Add Golang hooks and offsets

agent/go.mod

@@ -47,6 +47,7 @@ require (
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/beevik/etree v1.1.0 // indirect
@@ -89,6 +90,7 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mertyildiran/gqlparser/v2 v2.4.6 // indirect
+	github.com/mitchellh/go-ps v1.0.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect

agent/go.sum

@@ -77,6 +77,8 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
 github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
 github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
@@ -507,6 +509,8 @@ github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceT
 github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
+github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=

tap/go.mod

@@ -3,10 +3,12 @@ module github.com/up9inc/mizu/tap
 go 1.17
 
 require (
+	github.com/Masterminds/semver v1.5.0
 	github.com/cilium/ebpf v0.8.0
 	github.com/go-errors/errors v1.4.2
 	github.com/google/gopacket v1.1.19
 	github.com/hashicorp/golang-lru v0.5.4
+	github.com/mitchellh/go-ps v1.0.0
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/struCoder/pidusage v0.2.1
 	github.com/up9inc/mizu/logger v0.0.0

tap/go.sum

@@ -1,5 +1,7 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
@@ -91,6 +93,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
+github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -290,5 +294,6 @@ sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
-sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

tap/tlstapper/bpf/golang_uprobes.c

@@ -32,7 +32,11 @@ struct golang_read_write {
 
 BPF_LRU_HASH(golang_socket_dials, __u64, struct socket);
 BPF_LRU_HASH(golang_dial_writes, __u32, struct socket);
-BPF_RINGBUF(golang_read_writes);
+
+struct {
+    __uint(type, BPF_MAP_TYPE_RINGBUF);
+    __uint(max_entries, MAX_ENTRIES_RINGBUFF);
+} golang_read_writes SEC(".maps");
 
 const struct golang_read_write *unused __attribute__((unused));
 

tap/tlstapper/golang_hooks.go

@@ -0,0 +1,97 @@
+package tlstapper
+
+import (
+	"github.com/cilium/ebpf/link"
+	"github.com/go-errors/errors"
+)
+
+type golangHooks struct {
+	golangDialProbe   link.Link
+	golangSocketProbe link.Link
+	golangWriteProbe  link.Link
+	golangReadProbe   link.Link
+}
+
+func (s *golangHooks) installUprobes(bpfObjects *tlsTapperObjects, filePath string) error {
+	ex, err := link.OpenExecutable(filePath)
+
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
+	offsets, err := findGolangOffsets(filePath)
+
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
+	return s.installHooks(bpfObjects, ex, offsets)
+}
+
+func (s *golangHooks) installHooks(bpfObjects *tlsTapperObjects, ex *link.Executable, offsets golangOffsets) error {
+	var err error
+
+	// Relative offset points to
+	// [`net/http.(*Transport).dialConn+412`](https://github.com/golang/go/blob/fe4de36198794c447fbd9d7cc2d7199a506c76a5/src/net/http/transport.go#L1564)
+	s.golangDialProbe, err = ex.Uprobe(golangDialSymbol, bpfObjects.GolangNetHttpDialconnUprobe, &link.UprobeOptions{
+		Offset: offsets.GolangWriteOffset + 0x19c,
+	})
+
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
+	// Relative offset points to
+	// [`net.socket+127`](https://github.com/golang/go/blob/fe4de36198794c447fbd9d7cc2d7199a506c76a5/src/net/sock_posix.go#L23)
+	s.golangSocketProbe, err = ex.Uprobe(golangSocketSymbol, bpfObjects.GolangNetSocketUprobe, &link.UprobeOptions{
+		Offset: offsets.GolangWriteOffset + 0x7f,
+	})
+
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
+	// Symbol points to
+	// [`crypto/tls.(*Conn).Write`](https://github.com/golang/go/blob/fe4de36198794c447fbd9d7cc2d7199a506c76a5/src/crypto/tls/conn.go#L1109)
+	s.golangWriteProbe, err = ex.Uprobe(golangWriteSymbol, bpfObjects.GolangCryptoTlsWriteUprobe, &link.UprobeOptions{
+		Offset: offsets.GolangWriteOffset,
+	})
+
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
+	// Relative offset points to
+	// [`net/http.(*persistConn).Read+92`](https://github.com/golang/go/blob/fe4de36198794c447fbd9d7cc2d7199a506c76a5/src/net/http/transport.go#L1929)
+	s.golangReadProbe, err = ex.Uprobe(golangWriteSymbol, bpfObjects.GolangNetHttpReadUprobe, &link.UprobeOptions{
+		Offset: offsets.GolangReadOffset + 0x5c,
+	})
+
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
+	return nil
+}
+
+func (s *golangHooks) close() []error {
+	errors := make([]error, 0)
+
+	if err := s.golangDialProbe.Close(); err != nil {
+		errors = append(errors, err)
+	}
+
+	if err := s.golangSocketProbe.Close(); err != nil {
+		errors = append(errors, err)
+	}
+
+	if err := s.golangWriteProbe.Close(); err != nil {
+		errors = append(errors, err)
+	}
+
+	if err := s.golangReadProbe.Close(); err != nil {
+		errors = append(errors, err)
+	}
+
+	return errors
+}

tap/tlstapper/golang_offsets.go

@@ -0,0 +1,152 @@
+package tlstapper
+
+import (
+	"bufio"
+	"debug/elf"
+	"fmt"
+	"os"
+
+	"github.com/Masterminds/semver"
+	"github.com/cilium/ebpf/link"
+)
+
+type golangOffsets struct {
+	GolangDialOffset   uint64
+	GolangSocketOffset uint64
+	GolangWriteOffset  uint64
+	GolangReadOffset   uint64
+}
+
+const (
+	minimumSupportedGoVersion = "1.17.0"
+	golangVersionSymbol       = "runtime.buildVersion.str"
+	golangWriteSymbol         = "crypto/tls.(*Conn).Write"
+	golangReadSymbol          = "net/http.(*persistConn).Read"
+	golangSocketSymbol        = "net.socket"
+	golangDialSymbol          = "net/http.(*Transport).dialConn"
+)
+
+func findGolangOffsets(filePath string) (golangOffsets, error) {
+	offsets, err := getOffsets(filePath)
+	if err != nil {
+		return golangOffsets{}, err
+	}
+
+	goVersionOffset, err := getOffset(offsets, golangVersionSymbol)
+	if err != nil {
+		return golangOffsets{}, err
+	}
+
+	passed, goVersion, err := checkGoVersion(filePath, goVersionOffset)
+	if err != nil {
+		return golangOffsets{}, fmt.Errorf("Checking Go version: %s", err)
+	}
+
+	if !passed {
+		return golangOffsets{}, fmt.Errorf("Unsupported Go version: %s", goVersion)
+	}
+
+	dialOffset, err := getOffset(offsets, golangDialSymbol)
+	if err != nil {
+		return golangOffsets{}, fmt.Errorf("reading offset [%s]: %s", golangDialSymbol, err)
+	}
+
+	socketOffset, err := getOffset(offsets, golangSocketSymbol)
+	if err != nil {
+		return golangOffsets{}, fmt.Errorf("reading offset [%s]: %s", golangSocketSymbol, err)
+	}
+
+	writeOffset, err := getOffset(offsets, golangWriteSymbol)
+	if err != nil {
+		return golangOffsets{}, fmt.Errorf("reading offset [%s]: %s", golangWriteSymbol, err)
+	}
+
+	readOffset, err := getOffset(offsets, golangReadSymbol)
+	if err != nil {
+		return golangOffsets{}, fmt.Errorf("reading offset [%s]: %s", golangReadSymbol, err)
+	}
+
+	return golangOffsets{
+		GolangWriteOffset:  writeOffset,
+		GolangReadOffset:   readOffset,
+		GolangSocketOffset: socketOffset,
+		GolangDialOffset:   dialOffset,
+	}, nil
+}
+
+func getOffsets(filePath string) (offsets map[string]uint64, err error) {
+	offsets = make(map[string]uint64)
+	var fd *os.File
+	fd, err = os.Open(filePath)
+	if err != nil {
+		return
+	}
+	defer fd.Close()
+
+	var se *elf.File
+	se, err = elf.NewFile(fd)
+	if err != nil {
+		return nil, err
+	}
+
+	syms, err := se.Symbols()
+	for _, sym := range syms {
+		offset := sym.Value
+
+		for _, prog := range se.Progs {
+			if prog.Vaddr <= sym.Value && sym.Value < (prog.Vaddr+prog.Memsz) {
+				offset = sym.Value - prog.Vaddr + prog.Off
+				break
+			}
+		}
+
+		offsets[sym.Name] = offset
+	}
+
+	return
+}
+
+func getOffset(offsets map[string]uint64, symbol string) (uint64, error) {
+	if offset, ok := offsets[symbol]; ok {
+		return offset, nil
+	}
+	return 0, fmt.Errorf("symbol %s: %w", symbol, link.ErrNoSymbol)
+}
+
+func checkGoVersion(filePath string, offset uint64) (bool, string, error) {
+	fd, err := os.Open(filePath)
+	if err != nil {
+		return false, "", err
+	}
+	defer fd.Close()
+
+	reader := bufio.NewReader(fd)
+
+	_, err = reader.Discard(int(offset))
+	if err != nil {
+		return false, "", err
+	}
+
+	line, err := reader.ReadString(0)
+	if err != nil {
+		return false, "", err
+	}
+
+	if len(line) < 3 {
+		return false, "", fmt.Errorf("ELF data segment read error (corrupted result)")
+	}
+
+	goVersionStr := line[2 : len(line)-1]
+
+	goVersion, err := semver.NewVersion(goVersionStr)
+	if err != nil {
+		return false, goVersionStr, err
+	}
+
+	goVersionConstraint, err := semver.NewConstraint(fmt.Sprintf(">= %s", minimumSupportedGoVersion))
+	if err != nil {
+		return false, goVersionStr, err
+	}
+
+	return goVersionConstraint.Check(goVersion), goVersionStr, nil
+}

tap/tlstapper/tls_process_discoverer.go

@@ -28,7 +28,11 @@ func UpdateTapTargets(tls *TlsTapper, pods *[]v1.Pod, procfs string) error {
 	tls.ClearPids()
 
 	for pid, pod := range containerPids {
-		if err := tls.AddPid(procfs, pid, pod.Namespace); err != nil {
+		if err := tls.AddSsllibPid(procfs, pid, pod.Namespace); err != nil {
+			LogError(err)
+		}
+
+		if err := tls.AddGolangPid(procfs, pid, pod.Namespace); err != nil {
 			LogError(err)
 		}
 	}

tap/tlstapper/tls_tapper.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/cilium/ebpf/rlimit"
 	"github.com/go-errors/errors"
+	ps "github.com/mitchellh/go-ps"
 	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/tap/api"
 )
@@ -14,12 +15,13 @@ const GLOABL_TAP_PID = 0
 //go:generate go run github.com/cilium/ebpf/cmd/bpf2go@0d0727ef53e2f53b1731c73f4c61e0f58693083a -type golang_read_write tlsTapper bpf/tls_tapper.c -- -O2 -g -D__TARGET_ARCH_x86
 
 type TlsTapper struct {
-	bpfObjects      tlsTapperObjects
+	bpfObjects         tlsTapperObjects
-	syscallHooks    syscallHooks
+	syscallHooks       syscallHooks
-	sslHooksStructs []sslHooks
+	sslHooksStructs    []sslHooks
-	poller          *tlsPoller
+	golangHooksStructs []golangHooks
-	bpfLogger       *bpfLogger
+	poller             *tlsPoller
-	registeredPids  sync.Map
+	bpfLogger          *bpfLogger
+	registeredPids     sync.Map
 }
 
 func (t *TlsTapper) Init(chunksBufferSize int, logBufferSize int, procfs string, extension *api.Extension) error {
@@ -65,10 +67,10 @@ func (t *TlsTapper) PollForLogging() {
 }
 
 func (t *TlsTapper) GlobalTap(sslLibrary string) error {
-	return t.tapPid(GLOABL_TAP_PID, sslLibrary, api.UNKNOWN_NAMESPACE)
+	return t.tapLibsslPid(GLOABL_TAP_PID, sslLibrary, api.UNKNOWN_NAMESPACE)
 }
 
-func (t *TlsTapper) AddPid(procfs string, pid uint32, namespace string) error {
+func (t *TlsTapper) AddSsllibPid(procfs string, pid uint32, namespace string) error {
 	sslLibrary, err := findSsllib(procfs, pid)
 
 	if err != nil {
@@ -76,7 +78,18 @@ func (t *TlsTapper) AddPid(procfs string, pid uint32, namespace string) error {
 		return nil // hide the error on purpose, its OK for a process to not use libssl.so
 	}
 
-	return t.tapPid(pid, sslLibrary, namespace)
+	return t.tapLibsslPid(pid, sslLibrary, namespace)
+}
+
+func (t *TlsTapper) AddGolangPid(procfs string, pid uint32, namespace string) error {
+	p, err := ps.FindProcess(int(pid))
+
+	if err != nil {
+		logger.Log.Infof("PID skipped not found (pid: %d) %v", pid, err)
+		return nil
+	}
+
+	return t.tapGolangPid(pid, p.Executable(), namespace)
 }
 
 func (t *TlsTapper) RemovePid(pid uint32) error {
@@ -141,7 +154,7 @@ func setupRLimit() error {
 	return nil
 }
 
-func (t *TlsTapper) tapPid(pid uint32, sslLibrary string, namespace string) error {
+func (t *TlsTapper) tapLibsslPid(pid uint32, sslLibrary string, namespace string) error {
 	logger.Log.Infof("Tapping TLS (pid: %v) (sslLibrary: %v)", pid, sslLibrary)
 
 	newSsl := sslHooks{}
@@ -165,6 +178,28 @@ func (t *TlsTapper) tapPid(pid uint32, sslLibrary string, namespace string) erro
 	return nil
 }
 
+func (t *TlsTapper) tapGolangPid(pid uint32, exePath string, namespace string) error {
+	hooks := golangHooks{}
+
+	if err := hooks.installUprobes(&t.bpfObjects, exePath); err != nil {
+		return err
+	}
+
+	t.golangHooksStructs = append(t.golangHooksStructs, hooks)
+
+	t.poller.addPid(pid, namespace)
+
+	pids := t.bpfObjects.tlsTapperMaps.PidsMap
+
+	if err := pids.Put(pid, uint32(1)); err != nil {
+		return errors.Wrap(err, 0)
+	}
+
+	t.registeredPids.Store(pid, true)
+
+	return nil
+}
+
 func LogError(err error) {
 	var e *errors.Error
 	if errors.As(err, &e) {