diff --git a/agent/go.mod b/agent/go.mod index 7074cf75a..5a525f462 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -47,6 +47,7 @@ require ( github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect + github.com/Masterminds/semver v1.5.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/beevik/etree v1.1.0 // indirect @@ -89,6 +90,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mertyildiran/gqlparser/v2 v2.4.6 // indirect + github.com/mitchellh/go-ps v1.0.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect diff --git a/agent/go.sum b/agent/go.sum index a080e07bf..96aa71501 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -77,6 +77,8 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= @@ -507,6 +509,8 @@ github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceT github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= diff --git a/tap/go.mod b/tap/go.mod index 67b1c3174..7a7231b32 100644 --- a/tap/go.mod +++ b/tap/go.mod @@ -3,10 +3,12 @@ module github.com/up9inc/mizu/tap go 1.17 require ( + github.com/Masterminds/semver v1.5.0 github.com/cilium/ebpf v0.8.0 github.com/go-errors/errors v1.4.2 github.com/google/gopacket v1.1.19 github.com/hashicorp/golang-lru v0.5.4 + github.com/mitchellh/go-ps v1.0.0 github.com/shirou/gopsutil v3.21.11+incompatible github.com/struCoder/pidusage v0.2.1 github.com/up9inc/mizu/logger v0.0.0 diff --git a/tap/go.sum b/tap/go.sum index 786480904..823fde2ef 100644 --- a/tap/go.sum +++ b/tap/go.sum @@ -1,5 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= @@ -91,6 +93,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -290,5 +294,6 @@ sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/tap/tlstapper/bpf/golang_uprobes.c b/tap/tlstapper/bpf/golang_uprobes.c index 1099c5695..6718dd402 100644 --- a/tap/tlstapper/bpf/golang_uprobes.c +++ b/tap/tlstapper/bpf/golang_uprobes.c @@ -32,7 +32,11 @@ struct golang_read_write { BPF_LRU_HASH(golang_socket_dials, __u64, struct socket); BPF_LRU_HASH(golang_dial_writes, __u32, struct socket); -BPF_RINGBUF(golang_read_writes); + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, MAX_ENTRIES_RINGBUFF); +} golang_read_writes SEC(".maps"); const struct golang_read_write *unused __attribute__((unused)); diff --git a/tap/tlstapper/golang_hooks.go b/tap/tlstapper/golang_hooks.go new file mode 100644 index 000000000..d04da4f57 --- /dev/null +++ b/tap/tlstapper/golang_hooks.go @@ -0,0 +1,97 @@ +package tlstapper + +import ( + "github.com/cilium/ebpf/link" + "github.com/go-errors/errors" +) + +type golangHooks struct { + golangDialProbe link.Link + golangSocketProbe link.Link + golangWriteProbe link.Link + golangReadProbe link.Link +} + +func (s *golangHooks) installUprobes(bpfObjects *tlsTapperObjects, filePath string) error { + ex, err := link.OpenExecutable(filePath) + + if err != nil { + return errors.Wrap(err, 0) + } + + offsets, err := findGolangOffsets(filePath) + + if err != nil { + return errors.Wrap(err, 0) + } + + return s.installHooks(bpfObjects, ex, offsets) +} + +func (s *golangHooks) installHooks(bpfObjects *tlsTapperObjects, ex *link.Executable, offsets golangOffsets) error { + var err error + + // Relative offset points to + // [`net/http.(*Transport).dialConn+412`](https://github.com/golang/go/blob/fe4de36198794c447fbd9d7cc2d7199a506c76a5/src/net/http/transport.go#L1564) + s.golangDialProbe, err = ex.Uprobe(golangDialSymbol, bpfObjects.GolangNetHttpDialconnUprobe, &link.UprobeOptions{ + Offset: offsets.GolangWriteOffset + 0x19c, + }) + + if err != nil { + return errors.Wrap(err, 0) + } + + // Relative offset points to + // [`net.socket+127`](https://github.com/golang/go/blob/fe4de36198794c447fbd9d7cc2d7199a506c76a5/src/net/sock_posix.go#L23) + s.golangSocketProbe, err = ex.Uprobe(golangSocketSymbol, bpfObjects.GolangNetSocketUprobe, &link.UprobeOptions{ + Offset: offsets.GolangWriteOffset + 0x7f, + }) + + if err != nil { + return errors.Wrap(err, 0) + } + + // Symbol points to + // [`crypto/tls.(*Conn).Write`](https://github.com/golang/go/blob/fe4de36198794c447fbd9d7cc2d7199a506c76a5/src/crypto/tls/conn.go#L1109) + s.golangWriteProbe, err = ex.Uprobe(golangWriteSymbol, bpfObjects.GolangCryptoTlsWriteUprobe, &link.UprobeOptions{ + Offset: offsets.GolangWriteOffset, + }) + + if err != nil { + return errors.Wrap(err, 0) + } + + // Relative offset points to + // [`net/http.(*persistConn).Read+92`](https://github.com/golang/go/blob/fe4de36198794c447fbd9d7cc2d7199a506c76a5/src/net/http/transport.go#L1929) + s.golangReadProbe, err = ex.Uprobe(golangWriteSymbol, bpfObjects.GolangNetHttpReadUprobe, &link.UprobeOptions{ + Offset: offsets.GolangReadOffset + 0x5c, + }) + + if err != nil { + return errors.Wrap(err, 0) + } + + return nil +} + +func (s *golangHooks) close() []error { + errors := make([]error, 0) + + if err := s.golangDialProbe.Close(); err != nil { + errors = append(errors, err) + } + + if err := s.golangSocketProbe.Close(); err != nil { + errors = append(errors, err) + } + + if err := s.golangWriteProbe.Close(); err != nil { + errors = append(errors, err) + } + + if err := s.golangReadProbe.Close(); err != nil { + errors = append(errors, err) + } + + return errors +} diff --git a/tap/tlstapper/golang_offsets.go b/tap/tlstapper/golang_offsets.go new file mode 100644 index 000000000..ec4622f55 --- /dev/null +++ b/tap/tlstapper/golang_offsets.go @@ -0,0 +1,152 @@ +package tlstapper + +import ( + "bufio" + "debug/elf" + "fmt" + "os" + + "github.com/Masterminds/semver" + "github.com/cilium/ebpf/link" +) + +type golangOffsets struct { + GolangDialOffset uint64 + GolangSocketOffset uint64 + GolangWriteOffset uint64 + GolangReadOffset uint64 +} + +const ( + minimumSupportedGoVersion = "1.17.0" + golangVersionSymbol = "runtime.buildVersion.str" + golangWriteSymbol = "crypto/tls.(*Conn).Write" + golangReadSymbol = "net/http.(*persistConn).Read" + golangSocketSymbol = "net.socket" + golangDialSymbol = "net/http.(*Transport).dialConn" +) + +func findGolangOffsets(filePath string) (golangOffsets, error) { + offsets, err := getOffsets(filePath) + if err != nil { + return golangOffsets{}, err + } + + goVersionOffset, err := getOffset(offsets, golangVersionSymbol) + if err != nil { + return golangOffsets{}, err + } + + passed, goVersion, err := checkGoVersion(filePath, goVersionOffset) + if err != nil { + return golangOffsets{}, fmt.Errorf("Checking Go version: %s", err) + } + + if !passed { + return golangOffsets{}, fmt.Errorf("Unsupported Go version: %s", goVersion) + } + + dialOffset, err := getOffset(offsets, golangDialSymbol) + if err != nil { + return golangOffsets{}, fmt.Errorf("reading offset [%s]: %s", golangDialSymbol, err) + } + + socketOffset, err := getOffset(offsets, golangSocketSymbol) + if err != nil { + return golangOffsets{}, fmt.Errorf("reading offset [%s]: %s", golangSocketSymbol, err) + } + + writeOffset, err := getOffset(offsets, golangWriteSymbol) + if err != nil { + return golangOffsets{}, fmt.Errorf("reading offset [%s]: %s", golangWriteSymbol, err) + } + + readOffset, err := getOffset(offsets, golangReadSymbol) + if err != nil { + return golangOffsets{}, fmt.Errorf("reading offset [%s]: %s", golangReadSymbol, err) + } + + return golangOffsets{ + GolangWriteOffset: writeOffset, + GolangReadOffset: readOffset, + GolangSocketOffset: socketOffset, + GolangDialOffset: dialOffset, + }, nil +} + +func getOffsets(filePath string) (offsets map[string]uint64, err error) { + offsets = make(map[string]uint64) + var fd *os.File + fd, err = os.Open(filePath) + if err != nil { + return + } + defer fd.Close() + + var se *elf.File + se, err = elf.NewFile(fd) + if err != nil { + return nil, err + } + + syms, err := se.Symbols() + for _, sym := range syms { + offset := sym.Value + + for _, prog := range se.Progs { + if prog.Vaddr <= sym.Value && sym.Value < (prog.Vaddr+prog.Memsz) { + offset = sym.Value - prog.Vaddr + prog.Off + break + } + } + + offsets[sym.Name] = offset + } + + return +} + +func getOffset(offsets map[string]uint64, symbol string) (uint64, error) { + if offset, ok := offsets[symbol]; ok { + return offset, nil + } + return 0, fmt.Errorf("symbol %s: %w", symbol, link.ErrNoSymbol) +} + +func checkGoVersion(filePath string, offset uint64) (bool, string, error) { + fd, err := os.Open(filePath) + if err != nil { + return false, "", err + } + defer fd.Close() + + reader := bufio.NewReader(fd) + + _, err = reader.Discard(int(offset)) + if err != nil { + return false, "", err + } + + line, err := reader.ReadString(0) + if err != nil { + return false, "", err + } + + if len(line) < 3 { + return false, "", fmt.Errorf("ELF data segment read error (corrupted result)") + } + + goVersionStr := line[2 : len(line)-1] + + goVersion, err := semver.NewVersion(goVersionStr) + if err != nil { + return false, goVersionStr, err + } + + goVersionConstraint, err := semver.NewConstraint(fmt.Sprintf(">= %s", minimumSupportedGoVersion)) + if err != nil { + return false, goVersionStr, err + } + + return goVersionConstraint.Check(goVersion), goVersionStr, nil +} diff --git a/tap/tlstapper/tls_process_discoverer.go b/tap/tlstapper/tls_process_discoverer.go index f4b604399..7b653acd9 100644 --- a/tap/tlstapper/tls_process_discoverer.go +++ b/tap/tlstapper/tls_process_discoverer.go @@ -28,7 +28,11 @@ func UpdateTapTargets(tls *TlsTapper, pods *[]v1.Pod, procfs string) error { tls.ClearPids() for pid, pod := range containerPids { - if err := tls.AddPid(procfs, pid, pod.Namespace); err != nil { + if err := tls.AddSsllibPid(procfs, pid, pod.Namespace); err != nil { + LogError(err) + } + + if err := tls.AddGolangPid(procfs, pid, pod.Namespace); err != nil { LogError(err) } } diff --git a/tap/tlstapper/tls_tapper.go b/tap/tlstapper/tls_tapper.go index 67b38ccb3..ec71035a3 100644 --- a/tap/tlstapper/tls_tapper.go +++ b/tap/tlstapper/tls_tapper.go @@ -5,6 +5,7 @@ import ( "github.com/cilium/ebpf/rlimit" "github.com/go-errors/errors" + ps "github.com/mitchellh/go-ps" "github.com/up9inc/mizu/logger" "github.com/up9inc/mizu/tap/api" ) @@ -14,12 +15,13 @@ const GLOABL_TAP_PID = 0 //go:generate go run github.com/cilium/ebpf/cmd/bpf2go@0d0727ef53e2f53b1731c73f4c61e0f58693083a -type golang_read_write tlsTapper bpf/tls_tapper.c -- -O2 -g -D__TARGET_ARCH_x86 type TlsTapper struct { - bpfObjects tlsTapperObjects - syscallHooks syscallHooks - sslHooksStructs []sslHooks - poller *tlsPoller - bpfLogger *bpfLogger - registeredPids sync.Map + bpfObjects tlsTapperObjects + syscallHooks syscallHooks + sslHooksStructs []sslHooks + golangHooksStructs []golangHooks + poller *tlsPoller + bpfLogger *bpfLogger + registeredPids sync.Map } func (t *TlsTapper) Init(chunksBufferSize int, logBufferSize int, procfs string, extension *api.Extension) error { @@ -65,10 +67,10 @@ func (t *TlsTapper) PollForLogging() { } func (t *TlsTapper) GlobalTap(sslLibrary string) error { - return t.tapPid(GLOABL_TAP_PID, sslLibrary, api.UNKNOWN_NAMESPACE) + return t.tapLibsslPid(GLOABL_TAP_PID, sslLibrary, api.UNKNOWN_NAMESPACE) } -func (t *TlsTapper) AddPid(procfs string, pid uint32, namespace string) error { +func (t *TlsTapper) AddSsllibPid(procfs string, pid uint32, namespace string) error { sslLibrary, err := findSsllib(procfs, pid) if err != nil { @@ -76,7 +78,18 @@ func (t *TlsTapper) AddPid(procfs string, pid uint32, namespace string) error { return nil // hide the error on purpose, its OK for a process to not use libssl.so } - return t.tapPid(pid, sslLibrary, namespace) + return t.tapLibsslPid(pid, sslLibrary, namespace) +} + +func (t *TlsTapper) AddGolangPid(procfs string, pid uint32, namespace string) error { + p, err := ps.FindProcess(int(pid)) + + if err != nil { + logger.Log.Infof("PID skipped not found (pid: %d) %v", pid, err) + return nil + } + + return t.tapGolangPid(pid, p.Executable(), namespace) } func (t *TlsTapper) RemovePid(pid uint32) error { @@ -141,7 +154,7 @@ func setupRLimit() error { return nil } -func (t *TlsTapper) tapPid(pid uint32, sslLibrary string, namespace string) error { +func (t *TlsTapper) tapLibsslPid(pid uint32, sslLibrary string, namespace string) error { logger.Log.Infof("Tapping TLS (pid: %v) (sslLibrary: %v)", pid, sslLibrary) newSsl := sslHooks{} @@ -165,6 +178,28 @@ func (t *TlsTapper) tapPid(pid uint32, sslLibrary string, namespace string) erro return nil } +func (t *TlsTapper) tapGolangPid(pid uint32, exePath string, namespace string) error { + hooks := golangHooks{} + + if err := hooks.installUprobes(&t.bpfObjects, exePath); err != nil { + return err + } + + t.golangHooksStructs = append(t.golangHooksStructs, hooks) + + t.poller.addPid(pid, namespace) + + pids := t.bpfObjects.tlsTapperMaps.PidsMap + + if err := pids.Put(pid, uint32(1)); err != nil { + return errors.Wrap(err, 0) + } + + t.registeredPids.Store(pid, true) + + return nil +} + func LogError(err error) { var e *errors.Error if errors.As(err, &e) {