diff --git a/.github/workflows/static_code_analysis.yml b/.github/workflows/static_code_analysis.yml index 92bb6348b..2f91d49b2 100644 --- a/.github/workflows/static_code_analysis.yml +++ b/.github/workflows/static_code_analysis.yml @@ -26,6 +26,7 @@ jobs: run: | sudo apt update sudo apt install -y libpcap-dev + ./devops/install-capstone.sh - name: Check Agent modified files id: agent_modified_files @@ -37,7 +38,7 @@ jobs: with: version: latest working-directory: agent - args: --timeout=3m + args: --timeout=10m - name: Check shared modified files id: shared_modified_files @@ -49,7 +50,7 @@ jobs: with: version: latest working-directory: shared - args: --timeout=3m + args: --timeout=10m - name: Check tap modified files id: tap_modified_files @@ -61,7 +62,7 @@ jobs: with: version: latest working-directory: tap - args: --timeout=3m + args: --timeout=10m - name: Check cli modified files id: cli_modified_files @@ -73,7 +74,7 @@ jobs: with: version: latest working-directory: cli - args: --timeout=3m + args: --timeout=10m - name: Check acceptanceTests modified files id: acceptanceTests_modified_files @@ -85,7 +86,7 @@ jobs: with: version: latest working-directory: acceptanceTests - args: --timeout=3m + args: --timeout=10m - name: Check tap/api modified files id: tap_api_modified_files diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7abf91fa2..b8d6bd0bc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,6 +35,11 @@ jobs: run: | sudo apt-get install libpcap-dev + - name: Install Capstone + shell: bash + run: | + ./devops/install-capstone.sh + - name: Check CLI modified files id: cli_modified_files run: devops/check_modified_files.sh cli/ diff --git a/Dockerfile b/Dockerfile index 50ad7e7c3..1c3842ff6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,9 @@ RUN npm run build ### Base builder image for native builds architecture FROM golang:1.17-alpine AS builder-native-base ENV CGO_ENABLED=1 GOOS=linux -RUN apk add --no-cache libpcap-dev g++ perl-utils +RUN apk add --no-cache libpcap-dev g++ perl-utils curl build-base binutils-gold bash +COPY devops/install-capstone.sh . +RUN ./install-capstone.sh ### Intermediate builder image for x86-64 to x86-64 native builds @@ -39,15 +41,15 @@ ENV GOARCH=arm64 ### Builder image for x86-64 to AArch64 cross-compilation -FROM up9inc/linux-arm64-musl-go-libpcap AS builder-from-amd64-to-arm64v8 +FROM up9inc/linux-arm64-musl-go-libpcap-capstone AS builder-from-amd64-to-arm64v8 ENV CGO_ENABLED=1 GOOS=linux -ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap" +ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap -I/work/capstone/include" ### Builder image for AArch64 to x86-64 cross-compilation -FROM up9inc/linux-x86_64-musl-go-libpcap AS builder-from-arm64v8-to-amd64 +FROM up9inc/linux-x86_64-musl-go-libpcap-capstone AS builder-from-arm64v8-to-amd64 ENV CGO_ENABLED=1 GOOS=linux -ENV GOARCH=amd64 CGO_CFLAGS="-I/libpcap" +ENV GOARCH=amd64 CGO_CFLAGS="-I/libpcap -I/capstone/include" ### Final builder image where the build happens diff --git a/agent/go.mod b/agent/go.mod index 7074cf75a..52b2586a5 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -47,12 +47,13 @@ require ( github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect + github.com/Masterminds/semver v1.5.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/beevik/etree v1.1.0 // indirect github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b // indirect - github.com/cilium/ebpf v0.8.0 // indirect + github.com/cilium/ebpf v0.8.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect @@ -84,6 +85,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.14.2 // indirect + github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/agent/go.sum b/agent/go.sum index a080e07bf..6464e82aa 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -77,6 +77,8 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= @@ -126,8 +128,8 @@ github.com/chanced/openapi v0.0.8/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5Xyq github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.8.0 h1:2V6KSg3FRADVU2BMIRemZ0hV+9OM+aAHhZDjQyjJTAs= -github.com/cilium/ebpf v0.8.0/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMtrgk= +github.com/cilium/ebpf v0.8.1 h1:bLSSEbBLqGPXxls55pGr5qWZaTqcmfDJHhou7t254ao= +github.com/cilium/ebpf v0.8.1/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMtrgk= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -455,6 +457,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw= github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e h1:6J5obSn9umEThiYzWzndcPOZR0Qj/sVCZpH6V1G7yNE= +github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e/go.mod h1:1K5hEzsMBLTPdRJKEHqBFJ8Zt2VRqDhomcQ11KH0WW4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= diff --git a/devops/install-capstone.sh b/devops/install-capstone.sh new file mode 100755 index 000000000..4db790f1f --- /dev/null +++ b/devops/install-capstone.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +SUDO='' +if (( $EUID != 0 )); then + SUDO='sudo' +fi + +curl https://github.com/capstone-engine/capstone/archive/4.0.2.tar.gz -Lo ./capstone.tar.gz \ + && tar -xzf capstone.tar.gz && mv ./capstone-* ./capstone \ + && cd capstone \ + && CAPSTONE_ARCHS="aarch64 x86" ./make.sh \ + && $SUDO ./make.sh install diff --git a/devops/linux-arm64-musl-go-libpcap/Dockerfile b/devops/linux-arm64-musl-go-libpcap-capstone/Dockerfile similarity index 74% rename from devops/linux-arm64-musl-go-libpcap/Dockerfile rename to devops/linux-arm64-musl-go-libpcap-capstone/Dockerfile index 03f8fb7cf..6f9fb0c8a 100644 --- a/devops/linux-arm64-musl-go-libpcap/Dockerfile +++ b/devops/linux-arm64-musl-go-libpcap-capstone/Dockerfile @@ -17,4 +17,11 @@ RUN curl https://www.tcpdump.org/release/libpcap-1.10.1.tar.gz -Lo ./libpcap.tar WORKDIR /work/libpcap RUN ./configure --host=arm && make \ && cp /work/libpcap/libpcap.a /usr/xcc/aarch64-linux-musl-cross/lib/gcc/aarch64-linux-musl/*/ +WORKDIR /work +# Build and install Capstone from source +RUN curl https://github.com/capstone-engine/capstone/archive/4.0.2.tar.gz -Lo ./capstone.tar.gz \ + && tar -xzf capstone.tar.gz && mv ./capstone-* ./capstone +WORKDIR /work/capstone +RUN CAPSTONE_ARCHS="aarch64" CAPSTONE_STATIC=yes ./make.sh \ +&& cp /work/capstone/libcapstone.a /usr/xcc/aarch64-linux-musl-cross/lib/gcc/aarch64-linux-musl/*/ diff --git a/devops/linux-arm64-musl-go-libpcap-capstone/build-push.sh b/devops/linux-arm64-musl-go-libpcap-capstone/build-push.sh new file mode 100755 index 000000000..1048c1d52 --- /dev/null +++ b/devops/linux-arm64-musl-go-libpcap-capstone/build-push.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +docker build . -t up9inc/linux-arm64-musl-go-libpcap-capstone && docker push up9inc/linux-arm64-musl-go-libpcap-capstone diff --git a/devops/linux-arm64-musl-go-libpcap/build-push.sh b/devops/linux-arm64-musl-go-libpcap/build-push.sh deleted file mode 100755 index 904cb40f4..000000000 --- a/devops/linux-arm64-musl-go-libpcap/build-push.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -e - -docker build . -t up9inc/linux-arm64-musl-go-libpcap && docker push up9inc/linux-arm64-musl-go-libpcap diff --git a/devops/linux-x86_64-musl-go-libpcap/Dockerfile b/devops/linux-x86_64-musl-go-libpcap-capstone/Dockerfile similarity index 82% rename from devops/linux-x86_64-musl-go-libpcap/Dockerfile rename to devops/linux-x86_64-musl-go-libpcap-capstone/Dockerfile index b82454453..14b3e4903 100644 --- a/devops/linux-x86_64-musl-go-libpcap/Dockerfile +++ b/devops/linux-x86_64-musl-go-libpcap-capstone/Dockerfile @@ -29,3 +29,11 @@ RUN curl https://www.tcpdump.org/release/libpcap-1.10.1.tar.gz -Lo ./libpcap.tar WORKDIR /libpcap RUN ./configure --host=x86_64 && make \ && cp /libpcap/libpcap.a /usr/local/musl/lib/gcc/x86_64-unknown-linux-musl/*/ +WORKDIR / + +# Build and install Capstone from source +RUN curl https://github.com/capstone-engine/capstone/archive/4.0.2.tar.gz -Lo ./capstone.tar.gz \ + && tar -xzf capstone.tar.gz && mv ./capstone-* ./capstone +WORKDIR /capstone +RUN ./make.sh \ + && cp /capstone/libcapstone.a /usr/local/musl/lib/gcc/x86_64-unknown-linux-musl/*/ diff --git a/devops/linux-x86_64-musl-go-libpcap-capstone/build-push.sh b/devops/linux-x86_64-musl-go-libpcap-capstone/build-push.sh new file mode 100755 index 000000000..565f36705 --- /dev/null +++ b/devops/linux-x86_64-musl-go-libpcap-capstone/build-push.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +docker build . -t up9inc/linux-x86_64-musl-go-libpcap-capstone && docker push up9inc/linux-x86_64-musl-go-libpcap-capstone diff --git a/devops/linux-x86_64-musl-go-libpcap/build-push.sh b/devops/linux-x86_64-musl-go-libpcap/build-push.sh deleted file mode 100755 index 725366a64..000000000 --- a/devops/linux-x86_64-musl-go-libpcap/build-push.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -e - -docker build . -t up9inc/linux-x86_64-musl-go-libpcap && docker push up9inc/linux-x86_64-musl-go-libpcap diff --git a/tap/go.mod b/tap/go.mod index 67b1c3174..b3390fe0c 100644 --- a/tap/go.mod +++ b/tap/go.mod @@ -3,10 +3,12 @@ module github.com/up9inc/mizu/tap go 1.17 require ( - github.com/cilium/ebpf v0.8.0 + github.com/Masterminds/semver v1.5.0 + github.com/cilium/ebpf v0.8.1 github.com/go-errors/errors v1.4.2 github.com/google/gopacket v1.1.19 github.com/hashicorp/golang-lru v0.5.4 + github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e github.com/shirou/gopsutil v3.21.11+incompatible github.com/struCoder/pidusage v0.2.1 github.com/up9inc/mizu/logger v0.0.0 diff --git a/tap/go.sum b/tap/go.sum index 786480904..d57e92ed0 100644 --- a/tap/go.sum +++ b/tap/go.sum @@ -1,12 +1,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cilium/ebpf v0.8.0 h1:2V6KSg3FRADVU2BMIRemZ0hV+9OM+aAHhZDjQyjJTAs= -github.com/cilium/ebpf v0.8.0/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMtrgk= +github.com/cilium/ebpf v0.8.1 h1:bLSSEbBLqGPXxls55pGr5qWZaTqcmfDJHhou7t254ao= +github.com/cilium/ebpf v0.8.1/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMtrgk= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -81,6 +83,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e h1:6J5obSn9umEThiYzWzndcPOZR0Qj/sVCZpH6V1G7yNE= +github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e/go.mod h1:1K5hEzsMBLTPdRJKEHqBFJ8Zt2VRqDhomcQ11KH0WW4= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -290,5 +294,6 @@ sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/tap/passive_tapper.go b/tap/passive_tapper.go index 20d15a8a8..b62ce3b40 100644 --- a/tap/passive_tapper.go +++ b/tap/passive_tapper.go @@ -14,9 +14,9 @@ import ( "fmt" "os" "runtime" + "strconv" "strings" "time" - "strconv" "github.com/shirou/gopsutil/cpu" "github.com/struCoder/pidusage" @@ -59,8 +59,8 @@ var tls = flag.Bool("tls", false, "Enable TLS tapper") var memprofile = flag.String("memprofile", "", "Write memory profile") type TapOpts struct { - HostMode bool - IgnoredPorts []uint16 + HostMode bool + IgnoredPorts []uint16 } var extensions []*api.Extension // global @@ -100,7 +100,7 @@ func UpdateTapTargets(newTapTargets []v1.Pod) { packetSourceManager.UpdatePods(tapTargets, !*nodefrag, mainPacketInputChan) - if tlsTapperInstance != nil { + if tlsTapperInstance != nil && os.Getenv("MIZU_GLOBAL_GOLANG_PID") == "" { if err := tlstapper.UpdateTapTargets(tlsTapperInstance, &tapTargets, *procfs); err != nil { tlstapper.LogError(err) success = false @@ -278,7 +278,16 @@ func startTlsTapper(extension *api.Extension, outputItems chan *api.OutputChanne // A quick way to instrument libssl.so without PID filtering - used for debuging and troubleshooting // if os.Getenv("MIZU_GLOBAL_SSL_LIBRARY") != "" { - if err := tls.GlobalTap(os.Getenv("MIZU_GLOBAL_SSL_LIBRARY")); err != nil { + if err := tls.GlobalSsllibTap(os.Getenv("MIZU_GLOBAL_SSL_LIBRARY")); err != nil { + tlstapper.LogError(err) + return nil + } + } + + // A quick way to instrument Go `crypto/tls` without PID filtering - used for debuging and troubleshooting + // + if os.Getenv("MIZU_GLOBAL_GOLANG_PID") != "" { + if err := tls.GlobalGoTap(*procfs, os.Getenv("MIZU_GLOBAL_GOLANG_PID")); err != nil { tlstapper.LogError(err) return nil } diff --git a/tap/tlstapper/bpf-builder/Dockerfile b/tap/tlstapper/bpf-builder/Dockerfile index c817218cc..d16b41a2d 100644 --- a/tap/tlstapper/bpf-builder/Dockerfile +++ b/tap/tlstapper/bpf-builder/Dockerfile @@ -1,5 +1,5 @@ -FROM alpine:3.14 +FROM golang:1.17-alpine -RUN apk --no-cache update && apk --no-cache add clang llvm libbpf-dev go linux-headers +RUN apk --no-cache update && apk --no-cache add clang llvm libbpf-dev linux-headers WORKDIR /mizu diff --git a/tap/tlstapper/bpf/common.c b/tap/tlstapper/bpf/common.c new file mode 100644 index 000000000..d7f42f91b --- /dev/null +++ b/tap/tlstapper/bpf/common.c @@ -0,0 +1,145 @@ +/* +Note: This file is licenced differently from the rest of the project +SPDX-License-Identifier: GPL-2.0 +Copyright (C) UP9 Inc. +*/ + +#include "include/headers.h" +#include "include/util.h" +#include "include/maps.h" +#include "include/log.h" +#include "include/logger_messages.h" +#include "include/common.h" + + +static __always_inline int add_address_to_chunk(struct pt_regs *ctx, struct tls_chunk* chunk, __u64 id, __u32 fd) { + __u32 pid = id >> 32; + __u64 key = (__u64) pid << 32 | fd; + + struct fd_info *fdinfo = bpf_map_lookup_elem(&file_descriptor_to_ipv4, &key); + + if (fdinfo == NULL) { + return 0; + } + + int err = bpf_probe_read(chunk->address, sizeof(chunk->address), fdinfo->ipv4_addr); + chunk->flags |= (fdinfo->flags & FLAGS_IS_CLIENT_BIT); + + if (err != 0) { + log_error(ctx, LOG_ERROR_READING_FD_ADDRESS, id, err, 0l); + return 0; + } + + return 1; +} + +static __always_inline void send_chunk_part(struct pt_regs *ctx, __u8* buffer, __u64 id, + struct tls_chunk* chunk, int start, int end) { + size_t recorded = MIN(end - start, sizeof(chunk->data)); + + if (recorded <= 0) { + return; + } + + chunk->recorded = recorded; + chunk->start = start; + + // This ugly trick is for the ebpf verifier happiness + // + long err = 0; + if (chunk->recorded == sizeof(chunk->data)) { + err = bpf_probe_read(chunk->data, sizeof(chunk->data), buffer + start); + } else { + recorded &= (sizeof(chunk->data) - 1); // Buffer must be N^2 + err = bpf_probe_read(chunk->data, recorded, buffer + start); + } + + if (err != 0) { + log_error(ctx, LOG_ERROR_READING_FROM_SSL_BUFFER, id, err, 0l); + return; + } + + bpf_perf_event_output(ctx, &chunks_buffer, BPF_F_CURRENT_CPU, chunk, sizeof(struct tls_chunk)); +} + +static __always_inline void send_chunk(struct pt_regs *ctx, __u8* buffer, __u64 id, struct tls_chunk* chunk) { + // ebpf loops must be bounded at compile time, we can't use (i < chunk->len / CHUNK_SIZE) + // + // https://lwn.net/Articles/794934/ + // + // However we want to run in kernel older than 5.3, hence we use "#pragma unroll" anyway + // + #pragma unroll + for (int i = 0; i < MAX_CHUNKS_PER_OPERATION; i++) { + if (chunk->len <= (CHUNK_SIZE * i)) { + break; + } + + send_chunk_part(ctx, buffer, id, chunk, CHUNK_SIZE * i, chunk->len); + } +} + +static __always_inline void output_ssl_chunk(struct pt_regs *ctx, struct ssl_info* info, int count_bytes, __u64 id, __u32 flags) { + if (count_bytes <= 0) { + return; + } + + if (count_bytes > (CHUNK_SIZE * MAX_CHUNKS_PER_OPERATION)) { + log_error(ctx, LOG_ERROR_BUFFER_TOO_BIG, id, count_bytes, 0l); + return; + } + + struct tls_chunk* chunk; + int zero = 0; + + // If other thread, running on the same CPU get to this point at the same time like us (context switch) + // the data will be corrupted - protection may be added in the future + // + chunk = bpf_map_lookup_elem(&heap, &zero); + + if (!chunk) { + log_error(ctx, LOG_ERROR_ALLOCATING_CHUNK, id, 0l, 0l); + return; + } + + chunk->flags = flags; + chunk->pid = id >> 32; + chunk->tgid = id; + chunk->len = count_bytes; + chunk->fd = info->fd; + + if (!add_address_to_chunk(ctx, chunk, id, chunk->fd)) { + // Without an address, we drop the chunk because there is not much to do with it in Go + // + return; + } + + send_chunk(ctx, info->buffer, id, chunk); +} + +static __always_inline struct ssl_info new_ssl_info() { + struct ssl_info info = { .fd = invalid_fd, .created_at_nano = bpf_ktime_get_ns() }; + return info; +} + +static __always_inline struct ssl_info lookup_ssl_info(struct pt_regs *ctx, struct bpf_map_def* map_fd, __u64 pid_tgid) { + struct ssl_info *infoPtr = bpf_map_lookup_elem(map_fd, &pid_tgid); + struct ssl_info info = new_ssl_info(); + + if (infoPtr != NULL) { + long err = bpf_probe_read(&info, sizeof(struct ssl_info), infoPtr); + + if (err != 0) { + log_error(ctx, LOG_ERROR_READING_SSL_CONTEXT, pid_tgid, err, ORIGIN_SSL_UPROBE_CODE); + } + + if ((bpf_ktime_get_ns() - info.created_at_nano) > SSL_INFO_MAX_TTL_NANO) { + // If the ssl info is too old, we don't want to use its info because it may be incorrect. + // + info.fd = invalid_fd; + info.created_at_nano = bpf_ktime_get_ns(); + } + } + + return info; +} diff --git a/tap/tlstapper/bpf/fd_tracepoints.c b/tap/tlstapper/bpf/fd_tracepoints.c index 88add7f44..464da0ff0 100644 --- a/tap/tlstapper/bpf/fd_tracepoints.c +++ b/tap/tlstapper/bpf/fd_tracepoints.c @@ -28,7 +28,7 @@ void sys_enter_read(struct sys_enter_read_ctx *ctx) { return; } - struct ssl_info *infoPtr = bpf_map_lookup_elem(&ssl_read_context, &id); + struct ssl_info *infoPtr = bpf_map_lookup_elem(&openssl_read_context, &id); if (infoPtr == NULL) { return; @@ -44,7 +44,7 @@ void sys_enter_read(struct sys_enter_read_ctx *ctx) { info.fd = ctx->fd; - err = bpf_map_update_elem(&ssl_read_context, &id, &info, BPF_ANY); + err = bpf_map_update_elem(&openssl_read_context, &id, &info, BPF_ANY); if (err != 0) { log_error(ctx, LOG_ERROR_PUTTING_FILE_DESCRIPTOR, id, err, ORIGIN_SYS_ENTER_READ_CODE); @@ -68,7 +68,7 @@ void sys_enter_write(struct sys_enter_write_ctx *ctx) { return; } - struct ssl_info *infoPtr = bpf_map_lookup_elem(&ssl_write_context, &id); + struct ssl_info *infoPtr = bpf_map_lookup_elem(&openssl_write_context, &id); if (infoPtr == NULL) { return; @@ -84,7 +84,7 @@ void sys_enter_write(struct sys_enter_write_ctx *ctx) { info.fd = ctx->fd; - err = bpf_map_update_elem(&ssl_write_context, &id, &info, BPF_ANY); + err = bpf_map_update_elem(&openssl_write_context, &id, &info, BPF_ANY); if (err != 0) { log_error(ctx, LOG_ERROR_PUTTING_FILE_DESCRIPTOR, id, err, ORIGIN_SYS_ENTER_WRITE_CODE); diff --git a/tap/tlstapper/bpf/go_uprobes.c b/tap/tlstapper/bpf/go_uprobes.c new file mode 100644 index 000000000..d06930918 --- /dev/null +++ b/tap/tlstapper/bpf/go_uprobes.c @@ -0,0 +1,154 @@ +/* +Note: This file is licenced differently from the rest of the project +SPDX-License-Identifier: GPL-2.0 +Copyright (C) UP9 Inc. + + +--- + +README + +Go does not follow any platform ABI like x86-64 System V ABI. +Before 1.17, Go followed stack-based Plan9 (Bell Labs) calling convention. (ABI0) +After 1.17, Go switched to an internal register-based calling convention. (ABIInternal) +For now, the probes in this file supports only ABIInternal (Go 1.17+) + +`uretprobe` in Linux kernel uses trampoline pattern to jump to original return +address of the probed function. A Goroutine's stack size is 2Kb while a C thread is 2MB on Linux. +If stack size exceeds 2Kb, Go runtime relocates the stack. That causes the +return address to become incorrect in case of `uretprobe` and probed Go program crashes. +Therefore `uretprobe` CAN'T BE USED for a Go program. + +`_ex_uprobe` suffixed probes suppose to be `uretprobe`(s) are actually `uprobe`(s) +because of the non-standard ABI of Go. Therefore we probe all `ret` mnemonics under the symbol +by automatically finding them through reading the ELF binary and disassembling the symbols. +Disassembly related code located in `go_offsets.go` file and it uses Capstone Engine. +Solution based on: https://github.com/iovisor/bcc/issues/1320#issuecomment-407927542 +*Example* We probe an arbitrary point in a function body (offset +559): +https://github.com/golang/go/blob/go1.17.6/src/crypto/tls/conn.go#L1299 + +We get the file descriptor using the common $rax register that holds the address +of `go.itab.*net.TCPConn,net.Conn` and through a series of dereferencing +using `bpf_probe_read` calls in `go_crypto_tls_get_fd_from_tcp_conn` function. + +--- + +SOURCES: + +Tracing Go Functions with eBPF (before 1.17): https://www.grant.pizza/blog/tracing-go-functions-with-ebpf-part-2/ +Challenges of BPF Tracing Go: https://blog.0x74696d.com/posts/challenges-of-bpf-tracing-go/ +x86 calling conventions: https://en.wikipedia.org/wiki/X86_calling_conventions +Plan 9 from Bell Labs: https://en.wikipedia.org/wiki/Plan_9_from_Bell_Labs +The issue for calling convention change in Go: https://github.com/golang/go/issues/40724 +Proposal of Register-based Go calling convention: https://go.googlesource.com/proposal/+/master/design/40724-register-calling.md +Go internal ABI (1.17) specification: https://go.googlesource.com/go/+/refs/heads/dev.regabi/src/cmd/compile/internal-abi.md +Go internal ABI (current) specification: https://go.googlesource.com/go/+/refs/heads/master/src/cmd/compile/abi-internal.md +A Quick Guide to Go's Assembler: https://go.googlesource.com/go/+/refs/heads/dev.regabi/doc/asm.html +Dissecting Go Binaries: https://www.grant.pizza/blog/dissecting-go-binaries/ +Capstone Engine: https://www.capstone-engine.org/ +*/ + +#include "include/headers.h" +#include "include/util.h" +#include "include/maps.h" +#include "include/log.h" +#include "include/logger_messages.h" +#include "include/pids.h" +#include "include/common.h" +#include "include/go_abi_internal.h" +#include "include/go_types.h" + +static __always_inline __u32 go_crypto_tls_get_fd_from_tcp_conn(struct pt_regs *ctx) { + struct go_interface conn; + long err = bpf_probe_read(&conn, sizeof(conn), (void*)GO_ABI_INTERNAL_PT_REGS_R1(ctx)); + if (err != 0) { + return invalid_fd; + } + + void* net_fd_ptr; + err = bpf_probe_read(&net_fd_ptr, sizeof(net_fd_ptr), conn.ptr); + if (err != 0) { + return invalid_fd; + } + + __u32 fd; + err = bpf_probe_read(&fd, sizeof(fd), net_fd_ptr + 0x10); + if (err != 0) { + return invalid_fd; + } + + return fd; +} + +static __always_inline void go_crypto_tls_uprobe(struct pt_regs *ctx, struct bpf_map_def* go_context) { + __u64 pid_tgid = bpf_get_current_pid_tgid(); + __u64 pid = pid_tgid >> 32; + if (!should_tap(pid)) { + return; + } + + struct ssl_info info = new_ssl_info(); + + info.buffer_len = GO_ABI_INTERNAL_PT_REGS_R2(ctx); + info.buffer = (void*)GO_ABI_INTERNAL_PT_REGS_R4(ctx); + info.fd = go_crypto_tls_get_fd_from_tcp_conn(ctx); + + // GO_ABI_INTERNAL_PT_REGS_GP is Goroutine address + __u64 pid_fp = pid << 32 | GO_ABI_INTERNAL_PT_REGS_GP(ctx); + long err = bpf_map_update_elem(go_context, &pid_fp, &info, BPF_ANY); + + if (err != 0) { + log_error(ctx, LOG_ERROR_PUTTING_SSL_CONTEXT, pid_tgid, err, 0l); + } + + return; +} + +static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct bpf_map_def* go_context, __u32 flags) { + __u64 pid_tgid = bpf_get_current_pid_tgid(); + __u64 pid = pid_tgid >> 32; + if (!should_tap(pid)) { + return; + } + + // GO_ABI_INTERNAL_PT_REGS_GP is Goroutine address + __u64 pid_fp = pid << 32 | GO_ABI_INTERNAL_PT_REGS_GP(ctx); + struct ssl_info *info_ptr = bpf_map_lookup_elem(go_context, &pid_fp); + + if (info_ptr == NULL) { + return; + } + bpf_map_delete_elem(go_context, &pid_fp); + + struct ssl_info info; + long err = bpf_probe_read(&info, sizeof(struct ssl_info), info_ptr); + + if (err != 0) { + log_error(ctx, LOG_ERROR_READING_SSL_CONTEXT, pid_tgid, err, ORIGIN_SSL_URETPROBE_CODE); + return; + } + + output_ssl_chunk(ctx, &info, info.buffer_len, pid_tgid, flags); + + return; +} + +SEC("uprobe/go_crypto_tls_write") +void BPF_KPROBE(go_crypto_tls_write) { + go_crypto_tls_uprobe(ctx, &go_write_context); +} + +SEC("uprobe/go_crypto_tls_write_ex") +void BPF_KPROBE(go_crypto_tls_write_ex) { + go_crypto_tls_ex_uprobe(ctx, &go_write_context, 0); +} + +SEC("uprobe/go_crypto_tls_read") +void BPF_KPROBE(go_crypto_tls_read) { + go_crypto_tls_uprobe(ctx, &go_read_context); +} + +SEC("uprobe/go_crypto_tls_read_ex") +void BPF_KPROBE(go_crypto_tls_read_ex) { + go_crypto_tls_ex_uprobe(ctx, &go_read_context, FLAGS_IS_READ_BIT); +} diff --git a/tap/tlstapper/bpf/include/common.h b/tap/tlstapper/bpf/include/common.h new file mode 100644 index 000000000..613376396 --- /dev/null +++ b/tap/tlstapper/bpf/include/common.h @@ -0,0 +1,19 @@ +/* +Note: This file is licenced differently from the rest of the project +SPDX-License-Identifier: GPL-2.0 +Copyright (C) UP9 Inc. +*/ + +#ifndef __COMMON__ +#define __COMMON__ + +const __s32 invalid_fd = -1; + +static int add_address_to_chunk(struct pt_regs *ctx, struct tls_chunk* chunk, __u64 id, __u32 fd); +static void send_chunk_part(struct pt_regs *ctx, __u8* buffer, __u64 id, struct tls_chunk* chunk, int start, int end); +static void send_chunk(struct pt_regs *ctx, __u8* buffer, __u64 id, struct tls_chunk* chunk); +static void output_ssl_chunk(struct pt_regs *ctx, struct ssl_info* info, int count_bytes, __u64 id, __u32 flags); +static struct ssl_info new_ssl_info(); +static struct ssl_info lookup_ssl_info(struct pt_regs *ctx, struct bpf_map_def* map_fd, __u64 pid_tgid); + +#endif /* __COMMON__ */ diff --git a/tap/tlstapper/bpf/include/go_abi_internal.h b/tap/tlstapper/bpf/include/go_abi_internal.h new file mode 100644 index 000000000..368b32679 --- /dev/null +++ b/tap/tlstapper/bpf/include/go_abi_internal.h @@ -0,0 +1,141 @@ +/* +Note: This file is licenced differently from the rest of the project +SPDX-License-Identifier: GPL-2.0 +Copyright (C) UP9 Inc. +*/ + +#ifndef __GO_ABI_INTERNAL__ +#define __GO_ABI_INTERNAL__ + +/* +Go internal ABI specification +https://go.googlesource.com/go/+/refs/heads/master/src/cmd/compile/abi-internal.md +*/ + +/* Scan the ARCH passed in from ARCH env variable */ +#if defined(__TARGET_ARCH_x86) + #define bpf_target_x86 + #define bpf_target_defined +#elif defined(__TARGET_ARCH_s390) + #define bpf_target_s390 + #define bpf_target_defined +#elif defined(__TARGET_ARCH_arm) + #define bpf_target_arm + #define bpf_target_defined +#elif defined(__TARGET_ARCH_arm64) + #define bpf_target_arm64 + #define bpf_target_defined +#elif defined(__TARGET_ARCH_mips) + #define bpf_target_mips + #define bpf_target_defined +#elif defined(__TARGET_ARCH_powerpc) + #define bpf_target_powerpc + #define bpf_target_defined +#elif defined(__TARGET_ARCH_sparc) + #define bpf_target_sparc + #define bpf_target_defined +#else + #undef bpf_target_defined +#endif + +/* Fall back to what the compiler says */ +#ifndef bpf_target_defined +#if defined(__x86_64__) + #define bpf_target_x86 +#elif defined(__s390__) + #define bpf_target_s390 +#elif defined(__arm__) + #define bpf_target_arm +#elif defined(__aarch64__) + #define bpf_target_arm64 +#elif defined(__mips__) + #define bpf_target_mips +#elif defined(__powerpc__) + #define bpf_target_powerpc +#elif defined(__sparc__) + #define bpf_target_sparc +#endif +#endif + +#if defined(bpf_target_x86) + +#ifdef __i386__ + +/* +https://go.googlesource.com/go/+/refs/heads/dev.regabi/src/cmd/compile/internal-abi.md#amd64-architecture +https://github.com/golang/go/blob/go1.17.6/src/cmd/compile/internal/ssa/gen/AMD64Ops.go#L100 +*/ +#define GO_ABI_INTERNAL_PT_REGS_R1(x) ((x)->eax) +#define GO_ABI_INTERNAL_PT_REGS_P2(x) ((x)->ecx) +#define GO_ABI_INTERNAL_PT_REGS_P3(x) ((x)->edx) +#define GO_ABI_INTERNAL_PT_REGS_P4(x) 0 +#define GO_ABI_INTERNAL_PT_REGS_P5(x) 0 +#define GO_ABI_INTERNAL_PT_REGS_P6(x) 0 +#define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->esp) +#define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->ebp) +#define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->e14) + +#else + +#define GO_ABI_INTERNAL_PT_REGS_R1(x) ((x)->rax) +#define GO_ABI_INTERNAL_PT_REGS_R2(x) ((x)->rcx) +#define GO_ABI_INTERNAL_PT_REGS_R3(x) ((x)->rdx) +#define GO_ABI_INTERNAL_PT_REGS_R4(x) ((x)->rbx) +#define GO_ABI_INTERNAL_PT_REGS_R5(x) ((x)->rbp) +#define GO_ABI_INTERNAL_PT_REGS_R6(x) ((x)->rsi) +#define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->rsp) +#define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->rbp) +#define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->r14) + +#endif + +#elif defined(bpf_target_arm) + +/* +https://go.googlesource.com/go/+/refs/heads/master/src/cmd/compile/abi-internal.md#arm64-architecture +https://github.com/golang/go/blob/go1.17.6/src/cmd/compile/internal/ssa/gen/ARM64Ops.go#L129-L131 +*/ +#define GO_ABI_INTERNAL_PT_REGS_R1(x) ((x)->uregs[0]) +#define GO_ABI_INTERNAL_PT_REGS_R2(x) ((x)->uregs[1]) +#define GO_ABI_INTERNAL_PT_REGS_R3(x) ((x)->uregs[2]) +#define GO_ABI_INTERNAL_PT_REGS_R4(x) ((x)->uregs[3]) +#define GO_ABI_INTERNAL_PT_REGS_R5(x) ((x)->uregs[4]) +#define GO_ABI_INTERNAL_PT_REGS_R6(x) ((x)->uregs[5]) +#define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->uregs[14]) +#define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->uregs[29]) +#define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->uregs[28]) + +#elif defined(bpf_target_arm64) + +/* arm64 provides struct user_pt_regs instead of struct pt_regs to userspace */ +struct pt_regs; +#define PT_REGS_ARM64 const volatile struct user_pt_regs +#define GO_ABI_INTERNAL_PT_REGS_R1(x) (((PT_REGS_ARM64 *)(x))->regs[0]) +#define GO_ABI_INTERNAL_PT_REGS_R2(x) (((PT_REGS_ARM64 *)(x))->regs[1]) +#define GO_ABI_INTERNAL_PT_REGS_R3(x) (((PT_REGS_ARM64 *)(x))->regs[2]) +#define GO_ABI_INTERNAL_PT_REGS_R4(x) (((PT_REGS_ARM64 *)(x))->regs[3]) +#define GO_ABI_INTERNAL_PT_REGS_R5(x) (((PT_REGS_ARM64 *)(x))->regs[4]) +#define GO_ABI_INTERNAL_PT_REGS_R6(x) (((PT_REGS_ARM64 *)(x))->regs[5]) +#define GO_ABI_INTERNAL_PT_REGS_SP(x) (((PT_REGS_ARM64 *)(x))->regs[30]) +#define GO_ABI_INTERNAL_PT_REGS_FP(x) (((PT_REGS_ARM64 *)(x))->regs[29]) +#define GO_ABI_INTERNAL_PT_REGS_GP(x) (((PT_REGS_ARM64 *)(x))->regs[28]) + +#elif defined(bpf_target_powerpc) + +/* +https://go.googlesource.com/go/+/refs/heads/master/src/cmd/compile/abi-internal.md#ppc64-architecture +https://github.com/golang/go/blob/go1.17.6/src/cmd/compile/internal/ssa/gen/PPC64Ops.go#L125-L127 +*/ +#define GO_ABI_INTERNAL_PT_REGS_R1(x) ((x)->gpr[3]) +#define GO_ABI_INTERNAL_PT_REGS_R2(x) ((x)->gpr[4]) +#define GO_ABI_INTERNAL_PT_REGS_R3(x) ((x)->gpr[5]) +#define GO_ABI_INTERNAL_PT_REGS_R4(x) ((x)->gpr[6]) +#define GO_ABI_INTERNAL_PT_REGS_R5(x) ((x)->gpr[7]) +#define GO_ABI_INTERNAL_PT_REGS_R6(x) ((x)->gpr[8]) +#define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->sp) +#define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->gpr[12]) +#define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->gpr[30]) + +#endif + +#endif /* __GO_ABI_INTERNAL__ */ diff --git a/tap/tlstapper/bpf/include/go_types.h b/tap/tlstapper/bpf/include/go_types.h new file mode 100644 index 000000000..c865448c6 --- /dev/null +++ b/tap/tlstapper/bpf/include/go_types.h @@ -0,0 +1,15 @@ +/* +Note: This file is licenced differently from the rest of the project +SPDX-License-Identifier: GPL-2.0 +Copyright (C) UP9 Inc. +*/ + +#ifndef __GO_TYPES__ +#define __GO_TYPES__ + +struct go_interface { + __s64 type; + void* ptr; +}; + +#endif /* __GO_TYPES__ */ diff --git a/tap/tlstapper/bpf/include/maps.h b/tap/tlstapper/bpf/include/maps.h index 0774e775b..f8c799d13 100644 --- a/tap/tlstapper/bpf/include/maps.h +++ b/tap/tlstapper/bpf/include/maps.h @@ -16,11 +16,15 @@ Copyright (C) UP9 Inc. // One minute in nano seconds. Chosen by gut feeling. #define SSL_INFO_MAX_TTL_NANO (1000000000l * 60l) +#define MAX_ENTRIES_HASH (1 << 12) // 4096 +#define MAX_ENTRIES_PERF_OUTPUT (1 << 10) // 1024 +#define MAX_ENTRIES_LRU_HASH (1 << 14) // 16384 + // The same struct can be found in chunk.go // // Be careful when editing, alignment and padding should be exactly the same in go/c. // -struct tlsChunk { +struct tls_chunk { __u32 pid; __u32 tgid; __u32 len; @@ -34,6 +38,7 @@ struct tlsChunk { struct ssl_info { void* buffer; + __u32 buffer_len; __u32 fd; __u64 created_at_nano; @@ -48,6 +53,16 @@ struct fd_info { __u8 flags; }; +// Heap-like area for eBPF programs - stack size limited to 512 bytes, we must use maps for bigger (chunk) objects. +// +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct tls_chunk); +} heap SEC(".maps"); + + #define BPF_MAP(_name, _type, _key_type, _value_type, _max_entries) \ struct bpf_map_def SEC("maps") _name = { \ .type = _type, \ @@ -57,19 +72,26 @@ struct fd_info { }; #define BPF_HASH(_name, _key_type, _value_type) \ - BPF_MAP(_name, BPF_MAP_TYPE_HASH, _key_type, _value_type, 4096) + BPF_MAP(_name, BPF_MAP_TYPE_HASH, _key_type, _value_type, MAX_ENTRIES_HASH) #define BPF_PERF_OUTPUT(_name) \ - BPF_MAP(_name, BPF_MAP_TYPE_PERF_EVENT_ARRAY, int, __u32, 1024) - -#define BPF_LRU_HASH(_name, _key_type, _value_type) \ - BPF_MAP(_name, BPF_MAP_TYPE_LRU_HASH, _key_type, _value_type, 16384) + BPF_MAP(_name, BPF_MAP_TYPE_PERF_EVENT_ARRAY, int, __u32, MAX_ENTRIES_PERF_OUTPUT) +#define BPF_LRU_HASH(_name, _key_type, _value_type) \ + BPF_MAP(_name, BPF_MAP_TYPE_LRU_HASH, _key_type, _value_type, MAX_ENTRIES_LRU_HASH) + +// Generic BPF_HASH(pids_map, __u32, __u32); -BPF_LRU_HASH(ssl_write_context, __u64, struct ssl_info); -BPF_LRU_HASH(ssl_read_context, __u64, struct ssl_info); BPF_LRU_HASH(file_descriptor_to_ipv4, __u64, struct fd_info); BPF_PERF_OUTPUT(chunks_buffer); BPF_PERF_OUTPUT(log_buffer); +// OpenSSL specific +BPF_LRU_HASH(openssl_write_context, __u64, struct ssl_info); +BPF_LRU_HASH(openssl_read_context, __u64, struct ssl_info); + +// Go specific +BPF_LRU_HASH(go_write_context, __u64, struct ssl_info); +BPF_LRU_HASH(go_read_context, __u64, struct ssl_info); + #endif /* __MAPS__ */ diff --git a/tap/tlstapper/bpf/openssl_uprobes.c b/tap/tlstapper/bpf/openssl_uprobes.c index d4efa3392..2f5151da9 100644 --- a/tap/tlstapper/bpf/openssl_uprobes.c +++ b/tap/tlstapper/bpf/openssl_uprobes.c @@ -10,149 +10,35 @@ Copyright (C) UP9 Inc. #include "include/log.h" #include "include/logger_messages.h" #include "include/pids.h" +#include "include/common.h" -// Heap-like area for eBPF programs - stack size limited to 512 bytes, we must use maps for bigger (chunk) objects. -// -struct { - __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); - __uint(max_entries, 1); - __type(key, int); - __type(value, struct tlsChunk); -} heap SEC(".maps"); static __always_inline int get_count_bytes(struct pt_regs *ctx, struct ssl_info* info, __u64 id) { - int returnValue = PT_REGS_RC(ctx); - - if (info->count_ptr == NULL) { - // ssl_read and ssl_write return the number of bytes written/read - // - return returnValue; - } - - // ssl_read_ex and ssl_write_ex return 1 for success - // - if (returnValue != 1) { - return 0; - } - - // ssl_read_ex and ssl_write_ex write the number of bytes to an arg named *count - // - size_t countBytes; - long err = bpf_probe_read(&countBytes, sizeof(size_t), (void*) info->count_ptr); - - if (err != 0) { - log_error(ctx, LOG_ERROR_READING_BYTES_COUNT, id, err, 0l); - return 0; - } - - return countBytes; -} + int returnValue = PT_REGS_RC(ctx); -static __always_inline int add_address_to_chunk(struct pt_regs *ctx, struct tlsChunk* chunk, __u64 id, __u32 fd) { - __u32 pid = id >> 32; - __u64 key = (__u64) pid << 32 | fd; - - struct fd_info *fdinfo = bpf_map_lookup_elem(&file_descriptor_to_ipv4, &key); - - if (fdinfo == NULL) { - return 0; - } - - int err = bpf_probe_read(chunk->address, sizeof(chunk->address), fdinfo->ipv4_addr); - chunk->flags |= (fdinfo->flags & FLAGS_IS_CLIENT_BIT); - - if (err != 0) { - log_error(ctx, LOG_ERROR_READING_FD_ADDRESS, id, err, 0l); - return 0; - } - - return 1; -} + if (info->count_ptr == NULL) { + // ssl_read and ssl_write return the number of bytes written/read + // + return returnValue; + } -static __always_inline void send_chunk_part(struct pt_regs *ctx, __u8* buffer, __u64 id, - struct tlsChunk* chunk, int start, int end) { - size_t recorded = MIN(end - start, sizeof(chunk->data)); - - if (recorded <= 0) { - return; - } - - chunk->recorded = recorded; - chunk->start = start; - - // This ugly trick is for the ebpf verifier happiness - // - long err = 0; - if (chunk->recorded == sizeof(chunk->data)) { - err = bpf_probe_read(chunk->data, sizeof(chunk->data), buffer + start); - } else { - recorded &= (sizeof(chunk->data) - 1); // Buffer must be N^2 - err = bpf_probe_read(chunk->data, recorded, buffer + start); - } - - if (err != 0) { - log_error(ctx, LOG_ERROR_READING_FROM_SSL_BUFFER, id, err, 0l); - return; - } - - bpf_perf_event_output(ctx, &chunks_buffer, BPF_F_CURRENT_CPU, chunk, sizeof(struct tlsChunk)); -} + // ssl_read_ex and ssl_write_ex return 1 for success + // + if (returnValue != 1) { + return 0; + } -static __always_inline void send_chunk(struct pt_regs *ctx, __u8* buffer, __u64 id, struct tlsChunk* chunk) { - // ebpf loops must be bounded at compile time, we can't use (i < chunk->len / CHUNK_SIZE) - // - // https://lwn.net/Articles/794934/ - // - // However we want to run in kernel older than 5.3, hence we use "#pragma unroll" anyway - // - #pragma unroll - for (int i = 0; i < MAX_CHUNKS_PER_OPERATION; i++) { - if (chunk->len <= (CHUNK_SIZE * i)) { - break; - } - - send_chunk_part(ctx, buffer, id, chunk, CHUNK_SIZE * i, chunk->len); - } -} + // ssl_read_ex and ssl_write_ex write the number of bytes to an arg named *count + // + size_t countBytes; + long err = bpf_probe_read(&countBytes, sizeof(size_t), (void*) info->count_ptr); -static __always_inline void output_ssl_chunk(struct pt_regs *ctx, struct ssl_info* info, __u64 id, __u32 flags) { - int countBytes = get_count_bytes(ctx, info, id); - - if (countBytes <= 0) { - return; - } - - if (countBytes > (CHUNK_SIZE * MAX_CHUNKS_PER_OPERATION)) { - log_error(ctx, LOG_ERROR_BUFFER_TOO_BIG, id, countBytes, 0l); - return; - } - - struct tlsChunk* chunk; - int zero = 0; - - // If other thread, running on the same CPU get to this point at the same time like us (context switch) - // the data will be corrupted - protection may be added in the future - // - chunk = bpf_map_lookup_elem(&heap, &zero); - - if (!chunk) { - log_error(ctx, LOG_ERROR_ALLOCATING_CHUNK, id, 0l, 0l); - return; - } - - chunk->flags = flags; - chunk->pid = id >> 32; - chunk->tgid = id; - chunk->len = countBytes; - chunk->fd = info->fd; - - if (!add_address_to_chunk(ctx, chunk, id, chunk->fd)) { - // Without an address, we drop the chunk because there is not much to do with it in Go - // - return; - } - - send_chunk(ctx, info->buffer, id, chunk); + if (err != 0) { + log_error(ctx, LOG_ERROR_READING_BYTES_COUNT, id, err, 0l); + return 0; + } + + return countBytes; } static __always_inline void ssl_uprobe(struct pt_regs *ctx, void* ssl, void* buffer, int num, struct bpf_map_def* map_fd, size_t *count_ptr) { @@ -163,25 +49,7 @@ static __always_inline void ssl_uprobe(struct pt_regs *ctx, void* ssl, void* buf } struct ssl_info *infoPtr = bpf_map_lookup_elem(map_fd, &id); - struct ssl_info info = {}; - - if (infoPtr == NULL) { - info.fd = -1; - info.created_at_nano = bpf_ktime_get_ns(); - } else { - long err = bpf_probe_read(&info, sizeof(struct ssl_info), infoPtr); - - if (err != 0) { - log_error(ctx, LOG_ERROR_READING_SSL_CONTEXT, id, err, ORIGIN_SSL_UPROBE_CODE); - } - - if ((bpf_ktime_get_ns() - info.created_at_nano) > SSL_INFO_MAX_TTL_NANO) { - // If the ssl info is too old, we don't want to use its info because it may be incorrect. - // - info.fd = -1; - info.created_at_nano = bpf_ktime_get_ns(); - } - } + struct ssl_info info = lookup_ssl_info(ctx, &openssl_write_context, id); info.count_ptr = count_ptr; info.buffer = buffer; @@ -227,50 +95,52 @@ static __always_inline void ssl_uretprobe(struct pt_regs *ctx, struct bpf_map_de return; } - if (info.fd == -1) { + if (info.fd == invalid_fd) { log_error(ctx, LOG_ERROR_MISSING_FILE_DESCRIPTOR, id, 0l, 0l); return; } - - output_ssl_chunk(ctx, &info, id, flags); + + int count_bytes = get_count_bytes(ctx, &info, id); + + output_ssl_chunk(ctx, &info, count_bytes, id, flags); } SEC("uprobe/ssl_write") void BPF_KPROBE(ssl_write, void* ssl, void* buffer, int num) { - ssl_uprobe(ctx, ssl, buffer, num, &ssl_write_context, 0); + ssl_uprobe(ctx, ssl, buffer, num, &openssl_write_context, 0); } SEC("uretprobe/ssl_write") void BPF_KPROBE(ssl_ret_write) { - ssl_uretprobe(ctx, &ssl_write_context, 0); + ssl_uretprobe(ctx, &openssl_write_context, 0); } SEC("uprobe/ssl_read") void BPF_KPROBE(ssl_read, void* ssl, void* buffer, int num) { - ssl_uprobe(ctx, ssl, buffer, num, &ssl_read_context, 0); + ssl_uprobe(ctx, ssl, buffer, num, &openssl_read_context, 0); } SEC("uretprobe/ssl_read") void BPF_KPROBE(ssl_ret_read) { - ssl_uretprobe(ctx, &ssl_read_context, FLAGS_IS_READ_BIT); + ssl_uretprobe(ctx, &openssl_read_context, FLAGS_IS_READ_BIT); } SEC("uprobe/ssl_write_ex") void BPF_KPROBE(ssl_write_ex, void* ssl, void* buffer, size_t num, size_t *written) { - ssl_uprobe(ctx, ssl, buffer, num, &ssl_write_context, written); + ssl_uprobe(ctx, ssl, buffer, num, &openssl_write_context, written); } SEC("uretprobe/ssl_write_ex") void BPF_KPROBE(ssl_ret_write_ex) { - ssl_uretprobe(ctx, &ssl_write_context, 0); + ssl_uretprobe(ctx, &openssl_write_context, 0); } SEC("uprobe/ssl_read_ex") void BPF_KPROBE(ssl_read_ex, void* ssl, void* buffer, size_t num, size_t *readbytes) { - ssl_uprobe(ctx, ssl, buffer, num, &ssl_read_context, readbytes); + ssl_uprobe(ctx, ssl, buffer, num, &openssl_read_context, readbytes); } SEC("uretprobe/ssl_read_ex") void BPF_KPROBE(ssl_ret_read_ex) { - ssl_uretprobe(ctx, &ssl_read_context, FLAGS_IS_READ_BIT); + ssl_uretprobe(ctx, &openssl_read_context, FLAGS_IS_READ_BIT); } diff --git a/tap/tlstapper/bpf/tls_tapper.c b/tap/tlstapper/bpf/tls_tapper.c index 9878ef690..e5388b9f8 100644 --- a/tap/tlstapper/bpf/tls_tapper.c +++ b/tap/tlstapper/bpf/tls_tapper.c @@ -13,7 +13,9 @@ Copyright (C) UP9 Inc. // To avoid multiple .o files // +#include "common.c" #include "openssl_uprobes.c" +#include "go_uprobes.c" #include "fd_tracepoints.c" #include "fd_to_address_tracepoints.c" diff --git a/tap/tlstapper/bpf_logger_messages.go b/tap/tlstapper/bpf_logger_messages.go index bfcf84138..06bb642e4 100644 --- a/tap/tlstapper/bpf_logger_messages.go +++ b/tap/tlstapper/bpf_logger_messages.go @@ -2,7 +2,7 @@ package tlstapper // Must be synced with logger_messages.h // -var bpfLogMessages = []string { +var bpfLogMessages = []string{ /*0000*/ "[%d] Unable to read bytes count from _ex methods [err: %d]", /*0001*/ "[%d] Unable to read ipv4 address [err: %d]", /*0002*/ "[%d] Unable to read ssl buffer [err: %d]", @@ -20,6 +20,4 @@ var bpfLogMessages = []string { /*0014*/ "[%d] Unable to put connect info [err: %d]", /*0015*/ "[%d] Unable to get connect info", /*0016*/ "[%d] Unable to read connect info [err: %d]", - } - diff --git a/tap/tlstapper/chunk.go b/tap/tlstapper/chunk.go index 2ffc153aa..006585ec1 100644 --- a/tap/tlstapper/chunk.go +++ b/tap/tlstapper/chunk.go @@ -12,23 +12,7 @@ import ( const FLAGS_IS_CLIENT_BIT uint32 = (1 << 0) const FLAGS_IS_READ_BIT uint32 = (1 << 1) -// The same struct can be found in maps.h -// -// Be careful when editing, alignment and padding should be exactly the same in go/c. -// -type tlsChunk struct { - Pid uint32 // process id - Tgid uint32 // thread id inside the process - Len uint32 // the size of the native buffer used to read/write the tls data (may be bigger than tlsChunk.Data[]) - Start uint32 // the start offset withing the native buffer - Recorded uint32 // number of bytes copied from the native buffer to tlsChunk.Data[] - Fd uint32 // the file descriptor used to read/write the tls data (probably socket file descriptor) - Flags uint32 // bitwise flags - Address [16]byte // ipv4 address and port - Data [4096]byte // actual tls data -} - -func (c *tlsChunk) getAddress() (net.IP, uint16, error) { +func (c *tlsTapperTlsChunk) getAddress() (net.IP, uint16, error) { address := bytes.NewReader(c.Address[:]) var family uint16 var port uint16 @@ -51,31 +35,31 @@ func (c *tlsChunk) getAddress() (net.IP, uint16, error) { return ip, port, nil } -func (c *tlsChunk) isClient() bool { +func (c *tlsTapperTlsChunk) isClient() bool { return c.Flags&FLAGS_IS_CLIENT_BIT != 0 } -func (c *tlsChunk) isServer() bool { +func (c *tlsTapperTlsChunk) isServer() bool { return !c.isClient() } -func (c *tlsChunk) isRead() bool { +func (c *tlsTapperTlsChunk) isRead() bool { return c.Flags&FLAGS_IS_READ_BIT != 0 } -func (c *tlsChunk) isWrite() bool { +func (c *tlsTapperTlsChunk) isWrite() bool { return !c.isRead() } -func (c *tlsChunk) getRecordedData() []byte { +func (c *tlsTapperTlsChunk) getRecordedData() []byte { return c.Data[:c.Recorded] } -func (c *tlsChunk) isRequest() bool { +func (c *tlsTapperTlsChunk) isRequest() bool { return (c.isClient() && c.isWrite()) || (c.isServer() && c.isRead()) } -func (c *tlsChunk) getAddressPair() (addressPair, error) { +func (c *tlsTapperTlsChunk) getAddressPair() (addressPair, error) { ip, port, err := c.getAddress() if err != nil { diff --git a/tap/tlstapper/go_hooks.go b/tap/tlstapper/go_hooks.go new file mode 100644 index 000000000..25cdc89fe --- /dev/null +++ b/tap/tlstapper/go_hooks.go @@ -0,0 +1,105 @@ +package tlstapper + +import ( + "github.com/cilium/ebpf/link" + "github.com/go-errors/errors" +) + +type goHooks struct { + goWriteProbe link.Link + goWriteExProbes []link.Link + goReadProbe link.Link + goReadExProbes []link.Link +} + +func (s *goHooks) installUprobes(bpfObjects *tlsTapperObjects, filePath string) error { + ex, err := link.OpenExecutable(filePath) + + if err != nil { + return errors.Wrap(err, 0) + } + + offsets, err := findGoOffsets(filePath) + + if err != nil { + return errors.Wrap(err, 0) + } + + return s.installHooks(bpfObjects, ex, offsets) +} + +func (s *goHooks) installHooks(bpfObjects *tlsTapperObjects, ex *link.Executable, offsets goOffsets) error { + var err error + + // Symbol points to + // [`crypto/tls.(*Conn).Write`](https://github.com/golang/go/blob/go1.17.6/src/crypto/tls/conn.go#L1099) + s.goWriteProbe, err = ex.Uprobe(goWriteSymbol, bpfObjects.GoCryptoTlsWrite, &link.UprobeOptions{ + Offset: offsets.GoWriteOffset.enter, + }) + + if err != nil { + return errors.Wrap(err, 0) + } + + for _, offset := range offsets.GoWriteOffset.exits { + probe, err := ex.Uprobe(goWriteSymbol, bpfObjects.GoCryptoTlsWriteEx, &link.UprobeOptions{ + Offset: offset, + }) + + if err != nil { + return errors.Wrap(err, 0) + } + + s.goWriteExProbes = append(s.goWriteExProbes, probe) + } + + // Symbol points to + // [`crypto/tls.(*Conn).Read`](https://github.com/golang/go/blob/go1.17.6/src/crypto/tls/conn.go#L1263) + s.goReadProbe, err = ex.Uprobe(goReadSymbol, bpfObjects.GoCryptoTlsRead, &link.UprobeOptions{ + Offset: offsets.GoReadOffset.enter, + }) + + if err != nil { + return errors.Wrap(err, 0) + } + + for _, offset := range offsets.GoReadOffset.exits { + probe, err := ex.Uprobe(goReadSymbol, bpfObjects.GoCryptoTlsReadEx, &link.UprobeOptions{ + Offset: offset, + }) + + if err != nil { + return errors.Wrap(err, 0) + } + + s.goReadExProbes = append(s.goReadExProbes, probe) + } + + return nil +} + +func (s *goHooks) close() []error { + errors := make([]error, 0) + + if err := s.goWriteProbe.Close(); err != nil { + errors = append(errors, err) + } + + for _, probe := range s.goWriteExProbes { + if err := probe.Close(); err != nil { + errors = append(errors, err) + } + } + + if err := s.goReadProbe.Close(); err != nil { + errors = append(errors, err) + } + + for _, probe := range s.goReadExProbes { + if err := probe.Close(); err != nil { + errors = append(errors, err) + } + } + + return errors +} diff --git a/tap/tlstapper/go_offsets.go b/tap/tlstapper/go_offsets.go new file mode 100644 index 000000000..239b76bc2 --- /dev/null +++ b/tap/tlstapper/go_offsets.go @@ -0,0 +1,213 @@ +package tlstapper + +import ( + "bufio" + "debug/elf" + "fmt" + "os" + "runtime" + + "github.com/Masterminds/semver" + "github.com/cilium/ebpf/link" + "github.com/knightsc/gapstone" +) + +type goOffsets struct { + GoWriteOffset *goExtendedOffset + GoReadOffset *goExtendedOffset +} + +type goExtendedOffset struct { + enter uint64 + exits []uint64 +} + +const ( + minimumSupportedGoVersion = "1.17.0" + goVersionSymbol = "runtime.buildVersion.str" + goWriteSymbol = "crypto/tls.(*Conn).Write" + goReadSymbol = "crypto/tls.(*Conn).Read" +) + +func findGoOffsets(filePath string) (goOffsets, error) { + offsets, err := getOffsets(filePath) + if err != nil { + return goOffsets{}, err + } + + goVersionOffset, err := getOffset(offsets, goVersionSymbol) + if err != nil { + return goOffsets{}, err + } + + passed, goVersion, err := checkGoVersion(filePath, goVersionOffset) + if err != nil { + return goOffsets{}, fmt.Errorf("Checking Go version: %s", err) + } + + if !passed { + return goOffsets{}, fmt.Errorf("Unsupported Go version: %s", goVersion) + } + + writeOffset, err := getOffset(offsets, goWriteSymbol) + if err != nil { + return goOffsets{}, fmt.Errorf("reading offset [%s]: %s", goWriteSymbol, err) + } + + readOffset, err := getOffset(offsets, goReadSymbol) + if err != nil { + return goOffsets{}, fmt.Errorf("reading offset [%s]: %s", goReadSymbol, err) + } + + return goOffsets{ + GoWriteOffset: writeOffset, + GoReadOffset: readOffset, + }, nil +} + +func getOffsets(filePath string) (offsets map[string]*goExtendedOffset, err error) { + var engine gapstone.Engine + switch runtime.GOARCH { + case "amd64": + engine, err = gapstone.New( + gapstone.CS_ARCH_X86, + gapstone.CS_MODE_64, + ) + case "arm64": + engine, err = gapstone.New( + gapstone.CS_ARCH_ARM64, + gapstone.CS_MODE_ARM, + ) + default: + err = fmt.Errorf("Unsupported architecture: %v", runtime.GOARCH) + } + if err != nil { + return + } + + offsets = make(map[string]*goExtendedOffset) + var fd *os.File + fd, err = os.Open(filePath) + if err != nil { + return + } + defer fd.Close() + + var se *elf.File + se, err = elf.NewFile(fd) + if err != nil { + return + } + + textSection := se.Section(".text") + if textSection == nil { + err = fmt.Errorf("No text section") + return + } + + // extract the raw bytes from the .text section + var textSectionData []byte + textSectionData, err = textSection.Data() + if err != nil { + return + } + + syms, err := se.Symbols() + for _, sym := range syms { + offset := sym.Value + + var lastProg *elf.Prog + for _, prog := range se.Progs { + if prog.Vaddr <= sym.Value && sym.Value < (prog.Vaddr+prog.Memsz) { + offset = sym.Value - prog.Vaddr + prog.Off + lastProg = prog + break + } + } + + extendedOffset := &goExtendedOffset{enter: offset} + + // source: https://gist.github.com/grantseltzer/3efa8ecc5de1fb566e8091533050d608 + // skip over any symbols that aren't functinons/methods + if sym.Info != byte(2) && sym.Info != byte(18) { + offsets[sym.Name] = extendedOffset + continue + } + + // skip over empty symbols + if sym.Size == 0 { + offsets[sym.Name] = extendedOffset + continue + } + + // calculate starting and ending index of the symbol within the text section + symStartingIndex := sym.Value - textSection.Addr + symEndingIndex := symStartingIndex + sym.Size + + // collect the bytes of the symbol + symBytes := textSectionData[symStartingIndex:symEndingIndex] + + // disasemble the symbol + var instructions []gapstone.Instruction + instructions, err = engine.Disasm(symBytes, sym.Value, 0) + if err != nil { + return + } + + // iterate over each instruction and if the mnemonic is `ret` then that's an exit offset + for _, ins := range instructions { + if ins.Mnemonic == "ret" { + extendedOffset.exits = append(extendedOffset.exits, uint64(ins.Address)-lastProg.Vaddr+lastProg.Off) + } + } + + offsets[sym.Name] = extendedOffset + } + + return +} + +func getOffset(offsets map[string]*goExtendedOffset, symbol string) (*goExtendedOffset, error) { + if offset, ok := offsets[symbol]; ok { + return offset, nil + } + return nil, fmt.Errorf("symbol %s: %w", symbol, link.ErrNoSymbol) +} + +func checkGoVersion(filePath string, offset *goExtendedOffset) (bool, string, error) { + fd, err := os.Open(filePath) + if err != nil { + return false, "", err + } + defer fd.Close() + + reader := bufio.NewReader(fd) + + _, err = reader.Discard(int(offset.enter)) + if err != nil { + return false, "", err + } + + line, err := reader.ReadString(0) + if err != nil { + return false, "", err + } + + if len(line) < 3 { + return false, "", fmt.Errorf("ELF data segment read error (corrupted result)") + } + + goVersionStr := line[2 : len(line)-1] + + goVersion, err := semver.NewVersion(goVersionStr) + if err != nil { + return false, goVersionStr, err + } + + goVersionConstraint, err := semver.NewConstraint(fmt.Sprintf(">= %s", minimumSupportedGoVersion)) + if err != nil { + return false, goVersionStr, err + } + + return goVersionConstraint.Check(goVersion), goVersionStr, nil +} diff --git a/tap/tlstapper/ssllib_finder.go b/tap/tlstapper/ssllib_finder.go index 596772be7..3be5e18e6 100644 --- a/tap/tlstapper/ssllib_finder.go +++ b/tap/tlstapper/ssllib_finder.go @@ -46,7 +46,7 @@ func findLibraryByPid(procfs string, pid uint32, libraryName string) (string, er filepath := parts[5] - if !strings.Contains(filepath, libraryName) { + if libraryName != "" && !strings.Contains(filepath, libraryName) { continue } diff --git a/tap/tlstapper/tls_poller.go b/tap/tlstapper/tls_poller.go index 23643fab1..c0dddf9a4 100644 --- a/tap/tlstapper/tls_poller.go +++ b/tap/tlstapper/tls_poller.go @@ -20,8 +20,10 @@ import ( "github.com/up9inc/mizu/tap/api" ) -const fdCachedItemAvgSize = 40 -const fdCacheMaxItems = 500000 / fdCachedItemAvgSize +const ( + fdCachedItemAvgSize = 40 + fdCacheMaxItems = 500000 / fdCachedItemAvgSize +) type tlsPoller struct { tls *TlsTapper @@ -74,7 +76,8 @@ func (p *tlsPoller) close() error { } func (p *tlsPoller) poll(emitter api.Emitter, options *api.TrafficFilteringOptions, streamsMap api.TcpStreamMap) { - chunks := make(chan *tlsChunk) + // tlsTapperTlsChunk is generated by bpf2go. + chunks := make(chan *tlsTapperTlsChunk) go p.pollChunksPerfBuffer(chunks) @@ -94,7 +97,7 @@ func (p *tlsPoller) poll(emitter api.Emitter, options *api.TrafficFilteringOptio } } -func (p *tlsPoller) pollChunksPerfBuffer(chunks chan<- *tlsChunk) { +func (p *tlsPoller) pollChunksPerfBuffer(chunks chan<- *tlsTapperTlsChunk) { logger.Log.Infof("Start polling for tls events") for { @@ -118,7 +121,7 @@ func (p *tlsPoller) pollChunksPerfBuffer(chunks chan<- *tlsChunk) { buffer := bytes.NewReader(record.RawSample) - var chunk tlsChunk + var chunk tlsTapperTlsChunk if err := binary.Read(buffer, binary.LittleEndian, &chunk); err != nil { LogError(errors.Errorf("Error parsing chunk %v", err)) @@ -129,7 +132,7 @@ func (p *tlsPoller) pollChunksPerfBuffer(chunks chan<- *tlsChunk) { } } -func (p *tlsPoller) handleTlsChunk(chunk *tlsChunk, extension *api.Extension, emitter api.Emitter, +func (p *tlsPoller) handleTlsChunk(chunk *tlsTapperTlsChunk, extension *api.Extension, emitter api.Emitter, options *api.TrafficFilteringOptions, streamsMap api.TcpStreamMap) error { address, err := p.getSockfdAddressPair(chunk) @@ -158,11 +161,11 @@ func (p *tlsPoller) handleTlsChunk(chunk *tlsChunk, extension *api.Extension, em return nil } -func (p *tlsPoller) startNewTlsReader(chunk *tlsChunk, address *addressPair, key string, +func (p *tlsPoller) startNewTlsReader(chunk *tlsTapperTlsChunk, address *addressPair, key string, emitter api.Emitter, extension *api.Extension, options *api.TrafficFilteringOptions, streamsMap api.TcpStreamMap) *tlsReader { - tcpid := p.buildTcpId(chunk, address) + tcpid := p.buildTcpId(address) doneHandler := func(r *tlsReader) { p.closeReader(key, r) @@ -175,7 +178,7 @@ func (p *tlsPoller) startNewTlsReader(chunk *tlsChunk, address *addressPair, key reader := &tlsReader{ key: key, - chunks: make(chan *tlsChunk, 1), + chunks: make(chan *tlsTapperTlsChunk, 1), doneHandler: doneHandler, progress: &api.ReadProgress{}, tcpID: &tcpid, @@ -198,7 +201,7 @@ func (p *tlsPoller) startNewTlsReader(chunk *tlsChunk, address *addressPair, key return reader } -func dissect(extension *api.Extension, reader *tlsReader, options *api.TrafficFilteringOptions) { +func dissect(extension *api.Extension, reader api.TcpReader, options *api.TrafficFilteringOptions) { b := bufio.NewReader(reader) err := extension.Dissector.Dissect(b, reader, options) @@ -213,7 +216,7 @@ func (p *tlsPoller) closeReader(key string, r *tlsReader) { p.closedReaders <- key } -func (p *tlsPoller) getSockfdAddressPair(chunk *tlsChunk) (addressPair, error) { +func (p *tlsPoller) getSockfdAddressPair(chunk *tlsTapperTlsChunk) (addressPair, error) { address, err := getAddressBySockfd(p.procfs, chunk.Pid, chunk.Fd) fdCacheKey := fmt.Sprintf("%d:%d", chunk.Pid, chunk.Fd) @@ -252,7 +255,7 @@ func buildTlsKey(address addressPair) string { return fmt.Sprintf("%s:%d>%s:%d", address.srcIp, address.srcPort, address.dstIp, address.dstPort) } -func (p *tlsPoller) buildTcpId(chunk *tlsChunk, address *addressPair) api.TcpID { +func (p *tlsPoller) buildTcpId(address *addressPair) api.TcpID { return api.TcpID{ SrcIP: address.srcIp.String(), DstIP: address.dstIp.String(), @@ -289,7 +292,7 @@ func (p *tlsPoller) clearPids() { }) } -func (p *tlsPoller) logTls(chunk *tlsChunk, key string, reader *tlsReader) { +func (p *tlsPoller) logTls(chunk *tlsTapperTlsChunk, key string, reader *tlsReader) { var flagsStr string if chunk.isClient() { diff --git a/tap/tlstapper/tls_process_discoverer.go b/tap/tlstapper/tls_process_discoverer.go index f4b604399..d55536798 100644 --- a/tap/tlstapper/tls_process_discoverer.go +++ b/tap/tlstapper/tls_process_discoverer.go @@ -28,7 +28,11 @@ func UpdateTapTargets(tls *TlsTapper, pods *[]v1.Pod, procfs string) error { tls.ClearPids() for pid, pod := range containerPids { - if err := tls.AddPid(procfs, pid, pod.Namespace); err != nil { + if err := tls.AddSsllibPid(procfs, pid, pod.Namespace); err != nil { + LogError(err) + } + + if err := tls.AddGoPid(procfs, pid, pod.Namespace); err != nil { LogError(err) } } diff --git a/tap/tlstapper/tls_reader.go b/tap/tlstapper/tls_reader.go index b76a0008d..88c65ee56 100644 --- a/tap/tlstapper/tls_reader.go +++ b/tap/tlstapper/tls_reader.go @@ -9,7 +9,7 @@ import ( type tlsReader struct { key string - chunks chan *tlsChunk + chunks chan *tlsTapperTlsChunk seenChunks int data []byte doneHandler func(r *tlsReader) @@ -24,14 +24,14 @@ type tlsReader struct { reqResMatcher api.RequestResponseMatcher } -func (r *tlsReader) newChunk(chunk *tlsChunk) { +func (r *tlsReader) newChunk(chunk *tlsTapperTlsChunk) { r.captureTime = time.Now() r.seenChunks = r.seenChunks + 1 r.chunks <- chunk } func (r *tlsReader) Read(p []byte) (int, error) { - var chunk *tlsChunk + var chunk *tlsTapperTlsChunk for len(r.data) == 0 { var ok bool @@ -42,7 +42,7 @@ func (r *tlsReader) Read(p []byte) (int, error) { } r.data = chunk.getRecordedData() - case <-time.After(time.Second * 3): + case <-time.After(time.Second * 120): r.doneHandler(r) return 0, io.EOF } diff --git a/tap/tlstapper/tls_tapper.go b/tap/tlstapper/tls_tapper.go index b6fd173e1..4e7229ab6 100644 --- a/tap/tlstapper/tls_tapper.go +++ b/tap/tlstapper/tls_tapper.go @@ -1,6 +1,7 @@ package tlstapper import ( + "strconv" "sync" "github.com/cilium/ebpf/rlimit" @@ -11,12 +12,13 @@ import ( const GLOABL_TAP_PID = 0 -//go:generate go run github.com/cilium/ebpf/cmd/bpf2go tlsTapper bpf/tls_tapper.c -- -O2 -g -D__TARGET_ARCH_x86 +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go@0d0727ef53e2f53b1731c73f4c61e0f58693083a -type tls_chunk tlsTapper bpf/tls_tapper.c -- -O2 -g -D__TARGET_ARCH_x86 type TlsTapper struct { bpfObjects tlsTapperObjects syscallHooks syscallHooks sslHooksStructs []sslHooks + goHooksStructs []goHooks poller *tlsPoller bpfLogger *bpfLogger registeredPids sync.Map @@ -64,11 +66,20 @@ func (t *TlsTapper) PollForLogging() { t.bpfLogger.poll() } -func (t *TlsTapper) GlobalTap(sslLibrary string) error { - return t.tapPid(GLOABL_TAP_PID, sslLibrary, api.UNKNOWN_NAMESPACE) +func (t *TlsTapper) GlobalSsllibTap(sslLibrary string) error { + return t.tapSsllibPid(GLOABL_TAP_PID, sslLibrary, api.UNKNOWN_NAMESPACE) } -func (t *TlsTapper) AddPid(procfs string, pid uint32, namespace string) error { +func (t *TlsTapper) GlobalGoTap(procfs string, pid string) error { + _pid, err := strconv.Atoi(pid) + if err != nil { + return err + } + + return t.tapGoPid(procfs, uint32(_pid), api.UNKNOWN_NAMESPACE) +} + +func (t *TlsTapper) AddSsllibPid(procfs string, pid uint32, namespace string) error { sslLibrary, err := findSsllib(procfs, pid) if err != nil { @@ -76,7 +87,11 @@ func (t *TlsTapper) AddPid(procfs string, pid uint32, namespace string) error { return nil // hide the error on purpose, its OK for a process to not use libssl.so } - return t.tapPid(pid, sslLibrary, namespace) + return t.tapSsllibPid(pid, sslLibrary, namespace) +} + +func (t *TlsTapper) AddGoPid(procfs string, pid uint32, namespace string) error { + return t.tapGoPid(procfs, pid, namespace) } func (t *TlsTapper) RemovePid(pid uint32) error { @@ -120,6 +135,10 @@ func (t *TlsTapper) Close() []error { errors = append(errors, sslHooks.close()...) } + for _, goHooks := range t.goHooksStructs { + errors = append(errors, goHooks.close()...) + } + if err := t.bpfLogger.close(); err != nil { errors = append(errors, err) } @@ -141,7 +160,7 @@ func setupRLimit() error { return nil } -func (t *TlsTapper) tapPid(pid uint32, sslLibrary string, namespace string) error { +func (t *TlsTapper) tapSsllibPid(pid uint32, sslLibrary string, namespace string) error { logger.Log.Infof("Tapping TLS (pid: %v) (sslLibrary: %v)", pid, sslLibrary) newSsl := sslHooks{} @@ -165,6 +184,35 @@ func (t *TlsTapper) tapPid(pid uint32, sslLibrary string, namespace string) erro return nil } +func (t *TlsTapper) tapGoPid(procfs string, pid uint32, namespace string) error { + exePath, err := findLibraryByPid(procfs, pid, "") + if err != nil { + return err + } + + hooks := goHooks{} + + if err := hooks.installUprobes(&t.bpfObjects, exePath); err != nil { + return err + } + + logger.Log.Infof("Tapping TLS (pid: %v) (Go: %v)", pid, exePath) + + t.goHooksStructs = append(t.goHooksStructs, hooks) + + t.poller.addPid(pid, namespace) + + pids := t.bpfObjects.tlsTapperMaps.PidsMap + + if err := pids.Put(pid, uint32(1)); err != nil { + return errors.Wrap(err, 0) + } + + t.registeredPids.Store(pid, true) + + return nil +} + func LogError(err error) { var e *errors.Error if errors.As(err, &e) { diff --git a/tap/tlstapper/tlstapper_bpfeb.go b/tap/tlstapper/tlstapper_bpfeb.go index a4b07d7f4..c6b046e97 100644 --- a/tap/tlstapper/tlstapper_bpfeb.go +++ b/tap/tlstapper/tlstapper_bpfeb.go @@ -1,4 +1,5 @@ // Code generated by bpf2go; DO NOT EDIT. +//go:build arm64be || armbe || mips || mips64 || mips64p32 || ppc64 || s390 || s390x || sparc || sparc64 // +build arm64be armbe mips mips64 mips64p32 ppc64 s390 s390x sparc sparc64 package tlstapper @@ -12,6 +13,18 @@ import ( "github.com/cilium/ebpf" ) +type tlsTapperTlsChunk struct { + Pid uint32 + Tgid uint32 + Len uint32 + Start uint32 + Recorded uint32 + Fd uint32 + Flags uint32 + Address [16]uint8 + Data [4096]uint8 +} + // loadTlsTapper returns the embedded CollectionSpec for tlsTapper. func loadTlsTapper() (*ebpf.CollectionSpec, error) { reader := bytes.NewReader(_TlsTapperBytes) @@ -53,20 +66,24 @@ type tlsTapperSpecs struct { // // It can be passed ebpf.CollectionSpec.Assign. type tlsTapperProgramSpecs struct { - SslRead *ebpf.ProgramSpec `ebpf:"ssl_read"` - SslReadEx *ebpf.ProgramSpec `ebpf:"ssl_read_ex"` - SslRetRead *ebpf.ProgramSpec `ebpf:"ssl_ret_read"` - SslRetReadEx *ebpf.ProgramSpec `ebpf:"ssl_ret_read_ex"` - SslRetWrite *ebpf.ProgramSpec `ebpf:"ssl_ret_write"` - SslRetWriteEx *ebpf.ProgramSpec `ebpf:"ssl_ret_write_ex"` - SslWrite *ebpf.ProgramSpec `ebpf:"ssl_write"` - SslWriteEx *ebpf.ProgramSpec `ebpf:"ssl_write_ex"` - SysEnterAccept4 *ebpf.ProgramSpec `ebpf:"sys_enter_accept4"` - SysEnterConnect *ebpf.ProgramSpec `ebpf:"sys_enter_connect"` - SysEnterRead *ebpf.ProgramSpec `ebpf:"sys_enter_read"` - SysEnterWrite *ebpf.ProgramSpec `ebpf:"sys_enter_write"` - SysExitAccept4 *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"` - SysExitConnect *ebpf.ProgramSpec `ebpf:"sys_exit_connect"` + GoCryptoTlsRead *ebpf.ProgramSpec `ebpf:"go_crypto_tls_read"` + GoCryptoTlsReadEx *ebpf.ProgramSpec `ebpf:"go_crypto_tls_read_ex"` + GoCryptoTlsWrite *ebpf.ProgramSpec `ebpf:"go_crypto_tls_write"` + GoCryptoTlsWriteEx *ebpf.ProgramSpec `ebpf:"go_crypto_tls_write_ex"` + SslRead *ebpf.ProgramSpec `ebpf:"ssl_read"` + SslReadEx *ebpf.ProgramSpec `ebpf:"ssl_read_ex"` + SslRetRead *ebpf.ProgramSpec `ebpf:"ssl_ret_read"` + SslRetReadEx *ebpf.ProgramSpec `ebpf:"ssl_ret_read_ex"` + SslRetWrite *ebpf.ProgramSpec `ebpf:"ssl_ret_write"` + SslRetWriteEx *ebpf.ProgramSpec `ebpf:"ssl_ret_write_ex"` + SslWrite *ebpf.ProgramSpec `ebpf:"ssl_write"` + SslWriteEx *ebpf.ProgramSpec `ebpf:"ssl_write_ex"` + SysEnterAccept4 *ebpf.ProgramSpec `ebpf:"sys_enter_accept4"` + SysEnterConnect *ebpf.ProgramSpec `ebpf:"sys_enter_connect"` + SysEnterRead *ebpf.ProgramSpec `ebpf:"sys_enter_read"` + SysEnterWrite *ebpf.ProgramSpec `ebpf:"sys_enter_write"` + SysExitAccept4 *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"` + SysExitConnect *ebpf.ProgramSpec `ebpf:"sys_exit_connect"` } // tlsTapperMapSpecs contains maps before they are loaded into the kernel. @@ -77,11 +94,13 @@ type tlsTapperMapSpecs struct { ChunksBuffer *ebpf.MapSpec `ebpf:"chunks_buffer"` ConnectSyscallInfo *ebpf.MapSpec `ebpf:"connect_syscall_info"` FileDescriptorToIpv4 *ebpf.MapSpec `ebpf:"file_descriptor_to_ipv4"` + GoReadContext *ebpf.MapSpec `ebpf:"go_read_context"` + GoWriteContext *ebpf.MapSpec `ebpf:"go_write_context"` Heap *ebpf.MapSpec `ebpf:"heap"` LogBuffer *ebpf.MapSpec `ebpf:"log_buffer"` + OpensslReadContext *ebpf.MapSpec `ebpf:"openssl_read_context"` + OpensslWriteContext *ebpf.MapSpec `ebpf:"openssl_write_context"` PidsMap *ebpf.MapSpec `ebpf:"pids_map"` - SslReadContext *ebpf.MapSpec `ebpf:"ssl_read_context"` - SslWriteContext *ebpf.MapSpec `ebpf:"ssl_write_context"` } // tlsTapperObjects contains all objects after they have been loaded into the kernel. @@ -107,11 +126,13 @@ type tlsTapperMaps struct { ChunksBuffer *ebpf.Map `ebpf:"chunks_buffer"` ConnectSyscallInfo *ebpf.Map `ebpf:"connect_syscall_info"` FileDescriptorToIpv4 *ebpf.Map `ebpf:"file_descriptor_to_ipv4"` + GoReadContext *ebpf.Map `ebpf:"go_read_context"` + GoWriteContext *ebpf.Map `ebpf:"go_write_context"` Heap *ebpf.Map `ebpf:"heap"` LogBuffer *ebpf.Map `ebpf:"log_buffer"` + OpensslReadContext *ebpf.Map `ebpf:"openssl_read_context"` + OpensslWriteContext *ebpf.Map `ebpf:"openssl_write_context"` PidsMap *ebpf.Map `ebpf:"pids_map"` - SslReadContext *ebpf.Map `ebpf:"ssl_read_context"` - SslWriteContext *ebpf.Map `ebpf:"ssl_write_context"` } func (m *tlsTapperMaps) Close() error { @@ -120,11 +141,13 @@ func (m *tlsTapperMaps) Close() error { m.ChunksBuffer, m.ConnectSyscallInfo, m.FileDescriptorToIpv4, + m.GoReadContext, + m.GoWriteContext, m.Heap, m.LogBuffer, + m.OpensslReadContext, + m.OpensslWriteContext, m.PidsMap, - m.SslReadContext, - m.SslWriteContext, ) } @@ -132,24 +155,32 @@ func (m *tlsTapperMaps) Close() error { // // It can be passed to loadTlsTapperObjects or ebpf.CollectionSpec.LoadAndAssign. type tlsTapperPrograms struct { - SslRead *ebpf.Program `ebpf:"ssl_read"` - SslReadEx *ebpf.Program `ebpf:"ssl_read_ex"` - SslRetRead *ebpf.Program `ebpf:"ssl_ret_read"` - SslRetReadEx *ebpf.Program `ebpf:"ssl_ret_read_ex"` - SslRetWrite *ebpf.Program `ebpf:"ssl_ret_write"` - SslRetWriteEx *ebpf.Program `ebpf:"ssl_ret_write_ex"` - SslWrite *ebpf.Program `ebpf:"ssl_write"` - SslWriteEx *ebpf.Program `ebpf:"ssl_write_ex"` - SysEnterAccept4 *ebpf.Program `ebpf:"sys_enter_accept4"` - SysEnterConnect *ebpf.Program `ebpf:"sys_enter_connect"` - SysEnterRead *ebpf.Program `ebpf:"sys_enter_read"` - SysEnterWrite *ebpf.Program `ebpf:"sys_enter_write"` - SysExitAccept4 *ebpf.Program `ebpf:"sys_exit_accept4"` - SysExitConnect *ebpf.Program `ebpf:"sys_exit_connect"` + GoCryptoTlsRead *ebpf.Program `ebpf:"go_crypto_tls_read"` + GoCryptoTlsReadEx *ebpf.Program `ebpf:"go_crypto_tls_read_ex"` + GoCryptoTlsWrite *ebpf.Program `ebpf:"go_crypto_tls_write"` + GoCryptoTlsWriteEx *ebpf.Program `ebpf:"go_crypto_tls_write_ex"` + SslRead *ebpf.Program `ebpf:"ssl_read"` + SslReadEx *ebpf.Program `ebpf:"ssl_read_ex"` + SslRetRead *ebpf.Program `ebpf:"ssl_ret_read"` + SslRetReadEx *ebpf.Program `ebpf:"ssl_ret_read_ex"` + SslRetWrite *ebpf.Program `ebpf:"ssl_ret_write"` + SslRetWriteEx *ebpf.Program `ebpf:"ssl_ret_write_ex"` + SslWrite *ebpf.Program `ebpf:"ssl_write"` + SslWriteEx *ebpf.Program `ebpf:"ssl_write_ex"` + SysEnterAccept4 *ebpf.Program `ebpf:"sys_enter_accept4"` + SysEnterConnect *ebpf.Program `ebpf:"sys_enter_connect"` + SysEnterRead *ebpf.Program `ebpf:"sys_enter_read"` + SysEnterWrite *ebpf.Program `ebpf:"sys_enter_write"` + SysExitAccept4 *ebpf.Program `ebpf:"sys_exit_accept4"` + SysExitConnect *ebpf.Program `ebpf:"sys_exit_connect"` } func (p *tlsTapperPrograms) Close() error { return _TlsTapperClose( + p.GoCryptoTlsRead, + p.GoCryptoTlsReadEx, + p.GoCryptoTlsWrite, + p.GoCryptoTlsWriteEx, p.SslRead, p.SslReadEx, p.SslRetRead, diff --git a/tap/tlstapper/tlstapper_bpfeb.o b/tap/tlstapper/tlstapper_bpfeb.o index b4547a18d..5098e04ad 100644 Binary files a/tap/tlstapper/tlstapper_bpfeb.o and b/tap/tlstapper/tlstapper_bpfeb.o differ diff --git a/tap/tlstapper/tlstapper_bpfel.go b/tap/tlstapper/tlstapper_bpfel.go index eb6ca7281..a149b76f6 100644 --- a/tap/tlstapper/tlstapper_bpfel.go +++ b/tap/tlstapper/tlstapper_bpfel.go @@ -1,4 +1,5 @@ // Code generated by bpf2go; DO NOT EDIT. +//go:build 386 || amd64 || amd64p32 || arm || arm64 || mips64le || mips64p32le || mipsle || ppc64le || riscv64 // +build 386 amd64 amd64p32 arm arm64 mips64le mips64p32le mipsle ppc64le riscv64 package tlstapper @@ -12,6 +13,18 @@ import ( "github.com/cilium/ebpf" ) +type tlsTapperTlsChunk struct { + Pid uint32 + Tgid uint32 + Len uint32 + Start uint32 + Recorded uint32 + Fd uint32 + Flags uint32 + Address [16]uint8 + Data [4096]uint8 +} + // loadTlsTapper returns the embedded CollectionSpec for tlsTapper. func loadTlsTapper() (*ebpf.CollectionSpec, error) { reader := bytes.NewReader(_TlsTapperBytes) @@ -53,20 +66,24 @@ type tlsTapperSpecs struct { // // It can be passed ebpf.CollectionSpec.Assign. type tlsTapperProgramSpecs struct { - SslRead *ebpf.ProgramSpec `ebpf:"ssl_read"` - SslReadEx *ebpf.ProgramSpec `ebpf:"ssl_read_ex"` - SslRetRead *ebpf.ProgramSpec `ebpf:"ssl_ret_read"` - SslRetReadEx *ebpf.ProgramSpec `ebpf:"ssl_ret_read_ex"` - SslRetWrite *ebpf.ProgramSpec `ebpf:"ssl_ret_write"` - SslRetWriteEx *ebpf.ProgramSpec `ebpf:"ssl_ret_write_ex"` - SslWrite *ebpf.ProgramSpec `ebpf:"ssl_write"` - SslWriteEx *ebpf.ProgramSpec `ebpf:"ssl_write_ex"` - SysEnterAccept4 *ebpf.ProgramSpec `ebpf:"sys_enter_accept4"` - SysEnterConnect *ebpf.ProgramSpec `ebpf:"sys_enter_connect"` - SysEnterRead *ebpf.ProgramSpec `ebpf:"sys_enter_read"` - SysEnterWrite *ebpf.ProgramSpec `ebpf:"sys_enter_write"` - SysExitAccept4 *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"` - SysExitConnect *ebpf.ProgramSpec `ebpf:"sys_exit_connect"` + GoCryptoTlsRead *ebpf.ProgramSpec `ebpf:"go_crypto_tls_read"` + GoCryptoTlsReadEx *ebpf.ProgramSpec `ebpf:"go_crypto_tls_read_ex"` + GoCryptoTlsWrite *ebpf.ProgramSpec `ebpf:"go_crypto_tls_write"` + GoCryptoTlsWriteEx *ebpf.ProgramSpec `ebpf:"go_crypto_tls_write_ex"` + SslRead *ebpf.ProgramSpec `ebpf:"ssl_read"` + SslReadEx *ebpf.ProgramSpec `ebpf:"ssl_read_ex"` + SslRetRead *ebpf.ProgramSpec `ebpf:"ssl_ret_read"` + SslRetReadEx *ebpf.ProgramSpec `ebpf:"ssl_ret_read_ex"` + SslRetWrite *ebpf.ProgramSpec `ebpf:"ssl_ret_write"` + SslRetWriteEx *ebpf.ProgramSpec `ebpf:"ssl_ret_write_ex"` + SslWrite *ebpf.ProgramSpec `ebpf:"ssl_write"` + SslWriteEx *ebpf.ProgramSpec `ebpf:"ssl_write_ex"` + SysEnterAccept4 *ebpf.ProgramSpec `ebpf:"sys_enter_accept4"` + SysEnterConnect *ebpf.ProgramSpec `ebpf:"sys_enter_connect"` + SysEnterRead *ebpf.ProgramSpec `ebpf:"sys_enter_read"` + SysEnterWrite *ebpf.ProgramSpec `ebpf:"sys_enter_write"` + SysExitAccept4 *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"` + SysExitConnect *ebpf.ProgramSpec `ebpf:"sys_exit_connect"` } // tlsTapperMapSpecs contains maps before they are loaded into the kernel. @@ -77,11 +94,13 @@ type tlsTapperMapSpecs struct { ChunksBuffer *ebpf.MapSpec `ebpf:"chunks_buffer"` ConnectSyscallInfo *ebpf.MapSpec `ebpf:"connect_syscall_info"` FileDescriptorToIpv4 *ebpf.MapSpec `ebpf:"file_descriptor_to_ipv4"` + GoReadContext *ebpf.MapSpec `ebpf:"go_read_context"` + GoWriteContext *ebpf.MapSpec `ebpf:"go_write_context"` Heap *ebpf.MapSpec `ebpf:"heap"` LogBuffer *ebpf.MapSpec `ebpf:"log_buffer"` + OpensslReadContext *ebpf.MapSpec `ebpf:"openssl_read_context"` + OpensslWriteContext *ebpf.MapSpec `ebpf:"openssl_write_context"` PidsMap *ebpf.MapSpec `ebpf:"pids_map"` - SslReadContext *ebpf.MapSpec `ebpf:"ssl_read_context"` - SslWriteContext *ebpf.MapSpec `ebpf:"ssl_write_context"` } // tlsTapperObjects contains all objects after they have been loaded into the kernel. @@ -107,11 +126,13 @@ type tlsTapperMaps struct { ChunksBuffer *ebpf.Map `ebpf:"chunks_buffer"` ConnectSyscallInfo *ebpf.Map `ebpf:"connect_syscall_info"` FileDescriptorToIpv4 *ebpf.Map `ebpf:"file_descriptor_to_ipv4"` + GoReadContext *ebpf.Map `ebpf:"go_read_context"` + GoWriteContext *ebpf.Map `ebpf:"go_write_context"` Heap *ebpf.Map `ebpf:"heap"` LogBuffer *ebpf.Map `ebpf:"log_buffer"` + OpensslReadContext *ebpf.Map `ebpf:"openssl_read_context"` + OpensslWriteContext *ebpf.Map `ebpf:"openssl_write_context"` PidsMap *ebpf.Map `ebpf:"pids_map"` - SslReadContext *ebpf.Map `ebpf:"ssl_read_context"` - SslWriteContext *ebpf.Map `ebpf:"ssl_write_context"` } func (m *tlsTapperMaps) Close() error { @@ -120,11 +141,13 @@ func (m *tlsTapperMaps) Close() error { m.ChunksBuffer, m.ConnectSyscallInfo, m.FileDescriptorToIpv4, + m.GoReadContext, + m.GoWriteContext, m.Heap, m.LogBuffer, + m.OpensslReadContext, + m.OpensslWriteContext, m.PidsMap, - m.SslReadContext, - m.SslWriteContext, ) } @@ -132,24 +155,32 @@ func (m *tlsTapperMaps) Close() error { // // It can be passed to loadTlsTapperObjects or ebpf.CollectionSpec.LoadAndAssign. type tlsTapperPrograms struct { - SslRead *ebpf.Program `ebpf:"ssl_read"` - SslReadEx *ebpf.Program `ebpf:"ssl_read_ex"` - SslRetRead *ebpf.Program `ebpf:"ssl_ret_read"` - SslRetReadEx *ebpf.Program `ebpf:"ssl_ret_read_ex"` - SslRetWrite *ebpf.Program `ebpf:"ssl_ret_write"` - SslRetWriteEx *ebpf.Program `ebpf:"ssl_ret_write_ex"` - SslWrite *ebpf.Program `ebpf:"ssl_write"` - SslWriteEx *ebpf.Program `ebpf:"ssl_write_ex"` - SysEnterAccept4 *ebpf.Program `ebpf:"sys_enter_accept4"` - SysEnterConnect *ebpf.Program `ebpf:"sys_enter_connect"` - SysEnterRead *ebpf.Program `ebpf:"sys_enter_read"` - SysEnterWrite *ebpf.Program `ebpf:"sys_enter_write"` - SysExitAccept4 *ebpf.Program `ebpf:"sys_exit_accept4"` - SysExitConnect *ebpf.Program `ebpf:"sys_exit_connect"` + GoCryptoTlsRead *ebpf.Program `ebpf:"go_crypto_tls_read"` + GoCryptoTlsReadEx *ebpf.Program `ebpf:"go_crypto_tls_read_ex"` + GoCryptoTlsWrite *ebpf.Program `ebpf:"go_crypto_tls_write"` + GoCryptoTlsWriteEx *ebpf.Program `ebpf:"go_crypto_tls_write_ex"` + SslRead *ebpf.Program `ebpf:"ssl_read"` + SslReadEx *ebpf.Program `ebpf:"ssl_read_ex"` + SslRetRead *ebpf.Program `ebpf:"ssl_ret_read"` + SslRetReadEx *ebpf.Program `ebpf:"ssl_ret_read_ex"` + SslRetWrite *ebpf.Program `ebpf:"ssl_ret_write"` + SslRetWriteEx *ebpf.Program `ebpf:"ssl_ret_write_ex"` + SslWrite *ebpf.Program `ebpf:"ssl_write"` + SslWriteEx *ebpf.Program `ebpf:"ssl_write_ex"` + SysEnterAccept4 *ebpf.Program `ebpf:"sys_enter_accept4"` + SysEnterConnect *ebpf.Program `ebpf:"sys_enter_connect"` + SysEnterRead *ebpf.Program `ebpf:"sys_enter_read"` + SysEnterWrite *ebpf.Program `ebpf:"sys_enter_write"` + SysExitAccept4 *ebpf.Program `ebpf:"sys_exit_accept4"` + SysExitConnect *ebpf.Program `ebpf:"sys_exit_connect"` } func (p *tlsTapperPrograms) Close() error { return _TlsTapperClose( + p.GoCryptoTlsRead, + p.GoCryptoTlsReadEx, + p.GoCryptoTlsWrite, + p.GoCryptoTlsWriteEx, p.SslRead, p.SslReadEx, p.SslRetRead, diff --git a/tap/tlstapper/tlstapper_bpfel.o b/tap/tlstapper/tlstapper_bpfel.o index dd5697f6a..260994b7f 100644 Binary files a/tap/tlstapper/tlstapper_bpfel.o and b/tap/tlstapper/tlstapper_bpfel.o differ