github/kubeshark
1
0
Fork 0
mirror of https://github.com/kubeshark/kubeshark.git synced 2025-07-15 17:12:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix Go crypto/tls eBPF tracer on ARM64 (#1162)

Browse Source 
* Handle the Go `ABIInternal` differences on ARM64

* Log the Capstone version, arch and mode

* Upgrade Capstone from `4.0.2` to `5.0-rc2` to have ARM64 instructions fix

* Use the correct register on ARM64 for reading buffer length

* Fix the addresses on ARM64

* #run_acceptance_tests

* Update `x86.o`

* Update `arm64.o`

* #run_acceptance_tests
This commit is contained in:
M. Mert Yıldıran 2022-06-26 04:11:04 -07:00 committed by GitHub
parent 1ebc51b45e
commit cf91f3dab4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 72 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile
View File

@ -54,14 +54,14 @@ ENV BPF_TARGET=arm64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64"
### Builder image for x86-64 to AArch64 cross-compilation
FROM up9inc/linux-arm64-musl-go-libpcap-capstone-bpf AS builder-from-amd64-to-arm64v8
FROM up9inc/linux-arm64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 AS builder-from-amd64-to-arm64v8
ENV CGO_ENABLED=1 GOOS=linux
ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap -I/work/capstone/include"
ENV BPF_TARGET=arm64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64 -I/usr/xcc/aarch64-linux-musl-cross/aarch64-linux-musl/include/"
### Builder image for AArch64 to x86-64 cross-compilation
FROM up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf AS builder-from-arm64v8-to-amd64
FROM up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 AS builder-from-arm64v8-to-amd64
ENV CGO_ENABLED=1 GOOS=linux
ENV GOARCH=amd64 CGO_CFLAGS="-I/libpcap -I/capstone/include"
ENV BPF_TARGET=amd64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_x86 -I/usr/local/musl/x86_64-unknown-linux-musl/include/"

2
agent/go.mod
View File

@ -83,7 +83,7 @@ require (
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.14.2 // indirect
github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e // indirect
github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/mailru/easyjson v0.7.7 // indirect

4
agent/go.sum
View File

@ -453,8 +453,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw=
github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e h1:6J5obSn9umEThiYzWzndcPOZR0Qj/sVCZpH6V1G7yNE=
github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e/go.mod h1:1K5hEzsMBLTPdRJKEHqBFJ8Zt2VRqDhomcQ11KH0WW4=
github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9 h1:1KszOoXSFt0aRQ6wxxcKm7QKgfLPI0TWO47UcY/f+vA=
github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9/go.mod h1:1K5hEzsMBLTPdRJKEHqBFJ8Zt2VRqDhomcQ11KH0WW4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=

4
devops/install-capstone.sh
View File

@ -5,8 +5,8 @@ if (( $EUID != 0 )); then
SUDO='sudo'
fi
curl https://github.com/capstone-engine/capstone/archive/4.0.2.tar.gz -Lo ./capstone.tar.gz \
&& tar -xzf capstone.tar.gz && mv ./capstone-* ./capstone \
curl https://github.com/capstone-engine/capstone/releases/download/5.0-rc2/capstone-5.0-rc2.tar.xz -Lo ./capstone.tar.xz \
&& tar -xf capstone.tar.xz && mv ./capstone-* ./capstone \
&& cd capstone \
&& CAPSTONE_ARCHS="aarch64 x86" ./make.sh \
&& $SUDO ./make.sh install

4
devops/linux-arm64-musl-go-libpcap-capstone-bpf/Dockerfile
View File

@ -20,8 +20,8 @@ RUN ./configure --host=arm && make \
WORKDIR /work
# Build and install Capstone from source
RUN curl https://github.com/capstone-engine/capstone/archive/4.0.2.tar.gz -Lo ./capstone.tar.gz \
&& tar -xzf capstone.tar.gz && mv ./capstone-* ./capstone
RUN curl https://github.com/capstone-engine/capstone/releases/download/5.0-rc2/capstone-5.0-rc2.tar.xz -Lo ./capstone.tar.xz \
&& tar -xf capstone.tar.xz && mv ./capstone-* ./capstone
WORKDIR /work/capstone
RUN CAPSTONE_ARCHS="aarch64" CAPSTONE_STATIC=yes ./make.sh \
&& cp /work/capstone/libcapstone.a /usr/xcc/aarch64-linux-musl-cross/lib/gcc/aarch64-linux-musl/*/

3
devops/linux-arm64-musl-go-libpcap-capstone-bpf/build-push.sh
View File

@ -1,4 +1,5 @@
#!/bin/bash
set -e
docker build . -t up9inc/linux-arm64-musl-go-libpcap-capstone-bpf && docker push up9inc/linux-arm64-musl-go-libpcap-capstone-bpf
# Build it on x86_64
docker build . -t up9inc/linux-arm64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 && docker push up9inc/linux-arm64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2

4
devops/linux-x86_64-musl-go-libpcap-capstone-bpf/Dockerfile
View File

@ -44,8 +44,8 @@ RUN ./configure --host=x86_64 && make \
WORKDIR /
# Build and install Capstone from source
RUN curl https://github.com/capstone-engine/capstone/archive/4.0.2.tar.gz -Lo ./capstone.tar.gz \
&& tar -xzf capstone.tar.gz && mv ./capstone-* ./capstone
RUN curl https://github.com/capstone-engine/capstone/releases/download/5.0-rc2/capstone-5.0-rc2.tar.xz -Lo ./capstone.tar.xz \
&& tar -xf capstone.tar.xz && mv ./capstone-* ./capstone
WORKDIR /capstone
RUN CAPSTONE_ARCHS="x86" CAPSTONE_STATIC=yes ./make.sh \
&& cp /capstone/libcapstone.a /usr/local/musl/lib/gcc/x86_64-unknown-linux-musl/*/

3
devops/linux-x86_64-musl-go-libpcap-capstone-bpf/build-push.sh
View File

@ -1,4 +1,5 @@
#!/bin/bash
set -e
docker build . -t up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf && docker push up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf
# Build it on arm64
docker build . -t up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 && docker push up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2

2
tap/go.mod
View File

@ -8,7 +8,7 @@ require (
github.com/go-errors/errors v1.4.2
github.com/google/gopacket v1.1.19
github.com/hashicorp/golang-lru v0.5.4
github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e
github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9
github.com/shirou/gopsutil v3.21.11+incompatible
github.com/struCoder/pidusage v0.2.1
github.com/up9inc/mizu/logger v0.0.0

4
tap/go.sum
View File

@ -81,8 +81,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e h1:6J5obSn9umEThiYzWzndcPOZR0Qj/sVCZpH6V1G7yNE=
github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e/go.mod h1:1K5hEzsMBLTPdRJKEHqBFJ8Zt2VRqDhomcQ11KH0WW4=
github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9 h1:1KszOoXSFt0aRQ6wxxcKm7QKgfLPI0TWO47UcY/f+vA=
github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9/go.mod h1:1K5hEzsMBLTPdRJKEHqBFJ8Zt2VRqDhomcQ11KH0WW4=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=

32
tap/tlstapper/bpf/go_uprobes.c
View File

@ -60,7 +60,18 @@ Capstone Engine: https://www.capstone-engine.org/
static __always_inline __u32 go_crypto_tls_get_fd_from_tcp_conn(struct pt_regs *ctx) {
struct go_interface conn;
long err = bpf_probe_read(&conn, sizeof(conn), (void*)GO_ABI_INTERNAL_PT_REGS_R1(ctx));
long err;
__u64 addr;
#if defined(bpf_target_arm64)
err = bpf_probe_read(&addr, sizeof(addr), (void*)GO_ABI_INTERNAL_PT_REGS_SP(ctx)+0x8);
if (err != 0) {
return invalid_fd;
}
#else
addr = GO_ABI_INTERNAL_PT_REGS_R1(ctx);
#endif
err = bpf_probe_read(&conn, sizeof(conn), (void*)addr);
if (err != 0) {
return invalid_fd;
}
@ -88,14 +99,23 @@ static __always_inline void go_crypto_tls_uprobe(struct pt_regs *ctx, struct bpf
}
struct ssl_info info = new_ssl_info();
long err;
#if defined(bpf_target_arm64)
err = bpf_probe_read(&info.buffer_len, sizeof(__u32), (void*)GO_ABI_INTERNAL_PT_REGS_SP(ctx)+0x18);
if (err != 0) {
log_error(ctx, LOG_ERROR_READING_BYTES_COUNT, pid_tgid, err, ORIGIN_SSL_UPROBE_CODE);
return;
}
#else
info.buffer_len = GO_ABI_INTERNAL_PT_REGS_R2(ctx);
#endif
info.buffer = (void*)GO_ABI_INTERNAL_PT_REGS_R4(ctx);
info.fd = go_crypto_tls_get_fd_from_tcp_conn(ctx);
// GO_ABI_INTERNAL_PT_REGS_GP is Goroutine address
__u64 pid_fp = pid << 32 | GO_ABI_INTERNAL_PT_REGS_GP(ctx);
long err = bpf_map_update_elem(go_context, &pid_fp, &info, BPF_ANY);
err = bpf_map_update_elem(go_context, &pid_fp, &info, BPF_ANY);
if (err != 0) {
log_error(ctx, LOG_ERROR_PUTTING_SSL_CONTEXT, pid_tgid, err, 0l);
@ -130,7 +150,15 @@ static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct
// In case of read, the length is determined on return
if (flags == FLAGS_IS_READ_BIT) {
#if defined(bpf_target_arm64)
// On ARM64 we look at a general-purpose register as an indicator of error return
if (GO_ABI_INTERNAL_PT_REGS_R6(ctx) == 0x10) {
return;
}
info.buffer_len = GO_ABI_INTERNAL_PT_REGS_R7(ctx); // n in return n, nil
#else
info.buffer_len = GO_ABI_INTERNAL_PT_REGS_R1(ctx); // n in return n, nil
#endif
// This check achieves ignoring 0 length reads (the reads result with an error)
if (info.buffer_len <= 0) {
return;

19
tap/tlstapper/bpf/include/go_abi_internal.h
View File

@ -66,11 +66,12 @@ https://go.googlesource.com/go/+/refs/heads/dev.regabi/src/cmd/compile/internal-
https://github.com/golang/go/blob/go1.17.6/src/cmd/compile/internal/ssa/gen/AMD64Ops.go#L100
*/
#define GO_ABI_INTERNAL_PT_REGS_R1(x) ((x)->eax)
#define GO_ABI_INTERNAL_PT_REGS_P2(x) ((x)->ecx)
#define GO_ABI_INTERNAL_PT_REGS_P3(x) ((x)->edx)
#define GO_ABI_INTERNAL_PT_REGS_P4(x) 0
#define GO_ABI_INTERNAL_PT_REGS_P5(x) 0
#define GO_ABI_INTERNAL_PT_REGS_P6(x) 0
#define GO_ABI_INTERNAL_PT_REGS_R2(x) ((x)->ecx)
#define GO_ABI_INTERNAL_PT_REGS_R3(x) ((x)->edx)
#define GO_ABI_INTERNAL_PT_REGS_R4(x) 0
#define GO_ABI_INTERNAL_PT_REGS_R5(x) 0
#define GO_ABI_INTERNAL_PT_REGS_R6(x) 0
#define GO_ABI_INTERNAL_PT_REGS_R7(x) 0
#define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->esp)
#define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->ebp)
#define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->e14)
@ -83,6 +84,7 @@ https://github.com/golang/go/blob/go1.17.6/src/cmd/compile/internal/ssa/gen/AMD6
#define GO_ABI_INTERNAL_PT_REGS_R4(x) ((x)->rbx)
#define GO_ABI_INTERNAL_PT_REGS_R5(x) ((x)->rbp)
#define GO_ABI_INTERNAL_PT_REGS_R6(x) ((x)->rsi)
#define GO_ABI_INTERNAL_PT_REGS_R7(x) ((x)->rdi)
#define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->rsp)
#define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->rbp)
#define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->r14)
@ -101,7 +103,8 @@ https://github.com/golang/go/blob/go1.17.6/src/cmd/compile/internal/ssa/gen/ARM6
#define GO_ABI_INTERNAL_PT_REGS_R4(x) ((x)->uregs[3])
#define GO_ABI_INTERNAL_PT_REGS_R5(x) ((x)->uregs[4])
#define GO_ABI_INTERNAL_PT_REGS_R6(x) ((x)->uregs[5])
#define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->uregs[14])
#define GO_ABI_INTERNAL_PT_REGS_R7(x) ((x)->uregs[6])
#define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->uregs[13])
#define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->uregs[29])
#define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->uregs[28])
@ -116,7 +119,8 @@ struct pt_regs;
#define GO_ABI_INTERNAL_PT_REGS_R4(x) (((PT_REGS_ARM64 *)(x))->regs[3])
#define GO_ABI_INTERNAL_PT_REGS_R5(x) (((PT_REGS_ARM64 *)(x))->regs[4])
#define GO_ABI_INTERNAL_PT_REGS_R6(x) (((PT_REGS_ARM64 *)(x))->regs[5])
#define GO_ABI_INTERNAL_PT_REGS_SP(x) (((PT_REGS_ARM64 *)(x))->regs[30])
#define GO_ABI_INTERNAL_PT_REGS_R7(x) (((PT_REGS_ARM64 *)(x))->regs[6])
#define GO_ABI_INTERNAL_PT_REGS_SP(x) (((PT_REGS_ARM64 *)(x))->sp)
#define GO_ABI_INTERNAL_PT_REGS_FP(x) (((PT_REGS_ARM64 *)(x))->regs[29])
#define GO_ABI_INTERNAL_PT_REGS_GP(x) (((PT_REGS_ARM64 *)(x))->regs[28])
@ -132,6 +136,7 @@ https://github.com/golang/go/blob/go1.17.6/src/cmd/compile/internal/ssa/gen/PPC6
#define GO_ABI_INTERNAL_PT_REGS_R4(x) ((x)->gpr[6])
#define GO_ABI_INTERNAL_PT_REGS_R5(x) ((x)->gpr[7])
#define GO_ABI_INTERNAL_PT_REGS_R6(x) ((x)->gpr[8])
#define GO_ABI_INTERNAL_PT_REGS_R7(x) ((x)->gpr[9])
#define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->sp)
#define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->gpr[12])
#define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->gpr[30])

14
tap/tlstapper/go_offsets.go
View File

@ -77,7 +77,7 @@ func getOffsets(filePath string) (offsets map[string]*goExtendedOffset, err erro
case "arm64":
engine, err = gapstone.New(
gapstone.CS_ARCH_ARM64,
gapstone.CS_MODE_ARM,
gapstone.CS_MODE_LITTLE_ENDIAN,
)
default:
err = fmt.Errorf("Unsupported architecture: %v", runtime.GOARCH)
@ -86,6 +86,16 @@ func getOffsets(filePath string) (offsets map[string]*goExtendedOffset, err erro
return
}
engineMajor, engineMinor := engine.Version()
logger.Log.Infof(
"Disassembling %s with Capstone %d.%d (arch: %d, mode: %d)",
filePath,
engineMajor,
engineMinor,
engine.Arch(),
engine.Mode(),
)
offsets = make(map[string]*goExtendedOffset)
var fd *os.File
fd, err = os.Open(filePath)
@ -133,7 +143,7 @@ func getOffsets(filePath string) (offsets map[string]*goExtendedOffset, err erro
extendedOffset := &goExtendedOffset{enter: offset}
// source: https://gist.github.com/grantseltzer/3efa8ecc5de1fb566e8091533050d608
// skip over any symbols that aren't functinons/methods
// skip over any symbols that aren't functions/methods
if sym.Info != byte(2) && sym.Info != byte(18) {
offsets[sym.Name] = extendedOffset
continue

BIN
tap/tlstapper/tlstapper_bpfel_arm64.o
View File

Binary file not shown.

BIN
tap/tlstapper/tlstapper_bpfel_x86.o
View File

Binary file not shown.