diff --git a/Dockerfile b/Dockerfile index fa6a03849..9c7beb205 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,14 +54,14 @@ ENV BPF_TARGET=arm64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64" ### Builder image for x86-64 to AArch64 cross-compilation -FROM up9inc/linux-arm64-musl-go-libpcap-capstone-bpf AS builder-from-amd64-to-arm64v8 +FROM up9inc/linux-arm64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 AS builder-from-amd64-to-arm64v8 ENV CGO_ENABLED=1 GOOS=linux ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap -I/work/capstone/include" ENV BPF_TARGET=arm64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64 -I/usr/xcc/aarch64-linux-musl-cross/aarch64-linux-musl/include/" ### Builder image for AArch64 to x86-64 cross-compilation -FROM up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf AS builder-from-arm64v8-to-amd64 +FROM up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 AS builder-from-arm64v8-to-amd64 ENV CGO_ENABLED=1 GOOS=linux ENV GOARCH=amd64 CGO_CFLAGS="-I/libpcap -I/capstone/include" ENV BPF_TARGET=amd64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_x86 -I/usr/local/musl/x86_64-unknown-linux-musl/include/" diff --git a/agent/go.mod b/agent/go.mod index 9e44f8513..3fb330b82 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -83,7 +83,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.14.2 // indirect - github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e // indirect + github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/agent/go.sum b/agent/go.sum index 0dc224851..bce84c23e 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -453,8 +453,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw= github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e h1:6J5obSn9umEThiYzWzndcPOZR0Qj/sVCZpH6V1G7yNE= -github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e/go.mod h1:1K5hEzsMBLTPdRJKEHqBFJ8Zt2VRqDhomcQ11KH0WW4= +github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9 h1:1KszOoXSFt0aRQ6wxxcKm7QKgfLPI0TWO47UcY/f+vA= +github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9/go.mod h1:1K5hEzsMBLTPdRJKEHqBFJ8Zt2VRqDhomcQ11KH0WW4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= diff --git a/devops/install-capstone.sh b/devops/install-capstone.sh index 4db790f1f..28893a110 100755 --- a/devops/install-capstone.sh +++ b/devops/install-capstone.sh @@ -5,8 +5,8 @@ if (( $EUID != 0 )); then SUDO='sudo' fi -curl https://github.com/capstone-engine/capstone/archive/4.0.2.tar.gz -Lo ./capstone.tar.gz \ - && tar -xzf capstone.tar.gz && mv ./capstone-* ./capstone \ +curl https://github.com/capstone-engine/capstone/releases/download/5.0-rc2/capstone-5.0-rc2.tar.xz -Lo ./capstone.tar.xz \ + && tar -xf capstone.tar.xz && mv ./capstone-* ./capstone \ && cd capstone \ && CAPSTONE_ARCHS="aarch64 x86" ./make.sh \ && $SUDO ./make.sh install diff --git a/devops/linux-arm64-musl-go-libpcap-capstone-bpf/Dockerfile b/devops/linux-arm64-musl-go-libpcap-capstone-bpf/Dockerfile index d75a43621..afb9eef5f 100644 --- a/devops/linux-arm64-musl-go-libpcap-capstone-bpf/Dockerfile +++ b/devops/linux-arm64-musl-go-libpcap-capstone-bpf/Dockerfile @@ -20,8 +20,8 @@ RUN ./configure --host=arm && make \ WORKDIR /work # Build and install Capstone from source -RUN curl https://github.com/capstone-engine/capstone/archive/4.0.2.tar.gz -Lo ./capstone.tar.gz \ - && tar -xzf capstone.tar.gz && mv ./capstone-* ./capstone +RUN curl https://github.com/capstone-engine/capstone/releases/download/5.0-rc2/capstone-5.0-rc2.tar.xz -Lo ./capstone.tar.xz \ + && tar -xf capstone.tar.xz && mv ./capstone-* ./capstone WORKDIR /work/capstone RUN CAPSTONE_ARCHS="aarch64" CAPSTONE_STATIC=yes ./make.sh \ && cp /work/capstone/libcapstone.a /usr/xcc/aarch64-linux-musl-cross/lib/gcc/aarch64-linux-musl/*/ diff --git a/devops/linux-arm64-musl-go-libpcap-capstone-bpf/build-push.sh b/devops/linux-arm64-musl-go-libpcap-capstone-bpf/build-push.sh index 384b05fdd..f4092dd9c 100755 --- a/devops/linux-arm64-musl-go-libpcap-capstone-bpf/build-push.sh +++ b/devops/linux-arm64-musl-go-libpcap-capstone-bpf/build-push.sh @@ -1,4 +1,5 @@ #!/bin/bash set -e -docker build . -t up9inc/linux-arm64-musl-go-libpcap-capstone-bpf && docker push up9inc/linux-arm64-musl-go-libpcap-capstone-bpf +# Build it on x86_64 +docker build . -t up9inc/linux-arm64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 && docker push up9inc/linux-arm64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 diff --git a/devops/linux-x86_64-musl-go-libpcap-capstone-bpf/Dockerfile b/devops/linux-x86_64-musl-go-libpcap-capstone-bpf/Dockerfile index 55f941e04..90ce9fe2f 100644 --- a/devops/linux-x86_64-musl-go-libpcap-capstone-bpf/Dockerfile +++ b/devops/linux-x86_64-musl-go-libpcap-capstone-bpf/Dockerfile @@ -44,8 +44,8 @@ RUN ./configure --host=x86_64 && make \ WORKDIR / # Build and install Capstone from source -RUN curl https://github.com/capstone-engine/capstone/archive/4.0.2.tar.gz -Lo ./capstone.tar.gz \ - && tar -xzf capstone.tar.gz && mv ./capstone-* ./capstone +RUN curl https://github.com/capstone-engine/capstone/releases/download/5.0-rc2/capstone-5.0-rc2.tar.xz -Lo ./capstone.tar.xz \ + && tar -xf capstone.tar.xz && mv ./capstone-* ./capstone WORKDIR /capstone RUN CAPSTONE_ARCHS="x86" CAPSTONE_STATIC=yes ./make.sh \ && cp /capstone/libcapstone.a /usr/local/musl/lib/gcc/x86_64-unknown-linux-musl/*/ diff --git a/devops/linux-x86_64-musl-go-libpcap-capstone-bpf/build-push.sh b/devops/linux-x86_64-musl-go-libpcap-capstone-bpf/build-push.sh index 63c5e816e..26f8daba6 100755 --- a/devops/linux-x86_64-musl-go-libpcap-capstone-bpf/build-push.sh +++ b/devops/linux-x86_64-musl-go-libpcap-capstone-bpf/build-push.sh @@ -1,4 +1,5 @@ #!/bin/bash set -e -docker build . -t up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf && docker push up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf +# Build it on arm64 +docker build . -t up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 && docker push up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 diff --git a/tap/go.mod b/tap/go.mod index f2bd18d1a..ea065e22d 100644 --- a/tap/go.mod +++ b/tap/go.mod @@ -8,7 +8,7 @@ require ( github.com/go-errors/errors v1.4.2 github.com/google/gopacket v1.1.19 github.com/hashicorp/golang-lru v0.5.4 - github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e + github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9 github.com/shirou/gopsutil v3.21.11+incompatible github.com/struCoder/pidusage v0.2.1 github.com/up9inc/mizu/logger v0.0.0 diff --git a/tap/go.sum b/tap/go.sum index ef1565919..5f773059d 100644 --- a/tap/go.sum +++ b/tap/go.sum @@ -81,8 +81,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e h1:6J5obSn9umEThiYzWzndcPOZR0Qj/sVCZpH6V1G7yNE= -github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e/go.mod h1:1K5hEzsMBLTPdRJKEHqBFJ8Zt2VRqDhomcQ11KH0WW4= +github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9 h1:1KszOoXSFt0aRQ6wxxcKm7QKgfLPI0TWO47UcY/f+vA= +github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9/go.mod h1:1K5hEzsMBLTPdRJKEHqBFJ8Zt2VRqDhomcQ11KH0WW4= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= diff --git a/tap/tlstapper/bpf/go_uprobes.c b/tap/tlstapper/bpf/go_uprobes.c index fbee44f3d..19eef8603 100644 --- a/tap/tlstapper/bpf/go_uprobes.c +++ b/tap/tlstapper/bpf/go_uprobes.c @@ -60,7 +60,18 @@ Capstone Engine: https://www.capstone-engine.org/ static __always_inline __u32 go_crypto_tls_get_fd_from_tcp_conn(struct pt_regs *ctx) { struct go_interface conn; - long err = bpf_probe_read(&conn, sizeof(conn), (void*)GO_ABI_INTERNAL_PT_REGS_R1(ctx)); + long err; + __u64 addr; +#if defined(bpf_target_arm64) + err = bpf_probe_read(&addr, sizeof(addr), (void*)GO_ABI_INTERNAL_PT_REGS_SP(ctx)+0x8); + if (err != 0) { + return invalid_fd; + } +#else + addr = GO_ABI_INTERNAL_PT_REGS_R1(ctx); +#endif + + err = bpf_probe_read(&conn, sizeof(conn), (void*)addr); if (err != 0) { return invalid_fd; } @@ -88,14 +99,23 @@ static __always_inline void go_crypto_tls_uprobe(struct pt_regs *ctx, struct bpf } struct ssl_info info = new_ssl_info(); + long err; +#if defined(bpf_target_arm64) + err = bpf_probe_read(&info.buffer_len, sizeof(__u32), (void*)GO_ABI_INTERNAL_PT_REGS_SP(ctx)+0x18); + if (err != 0) { + log_error(ctx, LOG_ERROR_READING_BYTES_COUNT, pid_tgid, err, ORIGIN_SSL_UPROBE_CODE); + return; + } +#else info.buffer_len = GO_ABI_INTERNAL_PT_REGS_R2(ctx); +#endif info.buffer = (void*)GO_ABI_INTERNAL_PT_REGS_R4(ctx); info.fd = go_crypto_tls_get_fd_from_tcp_conn(ctx); // GO_ABI_INTERNAL_PT_REGS_GP is Goroutine address __u64 pid_fp = pid << 32 | GO_ABI_INTERNAL_PT_REGS_GP(ctx); - long err = bpf_map_update_elem(go_context, &pid_fp, &info, BPF_ANY); + err = bpf_map_update_elem(go_context, &pid_fp, &info, BPF_ANY); if (err != 0) { log_error(ctx, LOG_ERROR_PUTTING_SSL_CONTEXT, pid_tgid, err, 0l); @@ -130,7 +150,15 @@ static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct // In case of read, the length is determined on return if (flags == FLAGS_IS_READ_BIT) { +#if defined(bpf_target_arm64) + // On ARM64 we look at a general-purpose register as an indicator of error return + if (GO_ABI_INTERNAL_PT_REGS_R6(ctx) == 0x10) { + return; + } + info.buffer_len = GO_ABI_INTERNAL_PT_REGS_R7(ctx); // n in return n, nil +#else info.buffer_len = GO_ABI_INTERNAL_PT_REGS_R1(ctx); // n in return n, nil +#endif // This check achieves ignoring 0 length reads (the reads result with an error) if (info.buffer_len <= 0) { return; diff --git a/tap/tlstapper/bpf/include/go_abi_internal.h b/tap/tlstapper/bpf/include/go_abi_internal.h index 368b32679..cf5564ce0 100644 --- a/tap/tlstapper/bpf/include/go_abi_internal.h +++ b/tap/tlstapper/bpf/include/go_abi_internal.h @@ -66,11 +66,12 @@ https://go.googlesource.com/go/+/refs/heads/dev.regabi/src/cmd/compile/internal- https://github.com/golang/go/blob/go1.17.6/src/cmd/compile/internal/ssa/gen/AMD64Ops.go#L100 */ #define GO_ABI_INTERNAL_PT_REGS_R1(x) ((x)->eax) -#define GO_ABI_INTERNAL_PT_REGS_P2(x) ((x)->ecx) -#define GO_ABI_INTERNAL_PT_REGS_P3(x) ((x)->edx) -#define GO_ABI_INTERNAL_PT_REGS_P4(x) 0 -#define GO_ABI_INTERNAL_PT_REGS_P5(x) 0 -#define GO_ABI_INTERNAL_PT_REGS_P6(x) 0 +#define GO_ABI_INTERNAL_PT_REGS_R2(x) ((x)->ecx) +#define GO_ABI_INTERNAL_PT_REGS_R3(x) ((x)->edx) +#define GO_ABI_INTERNAL_PT_REGS_R4(x) 0 +#define GO_ABI_INTERNAL_PT_REGS_R5(x) 0 +#define GO_ABI_INTERNAL_PT_REGS_R6(x) 0 +#define GO_ABI_INTERNAL_PT_REGS_R7(x) 0 #define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->esp) #define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->ebp) #define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->e14) @@ -83,6 +84,7 @@ https://github.com/golang/go/blob/go1.17.6/src/cmd/compile/internal/ssa/gen/AMD6 #define GO_ABI_INTERNAL_PT_REGS_R4(x) ((x)->rbx) #define GO_ABI_INTERNAL_PT_REGS_R5(x) ((x)->rbp) #define GO_ABI_INTERNAL_PT_REGS_R6(x) ((x)->rsi) +#define GO_ABI_INTERNAL_PT_REGS_R7(x) ((x)->rdi) #define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->rsp) #define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->rbp) #define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->r14) @@ -101,7 +103,8 @@ https://github.com/golang/go/blob/go1.17.6/src/cmd/compile/internal/ssa/gen/ARM6 #define GO_ABI_INTERNAL_PT_REGS_R4(x) ((x)->uregs[3]) #define GO_ABI_INTERNAL_PT_REGS_R5(x) ((x)->uregs[4]) #define GO_ABI_INTERNAL_PT_REGS_R6(x) ((x)->uregs[5]) -#define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->uregs[14]) +#define GO_ABI_INTERNAL_PT_REGS_R7(x) ((x)->uregs[6]) +#define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->uregs[13]) #define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->uregs[29]) #define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->uregs[28]) @@ -116,7 +119,8 @@ struct pt_regs; #define GO_ABI_INTERNAL_PT_REGS_R4(x) (((PT_REGS_ARM64 *)(x))->regs[3]) #define GO_ABI_INTERNAL_PT_REGS_R5(x) (((PT_REGS_ARM64 *)(x))->regs[4]) #define GO_ABI_INTERNAL_PT_REGS_R6(x) (((PT_REGS_ARM64 *)(x))->regs[5]) -#define GO_ABI_INTERNAL_PT_REGS_SP(x) (((PT_REGS_ARM64 *)(x))->regs[30]) +#define GO_ABI_INTERNAL_PT_REGS_R7(x) (((PT_REGS_ARM64 *)(x))->regs[6]) +#define GO_ABI_INTERNAL_PT_REGS_SP(x) (((PT_REGS_ARM64 *)(x))->sp) #define GO_ABI_INTERNAL_PT_REGS_FP(x) (((PT_REGS_ARM64 *)(x))->regs[29]) #define GO_ABI_INTERNAL_PT_REGS_GP(x) (((PT_REGS_ARM64 *)(x))->regs[28]) @@ -132,6 +136,7 @@ https://github.com/golang/go/blob/go1.17.6/src/cmd/compile/internal/ssa/gen/PPC6 #define GO_ABI_INTERNAL_PT_REGS_R4(x) ((x)->gpr[6]) #define GO_ABI_INTERNAL_PT_REGS_R5(x) ((x)->gpr[7]) #define GO_ABI_INTERNAL_PT_REGS_R6(x) ((x)->gpr[8]) +#define GO_ABI_INTERNAL_PT_REGS_R7(x) ((x)->gpr[9]) #define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->sp) #define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->gpr[12]) #define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->gpr[30]) diff --git a/tap/tlstapper/go_offsets.go b/tap/tlstapper/go_offsets.go index d5de61462..7131d24b5 100644 --- a/tap/tlstapper/go_offsets.go +++ b/tap/tlstapper/go_offsets.go @@ -77,7 +77,7 @@ func getOffsets(filePath string) (offsets map[string]*goExtendedOffset, err erro case "arm64": engine, err = gapstone.New( gapstone.CS_ARCH_ARM64, - gapstone.CS_MODE_ARM, + gapstone.CS_MODE_LITTLE_ENDIAN, ) default: err = fmt.Errorf("Unsupported architecture: %v", runtime.GOARCH) @@ -86,6 +86,16 @@ func getOffsets(filePath string) (offsets map[string]*goExtendedOffset, err erro return } + engineMajor, engineMinor := engine.Version() + logger.Log.Infof( + "Disassembling %s with Capstone %d.%d (arch: %d, mode: %d)", + filePath, + engineMajor, + engineMinor, + engine.Arch(), + engine.Mode(), + ) + offsets = make(map[string]*goExtendedOffset) var fd *os.File fd, err = os.Open(filePath) @@ -133,7 +143,7 @@ func getOffsets(filePath string) (offsets map[string]*goExtendedOffset, err erro extendedOffset := &goExtendedOffset{enter: offset} // source: https://gist.github.com/grantseltzer/3efa8ecc5de1fb566e8091533050d608 - // skip over any symbols that aren't functinons/methods + // skip over any symbols that aren't functions/methods if sym.Info != byte(2) && sym.Info != byte(18) { offsets[sym.Name] = extendedOffset continue diff --git a/tap/tlstapper/tlstapper_bpfel_arm64.o b/tap/tlstapper/tlstapper_bpfel_arm64.o index 78016c738..3b150c59d 100644 Binary files a/tap/tlstapper/tlstapper_bpfel_arm64.o and b/tap/tlstapper/tlstapper_bpfel_arm64.o differ diff --git a/tap/tlstapper/tlstapper_bpfel_x86.o b/tap/tlstapper/tlstapper_bpfel_x86.o index c65c4516d..50937a913 100644 Binary files a/tap/tlstapper/tlstapper_bpfel_x86.o and b/tap/tlstapper/tlstapper_bpfel_x86.o differ