Merge pull request #3101 from kmjohansen/topics/ebpf

Enable building of bcc in kernel build
This commit is contained in:
Rolf Neugebauer 2018-08-16 14:57:43 +02:00 committed by GitHub
commit c0aecf8f26
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 294 additions and 146 deletions

31
docs/kernel-bcc.md Normal file
View File

@ -0,0 +1,31 @@
# Using the bcc utility with LinuxKit
The `bcc` utility is a standard Linux tool to access performance
counters, trace events and access various other kernel internals for
performance analysis.
The `bcc` utility needs to matched be with the kernel. For recent
kernel build, LinuxKit provides a `linuxkit/kernel-bcc` package with
a matching tag for each kernel under `linuxkit/kernel`.
The preferred way of using the `linuxkit/kernel-bcc` package is to
add it to the `init` section. This adds `/usr/share/bcc` to the
systems' root filesystem. From there it can be
- bind mounted into your container
- accessed via `/proc/1/root/usr/share/bcc/tools` from with in the `getty`
or `ssh` container.
- accessed via a nsenter of `/bin/ash` of proc 1.
If you want to use `bcc` you may also want to remove the `sysctl`
container, or alternatively, disable the kernel pointer restriction it
enables by default:
```
echo 0 > /proc/sys/kernel/kptr_restrict
```
Now, `bcc` is ready to use. The LinuxKit `bcc` package contains
the `bcc` binary, example and tool scripts, and kernel headers for the
associated kernel build.

112
kernel/Dockerfile.bcc Normal file
View File

@ -0,0 +1,112 @@
ARG IMAGE
FROM ${IMAGE} as ksrc
FROM linuxkit/alpine:8a89682421abf0d886d777235a05f4a04a736df2 AS build
RUN apk update && apk upgrade -a && \
apk add --no-cache \
argp-standalone \
autoconf \
automake \
bison \
build-base \
clang \
clang-dev \
clang-static \
cmake \
curl \
flex-dev \
fts-dev \
gettext-dev \
git \
iperf3 \
libedit-dev \
libtool \
llvm \
llvm-dev \
llvm-static \
luajit-dev \
m4 \
python \
zlib-dev \
&& true
RUN ln -s /usr/lib/cmake/llvm5/ /usr/lib/cmake/llvm && \
ln -s /usr/include/llvm5/llvm-c/ /usr/include/llvm-c && \
ln -s /usr/include/llvm5/llvm/ /usr/include/llvm
WORKDIR /build
COPY ./bcc.patches/ ./
RUN mv error.h /usr/include/
RUN mv cdefs.h /usr/include/sys/
ENV ELFUTILS_VERSION=0.165
ENV ELFUTILS_SHA256="a7fc9277192caaa5f30b47e8c0518dbcfd8c4a19c6493a63d511d804290ce972"
RUN curl -sSL -O https://fedorahosted.org/releases/e/l/elfutils/0.165/elfutils-$ELFUTILS_VERSION.tar.bz2
RUN echo "${ELFUTILS_SHA256} /build/elfutils-$ELFUTILS_VERSION.tar.bz2" | sha256sum -c - && \
tar xjf elfutils-$ELFUTILS_VERSION.tar.bz2 && \
cd elfutils-$ELFUTILS_VERSION && \
patch -p1 < ../100-musl-compat.patch && \
patch -p0 < ../decl.patch && \
patch -p0 < ../intl.patch
ENV BCC_COMMIT=6972806729da00ecda0235abac61d66c8fad7fad
RUN git clone https://github.com/iovisor/bcc.git && cd bcc && git checkout $BCC_COMMIT
RUN cd bcc && patch -p0 < ../bcc-gnuism.patch && patch -p0 < ../bcc-lua.patch
ENV LJSYSCALL_COMMIT=e587f8c55aad3955dddab3a4fa6c1968037b5c6e
RUN git clone https://github.com/justincormack/ljsyscall.git && cd ljsyscall && git checkout $LJSYSCALL_COMMIT
COPY --from=ksrc /kernel-headers.tar /build
RUN tar xf /build/kernel-headers.tar
COPY --from=ksrc /kernel-dev.tar /build
RUN tar xf /build/kernel-dev.tar
COPY --from=ksrc /kernel.tar /build
RUN tar xf /build/kernel.tar
RUN cd /build
RUN cd elfutils-$ELFUTILS_VERSION && \
aclocal && \
automake && \
./configure --prefix=/usr CFLAGS=-Wno-strict-aliasing && \
make -C libelf && make -C libelf install
RUN mkdir -p bcc/build && cd bcc/build && \
cmake .. -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_C_FLAGS="-I/build/usr/include" -DCMAKE_CXX_FLAGS="-I/build/usr/include" -DCMAKE_INSTALL_PREFIX=/usr -DLUAJIT_INCLUDE_DIR=/usr/include/luajit-2.1 && \
make && \
make install
RUN mkdir -p /usr/local/share/lua/5.1/ && cd ljsyscall && cp -a *.lua syscall /usr/local/share/lua/5.1/
RUN mkdir -p /out/usr/
RUN cp -a /build/usr/src /out/usr/
RUN cp -a /build/usr/include /out/usr
RUN mkdir -p /out/usr/lib
RUN cp -a /usr/lib/libelf* /out/usr/lib/
RUN cp -a /usr/lib/libstdc* /out/usr/lib/
RUN cp -a /usr/lib/libintl* /out/usr/lib/
RUN cp -a /usr/lib64/* /out/usr/lib/
RUN mkdir -p /out/usr/lib/python2.7
RUN cp -a /usr/lib/python2.7/site-packages /out/usr/lib/python2.7/
RUN mkdir -p /out/usr/share
RUN cp -a /usr/share/bcc /out/usr/share/
RUN mkdir -p /out/usr/bin
RUN cp -a /usr/bin/bcc-lua /out/usr/bin/
RUN mkdir -p /out/usr/local/share/
RUN cp -a /usr/local/share/lua /out/usr/local/share/
ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/lib64
FROM linuxkit/alpine:8a89682421abf0d886d777235a05f4a04a736df2 as mirror
RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/
RUN apk update && apk upgrade -a && \
apk add --no-cache --initdb -p /out \
busybox \
luajit \
python \
zlib \
&& true
FROM scratch
ENTRYPOINT []
CMD []
WORKDIR /
COPY --from=mirror /out /
COPY --from=build /out /

View File

@ -15,6 +15,7 @@
# Name and Org on Hub
ORG?=linuxkit
IMAGE:=kernel
IMAGE_BCC:=kernel-bcc
IMAGE_PERF:=kernel-perf
IMAGE_ZFS:=zfs-kmod
@ -192,6 +193,46 @@ push: push_perf_$(2)$(3)$(4)
forcepush: forcepush_perf_$(2)$(3)$(4)
endif
# For BCC, only build on x86 and only on latest LTS and latest stable.
# That's 4.14.x and 4.17.x at time of authorship.
ifeq ($(ARCH),x86_64)
ifeq ($(2), $(filter $(2),4.14.x 4.17.x))
build_bcc_$(2)$(3)$(4): build_$(2)$(3)$(4)
docker pull $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG)$(SUFFIX) || \
DOCKER_CONTENT_TRUST=0 docker build -f Dockerfile.bcc \
--build-arg IMAGE=$(ORG)/$(IMAGE):$(1)$(3)$(4)-$(TAG)$(SUFFIX) \
--no-cache $(LABEL) -t $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG)$(SUFFIX) .
forcebuild_bcc_$(2)$(3)$(4): build_$(2)$(3)$(4)
DOCKER_CONTENT_TRUST=0 docker build -f Dockerfile.bcc \
--build-arg IMAGE=$(ORG)/$(IMAGE):$(1)$(3)$(4)-$(TAG)$(SUFFIX) \
--no-cache $(LABEL) -t $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG)$(SUFFIX) .
push_bcc_$(2)$(3)$(4): build_bcc_$(2)$(3)$(4)
@if [ x"$(DIRTY)" != x ]; then echo "Your repository is not clean. Will not push image"; exit 1; fi
docker pull $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG)$(SUFFIX) || \
(docker push $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG)$(SUFFIX) && \
docker tag $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG)$(SUFFIX) $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)$(SUFFIX) && \
docker push $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)$(SUFFIX) && \
$(PUSH_MANIFEST) $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG) $(DOCKER_CONTENT_TRUST) && \
$(PUSH_MANIFEST) $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4) $(DOCKER_CONTENT_TRUST))
forcepush_bcc_$(2)$(3)$(4): forcebuild_bcc_$(2)$(3)$(4)
@if [ x"$(DIRTY)" != x ]; then echo "Your repository is not clean. Will not push image"; exit 1; fi
docker push $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG)$(SUFFIX) && \
docker tag $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG)$(SUFFIX) $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)$(SUFFIX) && \
docker push $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)$(SUFFIX) && \
$(PUSH_MANIFEST) $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG) $(DOCKER_CONTENT_TRUST) && \
$(PUSH_MANIFEST) $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4) $(DOCKER_CONTENT_TRUST)
build: build_bcc_$(2)$(3)$(4)
forcebuild: forcebuild_bcc_$(2)$(3)$(4)
push: push_bcc_$(2)$(3)$(4)
forcepush: forcepush_bcc_$(2)$(3)$(4)
endif
endif
ifeq ($(4),)
# ZFS does not compile against -dbg kernels because CONFIG_DEBUG_LOCK_ALLOC
# is incompatible with CDDL, apparently (this is ./configure check)

View File

@ -1,9 +1,9 @@
--- src/cc/usdt.h-orig
+++ src/cc/usdt.h
@@ -86,6 +86,24 @@
ArgumentParser(const char *arg) : arg_(arg), cur_pos_(0) {}
@@ -125,6 +125,24 @@ public:
ArgumentParser_powerpc64(const char *arg) : ArgumentParser(arg) {}
};
+#undef REG_A
+#undef REG_B
+#undef REG_C
@ -23,5 +23,6 @@
+#undef REG_RIP
+
class ArgumentParser_x64 : public ArgumentParser {
private:
enum Register {
REG_A,

View File

@ -0,0 +1,43 @@
--- src/cc/CMakeLists.txt
+++ src/cc/CMakeLists.txt
@@ -52,7 +52,7 @@ target_link_libraries(bcc-loader-static elf)
add_library(bcc-static STATIC
${bcc_common_sources} ${bcc_table_sources} ${bcc_util_sources})
set_target_properties(bcc-static PROPERTIES OUTPUT_NAME bcc)
-set(bcc-lua-static
+add_library(bcc-lua-static STATIC
${bcc_common_sources} ${bcc_table_sources} ${bcc_sym_sources} ${bcc_util_sources})
include(clang_libs)
@@ -64,9 +64,9 @@ set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${llvm_lib_exclude_f
set(bcc_common_libs_for_a b_frontend clang_frontend bpf-static
-Wl,--whole-archive ${clang_libs} ${llvm_libs} -Wl,--no-whole-archive
${LIBELF_LIBRARIES})
-set(bcc_common_libs_for_s ${bcc_common_libs_for_a})
-set(bcc_common_libs_for_lua b_frontend clang_frontend bpf-static
+set(bcc_common_libs_for_s b_frontend clang_frontend bpf-static
${clang_libs} ${llvm_libs} ${LIBELF_LIBRARIES})
+set(bcc_common_libs_for_lua ${bcc_common_libs_for_s})
if(ENABLE_CPP_API)
add_subdirectory(api)
@@ -87,7 +87,7 @@ add_subdirectory(frontends)
# Link against LLVM libraries
target_link_libraries(bcc-shared ${bcc_common_libs_for_s})
target_link_libraries(bcc-static ${bcc_common_libs_for_a} bcc-loader-static)
-set(bcc-lua-static ${bcc-lua-static} ${bcc_common_libs_for_lua})
+target_link_libraries(bcc-lua-static ${bcc_common_libs_for_lua})
install(TARGETS bcc-shared LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
install(FILES ${bcc_table_headers} DESTINATION include/bcc)
--- src/lua/CMakeLists.txt
+++ src/lua/CMakeLists.txt
@@ -23,7 +23,7 @@ if (LUAJIT_LIBRARIES AND LUAJIT)
add_executable(bcc-lua src/main.c bcc.o)
set_target_properties(bcc-lua PROPERTIES LINKER_LANGUAGE C)
target_link_libraries(bcc-lua ${LUAJIT_LIBRARIES})
- target_link_libraries(bcc-lua ${bcc-lua-static})
+ target_link_libraries(bcc-lua -Wl,--whole-archive bcc-lua-static -Wl,--no-whole-archive)
if (NOT COMPILER_NOPIE_FLAG EQUAL "")
target_link_libraries(bcc-lua ${COMPILER_NOPIE_FLAG})
endif()

View File

@ -1,18 +0,0 @@
# iovisor eBPF tools
The [iovisor eBPF tools](https://github.com/iovisor/bcc) are a
collection tools useful for debugging and performance analysis. This
project aims to provide an easy to consume packaging of these tools
for Moby.
It comes in two parts: [ebpf.build](ebpf.build/) is used for building the binaries on alpine and [ebpf.pkg](ebpf.pkg/) is supposed to package the binaries with the required kernel files.
**Note**: The packages currently do not build.
## Roadmap
**Near-term:**
- Make the package build again
**Mid-term:**
- Trim the distribution package to make it easier to consume

View File

@ -1,44 +0,0 @@
FROM alpine:3.5
RUN apk update && apk upgrade -a && \
apk add --no-cache \
argp-standalone \
autoconf \
automake \
bison \
build-base \
clang \
clang-dev \
cmake \
curl \
flex-dev \
fts-dev \
gettext-dev \
git \
iperf \
libedit-dev \
libtool \
llvm \
llvm-dev \
llvm-static \
luajit-dev \
m4 \
python \
zlib-dev \
&& true
WORKDIR /build
COPY . ./
COPY Dockerfile /
COPY error.h /usr/include/
COPY cdefs.h /usr/include/sys/
ENV ELFUTILS_VERSION=0.165
RUN curl -sSL -O https://fedorahosted.org/releases/e/l/elfutils/0.165/elfutils-$ELFUTILS_VERSION.tar.bz2
RUN cat elfutils-$ELFUTILS_VERSION.tar.bz2 | tar xjf - && \
cd elfutils-$ELFUTILS_VERSION && \
patch -p1 < ../100-musl-compat.patch && \
patch -p0 < ../decl.patch && \
patch -p0 < ../intl.patch
ENV BCC_COMMIT=d4fc95d92ec9bace9bd607dfd1833e9e06457486
RUN git clone https://github.com/iovisor/bcc.git && cd bcc && git checkout $BCC_COMMIT
RUN cd bcc && patch -p0 < ../bcc-gnuism.patch
ENV LJSYSCALL_COMMIT=0b266e8f4f751ae894299d24a2d40d16c6cf856f
RUN git clone https://github.com/justincormack/ljsyscall.git && cd ljsyscall && git checkout $LJSYSCALL_COMMIT

View File

@ -1,29 +0,0 @@
.PHONY: tag push
BASE=alpine:3.5
IMAGE=alpine-build-ebpf
default: push
hash: Dockerfile *.patch cdefs.h error.h
DOCKER_CONTENT_TRUST=1 docker pull $(BASE)
tar cf - $^ | docker build --no-cache -t $(IMAGE):build -
docker run --rm $(IMAGE):build sh -c 'cat /Dockerfile /build/*.patch /build/*.h /lib/apk/db/installed | sha1sum' | sed 's/ .*//' > hash
push: hash
docker pull mobylinux/$(IMAGE):$(shell cat hash) || \
(docker tag $(IMAGE):build mobylinux/$(IMAGE):$(shell cat hash) && \
docker push mobylinux/$(IMAGE):$(shell cat hash))
docker rmi $(IMAGE):build
rm -f hash
tag: hash
docker pull mobylinux/$(IMAGE):$(shell cat hash) || \
docker tag $(IMAGE):build mobylinux/$(IMAGE):$(shell cat hash)
docker rmi $(IMAGE):build
rm -f hash
clean:
rm -f hash
.DELETE_ON_ERROR:

View File

@ -1,22 +0,0 @@
## Using eBPF programs
There is now a development image `mobylinux/ebpf:_tag_`. These are currently being built
manually, I will tag one for each kernel release, as you should have a close one, eg
`mobylinux/ebpf:4.9` is currently available.
This image has all the kernel headers, `iovisor/bcc` built with support for C, Python and Lua,
and all sources installed. It is very large so if we are shipping stuff based on this we will
just extract compiled eBPF programs probably, but it is also usable for experiments, debug,
benchmarks etc.
You probably want to run with
`docker run -it -v /sys/kernel/debug:/sys/kernel/debug --privileged --pid=host mobylinux/ebpf:tag sh` for
interactive use as some things use debugfs. You need at least `CAP_SYS_ADMIN` to do anything.
There are examples in `bcc/examples` that should generally just work, I have tried several of
the Lua ones.
Some of the `iovisor/bcc` samples try to access the kernel symbols. For them to work correctly you should also execute:
```sh
echo 0 > /proc/sys/kernel/kptr_restrict
```

View File

@ -1,17 +0,0 @@
# Tag: cc5bf4d39442741eaeffad79bbd9ae049a9253fa
FROM mobylinux/alpine-build-ebpf@sha256:da37d99a8baad18e68c44d3807c86977767ed90ce03133b508842eea3b786d3a
ADD kernel-headers.tar /
ADD kernel-dev.tar /
ADD kernel-modules.tar /
RUN cd elfutils-$ELFUTILS_VERSION && \
automake && \
./configure --prefix=/usr CFLAGS=-Wno-strict-aliasing && \
make -C libelf && make -C libelf install
RUN mkdir -p bcc/build && cd bcc/build && \
cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DLUAJIT_INCLUDE_DIR=/usr/include/luajit-2.1 && \
make && \
make install
RUN mkdir -p /usr/local/share/lua/5.1/ && cd ljsyscall && cp -a *.lua syscall /usr/local/share/lua/5.1/
ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/lib64

View File

@ -1,12 +0,0 @@
KERNEL_FILES=-C ../alpine/kernel/x86_64 kernel-headers.tar kernel-dev.tar kernel-modules.tar
default: ebpf.tag
ebpf.tag: Dockerfile
BUILD=$$( tar cf - $^ $(KERNEL_FILES) | docker build -q - ) && [ -n "$$BUILD" ] && echo "Built $$BUILD" && \
echo $$BUILD > $@
clean:
rm -f ebpf.tag
.DELETE_ON_ERROR:

View File

@ -0,0 +1,9 @@
#!/bin/sh
nsenter -t 1 -m -- /bin/ash -c "/usr/share/bcc/tools/softirqs 1 1"
if [ "$?" -ne "0" ]; then
printf "bcc test suite FAILED\n" >&1
exit 1
fi;
printf "bcc test suite PASSED\n" >&1

View File

@ -0,0 +1,24 @@
#!/bin/sh
# SUMMARY: Check that the bcc package works
# LABELS: skip
# REPEAT:
set -e
# Source libraries. Uncomment if needed/defined
#. "${RT_LIB}"
. "${RT_PROJECT_ROOT}/_lib/lib.sh"
NAME=bcc
clean_up() {
rm -rf ${NAME}-*
}
trap clean_up EXIT
# Test code goes here
linuxkit build -format kernel+initrd -name "${NAME}" test.yml
RESULT="$(linuxkit run ${NAME})"
echo "${RESULT}"
echo "${RESULT}" | grep -q "suite PASSED"
exit 0

View File

@ -0,0 +1,29 @@
kernel:
image: linuxkit/kernel:4.14.54
cmdline: "console=ttyS0 console=ttyAMA0"
init:
- linuxkit/init:v0.5
- linuxkit/runc:v0.5
- linuxkit/kernel-bcc:4.14.54
onboot:
- name: check-bcc
image: alpine:3.8
binds:
- /check.sh:/check.sh
command: ["sh", "./check.sh"]
pid: host
net: host
ipc: host
capabilities:
- all
- name: poweroff
image: linuxkit/poweroff:2687ed712690766b8156a77ac600005b532d2c7d
command: ["/bin/sh", "/poweroff.sh", "10"]
files:
- path: check.sh
source: ./check.sh
trust:
org:
- linuxkit
image:
- alpine:3.8