Merge pull request #3101 from kmjohansen/topics/ebpf

Enable building of bcc in kernel build

docs/kernel-bcc.md

@@ -0,0 +1,31 @@
+# Using the bcc utility with LinuxKit
+
+The `bcc` utility is a standard Linux tool to access performance
+counters, trace events and access various other kernel internals for
+performance analysis.
+
+The `bcc` utility needs to matched be with the kernel. For recent
+kernel build, LinuxKit provides a `linuxkit/kernel-bcc` package with
+a matching tag for each kernel under `linuxkit/kernel`.
+
+The preferred way of using the `linuxkit/kernel-bcc` package is to
+add it to the `init` section. This adds `/usr/share/bcc` to the
+  systems' root filesystem. From there it can be
+  - bind mounted into your container
+  - accessed via `/proc/1/root/usr/share/bcc/tools` from with in the `getty`
+    or `ssh` container.
+  - accessed via a nsenter of `/bin/ash` of proc 1.
+
+If you want to use `bcc` you may also want to remove the `sysctl`
+container, or alternatively, disable the kernel pointer restriction it
+enables by default:
+
+```
+echo 0 > /proc/sys/kernel/kptr_restrict
+```
+
+Now, `bcc` is ready to use. The LinuxKit `bcc` package contains
+the `bcc` binary, example and tool scripts, and kernel headers for the
+associated kernel build.
+
+

kernel/Dockerfile.bcc

@@ -0,0 +1,112 @@
+ARG IMAGE
+FROM ${IMAGE} as ksrc
+
+FROM linuxkit/alpine:8a89682421abf0d886d777235a05f4a04a736df2 AS build
+RUN apk update && apk upgrade -a && \
+  apk add --no-cache \
+  argp-standalone \
+  autoconf \
+  automake \
+  bison \
+  build-base \
+  clang \
+  clang-dev \
+  clang-static \
+  cmake \
+  curl \
+  flex-dev \
+  fts-dev \
+  gettext-dev \
+  git \
+  iperf3 \
+  libedit-dev \
+  libtool \
+  llvm \
+  llvm-dev \
+  llvm-static \
+  luajit-dev \
+  m4 \
+  python \
+  zlib-dev \
+  && true
+
+RUN ln -s /usr/lib/cmake/llvm5/ /usr/lib/cmake/llvm && \
+    ln -s /usr/include/llvm5/llvm-c/ /usr/include/llvm-c && \
+    ln -s /usr/include/llvm5/llvm/ /usr/include/llvm
+
+WORKDIR /build
+COPY ./bcc.patches/ ./
+RUN mv error.h /usr/include/
+RUN mv cdefs.h /usr/include/sys/
+ENV ELFUTILS_VERSION=0.165
+ENV ELFUTILS_SHA256="a7fc9277192caaa5f30b47e8c0518dbcfd8c4a19c6493a63d511d804290ce972"
+RUN curl -sSL -O https://fedorahosted.org/releases/e/l/elfutils/0.165/elfutils-$ELFUTILS_VERSION.tar.bz2
+ 
+RUN echo "${ELFUTILS_SHA256}  /build/elfutils-$ELFUTILS_VERSION.tar.bz2" | sha256sum -c - && \
+  tar xjf elfutils-$ELFUTILS_VERSION.tar.bz2 && \
+  cd elfutils-$ELFUTILS_VERSION && \
+  patch -p1 < ../100-musl-compat.patch && \
+  patch -p0 < ../decl.patch && \
+  patch -p0 < ../intl.patch
+ENV BCC_COMMIT=6972806729da00ecda0235abac61d66c8fad7fad
+RUN git clone https://github.com/iovisor/bcc.git && cd bcc && git checkout $BCC_COMMIT
+RUN cd bcc && patch -p0 < ../bcc-gnuism.patch && patch -p0 < ../bcc-lua.patch
+ENV LJSYSCALL_COMMIT=e587f8c55aad3955dddab3a4fa6c1968037b5c6e
+RUN git clone https://github.com/justincormack/ljsyscall.git && cd ljsyscall && git checkout $LJSYSCALL_COMMIT
+
+COPY --from=ksrc /kernel-headers.tar /build
+RUN tar xf /build/kernel-headers.tar
+
+COPY --from=ksrc /kernel-dev.tar /build
+RUN tar xf /build/kernel-dev.tar
+
+COPY --from=ksrc /kernel.tar /build
+RUN tar xf /build/kernel.tar
+
+RUN cd /build
+RUN cd elfutils-$ELFUTILS_VERSION && \
+  aclocal && \
+  automake && \
+  ./configure --prefix=/usr CFLAGS=-Wno-strict-aliasing && \
+  make -C libelf && make -C libelf install
+RUN mkdir -p bcc/build && cd bcc/build && \
+  cmake .. -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_C_FLAGS="-I/build/usr/include" -DCMAKE_CXX_FLAGS="-I/build/usr/include" -DCMAKE_INSTALL_PREFIX=/usr -DLUAJIT_INCLUDE_DIR=/usr/include/luajit-2.1 && \
+  make && \
+  make install
+RUN mkdir -p /usr/local/share/lua/5.1/ && cd ljsyscall && cp -a *.lua syscall /usr/local/share/lua/5.1/
+RUN mkdir -p /out/usr/
+RUN cp -a /build/usr/src /out/usr/
+RUN cp -a /build/usr/include /out/usr
+RUN mkdir -p /out/usr/lib
+RUN cp -a /usr/lib/libelf* /out/usr/lib/
+RUN cp -a /usr/lib/libstdc* /out/usr/lib/
+RUN cp -a /usr/lib/libintl* /out/usr/lib/
+RUN cp -a /usr/lib64/* /out/usr/lib/
+RUN mkdir -p /out/usr/lib/python2.7
+RUN cp -a /usr/lib/python2.7/site-packages /out/usr/lib/python2.7/
+RUN mkdir -p /out/usr/share
+RUN cp -a /usr/share/bcc /out/usr/share/
+RUN mkdir -p /out/usr/bin
+RUN cp -a /usr/bin/bcc-lua /out/usr/bin/
+RUN mkdir -p /out/usr/local/share/
+RUN cp -a /usr/local/share/lua /out/usr/local/share/
+
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/lib64
+
+FROM linuxkit/alpine:8a89682421abf0d886d777235a05f4a04a736df2 as mirror
+RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/
+RUN apk update && apk upgrade -a && \
+  apk add --no-cache --initdb -p /out \
+  busybox \
+  luajit \
+  python \
+  zlib \
+  && true
+
+FROM scratch
+ENTRYPOINT []
+CMD []
+WORKDIR /
+COPY --from=mirror /out /
+COPY --from=build /out /
+

kernel/Makefile

@@ -15,6 +15,7 @@
 # Name and Org on Hub
 ORG?=linuxkit
 IMAGE:=kernel
+IMAGE_BCC:=kernel-bcc
 IMAGE_PERF:=kernel-perf
 IMAGE_ZFS:=zfs-kmod
 
@@ -192,6 +193,46 @@ push: push_perf_$(2)$(3)$(4)
 forcepush: forcepush_perf_$(2)$(3)$(4)
 endif
 
+# For BCC, only build on x86 and only on latest LTS and latest stable.
+# That's 4.14.x and 4.17.x at time of authorship.
+ifeq ($(ARCH),x86_64)
+ifeq ($(2), $(filter $(2),4.14.x 4.17.x))
+build_bcc_$(2)$(3)$(4): build_$(2)$(3)$(4)
+	docker pull $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG)$(SUFFIX) || \
+		 DOCKER_CONTENT_TRUST=0 docker build -f Dockerfile.bcc \
+			--build-arg IMAGE=$(ORG)/$(IMAGE):$(1)$(3)$(4)-$(TAG)$(SUFFIX) \
+			--no-cache $(LABEL) -t $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG)$(SUFFIX) .
+
+forcebuild_bcc_$(2)$(3)$(4): build_$(2)$(3)$(4)
+	 DOCKER_CONTENT_TRUST=0 docker build -f Dockerfile.bcc \
+		--build-arg IMAGE=$(ORG)/$(IMAGE):$(1)$(3)$(4)-$(TAG)$(SUFFIX) \
+		--no-cache $(LABEL) -t $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG)$(SUFFIX) .
+
+push_bcc_$(2)$(3)$(4): build_bcc_$(2)$(3)$(4)
+	@if [ x"$(DIRTY)" != x ]; then echo "Your repository is not clean. Will not push image"; exit 1; fi
+	docker pull $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG)$(SUFFIX) || \
+		(docker push $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG)$(SUFFIX) && \
+		 docker tag $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG)$(SUFFIX) $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)$(SUFFIX) && \
+		 docker push $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)$(SUFFIX) && \
+		 $(PUSH_MANIFEST) $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG) $(DOCKER_CONTENT_TRUST) && \
+		 $(PUSH_MANIFEST) $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4) $(DOCKER_CONTENT_TRUST))
+
+forcepush_bcc_$(2)$(3)$(4): forcebuild_bcc_$(2)$(3)$(4)
+	@if [ x"$(DIRTY)" != x ]; then echo "Your repository is not clean. Will not push image"; exit 1; fi
+	docker push $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG)$(SUFFIX) && \
+	docker tag $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG)$(SUFFIX) $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)$(SUFFIX) && \
+	docker push $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)$(SUFFIX) && \
+	$(PUSH_MANIFEST) $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4)-$(TAG) $(DOCKER_CONTENT_TRUST) && \
+	$(PUSH_MANIFEST) $(ORG)/$(IMAGE_BCC):$(1)$(3)$(4) $(DOCKER_CONTENT_TRUST)
+
+
+build: build_bcc_$(2)$(3)$(4)
+forcebuild: forcebuild_bcc_$(2)$(3)$(4)
+push: push_bcc_$(2)$(3)$(4)
+forcepush: forcepush_bcc_$(2)$(3)$(4)
+endif
+endif
+
 ifeq ($(4),)
 # ZFS does not compile against -dbg kernels because CONFIG_DEBUG_LOCK_ALLOC
 # is incompatible with CDDL, apparently (this is ./configure check)

kernel/bcc.patches/bcc-gnuism.patch

@@ -1,9 +1,9 @@
 --- src/cc/usdt.h-orig
 +++ src/cc/usdt.h
-@@ -86,6 +86,24 @@
+@@ -125,6 +125,24 @@ public:
-   ArgumentParser(const char *arg) : arg_(arg), cur_pos_(0) {}
+   ArgumentParser_powerpc64(const char *arg) : ArgumentParser(arg) {}
  };
- 
+
 +#undef REG_A
 +#undef REG_B
 +#undef REG_C
@@ -23,5 +23,6 @@
 +#undef REG_RIP
 +
  class ArgumentParser_x64 : public ArgumentParser {
+ private:
    enum Register {
-     REG_A,
+

kernel/bcc.patches/bcc-lua.patch

@@ -0,0 +1,43 @@
+--- src/cc/CMakeLists.txt
++++ src/cc/CMakeLists.txt
+@@ -52,7 +52,7 @@ target_link_libraries(bcc-loader-static elf)
+ add_library(bcc-static STATIC
+   ${bcc_common_sources} ${bcc_table_sources} ${bcc_util_sources})
+ set_target_properties(bcc-static PROPERTIES OUTPUT_NAME bcc)
+-set(bcc-lua-static
++add_library(bcc-lua-static STATIC
+   ${bcc_common_sources} ${bcc_table_sources} ${bcc_sym_sources} ${bcc_util_sources})
+ 
+ include(clang_libs)
+@@ -64,9 +64,9 @@ set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${llvm_lib_exclude_f
+ set(bcc_common_libs_for_a b_frontend clang_frontend bpf-static
+   -Wl,--whole-archive ${clang_libs} ${llvm_libs} -Wl,--no-whole-archive
+   ${LIBELF_LIBRARIES})
+-set(bcc_common_libs_for_s ${bcc_common_libs_for_a})
+-set(bcc_common_libs_for_lua b_frontend clang_frontend bpf-static
++set(bcc_common_libs_for_s b_frontend clang_frontend bpf-static
+   ${clang_libs} ${llvm_libs} ${LIBELF_LIBRARIES})
++set(bcc_common_libs_for_lua ${bcc_common_libs_for_s})
+ 
+ if(ENABLE_CPP_API)
+   add_subdirectory(api)
+@@ -87,7 +87,7 @@ add_subdirectory(frontends)
+ # Link against LLVM libraries
+ target_link_libraries(bcc-shared ${bcc_common_libs_for_s})
+ target_link_libraries(bcc-static ${bcc_common_libs_for_a} bcc-loader-static)
+-set(bcc-lua-static ${bcc-lua-static} ${bcc_common_libs_for_lua})
++target_link_libraries(bcc-lua-static ${bcc_common_libs_for_lua})
+ 
+ install(TARGETS bcc-shared LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
+ install(FILES ${bcc_table_headers} DESTINATION include/bcc)
+--- src/lua/CMakeLists.txt
++++ src/lua/CMakeLists.txt
+@@ -23,7 +23,7 @@ if (LUAJIT_LIBRARIES AND LUAJIT)
+ 	add_executable(bcc-lua src/main.c bcc.o)
+ 	set_target_properties(bcc-lua PROPERTIES LINKER_LANGUAGE C)
+ 	target_link_libraries(bcc-lua ${LUAJIT_LIBRARIES})
+-	target_link_libraries(bcc-lua ${bcc-lua-static})
++	target_link_libraries(bcc-lua -Wl,--whole-archive bcc-lua-static -Wl,--no-whole-archive)
+ 	if (NOT COMPILER_NOPIE_FLAG EQUAL "")
+ 		target_link_libraries(bcc-lua ${COMPILER_NOPIE_FLAG})
+ 	endif()

projects/ebpf/README.md

@@ -1,18 +0,0 @@
-# iovisor eBPF tools
-
-The [iovisor eBPF tools](https://github.com/iovisor/bcc) are a
-collection tools useful for debugging and performance analysis. This
-project aims to provide an easy to consume packaging of these tools
-for Moby.
-
-It comes in two parts: [ebpf.build](ebpf.build/) is used for building the binaries on alpine and [ebpf.pkg](ebpf.pkg/) is supposed to package the binaries with the required kernel files.
-
-**Note**: The packages currently do not build.
-
-## Roadmap
-
-**Near-term:**
-- Make the package build again
-
-**Mid-term:**
-- Trim the distribution package to make it easier to consume

projects/ebpf/ebpf.build/Dockerfile

@@ -1,44 +0,0 @@
-FROM alpine:3.5
-RUN apk update && apk upgrade -a && \
-  apk add --no-cache \
-  argp-standalone \
-  autoconf \
-  automake \
-  bison \
-  build-base \
-  clang \
-  clang-dev \
-  cmake \
-  curl \
-  flex-dev \
-  fts-dev \
-  gettext-dev \
-  git \
-  iperf \
-  libedit-dev \
-  libtool \
-  llvm \
-  llvm-dev \
-  llvm-static \
-  luajit-dev \
-  m4 \
-  python \
-  zlib-dev \
-  && true
-WORKDIR /build
-COPY . ./
-COPY Dockerfile /
-COPY error.h /usr/include/
-COPY cdefs.h /usr/include/sys/
-ENV ELFUTILS_VERSION=0.165
-RUN curl -sSL -O https://fedorahosted.org/releases/e/l/elfutils/0.165/elfutils-$ELFUTILS_VERSION.tar.bz2
-RUN cat elfutils-$ELFUTILS_VERSION.tar.bz2 | tar xjf - && \
-  cd elfutils-$ELFUTILS_VERSION && \
-  patch -p1 < ../100-musl-compat.patch && \
-  patch -p0 < ../decl.patch && \
-  patch -p0 < ../intl.patch
-ENV BCC_COMMIT=d4fc95d92ec9bace9bd607dfd1833e9e06457486
-RUN git clone https://github.com/iovisor/bcc.git && cd bcc && git checkout $BCC_COMMIT
-RUN cd bcc && patch -p0 < ../bcc-gnuism.patch
-ENV LJSYSCALL_COMMIT=0b266e8f4f751ae894299d24a2d40d16c6cf856f
-RUN git clone https://github.com/justincormack/ljsyscall.git && cd ljsyscall && git checkout $LJSYSCALL_COMMIT

projects/ebpf/ebpf.build/Makefile

@@ -1,29 +0,0 @@
-.PHONY: tag push
-
-BASE=alpine:3.5
-IMAGE=alpine-build-ebpf
-
-default: push
-
-hash: Dockerfile *.patch cdefs.h error.h
-	DOCKER_CONTENT_TRUST=1 docker pull $(BASE)
-	tar cf - $^ | docker build --no-cache -t $(IMAGE):build -
-	docker run --rm $(IMAGE):build sh -c 'cat /Dockerfile /build/*.patch /build/*.h /lib/apk/db/installed | sha1sum' | sed 's/ .*//' > hash
-
-push: hash
-	docker pull mobylinux/$(IMAGE):$(shell cat hash) || \
-		(docker tag $(IMAGE):build mobylinux/$(IMAGE):$(shell cat hash) && \
-		 docker push mobylinux/$(IMAGE):$(shell cat hash))
-	docker rmi $(IMAGE):build
-	rm -f hash
-
-tag: hash
-	docker pull mobylinux/$(IMAGE):$(shell cat hash) || \
-		docker tag $(IMAGE):build mobylinux/$(IMAGE):$(shell cat hash)
-	docker rmi $(IMAGE):build
-	rm -f hash
-
-clean:
-	rm -f hash
-
-.DELETE_ON_ERROR:

projects/ebpf/ebpf.md

@@ -1,22 +0,0 @@
-## Using eBPF programs
-
-There is now a development image `mobylinux/ebpf:_tag_`. These are currently being built
-manually, I will tag one for each kernel release, as you should have a close one, eg
-`mobylinux/ebpf:4.9` is currently available.
-
-This image has all the kernel headers, `iovisor/bcc` built with support for C, Python and Lua,
-and all sources installed. It is very large so if we are shipping stuff based on this we will
-just extract compiled eBPF programs probably, but it is also usable for experiments, debug,
-benchmarks etc.
-
-You probably want to run with
-
-`docker run -it -v /sys/kernel/debug:/sys/kernel/debug --privileged --pid=host mobylinux/ebpf:tag sh` for
-interactive use as some things use debugfs. You need at least `CAP_SYS_ADMIN` to do anything.
-There are examples in `bcc/examples` that should generally just work, I have tried several of
-the Lua ones.
-
-Some of the `iovisor/bcc` samples try to access the kernel symbols. For them to work correctly you should also execute:
-```sh
-echo 0 > /proc/sys/kernel/kptr_restrict
-```

projects/ebpf/ebpf.pkg/Dockerfile

@@ -1,17 +0,0 @@
-# Tag: cc5bf4d39442741eaeffad79bbd9ae049a9253fa
-FROM mobylinux/alpine-build-ebpf@sha256:da37d99a8baad18e68c44d3807c86977767ed90ce03133b508842eea3b786d3a
-
-ADD kernel-headers.tar /
-ADD kernel-dev.tar /
-ADD kernel-modules.tar /
-RUN cd elfutils-$ELFUTILS_VERSION && \
-  automake && \
-  ./configure --prefix=/usr CFLAGS=-Wno-strict-aliasing && \
-  make -C libelf && make -C libelf install
-RUN mkdir -p bcc/build && cd bcc/build && \
-  cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DLUAJIT_INCLUDE_DIR=/usr/include/luajit-2.1 && \
-  make && \
-  make install
-RUN mkdir -p /usr/local/share/lua/5.1/ && cd ljsyscall && cp -a *.lua syscall /usr/local/share/lua/5.1/
-
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/lib64

projects/ebpf/ebpf.pkg/Makefile

@@ -1,12 +0,0 @@
-KERNEL_FILES=-C ../alpine/kernel/x86_64 kernel-headers.tar kernel-dev.tar kernel-modules.tar
-
-default: ebpf.tag
-
-ebpf.tag: Dockerfile
-	BUILD=$$( tar cf - $^ $(KERNEL_FILES) | docker build -q - ) && [ -n "$$BUILD" ] && echo "Built $$BUILD" && \
-	echo $$BUILD > $@
-
-clean:
-	rm -f ebpf.tag
-
-.DELETE_ON_ERROR:

test/cases/040_packages/032_bcc/check.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+nsenter -t 1 -m --  /bin/ash -c "/usr/share/bcc/tools/softirqs 1 1"
+if [ "$?" -ne "0" ]; then
+	printf "bcc test suite FAILED\n" >&1
+	exit 1
+fi;
+
+printf "bcc test suite PASSED\n" >&1

test/cases/040_packages/032_bcc/test.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# SUMMARY: Check that the bcc package works
+# LABELS: skip
+# REPEAT:
+
+set -e
+
+# Source libraries. Uncomment if needed/defined
+#. "${RT_LIB}"
+. "${RT_PROJECT_ROOT}/_lib/lib.sh"
+NAME=bcc
+
+clean_up() {
+	rm -rf ${NAME}-*
+}
+trap clean_up EXIT
+
+# Test code goes here
+linuxkit build -format kernel+initrd -name "${NAME}" test.yml
+RESULT="$(linuxkit run ${NAME})"
+echo "${RESULT}"
+echo "${RESULT}" | grep -q "suite PASSED"
+
+exit 0

test/cases/040_packages/032_bcc/test.yml

@@ -0,0 +1,29 @@
+kernel:
+  image: linuxkit/kernel:4.14.54
+  cmdline: "console=ttyS0 console=ttyAMA0"
+init:
+  - linuxkit/init:v0.5
+  - linuxkit/runc:v0.5
+  - linuxkit/kernel-bcc:4.14.54
+onboot:
+  - name: check-bcc
+    image: alpine:3.8
+    binds:
+      - /check.sh:/check.sh
+    command: ["sh", "./check.sh"]
+    pid: host
+    net: host
+    ipc: host
+    capabilities:
+      - all
+  - name: poweroff
+    image: linuxkit/poweroff:2687ed712690766b8156a77ac600005b532d2c7d
+    command: ["/bin/sh", "/poweroff.sh", "10"]
+files:
+  - path: check.sh
+    source: ./check.sh
+trust:
+  org:
+    - linuxkit
+  image:
+    - alpine:3.8