Merge pull request #1295 from rneugeba/kernel

Update the kernel build system

examples/sshd.yaml

@@ -1,5 +1,5 @@
 kernel:
-  image: "mobylinux/kernel:c1229050671f22671f98fd401279b0f5f1e461f8"
+  image: "mobylinux/kernel:4.9.x"
   cmdline: "console=ttyS0 page_poison=1"
 init: "mobylinux/init:65d6491c93fbf2a65fa19305da6ac245b8070526"
 system:

kernel/Dockerfile

@@ -1,6 +1,7 @@
 FROM mobylinux/alpine-build-kernel:0e893fbf6fa7638d2f23354de03ea11017bb8065@sha256:3ef3f9d11f0802b759dbd9c43a7706cf0ec37263c99ae90e2b10c29ea85739fa
 
-ARG KERNEL_VERSION=4.9.14
+ARG KERNEL_VERSION
+ARG DEBUG=0
 
 ENV KERNEL_SOURCE=https://www.kernel.org/pub/linux/kernel/v4.x/linux-${KERNEL_VERSION}.tar.xz
 
@@ -11,8 +12,6 @@ RUN cat linux-${KERNEL_VERSION}.tar.xz | tar --absolute-names -xJ &&  mv /linux-
 COPY kernel_config /linux/arch/x86/configs/x86_64_defconfig
 COPY kernel_config.debug /linux/debug_config
 
-ARG DEBUG=0
-
 RUN if [ $DEBUG -ne "0" ]; then \
     sed -i 's/CONFIG_PANIC_ON_OOPS=y/# CONFIG_PANIC_ON_OOPS is not set/' /linux/arch/x86/configs/x86_64_defconfig; \
     cat /linux/debug_config >> /linux/arch/x86/configs/x86_64_defconfig; \

kernel/Dockerfile.aufs

@@ -1,100 +0,0 @@
-FROM mobylinux/alpine-build-kernel:0e893fbf6fa7638d2f23354de03ea11017bb8065@sha256:3ef3f9d11f0802b759dbd9c43a7706cf0ec37263c99ae90e2b10c29ea85739fa
-
-ARG KERNEL_VERSION=4.9.14
-
-ENV KERNEL_SOURCE=https://www.kernel.org/pub/linux/kernel/v4.x/linux-${KERNEL_VERSION}.tar.xz
-
-RUN curl -fsSL -o linux-${KERNEL_VERSION}.tar.xz ${KERNEL_SOURCE}
-
-RUN cat linux-${KERNEL_VERSION}.tar.xz | tar --absolute-names -xJ &&  mv /linux-${KERNEL_VERSION} /linux
-
-# this is aufs4.9 20170220
-ENV AUFS_REPO       https://github.com/sfjro/aufs4-standalone
-ENV AUFS_BRANCH     aufs4.9
-ENV AUFS_COMMIT     97ef00a26bd2e098fb91d87852cc926e517ec8f7
-
-# Download AUFS
-RUN git clone -b "$AUFS_BRANCH" "$AUFS_REPO" /aufs && \
-    cd /aufs && \
-    git checkout -q "$AUFS_COMMIT"
-
-# aufs-util 20151116
-ENV AUFS_TOOLS_REPO https://github.com/ncopa/aufs-util.git
-ENV AUFS_TOOLS_COMMIT 3b7c5e262b53598a8204a915e485489c46d4e7a4
-
-# Download aufs tools
-RUN git clone ${AUFS_TOOLS_REPO} && \
-    cd /aufs-util && \
-    git checkout "$AUFS_TOOLS_COMMIT" 
-
-#BUILD
-# patch kernel with aufs
-RUN cd /linux && \
-    cp -r /aufs/Documentation /linux && \
-    cp -r /aufs/fs /linux && \
-    cp -r /aufs/include/uapi/linux/aufs_type.h /linux/include/uapi/linux/ && \
-    set -e && for patch in \
-        /aufs/aufs*-kbuild.patch \
-        /aufs/aufs*-base.patch \
-        /aufs/aufs*-mmap.patch \
-        /aufs/aufs*-standalone.patch \
-        /aufs/aufs*-loopback.patch \
-        /aufs/lockdep-debug.patch \
-    ; do \
-        patch -p1 < "$patch"; \
-    done
-
-COPY kernel_config /linux/arch/x86/configs/x86_64_defconfig
-COPY kernel_config.debug /linux/debug_config
-COPY kernel_config.aufs /linux/aufs_config
-RUN cat /linux/aufs_config >> /linux/arch/x86/configs/x86_64_defconfig
-
-ARG DEBUG=0
-
-RUN if [ $DEBUG -ne "0" ]; then \
-    sed -i 's/CONFIG_PANIC_ON_OOPS=y/# CONFIG_PANIC_ON_OOPS is not set/' /linux/arch/x86/configs/x86_64_defconfig; \
-    cat /linux/debug_config >> /linux/arch/x86/configs/x86_64_defconfig; \
-    fi
-
-# Apply local patches
-COPY patches-4.9/* /patches/
-RUN cd /linux && \
-    set -e && for patch in /patches/*.patch; do \
-        echo "Applying $patch"; \
-        patch -p1 < "$patch"; \
-    done
-
-RUN cd /linux && \
-    make defconfig && \
-    make oldconfig && \
-    make -j "$(getconf _NPROCESSORS_ONLN)" KCFLAGS="-fno-pie"
-RUN cd /linux && \
-    make INSTALL_MOD_PATH=/tmp/kernel-modules modules_install && \
-    ( DVER=$(basename $(find /tmp/kernel-modules/lib/modules/ -mindepth 1 -maxdepth 1)) && \
-      cd /tmp/kernel-modules/lib/modules/$DVER && \
-      rm build source && \
-      ln -s /usr/src/linux-headers-$DVER build ) && \
-    mkdir -p /tmp/kernel-headers/usr && \
-    make INSTALL_HDR_PATH=/tmp/kernel-headers/usr headers_install && \
-    ( cd /tmp/kernel-headers && tar cf /kernel-headers.tar usr ) && \
-    ( cd /tmp/kernel-modules && tar cf /kernel-modules.tar lib ) && \
-    cp vmlinux arch/x86_64/boot/bzImage /
-
-RUN DVER=$(basename $(find /tmp/kernel-modules/lib/modules/ -mindepth 1 -maxdepth 1)) && \
-    dir=/tmp/usr/src/linux-headers-$DVER && \
-    mkdir -p $dir && \
-    cp /linux/.config $dir && \
-    cd /linux && \
-    cp -a include "$dir" && \
-    mkdir -p "$dir"/arch/x86 && cp -a arch/x86/include "$dir"/arch/x86/ && \
-    ( cd /tmp && tar cf /kernel-dev.tar usr/src )
-
-# Build aufs tools, do this here as they need kernel headers and to match aufs
-# Fortunately they are built statically linked
-RUN cd /aufs-util && \
-    CPPFLAGS="-I/tmp/include" CFLAGS=$CPPFLAGS LDFLAGS=$CPPFLAGS make && \
-    DESTDIR=/tmp/aufs-utils make install && \
-    rm -rf /tmp/aufs-utils/usr/lib /tmp/aufs-utils/usr/share && \
-    cd /tmp/aufs-utils && rm libau* && tar cf /aufs-utils.tar .
-
-RUN printf "KERNEL_SOURCE=${KERNEL_SOURCE}\nAUFS_REPO=${AUFS_REPO}\nAUFS_BRANCH=${AUFS_BRANCH}\nAUFS_COMMIT=${AUFS_COMMIT}\nAUFS_TOOLS_REPO=${AUFS_TOOLS_REPO}\nAUFS_TOOLS_COMMIT=${AUFS_TOOLS_COMMIT}\n" > /kernel-source-info

kernel/Makefile

@@ -2,17 +2,36 @@ DEBUG ?= 0
 
 all:	bzImage push
 
-ifdef AUFS
+# We push the image to hub twice, once with the full kernel version of
-DEPS=Dockerfile.aufs kernel_config kernel_config.debug kernel_config.aufs patches-4.9
+# "mobylinux/kernel:<kernel version>.<major version>.<minor version>-<n>",
-else
+# where "<n>" is a monotonically increasing config number, and as
+# "mobylinux/kernel:<kernel version>.<major version>.x". This version
+# number is stored in IMAGE_VERSION.
+#
+# We expect most users to us the "<kernel version>.<major version>.x"
+# variant as this simply is the latest version of a given major kernel
+# version. This version number is stored in IMAGE_MAJOR_VERSION.
+#
+# For IMAGE_VERSION, the "<n>" must be increased whenever
+# the kernel config or the patches change. We don't expect this to
+# happen very often as the minor version number gets update quite
+# frequently.
+#
+# IMAGE_VERSION is used to determine if a new image should be pushed to hub.
 ifdef LTS4.4
-DEPS=Dockerfile.4.4 kernel_config kernel_config.debug kernel_config.4.4 patches-4.4
+KERNEL_VERSION=4.4.53
+IMAGE_VERSION=$(KERNEL_VERSION)-0
+IMAGE_MAJOR_VERSION=4.4.x
+DEPS=Dockerfile.4.4 Makefile kernel_config kernel_config.debug kernel_config.4.4 patches-4.4
 else
-DEPS=Dockerfile kernel_config kernel_config.debug patches-4.9
+KERNEL_VERSION=4.9.14
-endif
+IMAGE_VERSION=$(KERNEL_VERSION)-0
+IMAGE_MAJOR_VERSION=4.9.x
+DEPS=Dockerfile Makefile kernel_config kernel_config.debug patches-4.9
 endif
+
 kernel.tag: $(DEPS)
-	BUILD=$$( tar cf - $^ | docker build -f $< --build-arg DEBUG=$(DEBUG) -q - ) && [ -n "$$BUILD" ] && echo "Built $$BUILD" && echo "$$BUILD" > $@
+	BUILD=$$( tar cf - $^ | docker build -f $< --build-arg DEBUG=$(DEBUG) --build-arg KERNEL_VERSION=$(KERNEL_VERSION) -q - ) && [ -n "$$BUILD" ] && echo "Built $$BUILD" && echo "$$BUILD" > $@
 
 bzImage: kernel.tag
 	rm -rf etc/kernel-patches
@@ -21,10 +40,9 @@ bzImage: kernel.tag
 	cp x86_64/kernel-modules.tar kernel.tar
 	cp x86_64/bzImage $@
 
-.PHONY: push tag
+.PHONY: image push tag
 
 MEDIA_TOYBOX=mobylinux/toybox-media:0a26fe5f574e444849983f9c4148ef74b3804d55@sha256:5ac38f77b66deb194c9016591b9b096e81fcdc9f7c3e6d01566294a6b4b4ebd2
-SHASUM=alpine:3.5
 
 BASE="$MEDIA_TOYBOX"
 IMAGE=kernel
@@ -34,25 +52,26 @@ default: push
 Dockerfile.media:
 	printf "FROM $(MEDIA_TOYBOX)\nADD . /\n" > $@
 
-hash: Dockerfile.media bzImage kernel.tar $(DEPS)
+image: Dockerfile.media bzImage kernel.tar $(DEPS)
 	tar cf - $^ | docker build --no-cache -t $(IMAGE):build -f Dockerfile.media -
-	tar cf - $^ | DOCKER_CONTENT_TRUST=1 docker run --rm -i $(SHASUM) sha1sum | sed 's/ .*//' > $@
 
-push: hash
+push: image
-	docker pull mobylinux/$(IMAGE):$(shell cat hash) || \
+	docker pull mobylinux/$(IMAGE):$(IMAGE_VERSION) || \
-		(docker tag $(IMAGE):build mobylinux/$(IMAGE):$(shell cat hash) && \
+		(docker tag $(IMAGE):build mobylinux/$(IMAGE):$(IMAGE_VERSION) && \
-		 docker push mobylinux/$(IMAGE):$(shell cat hash))
+		 docker push mobylinux/$(IMAGE):$(IMAGE_VERSION) && \
+		 docker tag $(IMAGE):build mobylinux/$(IMAGE):$(IMAGE_MAJOR_VERSION) && \
+		 docker push mobylinux/$(IMAGE):$(IMAGE_MAJOR_VERSION))
 	docker rmi $(IMAGE):build
 	rm -f hash
 
-tag: hash
+tag: image
-	docker pull mobylinux/$(IMAGE):$(shell cat hash) || \
+	(docker tag $(IMAGE):build mobylinux/$(IMAGE):$(IMAGE_VERSION) && \
-		docker tag $(IMAGE):build mobylinux/$(IMAGE):$(shell cat hash)
+	docker tag $(IMAGE):build mobylinux/$(IMAGE):$(IMAGE_MAJOR_VERSION))
 	docker rmi $(IMAGE):build
 	rm -f hash
 
 .PHONY: clean
 clean:
-	rm -rf x86_64 lib usr sbin kernel.tag hash Dockerfile.media bzImage kernel.tar
+	rm -rf x86_64 lib usr sbin kernel.tag Dockerfile.media bzImage kernel.tar
 
 .DELETE_ON_ERROR:

moby.yaml

@@ -1,5 +1,5 @@
 kernel:
-  image: "mobylinux/kernel:c1229050671f22671f98fd401279b0f5f1e461f8"
+  image: "mobylinux/kernel:4.9.x"
   cmdline: "console=ttyS0 page_poison=1"
 init: "mobylinux/init:65d6491c93fbf2a65fa19305da6ac245b8070526"
 system:

test.yaml

@@ -1,5 +1,5 @@
 kernel:
-  image: "mobylinux/kernel:c1229050671f22671f98fd401279b0f5f1e461f8"
+  image: "mobylinux/kernel:4.9.x"
   cmdline: "console=ttyS0"
 init: "mobylinux/init:65d6491c93fbf2a65fa19305da6ac245b8070526"
 system:

tools/test-ltp/test-ltp.yaml

@@ -1,5 +1,5 @@
 kernel:
-  image: "mobylinux/kernel-4.9.x-moby:c752c32911815b1cbdd580f10553767e934dc0fa"
+  image: "mobylinux/kernel:4.9.x"
   cmdline: "console=ttyS0"
 init: "mobylinux/init:2f1b5c1be1157cb17e5b1e6dee171dccdebb5277"
 system: