separate kernel series hashing (#4194)

* separate kernel series hashing

Signed-off-by: Chris Irrgang <chris.irrgang@gmx.de>

* fix issues with the update component sha script

- add bsd/gnu cross compatibility for sed
- also replace in */test.sh files
- replace potentially problematic xargs
- remove potentially problematic word boundary \b

Signed-off-by: Chris Irrgang <chris.irrgang@gmx.de>

* Move common kernel files to dedicated folder

Signed-off-by: Chris Irrgang <chris.irrgang@gmx.de>

* run update-kernel-yamls

Signed-off-by: Chris Irrgang <chris.irrgang@gmx.de>

---------

Signed-off-by: Chris Irrgang <chris.irrgang@gmx.de>

kernel/Makefile

@@ -25,14 +25,19 @@ EXTRA?=
 
 DEBUG?=
 
-ifeq ($(HASH),)
-HASH_COMMIT?=HEAD # Setting this is only really useful with the show-tag target
-HASH:=$(shell git ls-tree --full-tree $(HASH_COMMIT) -- $(CURDIR) | awk '{print $$3}')
-
-ifneq ($(HASH_COMMIT),HEAD) # Others can't be dirty by definition
+HASH_COMMIT?=HEAD
 DIRTY:=$(shell git update-index -q --refresh && git diff-index --quiet HEAD -- $(CURDIR) || echo "-dirty")
-endif
-endif
+
+# Shared files that affect all kernel builds (Dockerfiles, build yamls, scripts, etc.)
+# These are combined with series-specific folders to compute per-series hashes
+COMMON_DIR=common
+
+# Compute hash for shared files from above and include Makefile from current dir
+SHARED_HASH:=$(shell git ls-tree -r --full-tree $(HASH_COMMIT) -- $(CURDIR)/Makefile $(CURDIR)/$(COMMON_DIR) 2>/dev/null | git hash-object --stdin)
+
+# Function to compute per-series hash: combines shared files hash with series folder hash
+# Usage: $(call series_hash,6.12.x)
+series_hash=$(shell (echo $(SHARED_HASH); git ls-tree --full-tree $(HASH_COMMIT) -- $(CURDIR)/$(1) 2>/dev/null) | git hash-object --stdin)$(DIRTY)
 
 REPO_ROOT:=$(shell git rev-parse --show-toplevel)
 
@@ -55,8 +60,6 @@ endif
 
 BUILD_PLATFORM=linux/$(BUILDERARCH)
 
-HASHTAG=$(HASH)$(DIRTY)
-
 .PHONY: notdirty
 notdirty:
 	@if [ x"$(DIRTY)" !=  x ]; then echo "Your repository is not clean. Will not push image"; exit 1; fi
@@ -111,6 +114,7 @@ list:
 	@echo "Kernels: $(KERNELS)"
 	@echo "Deprecated: $(DEPRECATED)"
 	@echo "Tools: $(TOOLS)"
+	@echo "Shared hash: $(SHARED_HASH)"
 
 setforce:
 	$(eval FORCE=--force)
@@ -122,15 +126,17 @@ build-%: buildkernel-% buildtools-%;
 
 buildkernel-%: buildkerneldeps-% buildplainkernel-% builddebugkernel-%;
 
-buildkerneldeps-%: Dockerfile Makefile $(wildcard patches-$(call series,$*)/*) $(wildcard config-$(call series,$*)*) ;
+buildkerneldeps-%: $(COMMON_DIR)/Dockerfile Makefile $(wildcard patches-$(call series,$*)/*) $(wildcard config-$(call series,$*)*) ;
 
 buildplainkernel-%: buildkerneldeps-%
 	$(eval KERNEL_SERIES=$(call series,$*))
-	linuxkit pkg build . $(FORCE) --platforms $(BUILD_PLATFORM) --build-yml ./build-kernel.yml --tag "$*-{{.Hash}}" --build-arg-file $(KERNEL_SERIES)/build-args 
+	$(eval SERIES_HASH=$(call series_hash,$(KERNEL_SERIES)))
+	linuxkit pkg build . $(FORCE) --platforms $(BUILD_PLATFORM) --build-yml ./$(COMMON_DIR)/build-kernel.yml --hash $(SERIES_HASH) --tag "$*-{{.Hash}}" --build-arg-file $(KERNEL_SERIES)/build-args 
 
 builddebugkernel-%: buildkerneldeps-%
 	$(eval KERNEL_SERIES=$(call series,$*))
-	linuxkit pkg build . $(FORCE) --platforms $(BUILD_PLATFORM) --build-yml ./build-kernel.yml --tag "$*-dbg-{{.Hash}}" --build-arg-file $(KERNEL_SERIES)/build-args --build-arg-file build-args-debug
+	$(eval SERIES_HASH=$(call series_hash,$(KERNEL_SERIES)))
+	linuxkit pkg build . $(FORCE) --platforms $(BUILD_PLATFORM) --build-yml ./$(COMMON_DIR)/build-kernel.yml --hash $(SERIES_HASH) --tag "$*-dbg-{{.Hash}}" --build-arg-file $(KERNEL_SERIES)/build-args --build-arg-file $(COMMON_DIR)/build-args-debug
 
 push-%: notdirty build-% pushkernel-% tagbuilder-% pushtools-%;
 
@@ -145,19 +151,25 @@ tagbuilder-%: notdirty
 pushkernel-%: pushplainkernel-% pushdebugkernel-%;
 
 pushplainkernel-%: buildplainkernel-%
-	$(eval HASHED_IMAGE=$(shell linuxkit pkg show-tag . --build-yml ./build-kernel.yml --tag "$*-{{.Hash}}"))
-	$(eval PLAIN_IMAGE=$(shell linuxkit pkg show-tag . --build-yml ./build-kernel.yml --tag "$*"))
+	$(eval KERNEL_SERIES=$(call series,$*))
+	$(eval SERIES_HASH=$(call series_hash,$(KERNEL_SERIES)))
+	$(eval HASHED_IMAGE=$(shell linuxkit pkg show-tag . --build-yml ./$(COMMON_DIR)/build-kernel.yml --hash $(SERIES_HASH) --tag "$*-{{.Hash}}"))
+	$(eval PLAIN_IMAGE=$(shell linuxkit pkg show-tag . --build-yml ./$(COMMON_DIR)/build-kernel.yml --tag "$*"))
 	linuxkit cache push $(HASHED_IMAGE)
 	linuxkit cache push $(HASHED_IMAGE) --remote-name $(PLAIN_IMAGE)
 
 pushdebugkernel-%: builddebugkernel-%
-	$(eval HASHED_IMAGE=$(shell linuxkit pkg show-tag . --build-yml ./build-kernel.yml --tag "$*-dbg-{{.Hash}}"))
-	$(eval PLAIN_IMAGE=$(shell linuxkit pkg show-tag . --build-yml ./build-kernel.yml --tag "$*-dbg"))
+	$(eval KERNEL_SERIES=$(call series,$*))
+	$(eval SERIES_HASH=$(call series_hash,$(KERNEL_SERIES)))
+	$(eval HASHED_IMAGE=$(shell linuxkit pkg show-tag . --build-yml ./$(COMMON_DIR)/build-kernel.yml --hash $(SERIES_HASH) --tag "$*-dbg-{{.Hash}}"))
+	$(eval PLAIN_IMAGE=$(shell linuxkit pkg show-tag . --build-yml ./$(COMMON_DIR)/build-kernel.yml --tag "$*-dbg"))
 	linuxkit cache push $(HASHED_IMAGE)
 	linuxkit cache push $(HASHED_IMAGE) --remote-name $(PLAIN_IMAGE)
 
 show-tag-%:
-	@echo 	$(call baseimage,$*)-$(HASHTAG)
+	$(eval KERNEL_SERIES=$(call series,$*))
+	$(eval SERIES_HASH=$(call series_hash,$(KERNEL_SERIES)))
+	@echo	$(call baseimage,$*)-$(SERIES_HASH)
 
 buildtools-%: $(addprefix buildtool-%$(RELEASESEP),$(TOOLS));
 
@@ -165,7 +177,8 @@ buildtool-%:
 	$(eval TOOL=$(call toolname,$*))
 	$(eval KERNEL_VERSION=$(call toolkernel,$*))
 	$(eval KERNEL_SERIES=$(call series,$(KERNEL_VERSION)))
-	linuxkit pkg build . $(FORCE) --platforms $(BUILD_PLATFORM) --build-yml ./build-$(TOOL).yml --tag "$(KERNEL_VERSION)-{{.Hash}}" --build-arg-file $(KERNEL_SERIES)/build-args
+	$(eval SERIES_HASH=$(call series_hash,$(KERNEL_SERIES)))
+	linuxkit pkg build . $(FORCE) --platforms $(BUILD_PLATFORM) --build-yml ./$(COMMON_DIR)/build-$(TOOL).yml --tag "$(KERNEL_VERSION)-$(SERIES_HASH)" --build-arg-file $(KERNEL_SERIES)/build-args
 
 pushtools-%: $(addprefix pushtool-%$(RELEASESEP),$(TOOLS));
 
@@ -173,8 +186,9 @@ pushtool-%: buildtool-%
 	$(eval TOOL=$(call toolname,$*))
 	$(eval KERNEL_VERSION=$(call toolkernel,$*))
 	$(eval KERNEL_SERIES=$(call series,$(KERNEL_VERSION)))
-	$(eval HASHED_IMAGE=$(shell linuxkit pkg show-tag . --build-yml ./build-$(TOOL).yml --tag "$(KERNEL_VERSION)-{{.Hash}}"))
-	$(eval PLAIN_IMAGE=$(shell linuxkit pkg show-tag . --build-yml ./build-$(TOOL).yml --tag "$(KERNEL_VERSION)"))
+	$(eval SERIES_HASH=$(call series_hash,$(KERNEL_SERIES)))
+	$(eval HASHED_IMAGE=$(shell linuxkit pkg show-tag . --build-yml ./$(COMMON_DIR)/build-$(TOOL).yml --tag "$(KERNEL_VERSION)-$(SERIES_HASH)"))
+	$(eval PLAIN_IMAGE=$(shell linuxkit pkg show-tag . --build-yml ./$(COMMON_DIR)/build-$(TOOL).yml --tag "$(KERNEL_VERSION)"))
 	linuxkit cache push $(HASHED_IMAGE)
 	linuxkit cache push $(HASHED_IMAGE) --remote-name $(PLAIN_IMAGE)
 
@@ -184,11 +198,15 @@ pushtool-%: buildtool-%
 
 # get the tag for the normal kernel for a particular version. Accepts version or series
 tag-plainkernel-%:
-	@linuxkit pkg show-tag . --build-yml ./build-kernel.yml --tag "$*-{{.Hash}}"
+	$(eval KERNEL_SERIES=$(call series,$*))
+	$(eval SERIES_HASH=$(call series_hash,$(KERNEL_SERIES)))
+	@linuxkit pkg show-tag . --build-yml ./$(COMMON_DIR)/build-kernel.yml --hash $(SERIES_HASH) --tag "$*-{{.Hash}}"
 
 # get the tag for the debug kernel for a particular version. Accepts version or series
 tag-debugkernel-%:
-	@linuxkit pkg show-tag . --build-yml ./build-kernel.yml --tag "$*-dbg-{{.Hash}}"
+	$(eval KERNEL_SERIES=$(call series,$*))
+	$(eval SERIES_HASH=$(call series_hash,$(KERNEL_SERIES)))
+	@linuxkit pkg show-tag . --build-yml ./$(COMMON_DIR)/build-kernel.yml --hash $(SERIES_HASH) --tag "$*-dbg-{{.Hash}}"
 
 # find and replace any usage of the normal kernel with hash for a particular series
 # will update hash for same semver and/or patch version
@@ -214,7 +232,7 @@ KCONFIG_TAG_EXTENSION=-${KCONFIG_TAG}
 endif
 
 kconfig:
-	docker build --no-cache -f Dockerfile.kconfig \
+	docker build --no-cache -f $(COMMON_DIR)/Dockerfile.kconfig \
 		--build-arg KERNEL_VERSIONS="$(KERNEL_VERSIONS)" \
 		--build-arg BUILD_IMAGE=$(IMAGE_BUILDER) \
 		--platform $(BUILD_PLATFORM) \
@@ -222,7 +240,7 @@ kconfig:
 
 kconfigx:
 ifeq (${KCONFIG_TAG},)
-	docker buildx build --no-cache -f Dockerfile.kconfigx \
+	docker buildx build --no-cache -f $(COMMON_DIR)/Dockerfile.kconfigx \
 		--platform $(BUILD_PLATFORM) \
 		--output . \
 		--build-arg KERNEL_VERSIONS="$(KERNEL_VERSIONS)" \
@@ -232,7 +250,7 @@ ifeq (${KCONFIG_TAG},)
 	cp linux_amd64/config-${KERNEL_VERSIONS}-amd64 config-${KERNEL_SERIES}-x86_64
 	cp linux_amd64/config-${KERNEL_VERSIONS}-riscv64 config-${KERNEL_SERIES}-riscv64
 else
-	docker buildx build --no-cache -f Dockerfile.kconfigx \
+	docker buildx build --no-cache -f $(COMMON_DIR)/Dockerfile.kconfigx \
 		--platform $(BUILD_PLATFORM) --push \
 		--output . \
 		--build-arg KERNEL_VERSIONS="$(KERNEL_VERSIONS)" \

kernel/build-bcc.yml

@@ -1,3 +0,0 @@
-image: kernel-bcc
-network: true
-dockerfile: Dockerfile.bcc

kernel/build-kernel.yml

@@ -1,2 +0,0 @@
-image: kernel
-network: true

kernel/build-perf.yml

@@ -1,3 +0,0 @@
-image: kernel-perf
-network: true
-dockerfile: Dockerfile.perf

kernel/common/Dockerfile

@@ -55,7 +55,7 @@ RUN KERNEL_MAJOR=$(echo ${KERNEL_VERSION} | cut -d . -f 1) && \
     KERNEL_SHA256_SUMS=https://www.kernel.org/pub/linux/kernel/${KERNEL_MAJOR}/sha256sums.asc && \
     KERNEL_PGP2_SIGN=https://www.kernel.org/pub/linux/kernel/${KERNEL_MAJOR}/linux-${KERNEL_VERSION}.tar.sign && \
     curl -fsSLO ${KERNEL_SHA256_SUMS} && \
-    gpg2 -q --import /src/keys.asc && \
+    gpg2 -q --import /src/common/keys.asc && \
     gpg2 --verify sha256sums.asc && \
     KERNEL_SHA256=$(grep linux-${KERNEL_VERSION}.tar.xz sha256sums.asc | cut -d ' ' -f 1) && \
     [ -f linux-${KERNEL_VERSION}.tar.xz ] || curl -fsSLO ${KERNEL_SOURCE} && \
@@ -126,7 +126,7 @@ RUN case $(uname -m) in \
     sed -i "s/CONFIG_LOCALVERSION=\"-linuxkit\"/CONFIG_LOCALVERSION=\"-linuxkit${EXTRA}${DEBUG}\"/" ${KERNEL_DEF_CONF}; \
     if [ -n "${DEBUG}" ]; then \
         sed -i 's/CONFIG_PANIC_ON_OOPS=y/# CONFIG_PANIC_ON_OOPS is not set/' ${KERNEL_DEF_CONF}; \
-        cat /src/config${DEBUG} >> ${KERNEL_DEF_CONF}; \
+        cat /src/common/config${DEBUG} >> ${KERNEL_DEF_CONF}; \
     fi && \
     make defconfig && \
     make oldconfig && \

kernel/common/build-bcc.yml

@@ -0,0 +1,3 @@
+image: kernel-bcc
+network: true
+dockerfile: common/Dockerfile.bcc

kernel/common/build-kernel.yml

@@ -0,0 +1,3 @@
+image: kernel
+network: true
+dockerfile: common/Dockerfile

kernel/common/build-perf.yml

@@ -0,0 +1,3 @@
+image: kernel-perf
+network: true
+dockerfile: common/Dockerfile.perf

scripts/update-component-sha.sh

@@ -7,6 +7,14 @@ set -e
 # see usage() for usage and functionality
 #
 
+# Detect sed in-place flag (BSD vs GNU)
+if sed --version 2>/dev/null | grep -q GNU; then
+        SED_INPLACE="sed -i"
+else
+        # BSD sed requires an argument for -i
+        SED_INPLACE="sed -i ''"
+fi
+
 usage() {
     cat >&2 <<EOF
 $0 --<mode> <how-to-find> <new-hash>
@@ -49,7 +57,7 @@ updateImage() {
                 hash=$2
                 ;;
         esac
-        git grep -E -l "[[:space:]]$image:" -- '*.yml' '*.yaml' '*.yml.in' '*.yaml.in' '*/Dockerfile' '*/Makefile' | grep -v /vendor/ | xargs sed -i.bak -E -e "s,([[:space:]])($image):([^[:space:]]+), $image:$hash,g"
+        git grep -E -l "[[:space:]]$image:" -- '*.yml' '*.yaml' '*.yml.in' '*.yaml.in' '*/Dockerfile' '*/Makefile' '*/test.sh' | grep -v /vendor/ | while read -r file; do $SED_INPLACE -E -e "s,([[:space:]])($image):([^[:space:]]+),\1$image:$hash,g" "$file"; done
 }
 
 # backwards compatibility
@@ -69,7 +77,7 @@ case "${mode}" in
         fi
         old=$1
         new=$2
-        git grep -E -l "\b($old)([[:space:]]|$)" -- '*.yml' '*.yaml' '*.yml.in' '*.yaml.in' '*/Dockerfile' '*/Makefile' | grep -v /vendor/ | while read -r file; do sed -ri.bak -e "s,($old)([[:space:]]|$),$new\2,g" "$file"; done
+        git grep -E -l "($old)([[:space:]]|$)" -- '*.yml' '*.yaml' '*.yml.in' '*.yaml.in' '*/Dockerfile' '*/Makefile' '*/test.sh' | grep -v /vendor/ | while read -r file; do $SED_INPLACE -E -e "s,($old)([[:space:]]|$),$new\2,g" "$file"; done
         ;;
 --image)
 	if [ $# -lt 1 ] ; then
@@ -98,5 +106,3 @@ case "${mode}" in
 	exit 1
 	;;
 esac
-
-find . -name '*.bak' | xargs rm || true

test/cases/020_kernel/019_config_6.6.x/test.yml

@@ -1,5 +1,5 @@
 kernel:
-  image: linuxkit/kernel:6.6.71-fac3a2ad495a23a50c8d2d7f173ecc7145049c29
+  image: linuxkit/kernel:6.6.71-77b4563829536629b7d5768d4407453c26602d65
   cmdline: "console=ttyS0 console=ttyAMA0"
 init:
   - linuxkit/init:b5506cc74a6812dc40982cacfd2f4328f8a4b12a

test/cases/020_kernel/022_config_6.12.x/test.yml

@@ -1,5 +1,5 @@
 kernel:
-  image: linuxkit/kernel:6.12.59-fac3a2ad495a23a50c8d2d7f173ecc7145049c29
+  image: linuxkit/kernel:6.12.59-0ef72d722190ecfe0b3b37711f9a871a696e301a
   cmdline: "console=ttyS0 console=ttyAMA0"
 init:
   - linuxkit/init:b5506cc74a6812dc40982cacfd2f4328f8a4b12a

test/cases/020_kernel/119_kmod_6.6.x/Dockerfile

@@ -3,7 +3,7 @@
 # In the last stage, it creates a package, which can be used for
 # testing.
 
-FROM linuxkit/kernel:6.6.71-fac3a2ad495a23a50c8d2d7f173ecc7145049c29 AS ksrc
+FROM linuxkit/kernel:6.6.71-77b4563829536629b7d5768d4407453c26602d65 AS ksrc
 
 # Extract headers and compile module
 FROM linuxkit/kernel:6.6.71-builder AS build

test/cases/020_kernel/119_kmod_6.6.x/test.sh

@@ -19,7 +19,7 @@ clean_up() {
 trap clean_up EXIT
 
 # Make sure we have the latest kernel image
-docker pull linuxkit/kernel:6.6.71-fac3a2ad495a23a50c8d2d7f173ecc7145049c29
+docker pull linuxkit/kernel:6.6.71-77b4563829536629b7d5768d4407453c26602d65
 # Build a package
 docker build -t ${IMAGE_NAME} .
 

test/cases/020_kernel/119_kmod_6.6.x/test.yml

@@ -1,5 +1,5 @@
 kernel:
-  image: linuxkit/kernel:6.6.71-fac3a2ad495a23a50c8d2d7f173ecc7145049c29
+  image: linuxkit/kernel:6.6.71-77b4563829536629b7d5768d4407453c26602d65
   cmdline: "console=ttyS0 console=ttyAMA0"
 init:
   - linuxkit/init:b5506cc74a6812dc40982cacfd2f4328f8a4b12a

test/cases/020_kernel/122_kmod_6.12.x/Dockerfile

@@ -3,7 +3,7 @@
 # In the last stage, it creates a package, which can be used for
 # testing.
 
-FROM linuxkit/kernel:6.12.59-fac3a2ad495a23a50c8d2d7f173ecc7145049c29 AS ksrc
+FROM linuxkit/kernel:6.12.59-0ef72d722190ecfe0b3b37711f9a871a696e301a AS ksrc
 
 # Extract headers and compile module
 FROM linuxkit/kernel:6.6.71-builder AS build

test/cases/020_kernel/122_kmod_6.12.x/test.sh

@@ -19,7 +19,7 @@ clean_up() {
 trap clean_up EXIT
 
 # Make sure we have the latest kernel image
-docker pull linuxkit/kernel:6.12.59-fac3a2ad495a23a50c8d2d7f173ecc7145049c29
+docker pull linuxkit/kernel:6.12.59-0ef72d722190ecfe0b3b37711f9a871a696e301a
 # Build a package
 docker build -t ${IMAGE_NAME} .
 

test/cases/020_kernel/122_kmod_6.12.x/test.yml

@@ -1,5 +1,5 @@
 kernel:
-  image: linuxkit/kernel:6.12.59-fac3a2ad495a23a50c8d2d7f173ecc7145049c29
+  image: linuxkit/kernel:6.12.59-0ef72d722190ecfe0b3b37711f9a871a696e301a
   cmdline: "console=ttyS0 console=ttyAMA0"
 init:
   - linuxkit/init:b5506cc74a6812dc40982cacfd2f4328f8a4b12a

test/cases/020_kernel/219_tags_6.6.x/test.sh

@@ -5,7 +5,7 @@
 
 set -e
 
-KERNEL=linuxkit/kernel:6.6.71-fac3a2ad495a23a50c8d2d7f173ecc7145049c29
+KERNEL=linuxkit/kernel:6.6.71-77b4563829536629b7d5768d4407453c26602d65
 
 # just include the common test
 . ../tags.sh

test/cases/020_kernel/222_tags_6.12.x/test.sh

@@ -5,7 +5,7 @@
 
 set -e
 
-KERNEL=linuxkit/kernel:6.12.59-fac3a2ad495a23a50c8d2d7f173ecc7145049c29
+KERNEL=linuxkit/kernel:6.12.59-0ef72d722190ecfe0b3b37711f9a871a696e301a
 
 # just include the common test
 . ../tags.sh