kata-deploy: Add support for building confidential kernel and shim-v2 for Arm CCA

After supporting the Arm CCA, it will rely on the kernel kvm.h headers to build the
runtime. The kernel-headers currently quite new with the traditional one, so that we
rely on build the kernel header first and then inject it to the shim-v2 build container.

Signed-off-by: Kevin Zhao <kevin.zhao@linaro.org>
Co-authored-by: Seunguk Shin <seunguk.shin@arm.com>

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -45,6 +45,7 @@ jobs:
           - kernel
           - kernel-dragonball-experimental
           - kernel-nvidia-gpu
+          - kernel-cca-confidential
           - nydus
           - ovmf
           - qemu
@@ -126,7 +127,7 @@ jobs:
           if-no-files-found: error
 
       - name: store-extratarballs-artifact ${{ matrix.asset }}
-        if: ${{ startsWith(matrix.asset, 'kernel-nvidia-gpu') }}
+        if: ${{ startsWith(matrix.asset, 'kernel-nvidia-gpu') || startsWith(matrix.asset, 'kernel-cca-confidential') }}
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: kata-artifacts-arm64-${{ matrix.asset }}-headers${{ inputs.tarball-suffix }}

tools/packaging/kata-deploy/local-build/Makefile

@@ -49,6 +49,16 @@ BASE_TARBALLS = serial-targets \
 	virtiofsd-tarball
 BASE_SERIAL_TARBALLS = rootfs-image-tarball \
 	rootfs-initrd-tarball
+else ifeq ($(ARCH), aarch64)
+BASE_TARBALLS = serial-targets \
+	kernel-confidential-tarball \
+	kernel-cca-confidential-tarball \
+	kernel-tarball \
+	qemu-tarball \
+	shim-v2-tarball \
+	virtiofsd-tarball
+BASE_SERIAL_TARBALLS = rootfs-image-tarball \
+	rootfs-initrd-tarball
 endif
 
 define BUILD
@@ -135,6 +145,9 @@ kernel-tarball:
 kernel-confidential-tarball:
 	${MAKE} $@-build
 
+kernel-cca-confidential-tarball:
+	${MAKE} $@-build
+
 nydus-tarball:
 	${MAKE} $@-build
 
@@ -191,6 +204,12 @@ rootfs-initrd-nvidia-gpu-confidential-tarball: agent-tarball busybox-tarball pau
 shim-v2-tarball:
 	${MAKE} $@-build
 
+# The shim-v2 build for aarch64 needs the kernel-headers tar file from kernel-cca-confidential kernel.
+ifeq ($(ARCH), aarch64)
+shim-v2-tarball-build: kernel-cca-confidential-tarball-build
+	$(call BUILD,shim-v2)
+endif
+
 trace-forwarder-tarball: copy-scripts-for-the-tools-build
 	${MAKE} $@-build
 

tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh

@@ -114,6 +114,7 @@ options:
 	kata-manager
 	kernel
 	kernel-confidential
+	kernel-cca-confidential
 	kernel-dragonball-experimental
 	kernel-experimental
 	kernel-nvidia-gpu
@@ -160,17 +161,22 @@ get_kernel_modules_dir() {
 	local version=${kernel_version#v}
 	local numeric_final_version=${version}
 
-	# Every first release of a kernel is x.y, while the resulting folder would be x.y.0
-	local rc=$(echo ${version} | grep -oE "\-rc[0-9]+$")
-	if [ -n "${rc}" ]; then
-		numeric_final_version="${numeric_final_version%"${rc}"}"
-	fi
+	if [ -z "${kernel_ref}" ]; then
+		# Every first release of a kernel is x.y, while the resulting folder would be x.y.0
+		local rc=$(echo ${version} | grep -oE "\-rc[0-9]+$")
+		if [ -n "${rc}" ]; then
+			numeric_final_version="${numeric_final_version%"${rc}"}"
+		fi
 
-	local dots=$(echo ${version} | grep -o '\.' | wc -l)
-	[ "${dots}" == "1" ] && numeric_final_version="${numeric_final_version}.0"
+		local dots=$(echo ${version} | grep -o '\.' | wc -l)
+		[ "${dots}" == "1" ] && numeric_final_version="${numeric_final_version}.0"
 
-	if [ -n "${rc}" ]; then
-		numeric_final_version="${numeric_final_version}${rc}"
+		if [ -n "${rc}" ]; then
+			numeric_final_version="${numeric_final_version}${rc}"
+		fi
+	else
+		# kernel_version should be vx.y.z-rcn-hash format when git is used
+		numeric_final_version="${numeric_final_version%-*}+"
 	fi
 
 	local kernel_modules_dir="${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/${kernel_name}/builddir/kata-linux-${version}-${kernel_kata_config_version}/lib/modules/${numeric_final_version}"
@@ -608,7 +614,7 @@ install_cached_kernel_tarball_component() {
 		|| return 1
 
 	case ${kernel_name} in
-		"kernel-nvidia-gpu"*"")
+		"kernel-nvidia-gpu"*"" | "kernel-cca-confidential")
 			local kernel_headers_dir=$(get_kernel_headers_dir "${kernel_name}")
 			mkdir -p ${kernel_headers_dir} || true
 			tar xvf ${workdir}/${kernel_name}/builddir/kata-static-${kernel_name}-headers.tar.xz -C "${kernel_headers_dir}" || return 1
@@ -632,9 +638,10 @@ install_kernel_helper() {
 
 	export kernel_version="$(get_from_kata_deps .${kernel_yaml_path}.version)"
 	export kernel_url="$(get_from_kata_deps .${kernel_yaml_path}.url)"
+	export kernel_ref="$(get_from_kata_deps .${kernel_yaml_path}.ref)"
 	export kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)"
 
-	if [[ "${kernel_name}" == "kernel"*"-confidential" ]]; then
+	if [[ "${kernel_name}" == "kernel"*"-confidential" ]] && [[ "${ARCH}" == "x86_64" ]]; then
 		kernel_version="$(get_from_kata_deps .assets.kernel.confidential.version)"
 		kernel_url="$(get_from_kata_deps .assets.kernel.confidential.url)"
 	fi
@@ -645,7 +652,7 @@ install_kernel_helper() {
 		extra_tarballs="${kernel_modules_tarball_name}:${kernel_modules_tarball_path}"
 	fi
 
-	if [[ "${kernel_name}" == "kernel-nvidia-gpu*" ]]; then
+	if [[ "${kernel_name}" == "kernel-nvidia-gpu*" ]] || [[ "${kernel_name}" == "kernel-cca-confidential" ]]; then
 		local kernel_headers_tarball_name="kata-static-${kernel_name}-headers.tar.xz"
 		local kernel_headers_tarball_path="${workdir}/${kernel_headers_tarball_name}"
 		extra_tarballs+=" ${kernel_headers_tarball_name}:${kernel_headers_tarball_path}"
@@ -657,6 +664,9 @@ install_kernel_helper() {
 
 	info "build ${kernel_name}"
 	info "Kernel version ${kernel_version}"
+	if [ -n "${kernel_ref}" ]; then
+		extra_cmd+=" -r ${kernel_ref}"
+	fi
 	DESTDIR="${destdir}" PREFIX="${prefix}" "${kernel_builder}" -v "${kernel_version}" -f -u "${kernel_url}" "${extra_cmd}"
 }
 
@@ -681,6 +691,15 @@ install_kernel_confidential() {
 		"-x"
 }
 
+install_kernel_cca_confidential() {
+	export MEASURED_ROOTFS=yes
+
+	install_kernel_helper \
+		"assets.kernel-arm-experimental.confidential" \
+		"kernel-confidential" \
+		"-x -H deb"
+}
+
 install_kernel_dragonball_experimental() {
 	install_kernel_helper \
 		"assets.kernel-dragonball-experimental" \
@@ -1190,6 +1209,7 @@ handle_build() {
 		install_kata_manager
 		install_kernel
 		install_kernel_confidential
+		install_kernel_cca_confidential
 		install_kernel_dragonball_experimental
 		install_log_parser_rs
 		install_nydus
@@ -1233,6 +1253,8 @@ handle_build() {
 
 	kernel-confidential) install_kernel_confidential ;;
 
+  kernel-cca-confidential) install_kernel_cca_confidential ;;
+
 	kernel-dragonball-experimental) install_kernel_dragonball_experimental ;;
 
 	kernel-nvidia-gpu-dragonball-experimental) install_kernel_nvidia_gpu_dragonball_experimental ;;
@@ -1299,7 +1321,7 @@ handle_build() {
 	tar tvf "${final_tarball_path}"
 
 	case ${build_target} in
-		kernel-nvidia-gpu*)
+		kernel-nvidia-gpu* | kernel-cca-confidential)
 			local kernel_headers_final_tarball_path="${workdir}/kata-static-${build_target}-headers.tar.xz"
 			if [ ! -f "${kernel_headers_final_tarball_path}" ]; then
 				local kernel_headers_dir
@@ -1401,7 +1423,7 @@ handle_build() {
 					"kata-static-${build_target}-headers.tar.xz"
 				)
 				;;
-			kernel-nvidia-gpu-confidential)
+			kernel-nvidia-gpu-confidential | kernel-cca-confidential)
 				files_to_push+=(
 					"kata-static-${build_target}-modules.tar.xz"
 					"kata-static-${build_target}-headers.tar.xz"

tools/packaging/kernel/build-kernel.sh

@@ -65,6 +65,8 @@ PREFIX="${PREFIX:-/usr}"
 kernel_url=""
 #Linux headers for GPU guest fs module building
 linux_headers=""
+# Kernel Reference to download using git
+kernel_ref=""
 # Enable measurement of the guest rootfs at boot.
 measured_rootfs="false"
 
@@ -109,6 +111,7 @@ Options:
 	-m              : Enable measured rootfs.
 	-k <path>   	: Path to kernel to build.
 	-p <path>   	: Path to a directory with patches to apply to kernel.
+	-r <ref>        : Enable git mode to download kernel using ref.
 	-s          	: Skip .config checks
 	-t <hypervisor>	: Hypervisor_target.
 	-u <url>	: Kernel URL to be used to download the kernel tarball.
@@ -138,6 +141,26 @@ check_initramfs_or_die() {
 		die "Initramfs for measured rootfs not found at ${default_initramfs}"
 }
 
+get_git_kernel() {
+	local kernel_path="${2:-}"
+
+	if [ ! -d "${kernel_path}" ] ; then
+		mkdir -p "${kernel_path}"
+		pushd "${kernel_path}"
+		local kernel_git_url="https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git"
+		if [ -n "${kernel_url}" ]; then
+			kernel_git_url="${kernel_url}"
+		fi
+		git init
+		git remote add origin "${kernel_git_url}"
+		popd
+	fi
+	pushd "${kernel_path}"
+	git fetch --depth 1 origin "${kernel_ref}"
+	git checkout "${kernel_ref}"
+	popd
+}
+
 get_kernel() {
 	local version="${1:-}"
 
@@ -341,6 +364,10 @@ get_kernel_frag_path() {
 	results=$(grep "${not_in_string}" <<< "$results")
 	# Do not care about options that are in whitelist
 	results=$(grep -v -f ${default_config_whitelist} <<< "$results")
+	local version_config_whitelist="${default_config_whitelist%.*}-${kernel_version}.conf"
+	if [ -f ${version_config_whitelist} ]; then
+		results=$(grep -v -f ${version_config_whitelist} <<< "$results")
+	fi
 
 	[[ "${skip_config_checks}" == "true" ]] && echo "${config_path}" && return
 
@@ -441,7 +468,11 @@ setup_kernel() {
 		[ -n "$kernel_version" ] || die "failed to get kernel version: Kernel version is emtpy"
 
 		if [[ ${download_kernel} == "true" ]]; then
-			get_kernel "${kernel_version}" "${kernel_path}"
+			if [ -z "${kernel_ref}" ]; then
+				get_kernel "${kernel_version}" "${kernel_path}"
+			else
+				get_git_kernel "${kernel_version}" "${kernel_path}"
+			fi
 		fi
 
 		[ -n "$kernel_path" ] || die "failed to find kernel source path"
@@ -591,7 +622,7 @@ install_kata() {
 }
 
 main() {
-	while getopts "a:b:c:dD:eEfg:hH:k:mp:st:u:v:x" opt; do
+	while getopts "a:b:c:dD:eEfg:hH:k:mp:r:st:u:v:x" opt; do
 		case "$opt" in
 			a)
 				arch_target="${OPTARG}"
@@ -638,6 +669,9 @@ main() {
 			p)
 				patches_path="${OPTARG}"
 				;;
+			r)
+				kernel_ref="${OPTARG}"
+				;;
 			s)
 				skip_config_checks="true"
 				;;

tools/packaging/kernel/configs/fragments/arm64/confidential/cca.conf

@@ -0,0 +1,9 @@
+CONFIG_VIRT_DRIVERS=y
+CONFIG_TSM_REPORTS=y
+CONFIG_ARM_CCA_GUEST=y
+CONFIG_HW_RANDOM=y
+CONFIG_HW_RANDOM_VIRTIO=y
+CONFIG_ACPI_PROCESSOR=y
+CONFIG_HOTPLUG_CPU=y
+CONFIG_ACPI_HOTPLUG_CPU=y
+CONFIG_RODATA_FULL_DEFAULT_ENABLED=y

tools/packaging/kernel/configs/fragments/arm64/confidential/hotplug.conf

@@ -0,0 +1,3 @@
+# Define hotplugs to be online immediately. Speeds things up, and makes things
+# work smoother on some arch's.
+CONFIG_MHP_DEFAULT_ONLINE_TYPE_ONLINE_AUTO=y

tools/packaging/kernel/configs/fragments/arm64/confidential/rme.conf

@@ -0,0 +1,3 @@
+CONFIG_ARCH_HAS_CC_PLATFORM=y
+CONFIG_ARCH_HAS_MEM_ENCRYPT=y
+CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED=y

tools/packaging/kernel/configs/fragments/whitelist-6.15.0-rc1-916aeec68dd4500a1cdf4ebf214c5620955daf3f.conf

@@ -0,0 +1,9 @@
+# CONFIG_RANDOM_TRUST_CPU is removed from config since v6.2
+# https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=b9b01a5625b5a9e9d96d14d4a813a54e8a124f4b
+CONFIG_RANDOM_TRUST_CPU
+# CONFIG_ACPI_HOTPLUG_CPU is disabled arm64 and riscv since v6.8
+# https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=a02f66bb3cf475947b58dd3851b987b8ccd998c1
+CONFIG_ACPI_HOTPLUG_CPU
+# CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE is disabled since v6.14
+# https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=44d46b76c3a4b514a0cc9dab147ed430e5c1d699
+CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE

tools/packaging/kernel/kata_config_version

@@ -1 +1 @@
-160
+161

tools/packaging/static-build/kernel/Dockerfile

@@ -22,6 +22,7 @@ RUN apt-get update && \
 	    kmod \
 	    libelf-dev \
 	    libssl-dev \
+	    python3 \
 	    gettext \
 	    rsync \
 	    cpio \

tools/packaging/static-build/shim-v2/Dockerfile

@@ -58,3 +58,6 @@ RUN ARCH=$(uname -m); \
     curl -OL "https://storage.googleapis.com/golang/go${GO_VERSION}.${kernelname}-${goarch}.tar.gz" && \
     tar -C "${GO_HOME}" -xzf "go${GO_VERSION}.${kernelname}-${goarch}.tar.gz" && \
     rm "go${GO_VERSION}.${kernelname}-${goarch}.tar.gz"
+
+COPY pkg pkg
+RUN dpkg -i pkg/linux-* || true

tools/packaging/static-build/shim-v2/build.sh

@@ -49,6 +49,11 @@ if [ "${MEASURED_ROOTFS}" == "yes" ]; then
 	EXTRA_OPTS+=" ROOTMEASURECONFIG=\"${root_measure_config}\""
 fi
 
+# add kernel header package
+rm -rf "${script_dir}/pkg"
+mkdir -p "${script_dir}/pkg"
+find ${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/ -maxdepth 1 -name "kata-static-*-headers.tar.xz" -exec tar -Jxvf {} -C ${script_dir}/pkg \;
+
 docker pull ${container_image} || \
 	(docker ${BUILDX} build ${PLATFORM}  \
 		--build-arg GO_VERSION="${GO_VERSION}" \

versions.yaml

@@ -206,6 +206,11 @@ assets:
     description: "Linux kernel with cpu/mem hotplug support on arm64"
     url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/"
     version: "v5.15.138"
+    confidential:
+      description: "Linux kernel with RME support on arm64"
+      url: "https://gitlab.arm.com/linux-arm/linux-cca"
+      version: "v6.15.0-rc1-916aeec68dd4500a1cdf4ebf214c5620955daf3f"
+      ref: "916aeec68dd4500a1cdf4ebf214c5620955daf3f"
 
   kernel-dragonball-experimental:
     description: "Linux kernel with Dragonball VMM optimizations like upcall"