Merge pull request #6595 from zvonkok/gpu-snp-tdx-kernel

gpu: Build and Ship an GPU enabled Kernel

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -21,6 +21,9 @@ jobs:
           - kernel
           - kernel-dragonball-experimental
           - kernel-tdx-experimental
+          - kernel-gpu
+          - kernel-gpu-snp
+          - kernel-gpu-tdx-experimental
           - nydus
           - qemu
           - qemu-tdx-experimental

tools/packaging/kata-deploy/local-build/Makefile

@@ -26,6 +26,9 @@ all: serial-targets \
 	kernel-tarball \
 	kernel-dragonball-experimental-tarball \
 	kernel-tdx-experimental-tarball \
+	kernel-gpu \
+	kernel-gpu-snp-tarball \
+	kernel-gpu-tdx-experimental-tarball \
 	nydus-tarball \
 	qemu-tarball \
 	qemu-tdx-experimental-tarball \
@@ -54,6 +57,15 @@ kernel-tarball:
 kernel-dragonball-experimental-tarball:
 	${MAKE} $@-build
 
+kernel-gpu-tarball:
+	${MAKE} $@-build
+
+kernel-gpu-snp-tarball:
+	${MAKE} $@-build
+
+kernel-gpu-tdx-experimental-tarball:
+	${MAKE} $@-build	
+
 kernel-experimental-tarball:
 	${MAKE} $@-build
 

tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh

@@ -82,6 +82,9 @@ options:
 	kernel-dragonball-experimental
 	kernel-experimental
 	kernel-tdx-experimental
+	kernel-gpu
+	kernel-gpu-snp
+	kernel-gpu-tdx-experimental
 	nydus
 	qemu
 	qemu-tdx-experimental
@@ -213,6 +216,36 @@ install_kernel_dragonball_experimental() {
 		"-e -t dragonball"
 }
 
+#Install GPU enabled kernel asset
+install_kernel_gpu() {
+	local kernel_url="$(get_from_kata_deps assets.kernel.url)"
+
+	install_kernel_helper \
+		"assets.kernel.version" \
+		"kernel-gpu" \
+		"-g nvidia -u ${kernel_url} -H deb"
+}
+
+#Install GPU and SNP enabled kernel asset
+install_kernel_gpu_snp() {
+	local kernel_url="$(get_from_kata_deps assets.kernel.snp.url)"
+
+	install_kernel_helper \
+		"assets.kernel.snp.version" \
+		"kernel-gpu-snp" \
+		"-x snp -g nvidia -u ${kernel_url} -H deb"
+}
+
+#Install GPU and TDX experimental enabled kernel asset
+install_kernel_gpu_tdx_experimental() {
+	local kernel_url="$(get_from_kata_deps assets.kernel-tdx-experimental.url)"
+
+	install_kernel_helper \
+		"assets.kernel-tdx-experimental.version" \
+		"kernel-gpu-tdx" \
+		"-x tdx -g nvidia -u ${kernel_url} -H deb"
+}
+
 #Install experimental kernel asset
 install_kernel_experimental() {
 	install_kernel_helper \
@@ -448,6 +481,12 @@ handle_build() {
 
 	kernel-tdx-experimental) install_kernel_tdx_experimental ;;
 
+	kernel-gpu) install_kernel_gpu ;;
+
+	kernel-gpu-snp) install_kernel_gpu_snp;;
+
+	kernel-gpu-tdx-experimental) install_kernel_gpu_tdx_experimental;;
+
 	qemu) install_qemu ;;
 
 	qemu-tdx-experimental) install_qemu_tdx_experimental ;;

tools/packaging/kernel/README.md

@@ -47,6 +47,7 @@ Options:
 	-f          	: Enable force generate config when setup.
 	-g <vendor> 	: GPU vendor, intel or nvidia.
 	-h          	: Display this help.
+  -H <deb|rpm>  : Linux headers for guest fs module building.
 	-k <path>   	: Path to kernel to build.
 	-p <path>   	: Path to a directory with patches to apply to kernel, only patches in top-level directory are applied.
 	-t <hypervisor>	: Hypervisor_target.

tools/packaging/kernel/build-kernel.sh

@@ -61,6 +61,8 @@ DESTDIR="${DESTDIR:-/}"
 PREFIX="${PREFIX:-/usr}"
 #Kernel URL
 kernel_url=""
+#Linux headers for GPU guest fs module building
+linux_headers=""
 
 packaging_scripts_dir="${script_dir}/../scripts"
 source "${packaging_scripts_dir}/lib.sh"
@@ -95,6 +97,7 @@ Options:
 	-f          	: Enable force generate config when setup.
 	-g <vendor> 	: GPU vendor, intel or nvidia.
 	-h          	: Display this help.
+	-H <deb|rpm>	: Linux headers for guest fs module building.
 	-k <path>   	: Path to kernel to build.
 	-p <path>   	: Path to a directory with patches to apply to kernel.
 	-s          	: Skip .config checks
@@ -241,6 +244,23 @@ get_kernel_frag_path() {
 		info "Add kernel config for GPU due to '-g ${gpu_vendor}'"
 		local gpu_configs="$(ls ${gpu_path}/${gpu_vendor}.conf)"
 		all_configs="${all_configs} ${gpu_configs}"
+		# If conf_guest is set we need to update the CONFIG_LOCALVERSION
+		# to match the suffix created in install_kata
+		# -nvidia-gpu-{snp|tdx}, the linux headers will be named the very
+		# same if build with make deb-pkg for TDX or SNP.
+		if [[ "${conf_guest}" != "" ]];then
+			local gpu_cc_configs=$(mktemp).conf
+			local gpu_subst_configs="$(ls ${gpu_path}/${gpu_vendor}.conf.in)"
+
+			export CONF_GUEST_SUFFIX="-${conf_guest}"
+			envsubst <${gpu_subst_configs} >${gpu_cc_configs}
+			unset CONF_GUEST_SUFFIX
+
+			all_configs="${all_configs} ${gpu_cc_configs}"
+		else
+			local gpu_configs="$(ls ${gpu_path}/${gpu_vendor}.conf)"
+			all_configs="${all_configs} ${gpu_configs}"
+		fi
 	fi
 
 	if [[ "${conf_guest}" != "" ]];then
@@ -415,6 +435,24 @@ build_kernel() {
 	popd >>/dev/null
 }
 
+build_kernel_headers() {
+	local kernel_path=${1:-}
+	[ -n "${kernel_path}" ] || die "kernel_path not provided"
+	[ -d "${kernel_path}" ] || die "path to kernel does not exist, use ${script_name} setup"
+	[ -n "${arch_target}" ] || arch_target="$(uname -m)"
+	arch_target=$(arch_to_kernel "${arch_target}")
+	pushd "${kernel_path}" >>/dev/null
+
+	if [ "$linux_headers" == "deb" ]; then
+		make -j $(nproc ${CI:+--ignore 1}) deb-pkg ARCH="${arch_target}"
+	fi
+	if [ "$linux_headers" == "rpm" ]; then
+		make -j $(nproc ${CI:+--ignore 1}) rpm-pkg ARCH="${arch_target}"
+	fi
+
+	popd >>/dev/null
+}
+
 install_kata() {
 	local kernel_path=${1:-}
 	[ -n "${kernel_path}" ] || die "kernel_path not provided"
@@ -430,14 +468,15 @@ install_kata() {
 	if [[ ${build_type} != "" ]]; then
 		suffix="-${build_type}"
 	fi
-	if [[ ${gpu_vendor} != "" ]];then
-		suffix="-${gpu_vendor}-gpu${suffix}"
-	fi
 
 	if [[ ${conf_guest} != "" ]];then
 		suffix="-${conf_guest}${suffix}"
 	fi
 
+	if [[ ${gpu_vendor} != "" ]];then
+		suffix="-${gpu_vendor}-gpu${suffix}"
+	fi
+
 	vmlinuz="vmlinuz-${kernel_version}-${config_version}${suffix}"
 	vmlinux="vmlinux-${kernel_version}-${config_version}${suffix}"
 
@@ -475,7 +514,7 @@ install_kata() {
 }
 
 main() {
-	while getopts "a:b:c:deEfg:hk:p:t:u:v:x:" opt; do
+	while getopts "a:b:c:deEfg:hH:k:p:t:u:v:x:" opt; do
 		case "$opt" in
 			a)
 				arch_target="${OPTARG}"
@@ -506,6 +545,9 @@ main() {
 			h)
 				usage 0
 				;;
+			H)
+				linux_headers="${OPTARG}"
+				;;
 			k)
 				kernel_path="$(realpath ${OPTARG})"
 				;;
@@ -594,6 +636,9 @@ main() {
 		build)
 			build_kernel "${kernel_path}"
 			;;
+		build-headers)
+			build_kernel_headers "${kernel_path}"
+			;;
 		install)
 			install_kata "${kernel_path}"
 			;;

tools/packaging/kernel/configs/fragments/gpu/nvidia.conf.in

@@ -0,0 +1,14 @@
+# Support mmconfig PCI config space access.
+# It's used to enable the MMIO access method for PCIe devices.
+CONFIG_PCI_MMCONFIG=y
+
+# Support for loading modules.
+# It is used to support loading GPU drivers.
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+
+# CRYPTO_FIPS requires this config when loading modules is enabled.
+CONFIG_MODULE_SIG=y
+
+# Linux kernel version suffix
+CONFIG_LOCALVERSION="-nvidia-gpu${CONF_GUEST_SUFFIX}"

tools/packaging/kernel/kata_config_version

@@ -1 +1 @@
-104
+105

tools/packaging/static-build/kernel/Dockerfile

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 ENV DEBIAN_FRONTEND=noninteractive
 
 # kernel deps
@@ -18,6 +18,9 @@ RUN apt-get update && \
 	    iptables \
 	    kmod \
 	    libelf-dev \
+	    libssl-dev \
+	    gettext \
+	    rsync \
+	    cpio \
 	    patch && \
-    if [ "$(uname -m)" = "s390x" ]; then apt-get install -y --no-install-recommends libssl-dev; fi && \
-    apt-get clean && rm -rf /var/lib/lists/
+    apt-get clean && apt-get autoclean

tools/packaging/static-build/kernel/build.sh

@@ -38,3 +38,9 @@ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	--env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \
 	"${container_image}" \
 	bash -c "${kernel_builder} $* install"
+
+sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
+	-w "${PWD}" \
+	--env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \
+	"${container_image}" \
+	bash -c "${kernel_builder} $* build-headers"