From bc8360e8a91fdaf9361ae81bf27eac46dd6d1bbd Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Wed, 12 Feb 2025 01:04:27 +0000 Subject: [PATCH 01/13] gpu: Add proper config for module signing We want to enable module signing in Kata and Coco Signed-off-by: Zvonko Kaiser --- .../configs/fragments/gpu/nvidia.x86_64.conf.in | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/packaging/kernel/configs/fragments/gpu/nvidia.x86_64.conf.in b/tools/packaging/kernel/configs/fragments/gpu/nvidia.x86_64.conf.in index 85fc626ac3..a1386b239b 100644 --- a/tools/packaging/kernel/configs/fragments/gpu/nvidia.x86_64.conf.in +++ b/tools/packaging/kernel/configs/fragments/gpu/nvidia.x86_64.conf.in @@ -7,9 +7,6 @@ CONFIG_PCI_MMCONFIG=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y -# CRYPTO_FIPS requires this config when loading modules is enabled. -CONFIG_MODULE_SIG=y - # Linux kernel version suffix CONFIG_LOCALVERSION="-nvidia-gpu${CONF_GUEST_SUFFIX}" @@ -25,3 +22,11 @@ CONFIG_X86_PAT=y CONFIG_CRYPTO_ECC=y CONFIG_CRYPTO_ECDH=y CONFIG_CRYPTO_ECDSA=y + +# Module signing +CONFIG_MODULE_SIG=y +CONFIG_MODULE_SIG_FORCE=y +CONFIG_MODULE_SIG_ALL=y +CONFIG_MODULE_SIG_SHA512=y +CONFIG_SYSTEM_TRUSTED_KEYS="" +CONFIG_SYSTEM_TRUSTED_KEYRING=y From c2cb89532b826ae4299904b5c09ea9c17d389de7 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Wed, 12 Feb 2025 15:27:57 +0000 Subject: [PATCH 02/13] gpu: Add the proper handling in build-kernel.sh If KBUILD_SIGN_PIN is provided we can encrypt the signing key for out-of-tree builds and second round jobs in GHA Signed-off-by: Zvonko Kaiser --- tools/packaging/kernel/build-kernel.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/packaging/kernel/build-kernel.sh b/tools/packaging/kernel/build-kernel.sh index f96a2fd1a5..5307f2bc64 100755 --- a/tools/packaging/kernel/build-kernel.sh +++ b/tools/packaging/kernel/build-kernel.sh @@ -32,6 +32,7 @@ readonly default_initramfs="${script_dir}/initramfs.cpio.gz" # xPU vendor readonly VENDOR_INTEL="intel" readonly VENDOR_NVIDIA="nvidia" +readonly KBUILD_SIGN_PIN=${KBUILD_SIGN_PIN:-""} #Path to kernel directory kernel_path="" @@ -493,6 +494,15 @@ build_kernel_headers() { if [ "$linux_headers" == "rpm" ]; then make -j $(nproc) rpm-pkg ARCH="${arch_target}" fi + # If we encrypt the key earlier it will break the kernel_headers build. + # At this stage the kernel has created the certs/signing_key.pem + # encrypt it for later usage in another job or out-of-tree build + # only encrypt if we have KBUILD_SIGN_PIN set + local key="certs/signing_key.pem" + if [ -n "${KBUILD_SIGN_PIN}" ]; then + [ -e "${key}" ] || die "${key} missing but KBUILD_SIGN_PIN is set" + openssl rsa -aes256 -in ${key} -out ${key} -passout env:KBUILD_SIGN_PIN + fi popd >>/dev/null } From d815fb6f4616b44aa4cd7afe11bf69108edcede2 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Wed, 12 Feb 2025 15:35:45 +0000 Subject: [PATCH 03/13] gpu: Update kernel-headers Use the kernel-headers as the extra_tarball to move the encrypted key and cert from stage to stage Signed-off-by: Zvonko Kaiser --- .../kata-deploy/local-build/kata-deploy-binaries.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 48d04099c4..f82c614218 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -1246,7 +1246,12 @@ handle_build() { kernel_headers_dir=$(get_kernel_headers_dir "${build_target}") pushd "${kernel_headers_dir}" - find . -type f -name "*.${KERNEL_HEADERS_PKG_TYPE}" -exec tar cvfJ "${kernel_headers_final_tarball_path}" {} + + find . -type f -name "*.${KERNEL_HEADERS_PKG_TYPE}" -exec tar rvf kernel-headers.tar {} + + mv kata-linux-*/certs/signing_key.pem . + mv kata-linux-*/certs/signing_key.x509 . + tar -rvf kernel-headers.tar signing_key.pem signing_key.x509 --remove-files + xz -T0 kernel-headers.tar + mv kernel-headers.tar.xz "${kernel_headers_final_tarball_path}" popd fi tar tvf "${kernel_headers_final_tarball_path}" From 39d3b7fb90c4d8a902ebde58e85e12a505dee562 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Wed, 12 Feb 2025 15:46:50 +0000 Subject: [PATCH 04/13] gpu: Update NVIDIA chroot script We need to place the signing key and cert at the right place and hide the KBUILD_SIGN_PIN from echo'ing or xtrace Signed-off-by: Zvonko Kaiser --- .../rootfs-builder/nvidia/nvidia_chroot.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tools/osbuilder/rootfs-builder/nvidia/nvidia_chroot.sh b/tools/osbuilder/rootfs-builder/nvidia/nvidia_chroot.sh index 92ed8d078c..357195acde 100644 --- a/tools/osbuilder/rootfs-builder/nvidia/nvidia_chroot.sh +++ b/tools/osbuilder/rootfs-builder/nvidia/nvidia_chroot.sh @@ -5,7 +5,7 @@ # SPDX-License-Identifier: Apache-2.0 #!/bin/bash -set -xeuo pipefail +set -euo pipefail shopt -s nullglob shopt -s extglob @@ -21,6 +21,8 @@ base_os="jammy" APT_INSTALL="apt -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' -yqq --no-install-recommends install" +export KBUILD_SIGN_PIN=$6 + export DEBIAN_FRONTEND=noninteractive is_feature_enabled() { @@ -104,9 +106,13 @@ build_nvidia_drivers() { echo "chroot: Build NVIDIA drivers" pushd "${driver_source_files}" >> /dev/null + local certs_dir local kernel_version for version in /lib/modules/*; do kernel_version=$(basename "${version}") + certs_dir=/lib/modules/"${kernel_version}"/build/certs + signing_key=${certs_dir}/signing_key.pem + echo "chroot: Building GPU modules for: ${kernel_version}" cp /boot/System.map-"${kernel_version}" /lib/modules/"${kernel_version}"/build/System.map @@ -119,9 +125,16 @@ build_nvidia_drivers() { fi make -j "$(nproc)" CC=gcc SYSSRC=/lib/modules/"${kernel_version}"/build > /dev/null + + if [ -n "${KBUILD_SIGN_PIN}" ]; then + mkdir -p "${certs_dir}" && mv /signing_key.* "${certs_dir}"/. + fi + make INSTALL_MOD_STRIP=1 -j "$(nproc)" CC=gcc SYSSRC=/lib/modules/"${kernel_version}"/build modules_install make -j "$(nproc)" CC=gcc SYSSRC=/lib/modules/"${kernel_version}"/build clean > /dev/null - + # The make clean above should clear also the certs directory but just in case something + # went wroing make sure the signing_key.pem is removed + [ -e "${signing_key}" ] && rm -f "${signing_key}" done # Save the modules for later so that a linux-image purge does not remove it tar cvfa /lib/modules.save_from_purge.tar.zst /lib/modules From 9602ba6ccc5fb4f5a5f65849db96c3be6f83a600 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Wed, 12 Feb 2025 17:03:43 +0000 Subject: [PATCH 05/13] gpu: Add proper KBUILD_SIGN_PIN to entry script Update kata-deploy-binaries-in-docker.sh to read the env variable KBUILD_SIGN_PIN that either can be set via GHA or other means. Signed-off-by: Zvonko Kaiser --- .../local-build/kata-deploy-binaries-in-docker.sh | 2 ++ .../kata-deploy/local-build/kata-deploy-binaries.sh | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh index 36e77fbdb6..6f0b114c1d 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh @@ -104,6 +104,7 @@ PULL_TYPE="${PULL_TYPE:-default}" USE_CACHE="${USE_CACHE:-}" BUSYBOX_CONF_FILE=${BUSYBOX_CONF_FILE:-} NVIDIA_GPU_STACK="${NVIDIA_GPU_STACK:-}" +KBUILD_SIGN_PIN=${KBUILD_SIGN_PIN:-} docker run \ -v $HOME/.docker:/root/.docker \ @@ -135,6 +136,7 @@ docker run \ --env USE_CACHE="${USE_CACHE}" \ --env BUSYBOX_CONF_FILE="${BUSYBOX_CONF_FILE}" \ --env NVIDIA_GPU_STACK="${NVIDIA_GPU_STACK}" \ + --env KBUILD_SIGN_PIN="${KBUILD_SIGN_PIN}" \ --env AA_KBC="${AA_KBC:-}" \ --env HKD_PATH="$(realpath "${HKD_PATH:-}" 2> /dev/null || true)" \ --env SE_KERNEL_PARAMS="${SE_KERNEL_PARAMS:-}" \ diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index f82c614218..5fc0e6f3f1 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -1247,9 +1247,12 @@ handle_build() { pushd "${kernel_headers_dir}" find . -type f -name "*.${KERNEL_HEADERS_PKG_TYPE}" -exec tar rvf kernel-headers.tar {} + - mv kata-linux-*/certs/signing_key.pem . - mv kata-linux-*/certs/signing_key.x509 . - tar -rvf kernel-headers.tar signing_key.pem signing_key.x509 --remove-files + if [ -n "${KBUILD_SIGN_PIN}" ]; then + head -n1 kata-linux-*/certs/signing_key.pem | grep -q "ENCRYPTED PRIVATE KEY" || die "signing_key.pem is not encrypted" + mv kata-linux-*/certs/signing_key.pem . + mv kata-linux-*/certs/signing_key.x509 . + tar -rvf kernel-headers.tar signing_key.pem signing_key.x509 --remove-files + fi xz -T0 kernel-headers.tar mv kernel-headers.tar.xz "${kernel_headers_final_tarball_path}" popd From 0309b705228e0e84e1c40249ab42fa3b921b57cd Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Wed, 12 Feb 2025 17:05:24 +0000 Subject: [PATCH 06/13] gpu: Pass-through KBUILD_SIGN_PIN In kata-deploy-binaries.sh we need to pass-through the var KBUILD_SIGN_PIN to the other static builder scripts. Signed-off-by: Zvonko Kaiser --- tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 5fc0e6f3f1..e58b71f323 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -53,6 +53,7 @@ TARGET_BRANCH="${TARGET_BRANCH:-main}" PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-}" KERNEL_HEADERS_PKG_TYPE="${KERNEL_HEADERS_PKG_TYPE:-deb}" RELEASE="${RELEASE:-"no"}" +KBUILD_SIGN_PIN="${KBUILD_SIGN_PIN:-}" workdir="${WORKDIR:-$PWD}" From 493ba63c778b72db70d86d8774602a4d2a0cfd04 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Wed, 12 Feb 2025 17:06:45 +0000 Subject: [PATCH 07/13] gpu: Provide KBUILD_SIGN_PIN to the build.sh At the proper step pass-through the var KBUILD_SIGN_PIN so that the kernel_headers step has the PIN for encrypting the signing key. Signed-off-by: Zvonko Kaiser --- .../kernel/configs/fragments/gpu/nvidia.x86_64.conf.in | 1 + tools/packaging/static-build/kernel/build.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tools/packaging/kernel/configs/fragments/gpu/nvidia.x86_64.conf.in b/tools/packaging/kernel/configs/fragments/gpu/nvidia.x86_64.conf.in index a1386b239b..4285234e0f 100644 --- a/tools/packaging/kernel/configs/fragments/gpu/nvidia.x86_64.conf.in +++ b/tools/packaging/kernel/configs/fragments/gpu/nvidia.x86_64.conf.in @@ -30,3 +30,4 @@ CONFIG_MODULE_SIG_ALL=y CONFIG_MODULE_SIG_SHA512=y CONFIG_SYSTEM_TRUSTED_KEYS="" CONFIG_SYSTEM_TRUSTED_KEYRING=y + diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh index cbf973c608..889283663b 100755 --- a/tools/packaging/static-build/kernel/build.sh +++ b/tools/packaging/static-build/kernel/build.sh @@ -22,6 +22,7 @@ DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}" MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} +KBUILD_SIGN_PIN="${KBUILD_SIGN_PIN:-}" kernel_builder_args="-a ${ARCH} $*" if [ "${MEASURED_ROOTFS}" == "yes" ]; then @@ -71,6 +72,7 @@ docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${PWD}" \ --env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \ --env USER="${USER}" \ + --env KBUILD_SIGN_PIN="${KBUILD_SIGN_PIN}" \ --user "$(id -u)":"$(id -g)" \ "${container_image}" \ bash -c "${kernel_builder} ${kernel_builder_args} build-headers" From 5ab3192c516da4a45e0dd1e540ff8d52bc649277 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Thu, 13 Feb 2025 02:27:23 +0000 Subject: [PATCH 08/13] gpu: Update nvidia_rootfs.sh We need to handle KBUILD_SIGN_PIN so that the kbuild can decrypte the signing key Signed-off-by: Zvonko Kaiser --- tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh b/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh index 96ee8250d9..dcc3bdf651 100644 --- a/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh +++ b/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh @@ -14,6 +14,7 @@ readonly SCRIPT_DIR="${script_dir}/nvidia" # This will control how much output the inird/image will produce DEBUG="" +KBUILD_SIGN_PIN=${KBUILD_SIGN_PIN:-""} machine_arch=${ARCH} @@ -165,7 +166,7 @@ setup_nvidia_gpu_rootfs_stage_one() { mount -t proc /proc ./proc chroot . /bin/bash -c "/nvidia_chroot.sh $(uname -r) ${run_file_name} \ - ${run_fm_file_name} ${machine_arch} ${NVIDIA_GPU_STACK}" + ${run_fm_file_name} ${machine_arch} ${NVIDIA_GPU_STACK} ${KBUILD_SIGN_PIN}" umount -R ./dev umount ./proc From 4dadd07699138d84c7211931e93a6f178d2ec079 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Thu, 13 Feb 2025 02:27:50 +0000 Subject: [PATCH 09/13] gpu: Update rootfs.sh Pass-through KBUILD_SIGN_PIN to the rootfs build Signed-off-by: Zvonko Kaiser --- tools/osbuilder/rootfs-builder/rootfs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/osbuilder/rootfs-builder/rootfs.sh b/tools/osbuilder/rootfs-builder/rootfs.sh index fd7d98ff27..21ae5b9510 100755 --- a/tools/osbuilder/rootfs-builder/rootfs.sh +++ b/tools/osbuilder/rootfs-builder/rootfs.sh @@ -53,6 +53,7 @@ USE_DOCKER=${USE_DOCKER:-""} USE_PODMAN=${USE_PODMAN:-""} EXTRA_PKGS=${EXTRA_PKGS:-""} +KBUILD_SIGN_PIN=${KBUILD_SIGN_PIN:-""} NVIDIA_GPU_STACK=${NVIDIA_GPU_STACK:-""} nvidia_rootfs="${script_dir}/nvidia/nvidia_rootfs.sh" [ "${ARCH}" == "x86_64" ] || [ "${ARCH}" == "aarch64" ] && source "$nvidia_rootfs" @@ -564,6 +565,7 @@ build_rootfs_distro() --env AGENT_POLICY="${AGENT_POLICY}" \ --env CONFIDENTIAL_GUEST="${CONFIDENTIAL_GUEST}" \ --env NVIDIA_GPU_STACK="${NVIDIA_GPU_STACK}" \ + --env KBUILD_SIGN_PIN="${KBUILD_SIGN_PIN}" \ -v "${repo_dir}":"/kata-containers" \ -v "${ROOTFS_DIR}":"/rootfs" \ -v "${script_dir}/../scripts":"/scripts" \ From c4e4e14b32ac036ba004359068fb6d7194016297 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Fri, 28 Feb 2025 13:58:59 +0000 Subject: [PATCH 10/13] kernel: bump kata_config_version Mandatory update to have a unique kernel version name Signed-off-by: Zvonko Kaiser --- tools/packaging/kernel/build-kernel.sh | 1 + tools/packaging/kernel/kata_config_version | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/packaging/kernel/build-kernel.sh b/tools/packaging/kernel/build-kernel.sh index 5307f2bc64..44e0ddff9f 100755 --- a/tools/packaging/kernel/build-kernel.sh +++ b/tools/packaging/kernel/build-kernel.sh @@ -70,6 +70,7 @@ measured_rootfs="false" CROSS_BUILD_ARG="" packaging_scripts_dir="${script_dir}/../scripts" +# shellcheck source=tools/packaging/scripts/lib.sh source "${packaging_scripts_dir}/lib.sh" usage() { diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version index c748b568f7..0d667b5e3a 100644 --- a/tools/packaging/kernel/kata_config_version +++ b/tools/packaging/kernel/kata_config_version @@ -1 +1 @@ -147 +148 From af1d6c2407becc752c21f24063d39bccf3d057be Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Fri, 28 Feb 2025 16:23:42 +0000 Subject: [PATCH 11/13] shecllcheck: Update nvidia_chroot.sh Make shellcheck happy with the new rules new updates needed Signed-off-by: Zvonko Kaiser --- .../rootfs-builder/nvidia/nvidia_chroot.sh | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/tools/osbuilder/rootfs-builder/nvidia/nvidia_chroot.sh b/tools/osbuilder/rootfs-builder/nvidia/nvidia_chroot.sh index 357195acde..3948de1e85 100644 --- a/tools/osbuilder/rootfs-builder/nvidia/nvidia_chroot.sh +++ b/tools/osbuilder/rootfs-builder/nvidia/nvidia_chroot.sh @@ -21,14 +21,14 @@ base_os="jammy" APT_INSTALL="apt -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' -yqq --no-install-recommends install" -export KBUILD_SIGN_PIN=$6 +export KBUILD_SIGN_PIN="${6:-}" export DEBIAN_FRONTEND=noninteractive is_feature_enabled() { local feature="$1" # Check if feature is in the comma-separated list - if [[ ",$nvidia_gpu_stack," == *",$feature,"* ]]; then + if [[ ",${nvidia_gpu_stack}," == *",${feature},"* ]]; then return 0 else return 1 @@ -38,11 +38,11 @@ is_feature_enabled() { set_driver_version_type() { echo "chroot: Setting the correct driver version" - if [[ ",$nvidia_gpu_stack," == *",latest,"* ]]; then + if [[ ",${nvidia_gpu_stack}," == *",latest,"* ]]; then driver_version="latest" - elif [[ ",$nvidia_gpu_stack," == *",lts,"* ]]; then + elif [[ ",${nvidia_gpu_stack}," == *",lts,"* ]]; then driver_version="lts" - elif [[ "$nvidia_gpu_stack" =~ version=([^,]+) ]]; then + elif [[ "${nvidia_gpu_stack}" =~ version=([^,]+) ]]; then driver_version="${BASH_REMATCH[1]}" else echo "No known driver spec found. Please specify \"latest\", \"lts\", or \"version=\"." @@ -54,9 +54,9 @@ set_driver_version_type() { echo "chroot: Setting the correct driver type" # driver -> enable open or closed drivers - if [[ "$nvidia_gpu_stack" =~ (^|,)driver=open($|,) ]]; then + if [[ "${nvidia_gpu_stack}" =~ (^|,)driver=open($|,) ]]; then driver_type="-open" - elif [[ "$nvidia_gpu_stack" =~ (^|,)driver=closed($|,) ]]; then + elif [[ "${nvidia_gpu_stack}" =~ (^|,)driver=closed($|,) ]]; then driver_type="" fi @@ -76,7 +76,7 @@ install_nvidia_fabricmanager() { return } # if run_fm_file_name exists run it - if [ -f /"${run_fm_file_name}" ]; then + if [[ -f /"${run_fm_file_name}" ]]; then install_nvidia_fabricmanager_from_run_file else install_nvidia_fabricmanager_from_distribution @@ -116,17 +116,17 @@ build_nvidia_drivers() { echo "chroot: Building GPU modules for: ${kernel_version}" cp /boot/System.map-"${kernel_version}" /lib/modules/"${kernel_version}"/build/System.map - if [ "${arch_target}" == "aarch64" ]; then + if [[ "${arch_target}" == "aarch64" ]]; then ln -sf /lib/modules/"${kernel_version}"/build/arch/arm64 /lib/modules/"${kernel_version}"/build/arch/aarch64 fi - if [ "${arch_target}" == "x86_64" ]; then + if [[ "${arch_target}" == "x86_64" ]]; then ln -sf /lib/modules/"${kernel_version}"/build/arch/x86 /lib/modules/"${kernel_version}"/build/arch/amd64 fi make -j "$(nproc)" CC=gcc SYSSRC=/lib/modules/"${kernel_version}"/build > /dev/null - if [ -n "${KBUILD_SIGN_PIN}" ]; then + if [[ -n "${KBUILD_SIGN_PIN}" ]]; then mkdir -p "${certs_dir}" && mv /signing_key.* "${certs_dir}"/. fi @@ -134,7 +134,7 @@ build_nvidia_drivers() { make -j "$(nproc)" CC=gcc SYSSRC=/lib/modules/"${kernel_version}"/build clean > /dev/null # The make clean above should clear also the certs directory but just in case something # went wroing make sure the signing_key.pem is removed - [ -e "${signing_key}" ] && rm -f "${signing_key}" + [[ -e "${signing_key}" ]] && rm -f "${signing_key}" done # Save the modules for later so that a linux-image purge does not remove it tar cvfa /lib/modules.save_from_purge.tar.zst /lib/modules @@ -142,14 +142,14 @@ build_nvidia_drivers() { } install_userspace_components() { - if [ ! -f /"${run_file_name}" ]; then + if [[ ! -f /"${run_file_name}" ]]; then echo "chroot: Skipping NVIDIA userspace runfile components installation" return fi pushd /NVIDIA-* >> /dev/null # if aarch64 we need to remove --no-install-compat32-libs - if [ "${arch_target}" == "aarch64" ]; then + if [[ "${arch_target}" == "aarch64" ]]; then ./nvidia-installer --no-kernel-modules --no-systemd --no-nvidia-modprobe -s --x-prefix=/root else ./nvidia-installer --no-kernel-modules --no-systemd --no-nvidia-modprobe -s --x-prefix=/root --no-install-compat32-libs @@ -159,10 +159,10 @@ install_userspace_components() { } prepare_run_file_drivers() { - if [ "${driver_version}" == "latest" ]; then + if [[ "${driver_version}" == "latest" ]]; then driver_version="" echo "chroot: Resetting driver version not supported with run-file" - elif [ "${driver_version}" == "lts" ]; then + elif [[ "${driver_version}" == "lts" ]]; then driver_version="" echo "chroot: Resetting driver version not supported with run-file" fi @@ -176,16 +176,16 @@ prepare_run_file_drivers() { # Sooner or later RIM files will be only available remotely RIMFILE=$(ls NVIDIA-*/RIM_GH100PROD.swidtag) - if [ -e "${RIMFILE}" ]; then + if [[ -e "${RIMFILE}" ]]; then cp NVIDIA-*/RIM_GH100PROD.swidtag /usr/share/nvidia/rim/. fi popd >> /dev/null } prepare_distribution_drivers() { - if [ "${driver_version}" == "latest" ]; then + if [[ "${driver_version}" == "latest" ]]; then driver_version=$(apt-cache search --names-only 'nvidia-headless-no-dkms-.?.?.?-open' | awk '{ print $1 }' | tail -n 1 | cut -d'-' -f5) - elif [ "${driver_version}" == "lts" ]; then + elif [[ "${driver_version}" == "lts" ]]; then driver_version="550" fi @@ -208,11 +208,11 @@ prepare_distribution_drivers() { prepare_nvidia_drivers() { local driver_source_dir="" - if [ -f /"${run_file_name}" ]; then + if [[ -f /"${run_file_name}" ]]; then prepare_run_file_drivers for source_dir in /NVIDIA-*; do - if [ -d "${source_dir}" ]; then + if [[ -d "${source_dir}" ]]; then driver_source_files="${source_dir}"/kernel${driver_type} driver_source_dir="${source_dir}" break @@ -224,7 +224,7 @@ prepare_nvidia_drivers() { prepare_distribution_drivers for source_dir in /usr/src/nvidia*; do - if [ -d "${source_dir}" ]; then + if [[ -d "${source_dir}" ]]; then driver_source_files="${source_dir}" driver_source_dir="${source_dir}" break @@ -254,7 +254,7 @@ setup_apt_repositories() { # Changing the reference here also means changes needed for cuda_keyring # and cuda apt repository see install_dcgm for details - cat <<-CHROOT_EOF > /etc/apt/sources.list.d/${base_os}.list + cat <<-CHROOT_EOF > /etc/apt/sources.list.d/"${base_os}".list deb [arch=amd64] http://us.archive.ubuntu.com/ubuntu ${base_os} main restricted universe multiverse deb [arch=amd64] http://us.archive.ubuntu.com/ubuntu ${base_os}-updates main restricted universe multiverse deb [arch=amd64] http://us.archive.ubuntu.com/ubuntu ${base_os}-security main restricted universe multiverse @@ -286,13 +286,13 @@ get_supported_gpus_from_run_file() { local source_dir="$1" local supported_gpus_json="${source_dir}"/supported-gpus/supported-gpus.json - jq . < "${supported_gpus_json}" | grep '"devid"' | awk '{ print $2 }' | tr -d ',"' > ${supported_gpu_devids} + jq . < "${supported_gpus_json}" | grep '"devid"' | awk '{ print $2 }' | tr -d ',"' > "${supported_gpu_devids}" } get_supported_gpus_from_distro_drivers() { local supported_gpus_json=/usr/share/doc/nvidia-kernel-common-"${driver_version}"/supported-gpus.json - jq . < "${supported_gpus_json}" | grep '"devid"' | awk '{ print $2 }' | tr -d ',"' > ${supported_gpu_devids} + jq . < "${supported_gpus_json}" | grep '"devid"' | awk '{ print $2 }' | tr -d ',"' > "${supported_gpu_devids}" } export_driver_version() { @@ -315,8 +315,8 @@ install_nvidia_dcgm() { [[ ${base_os} == "jammy" ]] && osver="ubuntu2204" || die "Unknown base_os ${base_os} used" keyring="cuda-keyring_1.1-1_all.deb" - curl -O https://developer.download.nvidia.com/compute/cuda/repos/${osver}/${arch}/${keyring} - dpkg -i ${keyring} && rm -f ${keyring} + curl -O "https://developer.download.nvidia.com/compute/cuda/repos/${osver}/${arch}/${keyring}" + dpkg -i "${keyring}" && rm -f "${keyring}" apt update eval "${APT_INSTALL}" datacenter-gpu-manager @@ -328,7 +328,7 @@ cleanup_rootfs() { apt-mark hold libstdc++6 libzstd1 libgnutls30 pciutils # noble=libgnutls30t64 - if [ -n "${driver_version}" ]; then + if [[ -n "${driver_version}" ]]; then apt-mark hold libnvidia-cfg1-"${driver_version}" \ nvidia-compute-utils-"${driver_version}" \ nvidia-utils-"${driver_version}" \ @@ -355,7 +355,7 @@ cleanup_rootfs() { python3-pip software-properties-common ca-certificates \ linux-libc-dev nuitka python3-minimal - if [ -n "${driver_version}" ]; then + if [[ -n "${driver_version}" ]]; then apt purge -yqq nvidia-headless-no-dkms-"${driver_version}${driver_type}" \ nvidia-kernel-source-"${driver_version}${driver_type}" -yqq fi From 94579517d443dd1ef7fe20ff9f5c21f540db1aac Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Fri, 28 Feb 2025 16:36:05 +0000 Subject: [PATCH 12/13] shellcheck: Update nvidia_rootfs.sh With the new rules we need more updates. Signed-off-by: Zvonko Kaiser --- .../rootfs-builder/nvidia/nvidia_rootfs.sh | 72 ++++++++++--------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh b/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh index dcc3bdf651..76268387e3 100644 --- a/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh +++ b/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh @@ -5,7 +5,7 @@ # SPDX-License-Identifier: Apache-2.0 set -euo pipefail -[ -n "$DEBUG" ] && set -x +[[ -n "${DEBUG}" ]] && set -x readonly BUILD_DIR="/kata-containers/tools/packaging/kata-deploy/local-build/build/" # catch errors and then assign @@ -14,13 +14,17 @@ readonly SCRIPT_DIR="${script_dir}/nvidia" # This will control how much output the inird/image will produce DEBUG="" -KBUILD_SIGN_PIN=${KBUILD_SIGN_PIN:-""} +KBUILD_SIGN_PIN=${KBUILD_SIGN_PIN:-} -machine_arch=${ARCH} +NVIDIA_GPU_STACK=${NVIDIA_GPU_STACK:?NVIDIA_GPU_STACK must be set} +VARIANT=${VARIANT:?VARIANT must be set} +ARCH=${ARCH:?ARCH must be set} -if [[ "$machine_arch" == "aarch64" ]]; then +machine_arch="${ARCH}" + +if [[ "${machine_arch}" == "aarch64" ]]; then distro_arch="arm64" -elif [[ "$machine_arch" == "x86_64" ]]; then +elif [[ "${machine_arch}" == "x86_64" ]]; then distro_arch="amd64" else die "Unsupported architecture: ${machine_arch}" @@ -40,7 +44,7 @@ setup_nvidia-nvrc() { pushd "${TARGET_BUILD_DIR}" > /dev/null || exit 1 rm -rf "${PROJECT}" - git clone https://github.com/NVIDIA/${PROJECT}.git + git clone https://github.com/NVIDIA/"${PROJECT}".git pushd "${PROJECT}" > /dev/null || exit 1 @@ -68,8 +72,8 @@ setup_nvidia-gpu-admin-tools() { pushd "${TARGET_BUILD_DIR}" > /dev/null || exit 1 - rm -rf "$(basename ${TARGET_GIT})" - git clone ${TARGET_GIT} + rm -rf "$(basename "${TARGET_GIT}")" + git clone "${TARGET_GIT}" rm -rf dist # Installed via pipx local python environment @@ -99,14 +103,14 @@ setup_nvidia-dcgm-exporter() { local dex="dcgm-exporter" rm -rf "${dex}" - git clone --branch "${TARGET_VERSION}" https://github.com/NVIDIA/${dex} - make -C ${dex} binary + git clone --branch "${TARGET_VERSION}" https://github.com/NVIDIA/"${dex}" + make -C "${dex}" binary mkdir -p ../destdir/bin - mkdir -p ../destdir/etc/${dex} + mkdir -p ../destdir/etc/"${dex}" - cp ${dex}/cmd/${dex}/${dex} ../destdir/bin/. - cp ${dex}/etc/*.csv ../destdir/etc/${dex}/. + cp "${dex}"/cmd/"${dex}"/"${dex}" ../destdir/bin/. + cp "${dex}"/etc/*.csv ../destdir/etc/"${dex}"/. tar cvfa "${TARBALL}" -C ../destdir . tar tvf "${TARBALL}" @@ -115,7 +119,7 @@ setup_nvidia-dcgm-exporter() { } setup_nvidia_gpu_rootfs_stage_one() { - if [ -e "${BUILD_DIR}/kata-static-nvidia-gpu-rootfs-stage-one.tar.zst" ]; then + if [[ -e "${BUILD_DIR}/kata-static-nvidia-gpu-rootfs-stage-one.tar.zst" ]]; then info "nvidia: GPU rootfs stage one already exists" return fi @@ -124,11 +128,11 @@ setup_nvidia_gpu_rootfs_stage_one() { local rootfs_type=${1:-""} - info "nvidia: Setup GPU rootfs type=$rootfs_type" + info "nvidia: Setup GPU rootfs type=${rootfs_type}" for component in "nvidia-gpu-admin-tools" "nvidia-dcgm-exporter" "nvidia-nvrc"; do - if [ ! -e "${BUILD_DIR}/kata-static-${component}.tar.zst" ]; then - setup_${component} + if [[ ! -e "${BUILD_DIR}/kata-static-${component}.tar.zst" ]]; then + setup_"${component}" fi done @@ -137,28 +141,28 @@ setup_nvidia_gpu_rootfs_stage_one() { chmod +x ./nvidia_chroot.sh local appendix="" - if [ "$rootfs_type" == "confidential" ]; then + if [[ "${rootfs_type}" == "confidential" ]]; then appendix="-${rootfs_type}" fi - if echo "$NVIDIA_GPU_STACK" | grep -q '\'; then + if echo "${NVIDIA_GPU_STACK}" | grep -q '\'; then appendix="-dragonball-experimental" fi # We need the kernel packages for building the drivers cleanly will be # deinstalled and removed from the roofs once the build finishes. - tar -xvf ${BUILD_DIR}/kata-static-kernel-nvidia-gpu"${appendix}"-headers.tar.xz -C . + tar -xvf "${BUILD_DIR}"/kata-static-kernel-nvidia-gpu"${appendix}"-headers.tar.xz -C . # If we find a local downloaded run file build the kernel modules # with it, otherwise use the distribution packages. Run files may have # more recent drivers available then the distribution packages. local run_file_name="nvidia-driver.run" - if [ -f ${BUILD_DIR}/${run_file_name} ]; then - cp -L ${BUILD_DIR}/${run_file_name} ./${run_file_name} + if [[ -f ${BUILD_DIR}/${run_file_name} ]]; then + cp -L "${BUILD_DIR}"/"${run_file_name}" ./"${run_file_name}" fi local run_fm_file_name="nvidia-fabricmanager.run" - if [ -f ${BUILD_DIR}/${run_fm_file_name} ]; then - cp -L ${BUILD_DIR}/${run_fm_file_name} ./${run_fm_file_name} + if [[ -f ${BUILD_DIR}/${run_fm_file_name} ]]; then + cp -L "${BUILD_DIR}"/"${run_fm_file_name}" ./"${run_fm_file_name}" fi mount --rbind /dev ./dev @@ -310,7 +314,7 @@ compress_rootfs() { } toggle_debug() { - if echo "$NVIDIA_GPU_STACK" | grep -q '\'; then + if echo "${NVIDIA_GPU_STACK}" | grep -q '\'; then export DEBUG="true" fi } @@ -320,13 +324,13 @@ setup_nvidia_gpu_rootfs_stage_two() { readonly stage_two="${ROOTFS_DIR:?}" readonly stack="${NVIDIA_GPU_STACK:?}" - echo "nvidia: chisseling the following stack components: $stack" + echo "nvidia: chisseling the following stack components: ${stack}" - [ -e "${stage_one}" ] && rm -rf "${stage_one}" - [ ! -e "${stage_one}" ] && mkdir -p "${stage_one}" + [[ -e "${stage_one}" ]] && rm -rf "${stage_one}" + [[ ! -e "${stage_one}" ]] && mkdir -p "${stage_one}" - tar -C "${stage_one}" -xf ${BUILD_DIR}/kata-static-rootfs-nvidia-gpu-stage-one.tar.zst + tar -C "${stage_one}" -xf "${BUILD_DIR}"/kata-static-rootfs-nvidia-gpu-stage-one.tar.zst pushd "${stage_two}" >> /dev/null @@ -335,19 +339,19 @@ setup_nvidia_gpu_rootfs_stage_two() { chisseled_init chisseled_iptables - IFS=',' read -r -a stack_components <<< "$NVIDIA_GPU_STACK" + IFS=',' read -r -a stack_components <<< "${NVIDIA_GPU_STACK}" for component in "${stack_components[@]}"; do - if [ "$component" = "compute" ]; then + if [[ "${component}" = "compute" ]]; then echo "nvidia: processing \"compute\" component" chisseled_compute - elif [ "$component" = "dcgm" ]; then + elif [[ "${component}" = "dcgm" ]]; then echo "nvidia: processing DCGM component" chisseled_dcgm - elif [ "$component" = "nvswitch" ]; then + elif [[ "${component}" = "nvswitch" ]]; then echo "nvidia: processing NVSwitch component" chisseled_nvswitch - elif [ "$component" = "gpudirect" ]; then + elif [[ "${component}" = "gpudirect" ]]; then echo "nvidia: processing GPUDirect component" chisseled_gpudirect fi From d971e13446ac47623fbe6f22bef32f13af6023fb Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Fri, 28 Feb 2025 18:21:06 +0000 Subject: [PATCH 13/13] gpu: Update rootfs.sh Only source NV scripts if variant starts with "nvidia-gpu" Signed-off-by: Zvonko Kaiser --- tools/osbuilder/rootfs-builder/rootfs.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/osbuilder/rootfs-builder/rootfs.sh b/tools/osbuilder/rootfs-builder/rootfs.sh index 21ae5b9510..7deca9eda7 100755 --- a/tools/osbuilder/rootfs-builder/rootfs.sh +++ b/tools/osbuilder/rootfs-builder/rootfs.sh @@ -55,8 +55,9 @@ EXTRA_PKGS=${EXTRA_PKGS:-""} KBUILD_SIGN_PIN=${KBUILD_SIGN_PIN:-""} NVIDIA_GPU_STACK=${NVIDIA_GPU_STACK:-""} -nvidia_rootfs="${script_dir}/nvidia/nvidia_rootfs.sh" -[ "${ARCH}" == "x86_64" ] || [ "${ARCH}" == "aarch64" ] && source "$nvidia_rootfs" +VARIANT=${VARIANT:-""} + +[[ "${VARIANT}" == "nvidia-gpu"* ]] && source "${script_dir}/nvidia/nvidia_rootfs.sh" #For cross build CROSS_BUILD=${CROSS_BUILD:-false}