From a80d9152d29196390f8f048e1e8916df16ba33b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 11 Jun 2026 11:29:09 +0200 Subject: [PATCH] fixup! build: split NVIDIA rootfs into base-nvidia image + gpu addon --- .../rootfs-builder/nvidia/nvidia_rootfs.sh | 41 ++++++++++++++----- .../local-build/kata-deploy-binaries.sh | 3 ++ .../fragments/gpu/nvidia.arm64.conf.in | 8 ---- .../fragments/gpu/nvidia.x86_64.conf.in | 8 ---- tools/packaging/kernel/kata_config_version | 2 +- 5 files changed, 34 insertions(+), 28 deletions(-) diff --git a/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh b/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh index 5c6c4a41e7..826889efcc 100644 --- a/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh +++ b/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh @@ -69,7 +69,11 @@ setup_nvidia-nvrc() { local nvrc_tarball="${BUILD_DIR}/kata-static-nvrc.tar.zst" [[ -e "${nvrc_tarball}" ]] || \ die "NVRC tarball not found: ${nvrc_tarball} (build the 'nvrc' target first)" - tar --zstd -xvf "${nvrc_tarball}" -C . + # The tarball carries a ./bin/ entry; on a usr-merged rootfs /bin is a + # symlink to usr/bin, so extract with --keep-directory-symlink to follow it + # instead of clobbering the symlink with a real (near-empty) directory, + # which would hide /bin/bash and break the driver chroot below. + tar --keep-directory-symlink --zstd -xvf "${nvrc_tarball}" -C . } setup_nvidia_gpu_rootfs_stage_one() { @@ -333,7 +337,11 @@ compress_rootfs() { [[ ${machine_arch} == "aarch64" ]] && libdir="lib" [[ ${machine_arch} == "x86_64" ]] && libdir="lib64" - chmod +x "${libdir}"/ld-linux-* + # The gpu-addon layout ships no program interpreter (it lives in the + # base-nvidia image), so only fix up the loader when it is actually present. + if compgen -G "${libdir}/ld-linux-*" > /dev/null; then + chmod +x "${libdir}"/ld-linux-* + fi } copy_cdh_runtime_deps() { @@ -476,9 +484,10 @@ partition_gpu_addon() { } # Strip the GPU userspace from the chiseled tree, leaving a driver-agnostic -# base-nvidia: NVRC init + kata-agent + busybox + loader/libc + in-tree kernel -# modules. The empty /lib/firmware/nvidia directory is kept as the bind -# mountpoint NVRC uses for the addon firmware. Runs inside ${ROOTFS_DIR}. +# base-nvidia: NVRC init + kata-agent + busybox + loader/libc. No kernel +# modules are shipped (see below). The empty /lib/firmware/nvidia directory is +# kept as the bind mountpoint NVRC uses for the addon firmware. Runs inside +# ${ROOTFS_DIR}. partition_base() { echo "nvidia: building driver-agnostic base layout" @@ -502,14 +511,14 @@ partition_base() { rm -rf lib/firmware/nvidia mkdir -p lib/firmware/nvidia - # Reset kernel modules to the in-tree set (no NVIDIA driver modules) so the - # base stays driver-agnostic; the gpu addon ships the driver modules. + # Ship no kernel modules in the base: the NVIDIA driver modules are + # GPU-specific and live in the gpu addon (NVRC loads them via + # `modprobe --dirname `), and the remaining in-tree dependencies + # (mlx5, infiniband, ...) are built into the NVIDIA kernel. Keeping + # /lib/modules empty is what makes the base driver-agnostic and reusable + # across driver versions. rm -rf lib/modules mkdir -p lib/modules - local appendix="" - echo "${NVIDIA_GPU_STACK}" | grep -q '\' && appendix="-dragonball-experimental" - tar --zstd -xf "${BUILD_DIR}"/kata-static-kernel-nvidia-gpu"${appendix}"-modules.tar.zst \ - -C ./lib/modules/ } setup_nvidia_gpu_rootfs_stage_two() { @@ -539,6 +548,16 @@ setup_nvidia_gpu_rootfs_stage_two() { pushd "${stage_two}" >> /dev/null + # stage-one archives the full base+driver tree with `tar + # --remove-files`, emptying ${stage_two} on its first run. When + # stage-one is served from cache that emptying never happens, so the + # freshly-built distro rootfs is still here. The chisel assembles a + # minimal tree purely from ${stage_one}, so wipe any leftover distro + # content first to keep the result deterministic regardless of whether + # stage-one was cached (otherwise base/monolith layouts bloat with the + # full Ubuntu rootfs). + find . -mindepth 1 -maxdepth 1 -exec rm -rf {} + + chisseled_init chisseled_iptables diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index aaca6d5295..eba5561db4 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -918,6 +918,9 @@ install_image_nvidia_gpu_addon() { export MEASURED_ROOTFS="yes" export FS_TYPE="erofs" export SKIP_DAX_HEADER="yes" + # The gpu addon is GPU-userspace-only content mounted into base-nvidia; it + # ships no /sbin/init, so skip the rootfs init/agent sanity check. + export SKIP_ROOTFS_CHECK="yes" local version version=$(get_latest_nvidia_driver_version) EXTRA_PKGS="apt curl ${EXTRA_PKGS}" diff --git a/tools/packaging/kernel/configs/fragments/gpu/nvidia.arm64.conf.in b/tools/packaging/kernel/configs/fragments/gpu/nvidia.arm64.conf.in index acf1768239..498c1b9d42 100644 --- a/tools/packaging/kernel/configs/fragments/gpu/nvidia.arm64.conf.in +++ b/tools/packaging/kernel/configs/fragments/gpu/nvidia.arm64.conf.in @@ -27,11 +27,3 @@ CONFIG_ARM_SMMU_V3_SVA=y CONFIG_CRYPTO_ECC=y CONFIG_CRYPTO_ECDH=y CONFIG_CRYPTO_ECDSA=y - -# HGX/DGX platform -CONFIG_INFINIBAND=m -CONFIG_INFINIBAND_USER_MAD=m - -CONFIG_NET_VENDOR_MELLANOX=y -CONFIG_MLX5_CORE=m -CONFIG_MLX5_INFINIBAND=m diff --git a/tools/packaging/kernel/configs/fragments/gpu/nvidia.x86_64.conf.in b/tools/packaging/kernel/configs/fragments/gpu/nvidia.x86_64.conf.in index 3d92bf8796..4319f2caff 100644 --- a/tools/packaging/kernel/configs/fragments/gpu/nvidia.x86_64.conf.in +++ b/tools/packaging/kernel/configs/fragments/gpu/nvidia.x86_64.conf.in @@ -26,11 +26,3 @@ CONFIG_CRYPTO_ECDSA=y # Dependency of _CRYPTO_ CONFIG_MODULE_SIG=y - -# HGX/DGX platform -CONFIG_INFINIBAND=m -CONFIG_INFINIBAND_USER_MAD=m - -CONFIG_NET_VENDOR_MELLANOX=y -CONFIG_MLX5_CORE=m -CONFIG_MLX5_INFINIBAND=m diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version index 538165229c..ca55a6c593 100644 --- a/tools/packaging/kernel/kata_config_version +++ b/tools/packaging/kernel/kata_config_version @@ -1 +1 @@ -197 +198