From 1cdfc8ad42531b73693e1a160e6050815e6d4808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 9 Jul 2026 17:47:41 +0200 Subject: [PATCH] fixup! build: split NVIDIA rootfs into base-nvidia image + gpu extension --- .../rootfs-builder/nvidia/nvidia_rootfs.sh | 47 ++++++++++--------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh b/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh index 17ceeadb3e..c375a273bb 100644 --- a/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh +++ b/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh @@ -449,24 +449,28 @@ readonly nvidia_gpu_extension_lib_globs=( # Lay the GPU userspace out for the extension mount (/run/kata-extensions/gpu) and drop # everything else. Matches how NVRC consumes the extension: bins from /bin and -# /sbin, libraries via LD_LIBRARY_PATH=/usr/lib, kernel modules via -# `modprobe --dirname `, configs from /usr/share/nvidia, and -# /lib/firmware/nvidia bind-mounted onto /lib/firmware/nvidia. Runs inside -# the (full, chiseled) ${stage_two}/${ROOTFS_DIR}. +# /sbin, libraries from /usr/lib/ (into the loader cache NVRC +# rebuilds), kernel modules via `modprobe --dirname `, configs from +# /usr/share/nvidia, and /lib/firmware/nvidia bind-mounted onto +# /lib/firmware/nvidia. Runs inside the (full, chiseled) ${stage_two}/${ROOTFS_DIR}. # -# The GPU shared libraries go under /usr/lib (not a flat /lib) so -# that NVRC can run `nvidia-ctk cdi generate --driver-root=`: nvidia-ctk -# records the in-container mount path as the host path with the driver root -# stripped, so libraries at /usr/lib land at the canonical /usr/lib inside -# the container. Apps that scan /usr for the driver (e.g. NVIDIA NIM, which -# bails with "libnvidia-ml.so.1 not found under /usr") then find it; a flat -# /lib would strip to /lib and hide the driver from those checks. +# The GPU shared libraries go under /usr/lib/ (the multiarch dir, +# mirroring the monolith), not a flat /lib or /usr/lib, so that NVRC +# can run `nvidia-ctk cdi generate --driver-root=`: nvidia-ctk records the +# in-container mount path as the host path with the driver root stripped, so +# libraries at /usr/lib/ land at the canonical /usr/lib/ +# inside the container - the exact layout the monolith produces, so its CDI hooks +# (create-symlinks/update-ldcache) reconcile. A flat /usr/lib strips to +# /usr/lib, which those hooks can't reconcile and CUDA fails with "driver version +# is insufficient for CUDA runtime version". partition_gpu_extension() { echo "nvidia: building gpu extension layout" + local extlib="usr/lib/${machine_arch}-linux-gnu" + local extension extension="$(mktemp -d "${BUILD_DIR}/.nvidia-gpu-extension.XXXX")" - mkdir -p "${extension}/bin" "${extension}/sbin" "${extension}/usr/lib" \ + mkdir -p "${extension}/bin" "${extension}/sbin" "${extension}/${extlib}" \ "${extension}/usr/share/nvidia" "${extension}/lib/firmware" "${extension}/lib/modules" local f @@ -474,15 +478,16 @@ partition_gpu_extension() { [[ -e "${f}" ]] && install -D -m0755 "${f}" "${extension}/${f}" done - # Collect the GPU shared libraries under /usr/lib so both NVRC's - # LD_LIBRARY_PATH and `nvidia-ctk --driver-root=` resolve them and the - # container sees them at /usr/lib (see header); libc/loader stay in the base. + # Collect the GPU shared libraries under /usr/lib/ so NVRC's + # loader-cache rebuild and `nvidia-ctk --driver-root=` resolve them and + # the container sees them at /usr/lib/ (see header); libc/loader stay + # in the base. local md="lib/${machine_arch}-linux-gnu" local g for g in "${nvidia_gpu_extension_lib_globs[@]}"; do - find "${md}" -maxdepth 1 -name "${g}" -exec cp -a {} "${extension}/usr/lib/" \; + find "${md}" -maxdepth 1 -name "${g}" -exec cp -a {} "${extension}/${extlib}/" \; done - [[ -e lib/libgrpc_mgr.so ]] && cp -a lib/libgrpc_mgr.so "${extension}/usr/lib/" + [[ -e lib/libgrpc_mgr.so ]] && cp -a lib/libgrpc_mgr.so "${extension}/${extlib}/" # Materialize the SONAME symlinks (e.g. libcuda.so.1 -> libcuda.so.595.58.03) # inside the lib dir. The extension ships only the versioned files, so without @@ -493,7 +498,7 @@ partition_gpu_extension() { # for CUDA runtime version". `ldconfig -n` only creates the versioned symlinks # in the given dir (no cache, no chroot), which is all the loader-less extension # needs. The monolith gets the same links via the `chroot . ldconfig` below. - ldconfig -n "${extension}/usr/lib" + ldconfig -n "${extension}/${extlib}" # GPU configs (fabricmanager.cfg, nvlsm.conf, ...). [[ -d usr/share/nvidia ]] && cp -a usr/share/nvidia/. "${extension}/usr/share/nvidia/" @@ -734,9 +739,9 @@ setup_nvidia_gpu_rootfs_stage_two() { # content (no lib/-linux-gnu), so skip it there. [[ "${layout}" != "gpu-extension" ]] && copy_mkfs_ext4_runtime_deps compress_rootfs - # The gpu extension has no loader/ldconfig of its own; its libraries are found - # via NVRC's LD_LIBRARY_PATH, so skip the ld.so cache rebuild there. Its - # SONAME symlinks are created with `ldconfig -n` in partition_gpu_extension(). + # The gpu extension has no loader/ldconfig of its own; NVRC rebuilds the ld.so + # cache from its libraries at boot, so skip the cache rebuild here. Its SONAME + # symlinks are created with `ldconfig -n` in partition_gpu_extension(). [[ "${layout}" != "gpu-extension" ]] && chroot . ldconfig popd >> /dev/null