github/kata-containers
1
0
Fork 2
mirror of https://github.com/kata-containers/kata-containers.git synced 2026-07-13 20:27:41 +00:00
Code Issues Projects Releases Wiki Activity

fixup! build: split NVIDIA rootfs into base-nvidia image + gpu extension

Browse Source
This commit is contained in:
Fabiano Fidêncio
2026-07-09 17:47:41 +02:00
parent 7c74244d69
commit 1cdfc8ad42
1 changed files with 26 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh
View File

@@ -449,24 +449,28 @@ readonly nvidia_gpu_extension_lib_globs=(
# Lay the GPU userspace out for the extension mount (/run/kata-extensions/gpu) and drop
# everything else. Matches how NVRC consumes the extension: bins from <root>/bin and
# <root>/sbin, libraries via LD_LIBRARY_PATH=<root>/usr/lib, kernel modules via
# `modprobe --dirname <root>`, configs from <root>/usr/share/nvidia, and
# <root>/lib/firmware/nvidia bind-mounted onto /lib/firmware/nvidia. Runs inside
# the (full, chiseled) ${stage_two}/${ROOTFS_DIR}.
# <root>/sbin, libraries from <root>/usr/lib/<triplet> (into the loader cache NVRC
# rebuilds), kernel modules via `modprobe --dirname <root>`, configs from
# <root>/usr/share/nvidia, and <root>/lib/firmware/nvidia bind-mounted onto
# /lib/firmware/nvidia. Runs inside the (full, chiseled) ${stage_two}/${ROOTFS_DIR}.
#
# The GPU shared libraries go under <root>/usr/lib (not a flat <root>/lib) so
# that NVRC can run `nvidia-ctk cdi generate --driver-root=<root>`: nvidia-ctk
# records the in-container mount path as the host path with the driver root
# stripped, so libraries at <root>/usr/lib land at the canonical /usr/lib inside
# the container. Apps that scan /usr for the driver (e.g. NVIDIA NIM, which
# bails with "libnvidia-ml.so.1 not found under /usr") then find it; a flat
# <root>/lib would strip to /lib and hide the driver from those checks.
# The GPU shared libraries go under <root>/usr/lib/<triplet> (the multiarch dir,
# mirroring the monolith), not a flat <root>/lib or <root>/usr/lib, so that NVRC
# can run `nvidia-ctk cdi generate --driver-root=<root>`: nvidia-ctk records the
# in-container mount path as the host path with the driver root stripped, so
# libraries at <root>/usr/lib/<triplet> land at the canonical /usr/lib/<triplet>
# inside the container - the exact layout the monolith produces, so its CDI hooks
# (create-symlinks/update-ldcache) reconcile. A flat <root>/usr/lib strips to
# /usr/lib, which those hooks can't reconcile and CUDA fails with "driver version
# is insufficient for CUDA runtime version".
partition_gpu_extension() {
echo "nvidia: building gpu extension layout"
local extlib="usr/lib/${machine_arch}-linux-gnu"
local extension
extension="$(mktemp -d "${BUILD_DIR}/.nvidia-gpu-extension.XXXX")"
mkdir -p "${extension}/bin" "${extension}/sbin" "${extension}/usr/lib" \
mkdir -p "${extension}/bin" "${extension}/sbin" "${extension}/${extlib}" \
"${extension}/usr/share/nvidia" "${extension}/lib/firmware" "${extension}/lib/modules"
local f
@@ -474,15 +478,16 @@ partition_gpu_extension() {
[[ -e "${f}" ]] && install -D -m0755 "${f}" "${extension}/${f}"
done
# Collect the GPU shared libraries under <root>/usr/lib so both NVRC's
# LD_LIBRARY_PATH and `nvidia-ctk --driver-root=<root>` resolve them and the
# container sees them at /usr/lib (see header); libc/loader stay in the base.
# Collect the GPU shared libraries under <root>/usr/lib/<triplet> so NVRC's
# loader-cache rebuild and `nvidia-ctk --driver-root=<root>` resolve them and
# the container sees them at /usr/lib/<triplet> (see header); libc/loader stay
# in the base.
local md="lib/${machine_arch}-linux-gnu"
local g
for g in "${nvidia_gpu_extension_lib_globs[@]}"; do
find "${md}" -maxdepth 1 -name "${g}" -exec cp -a {} "${extension}/usr/lib/" \;
find "${md}" -maxdepth 1 -name "${g}" -exec cp -a {} "${extension}/${extlib}/" \;
done
[[ -e lib/libgrpc_mgr.so ]] && cp -a lib/libgrpc_mgr.so "${extension}/usr/lib/"
[[ -e lib/libgrpc_mgr.so ]] && cp -a lib/libgrpc_mgr.so "${extension}/${extlib}/"
# Materialize the SONAME symlinks (e.g. libcuda.so.1 -> libcuda.so.595.58.03)
# inside the lib dir. The extension ships only the versioned files, so without
@@ -493,7 +498,7 @@ partition_gpu_extension() {
# for CUDA runtime version". `ldconfig -n` only creates the versioned symlinks
# in the given dir (no cache, no chroot), which is all the loader-less extension
# needs. The monolith gets the same links via the `chroot . ldconfig` below.
ldconfig -n "${extension}/usr/lib"
ldconfig -n "${extension}/${extlib}"
# GPU configs (fabricmanager.cfg, nvlsm.conf, ...).
[[ -d usr/share/nvidia ]] && cp -a usr/share/nvidia/. "${extension}/usr/share/nvidia/"
@@ -734,9 +739,9 @@ setup_nvidia_gpu_rootfs_stage_two() {
# content (no lib/<arch>-linux-gnu), so skip it there.
[[ "${layout}" != "gpu-extension" ]] && copy_mkfs_ext4_runtime_deps
compress_rootfs
# The gpu extension has no loader/ldconfig of its own; its libraries are found
# via NVRC's LD_LIBRARY_PATH, so skip the ld.so cache rebuild there. Its
# SONAME symlinks are created with `ldconfig -n` in partition_gpu_extension().
# The gpu extension has no loader/ldconfig of its own; NVRC rebuilds the ld.so
# cache from its libraries at boot, so skip the cache rebuild here. Its SONAME
# symlinks are created with `ldconfig -n` in partition_gpu_extension().
[[ "${layout}" != "gpu-extension" ]] && chroot . ldconfig
popd >> /dev/null