From f4b0e1a88cbfe61b61b7b2c3a450bbf19550bc77 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Tue, 21 Jul 2026 22:26:51 +0000 Subject: [PATCH] nvidia-gpu: chisel NVIDIA libs from sandboxutils-filelist.json Replace the broad libnv*/libcuda.so.* globs in chisseled_compute() with nvidia_driver_capabilities(), a jq-based helper that queries the driver-authoritative sandboxutils-filelist.json shipped at usr/share/nvidia/files.d/ in every NVIDIA driver install. The function accepts a file type (LIB, BINARY, ICD, FIRMWARE, ...) and a list of capability categories, so the same filelist drives selection of libs, binaries, and ICD registration files consistently. Categories selected for LIB: cuda, opencl, nvml, nvpd, video, nvsandboxutils. Graphics categories (egl*, glx, vulkan, ngx, optix, gbm, utils, xdriver) are excluded, kata runs headless workloads only. libnvidia-allocator is trusted to be graphics-only as the filelist declares: no compute library (libcuda, libnvidia-ml, libnvidia-gpucomp, libnvidia-opencl) references it, only libnvidia-eglcore and libnvidia-glcore do, and its gbm backend role is display buffer allocation. No compute-side override is carried. Also fixes two gaps the old explicit copy list had missed: nvidia-cuda-mps-control/server missing despite being cuda category nvidia.icd missing, needed for OpenCL discovery Saves ~208 MB in usr/lib before strip/UPX: libnvoptix.so 46.8 MB OptiX ray tracing libnvidia-rtcore.so 40.1 MB ray tracing libnvidia-glcore.so 38.0 MB OpenGL libnvidia-eglcore.so 36.1 MB EGL libnvperf_dcgm_host.so 17.9 MB Nsight profiling SDK libnvidia-vksc-core.so 10.6 MB Vulkan SC libnvidia-glvkspirv.so 9.9 MB Vulkan SPIR-V libnvidia-pkcs11* is copied unconditionally: the gpu extension is built without a type tag, so any confidential-only gate strips the TEE attestation libs from extension images. The old explicit conditional also used the wrong libdir (lib/ instead of usr/lib/) and was dead code; it is removed. The libs cost a few hundred KB and the monolith split will make the distinction moot anyway. The capability arrays are validated to be non-empty after extraction: failures inside the mapfile process substitutions do not propagate under set -e, and an empty selection must fail the build instead of producing a rootfs with no NVIDIA userspace. Assisted-by: Claude Sonnet 4.6 (1M context) Signed-off-by: Zvonko Kaiser --- .../rootfs-builder/nvidia/nvidia_chroot.sh | 3 +- .../rootfs-builder/nvidia/nvidia_rootfs.sh | 121 +++++++++++++++--- .../rootfs-builder/ubuntu/Dockerfile.in | 1 + .../local-build/dockerbuild/Dockerfile | 1 + 4 files changed, 110 insertions(+), 16 deletions(-) diff --git a/tools/osbuilder/rootfs-builder/nvidia/nvidia_chroot.sh b/tools/osbuilder/rootfs-builder/nvidia/nvidia_chroot.sh index 8b29e0d868..3b17408fe9 100644 --- a/tools/osbuilder/rootfs-builder/nvidia/nvidia_chroot.sh +++ b/tools/osbuilder/rootfs-builder/nvidia/nvidia_chroot.sh @@ -61,7 +61,8 @@ install_userspace_components() { eval "${APT_INSTALL}" nvidia-imex nvidia-firmware \ libnvidia-cfg1 libnvidia-gl libnvidia-extra \ libnvidia-decode libnvidia-fbc1 libnvidia-encode \ - libnvidia-nscq libnvidia-compute nvidia-settings + libnvidia-nscq libnvidia-compute nvidia-settings \ + ocl-icd-libopencl1 apt-mark hold nvidia-imex nvidia-firmware \ libnvidia-cfg1 libnvidia-gl libnvidia-extra \ diff --git a/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh b/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh index 3cddf913af..c98111acb5 100644 --- a/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh +++ b/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh @@ -50,6 +50,47 @@ nvidia_stage_one_variant() { stage_one="${BUILD_DIR:?}/rootfs-$(nvidia_stage_one_variant)-stage-one" readonly stage_one +# The driver ships its own capability-to-file map so we do not have to maintain +# one ourselves or guess from filename prefixes. +readonly filelist="${stage_one}/usr/share/nvidia/files.d/sandboxutils-filelist.json" + +# sandboxutils-filelist.json schema (one object per file): +# { "name": "libcuda.so.570.00", +# "type": "LIB", # LIB | BINARY | FIRMWARE | ICD | SYMLINK | ... +# "category": ["cuda", "vulkan", ...], # capability tags; a file can belong to many +# "is_32bit_compat": "true" # present only on 32-bit compat entries +# } +# +# If this schema changes (key renamed, type values altered) jq will return +# empty output and nvidia_driver_capabilities_validate() will catch it early. +nvidia_driver_capabilities_validate() { + command -v jq > /dev/null || die "nvidia: jq is required but not installed" + [[ -f "${filelist}" ]] || die "sandboxutils filelist not found: ${filelist}" + # Require at least one entry with the mandatory keys (name, type, category). + local count + count=$(jq '[.[] | select(.name and .type and .category)] | length' "${filelist}") \ + || die "nvidia: sandboxutils-filelist.json is not valid JSON" + [[ "${count}" -gt 0 ]] \ + || die "nvidia: sandboxutils-filelist.json missing expected keys (name/type/category) — schema may have changed" +} + +# nvidia_driver_capabilities TYPE CATEGORY [CATEGORY ...] +# +# The driver knows better than we do which files belong to which capability. +# 32-bit compat entries are skipped: kata is 64-bit only. +nvidia_driver_capabilities() { + local filetype="${1:?type required}"; shift + command -v jq > /dev/null || die "jq is required for nvidia_driver_capabilities" + [[ -f "${filelist}" ]] || die "sandboxutils filelist not found: ${filelist}" + + jq -r --arg ftype "${filetype}" \ + '[.[] | select((.is_32bit_compat // false | tostring) != "true") + | select(.type == $ftype) + | select([.category[] | IN($ARGS.positional[])] | any) + | .name] | unique[]' \ + "${filelist}" --args "$@" +} + # Image layout produced from the chiseled tree: # monolith - the full GPU image (default; unchanged behaviour) # base - driver-agnostic nvidia base (NVRC init + agent + base libs) @@ -222,25 +263,57 @@ chisseled_compute() { cp -a "${stage_one}/${libdir}"/libkeyutils.so.1* "${libdir}"/. cp -a "${stage_one}/etc/netconfig" etc/. - [[ "${type}" == "confidential" ]] && cp -a "${stage_one}/${libdir}"/libnvidia-pkcs11* "${libdir}"/. - [[ ${machine_arch} == "aarch64" ]] && libdir="lib" [[ ${machine_arch} == "x86_64" ]] && libdir="lib64" cp -aL "${stage_one}/${libdir}"/ld-linux-* "${libdir}"/. + # Kata runs headless compute workloads; the graphics stack (EGL, OpenGL, + # Vulkan, NGX, OptiX) has no consumers in the guest and only inflates the + # image. video is included for headless ffmpeg (nvenc/nvdec go through + # /dev/nvidia0, not DRI, so no display stack is needed). + # nvapi is excluded: libnvidia-api.so is a Linux shim for the Windows + # NVAPI SDK and has no CUDA compute dependency. libdir=usr/lib/"${machine_arch}"-linux-gnu - cp -a "${stage_one}/${libdir}"/libnv* lib/"${machine_arch}"-linux-gnu/. - cp -a "${stage_one}/${libdir}"/libcuda.so.* lib/"${machine_arch}"-linux-gnu/. + destdir=lib/"${machine_arch}"-linux-gnu + + for lib in "${_nvidia_libs[@]}"; do + # vdpau needs a display server (X11/Wayland); cannot function headless. + # cudadebugger is cuda-gdb support; no production use in a VM. + # opticalflow is a specialised CV primitive, not general compute. + case "${lib}" in + libvdpau_nvidia.*|libcudadebugger.*|libnvidia-opticalflow.*) continue ;; + esac + # The filelist is a superset; skip files absent on this OS/SSL configuration. + [[ -e "${stage_one}/${libdir}/${lib}" ]] && cp -a "${stage_one}/${libdir}/${lib}" "${destdir}/." + done + + for binary in "${_nvidia_bins[@]}"; do + # MPS assumes a persistent host daemon model that conflicts with + # NVRC's per-container lifecycle. debugdump is a host-side tool. + case "${binary}" in + nvidia-cuda-mps-control|nvidia-cuda-mps-server|nvidia-debugdump) continue ;; + esac + [[ -e "${stage_one}/usr/bin/${binary}" ]] && cp -a "${stage_one}/usr/bin/${binary}" bin/. + done + + # OpenCL ICD registration so the OpenCL loader finds the NVIDIA driver. + mkdir -p etc/OpenCL/vendors + for icd in "${_nvidia_icds[@]}"; do + [[ -e "${stage_one}/etc/OpenCL/vendors/${icd}" ]] && cp -a "${stage_one}/etc/OpenCL/vendors/${icd}" etc/OpenCL/vendors/. + done - # basic GPU admin tools - cp -a "${stage_one}"/usr/bin/nvidia-persistenced bin/. - cp -a "${stage_one}"/usr/bin/nvidia-smi bin/. - cp -a "${stage_one}"/usr/bin/nvidia-ctk bin/. - cp -a "${stage_one}"/usr/bin/nvidia-cdi-hook bin/. ln -s ../bin usr/bin } +chisseled_ctk() { + echo "nvidia: chisseling container-toolkit" + # nvidia-ctk and nvidia-cdi-hook come from the container-toolkit package, + # not the driver, so they are not in the sandboxutils filelist. + cp -a "${stage_one}"/usr/bin/nvidia-ctk bin/. + cp -a "${stage_one}"/usr/bin/nvidia-cdi-hook bin/. +} + chisseled_gpudirect() { echo "nvidia: chisseling GPUDirect" echo "nvidia: not implemented yet" @@ -447,12 +520,8 @@ readonly nvidia_gpu_extension_bins=( # GPU shared-library globs (inside the multiarch lib dir, plus libgrpc_mgr in # /lib) owned by the gpu extension. # -# 'libnv*' is deliberately broad: the NVIDIA userspace ships many libraries that -# start with libnv but not libnvidia- (libnvcuvid, libnvrtc, libnvidia-*, -# libnvoptix, libnvToolsExt, ...), and we want all of them in the extension. This -# is safe because the source tree is a chiseled rootfs that only contains what the -# NVIDIA driver/CUDA install puts there - there is no unrelated libnv* (e.g. an -# NVMe libnvme) to accidentally sweep in. +# The chiseled rootfs is built from the filelist so only compute/video libs +# are present; the globs below cannot accidentally match graphics libs. readonly nvidia_gpu_extension_lib_globs=( 'libnv*' 'libcuda.so*' @@ -513,6 +582,13 @@ partition_gpu_extension() { # needs. The monolith gets the same links via the `chroot . ldconfig` below. ldconfig -n "${extension}/${extlib}" + # Generate a full ldcache so nvidia-ctk can locate libraries efficiently + # when running with --driver-root=. Without it nvidia-ctk falls + # back to directory scanning, which serialises CDI generation and slows + # container start significantly. + mkdir -p "${extension}/etc" + ldconfig -r "${extension}" 2>/dev/null || true + # GPU configs (fabricmanager.cfg, nvlsm.conf, ...). [[ -d usr/share/nvidia ]] && cp -a usr/share/nvidia/. "${extension}/usr/share/nvidia/" @@ -702,6 +778,20 @@ setup_nvidia_gpu_rootfs_stage_two() { tar -C "${stage_one}" -xf "${stage_one}".tar.zst + # Validate and populate capability arrays after stage-one is extracted + # so the filelist is actually present on disk. + nvidia_driver_capabilities_validate + mapfile -t _nvidia_libs < <(nvidia_driver_capabilities LIB cuda opencl nvml nvpd video nvsandboxutils) + mapfile -t _nvidia_bins < <(nvidia_driver_capabilities BINARY cuda nvml nvpd) + mapfile -t _nvidia_icds < <(nvidia_driver_capabilities ICD opencl) + + # Failures inside the process substitutions above do not propagate + # under set -e; empty arrays are the observable symptom, so fail fast + # on them instead of assembling a broken rootfs. + (( ${#_nvidia_libs[@]} > 0 )) || die "nvidia: no LIB entries selected from ${filelist}" + (( ${#_nvidia_bins[@]} > 0 )) || die "nvidia: no BINARY entries selected from ${filelist}" + (( ${#_nvidia_icds[@]} > 0 )) || die "nvidia: no ICD entries selected from ${filelist}" + pushd "${stage_two}" >> /dev/null # stage-one archives the full base+driver tree with `tar @@ -735,6 +825,7 @@ setup_nvidia_gpu_rootfs_stage_two() { fi done + chisseled_ctk coco_guest_components chisseled_nvat diff --git a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in index 80e82e983a..b9e8895349 100644 --- a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in +++ b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in @@ -32,6 +32,7 @@ RUN apt-get update && \ g++ \ git \ gnupg2 \ + jq \ libclang-dev \ make \ makedev \ diff --git a/tools/packaging/kata-deploy/local-build/dockerbuild/Dockerfile b/tools/packaging/kata-deploy/local-build/dockerbuild/Dockerfile index 68dbcae65f..d7b172e9b7 100644 --- a/tools/packaging/kata-deploy/local-build/dockerbuild/Dockerfile +++ b/tools/packaging/kata-deploy/local-build/dockerbuild/Dockerfile @@ -45,6 +45,7 @@ RUN apt-get update && \ build-essential \ cpio \ gcc \ + jq \ unzip \ git \ make \