Files
kata-containers/tools/osbuilder/rootfs-builder/nvidia/nvidia_rootfs.sh
Manuel Huber ed4233bf91 rootfs: cdh: Update CDH to new version
Update CDH to a newer version and:
- adjust the NVIDIA root filesystem build to reflect the change from
  using libcryptsetup to using the cryptsetup binary.
- adjust image-pull test cases to conduct parallel write operations
  on the /dev/trusted_store backed guest image pull location since
  issue #12721 has been solved on CDH side.

Fixes #12721

Signed-off-by: Manuel Huber <manuelh@nvidia.com>
2026-05-13 20:20:45 +02:00

452 lines
15 KiB
Bash

#!/usr/bin/env bash
#
# Copyright (c) 2024 NVIDIA Corporation
#
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
[[ -n "${DEBUG}" ]] && set -x
# Error helpers
trap 'echo "rootfs: ERROR at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR
die() {
local msg="${*:-fatal error}"
echo "rootfs: ${msg}" >&2
exit 1
}
readonly BUILD_DIR="/kata-containers/tools/packaging/kata-deploy/local-build/build/"
# catch errors and then assign
script_dir="$(dirname "$(readlink -f "$0")")"
readonly SCRIPT_DIR="${script_dir}/nvidia"
KBUILD_SIGN_PIN=${KBUILD_SIGN_PIN:-}
AGENT_POLICY="${AGENT_POLICY:-no}"
NVIDIA_GPU_STACK=${NVIDIA_GPU_STACK:?NVIDIA_GPU_STACK must be set}
BUILD_VARIANT=${BUILD_VARIANT:?BUILD_VARIANT must be set}
ARCH=${ARCH:?ARCH must be set}
machine_arch="${ARCH}"
if [[ "${machine_arch}" == "aarch64" ]]; then
distro_arch="arm64"
elif [[ "${machine_arch}" == "x86_64" ]]; then
distro_arch="amd64"
else
die "Unsupported architecture: ${machine_arch}"
fi
readonly stage_one="${BUILD_DIR:?}/rootfs-${BUILD_VARIANT:?}-stage-one"
setup_nvidia-nvrc() {
local url ver
local nvrc=NVRC-${machine_arch}-unknown-linux-musl
url=$(get_package_version_from_kata_yaml "externals.nvrc.url")
ver=$(get_package_version_from_kata_yaml "externals.nvrc.version")
local dl="${url}/${ver}"
curl -fsSL -o "${BUILD_DIR}/${nvrc}.tar.xz" "${dl}/${nvrc}.tar.xz"
curl -fsSL -o "${BUILD_DIR}/${nvrc}.tar.xz.sig" "${dl}/${nvrc}.tar.xz.sig"
curl -fsSL -o "${BUILD_DIR}/${nvrc}.tar.xz.cert" "${dl}/${nvrc}.tar.xz.cert"
local id="^https://github.com/NVIDIA/nvrc/.github/workflows/.+@refs/heads/main$"
local oidc="https://token.actions.githubusercontent.com"
# Only allow releases from the NVIDIA/nvrc main branch and build by github actions
cosign verify-blob \
--rekor-url https://rekor.sigstore.dev \
--certificate "${BUILD_DIR}/${nvrc}.tar.xz.cert" \
--signature "${BUILD_DIR}/${nvrc}.tar.xz.sig" \
--certificate-identity-regexp "${id}" \
--certificate-oidc-issuer "${oidc}" \
"${BUILD_DIR}/${nvrc}.tar.xz"
}
setup_nvidia_gpu_rootfs_stage_one() {
local rootfs_type=${1:-""}
if [[ -e "${stage_one}.tar.zst" ]]; then
info "nvidia: GPU rootfs stage one already exists"
return
fi
pushd "${ROOTFS_DIR:?}" >> /dev/null
info "nvidia: Setup GPU rootfs type=${rootfs_type}"
cp "${SCRIPT_DIR}/nvidia_chroot.sh" ./nvidia_chroot.sh
chmod +x ./nvidia_chroot.sh
local nvrc=NVRC-${machine_arch}-unknown-linux-musl
if [[ ! -e "${BUILD_DIR}/${nvrc}.tar.xz" ]]; then
setup_nvidia-nvrc
fi
tar -xvf "${BUILD_DIR}/${nvrc}.tar.xz" -C ./bin/
local appendix=""
if echo "${NVIDIA_GPU_STACK}" | grep -q '\<dragonball\>'; then
appendix="-dragonball-experimental"
fi
# Install the precompiled kernel modules shipped with the kernel
mkdir -p ./lib/modules/
tar --zstd -xvf "${BUILD_DIR}"/kata-static-kernel-nvidia-gpu"${appendix}"-modules.tar.zst -C ./lib/modules/
mount --rbind /dev ./dev
mount --make-rslave ./dev
mount -t proc /proc ./proc
local cuda_repo_url cuda_repo_pkg gpu_base_os_version ctk_version
cuda_repo_url=$(get_package_version_from_kata_yaml "externals.nvidia.cuda.repo.${machine_arch}.url")
cuda_repo_pkg=$(get_package_version_from_kata_yaml "externals.nvidia.cuda.repo.${machine_arch}.pkg")
gpu_base_os_version=$(get_package_version_from_kata_yaml "assets.image.architecture.x86_64.nvidia-gpu.version")
tools_repo_url=$(get_package_version_from_kata_yaml "externals.nvidia.tools.repo.${machine_arch}.url")
tools_repo_pkg=$(get_package_version_from_kata_yaml "externals.nvidia.tools.repo.${machine_arch}.pkg")
ctk_version=$(get_package_version_from_kata_yaml "externals.nvidia.ctk.version")
chroot . /bin/bash -c "/nvidia_chroot.sh ${machine_arch} ${NVIDIA_GPU_STACK} \
${gpu_base_os_version} ${cuda_repo_url} ${cuda_repo_pkg} ${tools_repo_url} ${tools_repo_pkg} ${ctk_version}"
umount -R ./dev
umount ./proc
rm ./nvidia_chroot.sh
tar cfa "${stage_one}.tar.zst" --remove-files -- *
popd >> /dev/null
pushd "${BUILD_DIR}" >> /dev/null
curl -LO "https://github.com/upx/upx/releases/download/v4.2.4/upx-4.2.4-${distro_arch}_linux.tar.xz"
tar xvf "upx-4.2.4-${distro_arch}_linux.tar.xz"
popd >> /dev/null
}
chisseled_iptables() {
echo "nvidia: chisseling iptables"
cp -a "${stage_one}"/usr/sbin/xtables-nft-multi sbin/.
ln -s ../sbin/xtables-nft-multi sbin/iptables-restore
ln -s ../sbin/xtables-nft-multi sbin/iptables-save
libdir=lib/"${machine_arch}"-linux-gnu
cp -a "${stage_one}/${libdir}"/libmnl.so.0* lib/.
libdir=usr/lib/"${machine_arch}"-linux-gnu
cp -a "${stage_one}/${libdir}"/libnftnl.so.11* lib/.
cp -a "${stage_one}/${libdir}"/libxtables.so.12* lib/.
}
# <= NVLINK4 nv-fabrimanager
# >= NVLINK5 nv-fabricmanager + nvlsm (TODO)
chisseled_nvswitch() {
echo "nvidia: chisseling NVSwitch"
mkdir -p usr/share/nvidia/nvswitch
cp -a "${stage_one}"/usr/bin/nv-fabricmanager bin/.
cp -a "${stage_one}"/usr/share/nvidia/nvswitch usr/share/nvidia/.
libdir=usr/lib/"${machine_arch}"-linux-gnu
cp -a "${stage_one}/${libdir}"/libnvidia-nscq.so.* lib/"${machine_arch}"-linux-gnu/.
# NVLINK SubnetManager dependencies
local nvlsm=usr/share/nvidia/nvlsm
mkdir -p "${nvlsm}"
cp -a "${stage_one}"/opt/nvidia/nvlsm/lib/libgrpc_mgr.so lib/.
cp -a "${stage_one}"/opt/nvidia/nvlsm/sbin/nvlsm sbin/.
cp -a "${stage_one}/${nvlsm}"/*.conf "${nvlsm}"/.
# Redirect all the logs to syslog instead of logging to file
sed -i 's|^LOG_USE_SYSLOG=.*|LOG_USE_SYSLOG=1|' usr/share/nvidia/nvswitch/fabricmanager.cfg
}
chisseled_dcgm() {
echo "nvidia: chisseling DCGM"
mkdir -p etc/dcgm-exporter
libdir=lib/"${machine_arch}"-linux-gnu
cp -a "${stage_one}"/usr/"${libdir}"/libdcgm.* "${libdir}"/.
cp -a "${stage_one}"/"${libdir}"/libgcc_s.so.1* "${libdir}"/.
cp -a "${stage_one}"/usr/bin/nv-hostengine bin/.
}
# copute always includes utility per default
chisseled_compute() {
echo "nvidia: chisseling GPU"
cp -a "${stage_one}"/lib/modules/* lib/modules/.
libdir="lib/${machine_arch}-linux-gnu"
cp -a "${stage_one}/${libdir}"/libdl.so.2* "${libdir}"/.
cp -a "${stage_one}/${libdir}"/libz.so.1* "${libdir}"/.
cp -a "${stage_one}/${libdir}"/libpthread.so.0* "${libdir}"/.
cp -a "${stage_one}/${libdir}"/libresolv.so.2* "${libdir}"/.
cp -a "${stage_one}/${libdir}"/libc.so.6* "${libdir}"/.
cp -a "${stage_one}/${libdir}"/libm.so.6* "${libdir}"/.
cp -a "${stage_one}/${libdir}"/librt.so.1* "${libdir}"/.
# nvidia-persistenced dependencies for CUDA repo and >= 590
cp -a "${stage_one}/${libdir}"/libtirpc.so.3* "${libdir}"/.
cp -a "${stage_one}/${libdir}"/libgssapi_krb5.so.2* "${libdir}"/.
cp -a "${stage_one}/${libdir}"/libkrb5.so.3* "${libdir}"/.
cp -a "${stage_one}/${libdir}"/libkrb5support.so.0* "${libdir}"/.
cp -a "${stage_one}/${libdir}"/libk5crypto.so.3* "${libdir}"/.
cp -a "${stage_one}/${libdir}"/libcom_err.so.2* "${libdir}"/.
cp -a "${stage_one}/${libdir}"/libkeyutils.so.1* "${libdir}"/.
cp -a "${stage_one}/etc/netconfig" etc/.
[[ "${type}" == "confidential" ]] && cp -a "${stage_one}/${libdir}"/libnvidia-pkcs11* "${libdir}"/.
[[ ${machine_arch} == "aarch64" ]] && libdir="lib"
[[ ${machine_arch} == "x86_64" ]] && libdir="lib64"
cp -aL "${stage_one}/${libdir}"/ld-linux-* "${libdir}"/.
libdir=usr/lib/"${machine_arch}"-linux-gnu
cp -a "${stage_one}/${libdir}"/libnv* lib/"${machine_arch}"-linux-gnu/.
cp -a "${stage_one}/${libdir}"/libcuda.so.* lib/"${machine_arch}"-linux-gnu/.
# basic GPU admin tools
cp -a "${stage_one}"/usr/bin/nvidia-persistenced bin/.
cp -a "${stage_one}"/usr/bin/nvidia-smi bin/.
cp -a "${stage_one}"/usr/bin/nvidia-ctk bin/.
cp -a "${stage_one}"/usr/bin/nvidia-cdi-hook bin/.
ln -s ../bin usr/bin
}
chisseled_gpudirect() {
echo "nvidia: chisseling GPUDirect"
echo "nvidia: not implemented yet"
exit 1
}
chisseled_nvat() {
if [[ "${type}" != "confidential" ]]; then
return
fi
echo "nvidia: chisseling NVAT"
local libdir="lib/${machine_arch}-linux-gnu"
# NVAT shared library (bundled via coco-guest-components tarball)
cp -a "${stage_one}"/usr/local/lib/libnvat.so* "${libdir}"/.
# NVAT runtime dependencies (per ldd on attestation-agent)
cp -a "${stage_one}/${libdir}"/libxml2.so.2* "${libdir}"/.
cp -a "${stage_one}/${libdir}"/libstdc++.so.6* "${libdir}"/.
cp -a "${stage_one}/${libdir}"/liblzma.so.5* "${libdir}"/.
cp -a "${stage_one}/${libdir}"/libicuuc.so.* "${libdir}"/.
cp -a "${stage_one}/${libdir}"/libicudata.so.* "${libdir}"/.
}
setup_nvrc_init_symlinks() {
local nvrc="NVRC-${machine_arch}-unknown-linux-musl"
# make sure NVRC is the init process for the initrd and image case
ln -sf /bin/"${nvrc}" init
ln -sf /bin/"${nvrc}" sbin/init
}
chisseled_init() {
echo "nvidia: chisseling init"
tar --zstd -xvf "${BUILD_DIR}"/kata-static-busybox.tar.zst -C .
mkdir -p dev etc proc run/cdi sys tmp usr var lib/modules lib/firmware \
usr/share/nvidia lib/"${machine_arch}"-linux-gnu lib64 \
usr/bin etc/modprobe.d etc/ssl/certs
ln -sf ../run var/run
ln -sf ../run var/log
ln -sf ../run var/cache
# Needed for various RUST static builds with LIBC=gnu
libdir=lib/"${machine_arch}"-linux-gnu
cp -a "${stage_one}"/"${libdir}"/libgcc_s.so.1* "${libdir}"/.
local nvrc="NVRC-${machine_arch}-unknown-linux-musl"
cp -a "${stage_one}/bin/${nvrc}" bin/.
cp -a "${stage_one}/bin/${nvrc}".cert bin/.
cp -a "${stage_one}/bin/${nvrc}".sig bin/.
setup_nvrc_init_symlinks
cp -a "${stage_one}"/usr/bin/kata-agent usr/bin/.
if [[ "${AGENT_POLICY}" == "yes" ]]; then
cp -a "${stage_one}"/etc/kata-opa etc/.
fi
cp -a "${stage_one}"/etc/resolv.conf etc/.
cp -a "${stage_one}"/lib/firmware/nvidia lib/firmware/.
cp -a "${stage_one}"/sbin/ldconfig.real sbin/ldconfig
cp -a "${stage_one}"/etc/ssl/certs/ca-certificates.crt etc/ssl/certs/.
local conf_file="etc/modprobe.d/0000-nvidia.conf"
echo 'options nvidia NVreg_DeviceFileMode=0660' > "${conf_file}"
}
compress_rootfs() {
echo "nvidia: compressing rootfs"
# For some unobvious reason libc has executable bit set
# clean this up otherwise the find -executable will not work correctly
find . -type f -name "*.so.*" | while IFS= read -r file; do
if ! file "${file}" | grep -q ELF; then
echo "nvidia: skip stripping file: ${file} ($(file -b "${file}"))"
continue
fi
chmod -x "${file}"
strip "${file}"
done
find . -type f -executable | while IFS= read -r file; do
# Skip files with setuid/setgid bits (UPX refuses to pack them)
if [[ -u "${file}" ]] || [[ -g "${file}" ]]; then
echo "nvidia: skip compressing executable (special permissions): ${file} ($(file -b "${file}"))"
continue
fi
if ! file "${file}" | grep -q ELF; then
echo "nvidia: skip compressing executable (not ELF): ${file} ($(file -b "${file}"))"
continue
fi
strip "${file}"
"${BUILD_DIR}"/upx-4.2.4-"${distro_arch}"_linux/upx --best --lzma "${file}"
done
# While I was playing with compression the executable flag on
# /lib64/ld-linux-x86-64.so.2 was lost...
# Since this is the program interpreter, it needs to be executable
# as well.. sigh
[[ ${machine_arch} == "aarch64" ]] && libdir="lib"
[[ ${machine_arch} == "x86_64" ]] && libdir="lib64"
chmod +x "${libdir}"/ld-linux-*
}
copy_cdh_runtime_deps() {
local libdir="lib/${machine_arch}-linux-gnu"
# Shared libraries required by /usr/local/bin/confidential-data-hub.
cp -a "${stage_one}/${libdir}"/libgcc_s.so.1* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libm.so.6* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libc.so.6* "${libdir}/."
# Shared libraries required by the cryptsetup, mkfs.ext4, and dd binaries
# used by CDH secure_mount.
#
# cryptsetup direct dependencies
cp -a "${stage_one}/${libdir}"/libcryptsetup.so.12* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libpopt.so.0* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libuuid.so.1* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libblkid.so.1* "${libdir}/."
# libcryptsetup transitive dependencies
cp -a "${stage_one}/${libdir}"/libdevmapper.so.1.02.1* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libcrypto.so.3* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libargon2.so.1* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libjson-c.so.5* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libselinux.so.1* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libudev.so.1* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libpcre2-8.so.0* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libcap.so.2* "${libdir}/."
# e2fsprogs (mke2fs/mkfs.ext4) runtime libs
cp -a "${stage_one}/${libdir}"/libext2fs.so.2* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libcom_err.so.2* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libe2p.so.2* "${libdir}/."
# cryptsetup, mkfs.ext4, and dd are used by CDH secure_mount.
mkdir -p sbin etc bin
cp -a "${stage_one}/sbin/cryptsetup" sbin/.
cp -a "${stage_one}/sbin/mke2fs" sbin/.
cp -a "${stage_one}/sbin/mkfs.ext4" sbin/.
cp -a "${stage_one}/etc/mke2fs.conf" etc/.
cp -a "${stage_one}/usr/bin/dd" bin/.
}
coco_guest_components() {
if [[ "${type}" != "confidential" ]]; then
return
fi
info "nvidia: installing the confidential containers guest components tarball"
local -r coco_bin_dir="usr/local/bin"
local -r etc_dir="etc"
local -r pause_dir="pause_bundle"
mkdir -p "${coco_bin_dir}"
cp -a "${stage_one}/${coco_bin_dir}"/attestation-agent-nv "${coco_bin_dir}/attestation-agent"
cp -a "${stage_one}/${coco_bin_dir}"/api-server-rest "${coco_bin_dir}/."
cp -a "${stage_one}/${coco_bin_dir}"/confidential-data-hub "${coco_bin_dir}/."
cp -a "${stage_one}/${etc_dir}"/ocicrypt_config.json "${etc_dir}/."
mkdir -p "${pause_dir}/rootfs"
cp -a "${stage_one}/${pause_dir}"/config.json "${pause_dir}/."
cp -a "${stage_one}/${pause_dir}"/rootfs/pause "${pause_dir}/rootfs/."
copy_cdh_runtime_deps
}
setup_nvidia_gpu_rootfs_stage_two() {
readonly stage_two="${ROOTFS_DIR:?}"
readonly stack="${NVIDIA_GPU_STACK:?}"
readonly type=${1:-""}
# If devkit flag is set, skip chisseling, use stage_one
if echo "${stack}" | grep -q '\<devkit\>'; then
echo "nvidia: devkit mode enabled - skip chisseling"
tar -C "${stage_two}" -xf "${stage_one}".tar.zst
pushd "${stage_two}" >> /dev/null
# Only step needed from stage_two (see chisseled_init)
setup_nvrc_init_symlinks
else
echo "nvidia: chisseling the following stack components: ${stack}"
[[ -e "${stage_one}" ]] && rm -rf "${stage_one}"
[[ ! -e "${stage_one}" ]] && mkdir -p "${stage_one}"
tar -C "${stage_one}" -xf "${stage_one}".tar.zst
pushd "${stage_two}" >> /dev/null
chisseled_init
chisseled_iptables
IFS=',' read -r -a stack_components <<< "${NVIDIA_GPU_STACK}"
for component in "${stack_components[@]}"; do
if [[ "${component}" = "compute" ]]; then
echo "nvidia: processing \"compute\" component"
chisseled_compute
elif [[ "${component}" = "dcgm" ]]; then
echo "nvidia: processing DCGM component"
chisseled_dcgm
elif [[ "${component}" = "nvswitch" ]]; then
echo "nvidia: processing NVSwitch component"
chisseled_nvswitch
elif [[ "${component}" = "gpudirect" ]]; then
echo "nvidia: processing GPUDirect component"
chisseled_gpudirect
fi
done
coco_guest_components
chisseled_nvat
fi
compress_rootfs
chroot . ldconfig
popd >> /dev/null
}