diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 5e333ea446..a3df32ee4c 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -465,12 +465,22 @@ endif KERNELPATH_NV = $(KERNELDIR)/$(KERNELNAME_NV) IMAGENAME_NV = $(PROJECT_TAG)-nvidia-gpu.img IMAGEPATH_NV = $(PKGDATADIR)/$(IMAGENAME_NV) + # Composable NVIDIA stack: driver-agnostic base-nvidia boot image plus the + # driver-versioned gpu extension (cold-plugged and mounted at /run/kata-extensions/gpu). + IMAGENAME_NV_BASE = $(PROJECT_TAG)-nvidia-gpu-base.img + IMAGEPATH_NV_BASE = $(PKGDATADIR)/$(IMAGENAME_NV_BASE) + IMAGENAME_NV_EXTENSION = $(PROJECT_TAG)-nvidia-gpu-extension.img + IMAGEPATH_NV_EXTENSION = $(PKGDATADIR)/$(IMAGENAME_NV_EXTENSION) KERNELPARAMS_NV := cgroup_no_v1=all pci=realloc pci=nocrs pci=assign-busses KERNELPARAMS_CONFIDENTIAL_NV = $(KERNELPARAMS_NV) KERNELPARAMS_CONFIDENTIAL_NV += "nvrc.smi.srs=1" + # Verity params for the base-nvidia boot image (root_hash_nvidia-gpu-base.txt) + # and for the gpu extension (root_hash_nvidia-gpu-extension.txt); filled in by the + # shim-v2 build from the emitted root-hash files. KERNELVERITYPARAMS_NV ?= + NVIDIAGPUEXTENSIONVERITYPARAMS ?= DEFAULTVCPUS_NV := 1 DEFAULTMEMORY_NV := 8192 DEFOVERHEADVCPUS_NV := 0.5 @@ -649,6 +659,11 @@ USER_VARS += IMAGECONFIDENTIALNAME USER_VARS += IMAGEPATH USER_VARS += IMAGEPATH_CLH_AZURE USER_VARS += IMAGEPATH_NV +USER_VARS += IMAGENAME_NV_BASE +USER_VARS += IMAGEPATH_NV_BASE +USER_VARS += IMAGENAME_NV_EXTENSION +USER_VARS += IMAGEPATH_NV_EXTENSION +USER_VARS += NVIDIAGPUEXTENSIONVERITYPARAMS USER_VARS += IMAGECONFIDENTIALPATH USER_VARS += COCOIMAGENAME USER_VARS += COCOIMAGEPATH diff --git a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in index ce7025f407..ef2ab4a995 100644 --- a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in @@ -16,7 +16,7 @@ [hypervisor.qemu] path = "@QEMUPATH@" kernel = "@KERNELPATH_NV@" -image = "@IMAGEPATH_NV@" +image = "@IMAGEPATH_NV_BASE@" machine_type = "@MACHINETYPE@" # rootfs filesystem type: @@ -850,3 +850,12 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@" # When set (together with a non-"no-port" cold_plug_vfio), the runtime can cold-plug # devices discovered via the Pod Resources API. Path is typically under kubelet root-dir. pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@" + +# The GPU userspace (driver binaries, libraries, configs, firmware and the +# NVIDIA kernel modules) ships in a separate, driver-versioned extension image that +# is cold-plugged alongside the base-nvidia boot image. NVRC mounts it at +# /run/kata-extensions/gpu and resolves the GPU stack from there. +[[hypervisor.qemu.guest_extension_images]] +name = "gpu" +path = "@IMAGEPATH_NV_EXTENSION@" +verity_params = "@NVIDIAGPUEXTENSIONVERITYPARAMS@" diff --git a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-snp-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-snp-runtime-rs.toml.in index 2665889009..0776643044 100644 --- a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-snp-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-snp-runtime-rs.toml.in @@ -16,7 +16,7 @@ [hypervisor.qemu] path = "@QEMUSNPPATH@" kernel = "@KERNELPATH_CONFIDENTIAL_NV@" -image = "@IMAGEPATH_CONFIDENTIAL_NV@" +image = "@IMAGEPATH_NV_BASE@" machine_type = "@MACHINETYPE@" # Enable confidential guest support. @@ -102,7 +102,7 @@ kernel_params = "@KERNELPARAMS_CONFIDENTIAL_NV@" # Optional dm-verity parameters (comma-separated key=value list): # root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=... # These are used by the runtime to assemble dm-verity kernel params. -kernel_verity_params = "@KERNELVERITYPARAMS_CONFIDENTIAL_NV@" +kernel_verity_params = "@KERNELVERITYPARAMS_NV@" # Path to the firmware. # If you want that qemu uses the default firmware leave this option empty @@ -782,3 +782,19 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@" # When set (together with a non-"no-port" cold_plug_vfio), the runtime can cold-plug # devices discovered via the Pod Resources API. Path is typically under kubelet root-dir. pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@" + +# The GPU userspace (driver binaries, libraries, configs, firmware and the +# NVIDIA kernel modules) ships in a separate, driver-versioned extension image that +# is cold-plugged alongside the base-nvidia boot image. NVRC mounts it at +# /run/kata-extensions/gpu and resolves the GPU stack from there. +[[hypervisor.qemu.guest_extension_images]] +name = "gpu" +path = "@IMAGEPATH_NV_EXTENSION@" +verity_params = "@NVIDIAGPUEXTENSIONVERITYPARAMS@" + +# Confidential guests additionally cold-plug the CoCo guest-components extension +# (attestation-agent, confidential-data-hub, ...) mounted at /run/kata-extensions/coco. +[[hypervisor.qemu.guest_extension_images]] +name = "coco" +path = "@COCOIMAGEPATH@" +verity_params = "@COCOVERITYPARAMS@" diff --git a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-tdx-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-tdx-runtime-rs.toml.in index 30452df6c6..5a7f2f7ac4 100644 --- a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-tdx-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-tdx-runtime-rs.toml.in @@ -16,7 +16,7 @@ [hypervisor.qemu] path = "@QEMUTDXPATH@" kernel = "@KERNELPATH_CONFIDENTIAL_NV@" -image = "@IMAGEPATH_CONFIDENTIAL_NV@" +image = "@IMAGEPATH_NV_BASE@" machine_type = "@MACHINETYPE@" tdx_quote_generation_service_socket_port = @QEMUTDXQUOTEGENERATIONSERVICESOCKETPORT@ @@ -78,7 +78,7 @@ kernel_params = "@KERNELPARAMS_CONFIDENTIAL_NV@" # Optional dm-verity parameters (comma-separated key=value list): # root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=... # These are used by the runtime to assemble dm-verity kernel params. -kernel_verity_params = "@KERNELVERITYPARAMS_CONFIDENTIAL_NV@" +kernel_verity_params = "@KERNELVERITYPARAMS_NV@" # Path to the firmware. # If you want that qemu uses the default firmware leave this option empty @@ -758,3 +758,19 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@" # When set (together with a non-"no-port" cold_plug_vfio), the runtime can cold-plug # devices discovered via the Pod Resources API. Path is typically under kubelet root-dir. pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@" + +# The GPU userspace (driver binaries, libraries, configs, firmware and the +# NVIDIA kernel modules) ships in a separate, driver-versioned extension image that +# is cold-plugged alongside the base-nvidia boot image. NVRC mounts it at +# /run/kata-extensions/gpu and resolves the GPU stack from there. +[[hypervisor.qemu.guest_extension_images]] +name = "gpu" +path = "@IMAGEPATH_NV_EXTENSION@" +verity_params = "@NVIDIAGPUEXTENSIONVERITYPARAMS@" + +# Confidential guests additionally cold-plug the CoCo guest-components extension +# (attestation-agent, confidential-data-hub, ...) mounted at /run/kata-extensions/coco. +[[hypervisor.qemu.guest_extension_images]] +name = "coco" +path = "@COCOIMAGEPATH@" +verity_params = "@COCOVERITYPARAMS@" diff --git a/tools/packaging/static-build/shim-v2/build.sh b/tools/packaging/static-build/shim-v2/build.sh index 9126d1db74..a07ab7c44a 100755 --- a/tools/packaging/static-build/shim-v2/build.sh +++ b/tools/packaging/static-build/shim-v2/build.sh @@ -45,17 +45,20 @@ esac # - rootfs-image (the base image, measured; root hash labelled "base") # - rootfs-image-confidential (monolithic CoCo image, root hash "confidential") # - rootfs-image-coco-extension -# - rootfs-image-nvidia-gpu -# - rootfs-image-nvidia-gpu-confidential +# - rootfs-image-nvidia-gpu / rootfs-image-nvidia-gpu-confidential (monolithic) +# - rootfs-image-nvidia-gpu-base (the driver-agnostic NVIDIA boot image) +# - rootfs-image-nvidia-gpu-extension (the driver-versioned gpu extension) # -# The CoCo configs come in two flavours during the transition to split images: +# Both the CoCo and the NVIDIA GPU configs come in two flavours during the +# transition to split images, and the two runtimes are built in separate `make` +# invocations so each maps the same make var to a different root hash: # - runtime-rs (Rust) uses the split layout: the measured base image -# (@KERNELVERITYPARAMS@) plus the CoCo extension (@COCOVERITYPARAMS@). -# - runtime (Go) still uses the monolithic confidential image, whose dm-verity -# hash is carried by @KERNELVERITYPARAMS@. -# Because the two runtimes are built in separate `make` invocations, each gets -# its own @KERNELVERITYPARAMS@ value (base hash for Rust, confidential hash for -# Go). The NVIDIA GPU verity params are shared by both runtimes. +# (@KERNELVERITYPARAMS@) plus the CoCo extension (@COCOVERITYPARAMS@), and for +# NVIDIA the base-nvidia image (@KERNELVERITYPARAMS_NV@) plus the gpu extension +# (@NVIDIAGPUEXTENSIONVERITYPARAMS@). +# - runtime (Go) still uses the monolithic images: the confidential image +# (@KERNELVERITYPARAMS@) and, for NVIDIA, nvidia-gpu (@KERNELVERITYPARAMS_NV@) +# and nvidia-gpu-confidential (@KERNELVERITYPARAMS_CONFIDENTIAL_NV@). # # shellcheck disable=SC2154 root_hash_dir="${repo_root_dir}/tools/packaging/kata-deploy/local-build/build" @@ -77,16 +80,24 @@ read_verity_param() { printf ' %s=%s' "${param_var}" "${root_measure_config}" } -# Shared by both runtimes (NVIDIA GPU variants). -EXTRA_OPTS+="$(read_verity_param "nvidia-gpu" "KERNELVERITYPARAMS_NV")" -EXTRA_OPTS+="$(read_verity_param "nvidia-gpu-confidential" "KERNELVERITYPARAMS_CONFIDENTIAL_NV")" +# The NVIDIA GPU verity params differ per runtime: runtime-rs boots the +# composable split layout (base-nvidia + gpu extension) while the Go runtime +# still boots the monolithic nvidia-gpu / nvidia-gpu-confidential images. Since +# both runtimes reuse KERNELVERITYPARAMS_NV but map it to different root hashes, +# the reads live in each runtime's own EXTRA_OPTS rather than the shared one. -# runtime-rs (Rust): split image -> base hash + CoCo extension hash. +# runtime-rs (Rust): split images -> base hash + CoCo extension hash, plus the +# base-nvidia and gpu extension hashes for the NVIDIA GPU configs. RUST_EXTRA_OPTS="$(read_verity_param "base" "KERNELVERITYPARAMS")" RUST_EXTRA_OPTS+="$(read_verity_param "coco-extension" "COCOVERITYPARAMS")" +RUST_EXTRA_OPTS+="$(read_verity_param "nvidia-gpu-base" "KERNELVERITYPARAMS_NV")" +RUST_EXTRA_OPTS+="$(read_verity_param "nvidia-gpu-extension" "NVIDIAGPUEXTENSIONVERITYPARAMS")" -# runtime (Go): monolithic confidential image -> confidential hash. +# runtime (Go): monolithic confidential image -> confidential hash, plus the +# monolithic nvidia-gpu / nvidia-gpu-confidential hashes for the NVIDIA configs. GO_EXTRA_OPTS="$(read_verity_param "confidential" "KERNELVERITYPARAMS")" +GO_EXTRA_OPTS+="$(read_verity_param "nvidia-gpu" "KERNELVERITYPARAMS_NV")" +GO_EXTRA_OPTS+="$(read_verity_param "nvidia-gpu-confidential" "KERNELVERITYPARAMS_CONFIDENTIAL_NV")" # shellcheck disable=SC2154,SC2086 docker pull "${container_image}" || \