config: boot composable base-nvidia image + gpu/coco addons

Flip the NVIDIA GPU configs (Go + runtime-rs, plain/tdx/snp) from the monolithic image to the driver-agnostic base-nvidia boot image plus a cold-plugged, driver-versioned gpu addon mounted at /run/kata-addons/gpu. Confidential configs additionally cold-plug the CoCo guest-components addon and enforce the base-nvidia dm-verity hash. Add the IMAGE{NAME,PATH}_NV_{BASE,ADDON} and NVIDIAGPUADDONVERITYPARAMS make vars, and repoint the shim-v2 verity_variants at the base-nvidia and gpu-addon root hashes. The monolithic images are still built and shipped during the transition, but no generated config references their hashes.
2026-07-10 07:05:51 +00:00 · 2026-06-11 07:29:17 +02:00
parent c029e9ed5b
commit 07d1e92cc6
9 changed files with 133 additions and 17 deletions
--- a/src/runtime-rs/Makefile
+++ b/src/runtime-rs/Makefile
@@ -447,12 +447,22 @@ endif
    KERNELPATH_NV = $(KERNELDIR)/$(KERNELNAME_NV)
    IMAGENAME_NV = $(PROJECT_TAG)-nvidia-gpu.img
    IMAGEPATH_NV = $(PKGDATADIR)/$(IMAGENAME_NV)
+    # Composable NVIDIA stack: driver-agnostic base-nvidia boot image plus the
+    # driver-versioned gpu addon (cold-plugged and mounted at /run/kata-addons/gpu).
+    IMAGENAME_NV_BASE = $(PROJECT_TAG)-nvidia-gpu-base.img
+    IMAGEPATH_NV_BASE = $(PKGDATADIR)/$(IMAGENAME_NV_BASE)
+    IMAGENAME_NV_ADDON = $(PROJECT_TAG)-nvidia-gpu-addon.img
+    IMAGEPATH_NV_ADDON = $(PKGDATADIR)/$(IMAGENAME_NV_ADDON)
    KERNELPARAMS_NV := cgroup_no_v1=all pci=realloc pci=nocrs pci=assign-busses

    KERNELPARAMS_CONFIDENTIAL_NV = $(KERNELPARAMS_NV)
    KERNELPARAMS_CONFIDENTIAL_NV += "nvrc.smi.srs=1"

+    # Verity params for the base-nvidia boot image (root_hash_nvidia-gpu-base.txt)
+    # and for the gpu addon (root_hash_nvidia-gpu-addon.txt); filled in by the
+    # shim-v2 build from the emitted root-hash files.
    KERNELVERITYPARAMS_NV ?=
+    NVIDIAGPUADDONVERITYPARAMS ?=
    DEFAULTVCPUS_NV := 1
    DEFAULTMEMORY_NV := 8192
    DEFAULTTIMEOUT_NV := 1200
@@ -628,6 +638,11 @@ USER_VARS += IMAGECONFIDENTIALNAME
 USER_VARS += IMAGEPATH
 USER_VARS += IMAGEPATH_CLH_AZURE
 USER_VARS += IMAGEPATH_NV
+USER_VARS += IMAGENAME_NV_BASE
+USER_VARS += IMAGEPATH_NV_BASE
+USER_VARS += IMAGENAME_NV_ADDON
+USER_VARS += IMAGEPATH_NV_ADDON
+USER_VARS += NVIDIAGPUADDONVERITYPARAMS
 USER_VARS += IMAGECONFIDENTIALPATH
 USER_VARS += COCOIMAGENAME
 USER_VARS += COCOIMAGEPATH
--- a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in
+++ b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in
@@ -16,7 +16,7 @@
 [hypervisor.qemu]
 path = "@QEMUPATH@"
 kernel = "@KERNELPATH_NV@"
-image = "@IMAGEPATH_NV@"
+image = "@IMAGEPATH_NV_BASE@"
 machine_type = "@MACHINETYPE@"

 # rootfs filesystem type:
@@ -806,3 +806,12 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@"
 # When set (together with a non-"no-port" cold_plug_vfio), the runtime can cold-plug
 # devices discovered via the Pod Resources API. Path is typically under kubelet root-dir.
 pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@"
+
+# The GPU userspace (driver binaries, libraries, configs, firmware and the
+# NVIDIA kernel modules) ships in a separate, driver-versioned addon image that
+# is cold-plugged alongside the base-nvidia boot image.  NVRC mounts it at
+# /run/kata-addons/gpu and resolves the GPU stack from there.
+[[hypervisor.qemu.extra_images]]
+name = "gpu"
+path = "@IMAGEPATH_NV_ADDON@"
+verity_params = "@NVIDIAGPUADDONVERITYPARAMS@"
--- a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-snp-runtime-rs.toml.in
+++ b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-snp-runtime-rs.toml.in
@@ -16,7 +16,7 @@
 [hypervisor.qemu]
 path = "@QEMUSNPPATH@"
 kernel = "@KERNELPATH_CONFIDENTIAL_NV@"
-image = "@IMAGEPATH_CONFIDENTIAL_NV@"
+image = "@IMAGEPATH_NV_BASE@"
 machine_type = "@MACHINETYPE@"

 # Enable confidential guest support.
@@ -102,7 +102,7 @@ kernel_params = "@KERNELPARAMS_CONFIDENTIAL_NV@"
 # Optional dm-verity parameters (comma-separated key=value list):
 # root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=...
 # These are used by the runtime to assemble dm-verity kernel params.
-kernel_verity_params = "@KERNELVERITYPARAMS_CONFIDENTIAL_NV@"
+kernel_verity_params = "@KERNELVERITYPARAMS_NV@"

 # Path to the firmware.
 # If you want that qemu uses the default firmware leave this option empty
@@ -738,3 +738,19 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@"
 # When set (together with a non-"no-port" cold_plug_vfio), the runtime can cold-plug
 # devices discovered via the Pod Resources API. Path is typically under kubelet root-dir.
 pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@"
+
+# The GPU userspace (driver binaries, libraries, configs, firmware and the
+# NVIDIA kernel modules) ships in a separate, driver-versioned addon image that
+# is cold-plugged alongside the base-nvidia boot image.  NVRC mounts it at
+# /run/kata-addons/gpu and resolves the GPU stack from there.
+[[hypervisor.qemu.extra_images]]
+name = "gpu"
+path = "@IMAGEPATH_NV_ADDON@"
+verity_params = "@NVIDIAGPUADDONVERITYPARAMS@"
+
+# Confidential guests additionally cold-plug the CoCo guest-components addon
+# (attestation-agent, confidential-data-hub, ...) mounted at /run/kata-addons/coco.
+[[hypervisor.qemu.extra_images]]
+name = "coco"
+path = "@COCOIMAGEPATH@"
+verity_params = "@COCOVERITYPARAMS@"
--- a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-tdx-runtime-rs.toml.in
+++ b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-tdx-runtime-rs.toml.in
@@ -16,7 +16,7 @@
 [hypervisor.qemu]
 path = "@QEMUTDXPATH@"
 kernel = "@KERNELPATH_CONFIDENTIAL_NV@"
-image = "@IMAGEPATH_CONFIDENTIAL_NV@"
+image = "@IMAGEPATH_NV_BASE@"
 machine_type = "@MACHINETYPE@"
 tdx_quote_generation_service_socket_port = @QEMUTDXQUOTEGENERATIONSERVICESOCKETPORT@

@@ -78,7 +78,7 @@ kernel_params = "@KERNELPARAMS_CONFIDENTIAL_NV@"
 # Optional dm-verity parameters (comma-separated key=value list):
 # root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=...
 # These are used by the runtime to assemble dm-verity kernel params.
-kernel_verity_params = "@KERNELVERITYPARAMS_CONFIDENTIAL_NV@"
+kernel_verity_params = "@KERNELVERITYPARAMS_NV@"

 # Path to the firmware.
 # If you want that qemu uses the default firmware leave this option empty
@@ -714,3 +714,19 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@"
 # When set (together with a non-"no-port" cold_plug_vfio), the runtime can cold-plug
 # devices discovered via the Pod Resources API. Path is typically under kubelet root-dir.
 pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@"
+
+# The GPU userspace (driver binaries, libraries, configs, firmware and the
+# NVIDIA kernel modules) ships in a separate, driver-versioned addon image that
+# is cold-plugged alongside the base-nvidia boot image.  NVRC mounts it at
+# /run/kata-addons/gpu and resolves the GPU stack from there.
+[[hypervisor.qemu.extra_images]]
+name = "gpu"
+path = "@IMAGEPATH_NV_ADDON@"
+verity_params = "@NVIDIAGPUADDONVERITYPARAMS@"
+
+# Confidential guests additionally cold-plug the CoCo guest-components addon
+# (attestation-agent, confidential-data-hub, ...) mounted at /run/kata-addons/coco.
+[[hypervisor.qemu.extra_images]]
+name = "coco"
+path = "@COCOIMAGEPATH@"
+verity_params = "@COCOVERITYPARAMS@"
--- a/src/runtime/Makefile
+++ b/src/runtime/Makefile
@@ -65,6 +65,10 @@ INITRDCONFIDENTIALNAME = $(PROJECT_TAG)-initrd-confidential.img

 IMAGENAME_NV = $(PROJECT_TAG)-nvidia-gpu.img
 IMAGENAME_CONFIDENTIAL_NV = $(PROJECT_TAG)-nvidia-gpu-confidential.img
+# Composable NVIDIA stack: driver-agnostic base-nvidia boot image plus the
+# driver-versioned gpu addon (cold-plugged and mounted at /run/kata-addons/gpu).
+IMAGENAME_NV_BASE = $(PROJECT_TAG)-nvidia-gpu-base.img
+IMAGENAME_NV_ADDON = $(PROJECT_TAG)-nvidia-gpu-addon.img

 TARGET = $(BIN_PREFIX)-runtime
 RUNTIME_OUTPUT = $(CURDIR)/$(TARGET)
@@ -137,6 +141,8 @@ INITRDCONFIDENTIALPATH := $(PKGDATADIR)/$(INITRDCONFIDENTIALNAME)

 IMAGEPATH_NV := $(PKGDATADIR)/$(IMAGENAME_NV)
 IMAGEPATH_CONFIDENTIAL_NV := $(PKGDATADIR)/$(IMAGENAME_CONFIDENTIAL_NV)
+IMAGEPATH_NV_BASE := $(PKGDATADIR)/$(IMAGENAME_NV_BASE)
+IMAGEPATH_NV_ADDON := $(PKGDATADIR)/$(IMAGENAME_NV_ADDON)

 ROOTFSTYPE_EXT4 := \"ext4\"
 ROOTFSTYPE_XFS := \"xfs\"
@@ -164,6 +170,9 @@ FIRMWARESNPPATH_NV := $(FIRMWARESNPPATH)
 KERNELVERITYPARAMS ?= ""
 KERNELVERITYPARAMS_NV ?= ""
 KERNELVERITYPARAMS_CONFIDENTIAL_NV ?= ""
+# Verity params for the gpu addon (root_hash_nvidia-gpu-addon.txt); the
+# base-nvidia boot image reuses KERNELVERITYPARAMS_NV.
+NVIDIAGPUADDONVERITYPARAMS ?= ""

 # Name of default configuration file the runtime will use.
 CONFIG_FILE = configuration.toml
@@ -695,6 +704,11 @@ USER_VARS += IMAGENAME_NV
 USER_VARS += IMAGENAME_CONFIDENTIAL_NV
 USER_VARS += IMAGEPATH_NV
 USER_VARS += IMAGEPATH_CONFIDENTIAL_NV
+USER_VARS += IMAGENAME_NV_BASE
+USER_VARS += IMAGEPATH_NV_BASE
+USER_VARS += IMAGENAME_NV_ADDON
+USER_VARS += IMAGEPATH_NV_ADDON
+USER_VARS += NVIDIAGPUADDONVERITYPARAMS
 USER_VARS += KERNELNAME_NV
 USER_VARS += KERNELPATH_NV
 USER_VARS += KERNELNAME_CONFIDENTIAL_NV
--- a/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in
+++ b/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in
@@ -15,7 +15,7 @@
 [hypervisor.qemu]
 path = "@QEMUSNPPATH@"
 kernel = "@KERNELPATH_CONFIDENTIAL_NV@"
-image = "@IMAGEPATH_CONFIDENTIAL_NV@"
+image = "@IMAGEPATH_NV_BASE@"

 machine_type = "@MACHINETYPE@"

@@ -95,7 +95,7 @@ kernel_params = "@KERNELPARAMS_CONFIDENTIAL_NV@"
 # Optional dm-verity parameters (comma-separated key=value list):
 # root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=...
 # These are used by the runtime to assemble dm-verity kernel params.
-kernel_verity_params = "@KERNELVERITYPARAMS_CONFIDENTIAL_NV@"
+kernel_verity_params = "@KERNELVERITYPARAMS_NV@"

 # Path to the firmware.
 # If you want that qemu uses the default firmware leave this option empty
@@ -796,3 +796,19 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@"
 #      cold_plug_vfio != no_port AND pod_resource_api_sock != "" => kubelet
 #              based cold plug.
 pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@"
+
+# The GPU userspace (driver binaries, libraries, configs, firmware and the
+# NVIDIA kernel modules) ships in a separate, driver-versioned addon image that
+# is cold-plugged alongside the base-nvidia boot image.  NVRC mounts it at
+# /run/kata-addons/gpu and resolves the GPU stack from there.
+[[hypervisor.qemu.extra_images]]
+name = "gpu"
+path = "@IMAGEPATH_NV_ADDON@"
+verity_params = "@NVIDIAGPUADDONVERITYPARAMS@"
+
+# Confidential guests additionally cold-plug the CoCo guest-components addon
+# (attestation-agent, confidential-data-hub, ...) mounted at /run/kata-addons/coco.
+[[hypervisor.qemu.extra_images]]
+name = "coco"
+path = "@COCOIMAGEPATH@"
+verity_params = "@COCOVERITYPARAMS@"
--- a/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in
+++ b/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in
@@ -14,7 +14,7 @@
 [hypervisor.qemu]
 path = "@QEMUTDXEXPERIMENTALPATH@"
 kernel = "@KERNELPATH_CONFIDENTIAL_NV@"
-image = "@IMAGEPATH_CONFIDENTIAL_NV@"
+image = "@IMAGEPATH_NV_BASE@"

 machine_type = "@MACHINETYPE@"
 tdx_quote_generation_service_socket_port = @QEMUTDXQUOTEGENERATIONSERVICESOCKETPORT@
@@ -72,7 +72,7 @@ kernel_params = "@KERNELPARAMS_CONFIDENTIAL_NV@"
 # Optional dm-verity parameters (comma-separated key=value list):
 # root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=...
 # These are used by the runtime to assemble dm-verity kernel params.
-kernel_verity_params = "@KERNELVERITYPARAMS_CONFIDENTIAL_NV@"
+kernel_verity_params = "@KERNELVERITYPARAMS_NV@"

 # Path to the firmware.
 # If you want that qemu uses the default firmware leave this option empty
@@ -773,3 +773,19 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@"
 #      cold_plug_vfio != no_port AND pod_resource_api_sock != "" => kubelet
 #              based cold plug.
 pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@"
+
+# The GPU userspace (driver binaries, libraries, configs, firmware and the
+# NVIDIA kernel modules) ships in a separate, driver-versioned addon image that
+# is cold-plugged alongside the base-nvidia boot image.  NVRC mounts it at
+# /run/kata-addons/gpu and resolves the GPU stack from there.
+[[hypervisor.qemu.extra_images]]
+name = "gpu"
+path = "@IMAGEPATH_NV_ADDON@"
+verity_params = "@NVIDIAGPUADDONVERITYPARAMS@"
+
+# Confidential guests additionally cold-plug the CoCo guest-components addon
+# (attestation-agent, confidential-data-hub, ...) mounted at /run/kata-addons/coco.
+[[hypervisor.qemu.extra_images]]
+name = "coco"
+path = "@COCOIMAGEPATH@"
+verity_params = "@COCOVERITYPARAMS@"
--- a/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in
+++ b/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in
@@ -14,7 +14,7 @@
 [hypervisor.qemu]
 path = "@QEMUPATH@"
 kernel = "@KERNELPATH_NV@"
-image = "@IMAGEPATH_NV@"
+image = "@IMAGEPATH_NV_BASE@"
 machine_type = "@MACHINETYPE@"

 # rootfs filesystem type:
@@ -770,3 +770,12 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@"
 #      cold_plug_vfio != no_port AND pod_resource_api_sock != "" => kubelet
 #              based cold plug.
 pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@"
+
+# The GPU userspace (driver binaries, libraries, configs, firmware and the
+# NVIDIA kernel modules) ships in a separate, driver-versioned addon image that
+# is cold-plugged alongside the base-nvidia boot image.  NVRC mounts it at
+# /run/kata-addons/gpu and resolves the GPU stack from there.
+[[hypervisor.qemu.extra_images]]
+name = "gpu"
+path = "@IMAGEPATH_NV_ADDON@"
+verity_params = "@NVIDIAGPUADDONVERITYPARAMS@"
--- a/tools/packaging/static-build/shim-v2/build.sh
+++ b/tools/packaging/static-build/shim-v2/build.sh
@@ -44,20 +44,25 @@ esac
 # Variants (targets) that build a measured rootfs as of now are:
 # - rootfs-image (the base image, measured; root hash labelled "base")
 # - rootfs-image-coco-addon
-# - rootfs-image-nvidia-gpu
-# - rootfs-image-nvidia-gpu-confidential
+# - rootfs-image-nvidia-gpu-base (the driver-agnostic NVIDIA boot image)
+# - rootfs-image-nvidia-gpu-addon (the driver-versioned gpu addon)
 #
-# The base image is shared between non-confidential and confidential guests.
-# Only confidential configurations enforce its dm-verity hash via
-# @KERNELVERITYPARAMS@; non-confidential guests boot the data partition directly.
+# The base images (generic + base-nvidia) are shared between non-confidential
+# and confidential guests.  Only confidential configurations enforce the generic
+# base dm-verity hash via @KERNELVERITYPARAMS@; the NVIDIA stack always boots
+# measured, so both the non-confidential and confidential NVIDIA configs enforce
+# @KERNELVERITYPARAMS_NV@ (base-nvidia) and @NVIDIAGPUADDONVERITYPARAMS@ (gpu addon).
+#
+# The monolithic nvidia-gpu / nvidia-gpu-confidential images are still built and
+# shipped during the transition, but no generated config references their hashes.
 #
 # shellcheck disable=SC2154
 root_hash_dir="${repo_root_dir}/tools/packaging/kata-deploy/local-build/build"
 verity_variants=(
 	"base:KERNELVERITYPARAMS"
 	"coco-addon:COCOVERITYPARAMS"
-	"nvidia-gpu:KERNELVERITYPARAMS_NV"
-	"nvidia-gpu-confidential:KERNELVERITYPARAMS_CONFIDENTIAL_NV"
+	"nvidia-gpu-base:KERNELVERITYPARAMS_NV"
+	"nvidia-gpu-addon:NVIDIAGPUADDONVERITYPARAMS"
 )
 for entry in "${verity_variants[@]}"; do
 	variant="${entry%%:*}"