From 98d71313374b07ff06c74d5bd021ceaf797d40df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 24 Jul 2026 15:18:15 +0200 Subject: [PATCH] runtime: switch confidential/GPU configs to composable images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Go runtime confidential and NVIDIA GPU QEMU configs booted the monolithic rootfs images (kata-containers-confidential.img, kata-containers-nvidia-gpu.img, kata-containers-nvidia-gpu-confidential.img) that bake every guest component into a single rootfs. Switch them to the composable layout already used by runtime-rs: boot the measured base image (kata-containers.img / kata-containers-nvidia.img) and cold-plug the purpose-specific guest extension images: - qemu-coco-dev, qemu-tdx, qemu-snp: base + coco extension - qemu-nvidia-gpu: nvidia base + gpu extension - qemu-nvidia-gpu-snp/-tdx: nvidia base + gpu + coco extensions The dm-verity params for the NVIDIA confidential configs move from KERNELVERITYPARAMS_CONFIDENTIAL_NV to KERNELVERITYPARAMS_NV (the nvidia base hash), and the Makefile gains the coco/gpu extension image path and verity vars (COCOIMAGEPATH/COCOVERITYPARAMS, IMAGEPATH_NV_EXTENSION/NVIDIAGPUEXTENSIONVERITYPARAMS). No runtime code change is needed: the Go QEMU backend already cold-plugs guest_extension_images. The main motivation is size: the monolithic images duplicate content that already ships in the base + extension images, and once the Go runtime no longer needs them we can stop building them (see follow-up commits). That keeps the released kata-static tarball within GitHub's release asset size limit (2 GiB). The monolithic images are no longer referenced by any shipped config, but their build targets are intentionally left in place so they can still be built manually. Note: this composable path had only been validated for runtime-rs so far; the Go-runtime TEE/GPU boots (TDX, SEV-SNP, NVIDIA GPU) still need validation on real hardware. Signed-off-by: Fabiano Fidêncio Assisted-by: Cursor Signed-off-by: Fabiano Fidêncio --- src/runtime/Makefile | 27 ++++++++++++++++--- .../configuration-qemu-coco-dev.toml.in | 10 ++++++- .../configuration-qemu-nvidia-gpu-snp.toml.in | 18 +++++++++++-- .../configuration-qemu-nvidia-gpu-tdx.toml.in | 18 +++++++++++-- .../configuration-qemu-nvidia-gpu.toml.in | 9 ++++++- .../config/configuration-qemu-snp.toml.in | 10 ++++++- .../config/configuration-qemu-tdx.toml.in | 10 ++++++- 7 files changed, 91 insertions(+), 11 deletions(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 440955ff37..adb50bd10e 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -62,11 +62,18 @@ IMAGENAME = $(PROJECT_TAG).img IMAGECONFIDENTIALNAME = $(PROJECT_TAG)-confidential.img INITRDNAME = $(PROJECT_TAG)-initrd.img INITRDCONFIDENTIALNAME = $(PROJECT_TAG)-initrd-confidential.img +# CoCo guest-components extension (attestation-agent, confidential-data-hub, ...), +# cold-plugged and mounted at /run/kata-extensions/coco on confidential guests. +COCOIMAGENAME = $(PROJECT_TAG)-coco-extension.img IMAGENAME_NV = $(PROJECT_TAG)-nvidia-gpu.img IMAGENAME_CONFIDENTIAL_NV = $(PROJECT_TAG)-nvidia-gpu-confidential.img -# Driver-agnostic NVIDIA base image (no GPU userspace), booted by qemu-nvidia-cpu. +# Driver-agnostic NVIDIA base image (no GPU userspace), booted by qemu-nvidia-cpu +# and the composable NVIDIA GPU configs. IMAGENAME_NV_BASE = $(PROJECT_TAG)-nvidia.img +# Driver-versioned GPU extension (GPU userspace), cold-plugged and mounted at +# /run/kata-extensions/gpu on GPU guests. +IMAGENAME_NV_EXTENSION = $(PROJECT_TAG)-nvidia-gpu-extension.img TARGET = $(BIN_PREFIX)-runtime RUNTIME_OUTPUT = $(CURDIR)/$(TARGET) @@ -131,12 +138,20 @@ KERNELDIR := $(PKGDATADIR) IMAGEPATH := $(PKGDATADIR)/$(IMAGENAME) IMAGECONFIDENTIALPATH := $(PKGDATADIR)/$(IMAGECONFIDENTIALNAME) +COCOIMAGEPATH := $(PKGDATADIR)/$(COCOIMAGENAME) +# dm-verity params for the CoCo extension (root_hash_coco-extension.txt); filled +# in by the shim-v2 build from the emitted root-hash file. +COCOVERITYPARAMS := INITRDPATH := $(PKGDATADIR)/$(INITRDNAME) INITRDCONFIDENTIALPATH := $(PKGDATADIR)/$(INITRDCONFIDENTIALNAME) IMAGEPATH_NV := $(PKGDATADIR)/$(IMAGENAME_NV) IMAGEPATH_CONFIDENTIAL_NV := $(PKGDATADIR)/$(IMAGENAME_CONFIDENTIAL_NV) IMAGEPATH_NV_BASE := $(PKGDATADIR)/$(IMAGENAME_NV_BASE) +IMAGEPATH_NV_EXTENSION := $(PKGDATADIR)/$(IMAGENAME_NV_EXTENSION) +# dm-verity params for the gpu extension (root_hash_nvidia-gpu-extension.txt); +# filled in by the shim-v2 build from the emitted root-hash file. +NVIDIAGPUEXTENSIONVERITYPARAMS ?= ROOTFSTYPE_EXT4 := \"ext4\" ROOTFSTYPE_XFS := \"xfs\" @@ -169,8 +184,8 @@ KERNELVERITYPARAMS ?= "" KERNELVERITYPARAMS_NV ?= "" KERNELVERITYPARAMS_CONFIDENTIAL_NV ?= "" # dm-verity params for the driver-agnostic nvidia base image (qemu-nvidia-cpu). -# The Go runtime maps KERNELVERITYPARAMS_NV to the monolithic nvidia-gpu hash, -# so qemu-nvidia-cpu needs its own var pointing at the "nvidia" base root hash. +# Both runtimes now map KERNELVERITYPARAMS_NV to the "nvidia" base root hash, so +# this mirrors it; kept as its own var to avoid churning the nvidia-cpu config. KERNELVERITYPARAMS_NV_BASE ?= "" # Name of default configuration file the runtime will use. @@ -676,6 +691,9 @@ USER_VARS += IMAGECONFIDENTIALNAME USER_VARS += IMAGEPATH USER_VARS += IMAGEPATH_CLH_AZURE USER_VARS += IMAGECONFIDENTIALPATH +USER_VARS += COCOIMAGENAME +USER_VARS += COCOIMAGEPATH +USER_VARS += COCOVERITYPARAMS USER_VARS += INITRDNAME USER_VARS += INITRDCONFIDENTIALNAME USER_VARS += INITRDPATH @@ -686,6 +704,9 @@ USER_VARS += IMAGEPATH_NV USER_VARS += IMAGEPATH_CONFIDENTIAL_NV USER_VARS += IMAGENAME_NV_BASE USER_VARS += IMAGEPATH_NV_BASE +USER_VARS += IMAGENAME_NV_EXTENSION +USER_VARS += IMAGEPATH_NV_EXTENSION +USER_VARS += NVIDIAGPUEXTENSIONVERITYPARAMS USER_VARS += KERNELNAME_NV USER_VARS += KERNELPATH_NV USER_VARS += KERNELNAME_CONFIDENTIAL_NV diff --git a/src/runtime/config/configuration-qemu-coco-dev.toml.in b/src/runtime/config/configuration-qemu-coco-dev.toml.in index 506fb43f5d..118829d46b 100644 --- a/src/runtime/config/configuration-qemu-coco-dev.toml.in +++ b/src/runtime/config/configuration-qemu-coco-dev.toml.in @@ -15,7 +15,7 @@ [hypervisor.qemu] path = "@QEMUPATH@" kernel = "@KERNELCONFIDENTIALPATH@" -image = "@IMAGECONFIDENTIALPATH@" +image = "@IMAGEPATH@" machine_type = "@MACHINETYPE@" # rootfs filesystem type: @@ -776,3 +776,11 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@" # cold_plug_vfio != no_port AND pod_resource_api_sock != "" => kubelet # based cold plug. pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK@" + +# Confidential guests boot the measured base image plus the CoCo guest-components +# extension (attestation-agent, confidential-data-hub, ...), cold-plugged and +# mounted at /run/kata-extensions/coco. +[[hypervisor.qemu.guest_extension_images]] +name = "coco" +path = "@COCOIMAGEPATH@" +verity_params = "@COCOVERITYPARAMS@" diff --git a/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in b/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in index 958e5e029a..483e6fccaf 100644 --- a/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in +++ b/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in @@ -15,7 +15,7 @@ [hypervisor.qemu] path = "@QEMUSNPPATH@" kernel = "@KERNELPATH_CONFIDENTIAL_NV@" -image = "@IMAGEPATH_CONFIDENTIAL_NV@" +image = "@IMAGEPATH_NV_BASE@" machine_type = "@MACHINETYPE@" @@ -95,7 +95,7 @@ kernel_params = "@KERNELPARAMS_CONFIDENTIAL_NV@" # Optional dm-verity parameters (comma-separated key=value list): # root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=... # These are used by the runtime to assemble dm-verity kernel params. -kernel_verity_params = "@KERNELVERITYPARAMS_CONFIDENTIAL_NV@" +kernel_verity_params = "@KERNELVERITYPARAMS_NV@" # Path to the firmware. # If you want that qemu uses the default firmware leave this option empty @@ -822,3 +822,17 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@" # cold_plug_vfio != no_port AND pod_resource_api_sock != "" => kubelet # based cold plug. pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@" + +# GPU guests boot the driver-agnostic nvidia base image plus the driver-versioned +# GPU extension (GPU userspace), cold-plugged and mounted at /run/kata-extensions/gpu. +[[hypervisor.qemu.guest_extension_images]] +name = "gpu" +path = "@IMAGEPATH_NV_EXTENSION@" +verity_params = "@NVIDIAGPUEXTENSIONVERITYPARAMS@" + +# Confidential GPU guests additionally cold-plug the CoCo guest-components extension +# (attestation-agent, confidential-data-hub, ...) mounted at /run/kata-extensions/coco. +[[hypervisor.qemu.guest_extension_images]] +name = "coco" +path = "@COCOIMAGEPATH@" +verity_params = "@COCOVERITYPARAMS@" diff --git a/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in b/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in index 9ca0f44837..b6c27a6cd0 100644 --- a/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in +++ b/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in @@ -14,7 +14,7 @@ [hypervisor.qemu] path = "@QEMUTDXEXPERIMENTALPATH@" kernel = "@KERNELPATH_CONFIDENTIAL_NV@" -image = "@IMAGEPATH_CONFIDENTIAL_NV@" +image = "@IMAGEPATH_NV_BASE@" machine_type = "@MACHINETYPE@" tdx_quote_generation_service_socket_port = @QEMUTDXQUOTEGENERATIONSERVICESOCKETPORT@ @@ -72,7 +72,7 @@ kernel_params = "@KERNELPARAMS_CONFIDENTIAL_NV@" # Optional dm-verity parameters (comma-separated key=value list): # root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=... # These are used by the runtime to assemble dm-verity kernel params. -kernel_verity_params = "@KERNELVERITYPARAMS_CONFIDENTIAL_NV@" +kernel_verity_params = "@KERNELVERITYPARAMS_NV@" # Path to the firmware. # If you want that qemu uses the default firmware leave this option empty @@ -799,3 +799,17 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@" # cold_plug_vfio != no_port AND pod_resource_api_sock != "" => kubelet # based cold plug. pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@" + +# GPU guests boot the driver-agnostic nvidia base image plus the driver-versioned +# GPU extension (GPU userspace), cold-plugged and mounted at /run/kata-extensions/gpu. +[[hypervisor.qemu.guest_extension_images]] +name = "gpu" +path = "@IMAGEPATH_NV_EXTENSION@" +verity_params = "@NVIDIAGPUEXTENSIONVERITYPARAMS@" + +# Confidential GPU guests additionally cold-plug the CoCo guest-components extension +# (attestation-agent, confidential-data-hub, ...) mounted at /run/kata-extensions/coco. +[[hypervisor.qemu.guest_extension_images]] +name = "coco" +path = "@COCOIMAGEPATH@" +verity_params = "@COCOVERITYPARAMS@" diff --git a/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in b/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in index 1201bb7919..ceab5f00ba 100644 --- a/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in +++ b/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in @@ -14,7 +14,7 @@ [hypervisor.qemu] path = "@QEMUPATH@" kernel = "@KERNELPATH_NV@" -image = "@IMAGEPATH_NV@" +image = "@IMAGEPATH_NV_BASE@" machine_type = "@MACHINETYPE@" # rootfs filesystem type: @@ -796,3 +796,10 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@" # cold_plug_vfio != no_port AND pod_resource_api_sock != "" => kubelet # based cold plug. pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@" + +# GPU guests boot the driver-agnostic nvidia base image plus the driver-versioned +# GPU extension (GPU userspace), cold-plugged and mounted at /run/kata-extensions/gpu. +[[hypervisor.qemu.guest_extension_images]] +name = "gpu" +path = "@IMAGEPATH_NV_EXTENSION@" +verity_params = "@NVIDIAGPUEXTENSIONVERITYPARAMS@" diff --git a/src/runtime/config/configuration-qemu-snp.toml.in b/src/runtime/config/configuration-qemu-snp.toml.in index 39b57798c9..c1a3842679 100644 --- a/src/runtime/config/configuration-qemu-snp.toml.in +++ b/src/runtime/config/configuration-qemu-snp.toml.in @@ -15,7 +15,7 @@ [hypervisor.qemu] path = "@QEMUPATH@" kernel = "@KERNELCONFIDENTIALPATH@" -image = "@IMAGECONFIDENTIALPATH@" +image = "@IMAGEPATH@" machine_type = "@MACHINETYPE@" # rootfs filesystem type: @@ -784,3 +784,11 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@" # cold_plug_vfio != no_port AND pod_resource_api_sock != "" => kubelet # based cold plug. pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK@" + +# Confidential guests boot the measured base image plus the CoCo guest-components +# extension (attestation-agent, confidential-data-hub, ...), cold-plugged and +# mounted at /run/kata-extensions/coco. +[[hypervisor.qemu.guest_extension_images]] +name = "coco" +path = "@COCOIMAGEPATH@" +verity_params = "@COCOVERITYPARAMS@" diff --git a/src/runtime/config/configuration-qemu-tdx.toml.in b/src/runtime/config/configuration-qemu-tdx.toml.in index da72270fcf..e753123d8d 100644 --- a/src/runtime/config/configuration-qemu-tdx.toml.in +++ b/src/runtime/config/configuration-qemu-tdx.toml.in @@ -14,7 +14,7 @@ [hypervisor.qemu] path = "@QEMUPATH@" kernel = "@KERNELCONFIDENTIALPATH@" -image = "@IMAGECONFIDENTIALPATH@" +image = "@IMAGEPATH@" machine_type = "@MACHINETYPE@" tdx_quote_generation_service_socket_port = @QEMUTDXQUOTEGENERATIONSERVICESOCKETPORT@ @@ -761,3 +761,11 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@" # cold_plug_vfio != no_port AND pod_resource_api_sock != "" => kubelet # based cold plug. pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK@" + +# Confidential guests boot the measured base image plus the CoCo guest-components +# extension (attestation-agent, confidential-data-hub, ...), cold-plugged and +# mounted at /run/kata-extensions/coco. +[[hypervisor.qemu.guest_extension_images]] +name = "coco" +path = "@COCOIMAGEPATH@" +verity_params = "@COCOVERITYPARAMS@"