diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 9d40132f14..c2cfa4c5a7 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -447,12 +447,22 @@ endif KERNELPATH_NV = $(KERNELDIR)/$(KERNELNAME_NV) IMAGENAME_NV = $(PROJECT_TAG)-nvidia-gpu.img IMAGEPATH_NV = $(PKGDATADIR)/$(IMAGENAME_NV) + # Composable NVIDIA stack: driver-agnostic base-nvidia boot image plus the + # driver-versioned gpu addon (cold-plugged and mounted at /run/kata-addons/gpu). + IMAGENAME_NV_BASE = $(PROJECT_TAG)-nvidia-gpu-base.img + IMAGEPATH_NV_BASE = $(PKGDATADIR)/$(IMAGENAME_NV_BASE) + IMAGENAME_NV_ADDON = $(PROJECT_TAG)-nvidia-gpu-addon.img + IMAGEPATH_NV_ADDON = $(PKGDATADIR)/$(IMAGENAME_NV_ADDON) KERNELPARAMS_NV := cgroup_no_v1=all pci=realloc pci=nocrs pci=assign-busses KERNELPARAMS_CONFIDENTIAL_NV = $(KERNELPARAMS_NV) KERNELPARAMS_CONFIDENTIAL_NV += "nvrc.smi.srs=1" + # Verity params for the base-nvidia boot image (root_hash_nvidia-gpu-base.txt) + # and for the gpu addon (root_hash_nvidia-gpu-addon.txt); filled in by the + # shim-v2 build from the emitted root-hash files. KERNELVERITYPARAMS_NV ?= + NVIDIAGPUADDONVERITYPARAMS ?= DEFAULTVCPUS_NV := 1 DEFAULTMEMORY_NV := 8192 DEFAULTTIMEOUT_NV := 1200 @@ -628,6 +638,11 @@ USER_VARS += IMAGECONFIDENTIALNAME USER_VARS += IMAGEPATH USER_VARS += IMAGEPATH_CLH_AZURE USER_VARS += IMAGEPATH_NV +USER_VARS += IMAGENAME_NV_BASE +USER_VARS += IMAGEPATH_NV_BASE +USER_VARS += IMAGENAME_NV_ADDON +USER_VARS += IMAGEPATH_NV_ADDON +USER_VARS += NVIDIAGPUADDONVERITYPARAMS USER_VARS += IMAGECONFIDENTIALPATH USER_VARS += COCOIMAGENAME USER_VARS += COCOIMAGEPATH diff --git a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in index 440803cf33..a551e2a47f 100644 --- a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in @@ -16,7 +16,7 @@ [hypervisor.qemu] path = "@QEMUPATH@" kernel = "@KERNELPATH_NV@" -image = "@IMAGEPATH_NV@" +image = "@IMAGEPATH_NV_BASE@" machine_type = "@MACHINETYPE@" # rootfs filesystem type: @@ -806,3 +806,12 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@" # When set (together with a non-"no-port" cold_plug_vfio), the runtime can cold-plug # devices discovered via the Pod Resources API. Path is typically under kubelet root-dir. pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@" + +# The GPU userspace (driver binaries, libraries, configs, firmware and the +# NVIDIA kernel modules) ships in a separate, driver-versioned addon image that +# is cold-plugged alongside the base-nvidia boot image. NVRC mounts it at +# /run/kata-addons/gpu and resolves the GPU stack from there. +[[hypervisor.qemu.extra_images]] +name = "gpu" +path = "@IMAGEPATH_NV_ADDON@" +verity_params = "@NVIDIAGPUADDONVERITYPARAMS@" diff --git a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-snp-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-snp-runtime-rs.toml.in index d9807edc93..4bf2b5e186 100644 --- a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-snp-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-snp-runtime-rs.toml.in @@ -16,7 +16,7 @@ [hypervisor.qemu] path = "@QEMUSNPPATH@" kernel = "@KERNELPATH_CONFIDENTIAL_NV@" -image = "@IMAGEPATH_CONFIDENTIAL_NV@" +image = "@IMAGEPATH_NV_BASE@" machine_type = "@MACHINETYPE@" # Enable confidential guest support. @@ -102,7 +102,7 @@ kernel_params = "@KERNELPARAMS_CONFIDENTIAL_NV@" # Optional dm-verity parameters (comma-separated key=value list): # root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=... # These are used by the runtime to assemble dm-verity kernel params. -kernel_verity_params = "@KERNELVERITYPARAMS_CONFIDENTIAL_NV@" +kernel_verity_params = "@KERNELVERITYPARAMS_NV@" # Path to the firmware. # If you want that qemu uses the default firmware leave this option empty @@ -738,3 +738,19 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@" # When set (together with a non-"no-port" cold_plug_vfio), the runtime can cold-plug # devices discovered via the Pod Resources API. Path is typically under kubelet root-dir. pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@" + +# The GPU userspace (driver binaries, libraries, configs, firmware and the +# NVIDIA kernel modules) ships in a separate, driver-versioned addon image that +# is cold-plugged alongside the base-nvidia boot image. NVRC mounts it at +# /run/kata-addons/gpu and resolves the GPU stack from there. +[[hypervisor.qemu.extra_images]] +name = "gpu" +path = "@IMAGEPATH_NV_ADDON@" +verity_params = "@NVIDIAGPUADDONVERITYPARAMS@" + +# Confidential guests additionally cold-plug the CoCo guest-components addon +# (attestation-agent, confidential-data-hub, ...) mounted at /run/kata-addons/coco. +[[hypervisor.qemu.extra_images]] +name = "coco" +path = "@COCOIMAGEPATH@" +verity_params = "@COCOVERITYPARAMS@" diff --git a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-tdx-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-tdx-runtime-rs.toml.in index b2fa39fa6e..a8e1f25609 100644 --- a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-tdx-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-tdx-runtime-rs.toml.in @@ -16,7 +16,7 @@ [hypervisor.qemu] path = "@QEMUTDXPATH@" kernel = "@KERNELPATH_CONFIDENTIAL_NV@" -image = "@IMAGEPATH_CONFIDENTIAL_NV@" +image = "@IMAGEPATH_NV_BASE@" machine_type = "@MACHINETYPE@" tdx_quote_generation_service_socket_port = @QEMUTDXQUOTEGENERATIONSERVICESOCKETPORT@ @@ -78,7 +78,7 @@ kernel_params = "@KERNELPARAMS_CONFIDENTIAL_NV@" # Optional dm-verity parameters (comma-separated key=value list): # root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=... # These are used by the runtime to assemble dm-verity kernel params. -kernel_verity_params = "@KERNELVERITYPARAMS_CONFIDENTIAL_NV@" +kernel_verity_params = "@KERNELVERITYPARAMS_NV@" # Path to the firmware. # If you want that qemu uses the default firmware leave this option empty @@ -714,3 +714,19 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@" # When set (together with a non-"no-port" cold_plug_vfio), the runtime can cold-plug # devices discovered via the Pod Resources API. Path is typically under kubelet root-dir. pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@" + +# The GPU userspace (driver binaries, libraries, configs, firmware and the +# NVIDIA kernel modules) ships in a separate, driver-versioned addon image that +# is cold-plugged alongside the base-nvidia boot image. NVRC mounts it at +# /run/kata-addons/gpu and resolves the GPU stack from there. +[[hypervisor.qemu.extra_images]] +name = "gpu" +path = "@IMAGEPATH_NV_ADDON@" +verity_params = "@NVIDIAGPUADDONVERITYPARAMS@" + +# Confidential guests additionally cold-plug the CoCo guest-components addon +# (attestation-agent, confidential-data-hub, ...) mounted at /run/kata-addons/coco. +[[hypervisor.qemu.extra_images]] +name = "coco" +path = "@COCOIMAGEPATH@" +verity_params = "@COCOVERITYPARAMS@" diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 93a034f1c5..c6820db1d2 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -65,6 +65,10 @@ INITRDCONFIDENTIALNAME = $(PROJECT_TAG)-initrd-confidential.img IMAGENAME_NV = $(PROJECT_TAG)-nvidia-gpu.img IMAGENAME_CONFIDENTIAL_NV = $(PROJECT_TAG)-nvidia-gpu-confidential.img +# Composable NVIDIA stack: driver-agnostic base-nvidia boot image plus the +# driver-versioned gpu addon (cold-plugged and mounted at /run/kata-addons/gpu). +IMAGENAME_NV_BASE = $(PROJECT_TAG)-nvidia-gpu-base.img +IMAGENAME_NV_ADDON = $(PROJECT_TAG)-nvidia-gpu-addon.img TARGET = $(BIN_PREFIX)-runtime RUNTIME_OUTPUT = $(CURDIR)/$(TARGET) @@ -137,6 +141,8 @@ INITRDCONFIDENTIALPATH := $(PKGDATADIR)/$(INITRDCONFIDENTIALNAME) IMAGEPATH_NV := $(PKGDATADIR)/$(IMAGENAME_NV) IMAGEPATH_CONFIDENTIAL_NV := $(PKGDATADIR)/$(IMAGENAME_CONFIDENTIAL_NV) +IMAGEPATH_NV_BASE := $(PKGDATADIR)/$(IMAGENAME_NV_BASE) +IMAGEPATH_NV_ADDON := $(PKGDATADIR)/$(IMAGENAME_NV_ADDON) ROOTFSTYPE_EXT4 := \"ext4\" ROOTFSTYPE_XFS := \"xfs\" @@ -164,6 +170,9 @@ FIRMWARESNPPATH_NV := $(FIRMWARESNPPATH) KERNELVERITYPARAMS ?= "" KERNELVERITYPARAMS_NV ?= "" KERNELVERITYPARAMS_CONFIDENTIAL_NV ?= "" +# Verity params for the gpu addon (root_hash_nvidia-gpu-addon.txt); the +# base-nvidia boot image reuses KERNELVERITYPARAMS_NV. +NVIDIAGPUADDONVERITYPARAMS ?= "" # Name of default configuration file the runtime will use. CONFIG_FILE = configuration.toml @@ -695,6 +704,11 @@ USER_VARS += IMAGENAME_NV USER_VARS += IMAGENAME_CONFIDENTIAL_NV USER_VARS += IMAGEPATH_NV USER_VARS += IMAGEPATH_CONFIDENTIAL_NV +USER_VARS += IMAGENAME_NV_BASE +USER_VARS += IMAGEPATH_NV_BASE +USER_VARS += IMAGENAME_NV_ADDON +USER_VARS += IMAGEPATH_NV_ADDON +USER_VARS += NVIDIAGPUADDONVERITYPARAMS USER_VARS += KERNELNAME_NV USER_VARS += KERNELPATH_NV USER_VARS += KERNELNAME_CONFIDENTIAL_NV diff --git a/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in b/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in index 0fcef210cc..039fee857c 100644 --- a/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in +++ b/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in @@ -15,7 +15,7 @@ [hypervisor.qemu] path = "@QEMUSNPPATH@" kernel = "@KERNELPATH_CONFIDENTIAL_NV@" -image = "@IMAGEPATH_CONFIDENTIAL_NV@" +image = "@IMAGEPATH_NV_BASE@" machine_type = "@MACHINETYPE@" @@ -95,7 +95,7 @@ kernel_params = "@KERNELPARAMS_CONFIDENTIAL_NV@" # Optional dm-verity parameters (comma-separated key=value list): # root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=... # These are used by the runtime to assemble dm-verity kernel params. -kernel_verity_params = "@KERNELVERITYPARAMS_CONFIDENTIAL_NV@" +kernel_verity_params = "@KERNELVERITYPARAMS_NV@" # Path to the firmware. # If you want that qemu uses the default firmware leave this option empty @@ -796,3 +796,19 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@" # cold_plug_vfio != no_port AND pod_resource_api_sock != "" => kubelet # based cold plug. pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@" + +# The GPU userspace (driver binaries, libraries, configs, firmware and the +# NVIDIA kernel modules) ships in a separate, driver-versioned addon image that +# is cold-plugged alongside the base-nvidia boot image. NVRC mounts it at +# /run/kata-addons/gpu and resolves the GPU stack from there. +[[hypervisor.qemu.extra_images]] +name = "gpu" +path = "@IMAGEPATH_NV_ADDON@" +verity_params = "@NVIDIAGPUADDONVERITYPARAMS@" + +# Confidential guests additionally cold-plug the CoCo guest-components addon +# (attestation-agent, confidential-data-hub, ...) mounted at /run/kata-addons/coco. +[[hypervisor.qemu.extra_images]] +name = "coco" +path = "@COCOIMAGEPATH@" +verity_params = "@COCOVERITYPARAMS@" diff --git a/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in b/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in index c0c481c6f9..c510beeb96 100644 --- a/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in +++ b/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in @@ -14,7 +14,7 @@ [hypervisor.qemu] path = "@QEMUTDXEXPERIMENTALPATH@" kernel = "@KERNELPATH_CONFIDENTIAL_NV@" -image = "@IMAGEPATH_CONFIDENTIAL_NV@" +image = "@IMAGEPATH_NV_BASE@" machine_type = "@MACHINETYPE@" tdx_quote_generation_service_socket_port = @QEMUTDXQUOTEGENERATIONSERVICESOCKETPORT@ @@ -72,7 +72,7 @@ kernel_params = "@KERNELPARAMS_CONFIDENTIAL_NV@" # Optional dm-verity parameters (comma-separated key=value list): # root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=... # These are used by the runtime to assemble dm-verity kernel params. -kernel_verity_params = "@KERNELVERITYPARAMS_CONFIDENTIAL_NV@" +kernel_verity_params = "@KERNELVERITYPARAMS_NV@" # Path to the firmware. # If you want that qemu uses the default firmware leave this option empty @@ -773,3 +773,19 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@" # cold_plug_vfio != no_port AND pod_resource_api_sock != "" => kubelet # based cold plug. pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@" + +# The GPU userspace (driver binaries, libraries, configs, firmware and the +# NVIDIA kernel modules) ships in a separate, driver-versioned addon image that +# is cold-plugged alongside the base-nvidia boot image. NVRC mounts it at +# /run/kata-addons/gpu and resolves the GPU stack from there. +[[hypervisor.qemu.extra_images]] +name = "gpu" +path = "@IMAGEPATH_NV_ADDON@" +verity_params = "@NVIDIAGPUADDONVERITYPARAMS@" + +# Confidential guests additionally cold-plug the CoCo guest-components addon +# (attestation-agent, confidential-data-hub, ...) mounted at /run/kata-addons/coco. +[[hypervisor.qemu.extra_images]] +name = "coco" +path = "@COCOIMAGEPATH@" +verity_params = "@COCOVERITYPARAMS@" diff --git a/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in b/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in index b9c45606b3..064baf7efe 100644 --- a/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in +++ b/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in @@ -14,7 +14,7 @@ [hypervisor.qemu] path = "@QEMUPATH@" kernel = "@KERNELPATH_NV@" -image = "@IMAGEPATH_NV@" +image = "@IMAGEPATH_NV_BASE@" machine_type = "@MACHINETYPE@" # rootfs filesystem type: @@ -770,3 +770,12 @@ kubelet_root_dir = "@DEFKUBELETROOTDIR@" # cold_plug_vfio != no_port AND pod_resource_api_sock != "" => kubelet # based cold plug. pod_resource_api_sock = "@DEFPODRESOURCEAPISOCK_NV@" + +# The GPU userspace (driver binaries, libraries, configs, firmware and the +# NVIDIA kernel modules) ships in a separate, driver-versioned addon image that +# is cold-plugged alongside the base-nvidia boot image. NVRC mounts it at +# /run/kata-addons/gpu and resolves the GPU stack from there. +[[hypervisor.qemu.extra_images]] +name = "gpu" +path = "@IMAGEPATH_NV_ADDON@" +verity_params = "@NVIDIAGPUADDONVERITYPARAMS@" diff --git a/tools/packaging/static-build/shim-v2/build.sh b/tools/packaging/static-build/shim-v2/build.sh index a9c5e6fd2f..75de5ff6f9 100755 --- a/tools/packaging/static-build/shim-v2/build.sh +++ b/tools/packaging/static-build/shim-v2/build.sh @@ -44,20 +44,25 @@ esac # Variants (targets) that build a measured rootfs as of now are: # - rootfs-image (the base image, measured; root hash labelled "base") # - rootfs-image-coco-addon -# - rootfs-image-nvidia-gpu -# - rootfs-image-nvidia-gpu-confidential +# - rootfs-image-nvidia-gpu-base (the driver-agnostic NVIDIA boot image) +# - rootfs-image-nvidia-gpu-addon (the driver-versioned gpu addon) # -# The base image is shared between non-confidential and confidential guests. -# Only confidential configurations enforce its dm-verity hash via -# @KERNELVERITYPARAMS@; non-confidential guests boot the data partition directly. +# The base images (generic + base-nvidia) are shared between non-confidential +# and confidential guests. Only confidential configurations enforce the generic +# base dm-verity hash via @KERNELVERITYPARAMS@; the NVIDIA stack always boots +# measured, so both the non-confidential and confidential NVIDIA configs enforce +# @KERNELVERITYPARAMS_NV@ (base-nvidia) and @NVIDIAGPUADDONVERITYPARAMS@ (gpu addon). +# +# The monolithic nvidia-gpu / nvidia-gpu-confidential images are still built and +# shipped during the transition, but no generated config references their hashes. # # shellcheck disable=SC2154 root_hash_dir="${repo_root_dir}/tools/packaging/kata-deploy/local-build/build" verity_variants=( "base:KERNELVERITYPARAMS" "coco-addon:COCOVERITYPARAMS" - "nvidia-gpu:KERNELVERITYPARAMS_NV" - "nvidia-gpu-confidential:KERNELVERITYPARAMS_CONFIDENTIAL_NV" + "nvidia-gpu-base:KERNELVERITYPARAMS_NV" + "nvidia-gpu-addon:NVIDIAGPUADDONVERITYPARAMS" ) for entry in "${verity_variants[@]}"; do variant="${entry%%:*}"