diff --git a/.github/workflows/run-k8s-tests-on-free-runner.yaml b/.github/workflows/run-k8s-tests-on-free-runner.yaml index 153ec53b5b..0d79d463e3 100644 --- a/.github/workflows/run-k8s-tests-on-free-runner.yaml +++ b/.github/workflows/run-k8s-tests-on-free-runner.yaml @@ -53,7 +53,7 @@ jobs: { vmm: clh-runtime-rs, containerd_version: latest }, { vmm: clh-runtime-rs, containerd_version: minimum }, { vmm: qemu-nvidia-cpu, containerd_version: latest }, - { vmm: qemu-nvidia-cpu-runtime-rs, containerd_version: latest }, + { vmm: qemu-nvidia-cpu-runtime-rs, containerd_version: latest, snapshotter: erofs, erofs_mode: memory, erofs_dmverity: dmverity }, ] concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-free-runner-${{ toJSON(matrix) }} @@ -74,6 +74,7 @@ jobs: CONTAINER_ENGINE_VERSION: ${{ matrix.environment.containerd_version }} SNAPSHOTTER: ${{ matrix.environment.snapshotter }} EROFS_SNAPSHOTTER_MODE: ${{ matrix.environment.erofs_mode }} + EROFS_DMVERITY: ${{ matrix.environment.erofs_dmverity }} EROFS_MERGE_MODE: ${{ matrix.environment.erofs_merge_mode }} GH_TOKEN: ${{ github.token }} steps: diff --git a/.github/workflows/run-k8s-tests-on-nvidia-cpu-arm64.yaml b/.github/workflows/run-k8s-tests-on-nvidia-cpu-arm64.yaml index 0c67b51966..98847322b4 100644 --- a/.github/workflows/run-k8s-tests-on-nvidia-cpu-arm64.yaml +++ b/.github/workflows/run-k8s-tests-on-nvidia-cpu-arm64.yaml @@ -38,8 +38,8 @@ jobs: fail-fast: false matrix: environment: [ - { name: nvidia-cpu, vmm: qemu-nvidia-cpu, runner: arm64-nvidia-gh200 }, - { name: nvidia-cpu-runtime-rs, vmm: qemu-nvidia-cpu-runtime-rs, runner: arm64-nvidia-gh200 }, + { name: nvidia-cpu, vmm: qemu-nvidia-cpu, runner: arm64-nvidia-gh200, snapshotter: "" }, + { name: nvidia-cpu-runtime-rs, vmm: qemu-nvidia-cpu-runtime-rs, runner: arm64-nvidia-gh200, snapshotter: "erofs" }, ] concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-${{ toJSON(matrix) }} @@ -54,6 +54,9 @@ jobs: KUBERNETES: kubeadm K8S_TEST_HOST_TYPE: baremetal TARGET_ARCH: aarch64 + SNAPSHOTTER: ${{ matrix.environment.snapshotter }} + EROFS_SNAPSHOTTER_MODE: ${{ matrix.environment.snapshotter == 'erofs' && 'memory' || '' }} + EROFS_DMVERITY: ${{ matrix.environment.snapshotter == 'erofs' && 'dmverity' || '' }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 613b85bd79..5250633d4e 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -149,6 +149,12 @@ ifeq ($(ARCH), aarch64) FIRMWAREPATH_NV := $(PREFIXDEPS)/share/$(EDK2_NAME)/AAVMF_CODE.fd endif +# The CPU-only NVIDIA runtime class does not need UEFI firmware (no OVMF/AAVMF). +# Use dedicated empty variables so it does not inherit the generic firmware +# paths; parity with src/runtime/Makefile FIRMWAREPATH_NV_CPU. +FIRMWAREPATH_NV_CPU := +FIRMWAREVOLUMEPATH_NV_CPU := + KERNELVERITYPARAMS ?= "" # TDX @@ -204,6 +210,7 @@ DEFENABLEANNOTATIONS_COCO := [\"enable_iommu\", \"kernel_params\", \"kernel_veri DEFDISABLEGUESTSECCOMP := true DEFEMPTYDIRMODE := shared-fs DEFEMPTYDIRMODE_COCO := block-encrypted +DEFEMPTYDIRMODE_NV := block-plain ##VAR DEFAULTEXPFEATURES=[features] Default experimental features enabled DEFAULTEXPFEATURES := [] DEFDISABLESELINUX := false @@ -703,7 +710,9 @@ USER_VARS += KERNELPATH USER_VARS += KERNELVIRTIOFSPATH USER_VARS += FIRMWAREPATH USER_VARS += FIRMWAREPATH_NV +USER_VARS += FIRMWAREPATH_NV_CPU USER_VARS += FIRMWAREVOLUMEPATH +USER_VARS += FIRMWAREVOLUMEPATH_NV_CPU USER_VARS += MACHINEACCELERATORS USER_VARS += CPUFEATURES USER_VARS += DEFMACHINETYPE_CLH @@ -751,6 +760,7 @@ USER_VARS += DEFNETWORKMODEL_QEMU USER_VARS += DEFNETWORKMODEL_FC USER_VARS += DEFEMPTYDIRMODE USER_VARS += DEFEMPTYDIRMODE_COCO +USER_VARS += DEFEMPTYDIRMODE_NV USER_VARS += DEFDISABLEGUESTSECCOMP USER_VARS += DEFDISABLESELINUX USER_VARS += DEFDISABLEGUESTSELINUX diff --git a/src/runtime-rs/config/configuration-qemu-nvidia-cpu-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-nvidia-cpu-runtime-rs.toml.in index 5e7318109d..899e469169 100644 --- a/src/runtime-rs/config/configuration-qemu-nvidia-cpu-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-nvidia-cpu-runtime-rs.toml.in @@ -65,13 +65,13 @@ kernel_verity_params = "@KERNELVERITYPARAMS_NV@" # Path to the firmware. # If you want that qemu uses the default firmware leave this option empty -firmware = "@FIRMWAREPATH@" +firmware = "@FIRMWAREPATH_NV_CPU@" # Path to the firmware volume. # firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables # as configuration) and FIRMWARE_CODE.fd (UEFI program image). UEFI variables # can be customized per each user while UEFI code is kept same. -firmware_volume = "@FIRMWAREVOLUMEPATH@" +firmware_volume = "@FIRMWAREVOLUMEPATH_NV_CPU@" # Machine accelerators # comma-separated list of machine accelerators to pass to the hypervisor. @@ -195,7 +195,7 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # - virtio-fs (default) # - virtio-fs-nydus # - none -shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@" +shared_fs = "none" # Path to vhost-user-fs daemon. virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" @@ -808,7 +808,7 @@ vfio_mode = "@DEFVFIOMODE_NV@" # - block-plain # Plugs a block device to be mounted directly in the guest. # -emptydir_mode = "@DEFEMPTYDIRMODE@" +emptydir_mode = "@DEFEMPTYDIRMODE_NV@" # Enabled experimental feature list, format: ["a", "b"]. # Experimental features are features not stable enough for production, diff --git a/tests/common.bash b/tests/common.bash index 36f15a72a6..708cbc7ae0 100644 --- a/tests/common.bash +++ b/tests/common.bash @@ -807,6 +807,31 @@ function enabling_hypervisor() { export KATA_CONFIG_PATH="${DEST_KATA_CONFIG}" } +# Docker and nerdctl smoke tests exercise Kata through the default overlayfs +# snapshotter path. Keep NVIDIA runtime-rs on virtio-fs for those tests; the +# shared_fs=none + EROFS snapshotter path is covered by Kubernetes CI instead. +function configure_nvidia_runtime_rs_shared_fs_dropin() { + case "${KATA_HYPERVISOR:-}" in + qemu-nvidia-cpu-runtime-rs) ;; + *) return 0 ;; + esac + + local -r cfg="${KATA_CONFIG_PATH:-}" + [[ -z "${cfg}" || ! -e "${cfg}" ]] && return 0 + + local -r dropin_dir="$(dirname "${cfg}")/config.d" + local -r dropin_path="${dropin_dir}/99-nvidia-runtime-rs-shared-fs.toml" + + info "Configuring NVIDIA runtime-rs shared-fs smoke test via ${dropin_path}" + sudo mkdir -p "${dropin_dir}" + sudo tee "${dropin_path}" >/dev/null <&3 return 0 @@ -128,7 +138,7 @@ wait_for_pod_terminated() { } teardown() { - if ! is_confidential_runtime_class; then + if ! is_shared_fs_none_runtime_class; then return fi diff --git a/tests/integration/nerdctl/gha-run.sh b/tests/integration/nerdctl/gha-run.sh index 5790cae3de..5a4dadde88 100755 --- a/tests/integration/nerdctl/gha-run.sh +++ b/tests/integration/nerdctl/gha-run.sh @@ -101,6 +101,7 @@ function run() { sudo nerdctl network create "${net2}" enabling_hypervisor + configure_nvidia_runtime_rs_shared_fs_dropin if [[ -n "${GITHUB_ENV:-}" ]]; then start_time=$(date '+%Y-%m-%d %H:%M:%S') diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-cpu.values.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-cpu.values.yaml index 991c96cb7b..d7505e55cd 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-cpu.values.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-cpu.values.yaml @@ -15,7 +15,35 @@ debug: false deploymentMode: job snapshotter: - setup: [] + setup: ["erofs"] + # The NVIDIA CPU runtime-rs handler runs with shared_fs=none and uses the + # EROFS snapshotter with memory-backed rw layers and dm-verity for lower + # (image) layer integrity. + erofsSnapshotterMode: "memory" + erofsDmverity: true + +# EROFS fs-verity requires the backing filesystem to support fs-verity, and we +# cannot guarantee that on an arbitrary node. Rather than gate this profile on a +# host feature we do not control, we disable fs-verity so it works on the widest +# possible range of setups; we take that hit deliberately to keep the profile +# generic. Combined with memory-backed EROFS rw layers, ship this as the default +# containerd erofs snapshotter drop-in for the NVIDIA CPU profile so it does not +# need to be injected out-of-band. It is loaded after kata-deploy's generated +# config, so it overrides the built-in enable_fsverity default. +# +# NOTE: This drop-in targets containerd's built-in EROFS snapshotter, which is +# only available on containerd >= 2.2.0 (config version 3, using the +# auto-imported conf.d drop-in mechanism). This is the same minimum kata-deploy +# enforces for the EROFS snapshotter (see check_containerd_erofs_version_support: +# "In order to use erofs-snapshotter containerd must be 2.2.0 or newer"). On +# older containerd releases the erofs snapshotter plugin does not exist and this +# specific "[plugins.'io.containerd.snapshotter.v1.erofs']" key layout will not +# be recognized, so do not expect this profile to work there. +containerd: + userDropIn: | + [plugins.'io.containerd.snapshotter.v1.erofs'] + enable_fsverity = false + default_size = "0" # Enable NVIDIA CPU shims # Disable all shims at once, then enable only the ones we need. @@ -38,12 +66,12 @@ shims: - arm64 allowedHypervisorAnnotations: [] containerd: - snapshotter: "" + snapshotter: "erofs" # Default shim per architecture. defaultShim: - amd64: qemu-nvidia-cpu - arm64: qemu-nvidia-cpu + amd64: qemu-nvidia-cpu-runtime-rs + arm64: qemu-nvidia-cpu-runtime-rs runtimeClasses: enabled: true diff --git a/tools/packaging/kata-deploy/shim-components.json b/tools/packaging/kata-deploy/shim-components.json index 3ef05a82b2..085aa8ed06 100644 --- a/tools/packaging/kata-deploy/shim-components.json +++ b/tools/packaging/kata-deploy/shim-components.json @@ -45,7 +45,7 @@ }, "qemu-nvidia-cpu-runtime-rs": { "x86_64": ["shim-v2-rust", "qemu", "virtiofsd", "kernel-nvidia-gpu", "rootfs-image-nvidia"], - "aarch64": ["shim-v2-rust", "qemu", "virtiofsd", "kernel-nvidia-gpu", "rootfs-image-nvidia", "ovmf"] + "aarch64": ["shim-v2-rust", "qemu", "virtiofsd", "kernel-nvidia-gpu", "rootfs-image-nvidia"] }, "qemu-nvidia-gpu-snp": { "x86_64": ["shim-v2-go", "qemu-snp-experimental", "kernel-nvidia-gpu", "rootfs-image-nvidia-gpu-confidential", "ovmf-sev"]