From a2d66aba14dce379cb94c14958a852c76bad675e Mon Sep 17 00:00:00 2001 From: Sudipta Pandit Date: Thu, 16 Jul 2026 15:40:51 +0000 Subject: [PATCH] tests: add support for runtime-rs VM templating Add support for testing the runtime-rs VM template lifecycle. Assisted-By: GitHub Copilot Signed-off-by: Sudipta Pandit --- .../run-k8s-tests-on-free-runner.yaml | 3 + .../kubernetes/k8s-vm-templating.bats | 112 ++++++++++++++---- .../busybox-template.yaml | 2 +- 3 files changed, 94 insertions(+), 23 deletions(-) diff --git a/.github/workflows/run-k8s-tests-on-free-runner.yaml b/.github/workflows/run-k8s-tests-on-free-runner.yaml index 0d79d463e3..c6ebaaeaed 100644 --- a/.github/workflows/run-k8s-tests-on-free-runner.yaml +++ b/.github/workflows/run-k8s-tests-on-free-runner.yaml @@ -51,6 +51,8 @@ jobs: { vmm: qemu-runtime-rs, containerd_version: latest }, { vmm: qemu-runtime-rs, containerd_version: minimum }, { vmm: clh-runtime-rs, containerd_version: latest }, + # TODO: Remove k8s_test_union once Cloud Hypervisor supports multi-layer flat VMDK images. + { vmm: clh-runtime-rs, containerd_version: latest, snapshotter: erofs, erofs_mode: disk, erofs_merge_mode: unmerged, k8s_test_union: k8s-vm-templating.bats }, { vmm: clh-runtime-rs, containerd_version: minimum }, { vmm: qemu-nvidia-cpu, containerd_version: latest }, { vmm: qemu-nvidia-cpu-runtime-rs, containerd_version: latest, snapshotter: erofs, erofs_mode: memory, erofs_dmverity: dmverity }, @@ -70,6 +72,7 @@ jobs: KATA_HYPERVISOR: ${{ matrix.environment.vmm }} KUBERNETES: vanilla K8S_TEST_HOST_TYPE: baremetal-no-attestation + K8S_TEST_UNION: ${{ matrix.environment.k8s_test_union }} CONTAINER_ENGINE: containerd CONTAINER_ENGINE_VERSION: ${{ matrix.environment.containerd_version }} SNAPSHOTTER: ${{ matrix.environment.snapshotter }} diff --git a/tests/integration/kubernetes/k8s-vm-templating.bats b/tests/integration/kubernetes/k8s-vm-templating.bats index c3fb0e01fa..20847bdd13 100644 --- a/tests/integration/kubernetes/k8s-vm-templating.bats +++ b/tests/integration/kubernetes/k8s-vm-templating.bats @@ -13,18 +13,63 @@ load "${BATS_TEST_DIRNAME}/tests_common.sh" # Returns 0 if the current environment supports VM templating, non-zero # otherwise. VM templating is only supported on non-confidential clh/qemu -# hypervisors, and because it uses shared_fs="none" it also requires a -# block-device-based snapshotter (blockfile or erofs). +# hypervisors (including runtime-rs CLH), and because it disables shared +# filesystem support it also requires a block-device-based snapshotter. vm_templating_supported() { - [[ "${KATA_HYPERVISOR}" == "clh" || "${KATA_HYPERVISOR}" == "qemu" ]] || return 1 + [[ "${KATA_HYPERVISOR}" == "clh" || "${KATA_HYPERVISOR}" == "clh-runtime-rs" || "${KATA_HYPERVISOR}" == "qemu" ]] || return 1 is_confidential_runtime_class && return 1 [[ "${SNAPSHOTTER:-}" =~ ^(blockfile|erofs)$ ]] || return 1 return 0 } +factory_command() { + local command="$1" + + case "${KATA_HYPERVISOR}" in + clh-runtime-rs) + exec_host "$node" "nsenter --mount=/proc/1/ns/mnt /opt/kata/bin/kata-ctl factory ${command}" + ;; + *) + exec_host "$node" "nsenter --mount=/proc/1/ns/mnt /opt/kata/bin/kata-runtime --config ${kata_config_path} factory ${command}" + ;; + esac +} + +configure_runtime_rs_factory() { + [[ "${KATA_HYPERVISOR}" == "clh-runtime-rs" ]] || return 0 + + local backup_id + backup_id="$(printf '%s' "${BATS_TEST_TMPDIR}" | sha256sum | cut -c1-12)" + kata_ctl_config_path="/etc/kata-containers/runtime-rs/configuration.toml" + kata_ctl_config_backup_path="${kata_ctl_config_path}.k8s-vm-templating.${backup_id}.bak" + + # Comment out shared_fs because drop-ins cannot remove an existing key. + # Point kata-ctl at that same per-shim config because it has no --config flag. + exec_host "$node" "set -e; \ + mkdir -p /etc/kata-containers/runtime-rs; \ + sed -i '/^[[:space:]]*shared_fs[[:space:]]*=/s/^/# k8s-vm-templating: /' ${kata_config_path}; \ + grep -q '^# k8s-vm-templating: [[:space:]]*shared_fs[[:space:]]*=' ${kata_config_path}; \ + if [ -e ${kata_ctl_config_path} ] || [ -L ${kata_ctl_config_path} ]; then \ + mv ${kata_ctl_config_path} ${kata_ctl_config_backup_path}; \ + fi; \ + ln -s ${kata_config_path} ${kata_ctl_config_path}" +} + +restore_runtime_rs_factory() { + [[ "${KATA_HYPERVISOR}" == "clh-runtime-rs" ]] || return 0 + [[ -n "${kata_ctl_config_path:-}" ]] || return 0 + + exec_host "$node" "set -e; \ + sed -i 's/^# k8s-vm-templating: //' ${kata_config_path}; \ + rm -f ${kata_ctl_config_path}; \ + if [ -e ${kata_ctl_config_backup_path} ] || [ -L ${kata_ctl_config_backup_path} ]; then \ + mv ${kata_ctl_config_backup_path} ${kata_ctl_config_path}; \ + fi" +} + setup() { if ! vm_templating_supported; then - skip "VM templating requires a non-confidential clh/qemu hypervisor and a blockfile/erofs snapshotter (KATA_HYPERVISOR=${KATA_HYPERVISOR}, SNAPSHOTTER=${SNAPSHOTTER:-unset})" + skip "VM templating requires a non-confidential clh/qemu hypervisor (including runtime-rs CLH) and a blockfile/erofs snapshotter (KATA_HYPERVISOR=${KATA_HYPERVISOR}, SNAPSHOTTER=${SNAPSHOTTER:-unset})" fi setup_common || die "setup_common failed" @@ -32,20 +77,28 @@ setup() { # Build a Kata runtime config drop-in that enables VM templating and # disables shared_fs (incompatible with templating). # QEMU VM templating requires an initrd, CLH does not. + local factory_section="factory" + local hypervisor_name="${KATA_HYPERVISOR}" local rootfs_override="" + local shared_fs_override='shared_fs = "none"' if [[ "${KATA_HYPERVISOR}" == "qemu" ]]; then rootfs_override=$'image = ""\ninitrd = "/opt/kata/share/kata-containers/kata-containers-initrd.img"' fi + if [[ "${KATA_HYPERVISOR}" == "clh-runtime-rs" ]]; then + factory_section="hypervisor.clh.factory" + hypervisor_name="clh" + shared_fs_override="" + fi local runtime_config_dropin_file="${BATS_TEST_TMPDIR}/99-k8s-vm-templating.toml" cat > "${runtime_config_dropin_file}" </, whereas a factory-spawned VM - # stores its state under a generated UUID and /run/vc/vm/ is a - # symlink pointing at it (see assignSandbox() in - # src/runtime/virtcontainers/vm.go). Inspect PID 1's mount namespace, where - # the shim creates these entries alongside the template tmpfs. - exec_host "$node" \ - "nsenter --mount=/proc/1/ns/mnt find /run/vc/vm -maxdepth 1 -mindepth 1 -type l ! -name template | grep -q . && echo symlink" \ - | grep -q symlink + if [[ "${KATA_HYPERVISOR}" != "clh-runtime-rs" ]]; then + # Confirm at least one VM sandbox under /run/vc/vm/ is a symlink, which + # proves the factory/template path was used. A non-templated VM creates a + # real directory at /run/vc/vm//, whereas a factory-spawned VM + # stores its state under a generated UUID and /run/vc/vm/ is a + # symlink pointing at it (see assignSandbox() in + # src/runtime/virtcontainers/vm.go). Inspect PID 1's mount namespace, where + # the shim creates these entries alongside the template tmpfs. + exec_host "$node" \ + "nsenter --mount=/proc/1/ns/mnt find /run/vc/vm -maxdepth 1 -mindepth 1 -type l ! -name template | grep -q . && echo symlink" \ + | grep -q symlink + fi } teardown() { @@ -104,8 +170,10 @@ teardown() { # Destroy the VM template and remove the config drop-in on the target node. # factory destroy must run in PID 1's mount namespace to unmount the template # tmpfs that factory init created there (see the @test for details). - exec_host "$node" "nsenter --mount=/proc/1/ns/mnt /opt/kata/bin/kata-runtime --config ${kata_config_path} factory destroy" \ + factory_command destroy \ || echo "Warning: Failed to destroy VM template on node $node" + restore_runtime_rs_factory \ + || echo "Warning: Failed to restore kata-ctl configuration on node $node" remove_kata_runtime_config_dropin_file "$node" "${dropin_path:-}" \ || echo "Warning: Failed to remove Kata runtime config drop-in on node $node" diff --git a/tests/integration/kubernetes/runtimeclass_workloads/busybox-template.yaml b/tests/integration/kubernetes/runtimeclass_workloads/busybox-template.yaml index ec5de5d966..d59781a05a 100644 --- a/tests/integration/kubernetes/runtimeclass_workloads/busybox-template.yaml +++ b/tests/integration/kubernetes/runtimeclass_workloads/busybox-template.yaml @@ -14,7 +14,7 @@ spec: supplementalGroups: [10] containers: - name: CTR_NAME - image: quay.io/prometheus/busybox:latest + image: docker.io/library/busybox:latest@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d command: - sleep - "120"