From 5524f1f276340dbc616a9fbbe10da17f8731fe3b Mon Sep 17 00:00:00 2001 From: Saul Paredes Date: Thu, 9 Jul 2026 09:51:06 -0700 Subject: [PATCH] tests: k8s: verify the sandbox processes share the pod cgroup Add an integration test for the runtime-rs fix that co-locates the shim, the VMM and virtiofsd in the pod's kubepods cgroup under sandbox_cgroup_only on cgroup v2. The regression it guards leaves the VMM and virtiofsd in system.slice/containerd.service while only the shim reaches kubepods. Start a BestEffort Kata pod and, from the host, assert the VMM and virtiofsd share the shim's cgroup and that it is under kubepods. The shim is selected by the pod UID in its cgroup path, so a pod in another namespace is not picked. Skips dragonball (in-process VMM). Generated-By: GitHub Copilot (Claude) Signed-off-by: Saul Paredes --- .../k8s-sandbox-cgroup-placement.bats | 100 ++++++++++++++++++ .../kubernetes/run_kubernetes_tests.sh | 1 + tests/integration/kubernetes/tests_common.sh | 21 ++++ 3 files changed, 122 insertions(+) create mode 100644 tests/integration/kubernetes/k8s-sandbox-cgroup-placement.bats diff --git a/tests/integration/kubernetes/k8s-sandbox-cgroup-placement.bats b/tests/integration/kubernetes/k8s-sandbox-cgroup-placement.bats new file mode 100644 index 0000000000..c24763c243 --- /dev/null +++ b/tests/integration/kubernetes/k8s-sandbox-cgroup-placement.bats @@ -0,0 +1,100 @@ +#!/usr/bin/env bats +# +# Copyright (c) 2026 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# Under sandbox_cgroup_only on a cgroup v2 node, the runtime must place the +# shim, the VMM and virtiofsd in the same pod (kubepods) cgroup. The +# regression this guards leaves the VMM and virtiofsd in +# system.slice/containerd.service while only the shim reaches kubepods. Runs +# on any hypervisor with a separate VMM process; all checks use exec_host. + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/../../common.bash" +load "${BATS_TEST_DIRNAME}/tests_common.sh" + +host_is_cgroup_v2() { + exec_host "${node}" "test -f /sys/fs/cgroup/cgroup.controllers" >/dev/null 2>&1 +} + +sandbox_cgroup_only_enabled() { + local cfg + cfg="$(get_kata_runtime_config_file "${node}")" || return 1 + exec_host "${node}" "grep -qE '^\s*sandbox_cgroup_only\s*=\s*true' '${cfg}'" +} + +# Echo the cgroup v2 path (the part after "0::") for a pid on the node. +# Returns non-zero if the path cannot be determined. +host_cgroup_v2_path() { + local pid="${1}" path + # On cgroup v2 the file is a single line "0::", e.g. + # 0::/kubepods.slice/.../cri-containerd-.scope + # Split on "::" and print the path that follows the "0::" prefix. + path="$(exec_host "${node}" "cat /proc/${pid}/cgroup" | awk -F'::' '/^0::/ {print $2}')" + [[ -n "${path}" ]] || return 1 + echo "${path}" +} + +setup() { + node="$(get_one_kata_node)" + has_separate_vmm || skip "test requires a hypervisor with a separate VMM process (KATA_HYPERVISOR=${KATA_HYPERVISOR})" + [[ -n "${node:-}" ]] || skip "no kata-runtime node found" + host_is_cgroup_v2 || skip "test requires the cgroup v2 unified hierarchy on the node" + sandbox_cgroup_only_enabled || skip "test requires sandbox_cgroup_only=true" + + setup_common || die "setup_common failed" + + pod_name="besteffort-test" + + yaml_file="${pod_config_dir}/pod-besteffort.yaml" + set_node "$yaml_file" "$node" + + auto_generate_policy "${pod_config_dir}" "${yaml_file}" +} + +@test "runtime, virtiofsd and VMM share the pod sandbox cgroup" { + kubectl create -f "${yaml_file}" + kubectl wait --for=condition=Ready --timeout="$timeout" pod "$pod_name" + + # Select the shim by pod's UID + local pod_uid + pod_uid="$(kubectl get pod "${pod_name}" -o jsonpath='{.metadata.uid}')" + + local shim_pid vmm_pid vfsd_pid + shim_pid="$(shim_pid_for_pod "${node}" "${pod_uid}")" + + vmm_pid="$(exec_host "${node}" "pgrep -P ${shim_pid} -f 'cloud-hypervisor|qemu-system|firecracker|stratovirt'")" + + vfsd_pid="$(exec_host "${node}" "pgrep -P ${shim_pid} virtiofsd || true")" + + info "pod uid: ${pod_uid}, shim pid: ${shim_pid}, VMM pid: ${vmm_pid}, virtiofsd pid: ${vfsd_pid:-}" + + local shim_cgroup + shim_cgroup="$(host_cgroup_v2_path "${shim_pid}")" + info "shim cgroup: ${shim_cgroup}" + echo "${shim_cgroup}" | grep -q "kubepods" || die "shim is not in a kubepods cgroup: ${shim_cgroup}" + + # The VMM must be co-located with the shim, not left in the spawn cgroup. + local vmm_cgroup + vmm_cgroup="$(host_cgroup_v2_path "${vmm_pid}")" + info "VMM cgroup: ${vmm_cgroup}" + [[ "${vmm_cgroup}" == "${shim_cgroup}" ]] || die "VMM cgroup (${vmm_cgroup}) differs from the shim/pod cgroup (${shim_cgroup}); guest RAM would be mischarged" + + # virtiofsd (when a separate daemon is used) must be co-located too. + if [[ -n "${vfsd_pid}" ]]; then + local vfsd_cgroup + vfsd_cgroup="$(host_cgroup_v2_path "${vfsd_pid}")" + info "virtiofsd cgroup: ${vfsd_cgroup}" + [[ "${vfsd_cgroup}" == "${shim_cgroup}" ]] || die "virtiofsd cgroup (${vfsd_cgroup}) differs from the shim/pod cgroup (${shim_cgroup})" + fi +} + +teardown() { + has_separate_vmm || skip "test requires a hypervisor with a separate VMM process (KATA_HYPERVISOR=${KATA_HYPERVISOR})" + [[ -n "${node:-}" ]] || skip "no kata-runtime node found" + host_is_cgroup_v2 || skip "test requires the cgroup v2 unified hierarchy on the node" + sandbox_cgroup_only_enabled || skip "test requires sandbox_cgroup_only=true" + + teardown_common "${node}" "${node_start_time:-}" +} diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 9fb3bf3d3b..6e51479520 100755 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -101,6 +101,7 @@ else "k8s-projected-volume.bats" \ "k8s-replication.bats" \ "k8s-sandbox-cgroup.bats" \ + "k8s-sandbox-cgroup-placement.bats" \ "k8s-seccomp.bats" \ "k8s-sysctls.bats" \ "k8s-security-context.bats" \ diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index 782ce842aa..9ce765bdff 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -226,6 +226,27 @@ is_runtime_rs() { [[ "${KATA_HYPERVISOR}" == *-runtime-rs ]] || [[ "${KATA_HYPERVISOR}" == "dragonball" ]] } +# True for hypervisors that run the guest in a separate VMM process (i.e. not +# dragonball, which runs the guest in-process in the shim). +has_separate_vmm() { + [[ "${KATA_HYPERVISOR}" != *dragonball* ]] +} + +# Echo the PID of the Kata shim for the pod with UID $2, on node $1. +# Returns non-zero if no matching shim is found. +# +# The UID is matched in both cgroup-path forms (dashed for cgroupfs, +# underscored for systemd) so a shim from another namespace is not selected. +# The pgrep pattern is bracketed ("[c]ontainerd") so it doesn't also match the +# `bash -c` wrapper exec_host runs, whose command line contains the pattern. +shim_pid_for_pod() { + local node_name="$1" uid_dashed="$2" uid_underscored pid + uid_underscored="${uid_dashed//-/_}" + pid="$(exec_host "${node_name}" "for p in \$(pgrep -f '[c]ontainerd-shim-kata-v2'); do grep -qE 'pod(${uid_dashed}|${uid_underscored})' /proc/\$p/cgroup 2>/dev/null && echo \$p && break; done; true")" + [[ -n "${pid}" ]] || return 1 + echo "${pid}" +} + # Copy the right combination of drop-ins from drop-in-examples/ into # genpolicy-settings.d/. Drop-ins are layered: 10-* for platform base, # 20-* for OCI version and other overlays.