diff --git a/tests/integration/kubernetes/k8s-sandbox-cgroup-placement.bats b/tests/integration/kubernetes/k8s-sandbox-cgroup-placement.bats new file mode 100644 index 0000000000..c24763c243 --- /dev/null +++ b/tests/integration/kubernetes/k8s-sandbox-cgroup-placement.bats @@ -0,0 +1,100 @@ +#!/usr/bin/env bats +# +# Copyright (c) 2026 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# Under sandbox_cgroup_only on a cgroup v2 node, the runtime must place the +# shim, the VMM and virtiofsd in the same pod (kubepods) cgroup. The +# regression this guards leaves the VMM and virtiofsd in +# system.slice/containerd.service while only the shim reaches kubepods. Runs +# on any hypervisor with a separate VMM process; all checks use exec_host. + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/../../common.bash" +load "${BATS_TEST_DIRNAME}/tests_common.sh" + +host_is_cgroup_v2() { + exec_host "${node}" "test -f /sys/fs/cgroup/cgroup.controllers" >/dev/null 2>&1 +} + +sandbox_cgroup_only_enabled() { + local cfg + cfg="$(get_kata_runtime_config_file "${node}")" || return 1 + exec_host "${node}" "grep -qE '^\s*sandbox_cgroup_only\s*=\s*true' '${cfg}'" +} + +# Echo the cgroup v2 path (the part after "0::") for a pid on the node. +# Returns non-zero if the path cannot be determined. +host_cgroup_v2_path() { + local pid="${1}" path + # On cgroup v2 the file is a single line "0::", e.g. + # 0::/kubepods.slice/.../cri-containerd-.scope + # Split on "::" and print the path that follows the "0::" prefix. + path="$(exec_host "${node}" "cat /proc/${pid}/cgroup" | awk -F'::' '/^0::/ {print $2}')" + [[ -n "${path}" ]] || return 1 + echo "${path}" +} + +setup() { + node="$(get_one_kata_node)" + has_separate_vmm || skip "test requires a hypervisor with a separate VMM process (KATA_HYPERVISOR=${KATA_HYPERVISOR})" + [[ -n "${node:-}" ]] || skip "no kata-runtime node found" + host_is_cgroup_v2 || skip "test requires the cgroup v2 unified hierarchy on the node" + sandbox_cgroup_only_enabled || skip "test requires sandbox_cgroup_only=true" + + setup_common || die "setup_common failed" + + pod_name="besteffort-test" + + yaml_file="${pod_config_dir}/pod-besteffort.yaml" + set_node "$yaml_file" "$node" + + auto_generate_policy "${pod_config_dir}" "${yaml_file}" +} + +@test "runtime, virtiofsd and VMM share the pod sandbox cgroup" { + kubectl create -f "${yaml_file}" + kubectl wait --for=condition=Ready --timeout="$timeout" pod "$pod_name" + + # Select the shim by pod's UID + local pod_uid + pod_uid="$(kubectl get pod "${pod_name}" -o jsonpath='{.metadata.uid}')" + + local shim_pid vmm_pid vfsd_pid + shim_pid="$(shim_pid_for_pod "${node}" "${pod_uid}")" + + vmm_pid="$(exec_host "${node}" "pgrep -P ${shim_pid} -f 'cloud-hypervisor|qemu-system|firecracker|stratovirt'")" + + vfsd_pid="$(exec_host "${node}" "pgrep -P ${shim_pid} virtiofsd || true")" + + info "pod uid: ${pod_uid}, shim pid: ${shim_pid}, VMM pid: ${vmm_pid}, virtiofsd pid: ${vfsd_pid:-}" + + local shim_cgroup + shim_cgroup="$(host_cgroup_v2_path "${shim_pid}")" + info "shim cgroup: ${shim_cgroup}" + echo "${shim_cgroup}" | grep -q "kubepods" || die "shim is not in a kubepods cgroup: ${shim_cgroup}" + + # The VMM must be co-located with the shim, not left in the spawn cgroup. + local vmm_cgroup + vmm_cgroup="$(host_cgroup_v2_path "${vmm_pid}")" + info "VMM cgroup: ${vmm_cgroup}" + [[ "${vmm_cgroup}" == "${shim_cgroup}" ]] || die "VMM cgroup (${vmm_cgroup}) differs from the shim/pod cgroup (${shim_cgroup}); guest RAM would be mischarged" + + # virtiofsd (when a separate daemon is used) must be co-located too. + if [[ -n "${vfsd_pid}" ]]; then + local vfsd_cgroup + vfsd_cgroup="$(host_cgroup_v2_path "${vfsd_pid}")" + info "virtiofsd cgroup: ${vfsd_cgroup}" + [[ "${vfsd_cgroup}" == "${shim_cgroup}" ]] || die "virtiofsd cgroup (${vfsd_cgroup}) differs from the shim/pod cgroup (${shim_cgroup})" + fi +} + +teardown() { + has_separate_vmm || skip "test requires a hypervisor with a separate VMM process (KATA_HYPERVISOR=${KATA_HYPERVISOR})" + [[ -n "${node:-}" ]] || skip "no kata-runtime node found" + host_is_cgroup_v2 || skip "test requires the cgroup v2 unified hierarchy on the node" + sandbox_cgroup_only_enabled || skip "test requires sandbox_cgroup_only=true" + + teardown_common "${node}" "${node_start_time:-}" +} diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 9fb3bf3d3b..6e51479520 100755 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -101,6 +101,7 @@ else "k8s-projected-volume.bats" \ "k8s-replication.bats" \ "k8s-sandbox-cgroup.bats" \ + "k8s-sandbox-cgroup-placement.bats" \ "k8s-seccomp.bats" \ "k8s-sysctls.bats" \ "k8s-security-context.bats" \ diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index 782ce842aa..9ce765bdff 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -226,6 +226,27 @@ is_runtime_rs() { [[ "${KATA_HYPERVISOR}" == *-runtime-rs ]] || [[ "${KATA_HYPERVISOR}" == "dragonball" ]] } +# True for hypervisors that run the guest in a separate VMM process (i.e. not +# dragonball, which runs the guest in-process in the shim). +has_separate_vmm() { + [[ "${KATA_HYPERVISOR}" != *dragonball* ]] +} + +# Echo the PID of the Kata shim for the pod with UID $2, on node $1. +# Returns non-zero if no matching shim is found. +# +# The UID is matched in both cgroup-path forms (dashed for cgroupfs, +# underscored for systemd) so a shim from another namespace is not selected. +# The pgrep pattern is bracketed ("[c]ontainerd") so it doesn't also match the +# `bash -c` wrapper exec_host runs, whose command line contains the pattern. +shim_pid_for_pod() { + local node_name="$1" uid_dashed="$2" uid_underscored pid + uid_underscored="${uid_dashed//-/_}" + pid="$(exec_host "${node_name}" "for p in \$(pgrep -f '[c]ontainerd-shim-kata-v2'); do grep -qE 'pod(${uid_dashed}|${uid_underscored})' /proc/\$p/cgroup 2>/dev/null && echo \$p && break; done; true")" + [[ -n "${pid}" ]] || return 1 + echo "${pid}" +} + # Copy the right combination of drop-ins from drop-in-examples/ into # genpolicy-settings.d/. Drop-ins are layered: 10-* for platform base, # 20-* for OCI version and other overlays.