diff --git a/src/runtime-rs/crates/resource/src/cgroups/resource_inner.rs b/src/runtime-rs/crates/resource/src/cgroups/resource_inner.rs index 848844c4c8..5dacbe45ec 100644 --- a/src/runtime-rs/crates/resource/src/cgroups/resource_inner.rs +++ b/src/runtime-rs/crates/resource/src/cgroups/resource_inner.rs @@ -72,6 +72,38 @@ impl CgroupsResourceInner { } } + /// Synchronously write the runtime's own pid into the sandbox cgroup's + /// `cgroup.procs` (cgroup v2 only; a no-op otherwise). + /// + /// `add_proc` already places the runtime, but on the systemd driver that + /// is an asynchronous call: the runtime may fork virtiofsd and the + /// VMM before systemd has moved it, so those children inherit its original + /// cgroup (e.g. `system.slice/containerd.service`). Under cgroup v2 + /// first-touch accounting the guest RAM is then charged there, not to the + /// pod. A direct write is synchronous, so children forked afterwards + /// inherit the sandbox cgroup. + fn place_runtime_in_sandbox_cgroup_v2_sync(cgroup: &CgroupManager) -> Result<()> { + if !cgroup.v2() { + return Ok(()); + } + let dir = cgroup + .cgroup_path(None) + .context("resolve sandbox cgroup path for runtime placement")?; + let procs_path = format!("{}/cgroup.procs", dir.trim_end_matches('/')); + let pid = process::id(); + let mut file = std::fs::OpenOptions::new() + .write(true) + .open(&procs_path) + .with_context(|| format!("open sandbox cgroup.procs {procs_path}"))?; + std::io::Write::write_all(&mut file, format!("{pid}\n").as_bytes()) + .with_context(|| format!("move runtime pid {pid} into sandbox cgroup {procs_path}"))?; + info!( + sl!(), + "synchronously placed runtime (pid {}) into sandbox cgroup: {}", pid, procs_path + ); + Ok(()) + } + /// Create cgroup managers according to the cgroup configuration. /// /// # Returns @@ -132,6 +164,10 @@ impl CgroupsResourceInner { pid, "add runtime to sandbox cgroup", )?; + // The systemd add_proc above is asynchronous; make sure we have + // really joined the sandbox cgroup before forking any child. + Self::place_runtime_in_sandbox_cgroup_v2_sync(&sandbox_cgroup) + .context("synchronously place runtime in sandbox cgroup")?; } Ok(Self { diff --git a/tests/integration/kubernetes/k8s-sandbox-cgroup-placement.bats b/tests/integration/kubernetes/k8s-sandbox-cgroup-placement.bats new file mode 100644 index 0000000000..c24763c243 --- /dev/null +++ b/tests/integration/kubernetes/k8s-sandbox-cgroup-placement.bats @@ -0,0 +1,100 @@ +#!/usr/bin/env bats +# +# Copyright (c) 2026 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# Under sandbox_cgroup_only on a cgroup v2 node, the runtime must place the +# shim, the VMM and virtiofsd in the same pod (kubepods) cgroup. The +# regression this guards leaves the VMM and virtiofsd in +# system.slice/containerd.service while only the shim reaches kubepods. Runs +# on any hypervisor with a separate VMM process; all checks use exec_host. + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/../../common.bash" +load "${BATS_TEST_DIRNAME}/tests_common.sh" + +host_is_cgroup_v2() { + exec_host "${node}" "test -f /sys/fs/cgroup/cgroup.controllers" >/dev/null 2>&1 +} + +sandbox_cgroup_only_enabled() { + local cfg + cfg="$(get_kata_runtime_config_file "${node}")" || return 1 + exec_host "${node}" "grep -qE '^\s*sandbox_cgroup_only\s*=\s*true' '${cfg}'" +} + +# Echo the cgroup v2 path (the part after "0::") for a pid on the node. +# Returns non-zero if the path cannot be determined. +host_cgroup_v2_path() { + local pid="${1}" path + # On cgroup v2 the file is a single line "0::", e.g. + # 0::/kubepods.slice/.../cri-containerd-.scope + # Split on "::" and print the path that follows the "0::" prefix. + path="$(exec_host "${node}" "cat /proc/${pid}/cgroup" | awk -F'::' '/^0::/ {print $2}')" + [[ -n "${path}" ]] || return 1 + echo "${path}" +} + +setup() { + node="$(get_one_kata_node)" + has_separate_vmm || skip "test requires a hypervisor with a separate VMM process (KATA_HYPERVISOR=${KATA_HYPERVISOR})" + [[ -n "${node:-}" ]] || skip "no kata-runtime node found" + host_is_cgroup_v2 || skip "test requires the cgroup v2 unified hierarchy on the node" + sandbox_cgroup_only_enabled || skip "test requires sandbox_cgroup_only=true" + + setup_common || die "setup_common failed" + + pod_name="besteffort-test" + + yaml_file="${pod_config_dir}/pod-besteffort.yaml" + set_node "$yaml_file" "$node" + + auto_generate_policy "${pod_config_dir}" "${yaml_file}" +} + +@test "runtime, virtiofsd and VMM share the pod sandbox cgroup" { + kubectl create -f "${yaml_file}" + kubectl wait --for=condition=Ready --timeout="$timeout" pod "$pod_name" + + # Select the shim by pod's UID + local pod_uid + pod_uid="$(kubectl get pod "${pod_name}" -o jsonpath='{.metadata.uid}')" + + local shim_pid vmm_pid vfsd_pid + shim_pid="$(shim_pid_for_pod "${node}" "${pod_uid}")" + + vmm_pid="$(exec_host "${node}" "pgrep -P ${shim_pid} -f 'cloud-hypervisor|qemu-system|firecracker|stratovirt'")" + + vfsd_pid="$(exec_host "${node}" "pgrep -P ${shim_pid} virtiofsd || true")" + + info "pod uid: ${pod_uid}, shim pid: ${shim_pid}, VMM pid: ${vmm_pid}, virtiofsd pid: ${vfsd_pid:-}" + + local shim_cgroup + shim_cgroup="$(host_cgroup_v2_path "${shim_pid}")" + info "shim cgroup: ${shim_cgroup}" + echo "${shim_cgroup}" | grep -q "kubepods" || die "shim is not in a kubepods cgroup: ${shim_cgroup}" + + # The VMM must be co-located with the shim, not left in the spawn cgroup. + local vmm_cgroup + vmm_cgroup="$(host_cgroup_v2_path "${vmm_pid}")" + info "VMM cgroup: ${vmm_cgroup}" + [[ "${vmm_cgroup}" == "${shim_cgroup}" ]] || die "VMM cgroup (${vmm_cgroup}) differs from the shim/pod cgroup (${shim_cgroup}); guest RAM would be mischarged" + + # virtiofsd (when a separate daemon is used) must be co-located too. + if [[ -n "${vfsd_pid}" ]]; then + local vfsd_cgroup + vfsd_cgroup="$(host_cgroup_v2_path "${vfsd_pid}")" + info "virtiofsd cgroup: ${vfsd_cgroup}" + [[ "${vfsd_cgroup}" == "${shim_cgroup}" ]] || die "virtiofsd cgroup (${vfsd_cgroup}) differs from the shim/pod cgroup (${shim_cgroup})" + fi +} + +teardown() { + has_separate_vmm || skip "test requires a hypervisor with a separate VMM process (KATA_HYPERVISOR=${KATA_HYPERVISOR})" + [[ -n "${node:-}" ]] || skip "no kata-runtime node found" + host_is_cgroup_v2 || skip "test requires the cgroup v2 unified hierarchy on the node" + sandbox_cgroup_only_enabled || skip "test requires sandbox_cgroup_only=true" + + teardown_common "${node}" "${node_start_time:-}" +} diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index e84ba426ce..08ba65c204 100755 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -102,6 +102,7 @@ else "k8s-projected-volume.bats" \ "k8s-replication.bats" \ "k8s-sandbox-cgroup.bats" \ + "k8s-sandbox-cgroup-placement.bats" \ "k8s-seccomp.bats" \ "k8s-sysctls.bats" \ "k8s-security-context.bats" \ diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index 782ce842aa..9ce765bdff 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -226,6 +226,27 @@ is_runtime_rs() { [[ "${KATA_HYPERVISOR}" == *-runtime-rs ]] || [[ "${KATA_HYPERVISOR}" == "dragonball" ]] } +# True for hypervisors that run the guest in a separate VMM process (i.e. not +# dragonball, which runs the guest in-process in the shim). +has_separate_vmm() { + [[ "${KATA_HYPERVISOR}" != *dragonball* ]] +} + +# Echo the PID of the Kata shim for the pod with UID $2, on node $1. +# Returns non-zero if no matching shim is found. +# +# The UID is matched in both cgroup-path forms (dashed for cgroupfs, +# underscored for systemd) so a shim from another namespace is not selected. +# The pgrep pattern is bracketed ("[c]ontainerd") so it doesn't also match the +# `bash -c` wrapper exec_host runs, whose command line contains the pattern. +shim_pid_for_pod() { + local node_name="$1" uid_dashed="$2" uid_underscored pid + uid_underscored="${uid_dashed//-/_}" + pid="$(exec_host "${node_name}" "for p in \$(pgrep -f '[c]ontainerd-shim-kata-v2'); do grep -qE 'pod(${uid_dashed}|${uid_underscored})' /proc/\$p/cgroup 2>/dev/null && echo \$p && break; done; true")" + [[ -n "${pid}" ]] || return 1 + echo "${pid}" +} + # Copy the right combination of drop-ins from drop-in-examples/ into # genpolicy-settings.d/. Drop-ins are layered: 10-* for platform base, # 20-* for OCI version and other overlays.