Merge pull request #13338 from microsoft/saul/sync_shim_cgroup

runtime-rs: resource: place the runtime in the sandbox cgroup synchronously

src/runtime-rs/crates/resource/src/cgroups/resource_inner.rs

@@ -72,6 +72,38 @@ impl CgroupsResourceInner {
         }
     }
 
+    /// Synchronously write the runtime's own pid into the sandbox cgroup's
+    /// `cgroup.procs` (cgroup v2 only; a no-op otherwise).
+    ///
+    /// `add_proc` already places the runtime, but on the systemd driver that
+    /// is an asynchronous call: the runtime may fork virtiofsd and the
+    /// VMM before systemd has moved it, so those children inherit its original
+    /// cgroup (e.g. `system.slice/containerd.service`). Under cgroup v2
+    /// first-touch accounting the guest RAM is then charged there, not to the
+    /// pod. A direct write is synchronous, so children forked afterwards
+    /// inherit the sandbox cgroup.
+    fn place_runtime_in_sandbox_cgroup_v2_sync(cgroup: &CgroupManager) -> Result<()> {
+        if !cgroup.v2() {
+            return Ok(());
+        }
+        let dir = cgroup
+            .cgroup_path(None)
+            .context("resolve sandbox cgroup path for runtime placement")?;
+        let procs_path = format!("{}/cgroup.procs", dir.trim_end_matches('/'));
+        let pid = process::id();
+        let mut file = std::fs::OpenOptions::new()
+            .write(true)
+            .open(&procs_path)
+            .with_context(|| format!("open sandbox cgroup.procs {procs_path}"))?;
+        std::io::Write::write_all(&mut file, format!("{pid}\n").as_bytes())
+            .with_context(|| format!("move runtime pid {pid} into sandbox cgroup {procs_path}"))?;
+        info!(
+            sl!(),
+            "synchronously placed runtime (pid {}) into sandbox cgroup: {}", pid, procs_path
+        );
+        Ok(())
+    }
+
     /// Create cgroup managers according to the cgroup configuration.
     ///
     /// # Returns
@@ -132,6 +164,10 @@ impl CgroupsResourceInner {
                 pid,
                 "add runtime to sandbox cgroup",
             )?;
+            // The systemd add_proc above is asynchronous; make sure we have
+            // really joined the sandbox cgroup before forking any child.
+            Self::place_runtime_in_sandbox_cgroup_v2_sync(&sandbox_cgroup)
+                .context("synchronously place runtime in sandbox cgroup")?;
         }
 
         Ok(Self {

tests/integration/kubernetes/k8s-sandbox-cgroup-placement.bats

@@ -0,0 +1,100 @@
+#!/usr/bin/env bats
+#
+# Copyright (c) 2026 Microsoft Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Under sandbox_cgroup_only on a cgroup v2 node, the runtime must place the
+# shim, the VMM and virtiofsd in the same pod (kubepods) cgroup. The
+# regression this guards leaves the VMM and virtiofsd in
+# system.slice/containerd.service while only the shim reaches kubepods. Runs
+# on any hypervisor with a separate VMM process; all checks use exec_host.
+
+load "${BATS_TEST_DIRNAME}/lib.sh"
+load "${BATS_TEST_DIRNAME}/../../common.bash"
+load "${BATS_TEST_DIRNAME}/tests_common.sh"
+
+host_is_cgroup_v2() {
+	exec_host "${node}" "test -f /sys/fs/cgroup/cgroup.controllers" >/dev/null 2>&1
+}
+
+sandbox_cgroup_only_enabled() {
+	local cfg
+	cfg="$(get_kata_runtime_config_file "${node}")" || return 1
+	exec_host "${node}" "grep -qE '^\s*sandbox_cgroup_only\s*=\s*true' '${cfg}'"
+}
+
+# Echo the cgroup v2 path (the part after "0::") for a pid on the node.
+# Returns non-zero if the path cannot be determined.
+host_cgroup_v2_path() {
+	local pid="${1}" path
+	# On cgroup v2 the file is a single line "0::<path>", e.g.
+	#   0::/kubepods.slice/.../cri-containerd-<cid>.scope
+	# Split on "::" and print the path that follows the "0::" prefix.
+	path="$(exec_host "${node}" "cat /proc/${pid}/cgroup" | awk -F'::' '/^0::/ {print $2}')"
+	[[ -n "${path}" ]] || return 1
+	echo "${path}"
+}
+
+setup() {
+	node="$(get_one_kata_node)"
+	has_separate_vmm || skip "test requires a hypervisor with a separate VMM process (KATA_HYPERVISOR=${KATA_HYPERVISOR})"
+	[[ -n "${node:-}" ]] || skip "no kata-runtime node found"
+	host_is_cgroup_v2 || skip "test requires the cgroup v2 unified hierarchy on the node"
+	sandbox_cgroup_only_enabled || skip "test requires sandbox_cgroup_only=true"
+
+	setup_common || die "setup_common failed"
+
+	pod_name="besteffort-test"
+
+	yaml_file="${pod_config_dir}/pod-besteffort.yaml"
+	set_node "$yaml_file" "$node"
+
+	auto_generate_policy "${pod_config_dir}" "${yaml_file}"
+}
+
+@test "runtime, virtiofsd and VMM share the pod sandbox cgroup" {
+	kubectl create -f "${yaml_file}"
+	kubectl wait --for=condition=Ready --timeout="$timeout" pod "$pod_name"
+
+	# Select the shim by pod's UID
+	local pod_uid
+	pod_uid="$(kubectl get pod "${pod_name}" -o jsonpath='{.metadata.uid}')"
+
+	local shim_pid vmm_pid vfsd_pid
+	shim_pid="$(shim_pid_for_pod "${node}" "${pod_uid}")"
+
+	vmm_pid="$(exec_host "${node}" "pgrep -P ${shim_pid} -f 'cloud-hypervisor|qemu-system|firecracker|stratovirt'")"
+
+	vfsd_pid="$(exec_host "${node}" "pgrep -P ${shim_pid} virtiofsd || true")"
+
+	info "pod uid: ${pod_uid}, shim pid: ${shim_pid}, VMM pid: ${vmm_pid}, virtiofsd pid: ${vfsd_pid:-<none>}"
+
+	local shim_cgroup
+	shim_cgroup="$(host_cgroup_v2_path "${shim_pid}")"
+	info "shim cgroup: ${shim_cgroup}"
+	echo "${shim_cgroup}" | grep -q "kubepods" || die "shim is not in a kubepods cgroup: ${shim_cgroup}"
+
+	# The VMM must be co-located with the shim, not left in the spawn cgroup.
+	local vmm_cgroup
+	vmm_cgroup="$(host_cgroup_v2_path "${vmm_pid}")"
+	info "VMM cgroup: ${vmm_cgroup}"
+	[[ "${vmm_cgroup}" == "${shim_cgroup}" ]] || die "VMM cgroup (${vmm_cgroup}) differs from the shim/pod cgroup (${shim_cgroup}); guest RAM would be mischarged"
+
+	# virtiofsd (when a separate daemon is used) must be co-located too.
+	if [[ -n "${vfsd_pid}" ]]; then
+		local vfsd_cgroup
+		vfsd_cgroup="$(host_cgroup_v2_path "${vfsd_pid}")"
+		info "virtiofsd cgroup: ${vfsd_cgroup}"
+		[[ "${vfsd_cgroup}" == "${shim_cgroup}" ]] || die "virtiofsd cgroup (${vfsd_cgroup}) differs from the shim/pod cgroup (${shim_cgroup})"
+	fi
+}
+
+teardown() {
+	has_separate_vmm || skip "test requires a hypervisor with a separate VMM process (KATA_HYPERVISOR=${KATA_HYPERVISOR})"
+	[[ -n "${node:-}" ]] || skip "no kata-runtime node found"
+	host_is_cgroup_v2 || skip "test requires the cgroup v2 unified hierarchy on the node"
+	sandbox_cgroup_only_enabled || skip "test requires sandbox_cgroup_only=true"
+
+	teardown_common "${node}" "${node_start_time:-}"
+}

tests/integration/kubernetes/run_kubernetes_tests.sh

@@ -102,6 +102,7 @@ else
 		"k8s-projected-volume.bats" \
 		"k8s-replication.bats" \
 		"k8s-sandbox-cgroup.bats" \
+		"k8s-sandbox-cgroup-placement.bats" \
 		"k8s-seccomp.bats" \
 		"k8s-sysctls.bats" \
 		"k8s-security-context.bats" \

tests/integration/kubernetes/tests_common.sh

@@ -226,6 +226,27 @@ is_runtime_rs() {
 	[[ "${KATA_HYPERVISOR}" == *-runtime-rs ]] || [[ "${KATA_HYPERVISOR}" == "dragonball" ]]
 }
 
+# True for hypervisors that run the guest in a separate VMM process (i.e. not
+# dragonball, which runs the guest in-process in the shim).
+has_separate_vmm() {
+	[[ "${KATA_HYPERVISOR}" != *dragonball* ]]
+}
+
+# Echo the PID of the Kata shim for the pod with UID $2, on node $1.
+# Returns non-zero if no matching shim is found.
+#
+# The UID is matched in both cgroup-path forms (dashed for cgroupfs,
+# underscored for systemd) so a shim from another namespace is not selected.
+# The pgrep pattern is bracketed ("[c]ontainerd") so it doesn't also match the
+# `bash -c` wrapper exec_host runs, whose command line contains the pattern.
+shim_pid_for_pod() {
+	local node_name="$1" uid_dashed="$2" uid_underscored pid
+	uid_underscored="${uid_dashed//-/_}"
+	pid="$(exec_host "${node_name}" "for p in \$(pgrep -f '[c]ontainerd-shim-kata-v2'); do grep -qE 'pod(${uid_dashed}|${uid_underscored})' /proc/\$p/cgroup 2>/dev/null && echo \$p && break; done; true")"
+	[[ -n "${pid}" ]] || return 1
+	echo "${pid}"
+}
+
 # Copy the right combination of drop-ins from drop-in-examples/ into
 # genpolicy-settings.d/. Drop-ins are layered: 10-* for platform base,
 # 20-* for OCI version and other overlays.