From 1d57808620c0469be56f695bb817053895c10a6c Mon Sep 17 00:00:00 2001 From: Saul Paredes Date: Wed, 8 Jul 2026 17:15:50 -0700 Subject: [PATCH 1/2] runtime-rs: resource: place the runtime in the sandbox cgroup synchronously Under sandbox_cgroup_only, the runtime adds itself to the sandbox cgroup via the cgroup manager. On the systemd driver that is an asynchronous call, so the runtime can fork virtiofsd and the VMM before it has joined the cgroup. Follow the manager placement with a synchronous cgroup.procs write, so the runtime is in the sandbox cgroup before any child is forked. No-op on cgroup v1. Generated-By: GitHub Copilot (Claude) Signed-off-by: Saul Paredes --- .../resource/src/cgroups/resource_inner.rs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/runtime-rs/crates/resource/src/cgroups/resource_inner.rs b/src/runtime-rs/crates/resource/src/cgroups/resource_inner.rs index bd56240b39..8836b88d8f 100644 --- a/src/runtime-rs/crates/resource/src/cgroups/resource_inner.rs +++ b/src/runtime-rs/crates/resource/src/cgroups/resource_inner.rs @@ -72,6 +72,38 @@ impl CgroupsResourceInner { } } + /// Synchronously write the runtime's own pid into the sandbox cgroup's + /// `cgroup.procs` (cgroup v2 only; a no-op otherwise). + /// + /// `add_proc` already places the runtime, but on the systemd driver that + /// is an asynchronous call: the runtime may fork virtiofsd and the + /// VMM before systemd has moved it, so those children inherit its original + /// cgroup (e.g. `system.slice/containerd.service`). Under cgroup v2 + /// first-touch accounting the guest RAM is then charged there, not to the + /// pod. A direct write is synchronous, so children forked afterwards + /// inherit the sandbox cgroup. + fn place_runtime_in_sandbox_cgroup_v2_sync(cgroup: &CgroupManager) -> Result<()> { + if !cgroup.v2() { + return Ok(()); + } + let dir = cgroup + .cgroup_path(None) + .context("resolve sandbox cgroup path for runtime placement")?; + let procs_path = format!("{}/cgroup.procs", dir.trim_end_matches('/')); + let pid = process::id(); + let mut file = std::fs::OpenOptions::new() + .write(true) + .open(&procs_path) + .with_context(|| format!("open sandbox cgroup.procs {procs_path}"))?; + std::io::Write::write_all(&mut file, format!("{pid}\n").as_bytes()) + .with_context(|| format!("move runtime pid {pid} into sandbox cgroup {procs_path}"))?; + info!( + sl!(), + "synchronously placed runtime (pid {}) into sandbox cgroup: {}", pid, procs_path + ); + Ok(()) + } + /// Create cgroup managers according to the cgroup configuration. /// /// # Returns @@ -132,6 +164,10 @@ impl CgroupsResourceInner { pid, "add runtime to sandbox cgroup", )?; + // The systemd add_proc above is asynchronous; make sure we have + // really joined the sandbox cgroup before forking any child. + Self::place_runtime_in_sandbox_cgroup_v2_sync(&sandbox_cgroup) + .context("synchronously place runtime in sandbox cgroup")?; } Ok(Self { From 5524f1f276340dbc616a9fbbe10da17f8731fe3b Mon Sep 17 00:00:00 2001 From: Saul Paredes Date: Thu, 9 Jul 2026 09:51:06 -0700 Subject: [PATCH 2/2] tests: k8s: verify the sandbox processes share the pod cgroup Add an integration test for the runtime-rs fix that co-locates the shim, the VMM and virtiofsd in the pod's kubepods cgroup under sandbox_cgroup_only on cgroup v2. The regression it guards leaves the VMM and virtiofsd in system.slice/containerd.service while only the shim reaches kubepods. Start a BestEffort Kata pod and, from the host, assert the VMM and virtiofsd share the shim's cgroup and that it is under kubepods. The shim is selected by the pod UID in its cgroup path, so a pod in another namespace is not picked. Skips dragonball (in-process VMM). Generated-By: GitHub Copilot (Claude) Signed-off-by: Saul Paredes --- .../k8s-sandbox-cgroup-placement.bats | 100 ++++++++++++++++++ .../kubernetes/run_kubernetes_tests.sh | 1 + tests/integration/kubernetes/tests_common.sh | 21 ++++ 3 files changed, 122 insertions(+) create mode 100644 tests/integration/kubernetes/k8s-sandbox-cgroup-placement.bats diff --git a/tests/integration/kubernetes/k8s-sandbox-cgroup-placement.bats b/tests/integration/kubernetes/k8s-sandbox-cgroup-placement.bats new file mode 100644 index 0000000000..c24763c243 --- /dev/null +++ b/tests/integration/kubernetes/k8s-sandbox-cgroup-placement.bats @@ -0,0 +1,100 @@ +#!/usr/bin/env bats +# +# Copyright (c) 2026 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# Under sandbox_cgroup_only on a cgroup v2 node, the runtime must place the +# shim, the VMM and virtiofsd in the same pod (kubepods) cgroup. The +# regression this guards leaves the VMM and virtiofsd in +# system.slice/containerd.service while only the shim reaches kubepods. Runs +# on any hypervisor with a separate VMM process; all checks use exec_host. + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/../../common.bash" +load "${BATS_TEST_DIRNAME}/tests_common.sh" + +host_is_cgroup_v2() { + exec_host "${node}" "test -f /sys/fs/cgroup/cgroup.controllers" >/dev/null 2>&1 +} + +sandbox_cgroup_only_enabled() { + local cfg + cfg="$(get_kata_runtime_config_file "${node}")" || return 1 + exec_host "${node}" "grep -qE '^\s*sandbox_cgroup_only\s*=\s*true' '${cfg}'" +} + +# Echo the cgroup v2 path (the part after "0::") for a pid on the node. +# Returns non-zero if the path cannot be determined. +host_cgroup_v2_path() { + local pid="${1}" path + # On cgroup v2 the file is a single line "0::", e.g. + # 0::/kubepods.slice/.../cri-containerd-.scope + # Split on "::" and print the path that follows the "0::" prefix. + path="$(exec_host "${node}" "cat /proc/${pid}/cgroup" | awk -F'::' '/^0::/ {print $2}')" + [[ -n "${path}" ]] || return 1 + echo "${path}" +} + +setup() { + node="$(get_one_kata_node)" + has_separate_vmm || skip "test requires a hypervisor with a separate VMM process (KATA_HYPERVISOR=${KATA_HYPERVISOR})" + [[ -n "${node:-}" ]] || skip "no kata-runtime node found" + host_is_cgroup_v2 || skip "test requires the cgroup v2 unified hierarchy on the node" + sandbox_cgroup_only_enabled || skip "test requires sandbox_cgroup_only=true" + + setup_common || die "setup_common failed" + + pod_name="besteffort-test" + + yaml_file="${pod_config_dir}/pod-besteffort.yaml" + set_node "$yaml_file" "$node" + + auto_generate_policy "${pod_config_dir}" "${yaml_file}" +} + +@test "runtime, virtiofsd and VMM share the pod sandbox cgroup" { + kubectl create -f "${yaml_file}" + kubectl wait --for=condition=Ready --timeout="$timeout" pod "$pod_name" + + # Select the shim by pod's UID + local pod_uid + pod_uid="$(kubectl get pod "${pod_name}" -o jsonpath='{.metadata.uid}')" + + local shim_pid vmm_pid vfsd_pid + shim_pid="$(shim_pid_for_pod "${node}" "${pod_uid}")" + + vmm_pid="$(exec_host "${node}" "pgrep -P ${shim_pid} -f 'cloud-hypervisor|qemu-system|firecracker|stratovirt'")" + + vfsd_pid="$(exec_host "${node}" "pgrep -P ${shim_pid} virtiofsd || true")" + + info "pod uid: ${pod_uid}, shim pid: ${shim_pid}, VMM pid: ${vmm_pid}, virtiofsd pid: ${vfsd_pid:-}" + + local shim_cgroup + shim_cgroup="$(host_cgroup_v2_path "${shim_pid}")" + info "shim cgroup: ${shim_cgroup}" + echo "${shim_cgroup}" | grep -q "kubepods" || die "shim is not in a kubepods cgroup: ${shim_cgroup}" + + # The VMM must be co-located with the shim, not left in the spawn cgroup. + local vmm_cgroup + vmm_cgroup="$(host_cgroup_v2_path "${vmm_pid}")" + info "VMM cgroup: ${vmm_cgroup}" + [[ "${vmm_cgroup}" == "${shim_cgroup}" ]] || die "VMM cgroup (${vmm_cgroup}) differs from the shim/pod cgroup (${shim_cgroup}); guest RAM would be mischarged" + + # virtiofsd (when a separate daemon is used) must be co-located too. + if [[ -n "${vfsd_pid}" ]]; then + local vfsd_cgroup + vfsd_cgroup="$(host_cgroup_v2_path "${vfsd_pid}")" + info "virtiofsd cgroup: ${vfsd_cgroup}" + [[ "${vfsd_cgroup}" == "${shim_cgroup}" ]] || die "virtiofsd cgroup (${vfsd_cgroup}) differs from the shim/pod cgroup (${shim_cgroup})" + fi +} + +teardown() { + has_separate_vmm || skip "test requires a hypervisor with a separate VMM process (KATA_HYPERVISOR=${KATA_HYPERVISOR})" + [[ -n "${node:-}" ]] || skip "no kata-runtime node found" + host_is_cgroup_v2 || skip "test requires the cgroup v2 unified hierarchy on the node" + sandbox_cgroup_only_enabled || skip "test requires sandbox_cgroup_only=true" + + teardown_common "${node}" "${node_start_time:-}" +} diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 9fb3bf3d3b..6e51479520 100755 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -101,6 +101,7 @@ else "k8s-projected-volume.bats" \ "k8s-replication.bats" \ "k8s-sandbox-cgroup.bats" \ + "k8s-sandbox-cgroup-placement.bats" \ "k8s-seccomp.bats" \ "k8s-sysctls.bats" \ "k8s-security-context.bats" \ diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index 782ce842aa..9ce765bdff 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -226,6 +226,27 @@ is_runtime_rs() { [[ "${KATA_HYPERVISOR}" == *-runtime-rs ]] || [[ "${KATA_HYPERVISOR}" == "dragonball" ]] } +# True for hypervisors that run the guest in a separate VMM process (i.e. not +# dragonball, which runs the guest in-process in the shim). +has_separate_vmm() { + [[ "${KATA_HYPERVISOR}" != *dragonball* ]] +} + +# Echo the PID of the Kata shim for the pod with UID $2, on node $1. +# Returns non-zero if no matching shim is found. +# +# The UID is matched in both cgroup-path forms (dashed for cgroupfs, +# underscored for systemd) so a shim from another namespace is not selected. +# The pgrep pattern is bracketed ("[c]ontainerd") so it doesn't also match the +# `bash -c` wrapper exec_host runs, whose command line contains the pattern. +shim_pid_for_pod() { + local node_name="$1" uid_dashed="$2" uid_underscored pid + uid_underscored="${uid_dashed//-/_}" + pid="$(exec_host "${node_name}" "for p in \$(pgrep -f '[c]ontainerd-shim-kata-v2'); do grep -qE 'pod(${uid_dashed}|${uid_underscored})' /proc/\$p/cgroup 2>/dev/null && echo \$p && break; done; true")" + [[ -n "${pid}" ]] || return 1 + echo "${pid}" +} + # Copy the right combination of drop-ins from drop-in-examples/ into # genpolicy-settings.d/. Drop-ins are layered: 10-* for platform base, # 20-* for OCI version and other overlays.