From 023c25838eb0cd764ff68bb57ee2516a58ed95db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sun, 31 May 2026 19:15:10 +0200 Subject: [PATCH] runtime-rs: cold-plug CDI VFIO devices for single containers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kubelet PodResources API is only available under Kubernetes. For single-container engines (Docker/nerdctl/podman) the CDI runtime applies the device's containerEdits directly to the OCI spec, so the VFIO nodes (e.g. /dev/vfio/devices/vfio0) show up in linux.devices instead. Discover those nodes from the OCI spec in addition to the PodResources API and feed both into the same cold-plug path, de-duplicating by host path. This reuses the existing Kubernetes machinery so GPU passthrough works for single containers with no changes to device handling. Signed-off-by: Fabiano Fidêncio Assisted-by: Cursor --- .../runtimes/virt_container/src/sandbox.rs | 83 ++++++++++++++----- 1 file changed, 64 insertions(+), 19 deletions(-) diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index b486063c19..e4f3db3bca 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -76,6 +76,7 @@ use resource::manager::ManagerArgs; use resource::network::{dan_config_path, DanNetworkConfig, NetworkConfig, NetworkWithNetNsConfig}; use resource::{ResourceConfig, ResourceManager}; use runtime_spec as spec; +use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::SystemTime; @@ -368,34 +369,55 @@ impl VirtSandbox { }; let config = self.resource_manager.config().await; + + // Collect the VFIO device nodes to cold-plug from two sources so that Kubernetes, docker, + // and nerdctl are handled by the same path: + // + // 1. Kubernetes: the kubelet PodResources API enumerates the CDI devices allocated to the + // pod. + // 2. Docker/nerdctl: the CDI runtime applies the device's containerEdits directly to the + // OCI spec, so the VFIO nodes show up in linux.devices (e.g. /dev/vfio/devices/vfio0). + let mut paths: Vec = Vec::new(); + let pod_resource_socket = &config.runtime.pod_resource_api_sock; info!( sl!(), "sandbox pod_resource_socket: {:?}", pod_resource_socket ); - if pod_resource_socket.is_empty() || !Path::new(pod_resource_socket).exists() { - return Ok(Vec::new()); + if !pod_resource_socket.is_empty() && Path::new(pod_resource_socket).exists() { + let annotations = &sandbox_config.annotations; + debug!( + sl!(), + "cold-plug: sandbox-name={:?} sandbox-namespace={:?}", + annotations.get("io.kubernetes.cri.sandbox-name"), + annotations.get("io.kubernetes.cri.sandbox-namespace") + ); + + let cdi_devices = pod_resources_rs::pod_resources::get_pod_cdi_devices( + pod_resource_socket, + annotations, + ) + .await + .context("failed to query Pod Resources CDI devices")?; + info!(sl!(), "pod cdi devices: {:?}", cdi_devices); + + let device_nodes = handle_cdi_devices(&cdi_devices).await?; + paths.extend( + device_nodes + .iter() + .filter_map(pod_resources_rs::device_node_host_path), + ); } - let annotations = &sandbox_config.annotations; - debug!( - sl!(), - "cold-plug: sandbox-name={:?} sandbox-namespace={:?}", - annotations.get("io.kubernetes.cri.sandbox-name"), - annotations.get("io.kubernetes.cri.sandbox-namespace") - ); + paths.extend(oci_spec_vfio_device_paths()); - let cdi_devices = - pod_resources_rs::pod_resources::get_pod_cdi_devices(pod_resource_socket, annotations) - .await - .context("failed to query Pod Resources CDI devices")?; - info!(sl!(), "pod cdi devices: {:?}", cdi_devices); + // De-duplicate while preserving discovery order. + let mut seen = HashSet::new(); + paths.retain(|path| seen.insert(path.clone())); - let device_nodes = handle_cdi_devices(&cdi_devices).await?; - let paths: Vec = device_nodes - .iter() - .filter_map(pod_resources_rs::device_node_host_path) - .collect(); + if paths.is_empty() { + return Ok(Vec::new()); + } let mut vfio_configs = Vec::new(); for path in paths.iter() { @@ -887,6 +909,29 @@ impl VirtSandbox { } } +/// Collect VFIO character device nodes (e.g. /dev/vfio/devices/vfio0) that a CDI +/// runtime injected directly into the OCI spec for the Docker/nerdctl/podman +/// flow, where there is no kubelet PodResources API to query. The legacy +/// `/dev/vfio/vfio` control node is skipped as it is not a pass-through device. +fn oci_spec_vfio_device_paths() -> Vec { + let Ok(spec) = load_oci_spec() else { + return Vec::new(); + }; + let Some(linux) = spec.linux() else { + return Vec::new(); + }; + let Some(devices) = linux.devices() else { + return Vec::new(); + }; + + devices + .iter() + .filter(|dev| dev.typ() == oci::LinuxDeviceType::C) + .map(|dev| dev.path().display().to_string()) + .filter(|path| path.starts_with("/dev/vfio") && path != "/dev/vfio/vfio") + .collect() +} + #[async_trait] impl Sandbox for VirtSandbox { #[instrument(name = "sb: start")]