github/kata-containers
1
0
Fork 2
mirror of https://github.com/kata-containers/kata-containers.git synced 2026-07-09 22:24:51 +00:00
Code Issues Projects Releases Wiki Activity

runtime-rs: cold-plug CDI VFIO devices for single containers

Browse Source 
The kubelet PodResources API is only available under Kubernetes. For
single-container engines (Docker/nerdctl/podman) the CDI runtime applies
the device's containerEdits directly to the OCI spec, so the VFIO nodes
(e.g. /dev/vfio/devices/vfio0) show up in linux.devices instead.

Discover those nodes from the OCI spec in addition to the PodResources
API and feed both into the same cold-plug path, de-duplicating by host
path. This reuses the existing Kubernetes machinery so GPU passthrough
works for single containers with no changes to device handling.

Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>
Assisted-by: Cursor <noreply@cursor.com>
This commit is contained in:
Fabiano Fidêncio
2026-05-31 19:15:10 +02:00
parent 403fc1400a
commit 023c25838e
1 changed files with 64 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs
View File

@@ -76,6 +76,7 @@ use resource::manager::ManagerArgs;
use resource::network::{dan_config_path, DanNetworkConfig, NetworkConfig, NetworkWithNetNsConfig};
use resource::{ResourceConfig, ResourceManager};
use runtime_spec as spec;
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::SystemTime;
@@ -368,34 +369,55 @@ impl VirtSandbox {
};
let config = self.resource_manager.config().await;
// Collect the VFIO device nodes to cold-plug from two sources so that Kubernetes, docker,
// and nerdctl are handled by the same path:
//
// 1. Kubernetes: the kubelet PodResources API enumerates the CDI devices allocated to the
// pod.
// 2. Docker/nerdctl: the CDI runtime applies the device's containerEdits directly to the
// OCI spec, so the VFIO nodes show up in linux.devices (e.g. /dev/vfio/devices/vfio0).
let mut paths: Vec<String> = Vec::new();
let pod_resource_socket = &config.runtime.pod_resource_api_sock;
info!(
sl!(),
"sandbox pod_resource_socket: {:?}", pod_resource_socket
);
if pod_resource_socket.is_empty() || !Path::new(pod_resource_socket).exists() {
return Ok(Vec::new());
if !pod_resource_socket.is_empty() && Path::new(pod_resource_socket).exists() {
let annotations = &sandbox_config.annotations;
debug!(
sl!(),
"cold-plug: sandbox-name={:?} sandbox-namespace={:?}",
annotations.get("io.kubernetes.cri.sandbox-name"),
annotations.get("io.kubernetes.cri.sandbox-namespace")
);
let cdi_devices = pod_resources_rs::pod_resources::get_pod_cdi_devices(
pod_resource_socket,
annotations,
)
.await
.context("failed to query Pod Resources CDI devices")?;
info!(sl!(), "pod cdi devices: {:?}", cdi_devices);
let device_nodes = handle_cdi_devices(&cdi_devices).await?;
paths.extend(
device_nodes
.iter()
.filter_map(pod_resources_rs::device_node_host_path),
);
}
let annotations = &sandbox_config.annotations;
debug!(
sl!(),
"cold-plug: sandbox-name={:?} sandbox-namespace={:?}",
annotations.get("io.kubernetes.cri.sandbox-name"),
annotations.get("io.kubernetes.cri.sandbox-namespace")
);
paths.extend(oci_spec_vfio_device_paths());
let cdi_devices =
pod_resources_rs::pod_resources::get_pod_cdi_devices(pod_resource_socket, annotations)
.await
.context("failed to query Pod Resources CDI devices")?;
info!(sl!(), "pod cdi devices: {:?}", cdi_devices);
// De-duplicate while preserving discovery order.
let mut seen = HashSet::new();
paths.retain(|path| seen.insert(path.clone()));
let device_nodes = handle_cdi_devices(&cdi_devices).await?;
let paths: Vec<String> = device_nodes
.iter()
.filter_map(pod_resources_rs::device_node_host_path)
.collect();
if paths.is_empty() {
return Ok(Vec::new());
}
let mut vfio_configs = Vec::new();
for path in paths.iter() {
@@ -887,6 +909,29 @@ impl VirtSandbox {
}
}
/// Collect VFIO character device nodes (e.g. /dev/vfio/devices/vfio0) that a CDI
/// runtime injected directly into the OCI spec for the Docker/nerdctl/podman
/// flow, where there is no kubelet PodResources API to query. The legacy
/// `/dev/vfio/vfio` control node is skipped as it is not a pass-through device.
fn oci_spec_vfio_device_paths() -> Vec<String> {
let Ok(spec) = load_oci_spec() else {
return Vec::new();
};
let Some(linux) = spec.linux() else {
return Vec::new();
};
let Some(devices) = linux.devices() else {
return Vec::new();
};
devices
.iter()
.filter(|dev| dev.typ() == oci::LinuxDeviceType::C)
.map(|dev| dev.path().display().to_string())
.filter(|path| path.starts_with("/dev/vfio") && path != "/dev/vfio/vfio")
.collect()
}
#[async_trait]
impl Sandbox for VirtSandbox {
#[instrument(name = "sb: start")]