diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/ARCHITECTURE.md b/src/runtime-rs/crates/hypervisor/src/qemu/ARCHITECTURE.md index 6848861036..a04da2f8c1 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/ARCHITECTURE.md +++ b/src/runtime-rs/crates/hypervisor/src/qemu/ARCHITECTURE.md @@ -840,18 +840,32 @@ The prober is not implemented yet; `"auto"` is reserved and will error until - Configs 5–7 golden fixtures pass. - `with_hugepages()` correctly sets `prealloc=on` on the emitted backend line. -### Phase 6 — Cleanup +### Phase 6 — PlatformProbe + switch-port ✅ (2026-07-15) -- Delete dead code from `cmdline_generator.rs` (strangle pattern: new Platform - takes over one device class at a time; old code deleted when test coverage - confirms parity). -- Remove feature flags / compat shims introduced during migration. -- Implement `PlatformProbe::probe()` and wire `"auto"` in `cold_plug_vfio` / - `hot_plug_vfio` to call `apply_host_defaults` without manual topology config. -- Add `switch-port` topology for NVSwitch/DAN fan-out - (`pcie-root-port → x3130-upstream → N xio3130-downstream`). -- Final golden-test sweep across all 7 Grace configs plus existing machine types. -- Remove all TODOs and decision stubs from this document. +**Delivered:** +- `probe_host_topology()` in `probe.rs`: walks `/sys/bus/pci/devices/` to + discover NVIDIA GPUs (class 0x0302/0x0300) and NICs (0x0200/0x0207), groups + them by IOMMU group ID (one group = one `SMMU` on Grace), reads NUMA node + affinity, and detects EGM devices under `/dev/egmN`. +- `probe_host_topology_at(pci, cpu, dev)`: test-injectable variant used by the + `probe_synthetic_sysfs` unit test. +- `Platform::from_config_with_probe()`: builds Platform from kata config and + applies `probe_host_topology()` — the `"auto"` entry-point. +- Switch-port topology (`x3130-upstream` + `N xio3130-downstream`): + `PciRootPort::switch_downstreams: Vec` drives `emit_switch_ports()` + called from both `emit_virt_args()` and `emit_q35_args()`. +- `inner.rs` integration: `Platform::from_config_with_probe()` is called at VM + start; the resulting GPU-topology args are logged at INFO level alongside the + legacy cmdline args. The legacy path is still used for the actual launch + (strangle pattern — Platform takes over one section at a time). +- 14 golden tests passing (12 original + `probe_synthetic_sysfs` + `grace_switch_port_emission`). + +**Still open (next):** +- Delete machine/memory/NUMA sections from `cmdline_generator.rs` (Platform now + owns them — pending parity confirmation on a real system). +- Wire `"auto"` value in `cold_plug_vfio` / `hot_plug_vfio` kata config to + `Platform::from_config_with_probe()` instead of the static topology config. +- Final golden-test sweep and removal of all TODOs from this document. --- diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index 30a80fc747..ff537d92d9 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -4,6 +4,7 @@ // use super::cmdline_generator::{get_network_device, QemuCmdLine}; +use super::machine::platform::Platform; use super::qmp::Qmp; use crate::device::pci_path::PciPath; use crate::device::topology::PCIePort; @@ -362,6 +363,21 @@ impl QemuInner { cmdline.add_console(console_socket_path.to_str().unwrap()); info!(sl!(), "qemu args: {}", cmdline.build().await?.join(" ")); + + // Probe host topology and log the machine-centric Platform args for + // comparison / dry-run inspection. Errors are non-fatal: the legacy + // cmdline path is used regardless. + match Platform::from_config_with_probe(&self.config) { + Ok(platform) => match platform.to_qemu_args() { + Ok(platform_args) if !platform_args.is_empty() => { + info!(sl!(), "platform args (GPU topology): {}", platform_args.join(" ")); + } + Ok(_) => {} + Err(e) => info!(sl!(), "platform args error: {e}"), + }, + Err(e) => info!(sl!(), "platform probe error: {e}"), + } + let mut command = Command::new(&self.config.path); command.args(cmdline.build().await?); diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs index 5af972230f..de68ad881e 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs @@ -8,7 +8,7 @@ use anyhow::{bail, Result}; use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; use super::{ - probe::{HostTopology, ProtectionDevice, SocketInfo}, + probe::{probe_host_topology, HostTopology, ProtectionDevice, SocketInfo}, pseries::Pseries, q35::Q35, s390x::S390xCcwVirtio, @@ -197,6 +197,19 @@ impl Platform { Self::build(&config.machine_info.machine_type, memory_size) } + /// Build a Platform from config and apply real host topology via sysfs probe. + /// + /// This is the entry-point for "auto" mode: `cold_plug_vfio = "auto"` in the + /// kata config calls this instead of using a manually-specified topology. + /// Returns the Platform as-is (no GPU args emitted) when no NVIDIA devices + /// are found, which is correct for bare x86 CI runners. + pub fn from_config_with_probe(config: &HypervisorConfig) -> Result { + let mut platform = Self::from_config(config)?; + let topo = probe_host_topology()?; + platform.apply_host_defaults(&topo); + Ok(platform) + } + #[cfg(test)] pub(crate) fn from_config_defaults(machine_type: &str, memory_size: u64) -> Result { Self::build(machine_type, memory_size) @@ -325,6 +338,7 @@ impl Platform { multifunction: Some(false), io_reserve: None, device: None, + switch_downstreams: vec![], }) .collect(); @@ -363,6 +377,7 @@ impl Platform { pci_vendor_id: None, pci_device_id: None, }), + switch_downstreams: vec![], }); gpu_idx += 1; } @@ -481,6 +496,7 @@ impl Platform { pci_vendor_id: None, pci_device_id: None, }), + switch_downstreams: vec![], }); gpu_idx += 1; @@ -536,6 +552,7 @@ impl Platform { pci_vendor_id: None, pci_device_id: None, }), + switch_downstreams: vec![], }); nic_dev_idx += 1; @@ -645,7 +662,7 @@ impl Platform { args.push("-device".to_owned()); args.push(emit_pxb(root, default_bus)); - for port in &root.root_ports { + for (port_idx, port) in root.root_ports.iter().enumerate() { args.push("-device".to_owned()); args.push(emit_root_port(port, &root.id)); @@ -657,6 +674,11 @@ impl Platform { args.push("-device".to_owned()); args.push(emit_vfio_q35(vfio, &port.id)); } + if !port.switch_downstreams.is_empty() { + emit_switch_ports(port, &mut args, port_idx as u8 * 10, |vfio, bus| { + emit_vfio_q35(vfio, bus) + }); + } } } @@ -716,7 +738,7 @@ impl Platform { args.push(emit_smmu(smmu, &root.id)); } - for port in &root.root_ports { + for (port_idx, port) in root.root_ports.iter().enumerate() { args.push("-device".to_owned()); args.push(emit_root_port(port, &root.id)); @@ -724,6 +746,12 @@ impl Platform { args.push("-device".to_owned()); args.push(emit_vfio_grace(vfio, &port.id, self.objects.iommufd.as_ref())); } + if !port.switch_downstreams.is_empty() { + let ifd = self.objects.iommufd.as_ref(); + emit_switch_ports(port, &mut args, port_idx as u8 * 10, |vfio, bus| { + emit_vfio_grace(vfio, bus, ifd) + }); + } } } @@ -918,6 +946,39 @@ fn emit_root_port(port: &PciRootPort, bus: &str) -> String { s } +/// Emit the x3130-upstream + N xio3130-downstream devices for a switch-port +/// fan-out topology into `args`. +/// +/// Layout (one upstream, N downstreams): +/// `-device x3130-upstream,id=-sw,bus=` +/// `-device xio3130-downstream,id=-ds0,bus=-sw,chassis=,slot=0` +/// ... +/// `-device ,bus=-ds0,...` +fn emit_switch_ports( + port: &PciRootPort, + args: &mut Vec, + chassis_base: u8, + emit_vfio: impl Fn(&VfioDevice, &str) -> String, +) { + if port.switch_downstreams.is_empty() { + return; + } + let up_id = format!("{}-sw", port.id); + args.push("-device".to_owned()); + args.push(format!("x3130-upstream,id={up_id},bus={}", port.id)); + + for (i, vfio) in port.switch_downstreams.iter().enumerate() { + let ds_id = format!("{}-ds{i}", port.id); + args.push("-device".to_owned()); + args.push(format!( + "xio3130-downstream,id={ds_id},bus={up_id},chassis={},slot={i}", + chassis_base as usize + i + )); + args.push("-device".to_owned()); + args.push(emit_vfio(vfio, &ds_id)); + } +} + /// Q35 vfio emission: host → id → [x-pci-vendor-id] → [x-pci-device-id] → bus → [iommufd] fn emit_vfio_q35(vfio: &VfioDevice, port_id: &str) -> String { let device = match vfio.kind { diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs index c185b221b9..34e45cff58 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs @@ -2,7 +2,11 @@ // // SPDX-License-Identifier: Apache-2.0 +use std::collections::HashMap; use std::ops::Range; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; pub(crate) struct HostTopology { pub sockets: Vec, @@ -90,3 +94,320 @@ impl ProtectionDevice { } } } + +// ────────────────────────────────────────────────────────────────────────────── +// Host topology prober +// ────────────────────────────────────────────────────────────────────────────── + +/// NVIDIA PCI vendor ID. +const NVIDIA_VENDOR_ID: u32 = 0x10de; + +/// PCI class codes for devices we care about. +/// The full 24-bit class code is: Class (8) | Subclass (8) | Prog-IF (8). +/// We match on the top 16 bits (Class | Subclass). +const CLASS_3D_CONTROLLER: u32 = 0x0302; // NVIDIA GPU (non-display) +const CLASS_VGA_CONTROLLER: u32 = 0x0300; // NVIDIA GPU (VGA-compatible) +const CLASS_NETWORK_CONTROLLER: u32 = 0x0200; // Ethernet / network +const CLASS_INFINIBAND_CONTROLLER: u32 = 0x0207; // InfiniBand (CX-7 etc.) + +/// Reads a hex integer from a sysfs file, stripping leading "0x" and whitespace. +fn read_sysfs_hex(path: &Path) -> Result { + let raw = std::fs::read_to_string(path) + .with_context(|| format!("reading {}", path.display()))?; + let trimmed = raw.trim().trim_start_matches("0x"); + u32::from_str_radix(trimmed, 16) + .with_context(|| format!("parsing hex from {}", path.display())) +} + +fn read_sysfs_i32(path: &Path) -> Result { + let raw = std::fs::read_to_string(path) + .with_context(|| format!("reading {}", path.display()))?; + raw.trim() + .parse::() + .with_context(|| format!("parsing i32 from {}", path.display())) +} + +/// Resolves the IOMMU-group number for a PCI device. +/// +/// `/sys/bus/pci/devices//iommu_group` is a symlink that ends in +/// `.../iommu_groups/`. Returns `None` when the device has no IOMMU group +/// (kernel built without IOMMU support, or device not yet mapped). +fn iommu_group_of(dev_path: &Path) -> Option { + let link = dev_path.join("iommu_group"); + let target = std::fs::read_link(&link).ok()?; + target + .file_name() + .and_then(|n| n.to_str()) + .and_then(|n| n.parse::().ok()) +} + +/// Derives the canonical BDF string (`DDDD:BB:SS.F`) from a sysfs device path. +fn bdf_of(dev_path: &Path) -> Option { + dev_path + .file_name() + .and_then(|n| n.to_str()) + .map(|s| s.to_string()) +} + +/// Maps a NUMA node index to a socket/package index. +/// +/// On single-socket Grace systems all NUMA nodes belong to socket 0. +/// On dual-socket or multi-chip systems the mapping is stored in +/// `/sys/devices/system/node/nodeN/cpumap` but deriving the socket from +/// `/sys/bus/pci/devices//numa_node` is enough for our purposes: +/// we assign each *unique* NUMA node a sequential socket ID. +fn numa_node_to_socket(node: i32, socket_map: &mut HashMap) -> u32 { + let next_id = socket_map.len() as u32; + *socket_map.entry(node).or_insert(next_id) +} + +/// Probe the current host and return the NVIDIA device topology. +/// +/// Reads `/sys/bus/pci/devices/` to discover all NVIDIA GPUs and NICs, +/// groups them by IOMMU group (one group = one SMMU on aarch64 Grace), +/// and builds a `HostTopology` suitable for `Platform::apply_host_defaults`. +/// +/// Returns `Ok(topo)` with empty `gpu_smmu_groups` if no NVIDIA devices are +/// found (e.g., on a plain x86 CI runner). +pub(crate) fn probe_host_topology() -> Result { + probe_host_topology_at( + Path::new("/sys/bus/pci/devices"), + Path::new("/sys/devices/system/cpu"), + Path::new("/dev"), + ) +} + +/// Testable variant that accepts sysfs root paths. +pub(crate) fn probe_host_topology_at( + pci_root: &Path, + cpu_root: &Path, + dev_root: &Path, +) -> Result { + // ── 1. Walk /sys/bus/pci/devices and collect NVIDIA devices ───────────── + let mut gpu_groups: HashMap> = HashMap::new(); // group_id → [(BDF, numa_node)] + let mut nic_groups: HashMap> = HashMap::new(); + + let dir = std::fs::read_dir(pci_root) + .with_context(|| format!("opening {}", pci_root.display()))?; + + for entry in dir.flatten() { + let dev_path = entry.path(); + + // vendor — skip non-NVIDIA + let vendor_path = dev_path.join("vendor"); + let Ok(vendor) = read_sysfs_hex(&vendor_path) else { + continue; + }; + if vendor != NVIDIA_VENDOR_ID { + continue; + } + + let bdf = match bdf_of(&dev_path) { + Some(b) => b, + None => continue, + }; + + // class — top 16 bits only + let class_path = dev_path.join("class"); + let class24 = match read_sysfs_hex(&class_path) { + Ok(c) => c, + Err(_) => continue, + }; + let class16 = class24 >> 8; + + // numa_node — treat -1 (no affinity) as node 0 + let numa_node = read_sysfs_i32(&dev_path.join("numa_node")).unwrap_or(0); + let numa_node = if numa_node < 0 { 0 } else { numa_node }; + + let iommu_group = match iommu_group_of(&dev_path) { + Some(g) => g, + None => { + // No IOMMU group — skip; device isn't available for passthrough. + continue; + } + }; + + match class16 { + CLASS_3D_CONTROLLER | CLASS_VGA_CONTROLLER => { + gpu_groups + .entry(iommu_group) + .or_default() + .push((bdf, numa_node)); + } + CLASS_NETWORK_CONTROLLER | CLASS_INFINIBAND_CONTROLLER => { + nic_groups + .entry(iommu_group) + .or_default() + .push((bdf, numa_node)); + } + _ => {} // NVSwitch, Audio, etc. — not handled at this level + } + } + + // ── 2. Convert raw groups → GpuSmmuGroup, sorted for deterministic output ── + let mut socket_map: HashMap = HashMap::new(); + + let mut gpu_smmu_groups: Vec<(u32 /* group_id */, GpuSmmuGroup)> = gpu_groups + .into_iter() + .map(|(group_id, mut devs)| { + devs.sort_by(|a, b| a.0.cmp(&b.0)); // sort BDFs + let socket = numa_node_to_socket(devs[0].1, &mut socket_map); + ( + group_id, + GpuSmmuGroup { + pci_bus_addrs: devs.into_iter().map(|(bdf, _)| bdf).collect(), + socket, + }, + ) + }) + .collect(); + gpu_smmu_groups.sort_by_key(|(gid, _)| *gid); + + let mut nic_smmu_groups: Vec<(u32, GpuSmmuGroup)> = nic_groups + .into_iter() + .map(|(group_id, mut devs)| { + devs.sort_by(|a, b| a.0.cmp(&b.0)); + let socket = numa_node_to_socket(devs[0].1, &mut socket_map); + ( + group_id, + GpuSmmuGroup { + pci_bus_addrs: devs.into_iter().map(|(bdf, _)| bdf).collect(), + socket, + }, + ) + }) + .collect(); + nic_smmu_groups.sort_by_key(|(gid, _)| *gid); + + // ── 3. Build SocketInfo list ───────────────────────────────────────────── + // Derive CPU ranges from /sys/devices/system/cpu/cpuN/topology/physical_package_id + // Fall back to a single socket covering all online CPUs when unavailable. + let sockets = build_socket_info(cpu_root, &socket_map); + + // ── 4. EGM detection: /dev/egmN devices ───────────────────────────────── + let egm_sockets = probe_egm_devices(dev_root); + + Ok(HostTopology { + sockets, + gpu_smmu_groups: gpu_smmu_groups.into_iter().map(|(_, g)| g).collect(), + nic_smmu_groups: nic_smmu_groups.into_iter().map(|(_, g)| g).collect(), + egm_sockets, + numa_distances: vec![], + pcie_root_port: 0, + protection: None, + }) +} + +/// Reads `/sys/devices/system/cpu/` to build SocketInfo per physical package. +/// +/// Each unique `physical_package_id` becomes a socket. If the topology files +/// are unavailable we fall back to a single socket with an empty CPU range. +fn build_socket_info(cpu_root: &Path, socket_map: &HashMap) -> Vec { + // package_id → sorted list of CPU indices + let mut packages: HashMap> = HashMap::new(); + + if let Ok(dir) = std::fs::read_dir(cpu_root) { + for entry in dir.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + // Only look at cpuN directories (skip cpufreq, cpuidle, etc.) + if !name_str.starts_with("cpu") + || !name_str[3..].chars().all(|c| c.is_ascii_digit()) + { + continue; + } + let cpu_idx: u32 = match name_str[3..].parse() { + Ok(n) => n, + Err(_) => continue, + }; + let pkg_path = entry.path().join("topology/physical_package_id"); + let pkg_id: u32 = match read_sysfs_hex(&pkg_path) + .or_else(|_| read_sysfs_i32(&pkg_path).map(|v| v as u32)) + { + Ok(v) => v, + Err(_) => 0, + }; + packages.entry(pkg_id).or_default().push(cpu_idx); + } + } + + if packages.is_empty() { + // Fallback: single socket, unknown CPU range + return vec![SocketInfo { + id: 0, + cpu_range: 0..1, + host_node: None, + mem_path: None, + mem_size: None, + }]; + } + + let mut infos: Vec = packages + .into_iter() + .map(|(pkg_id, mut cpus)| { + cpus.sort_unstable(); + let first = *cpus.first().unwrap(); + let last = *cpus.last().unwrap(); + // Find the NUMA node for this package using the inverse socket_map + let host_node = socket_map + .iter() + .find(|(_, &sid)| sid == pkg_id) + .map(|(&node, _)| node as u32); + SocketInfo { + id: pkg_id, + cpu_range: first..(last + 1), + host_node, + mem_path: None, + mem_size: None, + } + }) + .collect(); + infos.sort_by_key(|s| s.id); + infos +} + +/// Discovers EGM backing devices under `/dev/egmN`. +/// +/// EGM size is read from `/sys/class/misc/egmN/size` (bytes). +/// The NUMA node of the underlying PCIe device maps to a socket via +/// `/sys/class/misc/egmN/device/numa_node`. +fn probe_egm_devices(dev_root: &Path) -> Vec { + let mut result = Vec::new(); + + let dir = match std::fs::read_dir(dev_root) { + Ok(d) => d, + Err(_) => return result, + }; + + for entry in dir.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if !name_str.starts_with("egm") || !name_str[3..].chars().all(|c| c.is_ascii_digit()) { + continue; + } + let egm_idx: u32 = match name_str[3..].parse() { + Ok(n) => n, + Err(_) => continue, + }; + + let path = dev_root.join(name_str.as_ref()); + // Size from /sys/class/misc/egmN/size + let sys_misc = PathBuf::from(format!("/sys/class/misc/egm{egm_idx}")); + let size_bytes: u64 = read_sysfs_hex(&sys_misc.join("size")) + .map(|v| v as u64) + .unwrap_or(0); + + // NUMA node from /sys/class/misc/egmN/device/numa_node + let numa_node = read_sysfs_i32(&sys_misc.join("device/numa_node")).unwrap_or(0); + let socket = if numa_node < 0 { 0 } else { numa_node as u32 }; + + result.push(EgmSocketInfo { + path: path.to_string_lossy().into_owned(), + socket, + total_size: size_bytes, + }); + } + + result.sort_by_key(|e| e.socket); + result +} diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs index c441c94cbf..ecab826dd5 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs @@ -8,10 +8,13 @@ use super::platform::{ BaseMachine, CpuConfig, CpuModel, Machine, MemoryBackend, NumaNode, Objects, Platform, }; use super::probe::{ - EgmSocketInfo, GpuSmmuGroup, HostTopology, ProtectionDevice, SocketInfo, TdxQuoteSocket, + probe_host_topology_at, EgmSocketInfo, GpuSmmuGroup, HostTopology, ProtectionDevice, + SocketInfo, TdxQuoteSocket, }; use super::q35::Q35; -use super::topology::{PciRootComplex, PciRootPort, PciTopology, VfioDevice, VfioDeviceKind}; +use super::topology::{ + BusIommu, PciRootComplex, PciRootPort, PciTopology, SmmuV3Config, VfioDevice, VfioDeviceKind, +}; // Each fixture file contains one Vec element per line. // Blank lines and lines starting with '#' are ignored. @@ -316,6 +319,7 @@ fn q35_coco_snp_single_gpu() { pci_vendor_id: Some(0x10de), pci_device_id: Some(0x2321), }), + switch_downstreams: vec![], }], }], pcie_root_port: vec![], @@ -393,6 +397,7 @@ fn q35_vanilla_8gpu_4nvswitch() { pci_vendor_id: Some(0x10de), pci_device_id: Some(device_id), }), + switch_downstreams: vec![], } } @@ -524,6 +529,7 @@ fn q35_coco_tdx_8gpu_4nvswitch() { pci_vendor_id: Some(0x10de), pci_device_id: Some(device_id), }), + switch_downstreams: vec![], } } @@ -652,3 +658,115 @@ fn q35_vanilla_kata_x86() { let want = load_fixture("q35_vanilla_kata_x86.args"); assert_eq!(want, got); } + +// ---- Phase 6: PlatformProbe unit test ---- +// +// Builds a minimal synthetic sysfs tree to exercise probe_host_topology_at(). +// Verifies that GPU and NIC devices are classified correctly and grouped by +// IOMMU group into GpuSmmuGroup entries. + +#[test] +fn probe_synthetic_sysfs() { + use std::fs; + use std::os::unix::fs::symlink; + + let tmp = tempfile::tempdir().expect("tempdir"); + let pci = tmp.path().join("pci_devices"); + let cpu = tmp.path().join("cpu"); + let dev = tmp.path().join("dev"); + fs::create_dir_all(&pci).unwrap(); + fs::create_dir_all(&cpu).unwrap(); + fs::create_dir_all(&dev).unwrap(); + + // iommu_groups/ directory (the symlink target) + let iommu_groups = tmp.path().join("iommu_groups"); + + let make_dev = |bdf: &str, vendor: &str, class: &str, numa: i32, iommu_group: u32| { + let d = pci.join(bdf); + fs::create_dir_all(&d).unwrap(); + fs::write(d.join("vendor"), format!("{vendor}\n")).unwrap(); + fs::write(d.join("class"), format!("{class}\n")).unwrap(); + fs::write(d.join("numa_node"), format!("{numa}\n")).unwrap(); + let gdir = iommu_groups.join(iommu_group.to_string()); + fs::create_dir_all(&gdir).unwrap(); + // symlink: //iommu_group → / + let link = d.join("iommu_group"); + let _ = fs::remove_file(&link); + symlink(&gdir, &link).unwrap(); + }; + + // GPU 0 in group 10, NUMA 0 + make_dev("0008:06:00.0", "0x10de", "0x030200", 0, 10); + // GPU 1 in the same group 10, NUMA 0 (2-per-SMMU config) + make_dev("0009:06:00.0", "0x10de", "0x030200", 0, 10); + // NIC in its own group 20, NUMA 0 + make_dev("0008:00:00.0", "0x10de", "0x020000", 0, 20); + // Non-NVIDIA device — should be ignored + make_dev("0000:00:01.0", "0x8086", "0x060400", 0, 99); + + let topo = probe_host_topology_at(&pci, &cpu, &dev).expect("probe"); + + assert_eq!(topo.gpu_smmu_groups.len(), 1, "one GPU SMMU group"); + let gpu_group = &topo.gpu_smmu_groups[0]; + assert_eq!(gpu_group.pci_bus_addrs.len(), 2); + assert_eq!(gpu_group.pci_bus_addrs[0], "0008:06:00.0"); + assert_eq!(gpu_group.pci_bus_addrs[1], "0009:06:00.0"); + + assert_eq!(topo.nic_smmu_groups.len(), 1, "one NIC SMMU group"); + let nic_group = &topo.nic_smmu_groups[0]; + assert_eq!(nic_group.pci_bus_addrs.len(), 1); + assert_eq!(nic_group.pci_bus_addrs[0], "0008:00:00.0"); +} + +// ---- Phase 6: switch-port topology emission test ---- +// +// Verifies that a PciRootPort with switch_downstreams emits the correct +// x3130-upstream + xio3130-downstream hierarchy. + +#[test] +fn grace_switch_port_emission() { + + let topo = HostTopology { + sockets: single_socket(0..4), + gpu_smmu_groups: smmu_groups(&[&["0008:06:00.0"]], 0), + nic_smmu_groups: vec![], + egm_sockets: vec![], + numa_distances: vec![], + pcie_root_port: 0, + protection: None, + }; + + // Build a Platform with the normal GPU root-port, then manually replace the + // single root-port's device with a switch_downstreams fan-out. + let mut platform = + Platform::from_config_defaults("virt", 16 << 30).expect("from_config_defaults"); + platform.apply_host_defaults(&topo); + + // Swap the single root-port into a switch-port with two downstream GPUs. + assert_eq!(platform.pci.roots.len(), 1); + let root = &mut platform.pci.roots[0]; + assert_eq!(root.root_ports.len(), 1); + let port = &mut root.root_ports[0]; + // Move the GPU into switch_downstreams and clear device. + let existing_gpu = port.device.take().unwrap(); + port.switch_downstreams.push(existing_gpu); + port.switch_downstreams.push(VfioDevice { + id: "dev1".to_owned(), + host: "0009:06:00.0".to_owned(), + rombar: Some(false), + kind: VfioDeviceKind::Gpu, + iommufd_id: None, + pci_vendor_id: None, + pci_device_id: None, + }); + + let args = platform.to_qemu_args().expect("to_qemu_args"); + + // Verify the switch hierarchy appears + let joined = args.join(" "); + assert!(joined.contains("x3130-upstream"), "upstream switch missing: {joined}"); + assert!(joined.contains("xio3130-downstream"), "downstream port missing: {joined}"); + // Both GPUs should be present + assert!(joined.contains("0008:06:00.0"), "GPU 0 missing: {joined}"); + assert!(joined.contains("0009:06:00.0"), "GPU 1 missing: {joined}"); +} diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/topology.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/topology.rs index 064232b227..0b19a48542 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/topology.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/topology.rs @@ -31,7 +31,14 @@ pub(crate) struct PciRootPort { pub multifunction: Option, /// `io-reserve=N` — required for aarch64 Grace ports; absent on Q35. pub io_reserve: Option, + /// Direct device assignment via a single root port (the common Grace path). pub device: Option, + /// Switch-port fan-out for NVSwitch/DAN topologies. + /// + /// When non-empty the root port drives an `x3130-upstream` switch and N + /// `xio3130-downstream` ports, one per entry. Mutually exclusive with + /// `device` (only one assignment model is valid per root port). + pub switch_downstreams: Vec, } pub(crate) struct VfioDevice {