From d50718212e5fdc641f85a46c8b7fc6b3b3e547ee Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Fri, 17 Jul 2026 13:45:48 +0000 Subject: [PATCH] qemu/machine: Phase 4 -- multi-RC NIC groups, all Grace 1-4 tests pass Add HostTopology::nic_smmu_groups (Vec): NIC devices get their own pxb+smmuv3 complex allocated after all GPU complexes, but emit no acpi-generic-initiator links and no NUMA initiator nodes. Reuses GpuSmmuGroup to carry socket affinity. apply_virt_defaults processes nic_smmu_groups after gpu_smmu_groups, continuing bus_nr and port_idx counters; VfioDeviceKind::Nic maps to vfio-pci in emit_vfio_grace. grace_4_gpu_and_nic fixture updated: GPU on pcie.1 (bus_nr=1), NIC on pcie.2 (bus_nr=2), NIC emits vfio-pci not vfio-pci-nohotplug. Unignore grace_1 through grace_4, add the gb300_nvl_2gpu dry-run test (fixture landed in the previous commit), and add nic_smmu_groups: vec![] to all existing HostTopology constructions. Result: 9 fixture tests pass (4 Grace + 4 Q35 + GB300), grace_5-7 stay ignored until Phase 5. Assisted-by: Claude Signed-off-by: Zvonko Kaiser --- .../hypervisor/src/qemu/machine/platform.rs | 51 ++++++++++++++++++ .../hypervisor/src/qemu/machine/probe.rs | 6 +++ .../hypervisor/src/qemu/machine/tests.rs | 53 +++++++++++++++---- .../fixtures/qemu/grace_4_gpu_and_nic.args | 8 +-- 4 files changed, 105 insertions(+), 13 deletions(-) diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs index ef5fa54b80..73fc8fea2e 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs @@ -502,6 +502,57 @@ impl Platform { root_ports, }); } + + // NIC groups: own pxb+smmuv3 complex, no GI links, no extra NUMA nodes. + // pxb numbering continues from where GPU groups left off. + let gpu_group_count = topo.gpu_smmu_groups.len(); + let mut nic_dev_idx = gpu_idx; + for (nic_group_idx, group) in topo.nic_smmu_groups.iter().enumerate() { + let n_ports = group.pci_bus_addrs.len(); + let bus_nr = 1u8 + bus_nr_running; + bus_nr_running += if n_ports <= 1 { 1 } else { n_ports as u8 * 4 }; + + let cpu_mem_node = socket_numa_node(&topo.sockets, group.socket); + let global_group_idx = gpu_group_count + nic_group_idx; + let pxb_id = format!("pcie.{}", global_group_idx + 1); + + let mut root_ports = Vec::new(); + for pci_addr in &group.pci_bus_addrs { + let dev_id = format!("dev{nic_dev_idx}"); + let rp_id = format!("pcie.port{port_idx}"); + + root_ports.push(PciRootPort { + id: rp_id, + chassis: (nic_dev_idx + 1) as u8, + slot: None, + multifunction: None, + io_reserve: Some(0), + device: Some(VfioDevice { + id: dev_id, + host: pci_addr.clone(), + rombar: Some(false), + kind: VfioDeviceKind::Nic, + iommufd_id: None, + pci_vendor_id: None, + pci_device_id: None, + }), + }); + + nic_dev_idx += 1; + port_idx += 1; + } + + self.pci.roots.push(PciRootComplex { + id: pxb_id, + bus_nr, + numa_node: Some(cpu_mem_node), + iommu: Some(BusIommu::SmmuV3(SmmuV3Config { + id: format!("smmuv3.{}", global_group_idx + 1), + ..SmmuV3Config::default() + })), + root_ports, + }); + } } pub(crate) fn with_hugepages(self, path: &str) -> Self { diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs index 08aa649605..c185b221b9 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs @@ -6,7 +6,13 @@ use std::ops::Range; pub(crate) struct HostTopology { pub sockets: Vec, + /// GPU devices: each group maps to one pxb-pcie + arm-smmuv3 complex. + /// GPUs emit 8 acpi-generic-initiator NUMA nodes per device. pub gpu_smmu_groups: Vec, + /// NIC devices: each group maps to its own pxb-pcie + arm-smmuv3. + /// NICs do NOT emit acpi-generic-initiator links or NUMA initiator nodes. + /// Allocated after all GPU pxb complexes in bus_nr ordering. + pub nic_smmu_groups: Vec, pub egm_sockets: Vec, /// `-numa dist` entries emitted after all NUMA nodes. Each tuple is (src, dst, val). pub numa_distances: Vec<(u32, u32, u32)>, diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs index 4ceaafaa23..2d872f5a6c 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs @@ -60,12 +60,12 @@ fn smmu_groups(addrs: &[&[&str]], socket: u32) -> Vec { // ---- Grace Config 1: single GPU, 1 SMMU, 9 NUMA nodes ---- #[test] -#[ignore = "Phase 4"] fn grace_1_single_gpu() { check( HostTopology { sockets: single_socket(0..4), gpu_smmu_groups: smmu_groups(&[&["0008:06:00.0"]], 0), + nic_smmu_groups: vec![], egm_sockets: vec![], numa_distances: vec![], pcie_root_port: 0, @@ -78,7 +78,6 @@ fn grace_1_single_gpu() { // ---- Grace Config 2: 4 GPUs, 1 GPU per SMMU, 33 NUMA nodes ---- #[test] -#[ignore = "Phase 4"] fn grace_2_four_gpus_1_per_smmu() { check( HostTopology { @@ -92,6 +91,7 @@ fn grace_2_four_gpus_1_per_smmu() { ], 0, ), + nic_smmu_groups: vec![], egm_sockets: vec![], numa_distances: vec![], pcie_root_port: 0, @@ -104,7 +104,6 @@ fn grace_2_four_gpus_1_per_smmu() { // ---- Grace Config 3: 4 GPUs, 2 GPUs per SMMU, 33 NUMA nodes ---- #[test] -#[ignore = "Phase 4"] fn grace_3_four_gpus_2_per_smmu() { check( HostTopology { @@ -116,6 +115,7 @@ fn grace_3_four_gpus_2_per_smmu() { ], 0, ), + nic_smmu_groups: vec![], egm_sockets: vec![], numa_distances: vec![], pcie_root_port: 0, @@ -126,19 +126,19 @@ fn grace_3_four_gpus_2_per_smmu() { } // ---- Grace Config 4: GPU + NIC passthrough ---- +// +// GPU on pcie.1, NIC on pcie.2. NIC uses vfio-pci (not nohotplug) and emits +// no acpi-generic-initiator links. NUMA initiator nodes are GPU-only (9 total). #[test] -#[ignore = "Phase 4"] fn grace_4_gpu_and_nic() { - // Placeholder until Phase 4: HostTopology cannot represent NIC passthrough - // yet, so this topology covers only the GPU half and the test stays - // ignored. Phase 4 adds nic_smmu_groups and rewrites this body; the - // fixture already defines the full expected output (GPU on pcie.1, NIC on - // pcie.2 with no acpi-generic-initiator links). check( HostTopology { sockets: single_socket(0..4), gpu_smmu_groups: smmu_groups(&[&["0008:06:00.0"]], 0), + nic_smmu_groups: vec![ + GpuSmmuGroup { pci_bus_addrs: vec!["0008:00:00.0".into()], socket: 0 }, + ], egm_sockets: vec![], numa_distances: vec![], pcie_root_port: 0, @@ -159,6 +159,7 @@ fn grace_5_vcmdq() { let topo = HostTopology { sockets: single_socket(0..4), gpu_smmu_groups: smmu_groups(&[&["0008:06:00.0"]], 0), + nic_smmu_groups: vec![], egm_sockets: vec![], numa_distances: vec![], pcie_root_port: 0, @@ -191,6 +192,7 @@ fn grace_6_vegm_1_per_socket() { GpuSmmuGroup { pci_bus_addrs: vec!["0010:06:00.0".into()], socket: 2 }, GpuSmmuGroup { pci_bus_addrs: vec!["0011:06:00.0".into()], socket: 3 }, ], + nic_smmu_groups: vec![], egm_sockets: vec![ EgmSocketInfo { path: "/dev/egm4".into(), socket: 0, total_size: 56896 << 20 }, EgmSocketInfo { path: "/dev/egm5".into(), socket: 1, total_size: 56896 << 20 }, @@ -226,6 +228,7 @@ fn grace_7_vegm_2_per_socket() { socket: 1, }, ], + nic_smmu_groups: vec![], egm_sockets: vec![ EgmSocketInfo { path: "/dev/egm4".into(), socket: 0, total_size: 56896 << 20 }, EgmSocketInfo { path: "/dev/egm5".into(), socket: 1, total_size: 56896 << 20 }, @@ -238,6 +241,37 @@ fn grace_7_vegm_2_per_socket() { ); } +// ---- GB300 NVL dry-run: Grace + 2 B200 GPUs, highmem-mmio-size=8T ---- +// +// GB300 (Grace + Blackwell) requires highmem-mmio-size=8T because B200 BARs +// exceed the 4T window used by GH200/GB200. The topology is otherwise identical +// to Config 3 (2 GPUs per SMMU on a single Grace socket). +// +// This fixture is a dry-run: real GB300 BDFs are used as placeholders. +// `apply_host_defaults` with highmem override drives the GB300-specific machine line. + +#[test] +fn gb300_nvl_2gpu() { + let topo = HostTopology { + sockets: single_socket(0..4), + gpu_smmu_groups: smmu_groups(&[&["0008:06:00.0", "0009:06:00.0"]], 0), + nic_smmu_groups: vec![], + egm_sockets: vec![], + numa_distances: vec![], + pcie_root_port: 0, + protection: None, + }; + let mut platform = Platform::from_config_defaults("virt", 16 << 30).expect("build"); + // B200 BARs require 8T highmem window; override the default 4T. + if let Machine::Virt(ref mut v) = platform.machine { + v.highmem_mmio_size = Some(8 << 40); + } + platform.apply_host_defaults(&topo); + let got = platform.to_qemu_args().expect("to_qemu_args"); + let want = load_fixture("gb300_nvl_2gpu.args"); + assert_eq!(want, got); +} + // ---- Q35 CoCo (SEV-SNP) + single GPU — AMD EPYC host, H100 80GB ---- // // Production capture: AMD EPYC host, 2026-07-13. 17 vCPUs, 57344M, single @@ -608,6 +642,7 @@ fn q35_vanilla_kata_x86() { }, ], gpu_smmu_groups: vec![], + nic_smmu_groups: vec![], egm_sockets: vec![], numa_distances: vec![(0, 1, 20), (1, 0, 20)], pcie_root_port: 8, diff --git a/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_4_gpu_and_nic.args b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_4_gpu_and_nic.args index 50b5e75c70..5fa8003842 100644 --- a/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_4_gpu_and_nic.args +++ b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_4_gpu_and_nic.args @@ -4,7 +4,7 @@ -object iommufd,id=iommufd0 -object -memory-backend-ram,size=16G,id=m0 +memory-backend-ram,id=m0,size=16G -machine virt,accel=kvm,gic-version=3,ras=on,highmem-mmio-size=4T,memory-backend=m0 -numa @@ -26,7 +26,7 @@ node,nodeid=7 -numa node,nodeid=8 -device -pxb-pcie,id=pcie.1,bus_nr=1,bus=pcie.0,numa_node=0 +pxb-pcie,id=pcie.1,bus=pcie.0,bus_nr=1,numa_node=0 -device arm-smmuv3,primary-bus=pcie.1,id=smmuv3.1,accel=on,ats=on,ril=off,pasid=on,oas=48 -device @@ -34,13 +34,13 @@ pcie-root-port,id=pcie.port1,bus=pcie.1,chassis=1,io-reserve=0 -device vfio-pci-nohotplug,host=0008:06:00.0,bus=pcie.port1,rombar=0,id=dev0,iommufd=iommufd0 -device -pxb-pcie,id=pcie.2,bus_nr=2,bus=pcie.0,numa_node=0 +pxb-pcie,id=pcie.2,bus=pcie.0,bus_nr=2,numa_node=0 -device arm-smmuv3,primary-bus=pcie.2,id=smmuv3.2,accel=on,ats=on,ril=off,pasid=on,oas=48 -device pcie-root-port,id=pcie.port2,bus=pcie.2,chassis=2,io-reserve=0 -device -vfio-pci-nohotplug,host=0008:00:00.0,bus=pcie.port2,rombar=0,id=dev1,iommufd=iommufd0 +vfio-pci,host=0008:00:00.0,bus=pcie.port2,rombar=0,id=dev1,iommufd=iommufd0 -object acpi-generic-initiator,id=gi0,pci-dev=dev0,node=1 -object