github/kata-containers
1
0
Fork 2
mirror of https://github.com/kata-containers/kata-containers.git synced 2026-07-25 06:00:18 +00:00
Code Issues Projects Releases Wiki Activity

qemu/machine: Phase 4 -- multi-RC NIC groups, all Grace 1-4 tests pass

Browse Source 
Add HostTopology::nic_smmu_groups (Vec<GpuSmmuGroup>): NIC devices get
their own pxb+smmuv3 complex allocated after all GPU complexes, but emit
no acpi-generic-initiator links and no NUMA initiator nodes.  Reuses
GpuSmmuGroup to carry socket affinity.

apply_virt_defaults processes nic_smmu_groups after gpu_smmu_groups,
continuing bus_nr and port_idx counters; VfioDeviceKind::Nic maps to
vfio-pci in emit_vfio_grace.

grace_4_gpu_and_nic fixture updated: GPU on pcie.1 (bus_nr=1), NIC on
pcie.2 (bus_nr=2), NIC emits vfio-pci not vfio-pci-nohotplug.

Unignore grace_1 through grace_4, add the gb300_nvl_2gpu dry-run test
(fixture landed in the previous commit), and add nic_smmu_groups:
vec![] to all existing HostTopology constructions.

Result: 9 fixture tests pass (4 Grace + 4 Q35 + GB300), grace_5-7
stay ignored until Phase 5.

Assisted-by: Claude <noreply@anthropic.com>
Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
This commit is contained in:
Zvonko Kaiser
2026-07-17 13:45:48 +00:00
parent 94b584b517
commit d50718212e
4 changed files with 105 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs
View File

@@ -502,6 +502,57 @@ impl Platform {
root_ports,
});
}
// NIC groups: own pxb+smmuv3 complex, no GI links, no extra NUMA nodes.
// pxb numbering continues from where GPU groups left off.
let gpu_group_count = topo.gpu_smmu_groups.len();
let mut nic_dev_idx = gpu_idx;
for (nic_group_idx, group) in topo.nic_smmu_groups.iter().enumerate() {
let n_ports = group.pci_bus_addrs.len();
let bus_nr = 1u8 + bus_nr_running;
bus_nr_running += if n_ports <= 1 { 1 } else { n_ports as u8 * 4 };
let cpu_mem_node = socket_numa_node(&topo.sockets, group.socket);
let global_group_idx = gpu_group_count + nic_group_idx;
let pxb_id = format!("pcie.{}", global_group_idx + 1);
let mut root_ports = Vec::new();
for pci_addr in &group.pci_bus_addrs {
let dev_id = format!("dev{nic_dev_idx}");
let rp_id = format!("pcie.port{port_idx}");
root_ports.push(PciRootPort {
id: rp_id,
chassis: (nic_dev_idx + 1) as u8,
slot: None,
multifunction: None,
io_reserve: Some(0),
device: Some(VfioDevice {
id: dev_id,
host: pci_addr.clone(),
rombar: Some(false),
kind: VfioDeviceKind::Nic,
iommufd_id: None,
pci_vendor_id: None,
pci_device_id: None,
}),
});
nic_dev_idx += 1;
port_idx += 1;
}
self.pci.roots.push(PciRootComplex {
id: pxb_id,
bus_nr,
numa_node: Some(cpu_mem_node),
iommu: Some(BusIommu::SmmuV3(SmmuV3Config {
id: format!("smmuv3.{}", global_group_idx + 1),
..SmmuV3Config::default()
})),
root_ports,
});
}
}
pub(crate) fn with_hugepages(self, path: &str) -> Self {

6
src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs
View File

@@ -6,7 +6,13 @@ use std::ops::Range;
pub(crate) struct HostTopology {
pub sockets: Vec<SocketInfo>,
/// GPU devices: each group maps to one pxb-pcie + arm-smmuv3 complex.
/// GPUs emit 8 acpi-generic-initiator NUMA nodes per device.
pub gpu_smmu_groups: Vec<GpuSmmuGroup>,
/// NIC devices: each group maps to its own pxb-pcie + arm-smmuv3.
/// NICs do NOT emit acpi-generic-initiator links or NUMA initiator nodes.
/// Allocated after all GPU pxb complexes in bus_nr ordering.
pub nic_smmu_groups: Vec<GpuSmmuGroup>,
pub egm_sockets: Vec<EgmSocketInfo>,
/// `-numa dist` entries emitted after all NUMA nodes. Each tuple is (src, dst, val).
pub numa_distances: Vec<(u32, u32, u32)>,

53
src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs
View File

@@ -60,12 +60,12 @@ fn smmu_groups(addrs: &[&[&str]], socket: u32) -> Vec<GpuSmmuGroup> {
// ---- Grace Config 1: single GPU, 1 SMMU, 9 NUMA nodes ----
#[test]
#[ignore = "Phase 4"]
fn grace_1_single_gpu() {
check(
HostTopology {
sockets: single_socket(0..4),
gpu_smmu_groups: smmu_groups(&[&["0008:06:00.0"]], 0),
nic_smmu_groups: vec![],
egm_sockets: vec![],
numa_distances: vec![],
pcie_root_port: 0,
@@ -78,7 +78,6 @@ fn grace_1_single_gpu() {
// ---- Grace Config 2: 4 GPUs, 1 GPU per SMMU, 33 NUMA nodes ----
#[test]
#[ignore = "Phase 4"]
fn grace_2_four_gpus_1_per_smmu() {
check(
HostTopology {
@@ -92,6 +91,7 @@ fn grace_2_four_gpus_1_per_smmu() {
],
0,
),
nic_smmu_groups: vec![],
egm_sockets: vec![],
numa_distances: vec![],
pcie_root_port: 0,
@@ -104,7 +104,6 @@ fn grace_2_four_gpus_1_per_smmu() {
// ---- Grace Config 3: 4 GPUs, 2 GPUs per SMMU, 33 NUMA nodes ----
#[test]
#[ignore = "Phase 4"]
fn grace_3_four_gpus_2_per_smmu() {
check(
HostTopology {
@@ -116,6 +115,7 @@ fn grace_3_four_gpus_2_per_smmu() {
],
0,
),
nic_smmu_groups: vec![],
egm_sockets: vec![],
numa_distances: vec![],
pcie_root_port: 0,
@@ -126,19 +126,19 @@ fn grace_3_four_gpus_2_per_smmu() {
}
// ---- Grace Config 4: GPU + NIC passthrough ----
//
// GPU on pcie.1, NIC on pcie.2. NIC uses vfio-pci (not nohotplug) and emits
// no acpi-generic-initiator links. NUMA initiator nodes are GPU-only (9 total).
#[test]
#[ignore = "Phase 4"]
fn grace_4_gpu_and_nic() {
// Placeholder until Phase 4: HostTopology cannot represent NIC passthrough
// yet, so this topology covers only the GPU half and the test stays
// ignored. Phase 4 adds nic_smmu_groups and rewrites this body; the
// fixture already defines the full expected output (GPU on pcie.1, NIC on
// pcie.2 with no acpi-generic-initiator links).
check(
HostTopology {
sockets: single_socket(0..4),
gpu_smmu_groups: smmu_groups(&[&["0008:06:00.0"]], 0),
nic_smmu_groups: vec![
GpuSmmuGroup { pci_bus_addrs: vec!["0008:00:00.0".into()], socket: 0 },
],
egm_sockets: vec![],
numa_distances: vec![],
pcie_root_port: 0,
@@ -159,6 +159,7 @@ fn grace_5_vcmdq() {
let topo = HostTopology {
sockets: single_socket(0..4),
gpu_smmu_groups: smmu_groups(&[&["0008:06:00.0"]], 0),
nic_smmu_groups: vec![],
egm_sockets: vec![],
numa_distances: vec![],
pcie_root_port: 0,
@@ -191,6 +192,7 @@ fn grace_6_vegm_1_per_socket() {
GpuSmmuGroup { pci_bus_addrs: vec!["0010:06:00.0".into()], socket: 2 },
GpuSmmuGroup { pci_bus_addrs: vec!["0011:06:00.0".into()], socket: 3 },
],
nic_smmu_groups: vec![],
egm_sockets: vec![
EgmSocketInfo { path: "/dev/egm4".into(), socket: 0, total_size: 56896 << 20 },
EgmSocketInfo { path: "/dev/egm5".into(), socket: 1, total_size: 56896 << 20 },
@@ -226,6 +228,7 @@ fn grace_7_vegm_2_per_socket() {
socket: 1,
},
],
nic_smmu_groups: vec![],
egm_sockets: vec![
EgmSocketInfo { path: "/dev/egm4".into(), socket: 0, total_size: 56896 << 20 },
EgmSocketInfo { path: "/dev/egm5".into(), socket: 1, total_size: 56896 << 20 },
@@ -238,6 +241,37 @@ fn grace_7_vegm_2_per_socket() {
);
}
// ---- GB300 NVL dry-run: Grace + 2 B200 GPUs, highmem-mmio-size=8T ----
//
// GB300 (Grace + Blackwell) requires highmem-mmio-size=8T because B200 BARs
// exceed the 4T window used by GH200/GB200. The topology is otherwise identical
// to Config 3 (2 GPUs per SMMU on a single Grace socket).
//
// This fixture is a dry-run: real GB300 BDFs are used as placeholders.
// `apply_host_defaults` with highmem override drives the GB300-specific machine line.
#[test]
fn gb300_nvl_2gpu() {
let topo = HostTopology {
sockets: single_socket(0..4),
gpu_smmu_groups: smmu_groups(&[&["0008:06:00.0", "0009:06:00.0"]], 0),
nic_smmu_groups: vec![],
egm_sockets: vec![],
numa_distances: vec![],
pcie_root_port: 0,
protection: None,
};
let mut platform = Platform::from_config_defaults("virt", 16 << 30).expect("build");
// B200 BARs require 8T highmem window; override the default 4T.
if let Machine::Virt(ref mut v) = platform.machine {
v.highmem_mmio_size = Some(8 << 40);
}
platform.apply_host_defaults(&topo);
let got = platform.to_qemu_args().expect("to_qemu_args");
let want = load_fixture("gb300_nvl_2gpu.args");
assert_eq!(want, got);
}
// ---- Q35 CoCo (SEV-SNP) + single GPU — AMD EPYC host, H100 80GB ----
//
// Production capture: AMD EPYC host, 2026-07-13. 17 vCPUs, 57344M, single
@@ -608,6 +642,7 @@ fn q35_vanilla_kata_x86() {
},
],
gpu_smmu_groups: vec![],
nic_smmu_groups: vec![],
egm_sockets: vec![],
numa_distances: vec![(0, 1, 20), (1, 0, 20)],
pcie_root_port: 8,

8
src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_4_gpu_and_nic.args vendored
View File

@@ -4,7 +4,7 @@
-object
iommufd,id=iommufd0
-object
memory-backend-ram,size=16G,id=m0
memory-backend-ram,id=m0,size=16G
-machine
virt,accel=kvm,gic-version=3,ras=on,highmem-mmio-size=4T,memory-backend=m0
-numa
@@ -26,7 +26,7 @@ node,nodeid=7
-numa
node,nodeid=8
-device
pxb-pcie,id=pcie.1,bus_nr=1,bus=pcie.0,numa_node=0
pxb-pcie,id=pcie.1,bus=pcie.0,bus_nr=1,numa_node=0
-device
arm-smmuv3,primary-bus=pcie.1,id=smmuv3.1,accel=on,ats=on,ril=off,pasid=on,oas=48
-device
@@ -34,13 +34,13 @@ pcie-root-port,id=pcie.port1,bus=pcie.1,chassis=1,io-reserve=0
-device
vfio-pci-nohotplug,host=0008:06:00.0,bus=pcie.port1,rombar=0,id=dev0,iommufd=iommufd0
-device
pxb-pcie,id=pcie.2,bus_nr=2,bus=pcie.0,numa_node=0
pxb-pcie,id=pcie.2,bus=pcie.0,bus_nr=2,numa_node=0
-device
arm-smmuv3,primary-bus=pcie.2,id=smmuv3.2,accel=on,ats=on,ril=off,pasid=on,oas=48
-device
pcie-root-port,id=pcie.port2,bus=pcie.2,chassis=2,io-reserve=0
-device
vfio-pci-nohotplug,host=0008:00:00.0,bus=pcie.port2,rombar=0,id=dev1,iommufd=iommufd0
vfio-pci,host=0008:00:00.0,bus=pcie.port2,rombar=0,id=dev1,iommufd=iommufd0
-object
acpi-generic-initiator,id=gi0,pci-dev=dev0,node=1
-object