From f8af92b8be90cbd515fedd9940a793721cb189b0 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Mon, 13 Jul 2026 18:50:27 +0000 Subject: [PATCH] hypervisor/qemu: add golden test harness for platform generation Seven fixture files encode the expected QEMU argument list for each Grace passthrough topology derived from tested production deployments: grace_1_single_gpu -- 1 GPU, 1 SMMU, 9 NUMA nodes grace_2_four_gpus_1_per_smmu -- 4 GPUs x 1-per-SMMU, 33 NUMA nodes grace_3_four_gpus_2_per_smmu -- 4 GPUs x 2-per-SMMU, 33 NUMA nodes grace_4_gpu_and_nic -- GPU + NIC, NIC emits no generic-initiators grace_5_vcmdq -- hugepages backing + cmdqv=on on arm-smmuv3 grace_6_vegm_1_per_socket -- vEGM, 4 sockets, 1 GPU each grace_7_vegm_2_per_socket -- vEGM, 2 sockets, 2 GPUs each Each test is #[ignore] until the corresponding phase lands. The harness reads fixtures as one Vec element per line; blank lines and lines starting with '#' are skipped. The fixtures are the contract: any Platform implementation must reproduce them exactly before a phase PR can be merged. Refs: #12187, #12125 Signed-off-by: Zvonko Kaiser Assisted-by: Claude Sonnet 4.6 Signed-off-by: Zvonko Kaiser --- .../hypervisor/src/qemu/ARCHITECTURE.md | 4 - .../crates/hypervisor/src/qemu/machine/mod.rs | 2 +- .../hypervisor/src/qemu/machine/platform.rs | 3 +- .../hypervisor/src/qemu/machine/probe.rs | 2 +- .../hypervisor/src/qemu/machine/pseries.rs | 2 +- .../crates/hypervisor/src/qemu/machine/q35.rs | 2 +- .../hypervisor/src/qemu/machine/s390x.rs | 2 +- .../hypervisor/src/qemu/machine/tests.rs | 196 ++++++++++++++++++ .../hypervisor/src/qemu/machine/topology.rs | 2 +- .../hypervisor/src/qemu/machine/virt.rs | 2 +- .../fixtures/qemu/grace_1_single_gpu.args | 50 +++++ .../qemu/grace_2_four_gpus_1_per_smmu.args | 170 +++++++++++++++ .../qemu/grace_3_four_gpus_2_per_smmu.args | 162 +++++++++++++++ .../fixtures/qemu/grace_4_gpu_and_nic.args | 59 ++++++ .../tests/fixtures/qemu/grace_5_vcmdq.args | 51 +++++ .../qemu/grace_6_vegm_1_per_socket.args | 191 +++++++++++++++++ .../qemu/grace_7_vegm_2_per_socket.args | 176 ++++++++++++++++ 17 files changed, 1063 insertions(+), 13 deletions(-) create mode 100644 src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs create mode 100644 src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_1_single_gpu.args create mode 100644 src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_2_four_gpus_1_per_smmu.args create mode 100644 src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_3_four_gpus_2_per_smmu.args create mode 100644 src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_4_gpu_and_nic.args create mode 100644 src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_5_vcmdq.args create mode 100644 src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_6_vegm_1_per_socket.args create mode 100644 src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_7_vegm_2_per_socket.args diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/ARCHITECTURE.md b/src/runtime-rs/crates/hypervisor/src/qemu/ARCHITECTURE.md index d1e4fda2e4..ad8ec2fdc5 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/ARCHITECTURE.md +++ b/src/runtime-rs/crates/hypervisor/src/qemu/ARCHITECTURE.md @@ -299,10 +299,6 @@ generic-initiator NUMA nodes regardless of whether MIG is in use. The GPU driver (CUDA) uses these nodes to online GPU memory to the guest kernel. Total node count with 4 GPUs on a single-socket host: 1 CpuMem + 4 × 8 = 33 nodes. -**Memory hotplug placeholder:** QEMU assigns the last-defined NUMA node to the -hotplug region. When DIMM/NVDIMM hotplug is enabled, always append one empty -NUMA node after all real nodes so hotplug memory does not overlap GPU nodes. - **GPU memory spill prevention:** GPU NUMA nodes may attract page migration from `autonuma` or systemd NUMA policies. Mitigate with explicit NUMA distances: diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/mod.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/mod.rs index 44093c9e3f..36ce256db7 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/mod.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2025 NVIDIA Corporation +// Copyright (c) NVIDIA Corporation // // SPDX-License-Identifier: Apache-2.0 diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs index 65d0df9f82..b86e7f2c71 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2025 NVIDIA Corporation +// Copyright (c) NVIDIA Corporation // // SPDX-License-Identifier: Apache-2.0 @@ -30,7 +30,6 @@ pub(crate) enum Machine { /// Fields common to all machine types. pub(crate) struct BaseMachine { pub accel: String, - pub nvdimm: bool, /// ID of the primary memory backend, written as `-machine memory-backend=`. pub memory_backend: Option, } diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs index cacf7714b7..7013c6600f 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2025 NVIDIA Corporation +// Copyright (c) NVIDIA Corporation // // SPDX-License-Identifier: Apache-2.0 diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/pseries.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/pseries.rs index 26a205243e..1dc747d966 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/pseries.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/pseries.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2025 NVIDIA Corporation +// Copyright (c) NVIDIA Corporation // // SPDX-License-Identifier: Apache-2.0 diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/q35.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/q35.rs index 5a216cbfbe..95908ad690 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/q35.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/q35.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2025 NVIDIA Corporation +// Copyright (c) NVIDIA Corporation // // SPDX-License-Identifier: Apache-2.0 diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/s390x.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/s390x.rs index 7d230dcee0..850bc9a734 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/s390x.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/s390x.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2025 NVIDIA Corporation +// Copyright (c) NVIDIA Corporation // // SPDX-License-Identifier: Apache-2.0 diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs new file mode 100644 index 0000000000..6cd1c8ced3 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs @@ -0,0 +1,196 @@ +// Copyright (c) NVIDIA Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +use std::path::Path; + +use super::platform::Platform; +use super::probe::{EgmSocketInfo, GpuSmmuGroup, HostTopology, SocketInfo}; + +// Each fixture file contains one Vec element per line. +// Blank lines and lines starting with '#' are ignored. +fn load_fixture(name: &str) -> Vec { + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/qemu") + .join(name); + std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("fixture {}: {e}", path.display())) + .lines() + .map(str::trim) + .filter(|l| !l.is_empty() && !l.starts_with('#')) + .map(String::from) + .collect() +} + +fn check(topo: HostTopology, fixture: &str) { + let mut platform = Platform::from_config_defaults(16 << 30).expect("Platform::from_config_defaults"); + platform.apply_host_defaults(&topo); + let got = platform.to_qemu_args().expect("to_qemu_args"); + let want = load_fixture(fixture); + assert_eq!(want, got); +} + +fn single_socket(cpus: std::ops::Range) -> Vec { + vec![SocketInfo { id: 0, cpu_range: cpus }] +} + +fn smmu_groups(addrs: &[&[&str]], socket: u32) -> Vec { + addrs + .iter() + .map(|group| GpuSmmuGroup { + pci_bus_addrs: group.iter().map(|s| s.to_string()).collect(), + socket, + }) + .collect() +} + +// ---- Grace Config 1: single GPU, 1 SMMU, 9 NUMA nodes ---- + +#[test] +#[ignore = "Phase 4"] +fn grace_1_single_gpu() { + check( + HostTopology { + sockets: single_socket(0..4), + gpu_smmu_groups: smmu_groups(&[&["0008:06:00.0"]], 0), + egm_sockets: vec![], + }, + "grace_1_single_gpu.args", + ); +} + +// ---- Grace Config 2: 4 GPUs, 1 GPU per SMMU, 33 NUMA nodes ---- + +#[test] +#[ignore = "Phase 4"] +fn grace_2_four_gpus_1_per_smmu() { + check( + HostTopology { + sockets: single_socket(0..4), + gpu_smmu_groups: smmu_groups( + &[ + &["0008:06:00.0"], + &["0009:06:00.0"], + &["0010:06:00.0"], + &["0011:06:00.0"], + ], + 0, + ), + egm_sockets: vec![], + }, + "grace_2_four_gpus_1_per_smmu.args", + ); +} + +// ---- Grace Config 3: 4 GPUs, 2 GPUs per SMMU, 33 NUMA nodes ---- + +#[test] +#[ignore = "Phase 4"] +fn grace_3_four_gpus_2_per_smmu() { + check( + HostTopology { + sockets: single_socket(0..4), + gpu_smmu_groups: smmu_groups( + &[ + &["0008:06:00.0", "0009:06:00.0"], + &["0010:06:00.0", "0011:06:00.0"], + ], + 0, + ), + egm_sockets: vec![], + }, + "grace_3_four_gpus_2_per_smmu.args", + ); +} + +// ---- Grace Config 4: GPU + NIC passthrough ---- + +#[test] +#[ignore = "Phase 4"] +fn grace_4_gpu_and_nic() { + // NIC is represented differently; exact HostTopology shape TBD in Phase 4. + // Fixture defines the expected output; this test drives the API design. + check( + HostTopology { + sockets: single_socket(0..4), + gpu_smmu_groups: smmu_groups(&[&["0008:06:00.0"]], 0), + egm_sockets: vec![], + }, + "grace_4_gpu_and_nic.args", + ); +} + +// ---- Grace Config 5: vCMDQ, hugepages backing ---- + +#[test] +#[ignore = "Phase 5"] +fn grace_5_vcmdq() { + check( + HostTopology { + sockets: single_socket(0..4), + gpu_smmu_groups: smmu_groups(&[&["0008:06:00.0"]], 0), + egm_sockets: vec![], + }, + "grace_5_vcmdq.args", + ); +} + +// ---- Grace Config 6: vEGM, 1 GPU per socket, 4 sockets ---- + +#[test] +#[ignore = "Phase 5"] +fn grace_6_vegm_1_per_socket() { + check( + HostTopology { + sockets: vec![ + SocketInfo { id: 0, cpu_range: 0..1 }, + SocketInfo { id: 1, cpu_range: 1..2 }, + SocketInfo { id: 2, cpu_range: 2..3 }, + SocketInfo { id: 3, cpu_range: 3..4 }, + ], + gpu_smmu_groups: vec![ + GpuSmmuGroup { pci_bus_addrs: vec!["0008:06:00.0".into()], socket: 0 }, + GpuSmmuGroup { pci_bus_addrs: vec!["0009:06:00.0".into()], socket: 1 }, + GpuSmmuGroup { pci_bus_addrs: vec!["0010:06:00.0".into()], socket: 2 }, + GpuSmmuGroup { pci_bus_addrs: vec!["0011:06:00.0".into()], socket: 3 }, + ], + egm_sockets: vec![ + EgmSocketInfo { path: "/dev/egm4".into(), socket: 0, total_size: 56896 << 20 }, + EgmSocketInfo { path: "/dev/egm5".into(), socket: 1, total_size: 56896 << 20 }, + EgmSocketInfo { path: "/dev/egm6".into(), socket: 2, total_size: 56896 << 20 }, + EgmSocketInfo { path: "/dev/egm7".into(), socket: 3, total_size: 56896 << 20 }, + ], + }, + "grace_6_vegm_1_per_socket.args", + ); +} + +// ---- Grace Config 7: vEGM, 2 GPUs per socket, 2 sockets ---- + +#[test] +#[ignore = "Phase 5"] +fn grace_7_vegm_2_per_socket() { + check( + HostTopology { + sockets: vec![ + SocketInfo { id: 0, cpu_range: 0..2 }, + SocketInfo { id: 1, cpu_range: 2..4 }, + ], + gpu_smmu_groups: vec![ + GpuSmmuGroup { + pci_bus_addrs: vec!["0008:06:00.0".into(), "0009:06:00.0".into()], + socket: 0, + }, + GpuSmmuGroup { + pci_bus_addrs: vec!["0010:06:00.0".into(), "0011:06:00.0".into()], + socket: 1, + }, + ], + egm_sockets: vec![ + EgmSocketInfo { path: "/dev/egm4".into(), socket: 0, total_size: 56896 << 20 }, + EgmSocketInfo { path: "/dev/egm5".into(), socket: 1, total_size: 56896 << 20 }, + ], + }, + "grace_7_vegm_2_per_socket.args", + ); +} diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/topology.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/topology.rs index dabae9afc3..1bfdf360f3 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/topology.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/topology.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2025 NVIDIA Corporation +// Copyright (c) NVIDIA Corporation // // SPDX-License-Identifier: Apache-2.0 diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/virt.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/virt.rs index 9a12c98038..e99dda64c2 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/virt.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/virt.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2025 NVIDIA Corporation +// Copyright (c) NVIDIA Corporation // // SPDX-License-Identifier: Apache-2.0 diff --git a/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_1_single_gpu.args b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_1_single_gpu.args new file mode 100644 index 0000000000..00535add84 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_1_single_gpu.args @@ -0,0 +1,50 @@ +# Grace Config 1 - single GPU, 1 SMMU, 9 NUMA nodes +# HostTopology: 1 socket (cpus 0-3), 1 GpuSmmuGroup [0008:06:00.0], no EGM +-object +iommufd,id=iommufd0 +-object +memory-backend-ram,size=16G,id=m0 +-machine +virt,accel=kvm,gic-version=3,ras=on,highmem-mmio-size=4T,memory-backend=m0 +-numa +node,memdev=m0,cpus=0-3,nodeid=0 +-numa +node,nodeid=1 +-numa +node,nodeid=2 +-numa +node,nodeid=3 +-numa +node,nodeid=4 +-numa +node,nodeid=5 +-numa +node,nodeid=6 +-numa +node,nodeid=7 +-numa +node,nodeid=8 +-device +pxb-pcie,id=pcie.1,bus_nr=1,bus=pcie.0,numa_node=0 +-device +arm-smmuv3,primary-bus=pcie.1,id=smmuv3.1,accel=on,ats=on,ril=off,pasid=on,oas=48 +-device +pcie-root-port,id=pcie.port1,bus=pcie.1,chassis=1,io-reserve=0 +-device +vfio-pci-nohotplug,host=0008:06:00.0,bus=pcie.port1,rombar=0,id=dev0,iommufd=iommufd0 +-object +acpi-generic-initiator,id=gi0,pci-dev=dev0,node=1 +-object +acpi-generic-initiator,id=gi1,pci-dev=dev0,node=2 +-object +acpi-generic-initiator,id=gi2,pci-dev=dev0,node=3 +-object +acpi-generic-initiator,id=gi3,pci-dev=dev0,node=4 +-object +acpi-generic-initiator,id=gi4,pci-dev=dev0,node=5 +-object +acpi-generic-initiator,id=gi5,pci-dev=dev0,node=6 +-object +acpi-generic-initiator,id=gi6,pci-dev=dev0,node=7 +-object +acpi-generic-initiator,id=gi7,pci-dev=dev0,node=8 diff --git a/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_2_four_gpus_1_per_smmu.args b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_2_four_gpus_1_per_smmu.args new file mode 100644 index 0000000000..f5a8eb1b25 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_2_four_gpus_1_per_smmu.args @@ -0,0 +1,170 @@ +# Grace Config 2 - 4 GPUs, 1 GPU per SMMU, 33 NUMA nodes +# HostTopology: 1 socket (cpus 0-3), 4 GpuSmmuGroups each with 1 GPU, no EGM +-object +iommufd,id=iommufd0 +-object +memory-backend-ram,size=16G,id=m0 +-machine +virt,accel=kvm,gic-version=3,ras=on,highmem-mmio-size=4T,memory-backend=m0 +-numa +node,memdev=m0,cpus=0-3,nodeid=0 +-numa +node,nodeid=1 +-numa +node,nodeid=2 +-numa +node,nodeid=3 +-numa +node,nodeid=4 +-numa +node,nodeid=5 +-numa +node,nodeid=6 +-numa +node,nodeid=7 +-numa +node,nodeid=8 +-numa +node,nodeid=9 +-numa +node,nodeid=10 +-numa +node,nodeid=11 +-numa +node,nodeid=12 +-numa +node,nodeid=13 +-numa +node,nodeid=14 +-numa +node,nodeid=15 +-numa +node,nodeid=16 +-numa +node,nodeid=17 +-numa +node,nodeid=18 +-numa +node,nodeid=19 +-numa +node,nodeid=20 +-numa +node,nodeid=21 +-numa +node,nodeid=22 +-numa +node,nodeid=23 +-numa +node,nodeid=24 +-numa +node,nodeid=25 +-numa +node,nodeid=26 +-numa +node,nodeid=27 +-numa +node,nodeid=28 +-numa +node,nodeid=29 +-numa +node,nodeid=30 +-numa +node,nodeid=31 +-numa +node,nodeid=32 +-device +pxb-pcie,id=pcie.1,bus_nr=1,bus=pcie.0,numa_node=0 +-device +arm-smmuv3,primary-bus=pcie.1,id=smmuv3.1,accel=on,ats=on,ril=off,pasid=on,oas=48 +-device +pcie-root-port,id=pcie.port1,bus=pcie.1,chassis=1,io-reserve=0 +-device +vfio-pci-nohotplug,host=0008:06:00.0,bus=pcie.port1,rombar=0,id=dev0,iommufd=iommufd0 +-device +pxb-pcie,id=pcie.2,bus_nr=2,bus=pcie.0,numa_node=0 +-device +arm-smmuv3,primary-bus=pcie.2,id=smmuv3.2,accel=on,ats=on,ril=off,pasid=on,oas=48 +-device +pcie-root-port,id=pcie.port2,bus=pcie.2,chassis=2,io-reserve=0 +-device +vfio-pci-nohotplug,host=0009:06:00.0,bus=pcie.port2,rombar=0,id=dev1,iommufd=iommufd0 +-device +pxb-pcie,id=pcie.3,bus_nr=3,bus=pcie.0,numa_node=0 +-device +arm-smmuv3,primary-bus=pcie.3,id=smmuv3.3,accel=on,ats=on,ril=off,pasid=on,oas=48 +-device +pcie-root-port,id=pcie.port3,bus=pcie.3,chassis=3,io-reserve=0 +-device +vfio-pci-nohotplug,host=0010:06:00.0,bus=pcie.port3,rombar=0,id=dev2,iommufd=iommufd0 +-device +pxb-pcie,id=pcie.4,bus_nr=4,bus=pcie.0,numa_node=0 +-device +arm-smmuv3,primary-bus=pcie.4,id=smmuv3.4,accel=on,ats=on,ril=off,pasid=on,oas=48 +-device +pcie-root-port,id=pcie.port4,bus=pcie.4,chassis=4,io-reserve=0 +-device +vfio-pci-nohotplug,host=0011:06:00.0,bus=pcie.port4,rombar=0,id=dev3,iommufd=iommufd0 +-object +acpi-generic-initiator,id=gi0,pci-dev=dev0,node=1 +-object +acpi-generic-initiator,id=gi1,pci-dev=dev0,node=2 +-object +acpi-generic-initiator,id=gi2,pci-dev=dev0,node=3 +-object +acpi-generic-initiator,id=gi3,pci-dev=dev0,node=4 +-object +acpi-generic-initiator,id=gi4,pci-dev=dev0,node=5 +-object +acpi-generic-initiator,id=gi5,pci-dev=dev0,node=6 +-object +acpi-generic-initiator,id=gi6,pci-dev=dev0,node=7 +-object +acpi-generic-initiator,id=gi7,pci-dev=dev0,node=8 +-object +acpi-generic-initiator,id=gi8,pci-dev=dev1,node=9 +-object +acpi-generic-initiator,id=gi9,pci-dev=dev1,node=10 +-object +acpi-generic-initiator,id=gi10,pci-dev=dev1,node=11 +-object +acpi-generic-initiator,id=gi11,pci-dev=dev1,node=12 +-object +acpi-generic-initiator,id=gi12,pci-dev=dev1,node=13 +-object +acpi-generic-initiator,id=gi13,pci-dev=dev1,node=14 +-object +acpi-generic-initiator,id=gi14,pci-dev=dev1,node=15 +-object +acpi-generic-initiator,id=gi15,pci-dev=dev1,node=16 +-object +acpi-generic-initiator,id=gi16,pci-dev=dev2,node=17 +-object +acpi-generic-initiator,id=gi17,pci-dev=dev2,node=18 +-object +acpi-generic-initiator,id=gi18,pci-dev=dev2,node=19 +-object +acpi-generic-initiator,id=gi19,pci-dev=dev2,node=20 +-object +acpi-generic-initiator,id=gi20,pci-dev=dev2,node=21 +-object +acpi-generic-initiator,id=gi21,pci-dev=dev2,node=22 +-object +acpi-generic-initiator,id=gi22,pci-dev=dev2,node=23 +-object +acpi-generic-initiator,id=gi23,pci-dev=dev2,node=24 +-object +acpi-generic-initiator,id=gi24,pci-dev=dev3,node=25 +-object +acpi-generic-initiator,id=gi25,pci-dev=dev3,node=26 +-object +acpi-generic-initiator,id=gi26,pci-dev=dev3,node=27 +-object +acpi-generic-initiator,id=gi27,pci-dev=dev3,node=28 +-object +acpi-generic-initiator,id=gi28,pci-dev=dev3,node=29 +-object +acpi-generic-initiator,id=gi29,pci-dev=dev3,node=30 +-object +acpi-generic-initiator,id=gi30,pci-dev=dev3,node=31 +-object +acpi-generic-initiator,id=gi31,pci-dev=dev3,node=32 diff --git a/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_3_four_gpus_2_per_smmu.args b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_3_four_gpus_2_per_smmu.args new file mode 100644 index 0000000000..9640dca061 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_3_four_gpus_2_per_smmu.args @@ -0,0 +1,162 @@ +# Grace Config 3 - 4 GPUs, 2 GPUs per SMMU, 33 NUMA nodes +# HostTopology: 1 socket (cpus 0-3), 2 GpuSmmuGroups each with 2 GPUs, no EGM +-object +iommufd,id=iommufd0 +-object +memory-backend-ram,size=16G,id=m0 +-machine +virt,accel=kvm,gic-version=3,ras=on,highmem-mmio-size=4T,memory-backend=m0 +-numa +node,memdev=m0,cpus=0-3,nodeid=0 +-numa +node,nodeid=1 +-numa +node,nodeid=2 +-numa +node,nodeid=3 +-numa +node,nodeid=4 +-numa +node,nodeid=5 +-numa +node,nodeid=6 +-numa +node,nodeid=7 +-numa +node,nodeid=8 +-numa +node,nodeid=9 +-numa +node,nodeid=10 +-numa +node,nodeid=11 +-numa +node,nodeid=12 +-numa +node,nodeid=13 +-numa +node,nodeid=14 +-numa +node,nodeid=15 +-numa +node,nodeid=16 +-numa +node,nodeid=17 +-numa +node,nodeid=18 +-numa +node,nodeid=19 +-numa +node,nodeid=20 +-numa +node,nodeid=21 +-numa +node,nodeid=22 +-numa +node,nodeid=23 +-numa +node,nodeid=24 +-numa +node,nodeid=25 +-numa +node,nodeid=26 +-numa +node,nodeid=27 +-numa +node,nodeid=28 +-numa +node,nodeid=29 +-numa +node,nodeid=30 +-numa +node,nodeid=31 +-numa +node,nodeid=32 +-device +pxb-pcie,id=pcie.1,bus_nr=1,bus=pcie.0,numa_node=0 +-device +arm-smmuv3,primary-bus=pcie.1,id=smmuv3.1,accel=on,ats=on,ril=off,pasid=on,oas=48 +-device +pcie-root-port,id=pcie.port1,bus=pcie.1,chassis=1,io-reserve=0 +-device +vfio-pci-nohotplug,host=0008:06:00.0,bus=pcie.port1,rombar=0,id=dev0,iommufd=iommufd0 +-device +pcie-root-port,id=pcie.port2,bus=pcie.1,chassis=2,io-reserve=0 +-device +vfio-pci-nohotplug,host=0009:06:00.0,bus=pcie.port2,rombar=0,id=dev1,iommufd=iommufd0 +-device +pxb-pcie,id=pcie.2,bus_nr=9,bus=pcie.0,numa_node=0 +-device +arm-smmuv3,primary-bus=pcie.2,id=smmuv3.2,accel=on,ats=on,ril=off,pasid=on,oas=48 +-device +pcie-root-port,id=pcie.port3,bus=pcie.2,chassis=3,io-reserve=0 +-device +vfio-pci-nohotplug,host=0010:06:00.0,bus=pcie.port3,rombar=0,id=dev2,iommufd=iommufd0 +-device +pcie-root-port,id=pcie.port4,bus=pcie.2,chassis=4,io-reserve=0 +-device +vfio-pci-nohotplug,host=0011:06:00.0,bus=pcie.port4,rombar=0,id=dev3,iommufd=iommufd0 +-object +acpi-generic-initiator,id=gi0,pci-dev=dev0,node=1 +-object +acpi-generic-initiator,id=gi1,pci-dev=dev0,node=2 +-object +acpi-generic-initiator,id=gi2,pci-dev=dev0,node=3 +-object +acpi-generic-initiator,id=gi3,pci-dev=dev0,node=4 +-object +acpi-generic-initiator,id=gi4,pci-dev=dev0,node=5 +-object +acpi-generic-initiator,id=gi5,pci-dev=dev0,node=6 +-object +acpi-generic-initiator,id=gi6,pci-dev=dev0,node=7 +-object +acpi-generic-initiator,id=gi7,pci-dev=dev0,node=8 +-object +acpi-generic-initiator,id=gi8,pci-dev=dev1,node=9 +-object +acpi-generic-initiator,id=gi9,pci-dev=dev1,node=10 +-object +acpi-generic-initiator,id=gi10,pci-dev=dev1,node=11 +-object +acpi-generic-initiator,id=gi11,pci-dev=dev1,node=12 +-object +acpi-generic-initiator,id=gi12,pci-dev=dev1,node=13 +-object +acpi-generic-initiator,id=gi13,pci-dev=dev1,node=14 +-object +acpi-generic-initiator,id=gi14,pci-dev=dev1,node=15 +-object +acpi-generic-initiator,id=gi15,pci-dev=dev1,node=16 +-object +acpi-generic-initiator,id=gi16,pci-dev=dev2,node=17 +-object +acpi-generic-initiator,id=gi17,pci-dev=dev2,node=18 +-object +acpi-generic-initiator,id=gi18,pci-dev=dev2,node=19 +-object +acpi-generic-initiator,id=gi19,pci-dev=dev2,node=20 +-object +acpi-generic-initiator,id=gi20,pci-dev=dev2,node=21 +-object +acpi-generic-initiator,id=gi21,pci-dev=dev2,node=22 +-object +acpi-generic-initiator,id=gi22,pci-dev=dev2,node=23 +-object +acpi-generic-initiator,id=gi23,pci-dev=dev2,node=24 +-object +acpi-generic-initiator,id=gi24,pci-dev=dev3,node=25 +-object +acpi-generic-initiator,id=gi25,pci-dev=dev3,node=26 +-object +acpi-generic-initiator,id=gi26,pci-dev=dev3,node=27 +-object +acpi-generic-initiator,id=gi27,pci-dev=dev3,node=28 +-object +acpi-generic-initiator,id=gi28,pci-dev=dev3,node=29 +-object +acpi-generic-initiator,id=gi29,pci-dev=dev3,node=30 +-object +acpi-generic-initiator,id=gi30,pci-dev=dev3,node=31 +-object +acpi-generic-initiator,id=gi31,pci-dev=dev3,node=32 diff --git a/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_4_gpu_and_nic.args b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_4_gpu_and_nic.args new file mode 100644 index 0000000000..50b5e75c70 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_4_gpu_and_nic.args @@ -0,0 +1,59 @@ +# Grace Config 4 - GPU + NIC passthrough +# HostTopology: 1 socket (cpus 0-3), 1 GpuSmmuGroup [GPU], 1 NIC on separate SMMU +# NIC root port emits no acpi-generic-initiator objects +-object +iommufd,id=iommufd0 +-object +memory-backend-ram,size=16G,id=m0 +-machine +virt,accel=kvm,gic-version=3,ras=on,highmem-mmio-size=4T,memory-backend=m0 +-numa +node,memdev=m0,cpus=0-3,nodeid=0 +-numa +node,nodeid=1 +-numa +node,nodeid=2 +-numa +node,nodeid=3 +-numa +node,nodeid=4 +-numa +node,nodeid=5 +-numa +node,nodeid=6 +-numa +node,nodeid=7 +-numa +node,nodeid=8 +-device +pxb-pcie,id=pcie.1,bus_nr=1,bus=pcie.0,numa_node=0 +-device +arm-smmuv3,primary-bus=pcie.1,id=smmuv3.1,accel=on,ats=on,ril=off,pasid=on,oas=48 +-device +pcie-root-port,id=pcie.port1,bus=pcie.1,chassis=1,io-reserve=0 +-device +vfio-pci-nohotplug,host=0008:06:00.0,bus=pcie.port1,rombar=0,id=dev0,iommufd=iommufd0 +-device +pxb-pcie,id=pcie.2,bus_nr=2,bus=pcie.0,numa_node=0 +-device +arm-smmuv3,primary-bus=pcie.2,id=smmuv3.2,accel=on,ats=on,ril=off,pasid=on,oas=48 +-device +pcie-root-port,id=pcie.port2,bus=pcie.2,chassis=2,io-reserve=0 +-device +vfio-pci-nohotplug,host=0008:00:00.0,bus=pcie.port2,rombar=0,id=dev1,iommufd=iommufd0 +-object +acpi-generic-initiator,id=gi0,pci-dev=dev0,node=1 +-object +acpi-generic-initiator,id=gi1,pci-dev=dev0,node=2 +-object +acpi-generic-initiator,id=gi2,pci-dev=dev0,node=3 +-object +acpi-generic-initiator,id=gi3,pci-dev=dev0,node=4 +-object +acpi-generic-initiator,id=gi4,pci-dev=dev0,node=5 +-object +acpi-generic-initiator,id=gi5,pci-dev=dev0,node=6 +-object +acpi-generic-initiator,id=gi6,pci-dev=dev0,node=7 +-object +acpi-generic-initiator,id=gi7,pci-dev=dev0,node=8 diff --git a/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_5_vcmdq.args b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_5_vcmdq.args new file mode 100644 index 0000000000..b3b8702a9e --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_5_vcmdq.args @@ -0,0 +1,51 @@ +# Grace Config 5 - vCMDQ, hugepages backing + SMMU command-queue virtualisation +# HostTopology: 1 socket (cpus 0-3), 1 GpuSmmuGroup [0008:06:00.0], no EGM +# Differs from Config 1: memory-backend-file + cmdqv=on on arm-smmuv3 +-object +iommufd,id=iommufd0 +-object +memory-backend-file,id=m0,size=16G,mem-path=/dev/hugepages/,prealloc=on,share=on +-machine +virt,accel=kvm,gic-version=3,ras=on,highmem-mmio-size=4T,memory-backend=m0 +-numa +node,memdev=m0,cpus=0-3,nodeid=0 +-numa +node,nodeid=1 +-numa +node,nodeid=2 +-numa +node,nodeid=3 +-numa +node,nodeid=4 +-numa +node,nodeid=5 +-numa +node,nodeid=6 +-numa +node,nodeid=7 +-numa +node,nodeid=8 +-device +pxb-pcie,id=pcie.1,bus_nr=1,bus=pcie.0,numa_node=0 +-device +arm-smmuv3,primary-bus=pcie.1,id=smmuv3.1,accel=on,ats=on,ril=off,pasid=on,oas=48,cmdqv=on +-device +pcie-root-port,id=pcie.port1,bus=pcie.1,chassis=1,io-reserve=0 +-device +vfio-pci-nohotplug,host=0008:06:00.0,bus=pcie.port1,rombar=0,id=dev0,iommufd=iommufd0 +-object +acpi-generic-initiator,id=gi0,pci-dev=dev0,node=1 +-object +acpi-generic-initiator,id=gi1,pci-dev=dev0,node=2 +-object +acpi-generic-initiator,id=gi2,pci-dev=dev0,node=3 +-object +acpi-generic-initiator,id=gi3,pci-dev=dev0,node=4 +-object +acpi-generic-initiator,id=gi4,pci-dev=dev0,node=5 +-object +acpi-generic-initiator,id=gi5,pci-dev=dev0,node=6 +-object +acpi-generic-initiator,id=gi6,pci-dev=dev0,node=7 +-object +acpi-generic-initiator,id=gi7,pci-dev=dev0,node=8 diff --git a/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_6_vegm_1_per_socket.args b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_6_vegm_1_per_socket.args new file mode 100644 index 0000000000..9459ffa078 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_6_vegm_1_per_socket.args @@ -0,0 +1,191 @@ +# Grace Config 6 - vEGM, 1 GPU per socket, 4 sockets, 36 NUMA nodes +# HostTopology: 4 sockets, 4 GpuSmmuGroups (1 GPU each), 4 EgmSocketInfos +# CpuMem nodes 0-3 (one per socket), GPU initiator nodes 4-35 (8 per GPU) +-object +iommufd,id=iommufd0 +-object +memory-backend-file,id=m0,mem-path=/dev/egm4,size=56896M,share=on,prealloc=on +-object +memory-backend-file,id=m1,mem-path=/dev/egm5,size=56896M,share=on,prealloc=on +-object +memory-backend-file,id=m2,mem-path=/dev/egm6,size=56896M,share=on,prealloc=on +-object +memory-backend-file,id=m3,mem-path=/dev/egm7,size=56896M,share=on,prealloc=on +-machine +virt,accel=kvm,gic-version=3,ras=on,highmem-mmio-size=4T +-numa +node,memdev=m0,cpus=0,nodeid=0 +-numa +node,memdev=m1,cpus=1,nodeid=1 +-numa +node,memdev=m2,cpus=2,nodeid=2 +-numa +node,memdev=m3,cpus=3,nodeid=3 +-numa +node,nodeid=4 +-numa +node,nodeid=5 +-numa +node,nodeid=6 +-numa +node,nodeid=7 +-numa +node,nodeid=8 +-numa +node,nodeid=9 +-numa +node,nodeid=10 +-numa +node,nodeid=11 +-numa +node,nodeid=12 +-numa +node,nodeid=13 +-numa +node,nodeid=14 +-numa +node,nodeid=15 +-numa +node,nodeid=16 +-numa +node,nodeid=17 +-numa +node,nodeid=18 +-numa +node,nodeid=19 +-numa +node,nodeid=20 +-numa +node,nodeid=21 +-numa +node,nodeid=22 +-numa +node,nodeid=23 +-numa +node,nodeid=24 +-numa +node,nodeid=25 +-numa +node,nodeid=26 +-numa +node,nodeid=27 +-numa +node,nodeid=28 +-numa +node,nodeid=29 +-numa +node,nodeid=30 +-numa +node,nodeid=31 +-numa +node,nodeid=32 +-numa +node,nodeid=33 +-numa +node,nodeid=34 +-numa +node,nodeid=35 +-device +pxb-pcie,id=pcie.1,bus_nr=1,bus=pcie.0,numa_node=0 +-device +arm-smmuv3,primary-bus=pcie.1,id=smmuv3.1,accel=on,ats=on,ril=off,pasid=on,oas=48 +-device +pcie-root-port,id=pcie.port1,bus=pcie.1,chassis=1,io-reserve=0 +-device +vfio-pci-nohotplug,host=0008:06:00.0,bus=pcie.port1,rombar=0,id=dev0,iommufd=iommufd0 +-device +pxb-pcie,id=pcie.2,bus_nr=2,bus=pcie.0,numa_node=1 +-device +arm-smmuv3,primary-bus=pcie.2,id=smmuv3.2,accel=on,ats=on,ril=off,pasid=on,oas=48 +-device +pcie-root-port,id=pcie.port2,bus=pcie.2,chassis=2,io-reserve=0 +-device +vfio-pci-nohotplug,host=0009:06:00.0,bus=pcie.port2,rombar=0,id=dev1,iommufd=iommufd0 +-device +pxb-pcie,id=pcie.3,bus_nr=3,bus=pcie.0,numa_node=2 +-device +arm-smmuv3,primary-bus=pcie.3,id=smmuv3.3,accel=on,ats=on,ril=off,pasid=on,oas=48 +-device +pcie-root-port,id=pcie.port3,bus=pcie.3,chassis=3,io-reserve=0 +-device +vfio-pci-nohotplug,host=0010:06:00.0,bus=pcie.port3,rombar=0,id=dev2,iommufd=iommufd0 +-device +pxb-pcie,id=pcie.4,bus_nr=4,bus=pcie.0,numa_node=3 +-device +arm-smmuv3,primary-bus=pcie.4,id=smmuv3.4,accel=on,ats=on,ril=off,pasid=on,oas=48 +-device +pcie-root-port,id=pcie.port4,bus=pcie.4,chassis=4,io-reserve=0 +-device +vfio-pci-nohotplug,host=0011:06:00.0,bus=pcie.port4,rombar=0,id=dev3,iommufd=iommufd0 +-object +acpi-generic-initiator,id=gi0,pci-dev=dev0,node=4 +-object +acpi-generic-initiator,id=gi1,pci-dev=dev0,node=5 +-object +acpi-generic-initiator,id=gi2,pci-dev=dev0,node=6 +-object +acpi-generic-initiator,id=gi3,pci-dev=dev0,node=7 +-object +acpi-generic-initiator,id=gi4,pci-dev=dev0,node=8 +-object +acpi-generic-initiator,id=gi5,pci-dev=dev0,node=9 +-object +acpi-generic-initiator,id=gi6,pci-dev=dev0,node=10 +-object +acpi-generic-initiator,id=gi7,pci-dev=dev0,node=11 +-object +acpi-generic-initiator,id=gi8,pci-dev=dev1,node=12 +-object +acpi-generic-initiator,id=gi9,pci-dev=dev1,node=13 +-object +acpi-generic-initiator,id=gi10,pci-dev=dev1,node=14 +-object +acpi-generic-initiator,id=gi11,pci-dev=dev1,node=15 +-object +acpi-generic-initiator,id=gi12,pci-dev=dev1,node=16 +-object +acpi-generic-initiator,id=gi13,pci-dev=dev1,node=17 +-object +acpi-generic-initiator,id=gi14,pci-dev=dev1,node=18 +-object +acpi-generic-initiator,id=gi15,pci-dev=dev1,node=19 +-object +acpi-generic-initiator,id=gi16,pci-dev=dev2,node=20 +-object +acpi-generic-initiator,id=gi17,pci-dev=dev2,node=21 +-object +acpi-generic-initiator,id=gi18,pci-dev=dev2,node=22 +-object +acpi-generic-initiator,id=gi19,pci-dev=dev2,node=23 +-object +acpi-generic-initiator,id=gi20,pci-dev=dev2,node=24 +-object +acpi-generic-initiator,id=gi21,pci-dev=dev2,node=25 +-object +acpi-generic-initiator,id=gi22,pci-dev=dev2,node=26 +-object +acpi-generic-initiator,id=gi23,pci-dev=dev2,node=27 +-object +acpi-generic-initiator,id=gi24,pci-dev=dev3,node=28 +-object +acpi-generic-initiator,id=gi25,pci-dev=dev3,node=29 +-object +acpi-generic-initiator,id=gi26,pci-dev=dev3,node=30 +-object +acpi-generic-initiator,id=gi27,pci-dev=dev3,node=31 +-object +acpi-generic-initiator,id=gi28,pci-dev=dev3,node=32 +-object +acpi-generic-initiator,id=gi29,pci-dev=dev3,node=33 +-object +acpi-generic-initiator,id=gi30,pci-dev=dev3,node=34 +-object +acpi-generic-initiator,id=gi31,pci-dev=dev3,node=35 +-object +acpi-egm-memory,id=egm0,pci-dev=dev0,node=0 +-object +acpi-egm-memory,id=egm1,pci-dev=dev1,node=1 +-object +acpi-egm-memory,id=egm2,pci-dev=dev2,node=2 +-object +acpi-egm-memory,id=egm3,pci-dev=dev3,node=3 diff --git a/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_7_vegm_2_per_socket.args b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_7_vegm_2_per_socket.args new file mode 100644 index 0000000000..d78ba4b605 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/grace_7_vegm_2_per_socket.args @@ -0,0 +1,176 @@ +# Grace Config 7 - vEGM, 2 GPUs per socket, 2 sockets, 34 NUMA nodes +# HostTopology: 2 sockets, 2 GpuSmmuGroups (2 GPUs each), 2 EgmSocketInfos +# CpuMem nodes 0-1 (one per socket), GPU initiator nodes 2-33 (8 per GPU) +# Both GPUs on the same socket point to that socket's CpuMem node in acpi-egm-memory +-object +iommufd,id=iommufd0 +-object +memory-backend-file,id=m0,mem-path=/dev/egm4,size=56896M,share=on,prealloc=on +-object +memory-backend-file,id=m1,mem-path=/dev/egm5,size=56896M,share=on,prealloc=on +-machine +virt,accel=kvm,gic-version=3,ras=on,highmem-mmio-size=4T +-numa +node,memdev=m0,cpus=0-1,nodeid=0 +-numa +node,memdev=m1,cpus=2-3,nodeid=1 +-numa +node,nodeid=2 +-numa +node,nodeid=3 +-numa +node,nodeid=4 +-numa +node,nodeid=5 +-numa +node,nodeid=6 +-numa +node,nodeid=7 +-numa +node,nodeid=8 +-numa +node,nodeid=9 +-numa +node,nodeid=10 +-numa +node,nodeid=11 +-numa +node,nodeid=12 +-numa +node,nodeid=13 +-numa +node,nodeid=14 +-numa +node,nodeid=15 +-numa +node,nodeid=16 +-numa +node,nodeid=17 +-numa +node,nodeid=18 +-numa +node,nodeid=19 +-numa +node,nodeid=20 +-numa +node,nodeid=21 +-numa +node,nodeid=22 +-numa +node,nodeid=23 +-numa +node,nodeid=24 +-numa +node,nodeid=25 +-numa +node,nodeid=26 +-numa +node,nodeid=27 +-numa +node,nodeid=28 +-numa +node,nodeid=29 +-numa +node,nodeid=30 +-numa +node,nodeid=31 +-numa +node,nodeid=32 +-numa +node,nodeid=33 +-device +pxb-pcie,id=pcie.1,bus_nr=1,bus=pcie.0,numa_node=0 +-device +arm-smmuv3,primary-bus=pcie.1,id=smmuv3.1,accel=on,ats=on,ril=off,pasid=on,oas=48 +-device +pcie-root-port,id=pcie.port1,bus=pcie.1,chassis=1,io-reserve=0 +-device +vfio-pci-nohotplug,host=0008:06:00.0,bus=pcie.port1,rombar=0,id=dev0,iommufd=iommufd0 +-device +pcie-root-port,id=pcie.port2,bus=pcie.1,chassis=2,io-reserve=0 +-device +vfio-pci-nohotplug,host=0009:06:00.0,bus=pcie.port2,rombar=0,id=dev1,iommufd=iommufd0 +-device +pxb-pcie,id=pcie.2,bus_nr=9,bus=pcie.0,numa_node=1 +-device +arm-smmuv3,primary-bus=pcie.2,id=smmuv3.2,accel=on,ats=on,ril=off,pasid=on,oas=48 +-device +pcie-root-port,id=pcie.port3,bus=pcie.2,chassis=3,io-reserve=0 +-device +vfio-pci-nohotplug,host=0010:06:00.0,bus=pcie.port3,rombar=0,id=dev2,iommufd=iommufd0 +-device +pcie-root-port,id=pcie.port4,bus=pcie.2,chassis=4,io-reserve=0 +-device +vfio-pci-nohotplug,host=0011:06:00.0,bus=pcie.port4,rombar=0,id=dev3,iommufd=iommufd0 +-object +acpi-generic-initiator,id=gi0,pci-dev=dev0,node=2 +-object +acpi-generic-initiator,id=gi1,pci-dev=dev0,node=3 +-object +acpi-generic-initiator,id=gi2,pci-dev=dev0,node=4 +-object +acpi-generic-initiator,id=gi3,pci-dev=dev0,node=5 +-object +acpi-generic-initiator,id=gi4,pci-dev=dev0,node=6 +-object +acpi-generic-initiator,id=gi5,pci-dev=dev0,node=7 +-object +acpi-generic-initiator,id=gi6,pci-dev=dev0,node=8 +-object +acpi-generic-initiator,id=gi7,pci-dev=dev0,node=9 +-object +acpi-generic-initiator,id=gi8,pci-dev=dev1,node=10 +-object +acpi-generic-initiator,id=gi9,pci-dev=dev1,node=11 +-object +acpi-generic-initiator,id=gi10,pci-dev=dev1,node=12 +-object +acpi-generic-initiator,id=gi11,pci-dev=dev1,node=13 +-object +acpi-generic-initiator,id=gi12,pci-dev=dev1,node=14 +-object +acpi-generic-initiator,id=gi13,pci-dev=dev1,node=15 +-object +acpi-generic-initiator,id=gi14,pci-dev=dev1,node=16 +-object +acpi-generic-initiator,id=gi15,pci-dev=dev1,node=17 +-object +acpi-generic-initiator,id=gi16,pci-dev=dev2,node=18 +-object +acpi-generic-initiator,id=gi17,pci-dev=dev2,node=19 +-object +acpi-generic-initiator,id=gi18,pci-dev=dev2,node=20 +-object +acpi-generic-initiator,id=gi19,pci-dev=dev2,node=21 +-object +acpi-generic-initiator,id=gi20,pci-dev=dev2,node=22 +-object +acpi-generic-initiator,id=gi21,pci-dev=dev2,node=23 +-object +acpi-generic-initiator,id=gi22,pci-dev=dev2,node=24 +-object +acpi-generic-initiator,id=gi23,pci-dev=dev2,node=25 +-object +acpi-generic-initiator,id=gi24,pci-dev=dev3,node=26 +-object +acpi-generic-initiator,id=gi25,pci-dev=dev3,node=27 +-object +acpi-generic-initiator,id=gi26,pci-dev=dev3,node=28 +-object +acpi-generic-initiator,id=gi27,pci-dev=dev3,node=29 +-object +acpi-generic-initiator,id=gi28,pci-dev=dev3,node=30 +-object +acpi-generic-initiator,id=gi29,pci-dev=dev3,node=31 +-object +acpi-generic-initiator,id=gi30,pci-dev=dev3,node=32 +-object +acpi-generic-initiator,id=gi31,pci-dev=dev3,node=33 +-object +acpi-egm-memory,id=egm0,pci-dev=dev0,node=0 +-object +acpi-egm-memory,id=egm1,pci-dev=dev1,node=0 +-object +acpi-egm-memory,id=egm2,pci-dev=dev2,node=1 +-object +acpi-egm-memory,id=egm3,pci-dev=dev3,node=1