diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/ARCHITECTURE.md b/src/runtime-rs/crates/hypervisor/src/qemu/ARCHITECTURE.md index 7f6a90c4b0..9d8ee5795a 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/ARCHITECTURE.md +++ b/src/runtime-rs/crates/hypervisor/src/qemu/ARCHITECTURE.md @@ -126,7 +126,7 @@ pub struct Q35 { /// Global IOMMU device for Q35. Emitted as a top-level -device intel-iommu, /// not attached to any pxb-pcie. Contrast with SmmuV3 on PciRootComplex. pub intel_iommu: Option, - // pub runtime: RuntimeFeatures, -- Phase 3+ + // pub runtime: RuntimeFeatures, -- Phase 4+ } pub struct IntelIommuConfig { @@ -253,9 +253,10 @@ pub struct VfioDevice { } pub enum VfioDeviceKind { - Gpu, // vfio-pci-nohotplug (Grace aarch64) - GpuPci, // vfio-pci (Q35 / CoCo x86) - Nic, // vfio-pci + Gpu, // vfio-pci-nohotplug (Grace aarch64) + GpuPci, // vfio-pci (Q35 / CoCo x86 GPU) + NvSwitch, // vfio-pci (DGX/HGX NVSwitch; same device string as GpuPci) + Nic, // vfio-pci } /// IOMMU that attaches to a specific PCIe expander bus (pxb-pcie). @@ -376,7 +377,17 @@ pub struct EgmSocketInfo { pub enum ProtectionDevice { SevSnp { id: String, cbitpos: u8, reduced_phys_bits: u8, kernel_hashes: bool, policy: u64, host_data: Option }, - Tdx { id: String }, // fields TBD — no production capture yet + Tdx { id: String, quote_generation_socket: Option }, +} + +/// vsock address for the DCAP quote generation daemon. +/// Emitted as a JSON sub-object because QEMU's key=value parser cannot +/// represent nested structures. The production TDX capture uses +/// {"type":"vsock","cid":"2","port":"4050"}. +pub struct TdxQuoteSocket { + pub ty: String, // "vsock" + pub cid: String, // guest CID + pub port: String, // port number } ``` @@ -744,8 +755,15 @@ each supported machine type and that `HostTopology` round-trips through `vfio-pci-nohotplug` for Grace). - `PciRootPort` extended: `slot`, `multifunction`, `io_reserve` — all `Option` to cover both Q35 cold-plug and Grace io-reserve formats. -- Two Q35 fixture tests pass without `#[ignore]`: - `q35_vanilla_kata_x86`, `q35_coco_snp_single_gpu`. +- `ProtectionDevice::Tdx` fields added from TDX production capture: + `quote_generation_socket: Option` with `ty/cid/port`. + TDX protection object is emitted as JSON (`{"qom-type":"tdx-guest",...}`) rather + than key=value because QEMU's key=value parser cannot represent nested objects. +- `VfioDeviceKind::NvSwitch` added; maps to `vfio-pci` (same as `GpuPci`). + Production TDX capture shows NVSwitches use `pcie-root-port + vfio-pci` topology + (not switch-port hierarchy); distinguished in the type for probe-side classification. +- Three Q35 fixture tests pass without `#[ignore]`: + `q35_vanilla_kata_x86`, `q35_coco_snp_single_gpu`, `q35_coco_tdx_8gpu_4nvswitch`. ### Phase 4 — Multi-RC PCIe and NUMA layout @@ -898,30 +916,48 @@ Platform fields added in Phase 3: - `VfioDevice::iommufd_id: Option` — per-device iommufd for CoCo x86 - `VfioDevice::pci_vendor_id / pci_device_id` for CoCo attestation overrides -**TDX data still needed:** capture from a CoCo + GPU pod on an Intel TDX host. +**TDX production data captured** (Intel TDX host, 2026-07-15). +See the `q35_coco_tdx_8gpu_4nvswitch` fixture for full topology details. +Key differences from SNP: TDX object comes in JSON format; NUMA node 1 is +memory-only (no `cpus=`); NVSwitches share pxb-numa0 with GPUs. ### 8 GPUs + 4 NVSwitches (DGX/HGX topology) -NVSwitch passthrough adds a new device kind and a multi-level PCIe hierarchy -not present in the Grace configs: +**Production data captured** (Intel TDX host, 2026-07-15 — HGX H100 PPCIE). +Fixture: `q35_coco_tdx_8gpu_4nvswitch.args`. Test: `q35_coco_tdx_8gpu_4nvswitch` (passing, Phase 3). -- `VfioDeviceKind::NvSwitch` is not yet defined in `topology.rs` (only `Gpu` - and `Nic` exist). -- NVSwitches currently use `VfioDeviceConfig` (not `VfioDeviceGroup`) in the - legacy path (`add_gpu_nvswitch_setup` at cmdline_generator.rs:3373). -- PCIe hierarchy: root port → `x3130-upstream` → `xio3130-downstream` → - device (three levels vs. the two levels used for GPU direct attachment). -- `add_pcie_switch_ports` (cmdline_generator.rs:3508) emits this hierarchy; - `PciTopology` has no equivalent typed representation yet. +Key observations from the TDX + 8 GPU + 4 NVSwitch invocation: -New types needed before a fixture can be written: +- 2 `pxb-pcie` complexes: `pxb-numa0` (`bus_nr=32`, `numa_node=0`, chassis=10) and + `pxb-numa1` (`bus_nr=64`, `numa_node=1`, chassis=11). `bus_nr` is 32 apart to + reserve space for up to 32 subordinate buses per complex. +- pxb-numa0 carries both GPUs (slots 0-3, 0x10de:0x2330) and NVSwitches (slots 4-7, + 0x10de:0x22a3) on the same complex. NVSwitch NUMA affinity on this host matches + GPU NUMA node 0 so they share the pxb. +- pxb-numa1 carries 4 more GPUs (slots 0-3). The 4 NVSwitches are fabric chips + shared across all GPUs; they are homed on pxb-numa0 regardless. +- **NVSwitches use `pcie-root-port + vfio-pci` (root-port topology), NOT the + `pcie-root-port → x3130-upstream → xio3130-downstream` switch-port hierarchy.** + The legacy `add_pcie_switch_ports` path exists in `cmdline_generator.rs:3508` + for a different topology variant (DAN / NVSwitch fan-out) but was not used in + this TDX production deployment. Switch-port topology is tracked as Phase 4+. +- `VfioDeviceKind::NvSwitch` added in Phase 3; emits `vfio-pci` (same device + string as `GpuPci`). The kind distinction is used by probers and higher-level + code to classify devices without re-reading PCI IDs. +- NUMA node 1 is memory-only: the single vCPU (`smp 1`) lives on node 0. + NUMA distance 21 (symmetric) between nodes 0 and 1. +- TDX object emitted as JSON: `{"qom-type":"tdx-guest","id":"tdx","quote-generation-socket":{"type":"vsock","cid":"2","port":"4050"}}`. + QEMU's key=value parser cannot represent the nested `quote-generation-socket` + object, so the entire `-object` value is a JSON string. + +Platform fields added in Phase 3 from this capture: +- `ProtectionDevice::Tdx { quote_generation_socket: Option }` +- `TdxQuoteSocket { ty, cid, port }` and JSON emission in `emit_protection` - `VfioDeviceKind::NvSwitch` -- `PciSwitchPort { upstream: PcieUpstreamPort, downstream: Vec }` on `PciRootComplex` -- `HostTopology::nvswitch_addrs` or equivalent probe field +- Per-pxb `chassis` assignment (chassis=10 for pxb-numa0, chassis=11 for pxb-numa1) -**Data needed:** capture from a DGX/HGX or GB200 NVL system with 8 GPUs and -4 NVSwitches passed through. Exact bus_nr arithmetic and PCIe address -assignments must come from a live invocation, not from inference. +Phase 4 will wire `apply_host_defaults` end-to-end for TDX + NVSwitch topologies +and add `HostTopology` fields for NVSwitch device classification. --- diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs index 228da6a176..86af090b84 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/platform.rs @@ -761,7 +761,19 @@ fn emit_protection(prot: &ProtectionDevice) -> String { } s } - ProtectionDevice::Tdx { id } => format!("tdx-guest,id={id}"), + ProtectionDevice::Tdx { id, quote_generation_socket } => { + // QEMU's key=value parser cannot represent nested objects, so + // tdx-guest must be expressed as a JSON -object argument. + let mut json = format!(r#"{{"qom-type":"tdx-guest","id":"{id}""#); + if let Some(sock) = quote_generation_socket { + json.push_str(&format!( + r#","quote-generation-socket":{{"type":"{}","cid":"{}","port":"{}"}}"#, + sock.ty, sock.cid, sock.port + )); + } + json.push('}'); + json + } } } @@ -854,7 +866,7 @@ fn emit_root_port(port: &PciRootPort, bus: &str) -> String { fn emit_vfio_q35(vfio: &VfioDevice, port_id: &str) -> String { let device = match vfio.kind { VfioDeviceKind::Gpu => "vfio-pci-nohotplug", - VfioDeviceKind::GpuPci | VfioDeviceKind::Nic => "vfio-pci", + VfioDeviceKind::GpuPci | VfioDeviceKind::NvSwitch | VfioDeviceKind::Nic => "vfio-pci", }; let mut s = format!("{device},host={},id={}", vfio.host, vfio.id); if let Some(vid) = vfio.pci_vendor_id { @@ -878,7 +890,7 @@ fn emit_vfio_grace( ) -> String { let device = match vfio.kind { VfioDeviceKind::Gpu => "vfio-pci-nohotplug", - VfioDeviceKind::GpuPci | VfioDeviceKind::Nic => "vfio-pci", + VfioDeviceKind::GpuPci | VfioDeviceKind::NvSwitch | VfioDeviceKind::Nic => "vfio-pci", }; let mut s = format!("{device},host={},bus={port_id}", vfio.host); if let Some(rombar) = vfio.rombar { diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs index 810cc3ed22..08aa649605 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/probe.rs @@ -57,17 +57,30 @@ pub(crate) enum ProtectionDevice { policy: u64, host_data: Option, }, - /// TDX — fields TBD when a production capture is available. Tdx { id: String, + /// vsock address for the DCAP quote generation service. + /// Absent on TDs that do not perform local attestation. + quote_generation_socket: Option, }, } +/// vsock socket used by the TDX quote generation daemon (DCAP). +/// +/// Emitted as a JSON sub-object in the `tdx-guest` `-object` argument because +/// QEMU's key=value parser cannot represent nested structures. +#[derive(Clone)] +pub(crate) struct TdxQuoteSocket { + pub ty: String, // "vsock" + pub cid: String, // guest CID, e.g. "2" + pub port: String, // port number, e.g. "4050" +} + impl ProtectionDevice { pub(crate) fn id(&self) -> &str { match self { ProtectionDevice::SevSnp { id, .. } => id, - ProtectionDevice::Tdx { id } => id, + ProtectionDevice::Tdx { id, .. } => id, } } } diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs index 4fe05e32ba..b693ce3c23 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs @@ -7,7 +7,9 @@ use std::path::Path; use super::platform::{ BaseMachine, CpuConfig, CpuModel, Machine, MemoryBackend, NumaNode, Objects, Platform, }; -use super::probe::{EgmSocketInfo, GpuSmmuGroup, HostTopology, ProtectionDevice, SocketInfo}; +use super::probe::{ + EgmSocketInfo, GpuSmmuGroup, HostTopology, ProtectionDevice, SocketInfo, TdxQuoteSocket, +}; use super::q35::Q35; use super::topology::{PciRootComplex, PciRootPort, PciTopology, VfioDevice, VfioDeviceKind}; @@ -320,6 +322,137 @@ fn q35_coco_snp_single_gpu() { assert_eq!(want, got); } +// ---- Q35 CoCo (TDX) + 8 GPUs + 4 NVSwitches — Intel TDX host, HGX H100 PPCIE ---- +// +// Production capture: Intel TDX host, 2026-07-15. 1 vCPU, 8192M, 2 NUMA nodes. +// 4 H100 SXM5 GPUs (0x10de:0x2330) + 4 NVSwitch 3.0 (0x10de:0x22a3) on +// pxb-numa0 (bus_nr=32, chassis=10), 4 more H100 GPUs on pxb-numa1 (bus_nr=64, +// chassis=11). Node 1 is memory-only (no cpus= field). NUMA distance 21. +// TDX object emitted as JSON (not key=value); includes vsock quote-generation-socket. +// NVSwitches use root-port + vfio-pci topology — NOT x3130 switch-port hierarchy. +// +// Platform is built directly (not via apply_host_defaults) because per-device +// iommufd UUIDs are assigned by the runtime at VM launch time and because +// NvSwitch vs GPU classification is not yet in HostTopology (Phase 4). + +#[test] +fn q35_coco_tdx_8gpu_4nvswitch() { + use VfioDeviceKind::{GpuPci, NvSwitch}; + + fn rp( + id: &str, + chassis: u8, + slot: u8, + host: &str, + vfio_id: &str, + iommufd_id: &str, + kind: VfioDeviceKind, + device_id: u16, + ) -> PciRootPort { + PciRootPort { + id: id.to_owned(), + chassis, + slot: Some(slot), + multifunction: Some(false), + io_reserve: None, + device: Some(VfioDevice { + id: vfio_id.to_owned(), + host: host.to_owned(), + rombar: None, + kind, + iommufd_id: Some(iommufd_id.to_owned()), + pci_vendor_id: Some(0x10de), + pci_device_id: Some(device_id), + }), + } + } + + let platform = Platform { + machine: Machine::Q35(Q35 { + base: BaseMachine { + accel: "kvm".to_owned(), + memory_backend: None, + cpu: CpuConfig { model: CpuModel::Host { extra_features: vec![] } }, + }, + kernel_irqchip: Some("split".to_owned()), + confidential_guest_support: Some("tdx".to_owned()), + intel_iommu: None, + }), + pci: PciTopology { + default_bus: Some("pcie.0".to_owned()), + roots: vec![ + PciRootComplex { + id: "pxb-numa0".to_owned(), + bus_nr: 32, + numa_node: Some(0), + iommu: None, + root_ports: vec![ + rp("rp-numa0-0", 10, 0, "0000:1b:00.0", "vfio-4a182f997e753b150", "iommufdvfio-4a182f997e753b150", GpuPci, 0x2330), + rp("rp-numa0-1", 10, 1, "0000:43:00.0", "vfio-03bafd69f614ffb21", "iommufdvfio-03bafd69f614ffb21", GpuPci, 0x2330), + rp("rp-numa0-2", 10, 2, "0000:52:00.0", "vfio-947368d0843e55b42", "iommufdvfio-947368d0843e55b42", GpuPci, 0x2330), + rp("rp-numa0-3", 10, 3, "0000:61:00.0", "vfio-41adfe2466817df03", "iommufdvfio-41adfe2466817df03", GpuPci, 0x2330), + rp("rp-numa0-4", 10, 4, "0000:07:00.0", "vfio-7eff1447579be32f8", "iommufdvfio-7eff1447579be32f8", NvSwitch, 0x22a3), + rp("rp-numa0-5", 10, 5, "0000:08:00.0", "vfio-bfaa424dfcf1b24d9", "iommufdvfio-bfaa424dfcf1b24d9", NvSwitch, 0x22a3), + rp("rp-numa0-6", 10, 6, "0000:09:00.0", "vfio-585d94b2d0bd3b6b10", "iommufdvfio-585d94b2d0bd3b6b10", NvSwitch, 0x22a3), + rp("rp-numa0-7", 10, 7, "0000:0a:00.0", "vfio-44e4edb8e24e522911", "iommufdvfio-44e4edb8e24e522911", NvSwitch, 0x22a3), + ], + }, + PciRootComplex { + id: "pxb-numa1".to_owned(), + bus_nr: 64, + numa_node: Some(1), + iommu: None, + root_ports: vec![ + rp("rp-numa1-0", 11, 0, "0000:9d:00.0", "vfio-fd42e60b81a629b24", "iommufdvfio-fd42e60b81a629b24", GpuPci, 0x2330), + rp("rp-numa1-1", 11, 1, "0000:c3:00.0", "vfio-c3b4a6ef942a66a45", "iommufdvfio-c3b4a6ef942a66a45", GpuPci, 0x2330), + rp("rp-numa1-2", 11, 2, "0000:d1:00.0", "vfio-4ababdb3c8421bf06", "iommufdvfio-4ababdb3c8421bf06", GpuPci, 0x2330), + rp("rp-numa1-3", 11, 3, "0000:df:00.0", "vfio-2e0f5a646c1bbd557", "iommufdvfio-2e0f5a646c1bbd557", GpuPci, 0x2330), + ], + }, + ], + pcie_root_port: vec![], + }, + objects: Objects { + iommufd: None, + memory_backends: vec![ + MemoryBackend::Ram { + id: "numa-mem0".to_owned(), + size: 4096 << 20, + host_nodes: Some(0), + policy: Some("bind".to_owned()), + }, + MemoryBackend::Ram { + id: "numa-mem1".to_owned(), + size: 4096 << 20, + host_nodes: Some(1), + policy: Some("bind".to_owned()), + }, + ], + numa_nodes: vec![ + NumaNode { nodeid: 0, memdev: Some("numa-mem0".to_owned()), cpus: Some(0..1) }, + // Node 1 is memory-only: the single vCPU lives on node 0. + NumaNode { nodeid: 1, memdev: Some("numa-mem1".to_owned()), cpus: None }, + ], + numa_distances: vec![(0, 1, 21), (1, 0, 21)], + thread_contexts: vec![], + acpi_links: vec![], + rng: None, + protection: Some(ProtectionDevice::Tdx { + id: "tdx".to_owned(), + quote_generation_socket: Some(TdxQuoteSocket { + ty: "vsock".to_owned(), + cid: "2".to_owned(), + port: "4050".to_owned(), + }), + }), + }, + }; + + let got = platform.to_qemu_args().expect("to_qemu_args"); + let want = load_fixture("q35_coco_tdx_8gpu_4nvswitch.args"); + assert_eq!(want, got); +} + // ---- Q35 x86_64: vanilla kata, 2-socket NUMA, 8 cold-plug root ports ---- // // Production capture: DGX x86 host, 2026-07-07. 66 vCPUs, 73728M total, diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/topology.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/topology.rs index 36d9d2a824..064232b227 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/topology.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/topology.rs @@ -52,8 +52,13 @@ pub(crate) struct VfioDevice { pub(crate) enum VfioDeviceKind { /// `vfio-pci-nohotplug` — aarch64 Grace static binding. Gpu, - /// `vfio-pci` — x86 Q35 / CoCo passthrough and NIC. + /// `vfio-pci` — x86 Q35 / CoCo GPU passthrough. GpuPci, + /// `vfio-pci` — NVSwitch (DGX/HGX fabric chip). + /// + /// Uses the same device string as GpuPci; distinguished here so probers + /// and emitters can identify device type without re-reading PCI IDs. + NvSwitch, Nic, } diff --git a/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/q35_coco_tdx_8gpu_4nvswitch.args b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/q35_coco_tdx_8gpu_4nvswitch.args new file mode 100644 index 0000000000..8cb7fdf38d --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/q35_coco_tdx_8gpu_4nvswitch.args @@ -0,0 +1,114 @@ +# Q35 CoCo (TDX) — HGX H100 PPCIE, 8 GPUs + 4 NVSwitches, 2-NUMA x86_64 +# Source: production kata-containers log, 2026-07-15 (Intel TDX host, viking-prod-243) +# Binary: qemu-system-x86_64-tdx-experimental +# +# HostTopology (logically — Platform built directly in the test): +# socket 0: 1 vCPU (cpus 0-0), host-node 0, 4G RAM +# 4 H100 SXM5 GPUs (0x10de:0x2330) on pxb-numa0 slots 0-3 +# 4 NVSwitch 3.0 (0x10de:0x22a3) on pxb-numa0 slots 4-7 +# socket 1: no CPUs (memory-only node), host-node 1, 4G RAM +# 4 H100 SXM5 GPUs (0x10de:0x2330) on pxb-numa1 slots 0-3 +# NUMA distance 21 between nodes 0 and 1 +# Protection: TDX with vsock quote-generation-socket (cid=2, port=4050) +# iommufd is per-device; x-pci-vendor-id/device-id for CoCo attestation +# +# NVSwitches use pcie-root-port + vfio-pci (same topology as GPUs), NOT +# x3130-upstream/xio3130-downstream switch-port hierarchy — see ARCHITECTURE.md. +# +# NOT included (legacy generator scope): pcie-pci-bridge, virtio devices, +# -kernel, -append, -bios, -rtc, -smp, -m, etc. +# +# Emission order: protection → machine → (backend+node)×2 → dist×2 → +# pxb-numa0 (8 port+iommufd+vfio) → pxb-numa1 (4 port+iommufd+vfio) +-object +{"qom-type":"tdx-guest","id":"tdx","quote-generation-socket":{"type":"vsock","cid":"2","port":"4050"}} +-machine +q35,accel=kvm,kernel_irqchip=split,confidential-guest-support=tdx +-object +memory-backend-ram,id=numa-mem0,size=4G,host-nodes=0,policy=bind +-numa +node,memdev=numa-mem0,cpus=0,nodeid=0 +-object +memory-backend-ram,id=numa-mem1,size=4G,host-nodes=1,policy=bind +-numa +node,memdev=numa-mem1,nodeid=1 +-numa +dist,src=0,dst=1,val=21 +-numa +dist,src=1,dst=0,val=21 +-device +pxb-pcie,id=pxb-numa0,bus=pcie.0,bus_nr=32,numa_node=0 +-device +pcie-root-port,id=rp-numa0-0,bus=pxb-numa0,chassis=10,slot=0,multifunction=off +-object +iommufd,id=iommufdvfio-4a182f997e753b150 +-device +vfio-pci,host=0000:1b:00.0,id=vfio-4a182f997e753b150,x-pci-vendor-id=0x10de,x-pci-device-id=0x2330,bus=rp-numa0-0,iommufd=iommufdvfio-4a182f997e753b150 +-device +pcie-root-port,id=rp-numa0-1,bus=pxb-numa0,chassis=10,slot=1,multifunction=off +-object +iommufd,id=iommufdvfio-03bafd69f614ffb21 +-device +vfio-pci,host=0000:43:00.0,id=vfio-03bafd69f614ffb21,x-pci-vendor-id=0x10de,x-pci-device-id=0x2330,bus=rp-numa0-1,iommufd=iommufdvfio-03bafd69f614ffb21 +-device +pcie-root-port,id=rp-numa0-2,bus=pxb-numa0,chassis=10,slot=2,multifunction=off +-object +iommufd,id=iommufdvfio-947368d0843e55b42 +-device +vfio-pci,host=0000:52:00.0,id=vfio-947368d0843e55b42,x-pci-vendor-id=0x10de,x-pci-device-id=0x2330,bus=rp-numa0-2,iommufd=iommufdvfio-947368d0843e55b42 +-device +pcie-root-port,id=rp-numa0-3,bus=pxb-numa0,chassis=10,slot=3,multifunction=off +-object +iommufd,id=iommufdvfio-41adfe2466817df03 +-device +vfio-pci,host=0000:61:00.0,id=vfio-41adfe2466817df03,x-pci-vendor-id=0x10de,x-pci-device-id=0x2330,bus=rp-numa0-3,iommufd=iommufdvfio-41adfe2466817df03 +-device +pcie-root-port,id=rp-numa0-4,bus=pxb-numa0,chassis=10,slot=4,multifunction=off +-object +iommufd,id=iommufdvfio-7eff1447579be32f8 +-device +vfio-pci,host=0000:07:00.0,id=vfio-7eff1447579be32f8,x-pci-vendor-id=0x10de,x-pci-device-id=0x22a3,bus=rp-numa0-4,iommufd=iommufdvfio-7eff1447579be32f8 +-device +pcie-root-port,id=rp-numa0-5,bus=pxb-numa0,chassis=10,slot=5,multifunction=off +-object +iommufd,id=iommufdvfio-bfaa424dfcf1b24d9 +-device +vfio-pci,host=0000:08:00.0,id=vfio-bfaa424dfcf1b24d9,x-pci-vendor-id=0x10de,x-pci-device-id=0x22a3,bus=rp-numa0-5,iommufd=iommufdvfio-bfaa424dfcf1b24d9 +-device +pcie-root-port,id=rp-numa0-6,bus=pxb-numa0,chassis=10,slot=6,multifunction=off +-object +iommufd,id=iommufdvfio-585d94b2d0bd3b6b10 +-device +vfio-pci,host=0000:09:00.0,id=vfio-585d94b2d0bd3b6b10,x-pci-vendor-id=0x10de,x-pci-device-id=0x22a3,bus=rp-numa0-6,iommufd=iommufdvfio-585d94b2d0bd3b6b10 +-device +pcie-root-port,id=rp-numa0-7,bus=pxb-numa0,chassis=10,slot=7,multifunction=off +-object +iommufd,id=iommufdvfio-44e4edb8e24e522911 +-device +vfio-pci,host=0000:0a:00.0,id=vfio-44e4edb8e24e522911,x-pci-vendor-id=0x10de,x-pci-device-id=0x22a3,bus=rp-numa0-7,iommufd=iommufdvfio-44e4edb8e24e522911 +-device +pxb-pcie,id=pxb-numa1,bus=pcie.0,bus_nr=64,numa_node=1 +-device +pcie-root-port,id=rp-numa1-0,bus=pxb-numa1,chassis=11,slot=0,multifunction=off +-object +iommufd,id=iommufdvfio-fd42e60b81a629b24 +-device +vfio-pci,host=0000:9d:00.0,id=vfio-fd42e60b81a629b24,x-pci-vendor-id=0x10de,x-pci-device-id=0x2330,bus=rp-numa1-0,iommufd=iommufdvfio-fd42e60b81a629b24 +-device +pcie-root-port,id=rp-numa1-1,bus=pxb-numa1,chassis=11,slot=1,multifunction=off +-object +iommufd,id=iommufdvfio-c3b4a6ef942a66a45 +-device +vfio-pci,host=0000:c3:00.0,id=vfio-c3b4a6ef942a66a45,x-pci-vendor-id=0x10de,x-pci-device-id=0x2330,bus=rp-numa1-1,iommufd=iommufdvfio-c3b4a6ef942a66a45 +-device +pcie-root-port,id=rp-numa1-2,bus=pxb-numa1,chassis=11,slot=2,multifunction=off +-object +iommufd,id=iommufdvfio-4ababdb3c8421bf06 +-device +vfio-pci,host=0000:d1:00.0,id=vfio-4ababdb3c8421bf06,x-pci-vendor-id=0x10de,x-pci-device-id=0x2330,bus=rp-numa1-2,iommufd=iommufdvfio-4ababdb3c8421bf06 +-device +pcie-root-port,id=rp-numa1-3,bus=pxb-numa1,chassis=11,slot=3,multifunction=off +-object +iommufd,id=iommufdvfio-2e0f5a646c1bbd557 +-device +vfio-pci,host=0000:df:00.0,id=vfio-2e0f5a646c1bbd557,x-pci-vendor-id=0x10de,x-pci-device-id=0x2330,bus=rp-numa1-3,iommufd=iommufdvfio-2e0f5a646c1bbd557