diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/ARCHITECTURE.md b/src/runtime-rs/crates/hypervisor/src/qemu/ARCHITECTURE.md index 9d8ee5795a..c516d5b99e 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/ARCHITECTURE.md +++ b/src/runtime-rs/crates/hypervisor/src/qemu/ARCHITECTURE.md @@ -762,8 +762,9 @@ each supported machine type and that `HostTopology` round-trips through - `VfioDeviceKind::NvSwitch` added; maps to `vfio-pci` (same as `GpuPci`). Production TDX capture shows NVSwitches use `pcie-root-port + vfio-pci` topology (not switch-port hierarchy); distinguished in the type for probe-side classification. -- Three Q35 fixture tests pass without `#[ignore]`: - `q35_vanilla_kata_x86`, `q35_coco_snp_single_gpu`, `q35_coco_tdx_8gpu_4nvswitch`. +- Four Q35 fixture tests pass without `#[ignore]`: + `q35_vanilla_kata_x86`, `q35_coco_snp_single_gpu`, + `q35_coco_tdx_8gpu_4nvswitch`, `q35_vanilla_8gpu_4nvswitch`. ### Phase 4 — Multi-RC PCIe and NUMA layout @@ -959,6 +960,26 @@ Platform fields added in Phase 3 from this capture: Phase 4 will wire `apply_host_defaults` end-to-end for TDX + NVSwitch topologies and add `HostTopology` fields for NVSwitch device classification. +### Vanilla kata + GPU passthrough (non-CoCo) + +**Production data captured** (same host as TDX capture, 2026-07-15). +Fixture: `q35_vanilla_8gpu_4nvswitch.args`. Test: `q35_vanilla_8gpu_4nvswitch` (passing, Phase 3). + +Same physical topology as the TDX capture (8 GPUs + 4 NVSwitches, 2 NUMA nodes, NUMA +distance 21) but without CoCo. Key differences from the TDX capture: + +- No protection object; `-machine q35,accel=kvm` without `kernel_irqchip` or + `confidential-guest-support`. +- Memory backend is `memory-backend-file` via `/dev/shm` (same pattern as + `q35_vanilla_kata_x86`), not `memory-backend-ram` as in the CoCo captures. +- Per-device iommufd is still present: the modern VFIO iommufd interface is used + for GPU passthrough regardless of CoCo mode. +- `x-pci-vendor-id`/`x-pci-device-id` overrides are still present: the kata + runtime applies them for any GPU passthrough, not only for CoCo attestation. +- NUMA node 1 is memory-only (no `cpus=`). The single vCPU lives on node 0. + +No new Platform types were required; the fixture exercises the existing model. + --- ## Known Issues and Follow-up Items diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs b/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs index b693ce3c23..4ceaafaa23 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/machine/tests.rs @@ -322,6 +322,135 @@ fn q35_coco_snp_single_gpu() { assert_eq!(want, got); } +// ---- Q35 vanilla (non-CoCo) + 8 GPUs + 4 NVSwitches — HGX H100 PPCIE ---- +// +// Production capture: same host as the TDX capture, 2026-07-15. +// Same physical topology (8 H100 GPUs + 4 NVSwitches, 2 NUMA nodes, /dev/shm +// memory, NUMA distance 21) but without CoCo: +// - No protection object; -machine q35,accel=kvm (no kernel_irqchip) +// - memory-backend-file via /dev/shm (not memory-backend-ram) +// - Per-device iommufd retained (modern VFIO interface, CoCo-independent) +// - x-pci-vendor-id/device-id retained (kata applies them for any GPU passthrough) +// - NUMA node 1 is memory-only; the single vCPU lives on node 0 + +#[test] +fn q35_vanilla_8gpu_4nvswitch() { + use VfioDeviceKind::{GpuPci, NvSwitch}; + + fn rp( + id: &str, + chassis: u8, + slot: u8, + host: &str, + vfio_id: &str, + iommufd_id: &str, + kind: VfioDeviceKind, + device_id: u16, + ) -> PciRootPort { + PciRootPort { + id: id.to_owned(), + chassis, + slot: Some(slot), + multifunction: Some(false), + io_reserve: None, + device: Some(VfioDevice { + id: vfio_id.to_owned(), + host: host.to_owned(), + rombar: None, + kind, + iommufd_id: Some(iommufd_id.to_owned()), + pci_vendor_id: Some(0x10de), + pci_device_id: Some(device_id), + }), + } + } + + let platform = Platform { + machine: Machine::Q35(Q35 { + base: BaseMachine { + accel: "kvm".to_owned(), + memory_backend: None, + cpu: CpuConfig { model: CpuModel::Host { extra_features: vec![] } }, + }, + kernel_irqchip: None, + confidential_guest_support: None, + intel_iommu: None, + }), + pci: PciTopology { + default_bus: Some("pcie.0".to_owned()), + roots: vec![ + PciRootComplex { + id: "pxb-numa0".to_owned(), + bus_nr: 32, + numa_node: Some(0), + iommu: None, + root_ports: vec![ + rp("rp-numa0-0", 10, 0, "0000:1b:00.0", "vfio-5c7a307d86e7ae830", "iommufdvfio-5c7a307d86e7ae830", GpuPci, 0x2330), + rp("rp-numa0-1", 10, 1, "0000:43:00.0", "vfio-98077fd3aa014b541", "iommufdvfio-98077fd3aa014b541", GpuPci, 0x2330), + rp("rp-numa0-2", 10, 2, "0000:52:00.0", "vfio-35b65d16a30068022", "iommufdvfio-35b65d16a30068022", GpuPci, 0x2330), + rp("rp-numa0-3", 10, 3, "0000:61:00.0", "vfio-331446c1d73050393", "iommufdvfio-331446c1d73050393", GpuPci, 0x2330), + rp("rp-numa0-4", 10, 4, "0000:07:00.0", "vfio-b91919c2fa001cd18", "iommufdvfio-b91919c2fa001cd18", NvSwitch, 0x22a3), + rp("rp-numa0-5", 10, 5, "0000:08:00.0", "vfio-fab9e6a06185873e9", "iommufdvfio-fab9e6a06185873e9", NvSwitch, 0x22a3), + rp("rp-numa0-6", 10, 6, "0000:09:00.0", "vfio-3a77ef115d3f0ab610", "iommufdvfio-3a77ef115d3f0ab610", NvSwitch, 0x22a3), + rp("rp-numa0-7", 10, 7, "0000:0a:00.0", "vfio-2e0a336d490089a111", "iommufdvfio-2e0a336d490089a111", NvSwitch, 0x22a3), + ], + }, + PciRootComplex { + id: "pxb-numa1".to_owned(), + bus_nr: 64, + numa_node: Some(1), + iommu: None, + root_ports: vec![ + rp("rp-numa1-0", 11, 0, "0000:9d:00.0", "vfio-7f57f8dea75cdeca4", "iommufdvfio-7f57f8dea75cdeca4", GpuPci, 0x2330), + rp("rp-numa1-1", 11, 1, "0000:c3:00.0", "vfio-4461976d3de811155", "iommufdvfio-4461976d3de811155", GpuPci, 0x2330), + rp("rp-numa1-2", 11, 2, "0000:d1:00.0", "vfio-dc1944aed29728336", "iommufdvfio-dc1944aed29728336", GpuPci, 0x2330), + rp("rp-numa1-3", 11, 3, "0000:df:00.0", "vfio-5c974cdc2dcfe4cb7", "iommufdvfio-5c974cdc2dcfe4cb7", GpuPci, 0x2330), + ], + }, + ], + pcie_root_port: vec![], + }, + objects: Objects { + iommufd: None, + memory_backends: vec![ + MemoryBackend::File { + id: "numa-mem0".to_owned(), + size: 4096 << 20, + path: "/dev/shm".to_owned(), + prealloc: false, + share: true, + host_nodes: Some(0), + policy: Some("bind".to_owned()), + is_egm: false, + }, + MemoryBackend::File { + id: "numa-mem1".to_owned(), + size: 4096 << 20, + path: "/dev/shm".to_owned(), + prealloc: false, + share: true, + host_nodes: Some(1), + policy: Some("bind".to_owned()), + is_egm: false, + }, + ], + numa_nodes: vec![ + NumaNode { nodeid: 0, memdev: Some("numa-mem0".to_owned()), cpus: Some(0..1) }, + NumaNode { nodeid: 1, memdev: Some("numa-mem1".to_owned()), cpus: None }, + ], + numa_distances: vec![(0, 1, 21), (1, 0, 21)], + thread_contexts: vec![], + acpi_links: vec![], + rng: None, + protection: None, + }, + }; + + let got = platform.to_qemu_args().expect("to_qemu_args"); + let want = load_fixture("q35_vanilla_8gpu_4nvswitch.args"); + assert_eq!(want, got); +} + // ---- Q35 CoCo (TDX) + 8 GPUs + 4 NVSwitches — Intel TDX host, HGX H100 PPCIE ---- // // Production capture: Intel TDX host, 2026-07-15. 1 vCPU, 8192M, 2 NUMA nodes. diff --git a/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/q35_coco_tdx_8gpu_4nvswitch.args b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/q35_coco_tdx_8gpu_4nvswitch.args index 8cb7fdf38d..7c517cae86 100644 --- a/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/q35_coco_tdx_8gpu_4nvswitch.args +++ b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/q35_coco_tdx_8gpu_4nvswitch.args @@ -1,5 +1,5 @@ # Q35 CoCo (TDX) — HGX H100 PPCIE, 8 GPUs + 4 NVSwitches, 2-NUMA x86_64 -# Source: production kata-containers log, 2026-07-15 (Intel TDX host, viking-prod-243) +# Source: production kata-containers log, 2026-07-15 (Intel TDX host) # Binary: qemu-system-x86_64-tdx-experimental # # HostTopology (logically — Platform built directly in the test): diff --git a/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/q35_vanilla_8gpu_4nvswitch.args b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/q35_vanilla_8gpu_4nvswitch.args new file mode 100644 index 0000000000..7eafc78762 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/tests/fixtures/qemu/q35_vanilla_8gpu_4nvswitch.args @@ -0,0 +1,107 @@ +# Q35 vanilla (non-CoCo) — HGX H100 PPCIE, 8 GPUs + 4 NVSwitches, 2-NUMA x86_64 +# Source: production kata-containers log, 2026-07-15 +# Binary: /opt/kata/bin/qemu-system-x86_64 (standard build, not TDX/SNP experimental) +# BIOS: /opt/kata/share/ovmf/OVMF.fd (standard OVMF) +# +# Same physical topology as q35_coco_tdx_8gpu_4nvswitch; CoCo absent: +# - No protection object, no kernel_irqchip, no confidential-guest-support +# - Memory: memory-backend-file via /dev/shm with host-nodes pinning +# (same SHM pattern as q35_vanilla_kata_x86; not memory-backend-ram) +# - iommufd is per-device even without CoCo (modern VFIO interface) +# - x-pci-vendor-id/device-id present even without CoCo (kata always applies them) +# - NUMA node 1 is memory-only: the single vCPU lives on node 0 +# - NUMA distance 21 between nodes 0 and 1 +# +# Emission order differs from production log: our Platform iterates per-pxb then +# per-port (port+iommufd+vfio interleaved). The Go runtime batches all ports +# first, then all iommufd+vfio pairs in a different cross-pxb order. +-machine +q35,accel=kvm +-object +memory-backend-file,id=numa-mem0,size=4G,mem-path=/dev/shm,host-nodes=0,policy=bind,share=on +-numa +node,memdev=numa-mem0,cpus=0,nodeid=0 +-object +memory-backend-file,id=numa-mem1,size=4G,mem-path=/dev/shm,host-nodes=1,policy=bind,share=on +-numa +node,memdev=numa-mem1,nodeid=1 +-numa +dist,src=0,dst=1,val=21 +-numa +dist,src=1,dst=0,val=21 +-device +pxb-pcie,id=pxb-numa0,bus=pcie.0,bus_nr=32,numa_node=0 +-device +pcie-root-port,id=rp-numa0-0,bus=pxb-numa0,chassis=10,slot=0,multifunction=off +-object +iommufd,id=iommufdvfio-5c7a307d86e7ae830 +-device +vfio-pci,host=0000:1b:00.0,id=vfio-5c7a307d86e7ae830,x-pci-vendor-id=0x10de,x-pci-device-id=0x2330,bus=rp-numa0-0,iommufd=iommufdvfio-5c7a307d86e7ae830 +-device +pcie-root-port,id=rp-numa0-1,bus=pxb-numa0,chassis=10,slot=1,multifunction=off +-object +iommufd,id=iommufdvfio-98077fd3aa014b541 +-device +vfio-pci,host=0000:43:00.0,id=vfio-98077fd3aa014b541,x-pci-vendor-id=0x10de,x-pci-device-id=0x2330,bus=rp-numa0-1,iommufd=iommufdvfio-98077fd3aa014b541 +-device +pcie-root-port,id=rp-numa0-2,bus=pxb-numa0,chassis=10,slot=2,multifunction=off +-object +iommufd,id=iommufdvfio-35b65d16a30068022 +-device +vfio-pci,host=0000:52:00.0,id=vfio-35b65d16a30068022,x-pci-vendor-id=0x10de,x-pci-device-id=0x2330,bus=rp-numa0-2,iommufd=iommufdvfio-35b65d16a30068022 +-device +pcie-root-port,id=rp-numa0-3,bus=pxb-numa0,chassis=10,slot=3,multifunction=off +-object +iommufd,id=iommufdvfio-331446c1d73050393 +-device +vfio-pci,host=0000:61:00.0,id=vfio-331446c1d73050393,x-pci-vendor-id=0x10de,x-pci-device-id=0x2330,bus=rp-numa0-3,iommufd=iommufdvfio-331446c1d73050393 +-device +pcie-root-port,id=rp-numa0-4,bus=pxb-numa0,chassis=10,slot=4,multifunction=off +-object +iommufd,id=iommufdvfio-b91919c2fa001cd18 +-device +vfio-pci,host=0000:07:00.0,id=vfio-b91919c2fa001cd18,x-pci-vendor-id=0x10de,x-pci-device-id=0x22a3,bus=rp-numa0-4,iommufd=iommufdvfio-b91919c2fa001cd18 +-device +pcie-root-port,id=rp-numa0-5,bus=pxb-numa0,chassis=10,slot=5,multifunction=off +-object +iommufd,id=iommufdvfio-fab9e6a06185873e9 +-device +vfio-pci,host=0000:08:00.0,id=vfio-fab9e6a06185873e9,x-pci-vendor-id=0x10de,x-pci-device-id=0x22a3,bus=rp-numa0-5,iommufd=iommufdvfio-fab9e6a06185873e9 +-device +pcie-root-port,id=rp-numa0-6,bus=pxb-numa0,chassis=10,slot=6,multifunction=off +-object +iommufd,id=iommufdvfio-3a77ef115d3f0ab610 +-device +vfio-pci,host=0000:09:00.0,id=vfio-3a77ef115d3f0ab610,x-pci-vendor-id=0x10de,x-pci-device-id=0x22a3,bus=rp-numa0-6,iommufd=iommufdvfio-3a77ef115d3f0ab610 +-device +pcie-root-port,id=rp-numa0-7,bus=pxb-numa0,chassis=10,slot=7,multifunction=off +-object +iommufd,id=iommufdvfio-2e0a336d490089a111 +-device +vfio-pci,host=0000:0a:00.0,id=vfio-2e0a336d490089a111,x-pci-vendor-id=0x10de,x-pci-device-id=0x22a3,bus=rp-numa0-7,iommufd=iommufdvfio-2e0a336d490089a111 +-device +pxb-pcie,id=pxb-numa1,bus=pcie.0,bus_nr=64,numa_node=1 +-device +pcie-root-port,id=rp-numa1-0,bus=pxb-numa1,chassis=11,slot=0,multifunction=off +-object +iommufd,id=iommufdvfio-7f57f8dea75cdeca4 +-device +vfio-pci,host=0000:9d:00.0,id=vfio-7f57f8dea75cdeca4,x-pci-vendor-id=0x10de,x-pci-device-id=0x2330,bus=rp-numa1-0,iommufd=iommufdvfio-7f57f8dea75cdeca4 +-device +pcie-root-port,id=rp-numa1-1,bus=pxb-numa1,chassis=11,slot=1,multifunction=off +-object +iommufd,id=iommufdvfio-4461976d3de811155 +-device +vfio-pci,host=0000:c3:00.0,id=vfio-4461976d3de811155,x-pci-vendor-id=0x10de,x-pci-device-id=0x2330,bus=rp-numa1-1,iommufd=iommufdvfio-4461976d3de811155 +-device +pcie-root-port,id=rp-numa1-2,bus=pxb-numa1,chassis=11,slot=2,multifunction=off +-object +iommufd,id=iommufdvfio-dc1944aed29728336 +-device +vfio-pci,host=0000:d1:00.0,id=vfio-dc1944aed29728336,x-pci-vendor-id=0x10de,x-pci-device-id=0x2330,bus=rp-numa1-2,iommufd=iommufdvfio-dc1944aed29728336 +-device +pcie-root-port,id=rp-numa1-3,bus=pxb-numa1,chassis=11,slot=3,multifunction=off +-object +iommufd,id=iommufdvfio-5c974cdc2dcfe4cb7 +-device +vfio-pci,host=0000:df:00.0,id=vfio-5c974cdc2dcfe4cb7,x-pci-vendor-id=0x10de,x-pci-device-id=0x2330,bus=rp-numa1-3,iommufd=iommufdvfio-5c974cdc2dcfe4cb7