runtime-rs: Add PCIe topology cold-plug port management

Extend PCIeTopology to support cold-plug port reservation and release
for VFIO devices. New fields track the topology mode (NoPort, RootPort,
SwitchPort), whether cold-plug dynamic expansion is enabled, and a map
of reserved bus assignments per device.

PCIeTopology::new() now infers the mode from the configured root-port
and switch-port counts, pre-seeds the port structures, and makes
add_root_ports_on_bus() idempotent so that PortDevice::attach can
safely call it again after the topology has already been initialized.

New methods:
- reserve_bus_for_device: allocate a free root port or switch downstream
  port for a device, expanding the port map when cold_plug is enabled
- release_bus_for_device: free the previously reserved port
- find_free_root_port / find_free_switch_down_port: internal helpers
- release_root_port / release_switch_down_port: internal helpers

Signed-off-by: Alex Lyn <alex.lyn@antgroup.com>
Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>
This commit is contained in:
Alex Lyn
2026-04-12 16:25:23 +02:00
committed by Fabiano Fidêncio
parent 815c3b91c1
commit 2cd94ec92e

View File

@@ -227,7 +227,7 @@ impl std::fmt::Display for PCIePortBusPrefix {
/// PCIePort distinguishes between different types of PCIe ports.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
pub enum PCIePort {
// NoPort is for disabling VFIO hotplug/coldplug
// No root-port/switch, VFIO does not occupy ports
#[default]
NoPort,
@@ -327,15 +327,19 @@ pub enum AvailableNode {
#[derive(Clone, Debug, Default)]
pub struct PCIeTopology {
pub mode: PCIePort,
pub hypervisor_name: String,
pub root_complex: PCIeRootComplex,
pub bridges: u32,
pub cold_plug: bool,
pub pcie_root_ports: u32,
pub pcie_switch_ports: u32,
pub hotplug_vfio_on_root_bus: bool,
// pcie_port_devices keeps track of the devices attached to different types of PCI ports.
pub pcie_port_devices: HashMap<u32, TopologyPortDevice>,
// device_id -> (bus, bus_slot, port_id)
pub reserved_bus: HashMap<String, (String, u32, u32)>,
}
impl PCIeTopology {
@@ -354,15 +358,43 @@ impl PCIeTopology {
let total_rp = topo_config.device_info.pcie_root_port;
let total_swp = topo_config.device_info.pcie_switch_port;
Some(Self {
let mode = match (total_rp, total_swp) {
(0, 0) => PCIePort::NoPort,
(r, 0) if r > 0 => PCIePort::RootPort,
(r, s) if r > 0 && s > 0 => PCIePort::SwitchPort,
(0, s) if s > 0 => {
return None;
}
_ => return None,
};
let mut topo = Self {
hypervisor_name: topo_config.hypervisor_name.to_owned(),
root_complex,
bridges: topo_config.device_info.default_bridges,
cold_plug: true,
pcie_root_ports: total_rp,
pcie_switch_ports: total_swp,
hotplug_vfio_on_root_bus: topo_config.device_info.hotplug_vfio_on_root_bus,
pcie_port_devices: HashMap::new(),
})
mode,
reserved_bus: HashMap::new(),
};
if mode != PCIePort::NoPort {
if topo.add_root_ports_on_bus(total_rp).is_err() {
return None;
}
if mode == PCIePort::SwitchPort
&& topo
.add_switch_ports_with_strategy(1, total_swp, Strategy::SingleRootPort)
.is_err()
{
return None;
}
}
Some(topo)
}
pub fn insert_device(&mut self, ep: &mut PCIeEndpoint) -> Option<PciPath> {
@@ -566,10 +598,15 @@ impl PCIeTopology {
Ok(())
}
/// Adds a root port to pcie bus
/// Adds root ports `0..num_root_ports` on the root bus. Skips IDs that already exist so this
/// is safe when [`PCIeTopology::new`] already created the same ports from TOML and
/// [`crate::device::driver::port_device::PCIePortDevice`] attaches again (same pattern as
/// [`Self::add_switch_ports_single_root_port`]).
pub fn add_root_ports_on_bus(&mut self, num_root_ports: u32) -> Result<()> {
for index in 0..num_root_ports {
self.add_pcie_root_port(index)?;
if !self.pcie_port_devices.contains_key(&index) {
self.add_pcie_root_port(index)?;
}
}
Ok(())
@@ -681,24 +718,126 @@ impl PCIeTopology {
None
}
/// Reserve a PCIe bus/port for a cold-plugged device.
///
/// Full implementation is added alongside cold-plug port management;
/// this stub satisfies the VfioDeviceModern driver's PCIeDevice trait impl.
pub fn reserve_bus_for_device(
&mut self,
_device_id: &str,
_port: PCIePort,
device_id: &str,
mode: PCIePort,
) -> Result<Option<(String, u32, u32)>> {
todo!("PCIe cold-plug port reservation not yet wired")
if let Some(bus) = self.reserved_bus.get(device_id) {
return Ok(Some(bus.clone()));
}
let bus_port_id = match mode {
PCIePort::NoPort => return Ok(None),
PCIePort::RootPort => {
let rp = self
.find_free_root_port()
.ok_or_else(|| anyhow!("no free root port"))?;
// "rpX" starts from slot-6 and 0~5 are reserved for other virtio pci devices
(rp.port_id(), rp.id + 9, rp.id + 2)
}
PCIePort::SwitchPort => {
let dp = self
.find_free_switch_down_port()
.ok_or_else(|| anyhow!("no free switch downstream port"))?;
(dp.port_id(), dp.id + 9, dp.id + 2)
}
};
self.reserved_bus
.insert(device_id.to_string(), bus_port_id.clone());
Ok(Some(bus_port_id))
}
/// Release a previously reserved PCIe bus/port for a cold-plugged device.
///
/// Full implementation is added alongside cold-plug port management;
/// this stub satisfies the VfioDeviceModern driver's PCIeDevice trait impl.
pub fn release_bus_for_device(&mut self, _device_id: &str) -> Result<()> {
todo!("PCIe cold-plug port release not yet wired")
pub fn release_bus_for_device(&mut self, device_id: &str) -> Result<()> {
let bus = match self.reserved_bus.remove(device_id) {
Some(b) => b,
None => return Ok(()),
};
let rp_prefix = PCIePortBusPrefix::RootPort.to_string();
let swdp_prefix = PCIePortBusPrefix::SwitchDownstreamPort.to_string();
if bus.0.starts_with(&rp_prefix) {
self.release_root_port(&bus.0);
} else if bus.0.starts_with(&swdp_prefix) {
self.release_switch_down_port(&bus.0);
}
Ok(())
}
fn find_free_root_port(&mut self) -> Option<TopologyPortDevice> {
let mut ids: Vec<u32> = self.pcie_port_devices.keys().cloned().collect();
ids.sort();
for id in ids {
if let Some(rp) = self.pcie_port_devices.get_mut(&id) {
if !rp.allocated {
rp.allocated = true;
return Some(rp.clone());
}
}
}
if self.cold_plug {
let next_id = self
.pcie_port_devices
.keys()
.max()
.map(|&id| id + 1)
.unwrap_or(0);
let new_port = TopologyPortDevice {
id: next_id,
bus: "pcie.0".to_string(),
allocated: true,
connected_switch: None,
};
self.pcie_port_devices.insert(next_id, new_port.clone());
return Some(new_port);
}
None
}
fn find_free_switch_down_port(&mut self) -> Option<SwitchDownPort> {
for rp in self.pcie_port_devices.values_mut() {
if let Some(sw) = rp.connected_switch.as_mut() {
for dp in sw.switch_ports.values_mut() {
if !dp.allocated {
dp.allocated = true;
return Some(dp.clone());
}
}
}
}
None
}
fn release_root_port(&mut self, bus: &str) {
if let Some(id) = bus.strip_prefix("rp").and_then(|s| s.parse::<u32>().ok()) {
if let Some(rp) = self.pcie_port_devices.get_mut(&id) {
rp.allocated = false;
}
}
}
fn release_switch_down_port(&mut self, bus: &str) {
if let Some(id) = bus.strip_prefix("swdp").and_then(|s| s.parse::<u32>().ok()) {
for rp in self.pcie_port_devices.values_mut() {
if let Some(sw) = rp.connected_switch.as_mut() {
if let Some(dp) = sw.switch_ports.get_mut(&id) {
dp.allocated = false;
dp.connected_device = None;
break;
}
}
}
}
}
}
@@ -846,8 +985,9 @@ mod tests {
assert!(topology.add_root_ports_on_bus(3).is_ok());
assert_eq!(topology.pcie_port_devices.len(), 3);
// Adding more root ports than available should fail
assert!(topology.add_root_ports_on_bus(1).is_err());
// Idempotent: matches PCIeTopology::new pre-seeding + PortDevice::attach calling again.
assert!(topology.add_root_ports_on_bus(3).is_ok());
assert_eq!(topology.pcie_port_devices.len(), 3);
}
#[test]