diff --git a/src/runtime-rs/crates/hypervisor/src/device/topology.rs b/src/runtime-rs/crates/hypervisor/src/device/topology.rs index a1ab607f4..255996cae 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/topology.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/topology.rs @@ -61,7 +61,7 @@ use kata_types::config::hypervisor::TopologyConfigInfo; use super::pci_path::PciPath; -const DEFAULT_PCIE_ROOT_BUS: &str = "pcie.0"; +pub const DEFAULT_PCIE_ROOT_BUS: &str = "pcie.0"; // Currently, CLH and Dragonball support device attachment solely on the root bus. const DEFAULT_PCIE_ROOT_BUS_ADDRESS: &str = "0000:00"; pub const PCIE_ROOT_BUS_SLOTS_CAPACITY: u32 = 32; @@ -203,14 +203,132 @@ pub struct PCIeRootComplex { pub root_bus_devices: HashMap, } -#[derive(Debug, Default)] +/// PCIePortBusPrefix defines the naming scheme for PCIe ports. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum PCIePortBusPrefix { + RootPort, + SwitchPort, + SwitchUpstreamPort, + SwitchDownstreamPort, +} + +impl std::fmt::Display for PCIePortBusPrefix { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let prefix = match self { + PCIePortBusPrefix::RootPort => "rp", + PCIePortBusPrefix::SwitchPort => "sw", + PCIePortBusPrefix::SwitchUpstreamPort => "swup", + PCIePortBusPrefix::SwitchDownstreamPort => "swdp", + }; + write!(f, "{}", prefix) + } +} + +/// PCIePort distinguishes between different types of PCIe ports. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum PCIePort { + // NoPort is for disabling VFIO hotplug/coldplug + #[default] + NoPort, + + /// RootPort attach VFIO devices to a root-port + RootPort, + + // SwitchPort attach VFIO devices to a switch-port + SwitchPort, +} + +impl std::fmt::Display for PCIePort { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let port = match self { + PCIePort::NoPort => "no-port", + PCIePort::RootPort => "root-port", + PCIePort::SwitchPort => "switch-port", + }; + write!(f, "{}", port) + } +} + +/// Represents a PCIe port +#[derive(Default, Clone, Debug)] +pub struct SwitchDownPort { + pub id: u32, + pub bus: String, // swupX + pub allocated: bool, + pub connected_device: Option, +} + +impl SwitchDownPort { + pub fn port_id(&self) -> String { + format!("{}{}", PCIePortBusPrefix::SwitchDownstreamPort, self.id) + } +} + +/// Represents a PCIe switch +#[derive(Debug, Clone)] +pub struct PcieSwitch { + pub id: u32, + pub bus: String, //rpX + pub switch_ports: HashMap, // Switch ports +} + +impl PcieSwitch { + pub fn port_id(&self) -> String { + format!("{}{}", PCIePortBusPrefix::SwitchUpstreamPort, self.id) + } +} + +/// Represents a root port attached on root bus, TopologyPortDevice represents a PCIe device used for hotplugging. +#[derive(Debug, Clone)] +pub struct TopologyPortDevice { + pub id: u32, + pub bus: String, //pcie.0 + pub allocated: bool, + pub connected_switch: Option, // Connected PCIe switch +} + +impl TopologyPortDevice { + pub fn new(id: u32, bus: &str) -> Self { + Self { + id, + bus: bus.to_owned(), + allocated: false, + connected_switch: None, + } + } + + pub fn get_port_type(&self) -> PCIePort { + match self.connected_switch { + Some(_) => PCIePort::SwitchPort, + None => PCIePort::RootPort, + } + } + + pub fn port_id(&self) -> String { + format!("{}{}", PCIePortBusPrefix::RootPort, self.id) + } +} + +/// Represents strategy selection +#[derive(Debug)] +pub enum Strategy { + // Strategy 1: Use 1 Root Port to connect 1 PCIe Switch with multiple downstream switch ports + SingleRootPort, + // Strategy 2: Use multiple Root Ports to connect multiple PCIe Switches (each Switch provides at least 1 downstream port) + MultipleRootPorts, +} + +#[derive(Clone, Debug, Default)] pub struct PCIeTopology { pub hypervisor_name: String, pub root_complex: PCIeRootComplex, pub bridges: u32, pub pcie_root_ports: u32, + pub pcie_switch_ports: u32, pub hotplug_vfio_on_root_bus: bool, + // pcie_port_devices keeps track of the devices attached to different types of PCI ports. + pub pcie_port_devices: HashMap, } impl PCIeTopology { @@ -225,12 +343,18 @@ impl PCIeTopology { root_bus_devices: HashMap::with_capacity(PCIE_ROOT_BUS_SLOTS_CAPACITY as usize), }; + // initialize port devices within PCIe Topology + let total_rp = topo_config.device_info.pcie_root_port; + let total_swp = topo_config.device_info.pcie_switch_port; + Some(Self { hypervisor_name: topo_config.hypervisor_name.to_owned(), root_complex, bridges: topo_config.device_info.default_bridges, - pcie_root_ports: topo_config.device_info.pcie_root_port, + pcie_root_ports: total_rp, + pcie_switch_ports: total_swp, hotplug_vfio_on_root_bus: topo_config.device_info.hotplug_vfio_on_root_bus, + pcie_port_devices: HashMap::new(), }) } @@ -341,6 +465,189 @@ impl PCIeTopology { Ok(pci_path) } + + /// get pcie port and its total + pub fn get_pcie_port(&self) -> Option<(PCIePort, u32)> { + match (self.pcie_root_ports, self.pcie_switch_ports) { + (_, _) if self.pcie_root_ports > 0 && self.pcie_switch_ports > 0 => None, + (0, 0) => Some((PCIePort::NoPort, 0)), + (r, _) if r > 0 => Some((PCIePort::RootPort, r)), + (_, s) if s > 0 => Some((PCIePort::SwitchPort, s)), + _ => None, + } + } + + /// Adds a root port to pcie bus + fn add_pcie_root_port(&mut self, id: u32) -> Result<()> { + if self.pcie_port_devices.contains_key(&id) { + return Err(anyhow!("Root Port {} already exists.", id)); + } + + self.pcie_port_devices.insert( + id, + TopologyPortDevice { + id, + bus: DEFAULT_PCIE_ROOT_BUS.to_string(), + allocated: false, + connected_switch: None, + }, + ); + + Ok(()) + } + + /// Adds a PCIe switch to a root port + fn add_switch_to_root_port(&mut self, root_port_id: u32, switch_id: u32) -> Result<()> { + let root_port = self + .pcie_port_devices + .get_mut(&root_port_id) + .ok_or_else(|| anyhow!("Root Port {} does not exist.", root_port_id))?; + + if root_port.connected_switch.is_some() { + return Err(anyhow!( + "Root Port {} already has a connected switch.", + root_port_id + )); + } + let rp_bus = format!("{}{}", PCIePortBusPrefix::RootPort, root_port_id); + + root_port.allocated = true; + root_port.connected_switch = Some(PcieSwitch { + id: switch_id, + bus: rp_bus, + switch_ports: HashMap::new(), + }); + + Ok(()) + } + + /// Adds a switch port to a PCIe switch + fn add_switch_port_to_switch( + &mut self, + swup_bus: &str, + root_port_id: u32, + switch_port_id: u32, + ) -> Result<()> { + let root_port = self + .pcie_port_devices + .get_mut(&root_port_id) + .ok_or_else(|| anyhow!("Root Port {} does not exist.", root_port_id))?; + + let switch = root_port + .connected_switch + .as_mut() + .ok_or_else(|| anyhow!("Root Port {} has no connected switch.", root_port_id))?; + + if switch.switch_ports.contains_key(&switch_port_id) { + return Err(anyhow!( + "Switch Port {} already exists in Switch {}.", + switch_port_id, + switch.id + )); + } + + switch.switch_ports.insert( + switch_port_id, + SwitchDownPort { + id: switch_port_id, + bus: swup_bus.to_string(), + allocated: false, + connected_device: None, + }, + ); + + Ok(()) + } + + /// Adds a root port to pcie bus + pub fn add_root_ports_on_bus(&mut self, num_root_ports: u32) -> Result<()> { + for index in 0..num_root_ports { + self.add_pcie_root_port(index)?; + } + + Ok(()) + } + + /// Strategy selection for adding switch ports + pub fn add_switch_ports_with_strategy( + &mut self, + num_switches: u32, + num_switch_ports: u32, + strategy: Strategy, + ) -> Result<()> { + match strategy { + Strategy::SingleRootPort => self.add_switch_ports_single_root_port(num_switch_ports, 1), + Strategy::MultipleRootPorts => { + self.add_switch_ports_multiple_root_ports(num_switches, num_switch_ports) + } + } + } + + /// Strategy 1: Use 1 Root Port to connect 1 PCIe Switch with multiple downstream switch ports + fn add_switch_ports_single_root_port( + &mut self, + num_switch_ports: u32, + root_port_id: u32, + ) -> Result<()> { + if !self.pcie_port_devices.contains_key(&root_port_id) { + self.add_pcie_root_port(root_port_id)?; + } + + let switch_id = root_port_id; + self.add_switch_to_root_port(root_port_id, switch_id)?; + + let swup_bus = format!("{}{}", PCIePortBusPrefix::SwitchUpstreamPort, switch_id); + for i in 1..=num_switch_ports { + self.add_switch_port_to_switch(&swup_bus, root_port_id, i)?; + } + + Ok(()) + } + + /// Strategy 2: Use multiple Root Ports to connect multiple PCIe Switches (each Switch provides at least 1 downstream port) + fn add_switch_ports_multiple_root_ports( + &mut self, + num_switches: u32, + num_switch_ports: u32, + ) -> Result<()> { + // Base number of ports per switch + let ports_per_switch = num_switch_ports / num_switches; + // Remaining ports to distribute + let remainder = num_switch_ports % num_switches; + + // Track the total number of ports assigned + let mut total_ports_assigned = 0; + + for root_port_id in 1..=num_switches { + // let root_port_id = i; + if !self.pcie_port_devices.contains_key(&root_port_id) { + self.add_pcie_root_port(root_port_id)?; + } + + let switch_id = root_port_id; + self.add_switch_to_root_port(root_port_id, switch_id)?; + let swup_bus = format!("{}{}", PCIePortBusPrefix::SwitchUpstreamPort, switch_id); + + // Calculate the number of ports for the current switch + let ports_in_switch = if root_port_id <= remainder { + // First `remainder` switches get an extra port + ports_per_switch + 1 + } else { + ports_per_switch + }; + + // Assign ports + for j in 1..=ports_in_switch { + let switch_port_id = total_ports_assigned + j; // Ensure unique ID + self.add_switch_port_to_switch(&swup_bus, root_port_id, switch_port_id)?; + } + + // Update the total number of ports assigned + total_ports_assigned += ports_in_switch; + } + + Ok(()) + } } // do_add_pcie_endpoint do add a device into PCIe topology with pcie endpoint @@ -364,3 +671,158 @@ pub fn do_add_pcie_endpoint( topology.do_insert_or_update(pcie_endpoint) } + +#[cfg(test)] +mod tests { + use super::*; + + fn new_pcie_topology(rp_total: u32, sw_total: u32) -> PCIeTopology { + PCIeTopology { + pcie_root_ports: rp_total, + pcie_switch_ports: sw_total, + pcie_port_devices: HashMap::with_capacity(sw_total as usize), + ..Default::default() + } + } + + fn create_pcie_topo(rps: u32, swps: u32) -> PCIeTopology { + PCIeTopology { + pcie_root_ports: rps, + pcie_switch_ports: swps, + ..Default::default() + } + } + + #[test] + fn test_no_port() { + let pcie_topo = create_pcie_topo(0, 0); + assert_eq!(pcie_topo.get_pcie_port(), Some((PCIePort::NoPort, 0))); + } + + #[test] + fn test_root_port_only() { + let pcie_topo = create_pcie_topo(2, 0); + assert_eq!(pcie_topo.get_pcie_port(), Some((PCIePort::RootPort, 2))); + } + + #[test] + fn test_switch_port_only() { + let pcie_topo = create_pcie_topo(0, 1); + assert_eq!(pcie_topo.get_pcie_port(), Some((PCIePort::SwitchPort, 1))); + } + + #[test] + fn test_both_ports_invalid() { + let pcie_topo = create_pcie_topo(1, 1); + assert_eq!(pcie_topo.get_pcie_port(), None); + } + + #[test] + fn test_new_pcie_topology() { + let topology = new_pcie_topology(2, 3); + assert_eq!(topology.pcie_root_ports, 2); + assert_eq!(topology.pcie_switch_ports, 3); + assert!(topology.pcie_port_devices.is_empty()); + } + + #[test] + fn test_get_pcie_port() { + let topology = new_pcie_topology(2, 0); + assert_eq!(topology.get_pcie_port(), Some((PCIePort::RootPort, 2))); + + let topology = new_pcie_topology(0, 3); + assert_eq!(topology.get_pcie_port(), Some((PCIePort::SwitchPort, 3))); + + let topology = new_pcie_topology(0, 0); + assert_eq!(topology.get_pcie_port(), Some((PCIePort::NoPort, 0))); + + let topology = new_pcie_topology(2, 3); + assert_eq!(topology.get_pcie_port(), None); + } + + #[test] + fn test_add_pcie_root_port() { + let mut topology = new_pcie_topology(1, 0); + assert!(topology.add_pcie_root_port(1).is_ok()); + assert!(topology.pcie_port_devices.contains_key(&1)); + + // Adding the same root port again should fail + assert!(topology.add_pcie_root_port(1).is_err()); + } + + #[test] + fn test_add_switch_to_root_port() { + let mut topology = new_pcie_topology(1, 0); + topology.add_pcie_root_port(1).unwrap(); + assert!(topology.add_switch_to_root_port(1, 101).is_ok()); + + // Adding a switch to a non-existent root port should fail + assert!(topology.add_switch_to_root_port(2, 102).is_err()); + + // Adding a switch to a root port that already has a switch should fail + assert!(topology.add_switch_to_root_port(1, 103).is_err()); + } + + #[test] + fn test_add_switch_port_to_switch() { + let mut topology = new_pcie_topology(1, 0); + topology.add_pcie_root_port(1).unwrap(); + topology.add_switch_to_root_port(1, 101).unwrap(); + + let swup_bus = format!("{}{}", PCIePortBusPrefix::SwitchUpstreamPort, 101); + assert!(topology.add_switch_port_to_switch(&swup_bus, 1, 1).is_ok()); + + // Adding a switch port to a non-existent root port should fail + assert!(topology.add_switch_port_to_switch(&swup_bus, 2, 1).is_err()); + + // Adding a switch port to a root port without a switch should fail + let mut topology = new_pcie_topology(1, 0); + topology.add_pcie_root_port(1).unwrap(); + assert!(topology.add_switch_port_to_switch(&swup_bus, 1, 1).is_err()); + + // Adding a switch port with a duplicate ID should fail + let mut topology = new_pcie_topology(1, 0); + topology.add_pcie_root_port(1).unwrap(); + topology.add_switch_to_root_port(1, 101).unwrap(); + assert!(topology.add_switch_port_to_switch(&swup_bus, 1, 1).is_ok()); + assert!(topology.add_switch_port_to_switch(&swup_bus, 1, 1).is_err()); + } + + #[test] + fn test_add_root_ports_on_bus() { + let mut topology = new_pcie_topology(3, 0); + assert!(topology.add_root_ports_on_bus(3).is_ok()); + assert_eq!(topology.pcie_port_devices.len(), 3); + + // Adding more root ports than available should fail + assert!(topology.add_root_ports_on_bus(1).is_err()); + } + + #[test] + fn test_add_switch_ports_single_root_port() { + let mut topology = new_pcie_topology(0, 2); + assert!(topology + .add_switch_ports_with_strategy(1, 2, Strategy::SingleRootPort) + .is_ok()); + + let root_port = topology.pcie_port_devices.get(&1).unwrap(); + assert!(root_port.connected_switch.is_some()); + let switch = root_port.connected_switch.as_ref().unwrap(); + assert_eq!(switch.switch_ports.len(), 2); + } + + #[test] + fn test_add_switch_ports_multiple_root_ports() { + let mut topology = new_pcie_topology(2, 0); + assert!(topology + .add_switch_ports_with_strategy(2, 4, Strategy::MultipleRootPorts) + .is_ok()); + + for i in 1..=2 { + let root_port = topology.pcie_port_devices.get(&i).unwrap(); + assert!(root_port.connected_switch.is_some()); + let switch = root_port.connected_switch.as_ref().unwrap(); + assert_eq!(switch.switch_ports.len(), 2); + } + } +}