runtime-rs: Add PCIe topology mgmt for Root Port and Switch Port

This commit introduces an implementation for managing PCIe topologies,
focusing on the relationship between Root Ports and Switch Ports. The
design supports two strategies for generating Switch Ports:

Let's take the requirement of 4 switch ports as an example. There'll be
three possible solutions as below:
(1) Single Root Port + Single PCIe Switch: Uses 1 Root Port and 1 Switch
with 4 Downstream Ports.
(2) Multiple Root Ports + Multiple PCIe Switches: Uses 2 Root Ports and
2 Switches, each with 2 Downstream Ports.

The recommended strategy is Option 1 due to its simplicity, efficiency,
and scalability. The implementation includes data structures
(PcieTopology, RootPort, PcieSwitch, SwitchPort) and operations
(add_pcie_root_port, add_switch_to_root_port, add_switch_port_to_switch)
to manage the topology effectively.

Fxies #10361

Signed-off-by: alex.lyn <alex.lyn@antgroup.com>

src/runtime-rs/crates/hypervisor/src/device/topology.rs

@@ -61,7 +61,7 @@ use kata_types::config::hypervisor::TopologyConfigInfo;
 
 use super::pci_path::PciPath;
 
-const DEFAULT_PCIE_ROOT_BUS: &str = "pcie.0";
+pub const DEFAULT_PCIE_ROOT_BUS: &str = "pcie.0";
 // Currently, CLH and Dragonball support device attachment solely on the root bus.
 const DEFAULT_PCIE_ROOT_BUS_ADDRESS: &str = "0000:00";
 pub const PCIE_ROOT_BUS_SLOTS_CAPACITY: u32 = 32;
@@ -203,14 +203,132 @@ pub struct PCIeRootComplex {
     pub root_bus_devices: HashMap<String, PCIeEndpoint>,
 }
 
-#[derive(Debug, Default)]
+/// PCIePortBusPrefix defines the naming scheme for PCIe ports.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub enum PCIePortBusPrefix {
+    RootPort,
+    SwitchPort,
+    SwitchUpstreamPort,
+    SwitchDownstreamPort,
+}
+
+impl std::fmt::Display for PCIePortBusPrefix {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let prefix = match self {
+            PCIePortBusPrefix::RootPort => "rp",
+            PCIePortBusPrefix::SwitchPort => "sw",
+            PCIePortBusPrefix::SwitchUpstreamPort => "swup",
+            PCIePortBusPrefix::SwitchDownstreamPort => "swdp",
+        };
+        write!(f, "{}", prefix)
+    }
+}
+
+/// PCIePort distinguishes between different types of PCIe ports.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
+pub enum PCIePort {
+    // NoPort is for disabling VFIO hotplug/coldplug
+    #[default]
+    NoPort,
+
+    /// RootPort attach VFIO devices to a root-port
+    RootPort,
+
+    // SwitchPort attach VFIO devices to a switch-port
+    SwitchPort,
+}
+
+impl std::fmt::Display for PCIePort {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let port = match self {
+            PCIePort::NoPort => "no-port",
+            PCIePort::RootPort => "root-port",
+            PCIePort::SwitchPort => "switch-port",
+        };
+        write!(f, "{}", port)
+    }
+}
+
+/// Represents a PCIe port
+#[derive(Default, Clone, Debug)]
+pub struct SwitchDownPort {
+    pub id: u32,
+    pub bus: String, // swupX
+    pub allocated: bool,
+    pub connected_device: Option<String>,
+}
+
+impl SwitchDownPort {
+    pub fn port_id(&self) -> String {
+        format!("{}{}", PCIePortBusPrefix::SwitchDownstreamPort, self.id)
+    }
+}
+
+/// Represents a PCIe switch
+#[derive(Debug, Clone)]
+pub struct PcieSwitch {
+    pub id: u32,
+    pub bus: String,                                //rpX
+    pub switch_ports: HashMap<u32, SwitchDownPort>, // Switch ports
+}
+
+impl PcieSwitch {
+    pub fn port_id(&self) -> String {
+        format!("{}{}", PCIePortBusPrefix::SwitchUpstreamPort, self.id)
+    }
+}
+
+/// Represents a root port attached on root bus, TopologyPortDevice represents a PCIe device used for hotplugging.
+#[derive(Debug, Clone)]
+pub struct TopologyPortDevice {
+    pub id: u32,
+    pub bus: String, //pcie.0
+    pub allocated: bool,
+    pub connected_switch: Option<PcieSwitch>, // Connected PCIe switch
+}
+
+impl TopologyPortDevice {
+    pub fn new(id: u32, bus: &str) -> Self {
+        Self {
+            id,
+            bus: bus.to_owned(),
+            allocated: false,
+            connected_switch: None,
+        }
+    }
+
+    pub fn get_port_type(&self) -> PCIePort {
+        match self.connected_switch {
+            Some(_) => PCIePort::SwitchPort,
+            None => PCIePort::RootPort,
+        }
+    }
+
+    pub fn port_id(&self) -> String {
+        format!("{}{}", PCIePortBusPrefix::RootPort, self.id)
+    }
+}
+
+/// Represents strategy selection
+#[derive(Debug)]
+pub enum Strategy {
+    // Strategy 1: Use 1 Root Port to connect 1 PCIe Switch with multiple downstream switch ports
+    SingleRootPort,
+    // Strategy 2: Use multiple Root Ports to connect multiple PCIe Switches (each Switch provides at least 1 downstream port)
+    MultipleRootPorts,
+}
+
+#[derive(Clone, Debug, Default)]
 pub struct PCIeTopology {
     pub hypervisor_name: String,
     pub root_complex: PCIeRootComplex,
 
     pub bridges: u32,
     pub pcie_root_ports: u32,
+    pub pcie_switch_ports: u32,
     pub hotplug_vfio_on_root_bus: bool,
+    // pcie_port_devices keeps track of the devices attached to different types of PCI ports.
+    pub pcie_port_devices: HashMap<u32, TopologyPortDevice>,
 }
 
 impl PCIeTopology {
@@ -225,12 +343,18 @@ impl PCIeTopology {
             root_bus_devices: HashMap::with_capacity(PCIE_ROOT_BUS_SLOTS_CAPACITY as usize),
         };
 
+        // initialize port devices within PCIe Topology
+        let total_rp = topo_config.device_info.pcie_root_port;
+        let total_swp = topo_config.device_info.pcie_switch_port;
+
         Some(Self {
             hypervisor_name: topo_config.hypervisor_name.to_owned(),
             root_complex,
             bridges: topo_config.device_info.default_bridges,
-            pcie_root_ports: topo_config.device_info.pcie_root_port,
+            pcie_root_ports: total_rp,
+            pcie_switch_ports: total_swp,
             hotplug_vfio_on_root_bus: topo_config.device_info.hotplug_vfio_on_root_bus,
+            pcie_port_devices: HashMap::new(),
         })
     }
 
@@ -341,6 +465,189 @@ impl PCIeTopology {
 
         Ok(pci_path)
     }
+
+    /// get pcie port and its total
+    pub fn get_pcie_port(&self) -> Option<(PCIePort, u32)> {
+        match (self.pcie_root_ports, self.pcie_switch_ports) {
+            (_, _) if self.pcie_root_ports > 0 && self.pcie_switch_ports > 0 => None,
+            (0, 0) => Some((PCIePort::NoPort, 0)),
+            (r, _) if r > 0 => Some((PCIePort::RootPort, r)),
+            (_, s) if s > 0 => Some((PCIePort::SwitchPort, s)),
+            _ => None,
+        }
+    }
+
+    /// Adds a root port to pcie bus
+    fn add_pcie_root_port(&mut self, id: u32) -> Result<()> {
+        if self.pcie_port_devices.contains_key(&id) {
+            return Err(anyhow!("Root Port {} already exists.", id));
+        }
+
+        self.pcie_port_devices.insert(
+            id,
+            TopologyPortDevice {
+                id,
+                bus: DEFAULT_PCIE_ROOT_BUS.to_string(),
+                allocated: false,
+                connected_switch: None,
+            },
+        );
+
+        Ok(())
+    }
+
+    /// Adds a PCIe switch to a root port
+    fn add_switch_to_root_port(&mut self, root_port_id: u32, switch_id: u32) -> Result<()> {
+        let root_port = self
+            .pcie_port_devices
+            .get_mut(&root_port_id)
+            .ok_or_else(|| anyhow!("Root Port {} does not exist.", root_port_id))?;
+
+        if root_port.connected_switch.is_some() {
+            return Err(anyhow!(
+                "Root Port {} already has a connected switch.",
+                root_port_id
+            ));
+        }
+        let rp_bus = format!("{}{}", PCIePortBusPrefix::RootPort, root_port_id);
+
+        root_port.allocated = true;
+        root_port.connected_switch = Some(PcieSwitch {
+            id: switch_id,
+            bus: rp_bus,
+            switch_ports: HashMap::new(),
+        });
+
+        Ok(())
+    }
+
+    /// Adds a switch port to a PCIe switch
+    fn add_switch_port_to_switch(
+        &mut self,
+        swup_bus: &str,
+        root_port_id: u32,
+        switch_port_id: u32,
+    ) -> Result<()> {
+        let root_port = self
+            .pcie_port_devices
+            .get_mut(&root_port_id)
+            .ok_or_else(|| anyhow!("Root Port {} does not exist.", root_port_id))?;
+
+        let switch = root_port
+            .connected_switch
+            .as_mut()
+            .ok_or_else(|| anyhow!("Root Port {} has no connected switch.", root_port_id))?;
+
+        if switch.switch_ports.contains_key(&switch_port_id) {
+            return Err(anyhow!(
+                "Switch Port {} already exists in Switch {}.",
+                switch_port_id,
+                switch.id
+            ));
+        }
+
+        switch.switch_ports.insert(
+            switch_port_id,
+            SwitchDownPort {
+                id: switch_port_id,
+                bus: swup_bus.to_string(),
+                allocated: false,
+                connected_device: None,
+            },
+        );
+
+        Ok(())
+    }
+
+    /// Adds a root port to pcie bus
+    pub fn add_root_ports_on_bus(&mut self, num_root_ports: u32) -> Result<()> {
+        for index in 0..num_root_ports {
+            self.add_pcie_root_port(index)?;
+        }
+
+        Ok(())
+    }
+
+    /// Strategy selection for adding switch ports
+    pub fn add_switch_ports_with_strategy(
+        &mut self,
+        num_switches: u32,
+        num_switch_ports: u32,
+        strategy: Strategy,
+    ) -> Result<()> {
+        match strategy {
+            Strategy::SingleRootPort => self.add_switch_ports_single_root_port(num_switch_ports, 1),
+            Strategy::MultipleRootPorts => {
+                self.add_switch_ports_multiple_root_ports(num_switches, num_switch_ports)
+            }
+        }
+    }
+
+    /// Strategy 1: Use 1 Root Port to connect 1 PCIe Switch with multiple downstream switch ports
+    fn add_switch_ports_single_root_port(
+        &mut self,
+        num_switch_ports: u32,
+        root_port_id: u32,
+    ) -> Result<()> {
+        if !self.pcie_port_devices.contains_key(&root_port_id) {
+            self.add_pcie_root_port(root_port_id)?;
+        }
+
+        let switch_id = root_port_id;
+        self.add_switch_to_root_port(root_port_id, switch_id)?;
+
+        let swup_bus = format!("{}{}", PCIePortBusPrefix::SwitchUpstreamPort, switch_id);
+        for i in 1..=num_switch_ports {
+            self.add_switch_port_to_switch(&swup_bus, root_port_id, i)?;
+        }
+
+        Ok(())
+    }
+
+    /// Strategy 2: Use multiple Root Ports to connect multiple PCIe Switches (each Switch provides at least 1 downstream port)
+    fn add_switch_ports_multiple_root_ports(
+        &mut self,
+        num_switches: u32,
+        num_switch_ports: u32,
+    ) -> Result<()> {
+        // Base number of ports per switch
+        let ports_per_switch = num_switch_ports / num_switches;
+        // Remaining ports to distribute
+        let remainder = num_switch_ports % num_switches;
+
+        // Track the total number of ports assigned
+        let mut total_ports_assigned = 0;
+
+        for root_port_id in 1..=num_switches {
+            // let root_port_id = i;
+            if !self.pcie_port_devices.contains_key(&root_port_id) {
+                self.add_pcie_root_port(root_port_id)?;
+            }
+
+            let switch_id = root_port_id;
+            self.add_switch_to_root_port(root_port_id, switch_id)?;
+            let swup_bus = format!("{}{}", PCIePortBusPrefix::SwitchUpstreamPort, switch_id);
+
+            // Calculate the number of ports for the current switch
+            let ports_in_switch = if root_port_id <= remainder {
+                // First `remainder` switches get an extra port
+                ports_per_switch + 1
+            } else {
+                ports_per_switch
+            };
+
+            // Assign ports
+            for j in 1..=ports_in_switch {
+                let switch_port_id = total_ports_assigned + j; // Ensure unique ID
+                self.add_switch_port_to_switch(&swup_bus, root_port_id, switch_port_id)?;
+            }
+
+            // Update the total number of ports assigned
+            total_ports_assigned += ports_in_switch;
+        }
+
+        Ok(())
+    }
 }
 
 // do_add_pcie_endpoint do add a device into PCIe topology with pcie endpoint
@@ -364,3 +671,158 @@ pub fn do_add_pcie_endpoint(
 
     topology.do_insert_or_update(pcie_endpoint)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn new_pcie_topology(rp_total: u32, sw_total: u32) -> PCIeTopology {
+        PCIeTopology {
+            pcie_root_ports: rp_total,
+            pcie_switch_ports: sw_total,
+            pcie_port_devices: HashMap::with_capacity(sw_total as usize),
+            ..Default::default()
+        }
+    }
+
+    fn create_pcie_topo(rps: u32, swps: u32) -> PCIeTopology {
+        PCIeTopology {
+            pcie_root_ports: rps,
+            pcie_switch_ports: swps,
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn test_no_port() {
+        let pcie_topo = create_pcie_topo(0, 0);
+        assert_eq!(pcie_topo.get_pcie_port(), Some((PCIePort::NoPort, 0)));
+    }
+
+    #[test]
+    fn test_root_port_only() {
+        let pcie_topo = create_pcie_topo(2, 0);
+        assert_eq!(pcie_topo.get_pcie_port(), Some((PCIePort::RootPort, 2)));
+    }
+
+    #[test]
+    fn test_switch_port_only() {
+        let pcie_topo = create_pcie_topo(0, 1);
+        assert_eq!(pcie_topo.get_pcie_port(), Some((PCIePort::SwitchPort, 1)));
+    }
+
+    #[test]
+    fn test_both_ports_invalid() {
+        let pcie_topo = create_pcie_topo(1, 1);
+        assert_eq!(pcie_topo.get_pcie_port(), None);
+    }
+
+    #[test]
+    fn test_new_pcie_topology() {
+        let topology = new_pcie_topology(2, 3);
+        assert_eq!(topology.pcie_root_ports, 2);
+        assert_eq!(topology.pcie_switch_ports, 3);
+        assert!(topology.pcie_port_devices.is_empty());
+    }
+
+    #[test]
+    fn test_get_pcie_port() {
+        let topology = new_pcie_topology(2, 0);
+        assert_eq!(topology.get_pcie_port(), Some((PCIePort::RootPort, 2)));
+
+        let topology = new_pcie_topology(0, 3);
+        assert_eq!(topology.get_pcie_port(), Some((PCIePort::SwitchPort, 3)));
+
+        let topology = new_pcie_topology(0, 0);
+        assert_eq!(topology.get_pcie_port(), Some((PCIePort::NoPort, 0)));
+
+        let topology = new_pcie_topology(2, 3);
+        assert_eq!(topology.get_pcie_port(), None);
+    }
+
+    #[test]
+    fn test_add_pcie_root_port() {
+        let mut topology = new_pcie_topology(1, 0);
+        assert!(topology.add_pcie_root_port(1).is_ok());
+        assert!(topology.pcie_port_devices.contains_key(&1));
+
+        // Adding the same root port again should fail
+        assert!(topology.add_pcie_root_port(1).is_err());
+    }
+
+    #[test]
+    fn test_add_switch_to_root_port() {
+        let mut topology = new_pcie_topology(1, 0);
+        topology.add_pcie_root_port(1).unwrap();
+        assert!(topology.add_switch_to_root_port(1, 101).is_ok());
+
+        // Adding a switch to a non-existent root port should fail
+        assert!(topology.add_switch_to_root_port(2, 102).is_err());
+
+        // Adding a switch to a root port that already has a switch should fail
+        assert!(topology.add_switch_to_root_port(1, 103).is_err());
+    }
+
+    #[test]
+    fn test_add_switch_port_to_switch() {
+        let mut topology = new_pcie_topology(1, 0);
+        topology.add_pcie_root_port(1).unwrap();
+        topology.add_switch_to_root_port(1, 101).unwrap();
+
+        let swup_bus = format!("{}{}", PCIePortBusPrefix::SwitchUpstreamPort, 101);
+        assert!(topology.add_switch_port_to_switch(&swup_bus, 1, 1).is_ok());
+
+        // Adding a switch port to a non-existent root port should fail
+        assert!(topology.add_switch_port_to_switch(&swup_bus, 2, 1).is_err());
+
+        // Adding a switch port to a root port without a switch should fail
+        let mut topology = new_pcie_topology(1, 0);
+        topology.add_pcie_root_port(1).unwrap();
+        assert!(topology.add_switch_port_to_switch(&swup_bus, 1, 1).is_err());
+
+        // Adding a switch port with a duplicate ID should fail
+        let mut topology = new_pcie_topology(1, 0);
+        topology.add_pcie_root_port(1).unwrap();
+        topology.add_switch_to_root_port(1, 101).unwrap();
+        assert!(topology.add_switch_port_to_switch(&swup_bus, 1, 1).is_ok());
+        assert!(topology.add_switch_port_to_switch(&swup_bus, 1, 1).is_err());
+    }
+
+    #[test]
+    fn test_add_root_ports_on_bus() {
+        let mut topology = new_pcie_topology(3, 0);
+        assert!(topology.add_root_ports_on_bus(3).is_ok());
+        assert_eq!(topology.pcie_port_devices.len(), 3);
+
+        // Adding more root ports than available should fail
+        assert!(topology.add_root_ports_on_bus(1).is_err());
+    }
+
+    #[test]
+    fn test_add_switch_ports_single_root_port() {
+        let mut topology = new_pcie_topology(0, 2);
+        assert!(topology
+            .add_switch_ports_with_strategy(1, 2, Strategy::SingleRootPort)
+            .is_ok());
+
+        let root_port = topology.pcie_port_devices.get(&1).unwrap();
+        assert!(root_port.connected_switch.is_some());
+        let switch = root_port.connected_switch.as_ref().unwrap();
+        assert_eq!(switch.switch_ports.len(), 2);
+    }
+
+    #[test]
+    fn test_add_switch_ports_multiple_root_ports() {
+        let mut topology = new_pcie_topology(2, 0);
+        assert!(topology
+            .add_switch_ports_with_strategy(2, 4, Strategy::MultipleRootPorts)
+            .is_ok());
+
+        for i in 1..=2 {
+            let root_port = topology.pcie_port_devices.get(&i).unwrap();
+            assert!(root_port.connected_switch.is_some());
+            let switch = root_port.connected_switch.as_ref().unwrap();
+            assert_eq!(switch.switch_ports.len(), 2);
+        }
+    }
+}