diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index 1cdd13212..fbc54a6e4 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -830,6 +830,7 @@ dependencies = [ "num_cpus", "oci-spec", "once_cell", + "pci-ids", "rand", "runtime-spec", "safe-path", @@ -941,6 +942,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.3" @@ -992,6 +999,16 @@ dependencies = [ "pin-utils", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "ntapi" version = "0.4.1" @@ -1102,6 +1119,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "pci-ids" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d88ae3281b415d856e9c2ddbcdd5961e71c1a3e90138512c04d720241853a6af" +dependencies = [ + "nom", + "phf", + "phf_codegen", + "proc-macro2", + "quote", +] + [[package]] name = "petgraph" version = "0.5.1" @@ -1112,6 +1142,44 @@ dependencies = [ "indexmap", ] +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.0.12" @@ -1676,6 +1744,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "slab" version = "0.4.6" diff --git a/src/libs/kata-sys-util/Cargo.toml b/src/libs/kata-sys-util/Cargo.toml index 079339c9c..0c66c75fc 100644 --- a/src/libs/kata-sys-util/Cargo.toml +++ b/src/libs/kata-sys-util/Cargo.toml @@ -28,6 +28,7 @@ subprocess = "0.2.8" rand = "0.8.5" thiserror = "1.0.30" hex = "0.4.3" +pci-ids = "0.2.5" kata-types = { path = "../kata-types" } oci-spec = { version = "0.6.8", features = ["runtime"] } diff --git a/src/libs/kata-sys-util/src/lib.rs b/src/libs/kata-sys-util/src/lib.rs index 1e51c6e85..8131835c6 100644 --- a/src/libs/kata-sys-util/src/lib.rs +++ b/src/libs/kata-sys-util/src/lib.rs @@ -14,6 +14,7 @@ pub mod k8s; pub mod mount; pub mod netns; pub mod numa; +pub mod pcilibs; pub mod protection; pub mod rand; pub mod spec; diff --git a/src/libs/kata-sys-util/src/pcilibs/mod.rs b/src/libs/kata-sys-util/src/pcilibs/mod.rs new file mode 100644 index 000000000..2ff35b9d0 --- /dev/null +++ b/src/libs/kata-sys-util/src/pcilibs/mod.rs @@ -0,0 +1,5 @@ +// Copyright (c) 2024 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// +mod pci_manager; diff --git a/src/libs/kata-sys-util/src/pcilibs/pci_manager.rs b/src/libs/kata-sys-util/src/pcilibs/pci_manager.rs new file mode 100644 index 000000000..906e1683a --- /dev/null +++ b/src/libs/kata-sys-util/src/pcilibs/pci_manager.rs @@ -0,0 +1,292 @@ +// Copyright (c) 2024 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// +#![allow(dead_code)] +use std::collections::HashMap; +use std::fs; +use std::io; +use std::path::PathBuf; + +use pci_ids::{Classes, Vendors}; + +const PCI_DEV_DOMAIN: &str = "0000"; +const PCI_CONFIG_SPACE_SZ: u64 = 256; + +const UNKNOWN_DEVICE: &str = "UNKNOWN_DEVICE"; +const UNKNOWN_CLASS: &str = "UNKNOWN_CLASS"; + +const PCI_IOV_NUM_BAR: usize = 6; +const PCI_BASE_ADDRESS_MEM_TYPE_MASK: u64 = 0x06; + +pub(crate) const PCI_BASE_ADDRESS_MEM_TYPE32: u64 = 0x00; // 32 bit address +pub(crate) const PCI_BASE_ADDRESS_MEM_TYPE64: u64 = 0x04; // 64 bit address + +fn address_to_id(address: &str) -> u64 { + let cleaned_address = address.replace(":", "").replace(".", ""); + u64::from_str_radix(&cleaned_address, 16).unwrap_or(0) +} + +// Calculate the next power of 2. +fn calc_next_power_of_2(mut n: u64) -> u64 { + if n < 1 { + return 1_u64; + } + + n -= 1; + n |= n >> 1; + n |= n >> 2; + n |= n >> 4; + n |= n >> 8; + n |= n >> 16; + n |= n >> 32; + n + 1 +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct MemoryResource { + pub(crate) start: u64, + pub(crate) end: u64, + pub(crate) flags: u64, + pub(crate) path: PathBuf, +} + +pub(crate) type MemoryResources = HashMap; + +pub(crate) trait MemoryResourceTrait { + fn get_total_addressable_memory(&self, round_up: bool) -> (u64, u64); +} + +impl MemoryResourceTrait for MemoryResources { + fn get_total_addressable_memory(&self, round_up: bool) -> (u64, u64) { + let mut num_bar = 0; + let mut mem_size_32bit = 0u64; + let mut mem_size_64bit = 0u64; + + let mut keys: Vec<_> = self.keys().cloned().collect(); + keys.sort(); + + for key in keys { + if key as usize >= PCI_IOV_NUM_BAR || num_bar == PCI_IOV_NUM_BAR { + break; + } + num_bar += 1; + + if let Some(region) = self.get(&key) { + let flags = region.flags & PCI_BASE_ADDRESS_MEM_TYPE_MASK; + let mem_type_32bit = flags == PCI_BASE_ADDRESS_MEM_TYPE32; + let mem_type_64bit = flags == PCI_BASE_ADDRESS_MEM_TYPE64; + let mem_size = (region.end - region.start + 1) as u64; + + if mem_type_32bit { + mem_size_32bit += mem_size; + } + if mem_type_64bit { + mem_size_64bit += mem_size; + } + } + } + + if round_up { + mem_size_32bit = calc_next_power_of_2(mem_size_32bit); + mem_size_64bit = calc_next_power_of_2(mem_size_64bit); + } + + (mem_size_32bit, mem_size_64bit) + } +} + +pub trait PCIDevices { + fn get_pci_devices(&self, vendor: Option) -> Vec; +} + +#[derive(Clone, Debug, Default)] +pub struct PCIDevice { + pub(crate) device_path: PathBuf, + pub(crate) address: String, + pub(crate) vendor: u16, + pub(crate) class: u32, + pub(crate) class_name: String, + pub(crate) device: u16, + pub(crate) device_name: String, + pub(crate) driver: String, + pub(crate) iommu_group: i64, + pub(crate) numa_node: i64, + pub(crate) resources: MemoryResources, +} + +pub struct PCIDeviceManager { + pci_devices_root: PathBuf, +} + +impl PCIDeviceManager { + pub fn new(pci_devices_root: &str) -> Self { + PCIDeviceManager { + pci_devices_root: PathBuf::from(pci_devices_root), + } + } + + pub fn get_all_devices(&self, vendor: Option) -> io::Result> { + let mut pci_devices = Vec::new(); + let device_dirs = fs::read_dir(&self.pci_devices_root)?; + + let mut cache: HashMap = HashMap::new(); + + for entry in device_dirs { + let device_dir = entry?; + let device_address = device_dir.file_name().to_string_lossy().to_string(); + if let Ok(device) = self.get_device_by_pci_bus_id(&device_address, vendor, &mut cache) { + if let Some(dev) = device { + pci_devices.push(dev); + } + } + } + + pci_devices.sort_by_key(|dev| address_to_id(&dev.address)); + + Ok(pci_devices) + } + + fn get_device_by_pci_bus_id( + &self, + address: &str, + vendor: Option, + cache: &mut HashMap, + ) -> io::Result> { + if let Some(device) = cache.get(address) { + return Ok(Some(device.clone())); + } + + let device_path = self.pci_devices_root.join(address); + + // read vendor ID + let vendor_str = fs::read_to_string(device_path.join("vendor"))?; + let vendor_id = u16::from_str_radix(vendor_str.trim().trim_start_matches("0x"), 16) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + if let Some(vend_id) = vendor { + if vendor_id != vend_id { + return Ok(None); + } + } + + let class_str = fs::read_to_string(device_path.join("class"))?; + let class_id = u32::from_str_radix(class_str.trim().trim_start_matches("0x"), 16).unwrap(); + + let device_str = fs::read_to_string(device_path.join("device"))?; + let device_id = + u16::from_str_radix(device_str.trim().trim_start_matches("0x"), 16).unwrap(); + + let driver = match fs::read_link(device_path.join("driver")) { + Ok(path) => path.file_name().unwrap().to_string_lossy().to_string(), + Err(_) => String::new(), + }; + + let iommu_group = match fs::read_link(device_path.join("iommu_group")) { + Ok(path) => path + .file_name() + .unwrap() + .to_string_lossy() + .into_owned() + .parse::() + .unwrap_or(-1), + Err(_) => -1, + }; + + let numa_node = fs::read_to_string(device_path.join("numa_node")) + .map(|numa| numa.trim().parse::().unwrap_or(-1)) + .unwrap_or(-1); + + let resources = self.parse_resources(&device_path)?; + + let mut device_name = UNKNOWN_DEVICE.to_string(); + for vendor in Vendors::iter() { + for device in vendor.devices() { + if vendor.id() == vendor_id && device.id() == device_id { + device_name = device.name().to_owned(); + break; + } + } + } + + let mut class_name = UNKNOWN_CLASS.to_string(); + for class in Classes::iter() { + if u32::from(class.id()) == class_id { + class_name = class.name().to_owned(); + break; + } + } + + let pci_device = PCIDevice { + device_path, + address: address.to_string(), + vendor: vendor_id, + class: class_id, + device: device_id, + driver, + iommu_group, + numa_node, + resources, + device_name, + class_name, + }; + + cache.insert(address.to_string(), pci_device.clone()); + + Ok(Some(pci_device)) + } + + fn parse_resources(&self, device_path: &PathBuf) -> io::Result { + let content = fs::read_to_string(device_path.join("resource"))?; + let mut resources: MemoryResources = MemoryResources::new(); + for (i, line) in content.lines().enumerate() { + let values: Vec<&str> = line.split_whitespace().collect(); + if values.len() != 3 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("there's more than 3 entries in line '{}'", i), + )); + } + + let mem_start = u64::from_str_radix(values[0].trim_start_matches("0x"), 16) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + let mem_end = u64::from_str_radix(values[1].trim_start_matches("0x"), 16) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + let mem_flags = u64::from_str_radix(values[2].trim_start_matches("0x"), 16) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + if mem_end > mem_start { + resources.insert( + i, + MemoryResource { + start: mem_start, + end: mem_end, + flags: mem_flags, + path: device_path.join(format!("resource{}", i)), + }, + ); + } + } + + Ok(resources) + } +} + +/// Checks if the given BDF corresponds to a PCIe device. +/// The sysbus_pci_root is the path "/sys/bus/pci/devices" +pub fn is_pcie_device(bdf: &str, sysbus_pci_root: &str) -> bool { + let bdf_with_domain = if bdf.split(':').count() == 2 { + format!("{}:{}", PCI_DEV_DOMAIN, bdf) + } else { + bdf.to_string() + }; + + let config_path = PathBuf::from(sysbus_pci_root) + .join(bdf_with_domain) + .join("config"); + + match fs::metadata(config_path) { + Ok(metadata) => metadata.len() > PCI_CONFIG_SPACE_SZ, + // Error reading the file, assume it's not a PCIe device + Err(_) => false, + } +}