github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-07-19 01:39:48 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #8668 from Apokleos/pci-path-refactor

Browse Source 
runtime-rs: Refactor the code related to PCI paths and VFIO device driver initialize in DM.
This commit is contained in:
Alex.Lyn 2023-12-23 21:44:07 +08:00 committed by GitHub
commit 3a3f39aa2d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 258 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs
View File

@ -5,11 +5,11 @@
// SPDX-License-Identifier: Apache-2.0
use super::inner::CloudHypervisorInner;
use crate::device::pci_path::PciPath;
use crate::device::DeviceType;
use crate::HybridVsockDevice;
use crate::NetworkConfig;
use crate::NetworkDevice;
use crate::PciPath;
use crate::ShareFsConfig;
use crate::ShareFsDevice;
use crate::VfioDevice;
@ -277,7 +277,7 @@ impl CloudHypervisorInner {
));
}
PciPath::convert_from_string(toks[0])
PciPath::try_from(toks[0])
}
async fn handle_hvsock_device(&mut self, device: HybridVsockDevice) -> Result<DeviceType> {

2
src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
View File

@ -291,7 +291,7 @@ impl DeviceManager {
Arc::new(Mutex::new(VfioDevice::new(
device_id.clone(),
&vfio_dev_config,
)))
)?))
}
DeviceConfig::VhostUserBlkCfg(config) => {
// try to find the device, found and just return id.

190
src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs
View File

@ -12,8 +12,8 @@ mod virtio_net;
mod virtio_vsock;
pub use vfio::{
bind_device_to_host, bind_device_to_vfio, get_host_guest_map, get_vfio_device, HostDevice,
VfioBusMode, VfioConfig, VfioDevice,
bind_device_to_host, bind_device_to_vfio, get_vfio_device, HostDevice, VfioBusMode, VfioConfig,
VfioDevice,
};
pub use virtio_blk::{
BlockConfig, BlockDevice, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, KATA_NVDIMM_DEV_TYPE,
@ -29,189 +29,3 @@ pub use virtio_vsock::{
pub mod vhost_user_blk;
pub use vhost_user::{VhostUserConfig, VhostUserDevice, VhostUserType};
use anyhow::{anyhow, Context, Result};
// Tips:
// The Re-write `PciSlot` and `PciPath` with rust that it origins from `pcipath.go`:
//
// The PCI spec reserves 5 bits for slot number (a.k.a. device
// number), giving slots 0..31
const PCI_SLOT_BITS: u32 = 5;
const MAX_PCI_SLOTS: u32 = (1 << PCI_SLOT_BITS) - 1;
// A PciSlot describes where a PCI device sits on a single bus
//
// This encapsulates the PCI slot number a.k.a device number, which is
// limited to a 5 bit value [0x00..0x1f] by the PCI specification
//
// To support multifunction device's, It's needed to extend
// this to include the PCI 3-bit function number as well.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct PciSlot(pub u8);
impl PciSlot {
pub fn convert_from_string(s: &str) -> Result<PciSlot> {
if s.is_empty() || s.len() > 2 {
return Err(anyhow!("string given is invalid."));
}
let base = 16;
let n = u64::from_str_radix(s, base).context("convert string to number failed")?;
if n >> PCI_SLOT_BITS > 0 {
return Err(anyhow!(
"number {:?} exceeds MAX:{:?}, failed.",
n,
MAX_PCI_SLOTS
));
}
Ok(PciSlot(n as u8))
}
pub fn convert_from_u32(v: u32) -> Result<PciSlot> {
if v > MAX_PCI_SLOTS {
return Err(anyhow!("value {:?} exceeds MAX: {:?}", v, MAX_PCI_SLOTS));
}
Ok(PciSlot(v as u8))
}
pub fn convert_to_string(&self) -> String {
format!("{:02x}", self.0)
}
}
// A PciPath describes where a PCI sits in a PCI hierarchy.
//
// Consists of a list of PCI slots, giving the slot of each bridge
// that must be traversed from the PCI root to reach the device,
// followed by the slot of the device itself.
//
// When formatted into a string is written as "xx/.../yy/zz". Here,
// zz is the slot of the device on its PCI bridge, yy is the slot of
// the bridge on its parent bridge and so forth until xx is the slot
// of the "most upstream" bridge on the root bus.
//
// If a device is directly connected to the root bus, which used in
// lightweight hypervisors, such as dragonball/firecracker/clh, and
// its PciPath.slots will contains only one PciSlot.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct PciPath {
// list of PCI slots
slots: Vec<PciSlot>,
}
impl PciPath {
// method to format the PciPath into a string
pub fn convert_to_string(&self) -> String {
self.slots
.iter()
.map(|pci_slot| format!("{:02x}", pci_slot.0))
.collect::<Vec<String>>()
.join("/")
}
// method to parse a PciPath from a string
pub fn convert_from_string(path: &str) -> Result<PciPath> {
if path.is_empty() {
return Err(anyhow!("path given is empty."));
}
let mut pci_slots: Vec<PciSlot> = Vec::new();
let slots: Vec<&str> = path.split('/').collect();
for slot in slots {
match PciSlot::convert_from_string(slot) {
Ok(s) => pci_slots.push(s),
Err(e) => return Err(anyhow!("slot is invalid with: {:?}", e)),
}
}
Ok(PciPath { slots: pci_slots })
}
pub fn from_pci_slots(slots: Vec<PciSlot>) -> Option<PciPath> {
if slots.is_empty() {
return None;
}
Some(PciPath { slots })
}
// device_slot to get the slot of the device on its PCI bridge
pub fn get_device_slot(&self) -> Option<PciSlot> {
self.slots.last().cloned()
}
// root_slot to get the slot of the "most upstream" bridge on the root bus
pub fn get_root_slot(&self) -> Option<PciSlot> {
self.slots.first().cloned()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_pci_slot() {
// min
let pci_slot_01 = PciSlot::convert_from_string("00");
assert!(pci_slot_01.is_ok());
// max
let pci_slot_02 = PciSlot::convert_from_string("1f");
assert!(pci_slot_02.is_ok());
// exceed
let pci_slot_03 = PciSlot::convert_from_string("20");
assert!(pci_slot_03.is_err());
// valid number
let pci_slot_04 = PciSlot::convert_from_u32(1_u32);
assert!(pci_slot_04.is_ok());
assert_eq!(pci_slot_04.as_ref().unwrap().0, 1_u8);
let pci_slot_str = pci_slot_04.as_ref().unwrap().convert_to_string();
assert_eq!(pci_slot_str, format!("{:02x}", pci_slot_04.unwrap().0));
// max number
let pci_slot_05 = PciSlot::convert_from_u32(31_u32);
assert!(pci_slot_05.is_ok());
assert_eq!(pci_slot_05.unwrap().0, 31_u8);
// exceed and error
let pci_slot_06 = PciSlot::convert_from_u32(32_u32);
assert!(pci_slot_06.is_err());
}
#[test]
fn test_pci_patch() {
let pci_path_0 = PciPath::convert_from_string("01/0a/05");
assert!(pci_path_0.is_ok());
let pci_path_unwrap = pci_path_0.unwrap();
assert_eq!(pci_path_unwrap.slots[0].0, 1);
assert_eq!(pci_path_unwrap.slots[1].0, 10);
assert_eq!(pci_path_unwrap.slots[2].0, 5);
let pci_path_01 = PciPath::from_pci_slots(vec![PciSlot(1), PciSlot(10), PciSlot(5)]);
assert!(pci_path_01.is_some());
let pci_path = pci_path_01.unwrap();
let pci_path_02 = pci_path.convert_to_string();
assert_eq!(pci_path_02, "01/0a/05".to_string());
let dev_slot = pci_path.get_device_slot();
assert!(dev_slot.is_some());
assert_eq!(dev_slot.unwrap().0, 5);
let root_slot = pci_path.get_root_slot();
assert!(root_slot.is_some());
assert_eq!(root_slot.unwrap().0, 1);
}
#[test]
fn test_get_host_guest_map() {
// test unwrap is fine, no panic occurs.
let hg_map = get_host_guest_map("".to_owned());
assert!(hg_map.is_none());
}
}

47
src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs
View File

@ -20,12 +20,14 @@ use async_trait::async_trait;
use lazy_static::lazy_static;
use path_clean::PathClean;
use crate::{
device::{hypervisor, Device, DeviceType},
PciPath, PciSlot,
};
use kata_sys_util::fs::get_base_name;
use crate::device::{
hypervisor,
pci_path::{PciPath, PciSlot},
Device, DeviceType,
};
pub const SYS_BUS_PCI_DRIVER_PROBE: &str = "/sys/bus/pci/drivers_probe";
pub const SYS_BUS_PCI_DEVICES: &str = "/sys/bus/pci/devices";
pub const SYS_KERN_IOMMU_GROUPS: &str = "/sys/kernel/iommu_groups";
@ -66,16 +68,10 @@ pub fn generate_guest_pci_path(bdf: String) -> Result<PciPath> {
hg_map.write().unwrap().insert(host_bdf, guest_bdf);
Ok(PciPath {
slots: vec![PciSlot::convert_from_u32(slot.into()).context("pci slot convert failed.")?],
slots: vec![PciSlot::new(slot)],
})
}
// get host/guest mapping for info
pub fn get_host_guest_map(host_bdf: String) -> Option<String> {
// safe, just do unwrap as `HOST_GUEST_MAP` is always valid.
HOST_GUEST_MAP.read().unwrap().get(&host_bdf).cloned()
}
pub fn do_check_iommu_on() -> Result<bool> {
let element = std::fs::read_dir(SYS_CLASS_IOMMU)?
.filter_map(|e| e.ok())
@ -256,7 +252,7 @@ pub struct VfioDevice {
impl VfioDevice {
// new with VfioConfig
pub fn new(device_id: String, dev_info: &VfioConfig) -> Self {
pub fn new(device_id: String, dev_info: &VfioConfig) -> Result<Self> {
// devices and device_options are in a 1-1 mapping, used in
// vfio-pci handler for kata-agent.
let devices: Vec<HostDevice> = Vec::with_capacity(MAX_DEV_ID_SIZE);
@ -266,7 +262,7 @@ impl VfioDevice {
let dev_type = dev_info.dev_type.as_str();
let driver_type = VfioBusMode::driver_type(dev_type).to_owned();
Self {
let mut vfio_device = Self {
device_id,
attach_count: 0,
bus_mode: VfioBusMode::PCI,
@ -274,7 +270,13 @@ impl VfioDevice {
config: dev_info.clone(),
devices,
device_options,
}
};
vfio_device
.initialize_vfio_device()
.context("initialize vfio device failed.")?;
Ok(vfio_device)
}
fn get_host_path(&self) -> String {
@ -374,7 +376,7 @@ impl VfioDevice {
Ok(DeviceVendor(device, vendor))
}
async fn set_vfio_config(
fn set_vfio_config(
&mut self,
iommu_devs_path: PathBuf,
device_name: &str,
@ -426,11 +428,8 @@ impl VfioDevice {
_ => None,
}
}
}
#[async_trait]
impl Device for VfioDevice {
async fn attach(&mut self, h: &dyn hypervisor) -> Result<()> {
fn initialize_vfio_device(&mut self) -> Result<()> {
// host path: /dev/vfio/X
let host_path = self.get_host_path();
// vfio group: X
@ -464,7 +463,6 @@ impl Device for VfioDevice {
let mut hostdev: HostDevice = self
.set_vfio_config(iommu_devs_path.clone(), device)
.await
.context("set vfio config failed")?;
let dev_prefix = self.get_vfio_prefix();
hostdev.hostdev_id = make_device_nameid(&dev_prefix, index, MAX_DEV_ID_SIZE);
@ -472,6 +470,13 @@ impl Device for VfioDevice {
self.devices.push(hostdev);
}
Ok(())
}
}
#[async_trait]
impl Device for VfioDevice {
async fn attach(&mut self, h: &dyn hypervisor) -> Result<()> {
if self
.increase_attach_count()
.await
@ -505,7 +510,7 @@ impl Device for VfioDevice {
self.device_options.push(format!(
"0000:{}={}",
hostdev.bus_slot_func.clone(),
pci_path.convert_to_string()
pci_path.to_string()
));
}
}

6
src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs
View File

@ -4,6 +4,8 @@
// SPDX-License-Identifier: Apache-2.0
//
use crate::device::pci_path::PciPath;
#[derive(Debug, Clone)]
pub enum VhostUserType {
/// Blk - represents a block vhostuser device type
@ -50,8 +52,8 @@ pub struct VhostUserConfig {
pub device_type: VhostUserType,
/// guest block driver
pub driver_option: String,
/// pci_addr is the PCI address used to identify the slot at which the drive is attached.
pub pci_addr: Option<String>,
/// pci_path is the PCI Path used to identify the slot at which the device is attached.
pub pci_path: Option<PciPath>,
/// Block index of the device if assigned
/// type u64 is not OK

2
src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs
View File

@ -4,10 +4,10 @@
// SPDX-License-Identifier: Apache-2.0
//
use crate::device::pci_path::PciPath;
use crate::device::Device;
use crate::device::DeviceType;
use crate::Hypervisor as hypervisor;
use crate::PciPath;
use anyhow::{anyhow, Context, Result};
use async_trait::async_trait;

1
src/runtime-rs/crates/hypervisor/src/device/mod.rs
View File

@ -17,6 +17,7 @@ use async_trait::async_trait;
pub mod device_manager;
pub mod driver;
pub mod pci_path;
pub mod util;
#[derive(Debug)]

220
src/runtime-rs/crates/hypervisor/src/device/pci_path.rs Normal file
View File

@ -0,0 +1,220 @@
// Copyright (c) 2019-2023 Alibaba Cloud
// Copyright (c) 2019-2023 Ant Group
//
// SPDX-License-Identifier: Apache-2.0
//
use std::convert::TryFrom;
use anyhow::{anyhow, Context, Result};
// Tips:
// The Re-write `PciSlot` and `PciPath` with rust that it origins from `pcipath.go`:
//
// The PCI spec reserves 5 bits for slot number (a.k.a. device
// number), giving slots 0..31
const PCI_SLOT_BITS: u32 = 5;
const MAX_PCI_SLOTS: u32 = (1 << PCI_SLOT_BITS) - 1;
// A PciSlot describes where a PCI device sits on a single bus
//
// This encapsulates the PCI slot number a.k.a device number, which is
// limited to a 5 bit value [0x00..0x1f] by the PCI specification
//
// To support multifunction device's, It's needed to extend
// this to include the PCI 3-bit function number as well.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct PciSlot(pub u8);
impl PciSlot {
pub fn new(v: u8) -> PciSlot {
PciSlot(v)
}
}
impl ToString for PciSlot {
fn to_string(&self) -> String {
format!("{:02x}", self.0)
}
}
impl TryFrom<&str> for PciSlot {
type Error = anyhow::Error;
fn try_from(s: &str) -> Result<PciSlot> {
if s.is_empty() || s.len() > 2 {
return Err(anyhow!("string given is invalid."));
}
let base = 16;
let n = u64::from_str_radix(s, base).context(format!(
"convert string to number with base {:?} failed.",
base
))?;
if n >> PCI_SLOT_BITS > 0 {
return Err(anyhow!(
"number {:?} exceeds MAX:{:?}, failed.",
n,
MAX_PCI_SLOTS
));
}
Ok(PciSlot(n as u8))
}
}
impl TryFrom<u32> for PciSlot {
type Error = anyhow::Error;
fn try_from(v: u32) -> Result<PciSlot> {
if v > MAX_PCI_SLOTS {
return Err(anyhow!("value {:?} exceeds MAX: {:?}", v, MAX_PCI_SLOTS));
}
Ok(PciSlot(v as u8))
}
}
// A PciPath describes where a PCI sits in a PCI hierarchy.
//
// Consists of a list of PCI slots, giving the slot of each bridge
// that must be traversed from the PCI root to reach the device,
// followed by the slot of the device itself.
//
// When formatted into a string is written as "xx/.../yy/zz". Here,
// zz is the slot of the device on its PCI bridge, yy is the slot of
// the bridge on its parent bridge and so forth until xx is the slot
// of the "most upstream" bridge on the root bus.
//
// If a device is directly connected to the root bus, which used in
// lightweight hypervisors, such as dragonball/firecracker/clh, and
// its PciPath.slots will contains only one PciSlot.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct PciPath {
// list of PCI slots
pub slots: Vec<PciSlot>,
}
impl PciPath {
pub fn new(slots: Vec<PciSlot>) -> Option<PciPath> {
if slots.is_empty() {
return None;
}
Some(PciPath { slots })
}
// device_slot to get the slot of the device on its PCI bridge
pub fn get_device_slot(&self) -> Option<PciSlot> {
self.slots.last().cloned()
}
// root_slot to get the slot of the "most upstream" bridge on the root bus
pub fn get_root_slot(&self) -> Option<PciSlot> {
self.slots.first().cloned()
}
}
impl ToString for PciPath {
// method to format the PciPath into a string
fn to_string(&self) -> String {
self.slots
.iter()
.map(|pci_slot| format!("{:02x}", pci_slot.0))
.collect::<Vec<String>>()
.join("/")
}
}
// convert from u32
impl TryFrom<u32> for PciPath {
type Error = anyhow::Error;
fn try_from(slot: u32) -> Result<PciPath> {
Ok(PciPath {
slots: vec![PciSlot::try_from(slot).context("pci slot convert failed.")?],
})
}
}
impl TryFrom<&str> for PciPath {
type Error = anyhow::Error;
// method to parse a PciPath from a string
fn try_from(path: &str) -> Result<PciPath> {
if path.is_empty() {
return Err(anyhow!("path given is empty."));
}
let mut pci_slots: Vec<PciSlot> = Vec::new();
let slots: Vec<&str> = path.split('/').collect();
for slot in slots {
match PciSlot::try_from(slot) {
Ok(s) => pci_slots.push(s),
Err(e) => return Err(anyhow!("slot is invalid with: {:?}", e)),
}
}
Ok(PciPath { slots: pci_slots })
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_pci_slot() {
// min
let pci_slot_01 = PciSlot::try_from("00");
assert!(pci_slot_01.is_ok());
// max
let pci_slot_02 = PciSlot::try_from("1f");
assert!(pci_slot_02.is_ok());
// exceed
let pci_slot_03 = PciSlot::try_from("20");
assert!(pci_slot_03.is_err());
// valid number
let pci_slot_04 = PciSlot::try_from(1_u32);
assert!(pci_slot_04.is_ok());
assert_eq!(pci_slot_04.as_ref().unwrap().0, 1_u8);
let pci_slot_str = pci_slot_04.as_ref().unwrap().to_string();
assert_eq!(pci_slot_str, format!("{:02x}", pci_slot_04.unwrap().0));
// max number
let pci_slot_05 = PciSlot::try_from(31_u32);
assert!(pci_slot_05.is_ok());
assert_eq!(pci_slot_05.unwrap().0, 31_u8);
// exceed and error
let pci_slot_06 = PciSlot::try_from(32_u32);
assert!(pci_slot_06.is_err());
}
#[test]
fn test_pci_patch() {
let pci_path_0 = PciPath::try_from("01/0a/05");
assert!(pci_path_0.is_ok());
let pci_path_unwrap = pci_path_0.unwrap();
assert_eq!(pci_path_unwrap.slots[0].0, 1);
assert_eq!(pci_path_unwrap.slots[1].0, 10);
assert_eq!(pci_path_unwrap.slots[2].0, 5);
let pci_path_01 = PciPath::new(vec![PciSlot(1), PciSlot(10), PciSlot(5)]);
assert!(pci_path_01.is_some());
let pci_path = pci_path_01.unwrap();
let pci_path_02 = pci_path.to_string();
assert_eq!(pci_path_02, "01/0a/05".to_string());
let dev_slot = pci_path.get_device_slot();
assert!(dev_slot.is_some());
assert_eq!(dev_slot.unwrap().0, 5);
let root_slot = pci_path.get_root_slot();
assert!(root_slot.is_some());
assert_eq!(root_slot.unwrap().0, 1);
}
}

2
src/runtime-rs/crates/resource/src/manager_inner.rs
View File

@ -313,7 +313,7 @@ impl ResourceManagerInner {
// The following would work for drivers virtio-blk-pci and mmio.
// Once scsi support is added, need to handle scsi identifiers.
let id = if let Some(pci_path) = device.config.pci_path {
pci_path.convert_to_string()
pci_path.to_string()
} else {
device.config.virt_path.clone()
};