runtime-rs: support physical endpoint using device manager

use device manager to attach physical endpoint

Fixes: #7103
Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>

src/runtime-rs/crates/hypervisor/src/device/device_manager.rs

@@ -77,6 +77,7 @@ impl SharedInfo {
 }
 
 // Device manager will manage the lifecycle of sandbox device
+#[derive(Debug)]
 pub struct DeviceManager {
     devices: HashMap<String, ArcMutexDevice>,
     hypervisor: Arc<dyn Hypervisor>,

src/runtime-rs/crates/hypervisor/src/device/mod.rs

@@ -50,7 +50,7 @@ impl fmt::Display for DeviceType {
 }
 
 #[async_trait]
-pub trait Device: Send + Sync {
+pub trait Device: std::fmt::Debug + Send + Sync {
     // attach is to plug device into VM
     async fn attach(&mut self, h: &dyn hypervisor) -> Result<()>;
     // detach is to unplug device from VM

src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs

@@ -28,6 +28,7 @@ use std::{collections::HashSet, fs::create_dir_all, path::PathBuf};
 const DRAGONBALL_KERNEL: &str = "vmlinux";
 const DRAGONBALL_ROOT_FS: &str = "rootfs";
 
+#[derive(Debug)]
 pub struct DragonballInner {
     /// sandbox id
     pub(crate) id: String,

src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs

@@ -22,6 +22,7 @@ use tokio::sync::RwLock;
 
 use crate::{DeviceType, Hypervisor, VcpuThreadIds};
 
+#[derive(Debug)]
 pub struct Dragonball {
     inner: Arc<RwLock<DragonballInner>>,
 }

src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs

@@ -36,6 +36,7 @@ const DRAGONBALL_VERSION: &str = env!("CARGO_PKG_VERSION");
 const REQUEST_RETRY: u32 = 500;
 const KVM_DEVICE: &str = "/dev/kvm";
 
+#[derive(Debug)]
 pub struct VmmInstance {
     /// VMM instance info directly accessible from runtime
     vmm_shared_info: Arc<RwLock<InstanceInfo>>,

src/runtime-rs/crates/hypervisor/src/lib.rs

@@ -70,7 +70,7 @@ pub struct VcpuThreadIds {
 }
 
 #[async_trait]
-pub trait Hypervisor: Send + Sync {
+pub trait Hypervisor: std::fmt::Debug + Send + Sync {
     // vm manager
     async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()>;
     async fn start_vm(&self, timeout: i32) -> Result<()>;

src/runtime-rs/crates/hypervisor/src/qemu/inner.rs

@@ -11,7 +11,7 @@ use kata_types::capabilities::{Capabilities, CapabilityBits};
 const VSOCK_SCHEME: &str = "vsock";
 const VSOCK_AGENT_CID: u32 = 3;
 const VSOCK_AGENT_PORT: u32 = 1024;
-
+#[derive(Debug)]
 pub struct QemuInner {
     config: HypervisorConfig,
 }

src/runtime-rs/crates/hypervisor/src/qemu/mod.rs

@@ -18,6 +18,7 @@ use async_trait::async_trait;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 
+#[derive(Debug)]
 pub struct Qemu {
     inner: Arc<RwLock<QemuInner>>,
 }

src/runtime-rs/crates/resource/src/manager_inner.rs

@@ -142,10 +142,11 @@ impl ResourceManagerInner {
         // The solution is to block the future on the current thread, it is enabled by spawn an os thread, create a
         // tokio runtime, and block the task on it.
         let hypervisor = self.hypervisor.clone();
+        let device_manager = self.device_manager.clone();
         let network = thread::spawn(move || -> Result<Arc<dyn Network>> {
             let rt = runtime::Builder::new_current_thread().enable_io().build()?;
             let d = rt
-                .block_on(network::new(&network_config))
+                .block_on(network::new(&network_config, device_manager))
                 .context("new network")?;
             rt.block_on(d.setup(hypervisor.as_ref()))
                 .context("setup network")?;

src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs

@@ -5,12 +5,15 @@
 //
 
 use std::path::Path;
+use std::sync::Arc;
 
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
-use hypervisor::device::DeviceType;
+use hypervisor::device::device_manager::{do_handle_device, DeviceManager};
+use hypervisor::device::DeviceConfig;
 use hypervisor::{device::driver, Hypervisor};
-use hypervisor::{HostDevice, VfioDevice};
+use hypervisor::{get_vfio_device, VfioConfig};
+use tokio::sync::RwLock;
 
 use super::endpoint_persist::{EndpointState, PhysicalEndpointState};
 use super::Endpoint;
@@ -50,10 +53,11 @@ pub struct PhysicalEndpoint {
     bdf: String,
     driver: String,
     vendor_device_id: VendorDevice,
+    d: Arc<RwLock<DeviceManager>>,
 }
 
 impl PhysicalEndpoint {
-    pub fn new(name: &str, hardware_addr: &[u8]) -> Result<Self> {
+    pub fn new(name: &str, hardware_addr: &[u8], d: Arc<RwLock<DeviceManager>>) -> Result<Self> {
         let driver_info = link::get_driver_info(name).context("get driver info")?;
         let bdf = driver_info.bus_info;
         let sys_pci_devices_path = Path::new(SYS_PCI_DEVICES_PATH);
@@ -80,6 +84,7 @@ impl PhysicalEndpoint {
                 .context("new vendor device")?,
             driver,
             bdf,
+            d,
         })
     }
 }
@@ -94,7 +99,7 @@ impl Endpoint for PhysicalEndpoint {
         self.hard_addr.clone()
     }
 
-    async fn attach(&self, hypervisor: &dyn Hypervisor) -> Result<()> {
+    async fn attach(&self, _hypervisor: &dyn Hypervisor) -> Result<()> {
         // bind physical interface from host driver and bind to vfio
         driver::bind_device_to_vfio(
             &self.bdf,
@@ -103,24 +108,19 @@ impl Endpoint for PhysicalEndpoint {
         )
         .with_context(|| format!("bind physical endpoint from {} to vfio", &self.driver))?;
 
-        // set vfio's bus type, pci or mmio. Mostly use pci by default.
+        let vfio_device = get_vfio_device(self.bdf.clone()).context("get vfio device failed.")?;
-        let mode = match self.driver.as_str() {
+        let vfio_dev_config = &mut VfioConfig {
-            "virtio-pci" => "mmio",
+            host_path: vfio_device.clone(),
-            _ => "pci",
+            dev_type: "pci".to_string(),
+            hostdev_prefix: "physical_nic_".to_owned(),
+            ..Default::default()
         };
 
-        // add vfio device
+        // create and insert VFIO device into Kata VM
-        let d = DeviceType::Vfio(VfioDevice {
+        do_handle_device(&self.d, &DeviceConfig::VfioCfg(vfio_dev_config.clone()))
-            attach_count: 0,
+            .await
-            bus_mode: driver::VfioBusMode::new(mode),
+            .context("do handle device failed.")?;
-            devices: vec![HostDevice {
+
-                hostdev_id: format!("physical_nic_{}", self.name().await),
-                bus_slot_func: self.bdf.clone(),
-                ..Default::default()
-            }],
-            ..Default::default()
-        });
-        hypervisor.add_device(d).await.context("add device")?;
         Ok(())
     }
 

src/runtime-rs/crates/resource/src/network/mod.rs

@@ -5,6 +5,8 @@
 //
 
 mod endpoint;
+use std::sync::Arc;
+
 pub use endpoint::endpoint_persist::EndpointState;
 pub use endpoint::Endpoint;
 mod network_entity;
@@ -20,11 +22,11 @@ use network_pair::NetworkPair;
 mod utils;
 pub use utils::netns::{generate_netns_name, NetnsGuard};
 
-use std::sync::Arc;
+use tokio::sync::RwLock;
 
 use anyhow::{Context, Result};
 use async_trait::async_trait;
-use hypervisor::Hypervisor;
+use hypervisor::{device::device_manager::DeviceManager, Hypervisor};
 
 #[derive(Debug)]
 pub enum NetworkConfig {
@@ -41,10 +43,13 @@ pub trait Network: Send + Sync {
     async fn remove(&self, h: &dyn Hypervisor) -> Result<()>;
 }
 
-pub async fn new(config: &NetworkConfig) -> Result<Arc<dyn Network>> {
+pub async fn new(
+    config: &NetworkConfig,
+    d: Arc<RwLock<DeviceManager>>,
+) -> Result<Arc<dyn Network>> {
     match config {
         NetworkConfig::NetworkResourceWithNetNs(c) => Ok(Arc::new(
-            NetworkWithNetns::new(c)
+            NetworkWithNetns::new(c, d)
                 .await
                 .context("new network with netns")?,
         )),

src/runtime-rs/crates/resource/src/network/network_with_netns.rs

@@ -16,7 +16,7 @@ use super::endpoint::endpoint_persist::EndpointState;
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
 use futures::stream::TryStreamExt;
-use hypervisor::Hypervisor;
+use hypervisor::{device::device_manager::DeviceManager, Hypervisor};
 use netns_rs::get_from_path;
 use scopeguard::defer;
 use tokio::sync::RwLock;
@@ -47,13 +47,13 @@ struct NetworkWithNetnsInner {
 }
 
 impl NetworkWithNetnsInner {
-    async fn new(config: &NetworkWithNetNsConfig) -> Result<Self> {
+    async fn new(config: &NetworkWithNetNsConfig, d: Arc<RwLock<DeviceManager>>) -> Result<Self> {
         let entity_list = if config.netns_path.is_empty() {
             warn!(sl!(), "skip to scan for empty netns");
             vec![]
         } else {
             // get endpoint
-            get_entity_from_netns(config)
+            get_entity_from_netns(config, d)
                 .await
                 .context("get entity from netns")?
         };
@@ -70,9 +70,12 @@ pub(crate) struct NetworkWithNetns {
 }
 
 impl NetworkWithNetns {
-    pub(crate) async fn new(config: &NetworkWithNetNsConfig) -> Result<Self> {
+    pub(crate) async fn new(
+        config: &NetworkWithNetNsConfig,
+        d: Arc<RwLock<DeviceManager>>,
+    ) -> Result<Self> {
         Ok(Self {
-            inner: Arc::new(RwLock::new(NetworkWithNetnsInner::new(config).await?)),
+            inner: Arc::new(RwLock::new(NetworkWithNetnsInner::new(config, d).await?)),
         })
     }
 }
@@ -149,10 +152,13 @@ impl Network for NetworkWithNetns {
     }
 }
 
-async fn get_entity_from_netns(config: &NetworkWithNetNsConfig) -> Result<Vec<NetworkEntity>> {
+async fn get_entity_from_netns(
+    config: &NetworkWithNetNsConfig,
+    d: Arc<RwLock<DeviceManager>>,
+) -> Result<Vec<NetworkEntity>> {
     info!(
         sl!(),
-        "get network entity for config {:?} tid {:?}",
+        "get network entity from config {:?} tid {:?}",
         config,
         nix::unistd::gettid()
     );
@@ -178,9 +184,10 @@ async fn get_entity_from_netns(config: &NetworkWithNetNsConfig) -> Result<Vec<Ne
         }
 
         let idx = idx.fetch_add(1, Ordering::Relaxed);
-        let (endpoint, network_info) = create_endpoint(&handle, link.as_ref(), idx, config)
+        let (endpoint, network_info) =
-            .await
+            create_endpoint(&handle, link.as_ref(), idx, config, d.clone())
-            .context("create endpoint")?;
+                .await
+                .context("create endpoint")?;
 
         entity_list.push(NetworkEntity::new(endpoint, network_info));
     }
@@ -193,6 +200,7 @@ async fn create_endpoint(
     link: &dyn link::Link,
     idx: u32,
     config: &NetworkWithNetNsConfig,
+    d: Arc<RwLock<DeviceManager>>,
 ) -> Result<(Arc<dyn Endpoint>, Arc<dyn NetworkInfo>)> {
     let _netns_guard = netns::NetnsGuard::new(&config.netns_path)
         .context("net netns guard")
@@ -206,7 +214,7 @@ async fn create_endpoint(
             &attrs.name,
             nix::unistd::gettid()
         );
-        let t = PhysicalEndpoint::new(&attrs.name, &attrs.hardware_addr)
+        let t = PhysicalEndpoint::new(&attrs.name, &attrs.hardware_addr, d)
             .context("new physical endpoint")?;
         Arc::new(t)
     } else {