diff --git a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs index b490dc94aa..381a324fae 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs @@ -77,6 +77,7 @@ impl SharedInfo { } // Device manager will manage the lifecycle of sandbox device +#[derive(Debug)] pub struct DeviceManager { devices: HashMap, hypervisor: Arc, diff --git a/src/runtime-rs/crates/hypervisor/src/device/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/mod.rs index 4da2218e45..8154920574 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/mod.rs @@ -50,7 +50,7 @@ impl fmt::Display for DeviceType { } #[async_trait] -pub trait Device: Send + Sync { +pub trait Device: std::fmt::Debug + Send + Sync { // attach is to plug device into VM async fn attach(&mut self, h: &dyn hypervisor) -> Result<()>; // detach is to unplug device from VM diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs index 09afc4c1a0..c7c9cb082a 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -28,6 +28,7 @@ use std::{collections::HashSet, fs::create_dir_all, path::PathBuf}; const DRAGONBALL_KERNEL: &str = "vmlinux"; const DRAGONBALL_ROOT_FS: &str = "rootfs"; +#[derive(Debug)] pub struct DragonballInner { /// sandbox id pub(crate) id: String, diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs index 7072949376..23445bf94d 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs @@ -22,6 +22,7 @@ use tokio::sync::RwLock; use crate::{DeviceType, Hypervisor, VcpuThreadIds}; +#[derive(Debug)] pub struct Dragonball { inner: Arc>, } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs index ad3977eca7..1010f9d3f6 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs @@ -36,6 +36,7 @@ const DRAGONBALL_VERSION: &str = env!("CARGO_PKG_VERSION"); const REQUEST_RETRY: u32 = 500; const KVM_DEVICE: &str = "/dev/kvm"; +#[derive(Debug)] pub struct VmmInstance { /// VMM instance info directly accessible from runtime vmm_shared_info: Arc>, diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index 2386d09242..f9ec5109b4 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -70,7 +70,7 @@ pub struct VcpuThreadIds { } #[async_trait] -pub trait Hypervisor: Send + Sync { +pub trait Hypervisor: std::fmt::Debug + Send + Sync { // vm manager async fn prepare_vm(&self, id: &str, netns: Option) -> Result<()>; async fn start_vm(&self, timeout: i32) -> Result<()>; diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index 4b1f7cae38..3bfd3cbc17 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -11,7 +11,7 @@ use kata_types::capabilities::{Capabilities, CapabilityBits}; const VSOCK_SCHEME: &str = "vsock"; const VSOCK_AGENT_CID: u32 = 3; const VSOCK_AGENT_PORT: u32 = 1024; - +#[derive(Debug)] pub struct QemuInner { config: HypervisorConfig, } diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs index eb657dc2db..f8c66d7cf3 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs @@ -18,6 +18,7 @@ use async_trait::async_trait; use std::sync::Arc; use tokio::sync::RwLock; +#[derive(Debug)] pub struct Qemu { inner: Arc>, } diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index 23426ccd06..567af5bac3 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -142,10 +142,11 @@ impl ResourceManagerInner { // The solution is to block the future on the current thread, it is enabled by spawn an os thread, create a // tokio runtime, and block the task on it. let hypervisor = self.hypervisor.clone(); + let device_manager = self.device_manager.clone(); let network = thread::spawn(move || -> Result> { let rt = runtime::Builder::new_current_thread().enable_io().build()?; let d = rt - .block_on(network::new(&network_config)) + .block_on(network::new(&network_config, device_manager)) .context("new network")?; rt.block_on(d.setup(hypervisor.as_ref())) .context("setup network")?; diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs index fd73190b07..9a3a4d4983 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs @@ -5,12 +5,15 @@ // use std::path::Path; +use std::sync::Arc; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; -use hypervisor::device::DeviceType; +use hypervisor::device::device_manager::{do_handle_device, DeviceManager}; +use hypervisor::device::DeviceConfig; use hypervisor::{device::driver, Hypervisor}; -use hypervisor::{HostDevice, VfioDevice}; +use hypervisor::{get_vfio_device, VfioConfig}; +use tokio::sync::RwLock; use super::endpoint_persist::{EndpointState, PhysicalEndpointState}; use super::Endpoint; @@ -50,10 +53,11 @@ pub struct PhysicalEndpoint { bdf: String, driver: String, vendor_device_id: VendorDevice, + d: Arc>, } impl PhysicalEndpoint { - pub fn new(name: &str, hardware_addr: &[u8]) -> Result { + pub fn new(name: &str, hardware_addr: &[u8], d: Arc>) -> Result { let driver_info = link::get_driver_info(name).context("get driver info")?; let bdf = driver_info.bus_info; let sys_pci_devices_path = Path::new(SYS_PCI_DEVICES_PATH); @@ -80,6 +84,7 @@ impl PhysicalEndpoint { .context("new vendor device")?, driver, bdf, + d, }) } } @@ -94,7 +99,7 @@ impl Endpoint for PhysicalEndpoint { self.hard_addr.clone() } - async fn attach(&self, hypervisor: &dyn Hypervisor) -> Result<()> { + async fn attach(&self, _hypervisor: &dyn Hypervisor) -> Result<()> { // bind physical interface from host driver and bind to vfio driver::bind_device_to_vfio( &self.bdf, @@ -103,24 +108,19 @@ impl Endpoint for PhysicalEndpoint { ) .with_context(|| format!("bind physical endpoint from {} to vfio", &self.driver))?; - // set vfio's bus type, pci or mmio. Mostly use pci by default. - let mode = match self.driver.as_str() { - "virtio-pci" => "mmio", - _ => "pci", + let vfio_device = get_vfio_device(self.bdf.clone()).context("get vfio device failed.")?; + let vfio_dev_config = &mut VfioConfig { + host_path: vfio_device.clone(), + dev_type: "pci".to_string(), + hostdev_prefix: "physical_nic_".to_owned(), + ..Default::default() }; - // add vfio device - let d = DeviceType::Vfio(VfioDevice { - attach_count: 0, - bus_mode: driver::VfioBusMode::new(mode), - devices: vec![HostDevice { - hostdev_id: format!("physical_nic_{}", self.name().await), - bus_slot_func: self.bdf.clone(), - ..Default::default() - }], - ..Default::default() - }); - hypervisor.add_device(d).await.context("add device")?; + // create and insert VFIO device into Kata VM + do_handle_device(&self.d, &DeviceConfig::VfioCfg(vfio_dev_config.clone())) + .await + .context("do handle device failed.")?; + Ok(()) } diff --git a/src/runtime-rs/crates/resource/src/network/mod.rs b/src/runtime-rs/crates/resource/src/network/mod.rs index 0fe3aa2940..c922e817f7 100644 --- a/src/runtime-rs/crates/resource/src/network/mod.rs +++ b/src/runtime-rs/crates/resource/src/network/mod.rs @@ -5,6 +5,8 @@ // mod endpoint; +use std::sync::Arc; + pub use endpoint::endpoint_persist::EndpointState; pub use endpoint::Endpoint; mod network_entity; @@ -20,11 +22,11 @@ use network_pair::NetworkPair; mod utils; pub use utils::netns::{generate_netns_name, NetnsGuard}; -use std::sync::Arc; +use tokio::sync::RwLock; use anyhow::{Context, Result}; use async_trait::async_trait; -use hypervisor::Hypervisor; +use hypervisor::{device::device_manager::DeviceManager, Hypervisor}; #[derive(Debug)] pub enum NetworkConfig { @@ -41,10 +43,13 @@ pub trait Network: Send + Sync { async fn remove(&self, h: &dyn Hypervisor) -> Result<()>; } -pub async fn new(config: &NetworkConfig) -> Result> { +pub async fn new( + config: &NetworkConfig, + d: Arc>, +) -> Result> { match config { NetworkConfig::NetworkResourceWithNetNs(c) => Ok(Arc::new( - NetworkWithNetns::new(c) + NetworkWithNetns::new(c, d) .await .context("new network with netns")?, )), diff --git a/src/runtime-rs/crates/resource/src/network/network_with_netns.rs b/src/runtime-rs/crates/resource/src/network/network_with_netns.rs index bb5273ffcb..16faa88b3e 100644 --- a/src/runtime-rs/crates/resource/src/network/network_with_netns.rs +++ b/src/runtime-rs/crates/resource/src/network/network_with_netns.rs @@ -16,7 +16,7 @@ use super::endpoint::endpoint_persist::EndpointState; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use futures::stream::TryStreamExt; -use hypervisor::Hypervisor; +use hypervisor::{device::device_manager::DeviceManager, Hypervisor}; use netns_rs::get_from_path; use scopeguard::defer; use tokio::sync::RwLock; @@ -47,13 +47,13 @@ struct NetworkWithNetnsInner { } impl NetworkWithNetnsInner { - async fn new(config: &NetworkWithNetNsConfig) -> Result { + async fn new(config: &NetworkWithNetNsConfig, d: Arc>) -> Result { let entity_list = if config.netns_path.is_empty() { warn!(sl!(), "skip to scan for empty netns"); vec![] } else { // get endpoint - get_entity_from_netns(config) + get_entity_from_netns(config, d) .await .context("get entity from netns")? }; @@ -70,9 +70,12 @@ pub(crate) struct NetworkWithNetns { } impl NetworkWithNetns { - pub(crate) async fn new(config: &NetworkWithNetNsConfig) -> Result { + pub(crate) async fn new( + config: &NetworkWithNetNsConfig, + d: Arc>, + ) -> Result { Ok(Self { - inner: Arc::new(RwLock::new(NetworkWithNetnsInner::new(config).await?)), + inner: Arc::new(RwLock::new(NetworkWithNetnsInner::new(config, d).await?)), }) } } @@ -149,10 +152,13 @@ impl Network for NetworkWithNetns { } } -async fn get_entity_from_netns(config: &NetworkWithNetNsConfig) -> Result> { +async fn get_entity_from_netns( + config: &NetworkWithNetNsConfig, + d: Arc>, +) -> Result> { info!( sl!(), - "get network entity for config {:?} tid {:?}", + "get network entity from config {:?} tid {:?}", config, nix::unistd::gettid() ); @@ -178,9 +184,10 @@ async fn get_entity_from_netns(config: &NetworkWithNetNsConfig) -> Result>, ) -> Result<(Arc, Arc)> { let _netns_guard = netns::NetnsGuard::new(&config.netns_path) .context("net netns guard") @@ -206,7 +214,7 @@ async fn create_endpoint( &attrs.name, nix::unistd::gettid() ); - let t = PhysicalEndpoint::new(&attrs.name, &attrs.hardware_addr) + let t = PhysicalEndpoint::new(&attrs.name, &attrs.hardware_addr, d) .context("new physical endpoint")?; Arc::new(t) } else {