runtime-rs: device manager

Support device manager for runtime-rs, add block device handler for
device manager

Fixes:#5375
Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Signed-off-by: alex.lyn <alex.lyn@antgroup.com>

src/runtime-rs/Cargo.lock

@@ -61,6 +61,17 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "ahash"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
+dependencies = [
+ "getrandom 0.2.8",
+ "once_cell",
+ "version_check",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "0.7.20"
@@ -816,6 +827,12 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "dlv-list"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257"
+
 [[package]]
 name = "dragonball"
 version = "0.1.0"
@@ -1176,6 +1193,9 @@ name = "hashbrown"
 version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+dependencies = [
+ "ahash",
+]
 
 [[package]]
 name = "heck"
@@ -1281,6 +1301,7 @@ dependencies = [
 name = "hypervisor"
 version = "0.1.0"
 dependencies = [
+ "actix-rt",
  "anyhow",
  "async-trait",
  "ch-config",
@@ -1296,6 +1317,7 @@ dependencies = [
  "nix 0.24.3",
  "persist",
  "rand 0.8.5",
+ "rust-ini",
  "safe-path 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "seccompiler",
  "serde",
@@ -1979,6 +2001,16 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
+[[package]]
+name = "ordered-multimap"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a"
+dependencies = [
+ "dlv-list",
+ "hashbrown",
+]
+
 [[package]]
 name = "parking"
 version = "2.0.0"
@@ -2510,6 +2542,16 @@ dependencies = [
  "wasm_container",
 ]
 
+[[package]]
+name = "rust-ini"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df"
+dependencies = [
+ "cfg-if 1.0.0",
+ "ordered-multimap",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.21"

src/runtime-rs/crates/hypervisor/Cargo.toml

@@ -8,6 +8,7 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+actix-rt = "2.7.0"
 anyhow = "^1.0"
 async-trait = "0.1.48"
 dbs-utils = "0.2.0"
@@ -15,6 +16,7 @@ go-flag = "0.1.0"
 libc = ">=0.2.39"
 nix = "0.24.2"
 persist = { path = "../persist" }
+rust-ini = "0.18.0"
 seccompiler = "0.2.0"
 serde = { version = "1.0.138", features = ["derive"] }
 serde_json = ">=1.0.9"

src/runtime-rs/crates/hypervisor/src/ch/inner.rs

@@ -4,7 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use super::HypervisorState;
-use crate::device::Device;
+use crate::driver::DeviceConfig;
 use crate::VmmState;
 use anyhow::Result;
 use async_trait::async_trait;
@@ -44,7 +44,7 @@ pub struct CloudHypervisorInner {
     pub(crate) jailer_root: String,
 
     /// List of devices that will be added to the VM once it boots
-    pub(crate) pending_devices: Option<Vec<Device>>,
+    pub(crate) pending_devices: Option<Vec<DeviceConfig>>,
 
     pub(crate) _capabilities: Capabilities,
 

src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs

@@ -5,7 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use super::inner::CloudHypervisorInner;
-use crate::device::{Device, ShareFsDeviceConfig};
+use crate::driver::{DeviceConfig, ShareFsDeviceConfig};
 use crate::HybridVsockConfig;
 use crate::VmmState;
 use anyhow::{anyhow, Context, Result};
@@ -18,9 +18,10 @@ use std::path::PathBuf;
 const VIRTIO_FS: &str = "virtio-fs";
 
 impl CloudHypervisorInner {
-    pub(crate) async fn add_device(&mut self, device: Device) -> Result<()> {
+    pub(crate) async fn add_device(&mut self, device: DeviceConfig) -> Result<()> {
         if self.state != VmmState::VmRunning {
-            let mut devices: Vec<Device> = if let Some(devices) = self.pending_devices.take() {
+            let mut devices: Vec<DeviceConfig> = if let Some(devices) = self.pending_devices.take()
+            {
                 devices
             } else {
                 vec![]
@@ -38,10 +39,10 @@ impl CloudHypervisorInner {
         Ok(())
     }
 
-    async fn handle_add_device(&mut self, device: Device) -> Result<()> {
+    async fn handle_add_device(&mut self, device: DeviceConfig) -> Result<()> {
         match device {
-            Device::ShareFsDevice(cfg) => self.handle_share_fs_device(cfg).await,
+            DeviceConfig::ShareFsDevice(cfg) => self.handle_share_fs_device(cfg).await,
-            Device::HybridVsock(cfg) => self.handle_hvsock_device(&cfg).await,
+            DeviceConfig::HybridVsock(cfg) => self.handle_hvsock_device(&cfg).await,
             _ => Err(anyhow!("unhandled device: {:?}", device)),
         }
     }
@@ -66,7 +67,7 @@ impl CloudHypervisorInner {
         Ok(())
     }
 
-    pub(crate) async fn remove_device(&mut self, _device: Device) -> Result<()> {
+    pub(crate) async fn remove_device(&mut self, _device: DeviceConfig) -> Result<()> {
         Ok(())
     }
 
@@ -132,7 +133,7 @@ impl CloudHypervisorInner {
         if let Some(devices) = pending_root_devices {
             for dev in devices {
                 match dev {
-                    Device::ShareFsDevice(dev) => {
+                    DeviceConfig::ShareFsDevice(dev) => {
                         let settings = ShareFsSettings::new(dev, self.vm_path.clone());
 
                         let fs_cfg = FsConfig::try_from(settings)?;

src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs

@@ -7,7 +7,7 @@ use super::inner::CloudHypervisorInner;
 use crate::ch::utils::get_api_socket_path;
 use crate::ch::utils::{get_jailer_root, get_sandbox_path, get_vsock_path};
 use crate::kernel_param::KernelParams;
-use crate::Device;
+use crate::DeviceConfig;
 use crate::VsockConfig;
 use crate::VM_ROOTFS_DRIVER_PMEM;
 use crate::{VcpuThreadIds, VmmState};
@@ -419,7 +419,7 @@ impl CloudHypervisorInner {
 
         let vsock_cfg = VsockConfig::new(self.id.clone()).await?;
 
-        let dev = Device::Vsock(vsock_cfg);
+        let dev = DeviceConfig::Vsock(vsock_cfg);
         self.add_device(dev).await.context("add vsock device")?;
 
         self.start_hypervisor(self.timeout_secs).await?;

src/runtime-rs/crates/hypervisor/src/ch/mod.rs

@@ -4,7 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use super::HypervisorState;
-use crate::{device::Device, Hypervisor, VcpuThreadIds};
+use crate::{driver::DeviceConfig, Hypervisor, VcpuThreadIds};
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 use kata_types::capabilities::Capabilities;
@@ -78,12 +78,12 @@ impl Hypervisor for CloudHypervisor {
         inner.save_vm().await
     }
 
-    async fn add_device(&self, device: Device) -> Result<()> {
+    async fn add_device(&self, device: DeviceConfig) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.add_device(device).await
     }
 
-    async fn remove_device(&self, device: Device) -> Result<()> {
+    async fn remove_device(&self, device: DeviceConfig) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.remove_device(device).await
     }

src/runtime-rs/crates/hypervisor/src/device/block.rs

@@ -1,24 +0,0 @@
-// Copyright (c) 2019-2022 Alibaba Cloud
-// Copyright (c) 2019-2022 Ant Group
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-#[derive(Debug)]
-pub struct BlockConfig {
-    /// Unique identifier of the drive.
-    pub id: String,
-
-    /// Path of the drive.
-    pub path_on_host: String,
-
-    /// If set to true, the drive is opened in read-only mode. Otherwise, the
-    /// drive is opened as read-write.
-    pub is_readonly: bool,
-
-    /// Don't close `path_on_host` file when dropping the device.
-    pub no_drop: bool,
-
-    /// device index
-    pub index: u64,
-}

src/runtime-rs/crates/hypervisor/src/device/device_manager.rs

@@ -0,0 +1,199 @@
+// Copyright (c) 2019-2023 Alibaba Cloud
+// Copyright (c) 2019-2023 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::{collections::HashMap, sync::Arc};
+
+use anyhow::{anyhow, Context, Ok, Result};
+use kata_sys_util::rand::RandomBytes;
+use tokio::sync::Mutex;
+
+use crate::{
+    BlockConfig, DeviceConfig, Hypervisor, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE,
+    VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI,
+};
+
+use super::{
+    util::{get_host_path, get_virt_drive_name},
+    Device,
+};
+pub type ArcMutexBoxDevice = Arc<Mutex<dyn Device>>;
+
+/// block_index and released_block_index are used to search an available block index
+/// in Sandbox.
+///
+/// @block_index generally default is 1 for <vdb>;
+/// @released_block_index for blk devices removed and indexes will released at the same time.
+#[derive(Clone, Debug, Default)]
+struct SharedInfo {
+    block_index: u64,
+    released_block_index: Vec<u64>,
+}
+
+impl SharedInfo {
+    fn new() -> Self {
+        SharedInfo {
+            block_index: 1,
+            released_block_index: vec![],
+        }
+    }
+
+    // declare the available block index
+    fn declare_device_index(&mut self) -> Result<u64> {
+        let current_index = if let Some(index) = self.released_block_index.pop() {
+            index
+        } else {
+            self.block_index
+        };
+        self.block_index += 1;
+
+        Ok(current_index)
+    }
+
+    fn release_device_index(&mut self, index: u64) {
+        self.released_block_index.push(index);
+        self.released_block_index.sort_by(|a, b| b.cmp(a));
+    }
+}
+
+// Device manager will manage the lifecycle of sandbox device
+pub struct DeviceManager {
+    devices: HashMap<String, ArcMutexBoxDevice>,
+    hypervisor: Arc<dyn Hypervisor>,
+    shared_info: SharedInfo,
+}
+
+impl DeviceManager {
+    pub async fn new(hypervisor: Arc<dyn Hypervisor>) -> Result<Self> {
+        let devices = HashMap::<String, ArcMutexBoxDevice>::new();
+        Ok(DeviceManager {
+            devices,
+            hypervisor,
+            shared_info: SharedInfo::new(),
+        })
+    }
+
+    pub async fn new_device(&mut self, device_config: DeviceConfig) -> Result<String> {
+        let device_id = if let Some(dev) = self.find_device(&device_config).await {
+            dev
+        } else {
+            self.create_device(&device_config)
+                .await
+                .context("failed to create device")?
+        };
+        Ok(device_id)
+    }
+
+    pub async fn try_add_device(&mut self, device_id: String) -> Result<()> {
+        // get device
+        let device = self
+            .devices
+            .get(&device_id)
+            .context("failed to find device")?;
+        // attach device
+        let result = device.lock().await.attach(self.hypervisor.as_ref()).await;
+        // handle attach error
+        if let Err(e) = result {
+            if let DeviceConfig::Block(config) = device.lock().await.get_device_info().await {
+                self.shared_info.release_device_index(config.index);
+            };
+            self.devices.remove(&device_id);
+            return Err(e);
+        }
+        Ok(())
+    }
+
+    async fn find_device(&self, device_config: &DeviceConfig) -> Option<String> {
+        for (device_id, dev) in &self.devices {
+            match dev.lock().await.get_device_info().await {
+                DeviceConfig::Block(config) => match device_config {
+                    DeviceConfig::Block(ref config_new) => {
+                        if config_new.path_on_host == config.path_on_host {
+                            return Some(device_id.to_string());
+                        }
+                    }
+                    _ => {
+                        continue;
+                    }
+                },
+                _ => {
+                    // TODO: support find other device type
+                    continue;
+                }
+            }
+        }
+        None
+    }
+
+    async fn create_device(&mut self, device_config: &DeviceConfig) -> Result<String> {
+        // device ID must be generated by manager instead of device itself
+        // in case of ID collision
+        let device_id = self.new_device_id()?;
+        let dev: ArcMutexBoxDevice = match device_config {
+            DeviceConfig::Block(config) => self
+                .create_block_device(config, device_id.clone())
+                .await
+                .context("failed to create device")?,
+            _ => {
+                return Err(anyhow!("invliad device type"));
+            }
+        };
+        // register device to devices
+        self.devices.insert(device_id.clone(), dev.clone());
+        Ok(device_id)
+    }
+
+    async fn create_block_device(
+        &mut self,
+        config: &BlockConfig,
+        device_id: String,
+    ) -> Result<ArcMutexBoxDevice> {
+        let mut block_config = config.clone();
+        block_config.id = device_id.clone();
+        // get hypervisor block driver
+        let block_driver = match self
+            .hypervisor
+            .hypervisor_config()
+            .await
+            .blockdev_info
+            .block_device_driver
+            .as_str()
+        {
+            // convert the block driver to kata type
+            VIRTIO_BLOCK_MMIO => KATA_MMIO_BLK_DEV_TYPE.to_string(),
+            VIRTIO_BLOCK_PCI => KATA_BLK_DEV_TYPE.to_string(),
+            _ => "".to_string(),
+        };
+        block_config.driver_option = block_driver;
+        // generate virt path
+        let current_index = self.shared_info.declare_device_index()?;
+        block_config.index = current_index;
+        let drive_name = get_virt_drive_name(current_index as i32)?;
+        block_config.virt_path = format!("/dev/{}", drive_name);
+        // if the path on host is empty, we need to get device host path from major and minor
+        // Otherwise, it might be rawfile based block device
+        if block_config.path_on_host.is_empty() {
+            block_config.path_on_host = get_host_path("b".to_owned(), config.major, config.minor)
+                .context("failed to get host path")?;
+        }
+        Ok(Arc::new(Mutex::new(BlockConfig::new(block_config))))
+    }
+
+    // device ID must be generated by device manager instead of device itself
+    // in case of ID collision
+    fn new_device_id(&self) -> Result<String> {
+        for _ in 0..5 {
+            let rand_bytes = RandomBytes::new(8);
+            let id = format!("{:x}", rand_bytes);
+
+            // check collision in devices
+            if self.devices.get(&id).is_none() {
+                return Ok(id);
+            }
+        }
+
+        Err(anyhow!("ID are exhausted"))
+    }
+}

src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs

@@ -0,0 +1,36 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+mod virtio_blk;
+pub use virtio_blk::{
+    BlockConfig, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI,
+};
+mod virtio_net;
+pub use virtio_net::{Address, NetworkConfig};
+mod vfio;
+pub use vfio::{bind_device_to_host, bind_device_to_vfio, VfioBusMode, VfioConfig};
+mod virtio_fs;
+pub use virtio_fs::{ShareFsDeviceConfig, ShareFsMountConfig, ShareFsMountType, ShareFsOperation};
+mod virtio_vsock;
+use std::fmt;
+pub use virtio_vsock::{HybridVsockConfig, VsockConfig};
+
+#[derive(Debug)]
+pub enum DeviceConfig {
+    Block(BlockConfig),
+    Network(NetworkConfig),
+    ShareFsDevice(ShareFsDeviceConfig),
+    Vfio(VfioConfig),
+    ShareFsMount(ShareFsMountConfig),
+    Vsock(VsockConfig),
+    HybridVsock(HybridVsockConfig),
+}
+
+impl fmt::Display for DeviceConfig {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}

src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs

@@ -25,7 +25,7 @@ const VFIO_UNBIND_PATH: &str = "/sys/bus/pci/drivers/vfio-pci/unbind";
 
 pub const VFIO_PCI: &str = "vfio-pci";
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub enum VfioBusMode {
     PCI,
     MMIO,
@@ -40,7 +40,7 @@ impl VfioBusMode {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct VfioConfig {
     /// Unique identifier of the device
     pub id: String,

src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs

@@ -0,0 +1,124 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+pub const VIRTIO_BLOCK_MMIO: &str = "virtio-blk-mmio";
+use crate::Hypervisor as hypervisor;
+use crate::{device::Device, DeviceConfig};
+use anyhow::{anyhow, Context, Result};
+use async_trait::async_trait;
+/// VIRTIO_BLOCK_PCI indicates block driver is virtio-pci based
+pub const VIRTIO_BLOCK_PCI: &str = "virtio-blk-pci";
+pub const KATA_MMIO_BLK_DEV_TYPE: &str = "mmioblk";
+pub const KATA_BLK_DEV_TYPE: &str = "blk";
+
+#[derive(Debug, Clone, Default)]
+pub struct BlockConfig {
+    /// Unique identifier of the drive.
+    pub id: String,
+
+    /// Path of the drive.
+    pub path_on_host: String,
+
+    /// If set to true, the drive is opened in read-only mode. Otherwise, the
+    /// drive is opened as read-write.
+    pub is_readonly: bool,
+
+    /// Don't close `path_on_host` file when dropping the device.
+    pub no_drop: bool,
+
+    /// device index
+    pub index: u64,
+
+    /// driver type for block device
+    pub driver_option: String,
+
+    /// device path in guest
+    pub virt_path: String,
+
+    /// device attach count
+    pub attach_count: u64,
+
+    /// device major number
+    pub major: i64,
+
+    /// device minor number
+    pub minor: i64,
+}
+
+impl BlockConfig {
+    // new creates a new VirtioBlkDevice
+    pub fn new(dev_info: BlockConfig) -> Self {
+        dev_info
+    }
+}
+
+#[async_trait]
+impl Device for BlockConfig {
+    async fn attach(&mut self, h: &dyn hypervisor) -> Result<()> {
+        // increase attach count, skip attach the device if the device is already attached
+        if self
+            .increase_attach_count()
+            .await
+            .context("failed to increase attach count")?
+        {
+            return Ok(());
+        }
+        if let Err(e) = h.add_device(DeviceConfig::Block(self.clone())).await {
+            self.decrease_attach_count().await?;
+            return Err(e);
+        }
+        return Ok(());
+    }
+
+    async fn detach(&mut self, h: &dyn hypervisor) -> Result<Option<u64>> {
+        if self
+            .decrease_attach_count()
+            .await
+            .context("failed to decrease attach count")?
+        {
+            return Ok(None);
+        }
+        if let Err(e) = h.remove_device(DeviceConfig::Block(self.clone())).await {
+            self.increase_attach_count().await?;
+            return Err(e);
+        }
+        Ok(Some(self.index))
+    }
+
+    async fn get_device_info(&self) -> DeviceConfig {
+        DeviceConfig::Block(self.clone())
+    }
+
+    async fn increase_attach_count(&mut self) -> Result<bool> {
+        match self.attach_count {
+            0 => {
+                // do real attach
+                self.attach_count += 1;
+                Ok(false)
+            }
+            std::u64::MAX => Err(anyhow!("device was attached too many times")),
+            _ => {
+                self.attach_count += 1;
+                Ok(true)
+            }
+        }
+    }
+
+    async fn decrease_attach_count(&mut self) -> Result<bool> {
+        match self.attach_count {
+            0 => Err(anyhow!("detaching a device that wasn't attached")),
+            1 => {
+                // do real wrok
+                self.attach_count -= 1;
+                Ok(false)
+            }
+            _ => {
+                self.attach_count -= 1;
+                Ok(true)
+            }
+        }
+    }
+}

src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs

@@ -11,14 +11,14 @@ pub enum ShareFsOperation {
     Update,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub enum ShareFsMountType {
     PASSTHROUGH,
     RAFS,
 }
 
 /// ShareFsMountConfig: share fs mount config
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct ShareFsMountConfig {
     /// source: the passthrough fs exported dir or rafs meta file of rafs
     pub source: String,
@@ -41,3 +41,25 @@ pub struct ShareFsMountConfig {
     /// prefetch_list_path: path to file that contains file lists that should be prefetched by rafs
     pub prefetch_list_path: Option<String>,
 }
+
+/// ShareFsDeviceConfig: share fs device config
+#[derive(Debug, Clone)]
+pub struct ShareFsDeviceConfig {
+    /// fs_type: virtiofs or inline-virtiofs
+    pub fs_type: String,
+
+    /// socket_path: socket path for virtiofs
+    pub sock_path: String,
+
+    /// mount_tag: a label used as a hint to the guest.
+    pub mount_tag: String,
+
+    /// host_path: the host filesystem path for this volume.
+    pub host_path: String,
+
+    /// queue_size: queue size
+    pub queue_size: u64,
+
+    /// queue_num: queue number
+    pub queue_num: u64,
+}

src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs

@@ -6,6 +6,7 @@
 
 use std::fmt;
 
+#[derive(Clone)]
 pub struct Address(pub [u8; 6]);
 
 impl fmt::Debug for Address {
@@ -19,7 +20,7 @@ impl fmt::Debug for Address {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct NetworkConfig {
     /// Unique identifier of the device
     pub id: String,

src/runtime-rs/crates/hypervisor/src/device/mod.rs

@@ -1,37 +1,33 @@
-// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2023 Alibaba Cloud
-// Copyright (c) 2019-2022 Ant Group
+// Copyright (c) 2019-2023 Ant Group
 //
 // SPDX-License-Identifier: Apache-2.0
 //
 
-mod block;
+use crate::{DeviceConfig, Hypervisor as hypervisor};
-pub use block::BlockConfig;
+use anyhow::Result;
-mod network;
+use async_trait::async_trait;
-pub use network::{Address, NetworkConfig};
-mod share_fs_device;
-pub use share_fs_device::ShareFsDeviceConfig;
-mod vfio;
-pub use vfio::{bind_device_to_host, bind_device_to_vfio, VfioBusMode, VfioConfig};
-mod share_fs_mount;
-pub use share_fs_mount::{ShareFsMountConfig, ShareFsMountType, ShareFsOperation};
-mod vsock;
-pub use vsock::{HybridVsockConfig, VsockConfig};
 
-use std::fmt;
+pub mod device_manager;
+pub mod driver;
+pub mod util;
 
-#[derive(Debug)]
+#[async_trait]
-pub enum Device {
+pub trait Device: Send + Sync {
-    Block(BlockConfig),
+    // attach is to plug device into VM
-    Network(NetworkConfig),
+    async fn attach(&mut self, h: &dyn hypervisor) -> Result<()>;
-    ShareFsDevice(ShareFsDeviceConfig),
+    // detach is to unplug device from VM
-    Vfio(VfioConfig),
+    async fn detach(&mut self, h: &dyn hypervisor) -> Result<Option<u64>>;
-    ShareFsMount(ShareFsMountConfig),
+    // get_device_info returns device config
-    Vsock(VsockConfig),
+    async fn get_device_info(&self) -> DeviceConfig;
-    HybridVsock(HybridVsockConfig),
+    // increase_attach_count is used to increase the attach count for a device
-}
+    // return values:
-
+    // * true: no need to do real attach when current attach count is zero, skip following actions.
-impl fmt::Display for Device {
+    // * err error: error while do increase attach count
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    async fn increase_attach_count(&mut self) -> Result<bool>;
-        write!(f, "{:?}", self)
+    // decrease_attach_count is used to decrease the attach count for a device
-    }
+    // return values:
+    // * false: no need to do real dettach when current attach count is not zero, skip following actions.
+    // * err error: error while do decrease attach count
+    async fn decrease_attach_count(&mut self) -> Result<bool>;
 }

src/runtime-rs/crates/hypervisor/src/device/share_fs_device.rs

@@ -1,27 +0,0 @@
-// Copyright (c) 2019-2022 Alibaba Cloud
-// Copyright (c) 2019-2022 Ant Group
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-/// ShareFsDeviceConfig: share fs device config
-#[derive(Debug)]
-pub struct ShareFsDeviceConfig {
-    /// fs_type: virtiofs or inline-virtiofs
-    pub fs_type: String,
-
-    /// socket_path: socket path for virtiofs
-    pub sock_path: String,
-
-    /// mount_tag: a label used as a hint to the guest.
-    pub mount_tag: String,
-
-    /// host_path: the host filesystem path for this volume.
-    pub host_path: String,
-
-    /// queue_size: queue size
-    pub queue_size: u64,
-
-    /// queue_num: queue number
-    pub queue_num: u64,
-}

src/runtime-rs/crates/hypervisor/src/device/util.rs

@@ -0,0 +1,88 @@
+// Copyright (c) 2019-2023 Alibaba Cloud
+// Copyright (c) 2019-2023 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::{anyhow, Result};
+use ini::Ini;
+
+const SYS_DEV_PREFIX: &str = "/sys/dev";
+
+// get_host_path is used to fetch the host path for the device.
+// The path passed in the spec refers to the path that should appear inside the container.
+// We need to find the actual device path on the host based on the major-minor numbers of the device.
+pub fn get_host_path(dev_type: String, major: i64, minor: i64) -> Result<String> {
+    let path_comp = match dev_type.as_str() {
+        "c" | "u" => "char",
+        "b" => "block",
+        // for device type p will return an empty string
+        _ => return Ok(String::new()),
+    };
+    let format = format!("{}:{}", major, minor);
+    let sys_dev_path = std::path::Path::new(SYS_DEV_PREFIX)
+        .join(path_comp)
+        .join(format)
+        .join("uevent");
+    std::fs::metadata(&sys_dev_path)?;
+    let conf = Ini::load_from_file(&sys_dev_path)?;
+    let dev_name = conf
+        .section::<String>(None)
+        .ok_or_else(|| anyhow!("has no section"))?
+        .get("DEVNAME")
+        .ok_or_else(|| anyhow!("has no DEVNAME"))?;
+    Ok(format!("/dev/{}", dev_name))
+}
+
+// get_virt_drive_name returns the disk name format for virtio-blk
+// Reference: https://github.com/torvalds/linux/blob/master/drivers/block/virtio_blk.c @c0aa3e0916d7e531e69b02e426f7162dfb1c6c0
+pub(crate) fn get_virt_drive_name(mut index: i32) -> Result<String> {
+    if index < 0 {
+        return Err(anyhow!("Index cannot be negative"));
+    }
+
+    // Prefix used for virtio-block devices
+    const PREFIX: &str = "vd";
+
+    // Refer to DISK_NAME_LEN: https://github.com/torvalds/linux/blob/08c521a2011ff492490aa9ed6cc574be4235ce2b/include/linux/genhd.h#L61
+    let disk_name_len = 32usize;
+    let base = 26i32;
+
+    let suff_len = disk_name_len - PREFIX.len();
+    let mut disk_letters = vec![0u8; suff_len];
+
+    let mut i = 0usize;
+    while i < suff_len && index >= 0 {
+        let letter: u8 = b'a' + (index % base) as u8;
+        disk_letters[i] = letter;
+        index = (index / base) - 1;
+        i += 1;
+    }
+    if index >= 0 {
+        return Err(anyhow!("Index not supported"));
+    }
+    disk_letters.truncate(i);
+    disk_letters.reverse();
+    Ok(String::from(PREFIX) + std::str::from_utf8(&disk_letters)?)
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::device::util::get_virt_drive_name;
+
+    #[actix_rt::test]
+    async fn test_get_virt_drive_name() {
+        for &(input, output) in [
+            (0i32, "vda"),
+            (25, "vdz"),
+            (27, "vdab"),
+            (704, "vdaac"),
+            (18277, "vdzzz"),
+        ]
+        .iter()
+        {
+            let out = get_virt_drive_name(input).unwrap();
+            assert_eq!(&out, output);
+        }
+    }
+}

src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs

@@ -6,8 +6,8 @@
 
 use super::vmm_instance::VmmInstance;
 use crate::{
-    device::Device, hypervisor_persist::HypervisorState, kernel_param::KernelParams, VmmState,
+    driver::DeviceConfig, hypervisor_persist::HypervisorState, kernel_param::KernelParams,
-    DEV_HUGEPAGES, HUGETLBFS, HYPERVISOR_DRAGONBALL, SHMEM, VM_ROOTFS_DRIVER_BLK,
+    VmmState, DEV_HUGEPAGES, HUGETLBFS, HYPERVISOR_DRAGONBALL, SHMEM, VM_ROOTFS_DRIVER_BLK,
 };
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
@@ -56,7 +56,7 @@ pub struct DragonballInner {
     pub(crate) run_dir: String,
 
     /// pending device
-    pub(crate) pending_devices: Vec<Device>,
+    pub(crate) pending_devices: Vec<DeviceConfig>,
 
     /// cached block device
     pub(crate) cached_block_devices: HashSet<String>,

src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs

@@ -15,8 +15,8 @@ use dragonball::api::v1::{
 
 use super::DragonballInner;
 use crate::{
-    device::Device, HybridVsockConfig, NetworkConfig, ShareFsDeviceConfig, ShareFsMountConfig,
+    driver::DeviceConfig, HybridVsockConfig, NetworkConfig, ShareFsDeviceConfig,
-    ShareFsMountType, ShareFsOperation, VmmState,
+    ShareFsMountConfig, ShareFsMountType, ShareFsOperation, VmmState,
 };
 
 const MB_TO_B: u32 = 1024 * 1024;
@@ -31,7 +31,7 @@ pub(crate) fn drive_index_to_id(index: u64) -> String {
 }
 
 impl DragonballInner {
-    pub(crate) async fn add_device(&mut self, device: Device) -> Result<()> {
+    pub(crate) async fn add_device(&mut self, device: DeviceConfig) -> Result<()> {
         if self.state == VmmState::NotReady {
             info!(sl!(), "VMM not ready, queueing device {}", device);
 
@@ -44,11 +44,11 @@ impl DragonballInner {
 
         info!(sl!(), "dragonball add device {:?}", &device);
         match device {
-            Device::Network(config) => self.add_net_device(&config).context("add net device"),
+            DeviceConfig::Network(config) => self.add_net_device(&config).context("add net device"),
-            Device::Vfio(_config) => {
+            DeviceConfig::Vfio(_config) => {
                 todo!()
             }
-            Device::Block(config) => self
+            DeviceConfig::Block(config) => self
                 .add_block_device(
                     config.path_on_host.as_str(),
                     config.id.as_str(),
@@ -56,29 +56,29 @@ impl DragonballInner {
                     config.no_drop,
                 )
                 .context("add block device"),
-            Device::HybridVsock(config) => self.add_hvsock(&config).context("add vsock"),
+            DeviceConfig::HybridVsock(config) => self.add_hvsock(&config).context("add vsock"),
-            Device::ShareFsDevice(config) => self
+            DeviceConfig::ShareFsDevice(config) => self
                 .add_share_fs_device(&config)
                 .context("add share fs device"),
-            Device::ShareFsMount(config) => self
+            DeviceConfig::ShareFsMount(config) => self
                 .add_share_fs_mount(&config)
                 .context("add share fs mount"),
-            Device::Vsock(_) => {
+            DeviceConfig::Vsock(_) => {
                 todo!()
             }
         }
     }
 
-    pub(crate) async fn remove_device(&mut self, device: Device) -> Result<()> {
+    pub(crate) async fn remove_device(&mut self, device: DeviceConfig) -> Result<()> {
         info!(sl!(), "remove device {} ", device);
 
         match device {
-            Device::Block(config) => {
+            DeviceConfig::Block(config) => {
                 let drive_id = drive_index_to_id(config.index);
                 self.remove_block_drive(drive_id.as_str())
                     .context("remove block drive")
             }
-            Device::Vfio(_config) => {
+            DeviceConfig::Vfio(_config) => {
                 todo!()
             }
             _ => Err(anyhow!("unsupported device {:?}", device)),

src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs

@@ -32,7 +32,7 @@ impl DragonballInner {
 
         // prepare vsock
         let uds_path = [&self.jailer_root, DEFAULT_HYBRID_VSOCK_NAME].join("/");
-        let d = crate::device::Device::HybridVsock(crate::device::HybridVsockConfig {
+        let d = crate::driver::DeviceConfig::HybridVsock(crate::driver::HybridVsockConfig {
             id: format!("vsock-{}", &self.id),
             guest_cid: 3,
             uds_path,

src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs

@@ -20,7 +20,7 @@ use kata_types::capabilities::Capabilities;
 use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
 use tokio::sync::RwLock;
 
-use crate::{device::Device, Hypervisor, VcpuThreadIds};
+use crate::{driver::DeviceConfig, Hypervisor, VcpuThreadIds};
 
 pub struct Dragonball {
     inner: Arc<RwLock<DragonballInner>>,
@@ -77,12 +77,12 @@ impl Hypervisor for Dragonball {
         inner.save_vm().await
     }
 
-    async fn add_device(&self, device: Device) -> Result<()> {
+    async fn add_device(&self, device: DeviceConfig) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.add_device(device).await
     }
 
-    async fn remove_device(&self, device: Device) -> Result<()> {
+    async fn remove_device(&self, device: DeviceConfig) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.remove_device(device).await
     }

src/runtime-rs/crates/hypervisor/src/lib.rs

@@ -11,7 +11,8 @@ logging::logger_with_subsystem!(sl, "hypervisor");
 
 pub mod device;
 pub mod hypervisor_persist;
-pub use device::*;
+use device::driver;
+pub use device::driver::*;
 pub mod dragonball;
 mod kernel_param;
 pub mod qemu;
@@ -78,8 +79,8 @@ pub trait Hypervisor: Send + Sync {
     async fn resume_vm(&self) -> Result<()>;
 
     // device manager
-    async fn add_device(&self, device: device::Device) -> Result<()>;
+    async fn add_device(&self, device: driver::DeviceConfig) -> Result<()>;
-    async fn remove_device(&self, device: device::Device) -> Result<()>;
+    async fn remove_device(&self, device: driver::DeviceConfig) -> Result<()>;
 
     // utils
     async fn get_agent_socket(&self) -> Result<String>;

src/runtime-rs/crates/hypervisor/src/qemu/inner.rs

@@ -133,16 +133,16 @@ impl QemuInner {
     }
 }
 
-use crate::device::Device;
+use crate::driver::DeviceConfig;
 
 // device manager part of Hypervisor
 impl QemuInner {
-    pub(crate) async fn add_device(&mut self, device: Device) -> Result<()> {
+    pub(crate) async fn add_device(&mut self, device: DeviceConfig) -> Result<()> {
         info!(sl!(), "QemuInner::add_device() {}", device);
         todo!()
     }
 
-    pub(crate) async fn remove_device(&mut self, device: Device) -> Result<()> {
+    pub(crate) async fn remove_device(&mut self, device: DeviceConfig) -> Result<()> {
         info!(sl!(), "QemuInner::remove_device() {} ", device);
         todo!()
     }

src/runtime-rs/crates/hypervisor/src/qemu/mod.rs

@@ -5,7 +5,7 @@
 
 mod inner;
 
-use crate::device::Device;
+use crate::driver::DeviceConfig;
 use crate::hypervisor_persist::HypervisorState;
 use crate::Hypervisor;
 use crate::{HypervisorConfig, VcpuThreadIds};
@@ -73,12 +73,12 @@ impl Hypervisor for Qemu {
         inner.save_vm().await
     }
 
-    async fn add_device(&self, device: Device) -> Result<()> {
+    async fn add_device(&self, device: DeviceConfig) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.add_device(device).await
     }
 
-    async fn remove_device(&self, device: Device) -> Result<()> {
+    async fn remove_device(&self, device: DeviceConfig) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.remove_device(device).await
     }

src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs

@@ -13,7 +13,7 @@ use async_trait::async_trait;
 use super::Endpoint;
 use crate::network::network_model::TC_FILTER_NET_MODEL_STR;
 use crate::network::{utils, NetworkPair};
-use hypervisor::{device::NetworkConfig, Device, Hypervisor};
+use hypervisor::{device::driver::NetworkConfig, DeviceConfig, Hypervisor};
 
 // IPVlanEndpoint is the endpoint bridged to VM
 #[derive(Debug)]
@@ -67,7 +67,7 @@ impl Endpoint for IPVlanEndpoint {
             .await
             .context("error adding network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.add_device(Device::Network(config))
+        h.add_device(DeviceConfig::Network(config))
             .await
             .context("error adding device by hypervisor")?;
 
@@ -82,7 +82,7 @@ impl Endpoint for IPVlanEndpoint {
         let config = self
             .get_network_config()
             .context("error getting network config")?;
-        h.remove_device(Device::Network(config))
+        h.remove_device(DeviceConfig::Network(config))
             .await
             .context("error removing device by hypervisor")?;
 

src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs

@@ -11,7 +11,7 @@ use super::Endpoint;
 use crate::network::{utils, NetworkPair};
 use anyhow::{Context, Result};
 use async_trait::async_trait;
-use hypervisor::{device::NetworkConfig, Device, Hypervisor};
+use hypervisor::{device::driver::NetworkConfig, DeviceConfig, Hypervisor};
 
 #[derive(Debug)]
 pub struct MacVlanEndpoint {
@@ -64,7 +64,7 @@ impl Endpoint for MacVlanEndpoint {
             .await
             .context("add network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.add_device(Device::Network(config))
+        h.add_device(DeviceConfig::Network(config))
             .await
             .context("Error add device")?;
         Ok(())
@@ -76,7 +76,7 @@ impl Endpoint for MacVlanEndpoint {
             .await
             .context("del network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.remove_device(Device::Network(config))
+        h.remove_device(DeviceConfig::Network(config))
             .await
             .context("remove device")?;
         Ok(())

src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs

@@ -8,7 +8,7 @@ use std::path::Path;
 
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
-use hypervisor::{device, Hypervisor};
+use hypervisor::{device::driver, Hypervisor};
 
 use super::endpoint_persist::{EndpointState, PhysicalEndpointState};
 use super::Endpoint;
@@ -94,7 +94,7 @@ impl Endpoint for PhysicalEndpoint {
 
     async fn attach(&self, hypervisor: &dyn Hypervisor) -> Result<()> {
         // bind physical interface from host driver and bind to vfio
-        device::bind_device_to_vfio(
+        driver::bind_device_to_vfio(
             &self.bdf,
             &self.driver,
             &self.vendor_device_id.vendor_device_id(),
@@ -108,11 +108,11 @@ impl Endpoint for PhysicalEndpoint {
         };
 
         // add vfio device
-        let d = device::Device::Vfio(device::VfioConfig {
+        let d = driver::DeviceConfig::Vfio(driver::VfioConfig {
             id: format!("physical_nic_{}", self.name().await),
             sysfs_path: "".to_string(),
             bus_slot_func: self.bdf.clone(),
-            mode: device::VfioBusMode::new(mode)
+            mode: driver::VfioBusMode::new(mode)
                 .with_context(|| format!("new vfio bus mode {:?}", mode))?,
         });
         hypervisor.add_device(d).await.context("add device")?;
@@ -128,7 +128,7 @@ impl Endpoint for PhysicalEndpoint {
 
         // we do not need to enter the network namespace to bind back the
         // physical interface to host driver.
-        device::bind_device_to_host(
+        driver::bind_device_to_host(
             &self.bdf,
             &self.driver,
             &self.vendor_device_id.vendor_device_id(),

src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs

@@ -11,7 +11,7 @@ use super::Endpoint;
 use crate::network::{utils, NetworkPair};
 use anyhow::{Context, Result};
 use async_trait::async_trait;
-use hypervisor::{device::NetworkConfig, Device, Hypervisor};
+use hypervisor::{device::driver::NetworkConfig, DeviceConfig, Hypervisor};
 
 #[derive(Debug)]
 pub struct VethEndpoint {
@@ -64,7 +64,7 @@ impl Endpoint for VethEndpoint {
             .await
             .context("add network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.add_device(Device::Network(config))
+        h.add_device(DeviceConfig::Network(config))
             .await
             .context("Error add device")?;
         Ok(())
@@ -76,7 +76,7 @@ impl Endpoint for VethEndpoint {
             .await
             .context("del network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.remove_device(Device::Network(config))
+        h.remove_device(DeviceConfig::Network(config))
             .await
             .context("remove device")?;
         Ok(())

src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs

@@ -13,7 +13,7 @@ use super::endpoint_persist::{EndpointState, VlanEndpointState};
 use super::Endpoint;
 use crate::network::network_model::TC_FILTER_NET_MODEL_STR;
 use crate::network::{utils, NetworkPair};
-use hypervisor::{device::NetworkConfig, Device, Hypervisor};
+use hypervisor::{device::driver::NetworkConfig, DeviceConfig, Hypervisor};
 #[derive(Debug)]
 pub struct VlanEndpoint {
     pub(crate) net_pair: NetworkPair,
@@ -64,7 +64,7 @@ impl Endpoint for VlanEndpoint {
             .await
             .context("error adding network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.add_device(Device::Network(config))
+        h.add_device(DeviceConfig::Network(config))
             .await
             .context("error adding device by hypervisor")?;
 
@@ -79,7 +79,7 @@ impl Endpoint for VlanEndpoint {
         let config = self
             .get_network_config()
             .context("error getting network config")?;
-        h.remove_device(Device::Network(config))
+        h.remove_device(DeviceConfig::Network(config))
             .await
             .context("error removing device by hypervisor")?;
 

src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs

@@ -8,7 +8,9 @@ use std::path::Path;
 
 use anyhow::{Context, Result};
 use hypervisor::{
-    device::{Device as HypervisorDevice, ShareFsMountConfig, ShareFsMountType, ShareFsOperation},
+    device::driver::{
+        DeviceConfig as HypervisorDevice, ShareFsMountConfig, ShareFsMountType, ShareFsOperation,
+    },
     Hypervisor, ShareFsDeviceConfig,
 };
 use kata_sys_util::mount;