dragonball: support vhost-user-blk in device manager

This patch introduces a feature of supporting vhost-user-blk device.

Fixes: #8631

Signed-off-by: Qinqi Qu <quqinqi@linux.alibaba.com>

src/dragonball/Cargo.toml

@@ -13,16 +13,18 @@ edition = "2018"
 anyhow = "1.0.32"
 arc-swap = "1.5.0"
 bytes = "1.1.0"
-dbs-address-space =  { path = "./src/dbs_address_space" }
+dbs-address-space = { path = "./src/dbs_address_space" }
 dbs-allocator = { path = "./src/dbs_allocator" }
 dbs-arch = { path = "./src/dbs_arch" }
 dbs-boot = { path = "./src/dbs_boot" }
 dbs-device = { path = "./src/dbs_device" }
 dbs-interrupt = { path = "./src/dbs_interrupt", features = ["kvm-irq"] }
 dbs-legacy-devices = { path = "./src/dbs_legacy_devices" }
-dbs-upcall = { path = "./src/dbs_upcall" , optional = true }
+dbs-upcall = { path = "./src/dbs_upcall", optional = true }
 dbs-utils = { path = "./src/dbs_utils" }
-dbs-virtio-devices = { path = "./src/dbs_virtio_devices", optional = true, features = ["virtio-mmio"] }
+dbs-virtio-devices = { path = "./src/dbs_virtio_devices", optional = true, features = [
+    "virtio-mmio",
+] }
 derivative = "2.2.0"
 kvm-bindings = "0.6.0"
 kvm-ioctls = "0.12.0"
@@ -60,9 +62,18 @@ virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"]
 virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"]
 virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"]
 # virtio-fs only work on atomic-guest-memory
-virtio-fs = ["dbs-virtio-devices/virtio-fs-pro", "virtio-queue", "atomic-guest-memory"]
-virtio-mem = ["dbs-virtio-devices/virtio-mem", "virtio-queue", "atomic-guest-memory"]
+virtio-fs = [
+    "dbs-virtio-devices/virtio-fs-pro",
+    "virtio-queue",
+    "atomic-guest-memory",
+]
+virtio-mem = [
+    "dbs-virtio-devices/virtio-mem",
+    "virtio-queue",
+    "atomic-guest-memory",
+]
 virtio-balloon = ["dbs-virtio-devices/virtio-balloon", "virtio-queue"]
 vhost-net = ["dbs-virtio-devices/vhost-net"]
 vhost-user-fs = ["dbs-virtio-devices/vhost-user-fs"]
 vhost-user-net = ["dbs-virtio-devices/vhost-user-net"]
+vhost-user-blk = ["dbs-virtio-devices/vhost-user-blk"]

src/dragonball/src/api/v1/vmm_action.rs

@@ -26,7 +26,7 @@ use self::VmmActionError::MachineConfig;
 
 #[cfg(feature = "virtio-balloon")]
 pub use crate::device_manager::balloon_dev_mgr::{BalloonDeviceConfigInfo, BalloonDeviceError};
-#[cfg(feature = "virtio-blk")]
+#[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
 pub use crate::device_manager::blk_dev_mgr::{
     BlockDeviceConfigInfo, BlockDeviceConfigUpdateInfo, BlockDeviceError, BlockDeviceMgr,
 };
@@ -99,7 +99,7 @@ pub enum VmmActionError {
     #[error("failed to add virtio-vsock device: {0}")]
     Vsock(#[source] VsockDeviceError),
 
-    #[cfg(feature = "virtio-blk")]
+    #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
     /// Block device related errors.
     #[error("virtio-blk device error: {0}")]
     Block(#[source] BlockDeviceError),
@@ -186,16 +186,16 @@ pub enum VmmAction {
     /// booted. The response is sent using the `OutcomeSender`.
     InsertVsockDevice(VsockDeviceConfigInfo),
 
-    #[cfg(feature = "virtio-blk")]
+    #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
     /// Add a new block device or update one that already exists using the `BlockDeviceConfig` as
     /// input. This action can only be called before the microVM has booted.
     InsertBlockDevice(BlockDeviceConfigInfo),
 
-    #[cfg(feature = "virtio-blk")]
+    #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
     /// Remove a new block device for according to given drive_id
     RemoveBlockDevice(String),
 
-    #[cfg(feature = "virtio-blk")]
+    #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
     /// Update a block device, after microVM start. Currently, the only updatable properties
     /// are the RX and TX rate limiters.
     UpdateBlockDevice(BlockDeviceConfigUpdateInfo),
@@ -321,15 +321,15 @@ impl VmmService {
             VmmAction::EndHypervisorTracing => self.end_tracing(),
             #[cfg(feature = "virtio-vsock")]
             VmmAction::InsertVsockDevice(vsock_cfg) => self.add_vsock_device(vmm, vsock_cfg),
-            #[cfg(feature = "virtio-blk")]
+            #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
             VmmAction::InsertBlockDevice(block_device_config) => {
                 self.add_block_device(vmm, event_mgr, block_device_config)
             }
-            #[cfg(feature = "virtio-blk")]
+            #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
             VmmAction::UpdateBlockDevice(blk_update) => {
                 self.update_blk_rate_limiters(vmm, blk_update)
             }
-            #[cfg(feature = "virtio-blk")]
+            #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
             VmmAction::RemoveBlockDevice(drive_id) => {
                 self.remove_block_device(vmm, event_mgr, &drive_id)
             }
@@ -602,7 +602,7 @@ impl VmmService {
             .map_err(VmmActionError::Vsock)
     }
 
-    #[cfg(feature = "virtio-blk")]
+    #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
     // Only call this function as part of the API.
     // If the drive_id does not exist, a new Block Device Config is added to the list.
     #[instrument(skip(self, event_mgr))]
@@ -629,7 +629,7 @@ impl VmmService {
             .map_err(VmmActionError::Block)
     }
 
-    #[cfg(feature = "virtio-blk")]
+    #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
     /// Updates configuration for an emulated net device as described in `config`.
     #[instrument(skip(self))]
     fn update_blk_rate_limiters(
@@ -646,7 +646,7 @@ impl VmmService {
             .map_err(VmmActionError::Block)
     }
 
-    #[cfg(feature = "virtio-blk")]
+    #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
     // Remove the device
     #[instrument(skip(self, event_mgr))]
     fn remove_block_device(
@@ -1342,7 +1342,7 @@ mod tests {
         }
     }
 
-    #[cfg(feature = "virtio-blk")]
+    #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
     #[test]
     fn test_vmm_action_insert_block_device() {
         skip_if_not_root!();
@@ -1399,7 +1399,7 @@ mod tests {
         }
     }
 
-    #[cfg(feature = "virtio-blk")]
+    #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
     #[test]
     fn test_vmm_action_update_block_device() {
         skip_if_not_root!();
@@ -1432,7 +1432,7 @@ mod tests {
         }
     }
 
-    #[cfg(feature = "virtio-blk")]
+    #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
     #[test]
     fn test_vmm_action_remove_block_device() {
         skip_if_not_root!();

src/dragonball/src/device_manager/blk_dev_mgr.rs

@@ -17,6 +17,8 @@ use std::sync::Arc;
 
 use dbs_virtio_devices as virtio;
 use dbs_virtio_devices::block::{aio::Aio, io_uring::IoUring, Block, LocalFile, Ufile};
+#[cfg(feature = "vhost-user-blk")]
+use dbs_virtio_devices::vhost::vhost_user::block::VhostUserBlock;
 use serde_derive::{Deserialize, Serialize};
 
 use crate::address_space_manager::GuestAddressSpaceImpl;
@@ -404,9 +406,29 @@ impl BlockDeviceMgr {
                             BlockDeviceError::DeviceManager(e)
                         })
                     }
+                    #[cfg(feature = "vhost-user-blk")]
                     BlockDeviceType::Spool | BlockDeviceType::Spdk => {
-                        // TBD
-                        todo!()
+                        let device = Self::create_vhost_user_device(&config, &mut ctx)
+                            .map_err(BlockDeviceError::Virtio)?;
+                        let dev = DeviceManager::create_mmio_virtio_device(
+                            device,
+                            &mut ctx,
+                            config.use_shared_irq.unwrap_or(self.use_shared_irq),
+                            config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
+                        )
+                        .map_err(BlockDeviceError::DeviceManager)?;
+                        self.update_device_by_index(index, Arc::clone(&dev))?;
+                        ctx.insert_hotplug_mmio_device(&dev, None).map_err(|e| {
+                            let logger = ctx.logger().new(slog::o!());
+                            self.remove_device(ctx, &config.drive_id).unwrap();
+                            error!(
+                                logger,
+                                "failed to hot-add virtio block device {}, {:?}",
+                                &config.drive_id,
+                                e
+                            );
+                            BlockDeviceError::DeviceManager(e)
+                        })
                     }
                     _ => Err(BlockDeviceError::InvalidBlockDeviceType),
                 }
@@ -439,6 +461,25 @@ impl BlockDeviceMgr {
                     .map_err(BlockDeviceError::RegisterBlockDevice)?;
                     info.device = Some(device);
                 }
+                #[cfg(feature = "vhost-user-blk")]
+                BlockDeviceType::Spool | BlockDeviceType::Spdk => {
+                    info!(
+                        ctx.logger(),
+                        "attach vhost-user-blk device, drive_id {}, path {}",
+                        info.config.drive_id,
+                        info.config.path_on_host.to_str().unwrap_or("<unknown>")
+                    );
+                    let device = Self::create_vhost_user_device(&info.config, ctx)
+                        .map_err(BlockDeviceError::Virtio)?;
+                    let device = DeviceManager::create_mmio_virtio_device(
+                        device,
+                        ctx,
+                        info.config.use_shared_irq.unwrap_or(self.use_shared_irq),
+                        info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
+                    )
+                    .map_err(BlockDeviceError::RegisterBlockDevice)?;
+                    info.device = Some(device);
+                }
                 _ => {
                     return Err(BlockDeviceError::OpenBlockDevice(
                         std::io::Error::from_raw_os_error(libc::EINVAL),
@@ -573,6 +614,25 @@ impl BlockDeviceMgr {
         )?))
     }
 
+    #[cfg(feature = "vhost-user-blk")]
+    fn create_vhost_user_device(
+        cfg: &BlockDeviceConfigInfo,
+        ctx: &mut DeviceOpContext,
+    ) -> std::result::Result<Box<VhostUserBlock<GuestAddressSpaceImpl>>, virtio::Error> {
+        info!(
+            ctx.logger(),
+            "new vhost user block device {:?}", cfg.path_on_host
+        );
+        let epoll_mgr = ctx.epoll_mgr.clone().ok_or(virtio::Error::InvalidInput)?;
+        let path = cfg.path_on_host.to_str().unwrap().to_string();
+
+        Ok(Box::new(VhostUserBlock::new(
+            path,
+            Arc::new(cfg.queue_sizes()),
+            epoll_mgr,
+        )?))
+    }
+
     /// Generated guest kernel commandline related to root block device.
     pub fn generate_kernel_boot_args(
         &self,

src/dragonball/src/device_manager/mod.rs

@@ -68,10 +68,10 @@ pub mod vsock_dev_mgr;
 #[cfg(feature = "virtio-vsock")]
 use self::vsock_dev_mgr::VsockDeviceMgr;
 
-#[cfg(feature = "virtio-blk")]
+#[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
 /// virtio-block device manager
 pub mod blk_dev_mgr;
-#[cfg(feature = "virtio-blk")]
+#[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
 use self::blk_dev_mgr::BlockDeviceMgr;
 
 #[cfg(feature = "virtio-net")]
@@ -533,7 +533,7 @@ pub struct DeviceManager {
     #[cfg(feature = "virtio-vsock")]
     pub(crate) vsock_manager: VsockDeviceMgr,
 
-    #[cfg(feature = "virtio-blk")]
+    #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
     // If there is a Root Block Device, this should be added as the first element of the list.
     // This is necessary because we want the root to always be mounted on /dev/vda.
     pub(crate) block_manager: BlockDeviceMgr,
@@ -581,7 +581,7 @@ impl DeviceManager {
             mmio_device_info: HashMap::new(),
             #[cfg(feature = "virtio-vsock")]
             vsock_manager: VsockDeviceMgr::default(),
-            #[cfg(feature = "virtio-blk")]
+            #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
             block_manager: BlockDeviceMgr::default(),
             #[cfg(feature = "virtio-net")]
             virtio_net_manager: VirtioNetDeviceMgr::default(),
@@ -739,7 +739,7 @@ impl DeviceManager {
         self.create_legacy_devices(&mut ctx)?;
         self.init_legacy_devices(dmesg_fifo, com1_sock_path, &mut ctx)?;
 
-        #[cfg(feature = "virtio-blk")]
+        #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
         self.block_manager
             .attach_devices(&mut ctx)
             .map_err(StartMicroVmError::BlockDeviceError)?;
@@ -760,7 +760,7 @@ impl DeviceManager {
         #[cfg(feature = "virtio-vsock")]
         self.vsock_manager.attach_devices(&mut ctx)?;
 
-        #[cfg(feature = "virtio-blk")]
+        #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
         self.block_manager
             .generate_kernel_boot_args(kernel_config)
             .map_err(StartMicroVmError::DeviceManager)?;
@@ -1184,7 +1184,7 @@ mod tests {
                 res_manager,
 
                 legacy_manager: None,
-                #[cfg(feature = "virtio-blk")]
+                #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
                 block_manager: BlockDeviceMgr::default(),
                 #[cfg(any(feature = "virtio-fs", feature = "vhost-user-fs"))]
                 fs_manager: Arc::new(Mutex::new(FsDeviceMgr::default())),

src/dragonball/src/error.rs

@@ -174,7 +174,7 @@ pub enum StartMicroVmError {
     #[error("failure while connecting the upcall client: {0}")]
     UpcallConnectError(#[source] dbs_upcall::UpcallClientError),
 
-    #[cfg(feature = "virtio-blk")]
+    #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))]
     /// Virtio-blk errors.
     #[error("virtio-blk errors: {0}")]
     BlockDeviceError(#[source] device_manager::blk_dev_mgr::BlockDeviceError),