From 81ab174c16dfd6d7524ae1a0854f48ead4d129f2 Mon Sep 17 00:00:00 2001 From: Qinqi Qu Date: Tue, 12 Dec 2023 16:22:14 +0800 Subject: [PATCH] dragonball: support vhost-user-blk in device manager This patch introduces a feature of supporting vhost-user-blk device. Fixes: #8631 Signed-off-by: Qinqi Qu --- src/dragonball/Cargo.toml | 21 ++++-- src/dragonball/src/api/v1/vmm_action.rs | 28 ++++---- .../src/device_manager/blk_dev_mgr.rs | 64 ++++++++++++++++++- src/dragonball/src/device_manager/mod.rs | 14 ++-- src/dragonball/src/error.rs | 2 +- 5 files changed, 100 insertions(+), 29 deletions(-) diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index f4c43047b3..ca41c613fa 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -13,16 +13,18 @@ edition = "2018" anyhow = "1.0.32" arc-swap = "1.5.0" bytes = "1.1.0" -dbs-address-space = { path = "./src/dbs_address_space" } +dbs-address-space = { path = "./src/dbs_address_space" } dbs-allocator = { path = "./src/dbs_allocator" } dbs-arch = { path = "./src/dbs_arch" } dbs-boot = { path = "./src/dbs_boot" } dbs-device = { path = "./src/dbs_device" } dbs-interrupt = { path = "./src/dbs_interrupt", features = ["kvm-irq"] } dbs-legacy-devices = { path = "./src/dbs_legacy_devices" } -dbs-upcall = { path = "./src/dbs_upcall" , optional = true } +dbs-upcall = { path = "./src/dbs_upcall", optional = true } dbs-utils = { path = "./src/dbs_utils" } -dbs-virtio-devices = { path = "./src/dbs_virtio_devices", optional = true, features = ["virtio-mmio"] } +dbs-virtio-devices = { path = "./src/dbs_virtio_devices", optional = true, features = [ + "virtio-mmio", +] } derivative = "2.2.0" kvm-bindings = "0.6.0" kvm-ioctls = "0.12.0" @@ -60,9 +62,18 @@ virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"] virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"] virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"] # virtio-fs only work on atomic-guest-memory -virtio-fs = ["dbs-virtio-devices/virtio-fs-pro", "virtio-queue", "atomic-guest-memory"] -virtio-mem = ["dbs-virtio-devices/virtio-mem", "virtio-queue", "atomic-guest-memory"] +virtio-fs = [ + "dbs-virtio-devices/virtio-fs-pro", + "virtio-queue", + "atomic-guest-memory", +] +virtio-mem = [ + "dbs-virtio-devices/virtio-mem", + "virtio-queue", + "atomic-guest-memory", +] virtio-balloon = ["dbs-virtio-devices/virtio-balloon", "virtio-queue"] vhost-net = ["dbs-virtio-devices/vhost-net"] vhost-user-fs = ["dbs-virtio-devices/vhost-user-fs"] vhost-user-net = ["dbs-virtio-devices/vhost-user-net"] +vhost-user-blk = ["dbs-virtio-devices/vhost-user-blk"] diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 5d99b97185..0d2f678ed0 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -26,7 +26,7 @@ use self::VmmActionError::MachineConfig; #[cfg(feature = "virtio-balloon")] pub use crate::device_manager::balloon_dev_mgr::{BalloonDeviceConfigInfo, BalloonDeviceError}; -#[cfg(feature = "virtio-blk")] +#[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] pub use crate::device_manager::blk_dev_mgr::{ BlockDeviceConfigInfo, BlockDeviceConfigUpdateInfo, BlockDeviceError, BlockDeviceMgr, }; @@ -99,7 +99,7 @@ pub enum VmmActionError { #[error("failed to add virtio-vsock device: {0}")] Vsock(#[source] VsockDeviceError), - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] /// Block device related errors. #[error("virtio-blk device error: {0}")] Block(#[source] BlockDeviceError), @@ -186,16 +186,16 @@ pub enum VmmAction { /// booted. The response is sent using the `OutcomeSender`. InsertVsockDevice(VsockDeviceConfigInfo), - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] /// Add a new block device or update one that already exists using the `BlockDeviceConfig` as /// input. This action can only be called before the microVM has booted. InsertBlockDevice(BlockDeviceConfigInfo), - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] /// Remove a new block device for according to given drive_id RemoveBlockDevice(String), - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] /// Update a block device, after microVM start. Currently, the only updatable properties /// are the RX and TX rate limiters. UpdateBlockDevice(BlockDeviceConfigUpdateInfo), @@ -321,15 +321,15 @@ impl VmmService { VmmAction::EndHypervisorTracing => self.end_tracing(), #[cfg(feature = "virtio-vsock")] VmmAction::InsertVsockDevice(vsock_cfg) => self.add_vsock_device(vmm, vsock_cfg), - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] VmmAction::InsertBlockDevice(block_device_config) => { self.add_block_device(vmm, event_mgr, block_device_config) } - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] VmmAction::UpdateBlockDevice(blk_update) => { self.update_blk_rate_limiters(vmm, blk_update) } - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] VmmAction::RemoveBlockDevice(drive_id) => { self.remove_block_device(vmm, event_mgr, &drive_id) } @@ -602,7 +602,7 @@ impl VmmService { .map_err(VmmActionError::Vsock) } - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] // Only call this function as part of the API. // If the drive_id does not exist, a new Block Device Config is added to the list. #[instrument(skip(self, event_mgr))] @@ -629,7 +629,7 @@ impl VmmService { .map_err(VmmActionError::Block) } - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] /// Updates configuration for an emulated net device as described in `config`. #[instrument(skip(self))] fn update_blk_rate_limiters( @@ -646,7 +646,7 @@ impl VmmService { .map_err(VmmActionError::Block) } - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] // Remove the device #[instrument(skip(self, event_mgr))] fn remove_block_device( @@ -1342,7 +1342,7 @@ mod tests { } } - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] #[test] fn test_vmm_action_insert_block_device() { skip_if_not_root!(); @@ -1399,7 +1399,7 @@ mod tests { } } - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] #[test] fn test_vmm_action_update_block_device() { skip_if_not_root!(); @@ -1432,7 +1432,7 @@ mod tests { } } - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] #[test] fn test_vmm_action_remove_block_device() { skip_if_not_root!(); diff --git a/src/dragonball/src/device_manager/blk_dev_mgr.rs b/src/dragonball/src/device_manager/blk_dev_mgr.rs index ae6c534f8a..51bc5d13e1 100644 --- a/src/dragonball/src/device_manager/blk_dev_mgr.rs +++ b/src/dragonball/src/device_manager/blk_dev_mgr.rs @@ -17,6 +17,8 @@ use std::sync::Arc; use dbs_virtio_devices as virtio; use dbs_virtio_devices::block::{aio::Aio, io_uring::IoUring, Block, LocalFile, Ufile}; +#[cfg(feature = "vhost-user-blk")] +use dbs_virtio_devices::vhost::vhost_user::block::VhostUserBlock; use serde_derive::{Deserialize, Serialize}; use crate::address_space_manager::GuestAddressSpaceImpl; @@ -404,9 +406,29 @@ impl BlockDeviceMgr { BlockDeviceError::DeviceManager(e) }) } + #[cfg(feature = "vhost-user-blk")] BlockDeviceType::Spool | BlockDeviceType::Spdk => { - // TBD - todo!() + let device = Self::create_vhost_user_device(&config, &mut ctx) + .map_err(BlockDeviceError::Virtio)?; + let dev = DeviceManager::create_mmio_virtio_device( + device, + &mut ctx, + config.use_shared_irq.unwrap_or(self.use_shared_irq), + config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(BlockDeviceError::DeviceManager)?; + self.update_device_by_index(index, Arc::clone(&dev))?; + ctx.insert_hotplug_mmio_device(&dev, None).map_err(|e| { + let logger = ctx.logger().new(slog::o!()); + self.remove_device(ctx, &config.drive_id).unwrap(); + error!( + logger, + "failed to hot-add virtio block device {}, {:?}", + &config.drive_id, + e + ); + BlockDeviceError::DeviceManager(e) + }) } _ => Err(BlockDeviceError::InvalidBlockDeviceType), } @@ -439,6 +461,25 @@ impl BlockDeviceMgr { .map_err(BlockDeviceError::RegisterBlockDevice)?; info.device = Some(device); } + #[cfg(feature = "vhost-user-blk")] + BlockDeviceType::Spool | BlockDeviceType::Spdk => { + info!( + ctx.logger(), + "attach vhost-user-blk device, drive_id {}, path {}", + info.config.drive_id, + info.config.path_on_host.to_str().unwrap_or("") + ); + let device = Self::create_vhost_user_device(&info.config, ctx) + .map_err(BlockDeviceError::Virtio)?; + let device = DeviceManager::create_mmio_virtio_device( + device, + ctx, + info.config.use_shared_irq.unwrap_or(self.use_shared_irq), + info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(BlockDeviceError::RegisterBlockDevice)?; + info.device = Some(device); + } _ => { return Err(BlockDeviceError::OpenBlockDevice( std::io::Error::from_raw_os_error(libc::EINVAL), @@ -573,6 +614,25 @@ impl BlockDeviceMgr { )?)) } + #[cfg(feature = "vhost-user-blk")] + fn create_vhost_user_device( + cfg: &BlockDeviceConfigInfo, + ctx: &mut DeviceOpContext, + ) -> std::result::Result>, virtio::Error> { + info!( + ctx.logger(), + "new vhost user block device {:?}", cfg.path_on_host + ); + let epoll_mgr = ctx.epoll_mgr.clone().ok_or(virtio::Error::InvalidInput)?; + let path = cfg.path_on_host.to_str().unwrap().to_string(); + + Ok(Box::new(VhostUserBlock::new( + path, + Arc::new(cfg.queue_sizes()), + epoll_mgr, + )?)) + } + /// Generated guest kernel commandline related to root block device. pub fn generate_kernel_boot_args( &self, diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 888ecb7e49..090c135245 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -68,10 +68,10 @@ pub mod vsock_dev_mgr; #[cfg(feature = "virtio-vsock")] use self::vsock_dev_mgr::VsockDeviceMgr; -#[cfg(feature = "virtio-blk")] +#[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] /// virtio-block device manager pub mod blk_dev_mgr; -#[cfg(feature = "virtio-blk")] +#[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] use self::blk_dev_mgr::BlockDeviceMgr; #[cfg(feature = "virtio-net")] @@ -533,7 +533,7 @@ pub struct DeviceManager { #[cfg(feature = "virtio-vsock")] pub(crate) vsock_manager: VsockDeviceMgr, - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] // If there is a Root Block Device, this should be added as the first element of the list. // This is necessary because we want the root to always be mounted on /dev/vda. pub(crate) block_manager: BlockDeviceMgr, @@ -581,7 +581,7 @@ impl DeviceManager { mmio_device_info: HashMap::new(), #[cfg(feature = "virtio-vsock")] vsock_manager: VsockDeviceMgr::default(), - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] block_manager: BlockDeviceMgr::default(), #[cfg(feature = "virtio-net")] virtio_net_manager: VirtioNetDeviceMgr::default(), @@ -739,7 +739,7 @@ impl DeviceManager { self.create_legacy_devices(&mut ctx)?; self.init_legacy_devices(dmesg_fifo, com1_sock_path, &mut ctx)?; - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] self.block_manager .attach_devices(&mut ctx) .map_err(StartMicroVmError::BlockDeviceError)?; @@ -760,7 +760,7 @@ impl DeviceManager { #[cfg(feature = "virtio-vsock")] self.vsock_manager.attach_devices(&mut ctx)?; - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] self.block_manager .generate_kernel_boot_args(kernel_config) .map_err(StartMicroVmError::DeviceManager)?; @@ -1184,7 +1184,7 @@ mod tests { res_manager, legacy_manager: None, - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] block_manager: BlockDeviceMgr::default(), #[cfg(any(feature = "virtio-fs", feature = "vhost-user-fs"))] fs_manager: Arc::new(Mutex::new(FsDeviceMgr::default())), diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index b6cec47648..6dc427a2ab 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -174,7 +174,7 @@ pub enum StartMicroVmError { #[error("failure while connecting the upcall client: {0}")] UpcallConnectError(#[source] dbs_upcall::UpcallClientError), - #[cfg(feature = "virtio-blk")] + #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] /// Virtio-blk errors. #[error("virtio-blk errors: {0}")] BlockDeviceError(#[source] device_manager::blk_dev_mgr::BlockDeviceError),