diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index d44e115ca5..0f4aa582f9 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -51,6 +51,8 @@ hotplug = ["virtio-vsock"] virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"] virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"] virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"] +# virtio-fs only work on atomic-guest-memory +virtio-fs = ["dbs-virtio-devices/virtio-fs", "virtio-queue", "atomic-guest-memory"] [patch.'crates-io'] dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 0a021341ac..dfba2faf13 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -21,6 +21,10 @@ use crate::vmm::Vmm; use crate::device_manager::blk_dev_mgr::{ BlockDeviceConfigInfo, BlockDeviceConfigUpdateInfo, BlockDeviceError, BlockDeviceMgr, }; +#[cfg(feature = "virtio-fs")] +use crate::device_manager::fs_dev_mgr::{ + FsDeviceConfigInfo, FsDeviceConfigUpdateInfo, FsDeviceError, FsDeviceMgr, FsMountConfigInfo, +}; #[cfg(feature = "virtio-net")] use crate::device_manager::virtio_net_dev_mgr::{ VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo, VirtioNetDeviceError, @@ -74,6 +78,11 @@ pub enum VmmActionError { /// Net device related errors. #[error("virtio-net device error: {0}")] VirtioNet(#[source] VirtioNetDeviceError), + + #[cfg(feature = "virtio-fs")] + /// The action `InsertFsDevice` failed either because of bad user input or an internal error. + #[error("virtio-fs device: {0}")] + FsDevice(#[source] FsDeviceError), } /// This enum represents the public interface of the VMM. Each action contains various @@ -129,6 +138,22 @@ pub enum VmmAction { /// Update a network interface, after microVM start. Currently, the only updatable properties /// are the RX and TX rate limiters. UpdateNetworkInterface(VirtioNetDeviceConfigUpdateInfo), + + #[cfg(feature = "virtio-fs")] + /// Add a new shared fs device or update one that already exists using the + /// `FsDeviceConfig` as input. This action can only be called before the microVM has + /// booted. + InsertFsDevice(FsDeviceConfigInfo), + + #[cfg(feature = "virtio-fs")] + /// Attach a new virtiofs Backend fs or detach an existing virtiofs Backend fs using the + /// `FsMountConfig` as input. This action can only be called _after_ the microVM has + /// booted. + ManipulateFsBackendFs(FsMountConfigInfo), + + #[cfg(feature = "virtio-fs")] + /// Update fs rate limiter, after microVM start. + UpdateFsDevice(FsDeviceConfigUpdateInfo), } /// The enum represents the response sent by the VMM in case of success. The response is either @@ -218,6 +243,17 @@ impl VmmService { VmmAction::UpdateNetworkInterface(netif_update) => { self.update_net_rate_limiters(vmm, netif_update) } + #[cfg(feature = "virtio-fs")] + VmmAction::InsertFsDevice(fs_cfg) => self.add_fs_device(vmm, fs_cfg), + + #[cfg(feature = "virtio-fs")] + VmmAction::ManipulateFsBackendFs(fs_mount_cfg) => { + self.manipulate_fs_backend_fs(vmm, fs_mount_cfg) + } + #[cfg(feature = "virtio-fs")] + VmmAction::UpdateFsDevice(fs_update_cfg) => { + self.update_fs_rate_limiters(vmm, fs_update_cfg) + } }; debug!("send vmm response: {:?}", response); @@ -566,4 +602,63 @@ impl VmmService { .map(|_| VmmData::Empty) .map_err(VmmActionError::VirtioNet) } + + #[cfg(feature = "virtio-fs")] + fn add_fs_device(&mut self, vmm: &mut Vmm, config: FsDeviceConfigInfo) -> VmmRequestResult { + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + let hotplug = vm.is_vm_initialized(); + if !cfg!(feature = "hotplug") && hotplug { + return Err(VmmActionError::FsDevice( + FsDeviceError::UpdateNotAllowedPostBoot, + )); + } + + let ctx = vm.create_device_op_context(None).map_err(|e| { + info!("create device op context error: {:?}", e); + VmmActionError::FsDevice(FsDeviceError::UpdateNotAllowedPostBoot) + })?; + FsDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::FsDevice) + } + + #[cfg(feature = "virtio-fs")] + fn manipulate_fs_backend_fs( + &self, + vmm: &mut Vmm, + config: FsMountConfigInfo, + ) -> VmmRequestResult { + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + + if !vm.is_vm_initialized() { + return Err(VmmActionError::FsDevice(FsDeviceError::MicroVMNotRunning)); + } + + FsDeviceMgr::manipulate_backend_fs(vm.device_manager_mut(), config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::FsDevice) + } + + #[cfg(feature = "virtio-fs")] + fn update_fs_rate_limiters( + &self, + vmm: &mut Vmm, + config: FsDeviceConfigUpdateInfo, + ) -> VmmRequestResult { + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + + if !vm.is_vm_initialized() { + return Err(VmmActionError::FsDevice(FsDeviceError::MicroVMNotRunning)); + } + + FsDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::FsDevice) + } } diff --git a/src/dragonball/src/device_manager/blk_dev_mgr.rs b/src/dragonball/src/device_manager/blk_dev_mgr.rs index 38e46cd594..a624f68db6 100644 --- a/src/dragonball/src/device_manager/blk_dev_mgr.rs +++ b/src/dragonball/src/device_manager/blk_dev_mgr.rs @@ -176,7 +176,7 @@ pub struct BlockDeviceConfigInfo { pub no_drop: bool, /// Block device multi-queue pub num_queues: usize, - /// Virtio queue size. + /// Virtio queue size. Size: byte pub queue_size: u16, /// Rate Limiter for I/O operations. pub rate_limiter: Option, diff --git a/src/dragonball/src/device_manager/fs_dev_mgr.rs b/src/dragonball/src/device_manager/fs_dev_mgr.rs new file mode 100644 index 0000000000..c8fb553667 --- /dev/null +++ b/src/dragonball/src/device_manager/fs_dev_mgr.rs @@ -0,0 +1,523 @@ +// Copyright 2020-2022 Alibaba Cloud. All Rights Reserved. +// Copyright 2019 Intel Corporation. All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +use std::convert::TryInto; + +use dbs_utils::epoll_manager::EpollManager; +use dbs_virtio_devices::{self as virtio, Error as VirtIoError}; +use serde_derive::{Deserialize, Serialize}; +use slog::{error, info}; + +use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::config_manager::{ + get_bucket_update, ConfigItem, DeviceConfigInfo, DeviceConfigInfos, RateLimiterConfigInfo, +}; +use crate::device_manager::{ + DbsMmioV2Device, DeviceManager, DeviceMgrError, DeviceOpContext, DeviceVirtioRegionHandler, +}; + +use super::DbsVirtioDevice; + +// The flag of whether to use the shared irq. +const USE_SHARED_IRQ: bool = true; +// The flag of whether to use the generic irq. +const USE_GENERIC_IRQ: bool = true; +// Default cache size is 2 Gi since this is a typical VM memory size. +const DEFAULT_CACHE_SIZE: u64 = 2 * 1024 * 1024 * 1024; + +/// Errors associated with `FsDeviceConfig`. +#[derive(Debug, thiserror::Error)] +pub enum FsDeviceError { + /// Invalid fs, "virtio" or "vhostuser" is allowed. + #[error("the fs type is invalid, virtio or vhostuser is allowed")] + InvalidFs, + + /// Cannot access address space. + #[error("Cannot access address space.")] + AddressSpaceNotInitialized, + + /// Cannot convert RateLimterConfigInfo into RateLimiter. + #[error("failure while converting RateLimterConfigInfo into RateLimiter: {0}")] + RateLimterConfigInfoTryInto(#[source] std::io::Error), + + /// The fs device tag was already used for a different fs. + #[error("VirtioFs device tag {0} already exists")] + FsDeviceTagAlreadyExists(String), + + /// The fs device path was already used for a different fs. + #[error("VirtioFs device tag {0} already exists")] + FsDevicePathAlreadyExists(String), + + /// The update is not allowed after booting the microvm. + #[error("update operation is not allowed after boot")] + UpdateNotAllowedPostBoot, + + /// The attachbackendfs operation fails. + #[error("Fs device attach a backend fs failed")] + AttachBackendFailed(String), + + /// attach backend fs must be done when vm is running. + #[error("vm is not running when attaching a backend fs")] + MicroVMNotRunning, + + /// The mount tag doesn't exist. + #[error("fs tag'{0}' doesn't exist")] + TagNotExists(String), + + /// Failed to send patch message to VirtioFs epoll handler. + #[error("could not send patch message to the VirtioFs epoll handler")] + VirtioFsEpollHanderSendFail, + + /// Creating a shared-fs device fails (if the vhost-user socket cannot be open.) + #[error("cannot create shared-fs device: {0}")] + CreateFsDevice(#[source] VirtIoError), + + /// Cannot initialize a shared-fs device or add a device to the MMIO Bus. + #[error("failure while registering shared-fs device: {0}")] + RegisterFsDevice(#[source] DeviceMgrError), + + /// The device manager errors. + #[error("DeviceManager error: {0}")] + DeviceManager(#[source] DeviceMgrError), +} + +/// Configuration information for a vhost-user-fs device. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct FsDeviceConfigInfo { + /// vhost-user socket path. + pub sock_path: String, + /// virtiofs mount tag name used inside the guest. + /// used as the device name during mount. + pub tag: String, + /// Number of virtqueues to use. + pub num_queues: usize, + /// Size of each virtqueue. Unit: byte. + pub queue_size: u16, + /// DAX cache window size + pub cache_size: u64, + /// Number of thread pool workers. + pub thread_pool_size: u16, + /// The caching policy the file system should use (auto, always or never). + /// This cache policy is set for virtio-fs, visit https://gitlab.com/virtio-fs/virtiofsd to get further information. + pub cache_policy: String, + /// Writeback cache + pub writeback_cache: bool, + /// Enable no_open or not + pub no_open: bool, + /// Enable xattr or not + pub xattr: bool, + /// Drop CAP_SYS_RESOURCE or not + pub drop_sys_resource: bool, + /// virtio fs or vhostuser fs. + pub mode: String, + /// Enable kill_priv_v2 or not + pub fuse_killpriv_v2: bool, + /// Enable no_readdir or not + pub no_readdir: bool, + /// Rate Limiter for I/O operations. + pub rate_limiter: Option, + /// Use shared irq + pub use_shared_irq: Option, + /// Use generic irq + pub use_generic_irq: Option, +} + +impl std::default::Default for FsDeviceConfigInfo { + fn default() -> Self { + Self { + sock_path: String::default(), + tag: String::default(), + num_queues: 1, + queue_size: 1024, + cache_size: DEFAULT_CACHE_SIZE, + thread_pool_size: 0, + cache_policy: Self::default_cache_policy(), + writeback_cache: Self::default_writeback_cache(), + no_open: Self::default_no_open(), + fuse_killpriv_v2: Self::default_fuse_killpriv_v2(), + no_readdir: Self::default_no_readdir(), + xattr: Self::default_xattr(), + drop_sys_resource: Self::default_drop_sys_resource(), + mode: Self::default_fs_mode(), + rate_limiter: Some(RateLimiterConfigInfo::default()), + use_shared_irq: None, + use_generic_irq: None, + } + } +} + +impl FsDeviceConfigInfo { + /// The default mode is set to 'virtio' for 'virtio-fs' device. + pub fn default_fs_mode() -> String { + String::from("virtio") + } + + /// The default cache policy + pub fn default_cache_policy() -> String { + "always".to_string() + } + + /// The default setting of writeback cache + pub fn default_writeback_cache() -> bool { + true + } + + /// The default setting of no_open + pub fn default_no_open() -> bool { + true + } + + /// The default setting of killpriv_v2 + pub fn default_fuse_killpriv_v2() -> bool { + false + } + + /// The default setting of xattr + pub fn default_xattr() -> bool { + false + } + + /// The default setting of drop_sys_resource + pub fn default_drop_sys_resource() -> bool { + false + } + + /// The default setting of no_readdir + pub fn default_no_readdir() -> bool { + false + } + + /// The default setting of rate limiter + pub fn default_fs_rate_limiter() -> Option { + None + } +} + +/// Configuration information for virtio-fs. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct FsDeviceConfigUpdateInfo { + /// virtiofs mount tag name used inside the guest. + /// used as the device name during mount. + pub tag: String, + /// Rate Limiter for I/O operations. + pub rate_limiter: Option, +} + +impl FsDeviceConfigUpdateInfo { + /// Provides a `BucketUpdate` description for the bandwidth rate limiter. + pub fn bytes(&self) -> dbs_utils::rate_limiter::BucketUpdate { + get_bucket_update!(self, rate_limiter, bandwidth) + } + /// Provides a `BucketUpdate` description for the ops rate limiter. + pub fn ops(&self) -> dbs_utils::rate_limiter::BucketUpdate { + get_bucket_update!(self, rate_limiter, ops) + } +} + +impl ConfigItem for FsDeviceConfigInfo { + type Err = FsDeviceError; + + fn id(&self) -> &str { + &self.tag + } + + fn check_conflicts(&self, other: &Self) -> Result<(), FsDeviceError> { + if self.tag == other.tag { + Err(FsDeviceError::FsDeviceTagAlreadyExists(self.tag.clone())) + } else if self.mode.as_str() == "vhostuser" && self.sock_path == other.sock_path { + Err(FsDeviceError::FsDevicePathAlreadyExists( + self.sock_path.clone(), + )) + } else { + Ok(()) + } + } +} + +/// Configuration information of manipulating backend fs for a virtiofs device. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct FsMountConfigInfo { + /// Mount operations, mount, update, umount + pub ops: String, + /// The backend fs type to mount. + pub fstype: Option, + /// the source file/directory the backend fs points to + pub source: Option, + /// where the backend fs gets mounted + pub mountpoint: String, + /// backend fs config content in json format + pub config: Option, + /// virtiofs mount tag name used inside the guest. + /// used as the device name during mount. + pub tag: String, + /// Path to file that contains file lists that should be prefetched by rafs + pub prefetch_list_path: Option, + /// What size file supports dax + pub dax_threshold_size_kb: Option, +} + +pub(crate) type FsDeviceInfo = DeviceConfigInfo; + +impl ConfigItem for FsDeviceInfo { + type Err = FsDeviceError; + fn id(&self) -> &str { + &self.config.tag + } + + fn check_conflicts(&self, other: &Self) -> Result<(), FsDeviceError> { + if self.config.tag == other.config.tag { + Err(FsDeviceError::FsDeviceTagAlreadyExists( + self.config.tag.clone(), + )) + } else if self.config.sock_path == other.config.sock_path { + Err(FsDeviceError::FsDevicePathAlreadyExists( + self.config.sock_path.clone(), + )) + } else { + Ok(()) + } + } +} + +/// Wrapper for the collection that holds all the Fs Devices Configs +pub struct FsDeviceMgr { + /// A list of `FsDeviceConfig` objects. + pub(crate) info_list: DeviceConfigInfos, + pub(crate) use_shared_irq: bool, +} + +impl FsDeviceMgr { + /// Inserts `fs_cfg` in the shared-fs device configuration list. + pub fn insert_device( + device_mgr: &mut DeviceManager, + ctx: DeviceOpContext, + fs_cfg: FsDeviceConfigInfo, + ) -> std::result::Result<(), FsDeviceError> { + // It's too complicated to manage life cycle of shared-fs service process for hotplug. + if ctx.is_hotplug { + error!( + ctx.logger(), + "no support of shared-fs device hotplug"; + "subsystem" => "shared-fs", + "tag" => &fs_cfg.tag, + ); + return Err(FsDeviceError::UpdateNotAllowedPostBoot); + } + + info!( + ctx.logger(), + "add shared-fs device configuration"; + "subsystem" => "shared-fs", + "tag" => &fs_cfg.tag, + ); + device_mgr + .fs_manager + .lock() + .unwrap() + .info_list + .insert_or_update(&fs_cfg)?; + + Ok(()) + } + + /// Attaches all vhost-user-fs devices from the FsDevicesConfig. + pub fn attach_devices( + &mut self, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), FsDeviceError> { + let epoll_mgr = ctx + .epoll_mgr + .clone() + .ok_or(FsDeviceError::CreateFsDevice(virtio::Error::InvalidInput))?; + + for info in self.info_list.iter_mut() { + let device = Self::create_fs_device(&info.config, ctx, epoll_mgr.clone())?; + let mmio_device = DeviceManager::create_mmio_virtio_device( + device, + ctx, + info.config.use_shared_irq.unwrap_or(self.use_shared_irq), + info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(FsDeviceError::RegisterFsDevice)?; + + info.set_device(mmio_device); + } + + Ok(()) + } + + fn create_fs_device( + config: &FsDeviceConfigInfo, + ctx: &mut DeviceOpContext, + epoll_mgr: EpollManager, + ) -> std::result::Result { + match config.mode.as_str() { + "virtio" => Self::attach_virtio_fs_devices(config, ctx, epoll_mgr), + _ => Err(FsDeviceError::CreateFsDevice(virtio::Error::InvalidInput)), + } + } + + fn attach_virtio_fs_devices( + config: &FsDeviceConfigInfo, + ctx: &mut DeviceOpContext, + epoll_mgr: EpollManager, + ) -> std::result::Result { + info!( + ctx.logger(), + "add virtio-fs device configuration"; + "subsystem" => "virito-fs", + "tag" => &config.tag, + "dax_window_size" => &config.cache_size, + ); + + let limiter = if let Some(rlc) = config.rate_limiter.clone() { + Some( + rlc.try_into() + .map_err(FsDeviceError::RateLimterConfigInfoTryInto)?, + ) + } else { + None + }; + + let vm_as = ctx.get_vm_as().map_err(|e| { + error!(ctx.logger(), "virtio-fs get vm_as error: {:?}", e; + "subsystem" => "virito-fs"); + FsDeviceError::DeviceManager(e) + })?; + let address_space = match ctx.address_space.as_ref() { + Some(address_space) => address_space.clone(), + None => { + error!(ctx.logger(), "virtio-fs get address_space error"; "subsystem" => "virito-fs"); + return Err(FsDeviceError::AddressSpaceNotInitialized); + } + }; + let handler = DeviceVirtioRegionHandler { + vm_as, + address_space, + }; + + let device = Box::new( + virtio::fs::VirtioFs::new( + &config.tag, + config.num_queues, + config.queue_size, + config.cache_size, + &config.cache_policy, + config.thread_pool_size, + config.writeback_cache, + config.no_open, + config.fuse_killpriv_v2, + config.xattr, + config.drop_sys_resource, + config.no_readdir, + Box::new(handler), + epoll_mgr, + limiter, + ) + .map_err(FsDeviceError::CreateFsDevice)?, + ); + + Ok(device) + } + + /// Attach a backend fs to a VirtioFs device or detach a backend + /// fs from a Virtiofs device + pub fn manipulate_backend_fs( + device_mgr: &mut DeviceManager, + config: FsMountConfigInfo, + ) -> std::result::Result<(), FsDeviceError> { + let mut found = false; + + let mgr = &mut device_mgr.fs_manager.lock().unwrap(); + for info in mgr + .info_list + .iter() + .filter(|info| info.config.tag.as_str() == config.tag.as_str()) + { + found = true; + if let Some(device) = info.device.as_ref() { + if let Some(mmio_dev) = device.as_any().downcast_ref::() { + let mut guard = mmio_dev.state(); + let inner_dev = guard.get_inner_device_mut(); + if let Some(virtio_fs_dev) = inner_dev + .as_any_mut() + .downcast_mut::>() + { + return virtio_fs_dev + .manipulate_backend_fs( + config.source, + config.fstype, + &config.mountpoint, + config.config, + &config.ops, + config.prefetch_list_path, + config.dax_threshold_size_kb, + ) + .map(|_p| ()) + .map_err(|e| FsDeviceError::AttachBackendFailed(e.to_string())); + } + } + } + } + if !found { + Err(FsDeviceError::AttachBackendFailed( + "fs tag not found".to_string(), + )) + } else { + Ok(()) + } + } + + /// Gets the index of the device with the specified `tag` if it exists in the list. + pub fn get_index_of_tag(&self, tag: &str) -> Option { + self.info_list + .iter() + .position(|info| info.config.id().eq(tag)) + } + + /// Update the ratelimiter settings of a virtio fs device. + pub fn update_device_ratelimiters( + device_mgr: &mut DeviceManager, + new_cfg: FsDeviceConfigUpdateInfo, + ) -> std::result::Result<(), FsDeviceError> { + let mgr = &mut device_mgr.fs_manager.lock().unwrap(); + match mgr.get_index_of_tag(&new_cfg.tag) { + Some(index) => { + let config = &mut mgr.info_list[index].config; + config.rate_limiter = new_cfg.rate_limiter.clone(); + let device = mgr.info_list[index] + .device + .as_mut() + .ok_or_else(|| FsDeviceError::TagNotExists("".to_owned()))?; + + if let Some(mmio_dev) = device.as_any().downcast_ref::() { + let guard = mmio_dev.state(); + let inner_dev = guard.get_inner_device(); + if let Some(fs_dev) = inner_dev + .as_any() + .downcast_ref::>() + { + return fs_dev + .set_patch_rate_limiters(new_cfg.bytes(), new_cfg.ops()) + .map(|_p| ()) + .map_err(|_e| FsDeviceError::VirtioFsEpollHanderSendFail); + } + } + Ok(()) + } + None => Err(FsDeviceError::TagNotExists(new_cfg.tag)), + } + } +} + +impl Default for FsDeviceMgr { + /// Create a new `FsDeviceMgr` object.. + fn default() -> Self { + FsDeviceMgr { + info_list: DeviceConfigInfos::new(), + use_shared_irq: USE_SHARED_IRQ, + } + } +} diff --git a/src/dragonball/src/device_manager/memory_region_handler.rs b/src/dragonball/src/device_manager/memory_region_handler.rs new file mode 100644 index 0000000000..2be149ef97 --- /dev/null +++ b/src/dragonball/src/device_manager/memory_region_handler.rs @@ -0,0 +1,110 @@ +// Copyright 2022 Alibaba, Inc. or its affiliates. All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +use std::io; +use std::sync::Arc; + +use dbs_address_space::{AddressSpace, AddressSpaceRegion, AddressSpaceRegionType}; +use dbs_virtio_devices::{Error as VirtIoError, VirtioRegionHandler}; +use log::{debug, error}; +use vm_memory::{FileOffset, GuestAddressSpace, GuestMemoryRegion, GuestRegionMmap}; + +use crate::address_space_manager::GuestAddressSpaceImpl; + +/// This struct implements the VirtioRegionHandler trait, which inserts the memory +/// region of the virtio device into vm_as and address_space. +/// +/// * After region is inserted into the vm_as, the virtio device can read guest memory +/// data using vm_as.get_slice with GuestAddress. +/// +/// * Insert virtio memory into address_space so that the correct guest last address can +/// be found when initializing the e820 table. The e820 table is a table that describes +/// guest memory prepared before the guest startup. we need to config the correct guest +/// memory address and length in the table. The virtio device memory belongs to the MMIO +/// space and does not belong to the Guest Memory space. Therefore, it cannot be configured +/// into the e820 table. When creating AddressSpaceRegion we use +/// AddressSpaceRegionType::ReservedMemory type, in this way, address_space will know that +/// this region a special memory, it will don't put the this memory in e820 table. +/// +/// This function relies on the atomic-guest-memory feature. Without this feature enabled, memory +/// regions cannot be inserted into vm_as. Because the insert_region interface of vm_as does +/// not insert regions in place, but returns an array of inserted regions. We need to manually +/// replace this array of regions with vm_as, and that's what atomic-guest-memory feature does. +/// So we rely on the atomic-guest-memory feature here +pub struct DeviceVirtioRegionHandler { + pub(crate) vm_as: GuestAddressSpaceImpl, + pub(crate) address_space: AddressSpace, +} + +impl DeviceVirtioRegionHandler { + fn insert_address_space( + &mut self, + region: Arc, + ) -> std::result::Result<(), VirtIoError> { + let file_offset = match region.file_offset() { + // TODO: use from_arc + Some(f) => Some(FileOffset::new(f.file().try_clone()?, 0)), + None => None, + }; + + let as_region = Arc::new(AddressSpaceRegion::build( + AddressSpaceRegionType::DAXMemory, + region.start_addr(), + region.size() as u64, + None, + file_offset, + region.flags(), + false, + )); + + self.address_space.insert_region(as_region).map_err(|e| { + error!("inserting address apace error: {}", e); + // dbs-virtio-devices should not depend on dbs-address-space. + // So here io::Error is used instead of AddressSpaceError directly. + VirtIoError::IOError(io::Error::new( + io::ErrorKind::Other, + format!( + "invalid address space region ({0:#x}, {1:#x})", + region.start_addr().0, + region.len() + ), + )) + })?; + Ok(()) + } + + fn insert_vm_as( + &mut self, + region: Arc, + ) -> std::result::Result<(), VirtIoError> { + let vm_as_new = self.vm_as.memory().insert_region(region).map_err(|e| { + error!( + "DeviceVirtioRegionHandler failed to insert guest memory region: {:?}.", + e + ); + VirtIoError::InsertMmap(e) + })?; + // Do not expect poisoned lock here, so safe to unwrap(). + self.vm_as.lock().unwrap().replace(vm_as_new); + + Ok(()) + } +} + +impl VirtioRegionHandler for DeviceVirtioRegionHandler { + fn insert_region( + &mut self, + region: Arc, + ) -> std::result::Result<(), VirtIoError> { + debug!( + "add geust memory region to address_space/vm_as, new region: {:?}", + region + ); + + self.insert_address_space(region.clone())?; + self.insert_vm_as(region)?; + + Ok(()) + } +} diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 9313919f85..3cb0477e3d 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -71,6 +71,16 @@ pub mod virtio_net_dev_mgr; #[cfg(feature = "virtio-net")] use self::virtio_net_dev_mgr::VirtioNetDeviceMgr; +#[cfg(feature = "virtio-fs")] +/// virtio-block device manager +pub mod fs_dev_mgr; +#[cfg(feature = "virtio-fs")] +use self::fs_dev_mgr::FsDeviceMgr; +#[cfg(feature = "virtio-fs")] +mod memory_region_handler; +#[cfg(feature = "virtio-fs")] +pub use self::memory_region_handler::*; + macro_rules! info( ($l:expr, $($args:tt)+) => { slog::info!($l, $($args)+; slog::o!("subsystem" => "device_manager")) @@ -435,6 +445,9 @@ pub struct DeviceManager { #[cfg(feature = "virtio-net")] pub(crate) virtio_net_manager: VirtioNetDeviceMgr, + + #[cfg(feature = "virtio-fs")] + fs_manager: Arc>, } impl DeviceManager { @@ -463,6 +476,8 @@ impl DeviceManager { block_manager: BlockDeviceMgr::default(), #[cfg(feature = "virtio-net")] virtio_net_manager: VirtioNetDeviceMgr::default(), + #[cfg(feature = "virtio-fs")] + fs_manager: Arc::new(Mutex::new(FsDeviceMgr::default())), } } @@ -588,6 +603,14 @@ impl DeviceManager { .attach_devices(&mut ctx) .map_err(StartMicrovmError::BlockDeviceError)?; + #[cfg(feature = "virtio-fs")] + { + let mut fs_manager = self.fs_manager.lock().unwrap(); + fs_manager + .attach_devices(&mut ctx) + .map_err(StartMicrovmError::FsDeviceError)?; + } + #[cfg(feature = "virtio-net")] self.virtio_net_manager .attach_devices(&mut ctx) diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index 667405c5ec..2c6fbf6c44 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -168,6 +168,11 @@ pub enum StartMicroVmError { /// Virtio-net errors. #[error("virtio-net errors: {0}")] VirtioNetDeviceError(#[source] device_manager::virtio_net_dev_mgr::VirtioNetDeviceError), + + #[cfg(feature = "virtio-fs")] + /// Virtio-fs errors. + #[error("virtio-fs errors: {0}")] + FsDeviceError(#[source] device_manager::fs_dev_mgr::FsDeviceError), } /// Errors associated with starting the instance.