diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs index a5a9189490..985edd9789 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs @@ -10,8 +10,8 @@ use crate::BlockDevice; use crate::HybridVsockDevice; use crate::NetworkConfig; use crate::PciPath; +use crate::ShareFsConfig; use crate::ShareFsDevice; -use crate::ShareFsDeviceConfig; use crate::VfioDevice; use crate::VmmState; use anyhow::{anyhow, Context, Result}; @@ -108,6 +108,10 @@ impl CloudHypervisorInner { } } + pub(crate) async fn update_device(&mut self, _device: DeviceType) -> Result<()> { + Ok(()) + } + async fn handle_share_fs_device(&mut self, sharefs: ShareFsDevice) -> Result { let device: ShareFsDevice = sharefs.clone(); if device.config.fs_type != VIRTIO_FS { @@ -363,12 +367,12 @@ impl TryFrom for NetConfig { } #[derive(Debug)] pub struct ShareFsSettings { - cfg: ShareFsDeviceConfig, + cfg: ShareFsConfig, vm_path: String, } impl ShareFsSettings { - pub fn new(cfg: ShareFsDeviceConfig, vm_path: String) -> Self { + pub fn new(cfg: ShareFsConfig, vm_path: String) -> Self { ShareFsSettings { cfg, vm_path } } } diff --git a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs index 37f52d11cb..6f8b6c269d 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs @@ -89,6 +89,11 @@ impl Hypervisor for CloudHypervisor { inner.remove_device(device).await } + async fn update_device(&self, device: DeviceType) -> Result<()> { + let mut inner = self.inner.write().await; + inner.update_device(device).await + } + async fn get_agent_socket(&self) -> Result { let inner = self.inner.write().await; inner.get_agent_socket().await diff --git a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs index 4d14dea88f..8d71ecbccd 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs @@ -12,8 +12,8 @@ use tokio::sync::{Mutex, RwLock}; use crate::{ vhost_user_blk::VhostUserBlkDevice, BlockConfig, BlockDevice, HybridVsockDevice, Hypervisor, - NetworkDevice, VfioDevice, VhostUserConfig, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, - KATA_NVDIMM_DEV_TYPE, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM, + NetworkDevice, ShareFsDevice, VfioDevice, VhostUserConfig, KATA_BLK_DEV_TYPE, + KATA_MMIO_BLK_DEV_TYPE, KATA_NVDIMM_DEV_TYPE, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM, }; use super::{ @@ -226,6 +226,11 @@ impl DeviceManager { return Some(device_id.to_string()); } } + DeviceType::ShareFs(device) => { + if device.config.host_shared_path == host_path { + return Some(device_id.to_string()); + } + } _ => { // TODO: support find other device type continue; @@ -325,6 +330,22 @@ impl DeviceManager { // No need to do find device for hybrid vsock device. Arc::new(Mutex::new(HybridVsockDevice::new(&device_id, hvconfig))) } + DeviceConfig::ShareFsCfg(config) => { + // Try to find the sharefs device. If found, just return matched device id. + if let Some(device_id_matched) = + self.find_device(config.host_shared_path.clone()).await + { + info!( + sl!(), + "share-fs device with path:{:?} found, device id: {:?}", + config.host_shared_path, + device_id_matched + ); + return Ok(device_id_matched); + } + + Arc::new(Mutex::new(ShareFsDevice::new(&device_id, config))) + } _ => { return Err(anyhow!("invliad device type")); } @@ -437,6 +458,62 @@ impl DeviceManager { Err(anyhow!("ID are exhausted")) } + + async fn try_update_device(&mut self, updated_config: &DeviceConfig) -> Result<()> { + let device_id = match updated_config { + DeviceConfig::ShareFsCfg(config) => { + // Try to find the sharefs device. + // If found, just return the matched device id, otherwise return an error. + if let Some(device_id_matched) = + self.find_device(config.host_shared_path.clone()).await + { + device_id_matched + } else { + return Err(anyhow!( + "no matching device was found to do the update operation" + )); + } + } + // TODO for other Device Type + _ => { + return Err(anyhow!("update device with unsupported device type")); + } + }; + + // get the original device + let target_device = self + .get_device_info(&device_id) + .await + .context("get device failed")?; + + // update device with the updated configuration. + let updated_device: ArcMutexDevice = match target_device { + DeviceType::ShareFs(mut device) => { + if let DeviceConfig::ShareFsCfg(config) = updated_config { + // update the mount_config. + device.config.mount_config = config.mount_config.clone(); + } + Arc::new(Mutex::new(device)) + } + _ => return Err(anyhow!("update unsupported device type")), + }; + + // do handle update + if let Err(e) = updated_device + .lock() + .await + .update(self.hypervisor.as_ref()) + .await + { + debug!(sl!(), "update device with device id: {:?}", &device_id); + return Err(e); + } + + // Finally, we update the Map in Device Manager + self.devices.insert(device_id, updated_device); + + Ok(()) + } } // Many scenarios have similar steps when adding devices. so to reduce duplicated code, @@ -472,6 +549,19 @@ pub async fn do_handle_device( Ok(device_info) } +pub async fn do_update_device( + d: &RwLock, + updated_config: &DeviceConfig, +) -> Result<()> { + d.write() + .await + .try_update_device(updated_config) + .await + .context("failed to update device")?; + + Ok(()) +} + pub async fn get_block_driver(d: &RwLock) -> String { d.read().await.get_block_driver().await } diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs index 0818d7cec2..c8eb70e564 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs @@ -20,8 +20,7 @@ pub use virtio_blk::{ VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM, }; pub use virtio_fs::{ - ShareFsDevice, ShareFsDeviceConfig, ShareFsMountConfig, ShareFsMountDevice, ShareFsMountType, - ShareFsOperation, + ShareFsConfig, ShareFsDevice, ShareFsMountConfig, ShareFsMountOperation, ShareFsMountType, }; pub use virtio_net::{Address, Backend, NetworkConfig, NetworkDevice}; pub use virtio_vsock::{ diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs index a689bcb355..07890de659 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs @@ -543,6 +543,11 @@ impl Device for VfioDevice { Ok(device_index) } + async fn update(&mut self, _h: &dyn hypervisor) -> Result<()> { + // There's no need to do update for vfio device + Ok(()) + } + async fn increase_attach_count(&mut self) -> Result { match self.attach_count { 0 => { diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user_blk.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user_blk.rs index 0912f89f1a..5150f19563 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user_blk.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user_blk.rs @@ -86,6 +86,11 @@ impl Device for VhostUserBlkDevice { Ok(Some(self.config.index)) } + async fn update(&mut self, _h: &dyn hypervisor) -> Result<()> { + // There's no need to do update for vhost-user-blk + Ok(()) + } + async fn get_device_info(&self) -> DeviceType { DeviceType::VhostUserBlk(self.clone()) } diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs index 1de3ff3897..a93f8553d9 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs @@ -114,6 +114,11 @@ impl Device for BlockDevice { Ok(Some(self.config.index)) } + async fn update(&mut self, _h: &dyn hypervisor) -> Result<()> { + // There's no need to do update for virtio-blk + Ok(()) + } + async fn get_device_info(&self) -> DeviceType { DeviceType::Block(self.clone()) } diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs index 0a97845e71..e968606de3 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs @@ -4,21 +4,28 @@ // SPDX-License-Identifier: Apache-2.0 // -#[derive(Copy, Clone, Debug)] -pub enum ShareFsOperation { +use anyhow::{Context, Result}; +use async_trait::async_trait; + +use crate::device::{hypervisor, Device, DeviceType}; + +#[derive(Copy, Clone, Debug, Default)] +pub enum ShareFsMountOperation { + #[default] Mount, Umount, Update, } -#[derive(Debug, Clone)] +#[derive(Debug, Default, Clone)] pub enum ShareFsMountType { + #[default] PASSTHROUGH, RAFS, } /// ShareFsMountConfig: share fs mount config -#[derive(Debug, Clone)] +#[derive(Clone, Debug, Default)] pub struct ShareFsMountConfig { /// source: the passthrough fs exported dir or rafs meta file of rafs pub source: String, @@ -36,20 +43,19 @@ pub struct ShareFsMountConfig { pub tag: String, /// op: the operation to take, e.g. mount, umount or update - pub op: ShareFsOperation, + pub op: ShareFsMountOperation, /// prefetch_list_path: path to file that contains file lists that should be prefetched by rafs pub prefetch_list_path: Option, } -#[derive(Debug, Clone)] -pub struct ShareFsMountDevice { - pub config: ShareFsMountConfig, -} +/// ShareFsConfig: Sharefs config for virtio-fs devices and their corresponding mount configurations, +/// facilitating mount/umount/update operations. +#[derive(Clone, Debug, Default)] +pub struct ShareFsConfig { + /// host_shared_path: the upperdir of the passthrough fs exported dir or rafs meta file of rafs + pub host_shared_path: String, -/// ShareFsDeviceConfig: share fs device config -#[derive(Debug, Clone)] -pub struct ShareFsDeviceConfig { /// fs_type: virtiofs or inline-virtiofs pub fs_type: String, @@ -59,9 +65,6 @@ pub struct ShareFsDeviceConfig { /// mount_tag: a label used as a hint to the guest. pub mount_tag: String, - /// host_path: the host filesystem path for this volume. - pub host_path: String, - /// queue_size: queue size pub queue_size: u64, @@ -70,9 +73,65 @@ pub struct ShareFsDeviceConfig { /// options: virtiofs device's config options. pub options: Vec, + + /// mount config for sharefs mount/umount/update + pub mount_config: Option, } -#[derive(Debug, Clone)] +#[derive(Debug, Default, Clone)] pub struct ShareFsDevice { - pub config: ShareFsDeviceConfig, + /// device id for sharefs device in device manager + pub device_id: String, + + /// config for sharefs device + pub config: ShareFsConfig, +} + +impl ShareFsDevice { + // new creates a share-fs device + pub fn new(device_id: &str, config: &ShareFsConfig) -> Self { + Self { + device_id: device_id.to_string(), + config: config.clone(), + } + } +} + +#[async_trait] +impl Device for ShareFsDevice { + async fn attach(&mut self, h: &dyn hypervisor) -> Result<()> { + h.add_device(DeviceType::ShareFs(self.clone())) + .await + .context("add share-fs device.")?; + + Ok(()) + } + + async fn detach(&mut self, _h: &dyn hypervisor) -> Result> { + // no need to detach share-fs device + + Ok(None) + } + + async fn update(&mut self, h: &dyn hypervisor) -> Result<()> { + h.update_device(DeviceType::ShareFs(self.clone())) + .await + .context("update share-fs device.") + } + + async fn get_device_info(&self) -> DeviceType { + DeviceType::ShareFs(self.clone()) + } + + async fn increase_attach_count(&mut self) -> Result { + // share-fs devices will not be attached multiple times, Just return Ok(false) + + Ok(false) + } + + async fn decrease_attach_count(&mut self) -> Result { + // share-fs devices will not be detached multiple times, Just return Ok(false) + + Ok(false) + } } diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs index c05503ab53..eac28d81d9 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs @@ -95,6 +95,11 @@ impl Device for NetworkDevice { Ok(Some(self.config.index)) } + async fn update(&mut self, _h: &dyn hypervisor) -> Result<()> { + // There's no need to do update for network device + Ok(()) + } + async fn get_device_info(&self) -> DeviceType { DeviceType::Network(self.clone()) } diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs index 98f48e29a9..6ca8879536 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs @@ -62,6 +62,11 @@ impl Device for HybridVsockDevice { Ok(None) } + async fn update(&mut self, _h: &dyn hypervisor) -> Result<()> { + // There's no need to do update for hvsock device + Ok(()) + } + async fn get_device_info(&self) -> DeviceType { DeviceType::HybridVsock(self.clone()) } diff --git a/src/runtime-rs/crates/hypervisor/src/device/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/mod.rs index 59bb7540d8..b40f784ddb 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/mod.rs @@ -9,8 +9,8 @@ use std::fmt; use crate::device::driver::vhost_user_blk::VhostUserBlkDevice; use crate::{ BlockConfig, BlockDevice, HybridVsockConfig, HybridVsockDevice, Hypervisor as hypervisor, - NetworkConfig, NetworkDevice, ShareFsDevice, ShareFsDeviceConfig, ShareFsMountConfig, - ShareFsMountDevice, VfioConfig, VfioDevice, VhostUserConfig, VsockConfig, + NetworkConfig, NetworkDevice, ShareFsConfig, ShareFsDevice, VfioConfig, VfioDevice, + VhostUserConfig, VsockConfig, }; use anyhow::Result; use async_trait::async_trait; @@ -24,9 +24,8 @@ pub enum DeviceConfig { BlockCfg(BlockConfig), VhostUserBlkCfg(VhostUserConfig), NetworkCfg(NetworkConfig), - ShareFsCfg(ShareFsDeviceConfig), + ShareFsCfg(ShareFsConfig), VfioCfg(VfioConfig), - ShareFsMountCfg(ShareFsMountConfig), VsockCfg(VsockConfig), HybridVsockCfg(HybridVsockConfig), } @@ -38,7 +37,6 @@ pub enum DeviceType { Vfio(VfioDevice), Network(NetworkDevice), ShareFs(ShareFsDevice), - ShareFsMount(ShareFsMountDevice), HybridVsock(HybridVsockDevice), } @@ -54,6 +52,8 @@ pub trait Device: std::fmt::Debug + Send + Sync { async fn attach(&mut self, h: &dyn hypervisor) -> Result<()>; // detach is to unplug device from VM async fn detach(&mut self, h: &dyn hypervisor) -> Result>; + // update is to do update for some device + async fn update(&mut self, h: &dyn hypervisor) -> Result<()>; // get_device_info returns device config async fn get_device_info(&self) -> DeviceType; // increase_attach_count is used to increase the attach count for a device diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs index 90c1ae3169..7f86b4d3a8 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs @@ -14,8 +14,8 @@ use dragonball::device_manager::blk_dev_mgr::BlockDeviceType; use super::DragonballInner; use crate::{ - device::DeviceType, HybridVsockConfig, NetworkConfig, ShareFsDeviceConfig, ShareFsMountConfig, - ShareFsMountType, ShareFsOperation, VfioBusMode, VfioDevice, VmmState, JAILER_ROOT, + device::DeviceType, HybridVsockConfig, NetworkConfig, ShareFsConfig, ShareFsMountConfig, + ShareFsMountOperation, ShareFsMountType, VfioBusMode, VfioDevice, VmmState, JAILER_ROOT, }; const MB_TO_B: u32 = 1024 * 1024; @@ -67,9 +67,6 @@ impl DragonballInner { DeviceType::ShareFs(sharefs) => self .add_share_fs_device(&sharefs.config) .context("add share fs device"), - DeviceType::ShareFsMount(sharefs_mount) => self - .add_share_fs_mount(&sharefs_mount.config) - .context("add share fs mount"), } } @@ -101,6 +98,18 @@ impl DragonballInner { } } + pub(crate) async fn update_device(&mut self, device: DeviceType) -> Result<()> { + info!(sl!(), "dragonball update device {:?}", &device); + match device { + DeviceType::ShareFs(sharefs_mount) => { + // It's safe to unwrap mount config as mount_config is always there. + self.add_share_fs_mount(&sharefs_mount.config.mount_config.unwrap()) + .context("update share-fs device with mount operation.") + } + _ => Err(anyhow!("unsupported device {:?} to update.", device)), + } + } + fn add_vfio_device(&mut self, device: &VfioDevice) -> Result<()> { let vfio_device = device.clone(); @@ -285,7 +294,7 @@ impl DragonballInner { Ok(()) } - fn add_share_fs_device(&self, config: &ShareFsDeviceConfig) -> Result<()> { + fn add_share_fs_device(&self, config: &ShareFsConfig) -> Result<()> { let mut fs_cfg = FsDeviceConfigInfo { sock_path: config.sock_path.clone(), tag: config.mount_tag.clone(), @@ -337,9 +346,9 @@ impl DragonballInner { fn add_share_fs_mount(&mut self, config: &ShareFsMountConfig) -> Result<()> { let ops = match config.op { - ShareFsOperation::Mount => "mount", - ShareFsOperation::Umount => "umount", - ShareFsOperation::Update => "update", + ShareFsMountOperation::Mount => "mount", + ShareFsMountOperation::Umount => "umount", + ShareFsMountOperation::Update => "update", }; let fstype = match config.fstype { diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs index f4cb798bc4..399f8ca1b3 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs @@ -110,6 +110,11 @@ impl Hypervisor for Dragonball { inner.remove_device(device).await } + async fn update_device(&self, device: DeviceType) -> Result<()> { + let mut inner = self.inner.write().await; + inner.update_device(device).await + } + async fn get_agent_socket(&self) -> Result { let inner = self.inner.read().await; inner.get_agent_socket().await diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs index ec3b7b6fbd..4311fdf98d 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs @@ -26,7 +26,7 @@ use nix::sched::{setns, CloneFlags}; use seccompiler::BpfProgram; use vmm_sys_util::eventfd::EventFd; -use crate::ShareFsOperation; +use crate::ShareFsMountOperation; pub enum Request { Sync(VmmAction), @@ -238,7 +238,7 @@ impl VmmInstance { Ok(()) } - pub fn patch_fs(&self, cfg: &FsMountConfigInfo, op: ShareFsOperation) -> Result<()> { + pub fn patch_fs(&self, cfg: &FsMountConfigInfo, op: ShareFsMountOperation) -> Result<()> { self.handle_request(Request::Sync(VmmAction::ManipulateFsBackendFs(cfg.clone()))) .with_context(|| { format!( diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index deb7c92428..737133de50 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -87,6 +87,7 @@ pub trait Hypervisor: std::fmt::Debug + Send + Sync { // device manager async fn add_device(&self, device: DeviceType) -> Result; async fn remove_device(&self, device: DeviceType) -> Result<()>; + async fn update_device(&self, device: DeviceType) -> Result<()>; // utils async fn get_agent_socket(&self) -> Result; diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index 80f86a56eb..e4a3c4e0b0 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -155,4 +155,10 @@ impl QemuInner { info!(sl!(), "QemuInner::remove_device() {} ", device); todo!() } + + pub(crate) async fn update_device(&mut self, device: DeviceType) -> Result<()> { + info!(sl!(), "QemuInner::update_device() {:?}", &device); + + Ok(()) + } } diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs index d26468632e..65b4b6e57d 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs @@ -84,6 +84,11 @@ impl Hypervisor for Qemu { inner.remove_device(device).await } + async fn update_device(&self, device: DeviceType) -> Result<()> { + let mut inner = self.inner.write().await; + inner.update_device(device).await + } + async fn get_agent_socket(&self) -> Result { let inner = self.inner.read().await; inner.get_agent_socket().await diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index 53b3ac4a12..2ee6a5a054 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -102,7 +102,10 @@ impl ResourceManagerInner { { let share_fs = share_fs::new(&self.sid, &c).context("new share fs")?; share_fs - .setup_device_before_start_vm(self.hypervisor.as_ref()) + .setup_device_before_start_vm( + self.hypervisor.as_ref(), + &self.device_manager, + ) .await .context("setup share fs device before start vm")?; @@ -212,7 +215,7 @@ impl ResourceManagerInner { pub async fn setup_after_start_vm(&mut self) -> Result<()> { if let Some(share_fs) = self.share_fs.as_ref() { share_fs - .setup_device_after_start_vm(self.hypervisor.as_ref()) + .setup_device_after_start_vm(self.hypervisor.as_ref(), &self.device_manager) .await .context("setup share fs device after start vm")?; } @@ -227,6 +230,7 @@ impl ResourceManagerInner { .context("handle neighbors")?; self.handle_routes(network).await.context("handle routes")?; } + Ok(()) } diff --git a/src/runtime-rs/crates/resource/src/rootfs/mod.rs b/src/runtime-rs/crates/resource/src/rootfs/mod.rs index 75510ed027..8975929c6b 100644 --- a/src/runtime-rs/crates/resource/src/rootfs/mod.rs +++ b/src/runtime-rs/crates/resource/src/rootfs/mod.rs @@ -23,6 +23,7 @@ use self::{block_rootfs::is_block_rootfs, nydus_rootfs::NYDUS_ROOTFS_TYPE}; const ROOTFS: &str = "rootfs"; const HYBRID_ROOTFS_LOWER_DIR: &str = "rootfs_lower"; const TYPE_OVERLAY_FS: &str = "overlay"; + #[async_trait] pub trait Rootfs: Send + Sync { async fn get_guest_rootfs_path(&self) -> Result; @@ -102,9 +103,16 @@ impl RootFsResource { // handle nydus rootfs let share_rootfs: Arc = if layer.fs_type == NYDUS_ROOTFS_TYPE { Arc::new( - nydus_rootfs::NydusRootfs::new(share_fs, h, sid, cid, layer) - .await - .context("new nydus rootfs")?, + nydus_rootfs::NydusRootfs::new( + device_manager, + share_fs, + h, + sid, + cid, + layer, + ) + .await + .context("new nydus rootfs")?, ) } // handle sharefs rootfs diff --git a/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs index e6419bf7ec..a6c40489a8 100644 --- a/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs +++ b/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs @@ -39,6 +39,7 @@ pub(crate) struct NydusRootfs { impl NydusRootfs { pub async fn new( + d: &RwLock, share_fs: &Arc, h: &dyn Hypervisor, sid: &str, @@ -61,7 +62,8 @@ impl NydusRootfs { // rafs mount the metadata of nydus rootfs let rafs_mnt = do_get_guest_share_path(HYBRID_ROOTFS_LOWER_DIR, cid, true); rafs_mount( - h, + d, + sid, rafs_meta.to_string(), rafs_mnt, extra_options.config.clone(), diff --git a/src/runtime-rs/crates/resource/src/share_fs/mod.rs b/src/runtime-rs/crates/resource/src/share_fs/mod.rs index 4d70a6c7b4..81d35b09d5 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/mod.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/mod.rs @@ -22,12 +22,15 @@ pub mod sandbox_bind_mounts; use std::{collections::HashMap, fmt::Debug, path::PathBuf, sync::Arc}; -use agent::Storage; use anyhow::{anyhow, Context, Ok, Result}; use async_trait::async_trait; -use hypervisor::Hypervisor; +use tokio::sync::RwLock; + +use agent::Storage; use kata_types::config::hypervisor::SharedFsInfo; +use hypervisor::{device::device_manager::DeviceManager, Hypervisor}; + const VIRTIO_FS: &str = "virtio-fs"; const _VIRTIO_FS_NYDUS: &str = "virtio-fs-nydus"; const INLINE_VIRTIO_FS: &str = "inline-virtio-fs"; @@ -45,8 +48,16 @@ const RAFS_DIR: &str = "rafs"; #[async_trait] pub trait ShareFs: Send + Sync { fn get_share_fs_mount(&self) -> Arc; - async fn setup_device_before_start_vm(&self, h: &dyn Hypervisor) -> Result<()>; - async fn setup_device_after_start_vm(&self, h: &dyn Hypervisor) -> Result<()>; + async fn setup_device_before_start_vm( + &self, + h: &dyn Hypervisor, + d: &RwLock, + ) -> Result<()>; + async fn setup_device_after_start_vm( + &self, + h: &dyn Hypervisor, + d: &RwLock, + ) -> Result<()>; async fn get_storages(&self) -> Result>; fn mounted_info_set(&self) -> Arc>>; } diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs index c0449fa3d0..96b2676e97 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs @@ -7,18 +7,18 @@ use std::path::Path; use anyhow::{Context, Result}; +use nix::mount::MsFlags; +use tokio::sync::RwLock; + use hypervisor::{ device::{ - driver::{ - ShareFsDevice, ShareFsMountConfig, ShareFsMountDevice, ShareFsMountType, - ShareFsOperation, - }, - DeviceType, + device_manager::{do_handle_device, do_update_device, DeviceManager}, + driver::{ShareFsMountConfig, ShareFsMountOperation, ShareFsMountType}, + DeviceConfig, }, - Hypervisor, ShareFsDeviceConfig, + ShareFsConfig, }; use kata_sys_util::mount; -use nix::mount::MsFlags; use super::{utils, PASSTHROUGH_FS_DIR}; @@ -35,7 +35,7 @@ pub(crate) fn generate_sock_path(root: &str) -> String { } pub(crate) async fn prepare_virtiofs( - h: &dyn Hypervisor, + d: &RwLock, fs_type: &str, id: &str, root: &str, @@ -49,24 +49,26 @@ pub(crate) async fn prepare_virtiofs( mount::bind_mount_unchecked(&host_rw_dest, &host_ro_dest, true, MsFlags::MS_SLAVE) .context("bind mount shared_fs directory")?; - let share_fs_device = ShareFsDevice { - config: ShareFsDeviceConfig { - sock_path: generate_sock_path(root), - mount_tag: String::from(MOUNT_GUEST_TAG), - host_path: String::from(host_ro_dest.to_str().unwrap()), - fs_type: fs_type.to_string(), - queue_size: 0, - queue_num: 0, - options: vec![], - }, + let sharefs_config = ShareFsConfig { + host_shared_path: host_ro_dest.display().to_string(), + sock_path: generate_sock_path(root), + mount_tag: String::from(MOUNT_GUEST_TAG), + fs_type: fs_type.to_string(), + queue_size: 0, + queue_num: 0, + options: vec![], + mount_config: None, }; - h.add_device(DeviceType::ShareFs(share_fs_device)) + + // create and insert virtio-fs device into Guest + do_handle_device(d, &DeviceConfig::ShareFsCfg(sharefs_config)) .await - .context("add device")?; + .context("do add virtio-fs device failed.")?; + Ok(()) } -pub(crate) async fn setup_inline_virtiofs(id: &str, h: &dyn Hypervisor) -> Result<()> { +pub(crate) async fn setup_inline_virtiofs(d: &RwLock, id: &str) -> Result<()> { // - source is the absolute path of PASSTHROUGH_FS_DIR on host, e.g. // /run/kata-containers/shared/sandboxes//passthrough // - mount point is the path relative to KATA_GUEST_SHARE_DIR in guest @@ -75,34 +77,39 @@ pub(crate) async fn setup_inline_virtiofs(id: &str, h: &dyn Hypervisor) -> Resul let rw_source = utils::get_host_rw_shared_path(id).join(PASSTHROUGH_FS_DIR); utils::ensure_dir_exist(&rw_source).context("ensure directory exist")?; - let ro_source = utils::get_host_ro_shared_path(id).join(PASSTHROUGH_FS_DIR); - let source = String::from(ro_source.to_str().unwrap()); + let host_ro_shared_path = utils::get_host_ro_shared_path(id); + let source = host_ro_shared_path + .join(PASSTHROUGH_FS_DIR) + .display() + .to_string(); - let virtio_fs = ShareFsMountDevice { - config: ShareFsMountConfig { - source: source.clone(), - fstype: ShareFsMountType::PASSTHROUGH, - mount_point: mnt, - config: None, - tag: String::from(MOUNT_GUEST_TAG), - op: ShareFsOperation::Mount, - prefetch_list_path: None, - }, + let virtiofs_mount = ShareFsMountConfig { + source: source.clone(), + fstype: ShareFsMountType::PASSTHROUGH, + mount_point: mnt, + config: None, + tag: String::from(MOUNT_GUEST_TAG), + op: ShareFsMountOperation::Mount, + prefetch_list_path: None, }; - let result = h - .add_device(DeviceType::ShareFsMount(virtio_fs)) - .await - .with_context(|| format!("fail to attach passthrough fs {:?}", source)); + let sharefs_config = ShareFsConfig { + host_shared_path: host_ro_shared_path.display().to_string(), + mount_config: Some(virtiofs_mount), + ..Default::default() + }; - match result { - Ok(_) => Ok(()), - Err(e) => Err(e), - } + // update virtio-fs device with ShareFsMountConfig + do_update_device(d, &DeviceConfig::ShareFsCfg(sharefs_config)) + .await + .context("fail to attach passthrough fs.")?; + + Ok(()) } pub async fn rafs_mount( - h: &dyn Hypervisor, + d: &RwLock, + sid: &str, rafs_meta: String, rafs_mnt: String, config_content: String, @@ -112,19 +119,28 @@ pub async fn rafs_mount( sl!(), "Attaching rafs meta file {} to virtio-fs device, rafs mount point {}", rafs_meta, rafs_mnt ); - let virtio_fs = ShareFsMountDevice { - config: ShareFsMountConfig { - source: rafs_meta.clone(), - fstype: ShareFsMountType::RAFS, - mount_point: rafs_mnt, - config: Some(config_content), - tag: String::from(MOUNT_GUEST_TAG), - op: ShareFsOperation::Mount, - prefetch_list_path, - }, + + let rafs_config = ShareFsMountConfig { + source: rafs_meta.clone(), + fstype: ShareFsMountType::RAFS, + mount_point: rafs_mnt, + config: Some(config_content), + tag: String::from(MOUNT_GUEST_TAG), + op: ShareFsMountOperation::Mount, + prefetch_list_path, }; - h.add_device(DeviceType::ShareFsMount(virtio_fs)) + + let host_shared_path = utils::get_host_ro_shared_path(sid).display().to_string(); + let sharefs_config = ShareFsConfig { + host_shared_path, + mount_config: Some(rafs_config), + ..Default::default() + }; + + // update virtio-fs device with ShareFsMountConfig + do_update_device(d, &DeviceConfig::ShareFsCfg(sharefs_config)) .await .with_context(|| format!("fail to attach rafs {:?}", rafs_meta))?; + Ok(()) } diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs index 5dddefbfdd..fdac7c57bc 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs @@ -6,12 +6,13 @@ use std::collections::HashMap; -use agent::Storage; use anyhow::{Context, Result}; use async_trait::async_trait; -use hypervisor::Hypervisor; +use tokio::sync::{Mutex, RwLock}; + +use agent::Storage; +use hypervisor::{device::device_manager::DeviceManager, Hypervisor}; use kata_types::config::hypervisor::SharedFsInfo; -use tokio::sync::Mutex; use super::{ share_virtio_fs::{ @@ -52,19 +53,30 @@ impl ShareFs for ShareVirtioFsInline { self.share_fs_mount.clone() } - async fn setup_device_before_start_vm(&self, h: &dyn Hypervisor) -> Result<()> { - prepare_virtiofs(h, INLINE_VIRTIO_FS, &self.config.id, "") + async fn setup_device_before_start_vm( + &self, + _h: &dyn Hypervisor, + d: &RwLock, + ) -> Result<()> { + prepare_virtiofs(d, INLINE_VIRTIO_FS, &self.config.id, "") .await .context("prepare virtiofs")?; + Ok(()) } - async fn setup_device_after_start_vm(&self, h: &dyn Hypervisor) -> Result<()> { - setup_inline_virtiofs(&self.config.id, h) + async fn setup_device_after_start_vm( + &self, + _h: &dyn Hypervisor, + d: &RwLock, + ) -> Result<()> { + setup_inline_virtiofs(d, &self.config.id) .await .context("setup inline virtiofs")?; + Ok(()) } + async fn get_storages(&self) -> Result> { // setup storage let mut storages: Vec = Vec::new(); diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs index db421ada36..d281403a79 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs @@ -6,15 +6,8 @@ use std::{collections::HashMap, process::Stdio, sync::Arc}; -use crate::share_fs::share_virtio_fs::{ - prepare_virtiofs, FS_TYPE_VIRTIO_FS, KATA_VIRTIO_FS_DEV_TYPE, MOUNT_GUEST_TAG, -}; -use crate::share_fs::{KATA_GUEST_SHARE_DIR, VIRTIO_FS}; -use agent::Storage; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; -use hypervisor::Hypervisor; -use kata_types::config::hypervisor::SharedFsInfo; use tokio::{ io::{AsyncBufReadExt, BufReader}, process::{Child, Command}, @@ -24,10 +17,20 @@ use tokio::{ }, }; +use agent::Storage; +use hypervisor::{device::device_manager::DeviceManager, Hypervisor}; +use kata_types::config::hypervisor::SharedFsInfo; + use super::{ share_virtio_fs::generate_sock_path, utils::ensure_dir_exist, utils::get_host_ro_shared_path, virtio_fs_share_mount::VirtiofsShareMount, MountedInfo, ShareFs, ShareFsMount, }; +use crate::share_fs::{ + share_virtio_fs::{ + prepare_virtiofs, FS_TYPE_VIRTIO_FS, KATA_VIRTIO_FS_DEV_TYPE, MOUNT_GUEST_TAG, + }, + KATA_GUEST_SHARE_DIR, VIRTIO_FS, +}; #[derive(Debug, Clone)] pub struct ShareVirtioFsStandaloneConfig { @@ -172,15 +175,24 @@ impl ShareFs for ShareVirtioFsStandalone { self.share_fs_mount.clone() } - async fn setup_device_before_start_vm(&self, h: &dyn Hypervisor) -> Result<()> { - prepare_virtiofs(h, VIRTIO_FS, &self.config.id, &h.get_jailer_root().await?) + async fn setup_device_before_start_vm( + &self, + h: &dyn Hypervisor, + d: &RwLock, + ) -> Result<()> { + prepare_virtiofs(d, VIRTIO_FS, &self.config.id, &h.get_jailer_root().await?) .await .context("prepare virtiofs")?; self.setup_virtiofsd(h).await.context("setup virtiofsd")?; + Ok(()) } - async fn setup_device_after_start_vm(&self, _h: &dyn Hypervisor) -> Result<()> { + async fn setup_device_after_start_vm( + &self, + _h: &dyn Hypervisor, + _d: &RwLock, + ) -> Result<()> { Ok(()) }