diff --git a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs index 8d6d066520..cdce8956b1 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs @@ -66,7 +66,7 @@ pub struct DeviceManager { } impl DeviceManager { - pub async fn new(hypervisor: Arc) -> Result { + pub fn new(hypervisor: Arc) -> Result { let devices = HashMap::::new(); Ok(DeviceManager { devices, @@ -75,7 +75,7 @@ impl DeviceManager { }) } - pub async fn new_device(&mut self, device_config: DeviceConfig) -> Result { + pub async fn new_device(&mut self, device_config: &DeviceConfig) -> Result { let device_id = if let Some(dev) = self.find_device(&device_config).await { dev } else { @@ -86,11 +86,10 @@ impl DeviceManager { Ok(device_id) } - pub async fn try_add_device(&mut self, device_id: String) -> Result<()> { - // get device + pub async fn try_add_device(&mut self, device_id: &str) -> Result<()> { let device = self .devices - .get(&device_id) + .get(device_id) .context("failed to find device")?; // attach device let result = device.lock().await.attach(self.hypervisor.as_ref()).await; @@ -99,12 +98,22 @@ impl DeviceManager { if let DeviceConfig::Block(config) = device.lock().await.get_device_info().await { self.shared_info.release_device_index(config.index); }; - self.devices.remove(&device_id); + self.devices.remove(device_id); return Err(e); } Ok(()) } + pub async fn get_device_info(&self, device_id: &str) -> Result { + if let Some(dev) = self.devices.get(device_id) { + return Ok(dev.lock().await.get_device_info().await); + } + Err(anyhow!( + "device with specified ID hasn't been created. {}", + device_id + )) + } + async fn find_device(&self, device_config: &DeviceConfig) -> Option { for (device_id, dev) in &self.devices { match dev.lock().await.get_device_info().await { diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs index 4c43be0104..0366257d17 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs @@ -6,14 +6,13 @@ use std::{fs, path::Path, process::Command}; -use crate::{driver::hypervisor, DeviceConfig}; +use crate::Hypervisor as hypervisor; +use crate::{device::Device, DeviceConfig}; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] use anyhow::anyhow; use anyhow::{Context, Result}; use async_trait::async_trait; -use crate::Device; - fn override_driver(bdf: &str, driver: &str) -> Result<()> { let driver_override = format!("/sys/bus/pci/devices/{}/driver_override", bdf); fs::write(&driver_override, driver) @@ -152,11 +151,11 @@ pub fn bind_device_to_host(bdf: &str, host_driver: &str, _vendor_device_id: &str #[async_trait] impl Device for VfioConfig { - async fn attach(&self, _h: &dyn hypervisor) -> Result<()> { + async fn attach(&mut self, _h: &dyn hypervisor) -> Result<()> { todo!() } - async fn detach(&self, _h: &dyn hypervisor) -> Result { + async fn detach(&mut self, _h: &dyn hypervisor) -> Result> { todo!() } diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs index 53d4a0e019..bfde23239f 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs @@ -4,8 +4,8 @@ // SPDX-License-Identifier: Apache-2.0 // -use crate::Device; -use crate::{driver::hypervisor, DeviceConfig}; +use crate::Hypervisor as hypervisor; +use crate::{device::Device, DeviceConfig}; use anyhow::Result; use async_trait::async_trait; @@ -31,11 +31,11 @@ pub struct VhostUserConfig { #[async_trait] impl Device for VhostUserConfig { - async fn attach(&self, _h: &dyn hypervisor) -> Result<()> { + async fn attach(&mut self, _h: &dyn hypervisor) -> Result<()> { todo!() } - async fn detach(&self, _h: &dyn hypervisor) -> Result { + async fn detach(&mut self, _h: &dyn hypervisor) -> Result> { todo!() } diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index 6c6e4067af..16f88063bf 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -10,12 +10,12 @@ use crate::{network::NetworkConfig, resource_persist::ResourceState}; use agent::{Agent, Storage}; use anyhow::{anyhow, Context, Ok, Result}; use async_trait::async_trait; -use hypervisor::Hypervisor; +use hypervisor::{device::device_manager::DeviceManager, Hypervisor}; use kata_types::config::TomlConfig; use kata_types::mount::Mount; use oci::LinuxResources; use persist::sandbox_persist::Persist; -use tokio::runtime; +use tokio::{runtime, sync::RwLock}; use crate::{ cgroups::{CgroupArgs, CgroupsResource}, @@ -32,6 +32,7 @@ pub(crate) struct ResourceManagerInner { toml_config: Arc, agent: Arc, hypervisor: Arc, + device_manager: Arc>, network: Option>, share_fs: Option>, @@ -48,11 +49,17 @@ impl ResourceManagerInner { toml_config: Arc, ) -> Result { let cgroups_resource = CgroupsResource::new(sid, &toml_config)?; + + // create device manager + let dev_manager = + DeviceManager::new(hypervisor.clone()).context("failed to create device manager")?; + Ok(Self { sid: sid.to_string(), toml_config, agent, hypervisor, + device_manager: Arc::new(RwLock::new(dev_manager)), network: None, share_fs: None, rootfs_resource: RootFsResource::new(), @@ -212,6 +219,7 @@ impl ResourceManagerInner { self.rootfs_resource .handler_rootfs( &self.share_fs, + self.device_manager.as_ref(), self.hypervisor.as_ref(), &self.sid, cid, @@ -298,7 +306,8 @@ impl Persist for ResourceManagerInner { Ok(Self { sid: resource_args.sid, agent: resource_args.agent, - hypervisor: resource_args.hypervisor, + hypervisor: resource_args.hypervisor.clone(), + device_manager: Arc::new(RwLock::new(DeviceManager::new(resource_args.hypervisor)?)), network: None, share_fs: None, rootfs_resource: RootFsResource::new(), diff --git a/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs new file mode 100644 index 0000000000..56a2641da5 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs @@ -0,0 +1,126 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use super::{Rootfs, ROOTFS}; +use crate::share_fs::{do_get_guest_path, do_get_host_path}; +use agent::Storage; +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use hypervisor::{device::device_manager::DeviceManager, BlockConfig, DeviceConfig}; +use kata_types::mount::Mount; +use nix::sys::stat::{self, SFlag}; +use std::fs; +use tokio::sync::RwLock; + +pub(crate) struct BlockRootfs { + guest_path: String, + device_id: String, + mount: oci::Mount, + storage: Option, +} + +impl BlockRootfs { + pub async fn new( + d: &RwLock, + sid: &str, + cid: &str, + dev_id: u64, + rootfs: &Mount, + ) -> Result { + let container_path = do_get_guest_path(ROOTFS, cid, false, false); + let host_path = do_get_host_path(ROOTFS, sid, cid, false, false); + // Create rootfs dir on host to make sure mount point in guest exists, as readonly dir is + // shared to guest via virtiofs, and guest is unable to create rootfs dir. + fs::create_dir_all(&host_path) + .map_err(|e| anyhow!("failed to create rootfs dir {}: {:?}", host_path, e))?; + + let block_device_config = &mut BlockConfig { + major: stat::major(dev_id) as i64, + minor: stat::minor(dev_id) as i64, + ..Default::default() + }; + + let device_id = d + .write() + .await + .new_device(&DeviceConfig::Block(block_device_config.clone())) + .await + .context("failed to create deviec")?; + + d.write() + .await + .try_add_device(device_id.as_str()) + .await + .context("failed to add deivce")?; + + let mut storage = Storage { + fs_type: rootfs.fs_type.clone(), + mount_point: container_path.clone(), + options: rootfs.options.clone(), + ..Default::default() + }; + + // get complete device information + let dev_info = d + .read() + .await + .get_device_info(device_id.as_str()) + .await + .context("failed to get device info")?; + + if let DeviceConfig::Block(config) = dev_info { + storage.driver = config.driver_option; + storage.source = config.virt_path; + } + + Ok(Self { + guest_path: container_path.clone(), + device_id, + mount: oci::Mount { + ..Default::default() + }, + storage: Some(storage), + }) + } +} + +#[async_trait] +impl Rootfs for BlockRootfs { + async fn get_guest_rootfs_path(&self) -> Result { + Ok(self.guest_path.clone()) + } + + async fn get_rootfs_mount(&self) -> Result> { + Ok(vec![self.mount.clone()]) + } + + async fn get_storage(&self) -> Option { + self.storage.clone() + } + + async fn get_device_id(&self) -> Result> { + Ok(Some(self.device_id.clone())) + } + async fn cleanup(&self) -> Result<()> { + Ok(()) + } +} + +pub(crate) fn is_block_rootfs(file: &str) -> Option { + if file.is_empty() { + return None; + } + match stat::stat(file) { + Ok(fstat) => { + if SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFBLK { + let dev_id = fstat.st_rdev; + return Some(dev_id); + } + } + Err(_) => return None, + }; + None +} diff --git a/src/runtime-rs/crates/resource/src/rootfs/mod.rs b/src/runtime-rs/crates/resource/src/rootfs/mod.rs index 2c9160cec8..06487fc439 100644 --- a/src/runtime-rs/crates/resource/src/rootfs/mod.rs +++ b/src/runtime-rs/crates/resource/src/rootfs/mod.rs @@ -6,18 +6,18 @@ mod nydus_rootfs; mod share_fs_rootfs; - use agent::Storage; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; -use hypervisor::Hypervisor; use kata_types::mount::Mount; +mod block_rootfs; +use hypervisor::{device::device_manager::DeviceManager, Hypervisor}; use std::{sync::Arc, vec::Vec}; use tokio::sync::RwLock; use crate::share_fs::ShareFs; -use self::nydus_rootfs::NYDUS_ROOTFS_TYPE; +use self::{block_rootfs::is_block_rootfs, nydus_rootfs::NYDUS_ROOTFS_TYPE}; const ROOTFS: &str = "rootfs"; const HYBRID_ROOTFS_LOWER_DIR: &str = "rootfs_lower"; @@ -28,6 +28,7 @@ pub trait Rootfs: Send + Sync { async fn get_rootfs_mount(&self) -> Result>; async fn get_storage(&self) -> Option; async fn cleanup(&self) -> Result<()>; + async fn get_device_id(&self) -> Result>; } #[derive(Default)] @@ -56,7 +57,8 @@ impl RootFsResource { pub async fn handler_rootfs( &self, share_fs: &Option>, - hypervisor: &dyn Hypervisor, + device_manager: &RwLock, + h: &dyn Hypervisor, sid: &str, cid: &str, root: &oci::Root, @@ -67,7 +69,7 @@ impl RootFsResource { // if rootfs_mounts is empty mounts_vec if mounts_vec.is_empty() => { if let Some(share_fs) = share_fs { - // share fs rootfs + // handle share fs rootfs Ok(Arc::new( share_fs_rootfs::ShareFsRootfs::new( share_fs, @@ -85,16 +87,27 @@ impl RootFsResource { mounts_vec if is_single_layer_rootfs(mounts_vec) => { // Safe as single_layer_rootfs must have one layer let layer = &mounts_vec[0]; - let rootfs: Arc = if let Some(share_fs) = share_fs { - // nydus rootfs - if layer.fs_type == NYDUS_ROOTFS_TYPE { + let mut inner = self.inner.write().await; + let rootfs = if let Some(dev_id) = is_block_rootfs(&layer.source) { + // handle block rootfs + info!(sl!(), "block device: {}", dev_id); + let block_rootfs: Arc = Arc::new( + block_rootfs::BlockRootfs::new(device_manager, sid, cid, dev_id, layer) + .await + .context("new block rootfs")?, + ); + Ok(block_rootfs) + } else if let Some(share_fs) = share_fs { + // handle nydus rootfs + let share_rootfs: Arc = if layer.fs_type == NYDUS_ROOTFS_TYPE { Arc::new( - nydus_rootfs::NydusRootfs::new(share_fs, hypervisor, sid, cid, layer) + nydus_rootfs::NydusRootfs::new(share_fs, h, sid, cid, layer) .await .context("new nydus rootfs")?, ) - } else { - // share fs rootfs + } + // handle sharefs rootfs + else { Arc::new( share_fs_rootfs::ShareFsRootfs::new( share_fs, @@ -105,13 +118,12 @@ impl RootFsResource { .await .context("new share fs rootfs")?, ) - } + }; + Ok(share_rootfs) } else { - return Err(anyhow!("unsupported rootfs {:?}", &layer)); - }; - - let mut inner = self.inner.write().await; - inner.rootfs.push(Arc::clone(&rootfs)); + Err(anyhow!("unsupported rootfs {:?}", &layer)) + }?; + inner.rootfs.push(rootfs.clone()); Ok(rootfs) } _ => Err(anyhow!( diff --git a/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs index 008443b874..b8f9ed1009 100644 --- a/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs +++ b/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs @@ -150,6 +150,10 @@ impl Rootfs for NydusRootfs { Some(self.rootfs.clone()) } + async fn get_device_id(&self) -> Result> { + Ok(None) + } + async fn cleanup(&self) -> Result<()> { // TODO: Clean up NydusRootfs after the container is killed warn!(sl!(), "Cleaning up NydusRootfs is still unimplemented."); diff --git a/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs index b5d4136c1b..7a490215a2 100644 --- a/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs +++ b/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs @@ -4,12 +4,13 @@ // SPDX-License-Identifier: Apache-2.0 // +use std::sync::Arc; + use agent::Storage; use anyhow::{Context, Result}; use async_trait::async_trait; use kata_sys_util::mount::{umount_timeout, Mounter}; use kata_types::mount::Mount; -use std::sync::Arc; use super::{Rootfs, ROOTFS}; use crate::share_fs::{ShareFs, ShareFsRootfsConfig}; @@ -74,6 +75,10 @@ impl Rootfs for ShareFsRootfs { None } + async fn get_device_id(&self) -> Result> { + Ok(None) + } + async fn cleanup(&self) -> Result<()> { // Umount the mount point shared to guest let share_fs_mount = self.share_fs.get_share_fs_mount(); diff --git a/src/runtime-rs/crates/resource/src/share_fs/mod.rs b/src/runtime-rs/crates/resource/src/share_fs/mod.rs index 350c7ea71a..12bb64420d 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/mod.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/mod.rs @@ -12,7 +12,9 @@ mod share_virtio_fs_standalone; use share_virtio_fs_standalone::ShareVirtioFsStandalone; mod utils; use tokio::sync::Mutex; -pub use utils::{do_get_guest_path, do_get_guest_share_path, get_host_rw_shared_path}; +pub use utils::{ + do_get_guest_path, do_get_guest_share_path, do_get_host_path, get_host_rw_shared_path, +}; mod virtio_fs_share_mount; use virtio_fs_share_mount::VirtiofsShareMount; pub use virtio_fs_share_mount::EPHEMERAL_PATH; diff --git a/src/runtime-rs/crates/resource/src/share_fs/utils.rs b/src/runtime-rs/crates/resource/src/share_fs/utils.rs index 47f3df5331..c93cbec547 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/utils.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/utils.rs @@ -97,7 +97,7 @@ pub fn do_get_guest_share_path(target: &str, cid: &str, is_rafs: bool) -> String do_get_guest_any_path(target, cid, false, is_rafs, true) } -pub(crate) fn do_get_host_path( +pub fn do_get_host_path( target: &str, sid: &str, cid: &str,