runtime-rs: support block rootfs

support devmapper for block rootfs

Fixes: #5375

Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Signed-off-by: alex.lyn <alex.lyn@antgroup.com>

src/runtime-rs/crates/hypervisor/src/device/device_manager.rs

@@ -66,7 +66,7 @@ pub struct DeviceManager {
 }
 
 impl DeviceManager {
-    pub async fn new(hypervisor: Arc<dyn Hypervisor>) -> Result<Self> {
+    pub fn new(hypervisor: Arc<dyn Hypervisor>) -> Result<Self> {
         let devices = HashMap::<String, ArcMutexBoxDevice>::new();
         Ok(DeviceManager {
             devices,
@@ -75,7 +75,7 @@ impl DeviceManager {
         })
     }
 
-    pub async fn new_device(&mut self, device_config: DeviceConfig) -> Result<String> {
+    pub async fn new_device(&mut self, device_config: &DeviceConfig) -> Result<String> {
         let device_id = if let Some(dev) = self.find_device(&device_config).await {
             dev
         } else {
@@ -86,11 +86,10 @@ impl DeviceManager {
         Ok(device_id)
     }
 
-    pub async fn try_add_device(&mut self, device_id: String) -> Result<()> {
-        // get device
+    pub async fn try_add_device(&mut self, device_id: &str) -> Result<()> {
         let device = self
             .devices
-            .get(&device_id)
+            .get(device_id)
             .context("failed to find device")?;
         // attach device
         let result = device.lock().await.attach(self.hypervisor.as_ref()).await;
@@ -99,12 +98,22 @@ impl DeviceManager {
             if let DeviceConfig::Block(config) = device.lock().await.get_device_info().await {
                 self.shared_info.release_device_index(config.index);
             };
-            self.devices.remove(&device_id);
+            self.devices.remove(device_id);
             return Err(e);
         }
         Ok(())
     }
 
+    pub async fn get_device_info(&self, device_id: &str) -> Result<DeviceConfig> {
+        if let Some(dev) = self.devices.get(device_id) {
+            return Ok(dev.lock().await.get_device_info().await);
+        }
+        Err(anyhow!(
+            "device with specified ID hasn't been created. {}",
+            device_id
+        ))
+    }
+
     async fn find_device(&self, device_config: &DeviceConfig) -> Option<String> {
         for (device_id, dev) in &self.devices {
             match dev.lock().await.get_device_info().await {

src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs

@@ -6,14 +6,13 @@
 
 use std::{fs, path::Path, process::Command};
 
-use crate::{driver::hypervisor, DeviceConfig};
+use crate::Hypervisor as hypervisor;
+use crate::{device::Device, DeviceConfig};
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
 use anyhow::anyhow;
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 
-use crate::Device;
-
 fn override_driver(bdf: &str, driver: &str) -> Result<()> {
     let driver_override = format!("/sys/bus/pci/devices/{}/driver_override", bdf);
     fs::write(&driver_override, driver)
@@ -152,11 +151,11 @@ pub fn bind_device_to_host(bdf: &str, host_driver: &str, _vendor_device_id: &str
 
 #[async_trait]
 impl Device for VfioConfig {
-    async fn attach(&self, _h: &dyn hypervisor) -> Result<()> {
+    async fn attach(&mut self, _h: &dyn hypervisor) -> Result<()> {
         todo!()
     }
 
-    async fn detach(&self, _h: &dyn hypervisor) -> Result<u64> {
+    async fn detach(&mut self, _h: &dyn hypervisor) -> Result<Option<u64>> {
         todo!()
     }
 

src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs

@@ -4,8 +4,8 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use crate::Device;
-use crate::{driver::hypervisor, DeviceConfig};
+use crate::Hypervisor as hypervisor;
+use crate::{device::Device, DeviceConfig};
 use anyhow::Result;
 use async_trait::async_trait;
 
@@ -31,11 +31,11 @@ pub struct VhostUserConfig {
 
 #[async_trait]
 impl Device for VhostUserConfig {
-    async fn attach(&self, _h: &dyn hypervisor) -> Result<()> {
+    async fn attach(&mut self, _h: &dyn hypervisor) -> Result<()> {
         todo!()
     }
 
-    async fn detach(&self, _h: &dyn hypervisor) -> Result<u64> {
+    async fn detach(&mut self, _h: &dyn hypervisor) -> Result<Option<u64>> {
         todo!()
     }
 

src/runtime-rs/crates/resource/src/manager_inner.rs

@@ -10,12 +10,12 @@ use crate::{network::NetworkConfig, resource_persist::ResourceState};
 use agent::{Agent, Storage};
 use anyhow::{anyhow, Context, Ok, Result};
 use async_trait::async_trait;
-use hypervisor::Hypervisor;
+use hypervisor::{device::device_manager::DeviceManager, Hypervisor};
 use kata_types::config::TomlConfig;
 use kata_types::mount::Mount;
 use oci::LinuxResources;
 use persist::sandbox_persist::Persist;
-use tokio::runtime;
+use tokio::{runtime, sync::RwLock};
 
 use crate::{
     cgroups::{CgroupArgs, CgroupsResource},
@@ -32,6 +32,7 @@ pub(crate) struct ResourceManagerInner {
     toml_config: Arc<TomlConfig>,
     agent: Arc<dyn Agent>,
     hypervisor: Arc<dyn Hypervisor>,
+    device_manager: Arc<RwLock<DeviceManager>>,
     network: Option<Arc<dyn Network>>,
     share_fs: Option<Arc<dyn ShareFs>>,
 
@@ -48,11 +49,17 @@ impl ResourceManagerInner {
         toml_config: Arc<TomlConfig>,
     ) -> Result<Self> {
         let cgroups_resource = CgroupsResource::new(sid, &toml_config)?;
+
+        // create device manager
+        let dev_manager =
+            DeviceManager::new(hypervisor.clone()).context("failed to create device manager")?;
+
         Ok(Self {
             sid: sid.to_string(),
             toml_config,
             agent,
             hypervisor,
+            device_manager: Arc::new(RwLock::new(dev_manager)),
             network: None,
             share_fs: None,
             rootfs_resource: RootFsResource::new(),
@@ -212,6 +219,7 @@ impl ResourceManagerInner {
         self.rootfs_resource
             .handler_rootfs(
                 &self.share_fs,
+                self.device_manager.as_ref(),
                 self.hypervisor.as_ref(),
                 &self.sid,
                 cid,
@@ -298,7 +306,8 @@ impl Persist for ResourceManagerInner {
         Ok(Self {
             sid: resource_args.sid,
             agent: resource_args.agent,
-            hypervisor: resource_args.hypervisor,
+            hypervisor: resource_args.hypervisor.clone(),
+            device_manager: Arc::new(RwLock::new(DeviceManager::new(resource_args.hypervisor)?)),
             network: None,
             share_fs: None,
             rootfs_resource: RootFsResource::new(),

src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs

@@ -0,0 +1,126 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use super::{Rootfs, ROOTFS};
+use crate::share_fs::{do_get_guest_path, do_get_host_path};
+use agent::Storage;
+use anyhow::{anyhow, Context, Result};
+use async_trait::async_trait;
+use hypervisor::{device::device_manager::DeviceManager, BlockConfig, DeviceConfig};
+use kata_types::mount::Mount;
+use nix::sys::stat::{self, SFlag};
+use std::fs;
+use tokio::sync::RwLock;
+
+pub(crate) struct BlockRootfs {
+    guest_path: String,
+    device_id: String,
+    mount: oci::Mount,
+    storage: Option<agent::Storage>,
+}
+
+impl BlockRootfs {
+    pub async fn new(
+        d: &RwLock<DeviceManager>,
+        sid: &str,
+        cid: &str,
+        dev_id: u64,
+        rootfs: &Mount,
+    ) -> Result<Self> {
+        let container_path = do_get_guest_path(ROOTFS, cid, false, false);
+        let host_path = do_get_host_path(ROOTFS, sid, cid, false, false);
+        // Create rootfs dir on host to make sure mount point in guest exists, as readonly dir is
+        // shared to guest via virtiofs, and guest is unable to create rootfs dir.
+        fs::create_dir_all(&host_path)
+            .map_err(|e| anyhow!("failed to create rootfs dir {}: {:?}", host_path, e))?;
+
+        let block_device_config = &mut BlockConfig {
+            major: stat::major(dev_id) as i64,
+            minor: stat::minor(dev_id) as i64,
+            ..Default::default()
+        };
+
+        let device_id = d
+            .write()
+            .await
+            .new_device(&DeviceConfig::Block(block_device_config.clone()))
+            .await
+            .context("failed to create deviec")?;
+
+        d.write()
+            .await
+            .try_add_device(device_id.as_str())
+            .await
+            .context("failed to add deivce")?;
+
+        let mut storage = Storage {
+            fs_type: rootfs.fs_type.clone(),
+            mount_point: container_path.clone(),
+            options: rootfs.options.clone(),
+            ..Default::default()
+        };
+
+        // get complete device information
+        let dev_info = d
+            .read()
+            .await
+            .get_device_info(device_id.as_str())
+            .await
+            .context("failed to get device info")?;
+
+        if let DeviceConfig::Block(config) = dev_info {
+            storage.driver = config.driver_option;
+            storage.source = config.virt_path;
+        }
+
+        Ok(Self {
+            guest_path: container_path.clone(),
+            device_id,
+            mount: oci::Mount {
+                ..Default::default()
+            },
+            storage: Some(storage),
+        })
+    }
+}
+
+#[async_trait]
+impl Rootfs for BlockRootfs {
+    async fn get_guest_rootfs_path(&self) -> Result<String> {
+        Ok(self.guest_path.clone())
+    }
+
+    async fn get_rootfs_mount(&self) -> Result<Vec<oci::Mount>> {
+        Ok(vec![self.mount.clone()])
+    }
+
+    async fn get_storage(&self) -> Option<Storage> {
+        self.storage.clone()
+    }
+
+    async fn get_device_id(&self) -> Result<Option<String>> {
+        Ok(Some(self.device_id.clone()))
+    }
+    async fn cleanup(&self) -> Result<()> {
+        Ok(())
+    }
+}
+
+pub(crate) fn is_block_rootfs(file: &str) -> Option<u64> {
+    if file.is_empty() {
+        return None;
+    }
+    match stat::stat(file) {
+        Ok(fstat) => {
+            if SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFBLK {
+                let dev_id = fstat.st_rdev;
+                return Some(dev_id);
+            }
+        }
+        Err(_) => return None,
+    };
+    None
+}

src/runtime-rs/crates/resource/src/rootfs/mod.rs

@@ -6,18 +6,18 @@
 
 mod nydus_rootfs;
 mod share_fs_rootfs;
-
 use agent::Storage;
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
-use hypervisor::Hypervisor;
 use kata_types::mount::Mount;
+mod block_rootfs;
+use hypervisor::{device::device_manager::DeviceManager, Hypervisor};
 use std::{sync::Arc, vec::Vec};
 use tokio::sync::RwLock;
 
 use crate::share_fs::ShareFs;
 
-use self::nydus_rootfs::NYDUS_ROOTFS_TYPE;
+use self::{block_rootfs::is_block_rootfs, nydus_rootfs::NYDUS_ROOTFS_TYPE};
 
 const ROOTFS: &str = "rootfs";
 const HYBRID_ROOTFS_LOWER_DIR: &str = "rootfs_lower";
@@ -28,6 +28,7 @@ pub trait Rootfs: Send + Sync {
     async fn get_rootfs_mount(&self) -> Result<Vec<oci::Mount>>;
     async fn get_storage(&self) -> Option<Storage>;
     async fn cleanup(&self) -> Result<()>;
+    async fn get_device_id(&self) -> Result<Option<String>>;
 }
 
 #[derive(Default)]
@@ -56,7 +57,8 @@ impl RootFsResource {
     pub async fn handler_rootfs(
         &self,
         share_fs: &Option<Arc<dyn ShareFs>>,
-        hypervisor: &dyn Hypervisor,
+        device_manager: &RwLock<DeviceManager>,
+        h: &dyn Hypervisor,
         sid: &str,
         cid: &str,
         root: &oci::Root,
@@ -67,7 +69,7 @@ impl RootFsResource {
             // if rootfs_mounts is empty
             mounts_vec if mounts_vec.is_empty() => {
                 if let Some(share_fs) = share_fs {
-                    // share fs rootfs
+                    // handle share fs rootfs
                     Ok(Arc::new(
                         share_fs_rootfs::ShareFsRootfs::new(
                             share_fs,
@@ -85,16 +87,27 @@ impl RootFsResource {
             mounts_vec if is_single_layer_rootfs(mounts_vec) => {
                 // Safe as single_layer_rootfs must have one layer
                 let layer = &mounts_vec[0];
-                let rootfs: Arc<dyn Rootfs> = if let Some(share_fs) = share_fs {
-                    // nydus rootfs
-                    if layer.fs_type == NYDUS_ROOTFS_TYPE {
+                let mut inner = self.inner.write().await;
+                let rootfs = if let Some(dev_id) = is_block_rootfs(&layer.source) {
+                    // handle block rootfs
+                    info!(sl!(), "block device: {}", dev_id);
+                    let block_rootfs: Arc<dyn Rootfs> = Arc::new(
+                        block_rootfs::BlockRootfs::new(device_manager, sid, cid, dev_id, layer)
+                            .await
+                            .context("new block rootfs")?,
+                    );
+                    Ok(block_rootfs)
+                } else if let Some(share_fs) = share_fs {
+                    // handle nydus rootfs
+                    let share_rootfs: Arc<dyn Rootfs> = if layer.fs_type == NYDUS_ROOTFS_TYPE {
                         Arc::new(
-                            nydus_rootfs::NydusRootfs::new(share_fs, hypervisor, sid, cid, layer)
+                            nydus_rootfs::NydusRootfs::new(share_fs, h, sid, cid, layer)
                                 .await
                                 .context("new nydus rootfs")?,
                         )
-                    } else {
-                        // share fs rootfs
+                    }
+                    // handle sharefs rootfs
+                    else {
                         Arc::new(
                             share_fs_rootfs::ShareFsRootfs::new(
                                 share_fs,
@@ -105,13 +118,12 @@ impl RootFsResource {
                             .await
                             .context("new share fs rootfs")?,
                         )
-                    }
+                    };
+                    Ok(share_rootfs)
                 } else {
-                    return Err(anyhow!("unsupported rootfs {:?}", &layer));
-                };
-
-                let mut inner = self.inner.write().await;
-                inner.rootfs.push(Arc::clone(&rootfs));
+                    Err(anyhow!("unsupported rootfs {:?}", &layer))
+                }?;
+                inner.rootfs.push(rootfs.clone());
                 Ok(rootfs)
             }
             _ => Err(anyhow!(

src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs

@@ -150,6 +150,10 @@ impl Rootfs for NydusRootfs {
         Some(self.rootfs.clone())
     }
 
+    async fn get_device_id(&self) -> Result<Option<String>> {
+        Ok(None)
+    }
+
     async fn cleanup(&self) -> Result<()> {
         // TODO: Clean up NydusRootfs after the container is killed
         warn!(sl!(), "Cleaning up NydusRootfs is still unimplemented.");

src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs

@@ -4,12 +4,13 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use std::sync::Arc;
+
 use agent::Storage;
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 use kata_sys_util::mount::{umount_timeout, Mounter};
 use kata_types::mount::Mount;
-use std::sync::Arc;
 
 use super::{Rootfs, ROOTFS};
 use crate::share_fs::{ShareFs, ShareFsRootfsConfig};
@@ -74,6 +75,10 @@ impl Rootfs for ShareFsRootfs {
         None
     }
 
+    async fn get_device_id(&self) -> Result<Option<String>> {
+        Ok(None)
+    }
+
     async fn cleanup(&self) -> Result<()> {
         // Umount the mount point shared to guest
         let share_fs_mount = self.share_fs.get_share_fs_mount();

src/runtime-rs/crates/resource/src/share_fs/mod.rs

@@ -12,7 +12,9 @@ mod share_virtio_fs_standalone;
 use share_virtio_fs_standalone::ShareVirtioFsStandalone;
 mod utils;
 use tokio::sync::Mutex;
-pub use utils::{do_get_guest_path, do_get_guest_share_path, get_host_rw_shared_path};
+pub use utils::{
+    do_get_guest_path, do_get_guest_share_path, do_get_host_path, get_host_rw_shared_path,
+};
 mod virtio_fs_share_mount;
 use virtio_fs_share_mount::VirtiofsShareMount;
 pub use virtio_fs_share_mount::EPHEMERAL_PATH;

src/runtime-rs/crates/resource/src/share_fs/utils.rs

@@ -97,7 +97,7 @@ pub fn do_get_guest_share_path(target: &str, cid: &str, is_rafs: bool) -> String
     do_get_guest_any_path(target, cid, false, is_rafs, true)
 }
 
-pub(crate) fn do_get_host_path(
+pub fn do_get_host_path(
     target: &str,
     sid: &str,
     cid: &str,