runtime-rs: split the block volume into block and rawblock volume

(1) rawblock volume is directvol mount type.
(2) block volume is based on the bind mount type.

Fixes: #8300

Signed-off-by: alex.lyn <alex.lyn@antgroup.com>

src/runtime-rs/crates/resource/src/volume/block_volume.rs

@@ -4,20 +4,17 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use anyhow::{anyhow, Context, Result};
+use anyhow::{Context, Result};
 use async_trait::async_trait;
 use nix::sys::{stat, stat::SFlag};
 use tokio::sync::RwLock;
 
 use super::Volume;
-use crate::volume::utils::{
-    generate_shared_path, get_direct_volume_path, volume_mount_info, DEFAULT_VOLUME_FS_TYPE,
-    KATA_DIRECT_VOLUME_TYPE, KATA_MOUNT_BIND_TYPE,
-};
+use crate::volume::utils::{handle_block_volume, DEFAULT_VOLUME_FS_TYPE, KATA_MOUNT_BIND_TYPE};
 use hypervisor::{
     device::{
         device_manager::{do_handle_device, get_block_driver, DeviceManager},
-        DeviceConfig, DeviceType,
+        DeviceConfig,
     },
     BlockConfig,
 };
@@ -29,7 +26,7 @@ pub(crate) struct BlockVolume {
     device_id: String,
 }
 
-/// BlockVolume for bind-mount block volume and direct block volume
+/// BlockVolume for bind-mount block volume
 impl BlockVolume {
     pub(crate) async fn new(
         d: &RwLock<DeviceManager>,
@@ -38,57 +35,13 @@ impl BlockVolume {
         sid: &str,
     ) -> Result<Self> {
         let mnt_src: &str = &m.source;
-        // default block device fs type: ext4.
-        let mut blk_dev_fstype = DEFAULT_VOLUME_FS_TYPE.to_string();
-
         let block_driver = get_block_driver(d).await;
-
-        let block_device_config = match m.r#type.as_str() {
-            KATA_MOUNT_BIND_TYPE => {
-                let fstat = stat::stat(mnt_src).context(format!("stat {}", m.source))?;
-
-                BlockConfig {
-                    major: stat::major(fstat.st_rdev) as i64,
-                    minor: stat::minor(fstat.st_rdev) as i64,
-                    driver_option: block_driver,
-                    ..Default::default()
-                }
-            }
-            KATA_DIRECT_VOLUME_TYPE => {
-                // get volume mountinfo from mountinfo.json
-                let v = volume_mount_info(mnt_src)
-                    .context("deserde information from mountinfo.json")?;
-                // check volume type
-                if v.volume_type != KATA_DIRECT_VOLUME_TYPE {
-                    return Err(anyhow!("volume type {:?} is invalid", v.volume_type));
-                }
-
-                let fstat = stat::stat(v.device.as_str())
-                    .with_context(|| format!("stat volume device file: {}", v.device.clone()))?;
-                if SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFREG
-                    && SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFBLK
-                {
-                    return Err(anyhow!(
-                        "invalid volume device {:?} for volume type {:?}",
-                        v.device,
-                        v.volume_type
-                    ));
-                }
-
-                blk_dev_fstype = v.fs_type.clone();
-
-                BlockConfig {
-                    path_on_host: v.device,
-                    driver_option: block_driver,
-                    ..Default::default()
-                }
-            }
-            _ => {
-                return Err(anyhow!(
-                    "unsupport direct block volume r#type: {:?}",
-                    m.r#type.as_str()
-                ))
-            }
+        let fstat = stat::stat(mnt_src).context(format!("stat {}", m.source))?;
+        let block_device_config = BlockConfig {
+            major: stat::major(fstat.st_rdev) as i64,
+            minor: stat::minor(fstat.st_rdev) as i64,
+            driver_option: block_driver,
+            ..Default::default()
         };
 
         // create and insert block device into Kata VM
@@ -96,55 +49,15 @@ impl BlockVolume {
             .await
             .context("do handle device failed.")?;
 
-        // storage
-        let mut storage = agent::Storage {
-            options: if read_only {
-                vec!["ro".to_string()]
-            } else {
-                Vec::new()
-            },
-            ..Default::default()
-        };
-
-        // As the true Block Device wrapped in DeviceType, we need to
-        // get it out from the wrapper, and the device_id will be for
-        // BlockVolume.
-        // safe here, device_info is correct and only unwrap it.
-        let mut device_id = String::new();
-        if let DeviceType::Block(device) = device_info {
-            // blk, mmioblk
-            storage.driver = device.config.driver_option;
-            // /dev/vdX
-            storage.source = device.config.virt_path;
-            device_id = device.device_id;
-        }
-
-        // generate host guest shared path
-        let guest_path = generate_shared_path(m.destination.clone(), read_only, &device_id, sid)
-            .await
-            .context("generate host-guest shared path failed")?;
-        storage.mount_point = guest_path.clone();
-
-        // In some case, dest is device /dev/xxx
-        if m.destination.clone().starts_with("/dev") {
-            storage.fs_type = "bind".to_string();
-            storage.options.append(&mut m.options.clone());
-        } else {
-            // usually, the dest is directory.
-            storage.fs_type = blk_dev_fstype;
-        }
-
-        let mount = oci::Mount {
-            destination: m.destination.clone(),
-            r#type: storage.fs_type.clone(),
-            source: guest_path,
-            options: m.options.clone(),
-        };
+        let block_volume =
+            handle_block_volume(device_info, m, read_only, sid, DEFAULT_VOLUME_FS_TYPE)
+                .await
+                .context("do handle block volume failed")?;
 
         Ok(Self {
-            storage: Some(storage),
-            mount,
-            device_id,
+            storage: Some(block_volume.0),
+            mount: block_volume.1,
+            device_id: block_volume.2,
         })
     }
 }
@@ -178,28 +91,13 @@ impl Volume for BlockVolume {
     }
 }
 
-pub(crate) fn is_block_volume(m: &oci::Mount) -> Result<bool> {
-    let vol_types = [KATA_MOUNT_BIND_TYPE, KATA_DIRECT_VOLUME_TYPE];
-    if !vol_types.contains(&m.r#type.as_str()) {
-        return Ok(false);
+pub(crate) fn is_block_volume(m: &oci::Mount) -> bool {
+    if m.r#type.as_str() != KATA_MOUNT_BIND_TYPE {
+        return false;
     }
 
-    let source = if m.r#type.as_str() == KATA_DIRECT_VOLUME_TYPE {
-        get_direct_volume_path(&m.source).context("get direct volume path failed")?
-    } else {
-        m.source.clone()
-    };
-
-    let fstat =
-        stat::stat(source.as_str()).context(format!("stat mount source {} failed.", source))?;
-    let s_flag = SFlag::from_bits_truncate(fstat.st_mode);
-
-    match m.r#type.as_str() {
-        // case: mount bind and block device
-        KATA_MOUNT_BIND_TYPE if s_flag == SFlag::S_IFBLK => Ok(true),
-        // case: directvol and directory
-        KATA_DIRECT_VOLUME_TYPE if s_flag == SFlag::S_IFDIR => Ok(true),
-        // else: unsupported or todo for other volume type.
-        _ => Ok(false),
+    match stat::stat(m.source.as_str()) {
+        Ok(fstat) => SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFBLK,
+        Err(_) => false,
     }
 }

src/runtime-rs/crates/resource/src/volume/mod.rs

@@ -17,6 +17,9 @@ use vfio_volume::is_vfio_volume;
 pub mod spdk_volume;
 use spdk_volume::is_spdk_volume;
 
+pub mod rawblock_volume;
+use rawblock_volume::is_rawblock_volume;
+
 use std::{sync::Arc, vec::Vec};
 
 use anyhow::{Context, Result};
@@ -74,12 +77,19 @@ impl VolumeResource {
                     shm_volume::ShmVolume::new(m, shm_size)
                         .with_context(|| format!("new shm volume {:?}", m))?,
                 )
-            } else if is_block_volume(m).context("block volume type")? {
+            } else if is_block_volume(m) {
                 // handle block volume
                 Arc::new(
                     block_volume::BlockVolume::new(d, m, read_only, sid)
                         .await
-                        .with_context(|| format!("new share fs volume {:?}", m))?,
+                        .with_context(|| format!("new block volume {:?}", m))?,
+                )
+            } else if is_rawblock_volume(m)? {
+                // handle rawblock volume
+                Arc::new(
+                    rawblock_volume::RawblockVolume::new(d, m, read_only, sid)
+                        .await
+                        .with_context(|| format!("new rawblock volume {:?}", m))?,
                 )
             } else if is_vfio_volume(m) {
                 Arc::new(

src/runtime-rs/crates/resource/src/volume/rawblock_volume.rs

@@ -0,0 +1,123 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::{anyhow, Context, Result};
+use async_trait::async_trait;
+use nix::sys::{stat, stat::SFlag};
+use tokio::sync::RwLock;
+
+use super::Volume;
+use crate::volume::utils::{
+    get_direct_volume_path, handle_block_volume, volume_mount_info, KATA_DIRECT_VOLUME_TYPE,
+};
+use hypervisor::{
+    device::{
+        device_manager::{do_handle_device, get_block_driver, DeviceManager},
+        DeviceConfig,
+    },
+    BlockConfig,
+};
+
+#[derive(Clone)]
+pub(crate) struct RawblockVolume {
+    storage: Option<agent::Storage>,
+    mount: oci::Mount,
+    device_id: String,
+}
+
+/// RawblockVolume for raw block volume
+impl RawblockVolume {
+    pub(crate) async fn new(
+        d: &RwLock<DeviceManager>,
+        m: &oci::Mount,
+        read_only: bool,
+        sid: &str,
+    ) -> Result<Self> {
+        let mnt_src: &str = &m.source;
+        let block_driver = get_block_driver(d).await;
+        // get volume mountinfo from mountinfo.json
+        let v = volume_mount_info(mnt_src).context("deserde information from mountinfo.json")?;
+        // check volume type
+        if v.volume_type != KATA_DIRECT_VOLUME_TYPE {
+            return Err(anyhow!("volume type {:?} is invalid", v.volume_type));
+        }
+
+        let fstat = stat::stat(v.device.as_str())
+            .with_context(|| format!("stat volume device file: {}", v.device.clone()))?;
+        if SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFREG
+            && SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFBLK
+        {
+            return Err(anyhow!(
+                "invalid volume device {:?} for volume type {:?}",
+                v.device,
+                v.volume_type
+            ));
+        }
+
+        let block_config = BlockConfig {
+            path_on_host: v.device,
+            driver_option: block_driver,
+            ..Default::default()
+        };
+
+        // create and insert block device into Kata VM
+        let device_info = do_handle_device(d, &DeviceConfig::BlockCfg(block_config.clone()))
+            .await
+            .context("do handle device failed.")?;
+
+        let block_volume = handle_block_volume(device_info, m, read_only, sid, &v.fs_type)
+            .await
+            .context("do handle block volume failed")?;
+
+        Ok(Self {
+            storage: Some(block_volume.0),
+            mount: block_volume.1,
+            device_id: block_volume.2,
+        })
+    }
+}
+
+#[async_trait]
+impl Volume for RawblockVolume {
+    fn get_volume_mount(&self) -> Result<Vec<oci::Mount>> {
+        Ok(vec![self.mount.clone()])
+    }
+
+    fn get_storage(&self) -> Result<Vec<agent::Storage>> {
+        let s = if let Some(s) = self.storage.as_ref() {
+            vec![s.clone()]
+        } else {
+            vec![]
+        };
+
+        Ok(s)
+    }
+
+    async fn cleanup(&self, device_manager: &RwLock<DeviceManager>) -> Result<()> {
+        device_manager
+            .write()
+            .await
+            .try_remove_device(&self.device_id)
+            .await
+    }
+
+    fn get_device_id(&self) -> Result<Option<String>> {
+        Ok(Some(self.device_id.clone()))
+    }
+}
+
+pub(crate) fn is_rawblock_volume(m: &oci::Mount) -> Result<bool> {
+    // KATA_MOUNT_BIND_TYPE = "directvol"
+    if m.r#type.as_str() != KATA_DIRECT_VOLUME_TYPE {
+        return Ok(false);
+    }
+
+    let source = get_direct_volume_path(&m.source).context("get direct volume path failed")?;
+    let fstat =
+        stat::stat(source.as_str()).context(format!("stat mount source {} failed.", source))?;
+
+    Ok(SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFDIR)
+}

src/runtime-rs/crates/resource/src/volume/utils.rs

@@ -17,6 +17,8 @@ use kata_types::mount::{
     get_volume_mount_info, join_path, DirectVolumeMountInfo, KATA_DIRECT_VOLUME_ROOT_PATH,
 };
 
+use hypervisor::device::DeviceType;
+
 pub const DEFAULT_VOLUME_FS_TYPE: &str = "ext4";
 pub const KATA_MOUNT_BIND_TYPE: &str = "bind";
 pub const KATA_DIRECT_VOLUME_TYPE: &str = "directvol";
@@ -74,3 +76,58 @@ pub(crate) async fn generate_shared_path(
 
     Ok(guest_path)
 }
+
+pub async fn handle_block_volume(
+    device_info: DeviceType,
+    m: &oci::Mount,
+    read_only: bool,
+    sid: &str,
+    fstype: &str,
+) -> Result<(agent::Storage, oci::Mount, String)> {
+    // storage
+    let mut storage = agent::Storage {
+        options: if read_only {
+            vec!["ro".to_string()]
+        } else {
+            Vec::new()
+        },
+        ..Default::default()
+    };
+
+    // As the true Block Device wrapped in DeviceType, we need to
+    // get it out from the wrapper, and the device_id will be for
+    // BlockVolume.
+    // safe here, device_info is correct and only unwrap it.
+    let mut device_id = String::new();
+    if let DeviceType::Block(device) = device_info {
+        // blk, mmioblk
+        storage.driver = device.config.driver_option;
+        // /dev/vdX
+        storage.source = device.config.virt_path;
+        device_id = device.device_id;
+    }
+
+    // generate host guest shared path
+    let guest_path = generate_shared_path(m.destination.clone(), read_only, &device_id, sid)
+        .await
+        .context("generate host-guest shared path failed")?;
+    storage.mount_point = guest_path.clone();
+
+    // In some case, dest is device /dev/xxx
+    if m.destination.clone().starts_with("/dev") {
+        storage.fs_type = "bind".to_string();
+        storage.options.append(&mut m.options.clone());
+    } else {
+        // usually, the dest is directory.
+        storage.fs_type = fstype.to_owned();
+    }
+
+    let mount = oci::Mount {
+        destination: m.destination.clone(),
+        r#type: storage.fs_type.clone(),
+        source: guest_path,
+        options: m.options.clone(),
+    };
+
+    Ok((storage, mount, device_id))
+}