diff --git a/src/runtime-rs/crates/resource/src/volume/block_volume.rs b/src/runtime-rs/crates/resource/src/volume/block_volume.rs index fc79183d1e..43bfa0a949 100644 --- a/src/runtime-rs/crates/resource/src/volume/block_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/block_volume.rs @@ -4,20 +4,17 @@ // SPDX-License-Identifier: Apache-2.0 // -use anyhow::{anyhow, Context, Result}; +use anyhow::{Context, Result}; use async_trait::async_trait; use nix::sys::{stat, stat::SFlag}; use tokio::sync::RwLock; use super::Volume; -use crate::volume::utils::{ - generate_shared_path, get_direct_volume_path, volume_mount_info, DEFAULT_VOLUME_FS_TYPE, - KATA_DIRECT_VOLUME_TYPE, KATA_MOUNT_BIND_TYPE, -}; +use crate::volume::utils::{handle_block_volume, DEFAULT_VOLUME_FS_TYPE, KATA_MOUNT_BIND_TYPE}; use hypervisor::{ device::{ device_manager::{do_handle_device, get_block_driver, DeviceManager}, - DeviceConfig, DeviceType, + DeviceConfig, }, BlockConfig, }; @@ -29,7 +26,7 @@ pub(crate) struct BlockVolume { device_id: String, } -/// BlockVolume for bind-mount block volume and direct block volume +/// BlockVolume for bind-mount block volume impl BlockVolume { pub(crate) async fn new( d: &RwLock, @@ -38,57 +35,13 @@ impl BlockVolume { sid: &str, ) -> Result { let mnt_src: &str = &m.source; - // default block device fs type: ext4. - let mut blk_dev_fstype = DEFAULT_VOLUME_FS_TYPE.to_string(); - let block_driver = get_block_driver(d).await; - - let block_device_config = match m.r#type.as_str() { - KATA_MOUNT_BIND_TYPE => { - let fstat = stat::stat(mnt_src).context(format!("stat {}", m.source))?; - - BlockConfig { - major: stat::major(fstat.st_rdev) as i64, - minor: stat::minor(fstat.st_rdev) as i64, - driver_option: block_driver, - ..Default::default() - } - } - KATA_DIRECT_VOLUME_TYPE => { - // get volume mountinfo from mountinfo.json - let v = volume_mount_info(mnt_src) - .context("deserde information from mountinfo.json")?; - // check volume type - if v.volume_type != KATA_DIRECT_VOLUME_TYPE { - return Err(anyhow!("volume type {:?} is invalid", v.volume_type)); - } - - let fstat = stat::stat(v.device.as_str()) - .with_context(|| format!("stat volume device file: {}", v.device.clone()))?; - if SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFREG - && SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFBLK - { - return Err(anyhow!( - "invalid volume device {:?} for volume type {:?}", - v.device, - v.volume_type - )); - } - - blk_dev_fstype = v.fs_type.clone(); - - BlockConfig { - path_on_host: v.device, - driver_option: block_driver, - ..Default::default() - } - } - _ => { - return Err(anyhow!( - "unsupport direct block volume r#type: {:?}", - m.r#type.as_str() - )) - } + let fstat = stat::stat(mnt_src).context(format!("stat {}", m.source))?; + let block_device_config = BlockConfig { + major: stat::major(fstat.st_rdev) as i64, + minor: stat::minor(fstat.st_rdev) as i64, + driver_option: block_driver, + ..Default::default() }; // create and insert block device into Kata VM @@ -96,55 +49,15 @@ impl BlockVolume { .await .context("do handle device failed.")?; - // storage - let mut storage = agent::Storage { - options: if read_only { - vec!["ro".to_string()] - } else { - Vec::new() - }, - ..Default::default() - }; - - // As the true Block Device wrapped in DeviceType, we need to - // get it out from the wrapper, and the device_id will be for - // BlockVolume. - // safe here, device_info is correct and only unwrap it. - let mut device_id = String::new(); - if let DeviceType::Block(device) = device_info { - // blk, mmioblk - storage.driver = device.config.driver_option; - // /dev/vdX - storage.source = device.config.virt_path; - device_id = device.device_id; - } - - // generate host guest shared path - let guest_path = generate_shared_path(m.destination.clone(), read_only, &device_id, sid) - .await - .context("generate host-guest shared path failed")?; - storage.mount_point = guest_path.clone(); - - // In some case, dest is device /dev/xxx - if m.destination.clone().starts_with("/dev") { - storage.fs_type = "bind".to_string(); - storage.options.append(&mut m.options.clone()); - } else { - // usually, the dest is directory. - storage.fs_type = blk_dev_fstype; - } - - let mount = oci::Mount { - destination: m.destination.clone(), - r#type: storage.fs_type.clone(), - source: guest_path, - options: m.options.clone(), - }; + let block_volume = + handle_block_volume(device_info, m, read_only, sid, DEFAULT_VOLUME_FS_TYPE) + .await + .context("do handle block volume failed")?; Ok(Self { - storage: Some(storage), - mount, - device_id, + storage: Some(block_volume.0), + mount: block_volume.1, + device_id: block_volume.2, }) } } @@ -178,28 +91,13 @@ impl Volume for BlockVolume { } } -pub(crate) fn is_block_volume(m: &oci::Mount) -> Result { - let vol_types = [KATA_MOUNT_BIND_TYPE, KATA_DIRECT_VOLUME_TYPE]; - if !vol_types.contains(&m.r#type.as_str()) { - return Ok(false); +pub(crate) fn is_block_volume(m: &oci::Mount) -> bool { + if m.r#type.as_str() != KATA_MOUNT_BIND_TYPE { + return false; } - let source = if m.r#type.as_str() == KATA_DIRECT_VOLUME_TYPE { - get_direct_volume_path(&m.source).context("get direct volume path failed")? - } else { - m.source.clone() - }; - - let fstat = - stat::stat(source.as_str()).context(format!("stat mount source {} failed.", source))?; - let s_flag = SFlag::from_bits_truncate(fstat.st_mode); - - match m.r#type.as_str() { - // case: mount bind and block device - KATA_MOUNT_BIND_TYPE if s_flag == SFlag::S_IFBLK => Ok(true), - // case: directvol and directory - KATA_DIRECT_VOLUME_TYPE if s_flag == SFlag::S_IFDIR => Ok(true), - // else: unsupported or todo for other volume type. - _ => Ok(false), + match stat::stat(m.source.as_str()) { + Ok(fstat) => SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFBLK, + Err(_) => false, } } diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs index 490181a1df..c8e5021081 100644 --- a/src/runtime-rs/crates/resource/src/volume/mod.rs +++ b/src/runtime-rs/crates/resource/src/volume/mod.rs @@ -17,6 +17,9 @@ use vfio_volume::is_vfio_volume; pub mod spdk_volume; use spdk_volume::is_spdk_volume; +pub mod rawblock_volume; +use rawblock_volume::is_rawblock_volume; + use std::{sync::Arc, vec::Vec}; use anyhow::{Context, Result}; @@ -74,12 +77,19 @@ impl VolumeResource { shm_volume::ShmVolume::new(m, shm_size) .with_context(|| format!("new shm volume {:?}", m))?, ) - } else if is_block_volume(m).context("block volume type")? { + } else if is_block_volume(m) { // handle block volume Arc::new( block_volume::BlockVolume::new(d, m, read_only, sid) .await - .with_context(|| format!("new share fs volume {:?}", m))?, + .with_context(|| format!("new block volume {:?}", m))?, + ) + } else if is_rawblock_volume(m)? { + // handle rawblock volume + Arc::new( + rawblock_volume::RawblockVolume::new(d, m, read_only, sid) + .await + .with_context(|| format!("new rawblock volume {:?}", m))?, ) } else if is_vfio_volume(m) { Arc::new( diff --git a/src/runtime-rs/crates/resource/src/volume/rawblock_volume.rs b/src/runtime-rs/crates/resource/src/volume/rawblock_volume.rs new file mode 100644 index 0000000000..bad715de40 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/volume/rawblock_volume.rs @@ -0,0 +1,123 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use nix::sys::{stat, stat::SFlag}; +use tokio::sync::RwLock; + +use super::Volume; +use crate::volume::utils::{ + get_direct_volume_path, handle_block_volume, volume_mount_info, KATA_DIRECT_VOLUME_TYPE, +}; +use hypervisor::{ + device::{ + device_manager::{do_handle_device, get_block_driver, DeviceManager}, + DeviceConfig, + }, + BlockConfig, +}; + +#[derive(Clone)] +pub(crate) struct RawblockVolume { + storage: Option, + mount: oci::Mount, + device_id: String, +} + +/// RawblockVolume for raw block volume +impl RawblockVolume { + pub(crate) async fn new( + d: &RwLock, + m: &oci::Mount, + read_only: bool, + sid: &str, + ) -> Result { + let mnt_src: &str = &m.source; + let block_driver = get_block_driver(d).await; + // get volume mountinfo from mountinfo.json + let v = volume_mount_info(mnt_src).context("deserde information from mountinfo.json")?; + // check volume type + if v.volume_type != KATA_DIRECT_VOLUME_TYPE { + return Err(anyhow!("volume type {:?} is invalid", v.volume_type)); + } + + let fstat = stat::stat(v.device.as_str()) + .with_context(|| format!("stat volume device file: {}", v.device.clone()))?; + if SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFREG + && SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFBLK + { + return Err(anyhow!( + "invalid volume device {:?} for volume type {:?}", + v.device, + v.volume_type + )); + } + + let block_config = BlockConfig { + path_on_host: v.device, + driver_option: block_driver, + ..Default::default() + }; + + // create and insert block device into Kata VM + let device_info = do_handle_device(d, &DeviceConfig::BlockCfg(block_config.clone())) + .await + .context("do handle device failed.")?; + + let block_volume = handle_block_volume(device_info, m, read_only, sid, &v.fs_type) + .await + .context("do handle block volume failed")?; + + Ok(Self { + storage: Some(block_volume.0), + mount: block_volume.1, + device_id: block_volume.2, + }) + } +} + +#[async_trait] +impl Volume for RawblockVolume { + fn get_volume_mount(&self) -> Result> { + Ok(vec![self.mount.clone()]) + } + + fn get_storage(&self) -> Result> { + let s = if let Some(s) = self.storage.as_ref() { + vec![s.clone()] + } else { + vec![] + }; + + Ok(s) + } + + async fn cleanup(&self, device_manager: &RwLock) -> Result<()> { + device_manager + .write() + .await + .try_remove_device(&self.device_id) + .await + } + + fn get_device_id(&self) -> Result> { + Ok(Some(self.device_id.clone())) + } +} + +pub(crate) fn is_rawblock_volume(m: &oci::Mount) -> Result { + // KATA_MOUNT_BIND_TYPE = "directvol" + if m.r#type.as_str() != KATA_DIRECT_VOLUME_TYPE { + return Ok(false); + } + + let source = get_direct_volume_path(&m.source).context("get direct volume path failed")?; + let fstat = + stat::stat(source.as_str()).context(format!("stat mount source {} failed.", source))?; + + Ok(SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFDIR) +} diff --git a/src/runtime-rs/crates/resource/src/volume/utils.rs b/src/runtime-rs/crates/resource/src/volume/utils.rs index d5f17d44b4..034c3a9103 100644 --- a/src/runtime-rs/crates/resource/src/volume/utils.rs +++ b/src/runtime-rs/crates/resource/src/volume/utils.rs @@ -17,6 +17,8 @@ use kata_types::mount::{ get_volume_mount_info, join_path, DirectVolumeMountInfo, KATA_DIRECT_VOLUME_ROOT_PATH, }; +use hypervisor::device::DeviceType; + pub const DEFAULT_VOLUME_FS_TYPE: &str = "ext4"; pub const KATA_MOUNT_BIND_TYPE: &str = "bind"; pub const KATA_DIRECT_VOLUME_TYPE: &str = "directvol"; @@ -74,3 +76,58 @@ pub(crate) async fn generate_shared_path( Ok(guest_path) } + +pub async fn handle_block_volume( + device_info: DeviceType, + m: &oci::Mount, + read_only: bool, + sid: &str, + fstype: &str, +) -> Result<(agent::Storage, oci::Mount, String)> { + // storage + let mut storage = agent::Storage { + options: if read_only { + vec!["ro".to_string()] + } else { + Vec::new() + }, + ..Default::default() + }; + + // As the true Block Device wrapped in DeviceType, we need to + // get it out from the wrapper, and the device_id will be for + // BlockVolume. + // safe here, device_info is correct and only unwrap it. + let mut device_id = String::new(); + if let DeviceType::Block(device) = device_info { + // blk, mmioblk + storage.driver = device.config.driver_option; + // /dev/vdX + storage.source = device.config.virt_path; + device_id = device.device_id; + } + + // generate host guest shared path + let guest_path = generate_shared_path(m.destination.clone(), read_only, &device_id, sid) + .await + .context("generate host-guest shared path failed")?; + storage.mount_point = guest_path.clone(); + + // In some case, dest is device /dev/xxx + if m.destination.clone().starts_with("/dev") { + storage.fs_type = "bind".to_string(); + storage.options.append(&mut m.options.clone()); + } else { + // usually, the dest is directory. + storage.fs_type = fstype.to_owned(); + } + + let mount = oci::Mount { + destination: m.destination.clone(), + r#type: storage.fs_type.clone(), + source: guest_path, + options: m.options.clone(), + }; + + Ok((storage, mount, device_id)) +}