From d23867273f0aff28252c144de8b2b7ae5de61e7c Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Fri, 24 Nov 2023 23:30:30 +0800 Subject: [PATCH 1/8] runtime-rs: split the block volume into block and rawblock volume (1) rawblock volume is directvol mount type. (2) block volume is based on the bind mount type. Fixes: #8300 Signed-off-by: alex.lyn --- .../resource/src/volume/block_volume.rs | 148 +++--------------- .../crates/resource/src/volume/mod.rs | 14 +- .../resource/src/volume/rawblock_volume.rs | 123 +++++++++++++++ .../crates/resource/src/volume/utils.rs | 57 +++++++ 4 files changed, 215 insertions(+), 127 deletions(-) create mode 100644 src/runtime-rs/crates/resource/src/volume/rawblock_volume.rs diff --git a/src/runtime-rs/crates/resource/src/volume/block_volume.rs b/src/runtime-rs/crates/resource/src/volume/block_volume.rs index fc79183d1..43bfa0a94 100644 --- a/src/runtime-rs/crates/resource/src/volume/block_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/block_volume.rs @@ -4,20 +4,17 @@ // SPDX-License-Identifier: Apache-2.0 // -use anyhow::{anyhow, Context, Result}; +use anyhow::{Context, Result}; use async_trait::async_trait; use nix::sys::{stat, stat::SFlag}; use tokio::sync::RwLock; use super::Volume; -use crate::volume::utils::{ - generate_shared_path, get_direct_volume_path, volume_mount_info, DEFAULT_VOLUME_FS_TYPE, - KATA_DIRECT_VOLUME_TYPE, KATA_MOUNT_BIND_TYPE, -}; +use crate::volume::utils::{handle_block_volume, DEFAULT_VOLUME_FS_TYPE, KATA_MOUNT_BIND_TYPE}; use hypervisor::{ device::{ device_manager::{do_handle_device, get_block_driver, DeviceManager}, - DeviceConfig, DeviceType, + DeviceConfig, }, BlockConfig, }; @@ -29,7 +26,7 @@ pub(crate) struct BlockVolume { device_id: String, } -/// BlockVolume for bind-mount block volume and direct block volume +/// BlockVolume for bind-mount block volume impl BlockVolume { pub(crate) async fn new( d: &RwLock, @@ -38,57 +35,13 @@ impl BlockVolume { sid: &str, ) -> Result { let mnt_src: &str = &m.source; - // default block device fs type: ext4. - let mut blk_dev_fstype = DEFAULT_VOLUME_FS_TYPE.to_string(); - let block_driver = get_block_driver(d).await; - - let block_device_config = match m.r#type.as_str() { - KATA_MOUNT_BIND_TYPE => { - let fstat = stat::stat(mnt_src).context(format!("stat {}", m.source))?; - - BlockConfig { - major: stat::major(fstat.st_rdev) as i64, - minor: stat::minor(fstat.st_rdev) as i64, - driver_option: block_driver, - ..Default::default() - } - } - KATA_DIRECT_VOLUME_TYPE => { - // get volume mountinfo from mountinfo.json - let v = volume_mount_info(mnt_src) - .context("deserde information from mountinfo.json")?; - // check volume type - if v.volume_type != KATA_DIRECT_VOLUME_TYPE { - return Err(anyhow!("volume type {:?} is invalid", v.volume_type)); - } - - let fstat = stat::stat(v.device.as_str()) - .with_context(|| format!("stat volume device file: {}", v.device.clone()))?; - if SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFREG - && SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFBLK - { - return Err(anyhow!( - "invalid volume device {:?} for volume type {:?}", - v.device, - v.volume_type - )); - } - - blk_dev_fstype = v.fs_type.clone(); - - BlockConfig { - path_on_host: v.device, - driver_option: block_driver, - ..Default::default() - } - } - _ => { - return Err(anyhow!( - "unsupport direct block volume r#type: {:?}", - m.r#type.as_str() - )) - } + let fstat = stat::stat(mnt_src).context(format!("stat {}", m.source))?; + let block_device_config = BlockConfig { + major: stat::major(fstat.st_rdev) as i64, + minor: stat::minor(fstat.st_rdev) as i64, + driver_option: block_driver, + ..Default::default() }; // create and insert block device into Kata VM @@ -96,55 +49,15 @@ impl BlockVolume { .await .context("do handle device failed.")?; - // storage - let mut storage = agent::Storage { - options: if read_only { - vec!["ro".to_string()] - } else { - Vec::new() - }, - ..Default::default() - }; - - // As the true Block Device wrapped in DeviceType, we need to - // get it out from the wrapper, and the device_id will be for - // BlockVolume. - // safe here, device_info is correct and only unwrap it. - let mut device_id = String::new(); - if let DeviceType::Block(device) = device_info { - // blk, mmioblk - storage.driver = device.config.driver_option; - // /dev/vdX - storage.source = device.config.virt_path; - device_id = device.device_id; - } - - // generate host guest shared path - let guest_path = generate_shared_path(m.destination.clone(), read_only, &device_id, sid) - .await - .context("generate host-guest shared path failed")?; - storage.mount_point = guest_path.clone(); - - // In some case, dest is device /dev/xxx - if m.destination.clone().starts_with("/dev") { - storage.fs_type = "bind".to_string(); - storage.options.append(&mut m.options.clone()); - } else { - // usually, the dest is directory. - storage.fs_type = blk_dev_fstype; - } - - let mount = oci::Mount { - destination: m.destination.clone(), - r#type: storage.fs_type.clone(), - source: guest_path, - options: m.options.clone(), - }; + let block_volume = + handle_block_volume(device_info, m, read_only, sid, DEFAULT_VOLUME_FS_TYPE) + .await + .context("do handle block volume failed")?; Ok(Self { - storage: Some(storage), - mount, - device_id, + storage: Some(block_volume.0), + mount: block_volume.1, + device_id: block_volume.2, }) } } @@ -178,28 +91,13 @@ impl Volume for BlockVolume { } } -pub(crate) fn is_block_volume(m: &oci::Mount) -> Result { - let vol_types = [KATA_MOUNT_BIND_TYPE, KATA_DIRECT_VOLUME_TYPE]; - if !vol_types.contains(&m.r#type.as_str()) { - return Ok(false); +pub(crate) fn is_block_volume(m: &oci::Mount) -> bool { + if m.r#type.as_str() != KATA_MOUNT_BIND_TYPE { + return false; } - let source = if m.r#type.as_str() == KATA_DIRECT_VOLUME_TYPE { - get_direct_volume_path(&m.source).context("get direct volume path failed")? - } else { - m.source.clone() - }; - - let fstat = - stat::stat(source.as_str()).context(format!("stat mount source {} failed.", source))?; - let s_flag = SFlag::from_bits_truncate(fstat.st_mode); - - match m.r#type.as_str() { - // case: mount bind and block device - KATA_MOUNT_BIND_TYPE if s_flag == SFlag::S_IFBLK => Ok(true), - // case: directvol and directory - KATA_DIRECT_VOLUME_TYPE if s_flag == SFlag::S_IFDIR => Ok(true), - // else: unsupported or todo for other volume type. - _ => Ok(false), + match stat::stat(m.source.as_str()) { + Ok(fstat) => SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFBLK, + Err(_) => false, } } diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs index 490181a1d..c8e502108 100644 --- a/src/runtime-rs/crates/resource/src/volume/mod.rs +++ b/src/runtime-rs/crates/resource/src/volume/mod.rs @@ -17,6 +17,9 @@ use vfio_volume::is_vfio_volume; pub mod spdk_volume; use spdk_volume::is_spdk_volume; +pub mod rawblock_volume; +use rawblock_volume::is_rawblock_volume; + use std::{sync::Arc, vec::Vec}; use anyhow::{Context, Result}; @@ -74,12 +77,19 @@ impl VolumeResource { shm_volume::ShmVolume::new(m, shm_size) .with_context(|| format!("new shm volume {:?}", m))?, ) - } else if is_block_volume(m).context("block volume type")? { + } else if is_block_volume(m) { // handle block volume Arc::new( block_volume::BlockVolume::new(d, m, read_only, sid) .await - .with_context(|| format!("new share fs volume {:?}", m))?, + .with_context(|| format!("new block volume {:?}", m))?, + ) + } else if is_rawblock_volume(m)? { + // handle rawblock volume + Arc::new( + rawblock_volume::RawblockVolume::new(d, m, read_only, sid) + .await + .with_context(|| format!("new rawblock volume {:?}", m))?, ) } else if is_vfio_volume(m) { Arc::new( diff --git a/src/runtime-rs/crates/resource/src/volume/rawblock_volume.rs b/src/runtime-rs/crates/resource/src/volume/rawblock_volume.rs new file mode 100644 index 000000000..bad715de4 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/volume/rawblock_volume.rs @@ -0,0 +1,123 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use nix::sys::{stat, stat::SFlag}; +use tokio::sync::RwLock; + +use super::Volume; +use crate::volume::utils::{ + get_direct_volume_path, handle_block_volume, volume_mount_info, KATA_DIRECT_VOLUME_TYPE, +}; +use hypervisor::{ + device::{ + device_manager::{do_handle_device, get_block_driver, DeviceManager}, + DeviceConfig, + }, + BlockConfig, +}; + +#[derive(Clone)] +pub(crate) struct RawblockVolume { + storage: Option, + mount: oci::Mount, + device_id: String, +} + +/// RawblockVolume for raw block volume +impl RawblockVolume { + pub(crate) async fn new( + d: &RwLock, + m: &oci::Mount, + read_only: bool, + sid: &str, + ) -> Result { + let mnt_src: &str = &m.source; + let block_driver = get_block_driver(d).await; + // get volume mountinfo from mountinfo.json + let v = volume_mount_info(mnt_src).context("deserde information from mountinfo.json")?; + // check volume type + if v.volume_type != KATA_DIRECT_VOLUME_TYPE { + return Err(anyhow!("volume type {:?} is invalid", v.volume_type)); + } + + let fstat = stat::stat(v.device.as_str()) + .with_context(|| format!("stat volume device file: {}", v.device.clone()))?; + if SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFREG + && SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFBLK + { + return Err(anyhow!( + "invalid volume device {:?} for volume type {:?}", + v.device, + v.volume_type + )); + } + + let block_config = BlockConfig { + path_on_host: v.device, + driver_option: block_driver, + ..Default::default() + }; + + // create and insert block device into Kata VM + let device_info = do_handle_device(d, &DeviceConfig::BlockCfg(block_config.clone())) + .await + .context("do handle device failed.")?; + + let block_volume = handle_block_volume(device_info, m, read_only, sid, &v.fs_type) + .await + .context("do handle block volume failed")?; + + Ok(Self { + storage: Some(block_volume.0), + mount: block_volume.1, + device_id: block_volume.2, + }) + } +} + +#[async_trait] +impl Volume for RawblockVolume { + fn get_volume_mount(&self) -> Result> { + Ok(vec![self.mount.clone()]) + } + + fn get_storage(&self) -> Result> { + let s = if let Some(s) = self.storage.as_ref() { + vec![s.clone()] + } else { + vec![] + }; + + Ok(s) + } + + async fn cleanup(&self, device_manager: &RwLock) -> Result<()> { + device_manager + .write() + .await + .try_remove_device(&self.device_id) + .await + } + + fn get_device_id(&self) -> Result> { + Ok(Some(self.device_id.clone())) + } +} + +pub(crate) fn is_rawblock_volume(m: &oci::Mount) -> Result { + // KATA_MOUNT_BIND_TYPE = "directvol" + if m.r#type.as_str() != KATA_DIRECT_VOLUME_TYPE { + return Ok(false); + } + + let source = get_direct_volume_path(&m.source).context("get direct volume path failed")?; + let fstat = + stat::stat(source.as_str()).context(format!("stat mount source {} failed.", source))?; + + Ok(SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFDIR) +} diff --git a/src/runtime-rs/crates/resource/src/volume/utils.rs b/src/runtime-rs/crates/resource/src/volume/utils.rs index d5f17d44b..034c3a910 100644 --- a/src/runtime-rs/crates/resource/src/volume/utils.rs +++ b/src/runtime-rs/crates/resource/src/volume/utils.rs @@ -17,6 +17,8 @@ use kata_types::mount::{ get_volume_mount_info, join_path, DirectVolumeMountInfo, KATA_DIRECT_VOLUME_ROOT_PATH, }; +use hypervisor::device::DeviceType; + pub const DEFAULT_VOLUME_FS_TYPE: &str = "ext4"; pub const KATA_MOUNT_BIND_TYPE: &str = "bind"; pub const KATA_DIRECT_VOLUME_TYPE: &str = "directvol"; @@ -74,3 +76,58 @@ pub(crate) async fn generate_shared_path( Ok(guest_path) } + +pub async fn handle_block_volume( + device_info: DeviceType, + m: &oci::Mount, + read_only: bool, + sid: &str, + fstype: &str, +) -> Result<(agent::Storage, oci::Mount, String)> { + // storage + let mut storage = agent::Storage { + options: if read_only { + vec!["ro".to_string()] + } else { + Vec::new() + }, + ..Default::default() + }; + + // As the true Block Device wrapped in DeviceType, we need to + // get it out from the wrapper, and the device_id will be for + // BlockVolume. + // safe here, device_info is correct and only unwrap it. + let mut device_id = String::new(); + if let DeviceType::Block(device) = device_info { + // blk, mmioblk + storage.driver = device.config.driver_option; + // /dev/vdX + storage.source = device.config.virt_path; + device_id = device.device_id; + } + + // generate host guest shared path + let guest_path = generate_shared_path(m.destination.clone(), read_only, &device_id, sid) + .await + .context("generate host-guest shared path failed")?; + storage.mount_point = guest_path.clone(); + + // In some case, dest is device /dev/xxx + if m.destination.clone().starts_with("/dev") { + storage.fs_type = "bind".to_string(); + storage.options.append(&mut m.options.clone()); + } else { + // usually, the dest is directory. + storage.fs_type = fstype.to_owned(); + } + + let mount = oci::Mount { + destination: m.destination.clone(), + r#type: storage.fs_type.clone(), + source: guest_path, + options: m.options.clone(), + }; + + Ok((storage, mount, device_id)) +} From 6731466b13379766722508c732bfe419c2e803bc Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Sat, 25 Nov 2023 19:51:12 +0800 Subject: [PATCH 2/8] runtime-rs: set a standard NotFound when direct volume path not found. Fixes: #8300 Signed-off-by: alex.lyn --- src/libs/kata-types/src/mount.rs | 2 +- src/tools/kata-ctl/src/ops/volume_ops.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libs/kata-types/src/mount.rs b/src/libs/kata-types/src/mount.rs index 831cc8f1c..d5e378e7d 100644 --- a/src/libs/kata-types/src/mount.rs +++ b/src/libs/kata-types/src/mount.rs @@ -483,7 +483,7 @@ impl StorageHandlerManager { /// The `volume_path` is base64-url-encoded and then safely joined to the `prefix` pub fn join_path(prefix: &str, volume_path: &str) -> Result { if volume_path.is_empty() { - return Err(anyhow!("volume path must not be empty")); + return Err(anyhow!(std::io::ErrorKind::NotFound)); } let b64_url_encoded_path = base64::encode_config(volume_path.as_bytes(), base64::URL_SAFE); diff --git a/src/tools/kata-ctl/src/ops/volume_ops.rs b/src/tools/kata-ctl/src/ops/volume_ops.rs index 56768cf15..2f7fded4e 100644 --- a/src/tools/kata-ctl/src/ops/volume_ops.rs +++ b/src/tools/kata-ctl/src/ops/volume_ops.rs @@ -219,7 +219,7 @@ mod tests { TestData { rootfs: root_fs_str, volume_path: "", - result: Err(anyhow!("volume path must not be empty")), + result: Err(anyhow!(std::io::ErrorKind::NotFound)), }, TestData { rootfs: root_fs_str, From 17d2d465d1504865ecacba3c631ef303285c24ec Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Sat, 25 Nov 2023 21:04:55 +0800 Subject: [PATCH 3/8] runtime-rs: re-organize the volumes with adding new direct_volumes. Add a new dire direct_volumes containing spdk, rawblock and vfio volume. Fixes: #8300 Signed-off-by: alex.lyn --- .../resource/src/volume/direct_volumes/mod.rs | 32 +++++++++++++++++++ .../{ => direct_volumes}/rawblock_volume.rs | 14 ++++---- .../{ => direct_volumes}/spdk_volume.rs | 11 ++++--- .../{ => direct_volumes}/vfio_volume.rs | 10 +++--- .../crates/resource/src/volume/mod.rs | 20 ++++++------ .../crates/resource/src/volume/utils.rs | 20 ------------ 6 files changed, 61 insertions(+), 46 deletions(-) create mode 100644 src/runtime-rs/crates/resource/src/volume/direct_volumes/mod.rs rename src/runtime-rs/crates/resource/src/volume/{ => direct_volumes}/rawblock_volume.rs (93%) rename src/runtime-rs/crates/resource/src/volume/{ => direct_volumes}/spdk_volume.rs (96%) rename src/runtime-rs/crates/resource/src/volume/{ => direct_volumes}/vfio_volume.rs (96%) diff --git a/src/runtime-rs/crates/resource/src/volume/direct_volumes/mod.rs b/src/runtime-rs/crates/resource/src/volume/direct_volumes/mod.rs new file mode 100644 index 000000000..77c0ab9ec --- /dev/null +++ b/src/runtime-rs/crates/resource/src/volume/direct_volumes/mod.rs @@ -0,0 +1,32 @@ +// Copyright (c) 2023 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::{Context, Result}; + +use kata_types::mount::{ + get_volume_mount_info, join_path, DirectVolumeMountInfo, KATA_DIRECT_VOLUME_ROOT_PATH, +}; + +pub mod rawblock_volume; +pub mod spdk_volume; +pub mod vfio_volume; + +pub const KATA_DIRECT_VOLUME_TYPE: &str = "directvol"; +pub const KATA_VFIO_VOLUME_TYPE: &str = "vfiovol"; +pub const KATA_SPDK_VOLUME_TYPE: &str = "spdkvol"; +pub const KATA_SPOOL_VOLUME_TYPE: &str = "spoolvol"; + +// volume mount info load infomation from mountinfo.json +pub fn volume_mount_info(volume_path: &str) -> Result { + get_volume_mount_info(volume_path) +} + +// get direct volume path whose volume_path encoded with base64 +pub fn get_direct_volume_path(volume_path: &str) -> Result { + let volume_full_path = + join_path(KATA_DIRECT_VOLUME_ROOT_PATH, volume_path).context("failed to join path.")?; + + Ok(volume_full_path.display().to_string()) +} diff --git a/src/runtime-rs/crates/resource/src/volume/rawblock_volume.rs b/src/runtime-rs/crates/resource/src/volume/direct_volumes/rawblock_volume.rs similarity index 93% rename from src/runtime-rs/crates/resource/src/volume/rawblock_volume.rs rename to src/runtime-rs/crates/resource/src/volume/direct_volumes/rawblock_volume.rs index bad715de4..bcd6ca371 100644 --- a/src/runtime-rs/crates/resource/src/volume/rawblock_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/direct_volumes/rawblock_volume.rs @@ -1,5 +1,5 @@ -// Copyright (c) 2019-2022 Alibaba Cloud -// Copyright (c) 2019-2022 Ant Group +// Copyright (c) 2023 Alibaba Cloud +// Copyright (c) 2023 Ant Group // // SPDX-License-Identifier: Apache-2.0 // @@ -9,10 +9,6 @@ use async_trait::async_trait; use nix::sys::{stat, stat::SFlag}; use tokio::sync::RwLock; -use super::Volume; -use crate::volume::utils::{ - get_direct_volume_path, handle_block_volume, volume_mount_info, KATA_DIRECT_VOLUME_TYPE, -}; use hypervisor::{ device::{ device_manager::{do_handle_device, get_block_driver, DeviceManager}, @@ -21,6 +17,12 @@ use hypervisor::{ BlockConfig, }; +use crate::volume::{ + direct_volumes::{get_direct_volume_path, volume_mount_info, KATA_DIRECT_VOLUME_TYPE}, + utils::handle_block_volume, + Volume, +}; + #[derive(Clone)] pub(crate) struct RawblockVolume { storage: Option, diff --git a/src/runtime-rs/crates/resource/src/volume/spdk_volume.rs b/src/runtime-rs/crates/resource/src/volume/direct_volumes/spdk_volume.rs similarity index 96% rename from src/runtime-rs/crates/resource/src/volume/spdk_volume.rs rename to src/runtime-rs/crates/resource/src/volume/direct_volumes/spdk_volume.rs index 6789d6d37..f63fd33ab 100644 --- a/src/runtime-rs/crates/resource/src/volume/spdk_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/direct_volumes/spdk_volume.rs @@ -9,11 +9,6 @@ use async_trait::async_trait; use nix::sys::{stat, stat::SFlag}; use tokio::sync::RwLock; -use super::Volume; -use crate::volume::utils::{ - generate_shared_path, volume_mount_info, DEFAULT_VOLUME_FS_TYPE, KATA_SPDK_VOLUME_TYPE, - KATA_SPOOL_VOLUME_TYPE, -}; use hypervisor::{ device::{ device_manager::{do_handle_device, get_block_driver, DeviceManager}, @@ -22,6 +17,12 @@ use hypervisor::{ VhostUserConfig, VhostUserType, }; +use crate::volume::{ + direct_volumes::{volume_mount_info, KATA_SPDK_VOLUME_TYPE, KATA_SPOOL_VOLUME_TYPE}, + utils::{generate_shared_path, DEFAULT_VOLUME_FS_TYPE}, + Volume, +}; + /// SPDKVolume: spdk block device volume #[derive(Clone)] pub(crate) struct SPDKVolume { diff --git a/src/runtime-rs/crates/resource/src/volume/vfio_volume.rs b/src/runtime-rs/crates/resource/src/volume/direct_volumes/vfio_volume.rs similarity index 96% rename from src/runtime-rs/crates/resource/src/volume/vfio_volume.rs rename to src/runtime-rs/crates/resource/src/volume/direct_volumes/vfio_volume.rs index dab8c7dae..a4c226c4b 100644 --- a/src/runtime-rs/crates/resource/src/volume/vfio_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/direct_volumes/vfio_volume.rs @@ -8,10 +8,6 @@ use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use tokio::sync::RwLock; -use super::Volume; -use crate::volume::utils::{ - generate_shared_path, volume_mount_info, DEFAULT_VOLUME_FS_TYPE, KATA_VFIO_VOLUME_TYPE, -}; use hypervisor::{ device::{ device_manager::{do_handle_device, DeviceManager}, @@ -20,6 +16,12 @@ use hypervisor::{ get_vfio_device, VfioConfig, }; +use crate::volume::{ + direct_volumes::{volume_mount_info, KATA_VFIO_VOLUME_TYPE}, + utils::{generate_shared_path, DEFAULT_VOLUME_FS_TYPE}, + Volume, +}; + pub(crate) struct VfioVolume { storage: Option, mount: oci::Mount, diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs index c8e502108..cdae44d6e 100644 --- a/src/runtime-rs/crates/resource/src/volume/mod.rs +++ b/src/runtime-rs/crates/resource/src/volume/mod.rs @@ -11,14 +11,12 @@ mod share_fs_volume; mod shm_volume; pub mod utils; -pub mod vfio_volume; -use vfio_volume::is_vfio_volume; - -pub mod spdk_volume; -use spdk_volume::is_spdk_volume; - -pub mod rawblock_volume; -use rawblock_volume::is_rawblock_volume; +pub mod direct_volumes; +use direct_volumes::{ + rawblock_volume::{is_rawblock_volume, RawblockVolume}, + spdk_volume::{is_spdk_volume, SPDKVolume}, + vfio_volume::{is_vfio_volume, VfioVolume}, +}; use std::{sync::Arc, vec::Vec}; @@ -87,19 +85,19 @@ impl VolumeResource { } else if is_rawblock_volume(m)? { // handle rawblock volume Arc::new( - rawblock_volume::RawblockVolume::new(d, m, read_only, sid) + RawblockVolume::new(d, m, read_only, sid) .await .with_context(|| format!("new rawblock volume {:?}", m))?, ) } else if is_vfio_volume(m) { Arc::new( - vfio_volume::VfioVolume::new(d, m, read_only, cid, sid) + VfioVolume::new(d, m, read_only, cid, sid) .await .with_context(|| format!("new vfio volume {:?}", m))?, ) } else if is_spdk_volume(m) { Arc::new( - spdk_volume::SPDKVolume::new(d, m, read_only, cid, sid) + SPDKVolume::new(d, m, read_only, cid, sid) .await .with_context(|| format!("create spdk volume {:?}", m))?, ) diff --git a/src/runtime-rs/crates/resource/src/volume/utils.rs b/src/runtime-rs/crates/resource/src/volume/utils.rs index 034c3a910..1fb361426 100644 --- a/src/runtime-rs/crates/resource/src/volume/utils.rs +++ b/src/runtime-rs/crates/resource/src/volume/utils.rs @@ -13,31 +13,11 @@ use crate::{ volume::share_fs_volume::generate_mount_path, }; use kata_sys_util::eother; -use kata_types::mount::{ - get_volume_mount_info, join_path, DirectVolumeMountInfo, KATA_DIRECT_VOLUME_ROOT_PATH, -}; use hypervisor::device::DeviceType; pub const DEFAULT_VOLUME_FS_TYPE: &str = "ext4"; pub const KATA_MOUNT_BIND_TYPE: &str = "bind"; -pub const KATA_DIRECT_VOLUME_TYPE: &str = "directvol"; -pub const KATA_VFIO_VOLUME_TYPE: &str = "vfiovol"; -pub const KATA_SPDK_VOLUME_TYPE: &str = "spdkvol"; -pub const KATA_SPOOL_VOLUME_TYPE: &str = "spoolvol"; - -// volume mount info load infomation from mountinfo.json -pub fn volume_mount_info(volume_path: &str) -> Result { - get_volume_mount_info(volume_path) -} - -// get direct volume path whose volume_path encoded with base64 -pub fn get_direct_volume_path(volume_path: &str) -> Result { - let volume_full_path = - join_path(KATA_DIRECT_VOLUME_ROOT_PATH, volume_path).context("failed to join path.")?; - - Ok(volume_full_path.display().to_string()) -} pub fn get_file_name>(src: P) -> Result { let file_name = src From b952c5c5ce8b8745ca883d8ab31aa1f862d1efdc Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Sat, 25 Nov 2023 21:13:03 +0800 Subject: [PATCH 4/8] runtime-rs: add support kata/multi-containers sharing one spdk volume. Fiexes: #8300 Signed-off-by: alex.lyn --- .../src/volume/direct_volumes/spdk_volume.rs | 24 +++++++++---------- .../crates/resource/src/volume/mod.rs | 2 +- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/runtime-rs/crates/resource/src/volume/direct_volumes/spdk_volume.rs b/src/runtime-rs/crates/resource/src/volume/direct_volumes/spdk_volume.rs index f63fd33ab..0e66918ca 100644 --- a/src/runtime-rs/crates/resource/src/volume/direct_volumes/spdk_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/direct_volumes/spdk_volume.rs @@ -36,7 +36,6 @@ impl SPDKVolume { d: &RwLock, m: &oci::Mount, read_only: bool, - cid: &str, sid: &str, ) -> Result { let mnt_src: &str = &m.source; @@ -100,23 +99,16 @@ impl SPDKVolume { .await .context("do handle device failed.")?; - // generate host guest shared path - let guest_path = generate_shared_path(m.destination.clone(), read_only, cid, sid) - .await - .context("generate host-guest shared path failed")?; - // storage let mut storage = agent::Storage { - mount_point: guest_path.clone(), + options: if read_only { + vec!["ro".to_string()] + } else { + Vec::new() + }, ..Default::default() }; - storage.options = if read_only { - vec!["ro".to_string()] - } else { - Vec::new() - }; - let mut device_id = String::new(); if let DeviceType::VhostUserBlk(device) = device_info { // blk, mmioblk @@ -126,6 +118,12 @@ impl SPDKVolume { device_id = device.device_id; } + // generate host guest shared path + let guest_path = generate_shared_path(m.destination.clone(), read_only, &device_id, sid) + .await + .context("generate host-guest shared path failed")?; + storage.mount_point = guest_path.clone(); + if m.r#type != "bind" { storage.fs_type = v.fs_type.clone(); } else { diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs index cdae44d6e..156fc535c 100644 --- a/src/runtime-rs/crates/resource/src/volume/mod.rs +++ b/src/runtime-rs/crates/resource/src/volume/mod.rs @@ -97,7 +97,7 @@ impl VolumeResource { ) } else if is_spdk_volume(m) { Arc::new( - SPDKVolume::new(d, m, read_only, cid, sid) + SPDKVolume::new(d, m, read_only, sid) .await .with_context(|| format!("create spdk volume {:?}", m))?, ) From e3becea5666c8e0bfded2bb345ddc3e7ae9be16d Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Sat, 25 Nov 2023 21:15:42 +0800 Subject: [PATCH 5/8] runtime-rs: add support kata/multi-containers sharing one vfio volume. Fiexes: #8300 Signed-off-by: alex.lyn --- .../src/volume/direct_volumes/vfio_volume.rs | 13 ++++++------- src/runtime-rs/crates/resource/src/volume/mod.rs | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/runtime-rs/crates/resource/src/volume/direct_volumes/vfio_volume.rs b/src/runtime-rs/crates/resource/src/volume/direct_volumes/vfio_volume.rs index a4c226c4b..132827fd3 100644 --- a/src/runtime-rs/crates/resource/src/volume/direct_volumes/vfio_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/direct_volumes/vfio_volume.rs @@ -34,7 +34,6 @@ impl VfioVolume { d: &RwLock, m: &oci::Mount, read_only: bool, - cid: &str, sid: &str, ) -> Result { let mnt_src: &str = &m.source; @@ -59,11 +58,6 @@ impl VfioVolume { .await .context("do handle device failed.")?; - // generate host guest shared path - let guest_path = generate_shared_path(m.destination.clone(), read_only, cid, sid) - .await - .context("generate host-guest shared path failed")?; - let storage_options = if read_only { vec!["ro".to_string()] } else { @@ -72,7 +66,6 @@ impl VfioVolume { let mut storage = agent::Storage { options: storage_options, - mount_point: guest_path.clone(), ..Default::default() }; @@ -84,6 +77,12 @@ impl VfioVolume { storage.source = device.config.virt_path.unwrap().1; } + // generate host guest shared path + let guest_path = generate_shared_path(m.destination.clone(), read_only, &device_id, sid) + .await + .context("generate host-guest shared path failed")?; + storage.mount_point = guest_path.clone(); + if m.r#type != "bind" { storage.fs_type = v.fs_type.clone(); } else { diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs index 156fc535c..c6ecc57ca 100644 --- a/src/runtime-rs/crates/resource/src/volume/mod.rs +++ b/src/runtime-rs/crates/resource/src/volume/mod.rs @@ -91,7 +91,7 @@ impl VolumeResource { ) } else if is_vfio_volume(m) { Arc::new( - VfioVolume::new(d, m, read_only, cid, sid) + VfioVolume::new(d, m, read_only, sid) .await .with_context(|| format!("new vfio volume {:?}", m))?, ) From f9737290293d890f08dcceda0f3dbaafeb7c5b63 Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Sat, 25 Nov 2023 23:23:27 +0800 Subject: [PATCH 6/8] runtime-rs: Enhancing DirectVolMount Handling for current Infra. The current infra(K8S, CSI, CRI, Containerd) for Kata containers is unable to properly handle direct volumes, resulting in the need for workarounds like searching/comparision and then patch up volume type. In this commit, reimplement of handling method is added to support raw block volume which backends may be rawdisk or other format file. Fixes: #8300 Signed-off-by: alex.lyn --- .../resource/src/volume/direct_volume.rs | 87 +++++++++++++++++++ .../volume/direct_volumes/rawblock_volume.rs | 44 ++++------ .../crates/resource/src/volume/mod.rs | 18 ++-- 3 files changed, 113 insertions(+), 36 deletions(-) create mode 100644 src/runtime-rs/crates/resource/src/volume/direct_volume.rs diff --git a/src/runtime-rs/crates/resource/src/volume/direct_volume.rs b/src/runtime-rs/crates/resource/src/volume/direct_volume.rs new file mode 100644 index 000000000..51f2006b5 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/volume/direct_volume.rs @@ -0,0 +1,87 @@ +// Copyright (c) 2023 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::sync::Arc; + +use anyhow::{anyhow, Context, Result}; +use kata_types::mount::DirectVolumeMountInfo; +use nix::sys::{stat, stat::SFlag}; +use tokio::sync::RwLock; + +use hypervisor::device::device_manager::DeviceManager; + +use crate::volume::{ + direct_volumes::{ + get_direct_volume_path, rawblock_volume, volume_mount_info, KATA_DIRECT_VOLUME_TYPE, + }, + utils::KATA_MOUNT_BIND_TYPE, + Volume, +}; + +pub(crate) async fn handle_direct_volume( + d: &RwLock, + m: &oci::Mount, + read_only: bool, + sid: &str, +) -> Result>> { + // In the direct volume scenario, we check if the source of a mount is in the + // /run/kata-containers/shared/direct-volumes/SID path by iterating over all the mounts. + // If the source is not in the path with error kind *NotFound*, we ignore the error + // and we treat it as block volume with oci Mount.type *bind*. Just fill in the block + // volume info in the DirectVolumeMountInfo + let mount_info: DirectVolumeMountInfo = match volume_mount_info(&m.source) { + Ok(mount_info) => mount_info, + Err(e) => { + // First, We need filter the non-io::ErrorKind. + if !e.is::() { + return Err(anyhow!(format!( + "unexpected error occurs when parse mount info for {:?}, with error {:?}", + &m.source, + e.to_string() + ))); + } + + // Second, we need filter non-NotFound error. + // Safe to unwrap here, as the error is of type std::io::ErrorKind. + let error_kind = e.downcast_ref::().unwrap(); + if *error_kind != std::io::ErrorKind::NotFound { + return Err(anyhow!(format!( + "failed to parse volume mount info for {:?}, with error {:?}", + &m.source, + e.to_string() + ))); + } + + // Third, if the case is *NotFound* , we just return Ok(None). + return Ok(None); + } + }; + + let direct_volume: Arc = Arc::new( + rawblock_volume::RawblockVolume::new(d, m, &mount_info, read_only, sid) + .await + .with_context(|| format!("new sid {:?} rawblock volume {:?}", &sid, m))?, + ); + + Ok(Some(direct_volume)) +} + +pub(crate) fn is_direct_volume(m: &oci::Mount) -> Result { + let mount_type = m.r#type.as_str(); + // Filter the non-bind volume and non-direct-vol volume + let vol_types = [KATA_MOUNT_BIND_TYPE, KATA_DIRECT_VOLUME_TYPE]; + if !vol_types.contains(&mount_type) { + return Ok(false); + } + + match get_direct_volume_path(&m.source) { + Ok(directvol_path) => { + let fstat = stat::stat(directvol_path.as_str()) + .context(format!("stat mount source {} failed.", directvol_path))?; + Ok(SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFDIR) + } + Err(_) => Ok(false), + } +} diff --git a/src/runtime-rs/crates/resource/src/volume/direct_volumes/rawblock_volume.rs b/src/runtime-rs/crates/resource/src/volume/direct_volumes/rawblock_volume.rs index bcd6ca371..5ee02e4f2 100644 --- a/src/runtime-rs/crates/resource/src/volume/direct_volumes/rawblock_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/direct_volumes/rawblock_volume.rs @@ -16,12 +16,9 @@ use hypervisor::{ }, BlockConfig, }; +use kata_types::mount::DirectVolumeMountInfo; -use crate::volume::{ - direct_volumes::{get_direct_volume_path, volume_mount_info, KATA_DIRECT_VOLUME_TYPE}, - utils::handle_block_volume, - Volume, -}; +use crate::volume::{direct_volumes::KATA_DIRECT_VOLUME_TYPE, utils::handle_block_volume, Volume}; #[derive(Clone)] pub(crate) struct RawblockVolume { @@ -35,32 +32,34 @@ impl RawblockVolume { pub(crate) async fn new( d: &RwLock, m: &oci::Mount, + mount_info: &DirectVolumeMountInfo, read_only: bool, sid: &str, ) -> Result { - let mnt_src: &str = &m.source; let block_driver = get_block_driver(d).await; - // get volume mountinfo from mountinfo.json - let v = volume_mount_info(mnt_src).context("deserde information from mountinfo.json")?; + // check volume type - if v.volume_type != KATA_DIRECT_VOLUME_TYPE { - return Err(anyhow!("volume type {:?} is invalid", v.volume_type)); + if mount_info.volume_type != KATA_DIRECT_VOLUME_TYPE { + return Err(anyhow!( + "volume type {:?} is invalid", + mount_info.volume_type + )); } - let fstat = stat::stat(v.device.as_str()) - .with_context(|| format!("stat volume device file: {}", v.device.clone()))?; + let fstat = stat::stat(mount_info.device.as_str()) + .with_context(|| format!("stat volume device file: {}", mount_info.device.clone()))?; if SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFREG && SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFBLK { return Err(anyhow!( "invalid volume device {:?} for volume type {:?}", - v.device, - v.volume_type + mount_info.device, + mount_info.volume_type )); } let block_config = BlockConfig { - path_on_host: v.device, + path_on_host: mount_info.device.clone(), driver_option: block_driver, ..Default::default() }; @@ -70,7 +69,7 @@ impl RawblockVolume { .await .context("do handle device failed.")?; - let block_volume = handle_block_volume(device_info, m, read_only, sid, &v.fs_type) + let block_volume = handle_block_volume(device_info, m, read_only, sid, &mount_info.fs_type) .await .context("do handle block volume failed")?; @@ -110,16 +109,3 @@ impl Volume for RawblockVolume { Ok(Some(self.device_id.clone())) } } - -pub(crate) fn is_rawblock_volume(m: &oci::Mount) -> Result { - // KATA_MOUNT_BIND_TYPE = "directvol" - if m.r#type.as_str() != KATA_DIRECT_VOLUME_TYPE { - return Ok(false); - } - - let source = get_direct_volume_path(&m.source).context("get direct volume path failed")?; - let fstat = - stat::stat(source.as_str()).context(format!("stat mount source {} failed.", source))?; - - Ok(SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFDIR) -} diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs index c6ecc57ca..f4857fa72 100644 --- a/src/runtime-rs/crates/resource/src/volume/mod.rs +++ b/src/runtime-rs/crates/resource/src/volume/mod.rs @@ -11,9 +11,11 @@ mod share_fs_volume; mod shm_volume; pub mod utils; +pub mod direct_volume; +use crate::volume::direct_volume::is_direct_volume; + pub mod direct_volumes; use direct_volumes::{ - rawblock_volume::{is_rawblock_volume, RawblockVolume}, spdk_volume::{is_spdk_volume, SPDKVolume}, vfio_volume::{is_vfio_volume, VfioVolume}, }; @@ -82,13 +84,15 @@ impl VolumeResource { .await .with_context(|| format!("new block volume {:?}", m))?, ) - } else if is_rawblock_volume(m)? { + } else if is_direct_volume(m)? { // handle rawblock volume - Arc::new( - RawblockVolume::new(d, m, read_only, sid) - .await - .with_context(|| format!("new rawblock volume {:?}", m))?, - ) + match direct_volume::handle_direct_volume(d, m, read_only, sid) + .await + .context("handle direct volume")? + { + Some(directvol) => directvol, + None => continue, + } } else if is_vfio_volume(m) { Arc::new( VfioVolume::new(d, m, read_only, sid) From e3fd40312640644b302eb55d7e208f9b349c42fa Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Sun, 26 Nov 2023 00:19:36 +0800 Subject: [PATCH 7/8] runtime-rs: enhancement of spdk volume. (1) Add enum DirectVolumeType for direct volumes. (2) Reimplement spdk volume into direct_volume and do alignment of rawblock volume. Fixes: #8300 Signed-off-by: alex.lyn --- .../resource/src/volume/direct_volume.rs | 39 +++++++++++++++---- .../src/volume/direct_volumes/spdk_volume.rs | 38 +++++++----------- .../crates/resource/src/volume/mod.rs | 11 +----- 3 files changed, 46 insertions(+), 42 deletions(-) diff --git a/src/runtime-rs/crates/resource/src/volume/direct_volume.rs b/src/runtime-rs/crates/resource/src/volume/direct_volume.rs index 51f2006b5..70569d4ad 100644 --- a/src/runtime-rs/crates/resource/src/volume/direct_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/direct_volume.rs @@ -14,12 +14,25 @@ use hypervisor::device::device_manager::DeviceManager; use crate::volume::{ direct_volumes::{ - get_direct_volume_path, rawblock_volume, volume_mount_info, KATA_DIRECT_VOLUME_TYPE, + get_direct_volume_path, rawblock_volume, spdk_volume, volume_mount_info, + KATA_DIRECT_VOLUME_TYPE, KATA_SPDK_VOLUME_TYPE, KATA_SPOOL_VOLUME_TYPE, }, utils::KATA_MOUNT_BIND_TYPE, Volume, }; +enum DirectVolumeType { + RawBlock, + Spdk, +} + +fn to_volume_type(volume_type: &str) -> DirectVolumeType { + match volume_type { + KATA_SPDK_VOLUME_TYPE | KATA_SPOOL_VOLUME_TYPE => DirectVolumeType::Spdk, + _ => DirectVolumeType::RawBlock, + } +} + pub(crate) async fn handle_direct_volume( d: &RwLock, m: &oci::Mount, @@ -59,11 +72,18 @@ pub(crate) async fn handle_direct_volume( } }; - let direct_volume: Arc = Arc::new( - rawblock_volume::RawblockVolume::new(d, m, &mount_info, read_only, sid) - .await - .with_context(|| format!("new sid {:?} rawblock volume {:?}", &sid, m))?, - ); + let direct_volume: Arc = match to_volume_type(mount_info.volume_type.as_str()) { + DirectVolumeType::RawBlock => Arc::new( + rawblock_volume::RawblockVolume::new(d, m, &mount_info, read_only, sid) + .await + .with_context(|| format!("new sid {:?} rawblock volume {:?}", &sid, m))?, + ), + DirectVolumeType::Spdk => Arc::new( + spdk_volume::SPDKVolume::new(d, m, &mount_info, read_only, sid) + .await + .with_context(|| format!("create spdk volume {:?}", m))?, + ), + }; Ok(Some(direct_volume)) } @@ -71,7 +91,12 @@ pub(crate) async fn handle_direct_volume( pub(crate) fn is_direct_volume(m: &oci::Mount) -> Result { let mount_type = m.r#type.as_str(); // Filter the non-bind volume and non-direct-vol volume - let vol_types = [KATA_MOUNT_BIND_TYPE, KATA_DIRECT_VOLUME_TYPE]; + let vol_types = [ + KATA_MOUNT_BIND_TYPE, + KATA_DIRECT_VOLUME_TYPE, + KATA_SPDK_VOLUME_TYPE, + KATA_SPOOL_VOLUME_TYPE, + ]; if !vol_types.contains(&mount_type) { return Ok(false); } diff --git a/src/runtime-rs/crates/resource/src/volume/direct_volumes/spdk_volume.rs b/src/runtime-rs/crates/resource/src/volume/direct_volumes/spdk_volume.rs index 0e66918ca..7e28de9ca 100644 --- a/src/runtime-rs/crates/resource/src/volume/direct_volumes/spdk_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/direct_volumes/spdk_volume.rs @@ -6,6 +6,7 @@ use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; +use kata_types::mount::DirectVolumeMountInfo; use nix::sys::{stat, stat::SFlag}; use tokio::sync::RwLock; @@ -18,7 +19,7 @@ use hypervisor::{ }; use crate::volume::{ - direct_volumes::{volume_mount_info, KATA_SPDK_VOLUME_TYPE, KATA_SPOOL_VOLUME_TYPE}, + direct_volumes::{KATA_SPDK_VOLUME_TYPE, KATA_SPOOL_VOLUME_TYPE}, utils::{generate_shared_path, DEFAULT_VOLUME_FS_TYPE}, Volume, }; @@ -35,26 +36,23 @@ impl SPDKVolume { pub(crate) async fn new( d: &RwLock, m: &oci::Mount, + mount_info: &DirectVolumeMountInfo, read_only: bool, sid: &str, ) -> Result { - let mnt_src: &str = &m.source; - - // deserde Information from mountinfo.json - let v = volume_mount_info(mnt_src).context("deserde information from mountinfo.json")?; - let device = match v.volume_type.as_str() { + let device = match mount_info.volume_type.as_str() { KATA_SPDK_VOLUME_TYPE => { - if v.device.starts_with("spdk://") { - v.device.clone() + if mount_info.device.starts_with("spdk://") { + mount_info.device.clone() } else { - format!("spdk://{}", v.device.as_str()) + format!("spdk://{}", mount_info.device.as_str()) } } KATA_SPOOL_VOLUME_TYPE => { - if v.device.starts_with("spool://") { - v.device.clone() + if mount_info.device.starts_with("spool://") { + mount_info.device.clone() } else { - format!("spool://{}", v.device.as_str()) + format!("spool://{}", mount_info.device.as_str()) } } _ => return Err(anyhow!("mountinfo.json is invalid")), @@ -82,12 +80,12 @@ impl SPDKVolume { ..Default::default() }; - if let Some(num) = v.metadata.get("num_queues") { + if let Some(num) = mount_info.metadata.get("num_queues") { vhu_blk_config.num_queues = num .parse::() .context("num queues parse usize failed.")?; } - if let Some(size) = v.metadata.get("queue_size") { + if let Some(size) = mount_info.metadata.get("queue_size") { vhu_blk_config.queue_size = size .parse::() .context("num queues parse u32 failed.")?; @@ -125,7 +123,7 @@ impl SPDKVolume { storage.mount_point = guest_path.clone(); if m.r#type != "bind" { - storage.fs_type = v.fs_type.clone(); + storage.fs_type = mount_info.fs_type.clone(); } else { storage.fs_type = DEFAULT_VOLUME_FS_TYPE.to_string(); } @@ -179,13 +177,3 @@ impl Volume for SPDKVolume { Ok(Some(self.device_id.clone())) } } - -pub(crate) fn is_spdk_volume(m: &oci::Mount) -> bool { - // spdkvol or spoolvol will share the same implementation - let vol_types = [KATA_SPDK_VOLUME_TYPE, KATA_SPOOL_VOLUME_TYPE]; - if vol_types.contains(&m.r#type.as_str()) { - return true; - } - - false -} diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs index f4857fa72..6c40d9348 100644 --- a/src/runtime-rs/crates/resource/src/volume/mod.rs +++ b/src/runtime-rs/crates/resource/src/volume/mod.rs @@ -15,10 +15,7 @@ pub mod direct_volume; use crate::volume::direct_volume::is_direct_volume; pub mod direct_volumes; -use direct_volumes::{ - spdk_volume::{is_spdk_volume, SPDKVolume}, - vfio_volume::{is_vfio_volume, VfioVolume}, -}; +use direct_volumes::vfio_volume::{is_vfio_volume, VfioVolume}; use std::{sync::Arc, vec::Vec}; @@ -99,12 +96,6 @@ impl VolumeResource { .await .with_context(|| format!("new vfio volume {:?}", m))?, ) - } else if is_spdk_volume(m) { - Arc::new( - SPDKVolume::new(d, m, read_only, sid) - .await - .with_context(|| format!("create spdk volume {:?}", m))?, - ) } else if let Some(options) = get_huge_page_option(m).context("failed to check huge page")? { From fe68f25beaf8e1bf3fac08e2162de2781b93054f Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Sun, 26 Nov 2023 00:25:28 +0800 Subject: [PATCH 8/8] runtime-rs: enhancement of vfio volume. Reimplement vfio volume into direct_volume and do alignment of rawblock/spdk volume. Fixes: #8300 Signed-off-by: alex.lyn --- .../resource/src/volume/direct_volume.rs | 11 +++++++- .../src/volume/direct_volumes/vfio_volume.rs | 28 +++++-------------- .../crates/resource/src/volume/mod.rs | 10 +------ 3 files changed, 18 insertions(+), 31 deletions(-) diff --git a/src/runtime-rs/crates/resource/src/volume/direct_volume.rs b/src/runtime-rs/crates/resource/src/volume/direct_volume.rs index 70569d4ad..08849c98c 100644 --- a/src/runtime-rs/crates/resource/src/volume/direct_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/direct_volume.rs @@ -14,8 +14,9 @@ use hypervisor::device::device_manager::DeviceManager; use crate::volume::{ direct_volumes::{ - get_direct_volume_path, rawblock_volume, spdk_volume, volume_mount_info, + get_direct_volume_path, rawblock_volume, spdk_volume, vfio_volume, volume_mount_info, KATA_DIRECT_VOLUME_TYPE, KATA_SPDK_VOLUME_TYPE, KATA_SPOOL_VOLUME_TYPE, + KATA_VFIO_VOLUME_TYPE, }, utils::KATA_MOUNT_BIND_TYPE, Volume, @@ -24,11 +25,13 @@ use crate::volume::{ enum DirectVolumeType { RawBlock, Spdk, + Vfio, } fn to_volume_type(volume_type: &str) -> DirectVolumeType { match volume_type { KATA_SPDK_VOLUME_TYPE | KATA_SPOOL_VOLUME_TYPE => DirectVolumeType::Spdk, + KATA_VFIO_VOLUME_TYPE => DirectVolumeType::Vfio, _ => DirectVolumeType::RawBlock, } } @@ -83,6 +86,11 @@ pub(crate) async fn handle_direct_volume( .await .with_context(|| format!("create spdk volume {:?}", m))?, ), + DirectVolumeType::Vfio => Arc::new( + vfio_volume::VfioVolume::new(d, m, &mount_info, read_only, sid) + .await + .with_context(|| format!("new vfio volume {:?}", m))?, + ), }; Ok(Some(direct_volume)) @@ -94,6 +102,7 @@ pub(crate) fn is_direct_volume(m: &oci::Mount) -> Result { let vol_types = [ KATA_MOUNT_BIND_TYPE, KATA_DIRECT_VOLUME_TYPE, + KATA_VFIO_VOLUME_TYPE, KATA_SPDK_VOLUME_TYPE, KATA_SPOOL_VOLUME_TYPE, ]; diff --git a/src/runtime-rs/crates/resource/src/volume/direct_volumes/vfio_volume.rs b/src/runtime-rs/crates/resource/src/volume/direct_volumes/vfio_volume.rs index 132827fd3..f6dece635 100644 --- a/src/runtime-rs/crates/resource/src/volume/direct_volumes/vfio_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/direct_volumes/vfio_volume.rs @@ -4,7 +4,7 @@ // SPDX-License-Identifier: Apache-2.0 // -use anyhow::{anyhow, Context, Result}; +use anyhow::{Context, Result}; use async_trait::async_trait; use tokio::sync::RwLock; @@ -15,9 +15,9 @@ use hypervisor::{ }, get_vfio_device, VfioConfig, }; +use kata_types::mount::DirectVolumeMountInfo; use crate::volume::{ - direct_volumes::{volume_mount_info, KATA_VFIO_VOLUME_TYPE}, utils::{generate_shared_path, DEFAULT_VOLUME_FS_TYPE}, Volume, }; @@ -33,19 +33,13 @@ impl VfioVolume { pub(crate) async fn new( d: &RwLock, m: &oci::Mount, + mount_info: &DirectVolumeMountInfo, read_only: bool, sid: &str, ) -> Result { - let mnt_src: &str = &m.source; - - // deserde Information from mountinfo.json - let v = volume_mount_info(mnt_src).context("deserde information from mountinfo.json")?; - if v.volume_type != KATA_VFIO_VOLUME_TYPE { - return Err(anyhow!("volume type is invalid")); - } - // support both /dev/vfio/X and BDF or BDF - let vfio_device = get_vfio_device(v.device).context("get vfio device failed.")?; + let vfio_device = + get_vfio_device(mount_info.device.clone()).context("get vfio device failed.")?; let vfio_dev_config = &mut VfioConfig { host_path: vfio_device.clone(), dev_type: "b".to_string(), @@ -84,14 +78,14 @@ impl VfioVolume { storage.mount_point = guest_path.clone(); if m.r#type != "bind" { - storage.fs_type = v.fs_type.clone(); + storage.fs_type = mount_info.fs_type.clone(); } else { storage.fs_type = DEFAULT_VOLUME_FS_TYPE.to_string(); } let mount = oci::Mount { destination: m.destination.clone(), - r#type: v.fs_type, + r#type: mount_info.fs_type.clone(), source: guest_path, options: m.options.clone(), }; @@ -132,11 +126,3 @@ impl Volume for VfioVolume { Ok(Some(self.device_id.clone())) } } - -pub(crate) fn is_vfio_volume(m: &oci::Mount) -> bool { - if m.r#type == KATA_VFIO_VOLUME_TYPE { - return true; - } - - false -} diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs index 6c40d9348..08a8a6892 100644 --- a/src/runtime-rs/crates/resource/src/volume/mod.rs +++ b/src/runtime-rs/crates/resource/src/volume/mod.rs @@ -13,9 +13,7 @@ pub mod utils; pub mod direct_volume; use crate::volume::direct_volume::is_direct_volume; - pub mod direct_volumes; -use direct_volumes::vfio_volume::{is_vfio_volume, VfioVolume}; use std::{sync::Arc, vec::Vec}; @@ -82,7 +80,7 @@ impl VolumeResource { .with_context(|| format!("new block volume {:?}", m))?, ) } else if is_direct_volume(m)? { - // handle rawblock volume + // handle direct volumes match direct_volume::handle_direct_volume(d, m, read_only, sid) .await .context("handle direct volume")? @@ -90,12 +88,6 @@ impl VolumeResource { Some(directvol) => directvol, None => continue, } - } else if is_vfio_volume(m) { - Arc::new( - VfioVolume::new(d, m, read_only, sid) - .await - .with_context(|| format!("new vfio volume {:?}", m))?, - ) } else if let Some(options) = get_huge_page_option(m).context("failed to check huge page")? {