From f9737290293d890f08dcceda0f3dbaafeb7c5b63 Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Sat, 25 Nov 2023 23:23:27 +0800 Subject: [PATCH] runtime-rs: Enhancing DirectVolMount Handling for current Infra. The current infra(K8S, CSI, CRI, Containerd) for Kata containers is unable to properly handle direct volumes, resulting in the need for workarounds like searching/comparision and then patch up volume type. In this commit, reimplement of handling method is added to support raw block volume which backends may be rawdisk or other format file. Fixes: #8300 Signed-off-by: alex.lyn --- .../resource/src/volume/direct_volume.rs | 87 +++++++++++++++++++ .../volume/direct_volumes/rawblock_volume.rs | 44 ++++------ .../crates/resource/src/volume/mod.rs | 18 ++-- 3 files changed, 113 insertions(+), 36 deletions(-) create mode 100644 src/runtime-rs/crates/resource/src/volume/direct_volume.rs diff --git a/src/runtime-rs/crates/resource/src/volume/direct_volume.rs b/src/runtime-rs/crates/resource/src/volume/direct_volume.rs new file mode 100644 index 0000000000..51f2006b5c --- /dev/null +++ b/src/runtime-rs/crates/resource/src/volume/direct_volume.rs @@ -0,0 +1,87 @@ +// Copyright (c) 2023 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::sync::Arc; + +use anyhow::{anyhow, Context, Result}; +use kata_types::mount::DirectVolumeMountInfo; +use nix::sys::{stat, stat::SFlag}; +use tokio::sync::RwLock; + +use hypervisor::device::device_manager::DeviceManager; + +use crate::volume::{ + direct_volumes::{ + get_direct_volume_path, rawblock_volume, volume_mount_info, KATA_DIRECT_VOLUME_TYPE, + }, + utils::KATA_MOUNT_BIND_TYPE, + Volume, +}; + +pub(crate) async fn handle_direct_volume( + d: &RwLock, + m: &oci::Mount, + read_only: bool, + sid: &str, +) -> Result>> { + // In the direct volume scenario, we check if the source of a mount is in the + // /run/kata-containers/shared/direct-volumes/SID path by iterating over all the mounts. + // If the source is not in the path with error kind *NotFound*, we ignore the error + // and we treat it as block volume with oci Mount.type *bind*. Just fill in the block + // volume info in the DirectVolumeMountInfo + let mount_info: DirectVolumeMountInfo = match volume_mount_info(&m.source) { + Ok(mount_info) => mount_info, + Err(e) => { + // First, We need filter the non-io::ErrorKind. + if !e.is::() { + return Err(anyhow!(format!( + "unexpected error occurs when parse mount info for {:?}, with error {:?}", + &m.source, + e.to_string() + ))); + } + + // Second, we need filter non-NotFound error. + // Safe to unwrap here, as the error is of type std::io::ErrorKind. + let error_kind = e.downcast_ref::().unwrap(); + if *error_kind != std::io::ErrorKind::NotFound { + return Err(anyhow!(format!( + "failed to parse volume mount info for {:?}, with error {:?}", + &m.source, + e.to_string() + ))); + } + + // Third, if the case is *NotFound* , we just return Ok(None). + return Ok(None); + } + }; + + let direct_volume: Arc = Arc::new( + rawblock_volume::RawblockVolume::new(d, m, &mount_info, read_only, sid) + .await + .with_context(|| format!("new sid {:?} rawblock volume {:?}", &sid, m))?, + ); + + Ok(Some(direct_volume)) +} + +pub(crate) fn is_direct_volume(m: &oci::Mount) -> Result { + let mount_type = m.r#type.as_str(); + // Filter the non-bind volume and non-direct-vol volume + let vol_types = [KATA_MOUNT_BIND_TYPE, KATA_DIRECT_VOLUME_TYPE]; + if !vol_types.contains(&mount_type) { + return Ok(false); + } + + match get_direct_volume_path(&m.source) { + Ok(directvol_path) => { + let fstat = stat::stat(directvol_path.as_str()) + .context(format!("stat mount source {} failed.", directvol_path))?; + Ok(SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFDIR) + } + Err(_) => Ok(false), + } +} diff --git a/src/runtime-rs/crates/resource/src/volume/direct_volumes/rawblock_volume.rs b/src/runtime-rs/crates/resource/src/volume/direct_volumes/rawblock_volume.rs index bcd6ca3711..5ee02e4f20 100644 --- a/src/runtime-rs/crates/resource/src/volume/direct_volumes/rawblock_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/direct_volumes/rawblock_volume.rs @@ -16,12 +16,9 @@ use hypervisor::{ }, BlockConfig, }; +use kata_types::mount::DirectVolumeMountInfo; -use crate::volume::{ - direct_volumes::{get_direct_volume_path, volume_mount_info, KATA_DIRECT_VOLUME_TYPE}, - utils::handle_block_volume, - Volume, -}; +use crate::volume::{direct_volumes::KATA_DIRECT_VOLUME_TYPE, utils::handle_block_volume, Volume}; #[derive(Clone)] pub(crate) struct RawblockVolume { @@ -35,32 +32,34 @@ impl RawblockVolume { pub(crate) async fn new( d: &RwLock, m: &oci::Mount, + mount_info: &DirectVolumeMountInfo, read_only: bool, sid: &str, ) -> Result { - let mnt_src: &str = &m.source; let block_driver = get_block_driver(d).await; - // get volume mountinfo from mountinfo.json - let v = volume_mount_info(mnt_src).context("deserde information from mountinfo.json")?; + // check volume type - if v.volume_type != KATA_DIRECT_VOLUME_TYPE { - return Err(anyhow!("volume type {:?} is invalid", v.volume_type)); + if mount_info.volume_type != KATA_DIRECT_VOLUME_TYPE { + return Err(anyhow!( + "volume type {:?} is invalid", + mount_info.volume_type + )); } - let fstat = stat::stat(v.device.as_str()) - .with_context(|| format!("stat volume device file: {}", v.device.clone()))?; + let fstat = stat::stat(mount_info.device.as_str()) + .with_context(|| format!("stat volume device file: {}", mount_info.device.clone()))?; if SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFREG && SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFBLK { return Err(anyhow!( "invalid volume device {:?} for volume type {:?}", - v.device, - v.volume_type + mount_info.device, + mount_info.volume_type )); } let block_config = BlockConfig { - path_on_host: v.device, + path_on_host: mount_info.device.clone(), driver_option: block_driver, ..Default::default() }; @@ -70,7 +69,7 @@ impl RawblockVolume { .await .context("do handle device failed.")?; - let block_volume = handle_block_volume(device_info, m, read_only, sid, &v.fs_type) + let block_volume = handle_block_volume(device_info, m, read_only, sid, &mount_info.fs_type) .await .context("do handle block volume failed")?; @@ -110,16 +109,3 @@ impl Volume for RawblockVolume { Ok(Some(self.device_id.clone())) } } - -pub(crate) fn is_rawblock_volume(m: &oci::Mount) -> Result { - // KATA_MOUNT_BIND_TYPE = "directvol" - if m.r#type.as_str() != KATA_DIRECT_VOLUME_TYPE { - return Ok(false); - } - - let source = get_direct_volume_path(&m.source).context("get direct volume path failed")?; - let fstat = - stat::stat(source.as_str()).context(format!("stat mount source {} failed.", source))?; - - Ok(SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFDIR) -} diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs index c6ecc57caf..f4857fa72d 100644 --- a/src/runtime-rs/crates/resource/src/volume/mod.rs +++ b/src/runtime-rs/crates/resource/src/volume/mod.rs @@ -11,9 +11,11 @@ mod share_fs_volume; mod shm_volume; pub mod utils; +pub mod direct_volume; +use crate::volume::direct_volume::is_direct_volume; + pub mod direct_volumes; use direct_volumes::{ - rawblock_volume::{is_rawblock_volume, RawblockVolume}, spdk_volume::{is_spdk_volume, SPDKVolume}, vfio_volume::{is_vfio_volume, VfioVolume}, }; @@ -82,13 +84,15 @@ impl VolumeResource { .await .with_context(|| format!("new block volume {:?}", m))?, ) - } else if is_rawblock_volume(m)? { + } else if is_direct_volume(m)? { // handle rawblock volume - Arc::new( - RawblockVolume::new(d, m, read_only, sid) - .await - .with_context(|| format!("new rawblock volume {:?}", m))?, - ) + match direct_volume::handle_direct_volume(d, m, read_only, sid) + .await + .context("handle direct volume")? + { + Some(directvol) => directvol, + None => continue, + } } else if is_vfio_volume(m) { Arc::new( VfioVolume::new(d, m, read_only, sid)