runtime-rs: support block rootfs

support devmapper for block rootfs

Fixes: #5375

Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Signed-off-by: alex.lyn <alex.lyn@antgroup.com>
This commit is contained in:
Zhongtao Hu
2023-05-10 17:25:27 +08:00
parent b076d46db3
commit a8bfac90b1
10 changed files with 204 additions and 38 deletions

View File

@@ -66,7 +66,7 @@ pub struct DeviceManager {
}
impl DeviceManager {
pub async fn new(hypervisor: Arc<dyn Hypervisor>) -> Result<Self> {
pub fn new(hypervisor: Arc<dyn Hypervisor>) -> Result<Self> {
let devices = HashMap::<String, ArcMutexBoxDevice>::new();
Ok(DeviceManager {
devices,
@@ -75,7 +75,7 @@ impl DeviceManager {
})
}
pub async fn new_device(&mut self, device_config: DeviceConfig) -> Result<String> {
pub async fn new_device(&mut self, device_config: &DeviceConfig) -> Result<String> {
let device_id = if let Some(dev) = self.find_device(&device_config).await {
dev
} else {
@@ -86,11 +86,10 @@ impl DeviceManager {
Ok(device_id)
}
pub async fn try_add_device(&mut self, device_id: String) -> Result<()> {
// get device
pub async fn try_add_device(&mut self, device_id: &str) -> Result<()> {
let device = self
.devices
.get(&device_id)
.get(device_id)
.context("failed to find device")?;
// attach device
let result = device.lock().await.attach(self.hypervisor.as_ref()).await;
@@ -99,12 +98,22 @@ impl DeviceManager {
if let DeviceConfig::Block(config) = device.lock().await.get_device_info().await {
self.shared_info.release_device_index(config.index);
};
self.devices.remove(&device_id);
self.devices.remove(device_id);
return Err(e);
}
Ok(())
}
pub async fn get_device_info(&self, device_id: &str) -> Result<DeviceConfig> {
if let Some(dev) = self.devices.get(device_id) {
return Ok(dev.lock().await.get_device_info().await);
}
Err(anyhow!(
"device with specified ID hasn't been created. {}",
device_id
))
}
async fn find_device(&self, device_config: &DeviceConfig) -> Option<String> {
for (device_id, dev) in &self.devices {
match dev.lock().await.get_device_info().await {

View File

@@ -6,14 +6,13 @@
use std::{fs, path::Path, process::Command};
use crate::{driver::hypervisor, DeviceConfig};
use crate::Hypervisor as hypervisor;
use crate::{device::Device, DeviceConfig};
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
use anyhow::anyhow;
use anyhow::{Context, Result};
use async_trait::async_trait;
use crate::Device;
fn override_driver(bdf: &str, driver: &str) -> Result<()> {
let driver_override = format!("/sys/bus/pci/devices/{}/driver_override", bdf);
fs::write(&driver_override, driver)
@@ -152,11 +151,11 @@ pub fn bind_device_to_host(bdf: &str, host_driver: &str, _vendor_device_id: &str
#[async_trait]
impl Device for VfioConfig {
async fn attach(&self, _h: &dyn hypervisor) -> Result<()> {
async fn attach(&mut self, _h: &dyn hypervisor) -> Result<()> {
todo!()
}
async fn detach(&self, _h: &dyn hypervisor) -> Result<u64> {
async fn detach(&mut self, _h: &dyn hypervisor) -> Result<Option<u64>> {
todo!()
}

View File

@@ -4,8 +4,8 @@
// SPDX-License-Identifier: Apache-2.0
//
use crate::Device;
use crate::{driver::hypervisor, DeviceConfig};
use crate::Hypervisor as hypervisor;
use crate::{device::Device, DeviceConfig};
use anyhow::Result;
use async_trait::async_trait;
@@ -31,11 +31,11 @@ pub struct VhostUserConfig {
#[async_trait]
impl Device for VhostUserConfig {
async fn attach(&self, _h: &dyn hypervisor) -> Result<()> {
async fn attach(&mut self, _h: &dyn hypervisor) -> Result<()> {
todo!()
}
async fn detach(&self, _h: &dyn hypervisor) -> Result<u64> {
async fn detach(&mut self, _h: &dyn hypervisor) -> Result<Option<u64>> {
todo!()
}

View File

@@ -10,12 +10,12 @@ use crate::{network::NetworkConfig, resource_persist::ResourceState};
use agent::{Agent, Storage};
use anyhow::{anyhow, Context, Ok, Result};
use async_trait::async_trait;
use hypervisor::Hypervisor;
use hypervisor::{device::device_manager::DeviceManager, Hypervisor};
use kata_types::config::TomlConfig;
use kata_types::mount::Mount;
use oci::LinuxResources;
use persist::sandbox_persist::Persist;
use tokio::runtime;
use tokio::{runtime, sync::RwLock};
use crate::{
cgroups::{CgroupArgs, CgroupsResource},
@@ -32,6 +32,7 @@ pub(crate) struct ResourceManagerInner {
toml_config: Arc<TomlConfig>,
agent: Arc<dyn Agent>,
hypervisor: Arc<dyn Hypervisor>,
device_manager: Arc<RwLock<DeviceManager>>,
network: Option<Arc<dyn Network>>,
share_fs: Option<Arc<dyn ShareFs>>,
@@ -48,11 +49,17 @@ impl ResourceManagerInner {
toml_config: Arc<TomlConfig>,
) -> Result<Self> {
let cgroups_resource = CgroupsResource::new(sid, &toml_config)?;
// create device manager
let dev_manager =
DeviceManager::new(hypervisor.clone()).context("failed to create device manager")?;
Ok(Self {
sid: sid.to_string(),
toml_config,
agent,
hypervisor,
device_manager: Arc::new(RwLock::new(dev_manager)),
network: None,
share_fs: None,
rootfs_resource: RootFsResource::new(),
@@ -212,6 +219,7 @@ impl ResourceManagerInner {
self.rootfs_resource
.handler_rootfs(
&self.share_fs,
self.device_manager.as_ref(),
self.hypervisor.as_ref(),
&self.sid,
cid,
@@ -298,7 +306,8 @@ impl Persist for ResourceManagerInner {
Ok(Self {
sid: resource_args.sid,
agent: resource_args.agent,
hypervisor: resource_args.hypervisor,
hypervisor: resource_args.hypervisor.clone(),
device_manager: Arc::new(RwLock::new(DeviceManager::new(resource_args.hypervisor)?)),
network: None,
share_fs: None,
rootfs_resource: RootFsResource::new(),

View File

@@ -0,0 +1,126 @@
// Copyright (c) 2019-2022 Alibaba Cloud
// Copyright (c) 2019-2022 Ant Group
//
// SPDX-License-Identifier: Apache-2.0
//
use super::{Rootfs, ROOTFS};
use crate::share_fs::{do_get_guest_path, do_get_host_path};
use agent::Storage;
use anyhow::{anyhow, Context, Result};
use async_trait::async_trait;
use hypervisor::{device::device_manager::DeviceManager, BlockConfig, DeviceConfig};
use kata_types::mount::Mount;
use nix::sys::stat::{self, SFlag};
use std::fs;
use tokio::sync::RwLock;
pub(crate) struct BlockRootfs {
guest_path: String,
device_id: String,
mount: oci::Mount,
storage: Option<agent::Storage>,
}
impl BlockRootfs {
pub async fn new(
d: &RwLock<DeviceManager>,
sid: &str,
cid: &str,
dev_id: u64,
rootfs: &Mount,
) -> Result<Self> {
let container_path = do_get_guest_path(ROOTFS, cid, false, false);
let host_path = do_get_host_path(ROOTFS, sid, cid, false, false);
// Create rootfs dir on host to make sure mount point in guest exists, as readonly dir is
// shared to guest via virtiofs, and guest is unable to create rootfs dir.
fs::create_dir_all(&host_path)
.map_err(|e| anyhow!("failed to create rootfs dir {}: {:?}", host_path, e))?;
let block_device_config = &mut BlockConfig {
major: stat::major(dev_id) as i64,
minor: stat::minor(dev_id) as i64,
..Default::default()
};
let device_id = d
.write()
.await
.new_device(&DeviceConfig::Block(block_device_config.clone()))
.await
.context("failed to create deviec")?;
d.write()
.await
.try_add_device(device_id.as_str())
.await
.context("failed to add deivce")?;
let mut storage = Storage {
fs_type: rootfs.fs_type.clone(),
mount_point: container_path.clone(),
options: rootfs.options.clone(),
..Default::default()
};
// get complete device information
let dev_info = d
.read()
.await
.get_device_info(device_id.as_str())
.await
.context("failed to get device info")?;
if let DeviceConfig::Block(config) = dev_info {
storage.driver = config.driver_option;
storage.source = config.virt_path;
}
Ok(Self {
guest_path: container_path.clone(),
device_id,
mount: oci::Mount {
..Default::default()
},
storage: Some(storage),
})
}
}
#[async_trait]
impl Rootfs for BlockRootfs {
async fn get_guest_rootfs_path(&self) -> Result<String> {
Ok(self.guest_path.clone())
}
async fn get_rootfs_mount(&self) -> Result<Vec<oci::Mount>> {
Ok(vec![self.mount.clone()])
}
async fn get_storage(&self) -> Option<Storage> {
self.storage.clone()
}
async fn get_device_id(&self) -> Result<Option<String>> {
Ok(Some(self.device_id.clone()))
}
async fn cleanup(&self) -> Result<()> {
Ok(())
}
}
pub(crate) fn is_block_rootfs(file: &str) -> Option<u64> {
if file.is_empty() {
return None;
}
match stat::stat(file) {
Ok(fstat) => {
if SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFBLK {
let dev_id = fstat.st_rdev;
return Some(dev_id);
}
}
Err(_) => return None,
};
None
}

View File

@@ -6,18 +6,18 @@
mod nydus_rootfs;
mod share_fs_rootfs;
use agent::Storage;
use anyhow::{anyhow, Context, Result};
use async_trait::async_trait;
use hypervisor::Hypervisor;
use kata_types::mount::Mount;
mod block_rootfs;
use hypervisor::{device::device_manager::DeviceManager, Hypervisor};
use std::{sync::Arc, vec::Vec};
use tokio::sync::RwLock;
use crate::share_fs::ShareFs;
use self::nydus_rootfs::NYDUS_ROOTFS_TYPE;
use self::{block_rootfs::is_block_rootfs, nydus_rootfs::NYDUS_ROOTFS_TYPE};
const ROOTFS: &str = "rootfs";
const HYBRID_ROOTFS_LOWER_DIR: &str = "rootfs_lower";
@@ -28,6 +28,7 @@ pub trait Rootfs: Send + Sync {
async fn get_rootfs_mount(&self) -> Result<Vec<oci::Mount>>;
async fn get_storage(&self) -> Option<Storage>;
async fn cleanup(&self) -> Result<()>;
async fn get_device_id(&self) -> Result<Option<String>>;
}
#[derive(Default)]
@@ -56,7 +57,8 @@ impl RootFsResource {
pub async fn handler_rootfs(
&self,
share_fs: &Option<Arc<dyn ShareFs>>,
hypervisor: &dyn Hypervisor,
device_manager: &RwLock<DeviceManager>,
h: &dyn Hypervisor,
sid: &str,
cid: &str,
root: &oci::Root,
@@ -67,7 +69,7 @@ impl RootFsResource {
// if rootfs_mounts is empty
mounts_vec if mounts_vec.is_empty() => {
if let Some(share_fs) = share_fs {
// share fs rootfs
// handle share fs rootfs
Ok(Arc::new(
share_fs_rootfs::ShareFsRootfs::new(
share_fs,
@@ -85,16 +87,27 @@ impl RootFsResource {
mounts_vec if is_single_layer_rootfs(mounts_vec) => {
// Safe as single_layer_rootfs must have one layer
let layer = &mounts_vec[0];
let rootfs: Arc<dyn Rootfs> = if let Some(share_fs) = share_fs {
// nydus rootfs
if layer.fs_type == NYDUS_ROOTFS_TYPE {
let mut inner = self.inner.write().await;
let rootfs = if let Some(dev_id) = is_block_rootfs(&layer.source) {
// handle block rootfs
info!(sl!(), "block device: {}", dev_id);
let block_rootfs: Arc<dyn Rootfs> = Arc::new(
block_rootfs::BlockRootfs::new(device_manager, sid, cid, dev_id, layer)
.await
.context("new block rootfs")?,
);
Ok(block_rootfs)
} else if let Some(share_fs) = share_fs {
// handle nydus rootfs
let share_rootfs: Arc<dyn Rootfs> = if layer.fs_type == NYDUS_ROOTFS_TYPE {
Arc::new(
nydus_rootfs::NydusRootfs::new(share_fs, hypervisor, sid, cid, layer)
nydus_rootfs::NydusRootfs::new(share_fs, h, sid, cid, layer)
.await
.context("new nydus rootfs")?,
)
} else {
// share fs rootfs
}
// handle sharefs rootfs
else {
Arc::new(
share_fs_rootfs::ShareFsRootfs::new(
share_fs,
@@ -105,13 +118,12 @@ impl RootFsResource {
.await
.context("new share fs rootfs")?,
)
}
};
Ok(share_rootfs)
} else {
return Err(anyhow!("unsupported rootfs {:?}", &layer));
};
let mut inner = self.inner.write().await;
inner.rootfs.push(Arc::clone(&rootfs));
Err(anyhow!("unsupported rootfs {:?}", &layer))
}?;
inner.rootfs.push(rootfs.clone());
Ok(rootfs)
}
_ => Err(anyhow!(

View File

@@ -150,6 +150,10 @@ impl Rootfs for NydusRootfs {
Some(self.rootfs.clone())
}
async fn get_device_id(&self) -> Result<Option<String>> {
Ok(None)
}
async fn cleanup(&self) -> Result<()> {
// TODO: Clean up NydusRootfs after the container is killed
warn!(sl!(), "Cleaning up NydusRootfs is still unimplemented.");

View File

@@ -4,12 +4,13 @@
// SPDX-License-Identifier: Apache-2.0
//
use std::sync::Arc;
use agent::Storage;
use anyhow::{Context, Result};
use async_trait::async_trait;
use kata_sys_util::mount::{umount_timeout, Mounter};
use kata_types::mount::Mount;
use std::sync::Arc;
use super::{Rootfs, ROOTFS};
use crate::share_fs::{ShareFs, ShareFsRootfsConfig};
@@ -74,6 +75,10 @@ impl Rootfs for ShareFsRootfs {
None
}
async fn get_device_id(&self) -> Result<Option<String>> {
Ok(None)
}
async fn cleanup(&self) -> Result<()> {
// Umount the mount point shared to guest
let share_fs_mount = self.share_fs.get_share_fs_mount();

View File

@@ -12,7 +12,9 @@ mod share_virtio_fs_standalone;
use share_virtio_fs_standalone::ShareVirtioFsStandalone;
mod utils;
use tokio::sync::Mutex;
pub use utils::{do_get_guest_path, do_get_guest_share_path, get_host_rw_shared_path};
pub use utils::{
do_get_guest_path, do_get_guest_share_path, do_get_host_path, get_host_rw_shared_path,
};
mod virtio_fs_share_mount;
use virtio_fs_share_mount::VirtiofsShareMount;
pub use virtio_fs_share_mount::EPHEMERAL_PATH;

View File

@@ -97,7 +97,7 @@ pub fn do_get_guest_share_path(target: &str, cid: &str, is_rafs: bool) -> String
do_get_guest_any_path(target, cid, false, is_rafs, true)
}
pub(crate) fn do_get_host_path(
pub fn do_get_host_path(
target: &str,
sid: &str,
cid: &str,