diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs index c304be5b0e..aaadd068a2 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs @@ -14,7 +14,7 @@ pub mod vmm_instance; use std::sync::Arc; -use anyhow::Result; +use anyhow::{Context, Result}; use async_trait::async_trait; use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; use tokio::sync::RwLock; @@ -128,8 +128,8 @@ impl Hypervisor for Dragonball { inner.get_jailer_root().await } - async fn save(&self) -> Result { - Hypervisor::save(self).await + async fn save_state(&self) -> Result { + self.save().await } } @@ -140,7 +140,7 @@ impl Persist for Dragonball { /// Save a state of the component. async fn save(&self) -> Result { let inner = self.inner.read().await; - inner.save().await + inner.save().await.context("save hypervisor state") } /// Restore a component from a specified state. async fn restore( diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index b44896236f..b9296d81ef 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -64,5 +64,5 @@ pub trait Hypervisor: Send + Sync { async fn cleanup(&self) -> Result<()>; async fn check(&self) -> Result<()>; async fn get_jailer_root(&self) -> Result; - async fn save(&self) -> Result; + async fn save_state(&self) -> Result; } diff --git a/src/runtime-rs/crates/persist/Cargo.toml b/src/runtime-rs/crates/persist/Cargo.toml index 05553f4391..e0c6e5b1ff 100644 --- a/src/runtime-rs/crates/persist/Cargo.toml +++ b/src/runtime-rs/crates/persist/Cargo.toml @@ -8,9 +8,9 @@ edition = "2018" async-trait = "0.1.48" anyhow = "^1.0" kata-sys-util = { path = "../../../libs/kata-sys-util"} +kata-types = { path = "../../../libs/kata-types" } libc = "0.2" rustc-serialize = "0.3.24" serde = { version = "1.0.138", features = ["derive"] } serde_json = "1.0.82" safe-path = { path = "../../../libs/safe-path"} - diff --git a/src/runtime-rs/crates/resource/src/cgroups/mod.rs b/src/runtime-rs/crates/resource/src/cgroups/mod.rs index e3e816b35d..beb4850088 100644 --- a/src/runtime-rs/crates/resource/src/cgroups/mod.rs +++ b/src/runtime-rs/crates/resource/src/cgroups/mod.rs @@ -24,6 +24,11 @@ use oci::LinuxResources; use persist::sandbox_persist::Persist; use tokio::sync::RwLock; +pub struct CgroupArgs { + pub sid: String, + pub config: TomlConfig, +} + pub struct CgroupConfig { pub path: String, pub overhead_path: String, @@ -228,7 +233,7 @@ impl CgroupsResource { #[async_trait] impl Persist for CgroupsResource { type State = CgroupState; - type ConstructorArgs = (); + type ConstructorArgs = CgroupArgs; /// Save a state of the component. async fn save(&self) -> Result { Ok(CgroupState { @@ -239,15 +244,11 @@ impl Persist for CgroupsResource { } /// Restore a component from a specified state. async fn restore( - _resource_args: Self::ConstructorArgs, + cgroup_args: Self::ConstructorArgs, cgroup_state: Self::State, ) -> Result { let hier = cgroups_rs::hierarchies::auto(); - let config = CgroupConfig { - path: "".to_string(), - overhead_path: "".to_string(), - sandbox_cgroup_only: true, - }; + let config = CgroupConfig::new(&cgroup_args.sid, &cgroup_args.config)?; let path = cgroup_state.path.unwrap_or_default(); let cgroup_manager = Cgroup::load(hier, path.as_str()); Ok(Self { diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index 9a54a5bc18..dac07d02bb 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -17,7 +17,7 @@ use oci::LinuxResources; use persist::sandbox_persist::Persist; use crate::{ - cgroups::CgroupsResource, + cgroups::{CgroupArgs, CgroupsResource}, manager::ManagerArgs, network::{self, Network}, rootfs::{RootFsResource, Rootfs}, @@ -228,8 +228,12 @@ impl Persist for ResourceManagerInner { resource_args: Self::ConstructorArgs, resource_state: Self::State, ) -> Result { + let args = CgroupArgs { + sid: resource_args.sid.clone(), + config: resource_args.config, + }; Ok(Self { - sid: "".to_string(), + sid: resource_args.sid, agent: resource_args.agent, hypervisor: resource_args.hypervisor, network: None, @@ -237,11 +241,11 @@ impl Persist for ResourceManagerInner { rootfs_resource: RootFsResource::new(), volume_resource: VolumeResource::new(), cgroups_resource: CgroupsResource::restore( - (), + args, resource_state.cgroup_state.unwrap_or_default(), ) .await?, - toml_config: Arc::new(resource_args.config), + toml_config: Arc::new(TomlConfig::default()), }) } } diff --git a/src/runtime-rs/crates/runtimes/Cargo.toml b/src/runtime-rs/crates/runtimes/Cargo.toml index 304a7639bc..8d7630b306 100644 --- a/src/runtime-rs/crates/runtimes/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/Cargo.toml @@ -15,7 +15,7 @@ common = { path = "./common" } kata-types = { path = "../../../libs/kata-types" } logging = { path = "../../../libs/logging"} oci = { path = "../../../libs/oci" } - +persist = { path = "../persist" } # runtime handler linux_container = { path = "./linux_container", optional = true } virt_container = { path = "./virt_container", optional = true } diff --git a/src/runtime-rs/crates/runtimes/src/lib.rs b/src/runtime-rs/crates/runtimes/src/lib.rs index 64c57feeae..0853ee9c56 100644 --- a/src/runtime-rs/crates/runtimes/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/src/lib.rs @@ -9,5 +9,5 @@ extern crate slog; logging::logger_with_subsystem!(sl, "runtimes"); -mod manager; +pub mod manager; pub use manager::RuntimeHandlerManager; diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs index 10a4a427bb..c2c6c84c65 100644 --- a/src/runtime-rs/crates/runtimes/src/manager.rs +++ b/src/runtime-rs/crates/runtimes/src/manager.rs @@ -10,13 +10,16 @@ use anyhow::{anyhow, Context, Result}; use common::{ message::Message, types::{Request, Response}, - RuntimeHandler, RuntimeInstance, + RuntimeHandler, RuntimeInstance, Sandbox, }; use kata_types::{annotations::Annotation, config::TomlConfig}; -use tokio::sync::{mpsc::Sender, RwLock}; - #[cfg(feature = "linux")] use linux_container::LinuxContainer; +use persist::sandbox_persist::Persist; +use tokio::sync::{mpsc::Sender, RwLock}; +use virt_container::sandbox::SandboxRestoreArgs; +use virt_container::sandbox::VirtSandbox; +use virt_container::sandbox_persist::{SandboxState, SandboxTYPE}; #[cfg(feature = "virt")] use virt_container::VirtContainer; #[cfg(feature = "wasm")] @@ -127,8 +130,36 @@ impl RuntimeHandlerManager { }) } - pub fn cleanup(_id: &str) -> Result<()> { - // TODO: load runtime from persist and cleanup + pub async fn cleanup(&self) -> Result<()> { + let inner = self.inner.read().await; + let sender = inner.msg_sender.clone(); + let sandbox_state = persist::from_disk::(&inner.id) + .context("failed to load the sandbox state")?; + let sandbox_args = SandboxRestoreArgs { + sid: inner.id.clone(), + toml_config: TomlConfig::default(), + sender, + }; + match sandbox_state.sandbox_type { + SandboxTYPE::VIRTCONTAINER => { + let sandbox = VirtSandbox::restore(sandbox_args, sandbox_state) + .await + .context("failed to restore the sandbox")?; + sandbox + .cleanup(&inner.id) + .await + .context("failed to cleanup the resource")?; + } + SandboxTYPE::LINUXCONTAINER => { + // TODO :support linux container (https://github.com/kata-containers/kata-containers/issues/4905) + return Ok(()); + } + SandboxTYPE::WASMCONTAINER => { + // TODO :support wasm container (https://github.com/kata-containers/kata-containers/issues/4906) + return Ok(()); + } + } + Ok(()) } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index d596d81a03..c5c0ff1c4b 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -21,16 +21,14 @@ use resource::{ network::{NetworkConfig, NetworkWithNetNsConfig}, ResourceConfig, ResourceManager, }; -use tokio::sync::{ - mpsc::{channel, Sender}, - Mutex, RwLock, -}; +use tokio::sync::{mpsc::Sender, Mutex, RwLock}; use crate::{health_check::HealthCheck, sandbox_persist::SandboxTYPE}; use persist::{self, sandbox_persist::Persist}; pub struct SandboxRestoreArgs { pub sid: String, pub toml_config: TomlConfig, + pub sender: Sender, } #[derive(Clone, Copy, PartialEq, Debug)] @@ -171,7 +169,12 @@ impl Sandbox for VirtSandbox { .context("get storages for sandbox")?, sandbox_pidns: false, sandbox_id: id.to_string(), - guest_hook_path: "".to_string(), + guest_hook_path: self + .hypervisor + .hypervisor_config() + .await + .security_info + .guest_hook_path, kernel_modules: vec![], }; @@ -252,7 +255,9 @@ impl Sandbox for VirtSandbox { } async fn cleanup(&self, _id: &str) -> Result<()> { - // TODO: cleanup + self.resource_manager.delete_cgroups().await?; + self.hypervisor.cleanup().await?; + // TODO: cleanup other snadbox resource Ok(()) } } @@ -267,7 +272,7 @@ impl Persist for VirtSandbox { let sandbox_state = crate::sandbox_persist::SandboxState { sandbox_type: SandboxTYPE::VIRTCONTAINER, resource: Some(self.resource_manager.save().await?), - hypervisor: Some(self.hypervisor.save().await?), + hypervisor: Some(self.hypervisor.save_state().await?), }; persist::to_disk(&sandbox_state, &self.sid)?; Ok(sandbox_state) @@ -306,10 +311,9 @@ impl Persist for VirtSandbox { config, }; let resource_manager = Arc::new(ResourceManager::restore(args, r).await?); - let (sender, _receiver) = channel::(1); Ok(Self { sid: sid.to_string(), - msg_sender: Arc::new(Mutex::new(sender)), + msg_sender: Arc::new(Mutex::new(sandbox_args.sender)), inner: Arc::new(RwLock::new(SandboxInner::new())), agent, hypervisor, diff --git a/src/runtime-rs/crates/service/src/manager.rs b/src/runtime-rs/crates/service/src/manager.rs index de5ff22e3f..214db1e47d 100644 --- a/src/runtime-rs/crates/service/src/manager.rs +++ b/src/runtime-rs/crates/service/src/manager.rs @@ -26,7 +26,6 @@ use tokio::{ use ttrpc::asynchronous::Server; use crate::task_service::TaskService; - /// message buffer size const MESSAGE_BUFFER_SIZE: usize = 8; use persist::KATA_PATH; @@ -151,7 +150,12 @@ impl ServiceManager { Ok(()) } - pub fn cleanup(sid: &str) -> Result<()> { + pub async fn cleanup(sid: &str) -> Result<()> { + let (sender, _receiver) = channel::(MESSAGE_BUFFER_SIZE); + let handler = RuntimeHandlerManager::new(sid, sender) + .await + .context("new runtime handler")?; + handler.cleanup().await?; let temp_dir = [KATA_PATH, sid].join("/"); if std::fs::metadata(temp_dir.as_str()).is_ok() { // try to remove dir and skip the result diff --git a/src/runtime-rs/crates/shim/src/bin/main.rs b/src/runtime-rs/crates/shim/src/bin/main.rs index 262f9e2385..aa484c694e 100644 --- a/src/runtime-rs/crates/shim/src/bin/main.rs +++ b/src/runtime-rs/crates/shim/src/bin/main.rs @@ -142,7 +142,11 @@ fn real_main() -> Result<()> { let action = parse_args(&args).context("parse args")?; match action { Action::Start(args) => ShimExecutor::new(args).start().context("shim start")?, - Action::Delete(args) => ShimExecutor::new(args).delete().context("shim delete")?, + Action::Delete(args) => { + let mut shim = ShimExecutor::new(args); + let rt = get_tokio_runtime().context("get tokio runtime")?; + rt.block_on(shim.delete())? + } Action::Run(args) => { // set mnt namespace // need setup before other async call diff --git a/src/runtime-rs/crates/shim/src/shim_delete.rs b/src/runtime-rs/crates/shim/src/shim_delete.rs index fd90775662..8429d34aeb 100644 --- a/src/runtime-rs/crates/shim/src/shim_delete.rs +++ b/src/runtime-rs/crates/shim/src/shim_delete.rs @@ -12,15 +12,15 @@ use std::{fs, path::Path}; use crate::{shim::ShimExecutor, Error}; impl ShimExecutor { - pub fn delete(&mut self) -> Result<()> { + pub async fn delete(&mut self) -> Result<()> { self.args.validate(true).context("validate")?; - let rsp = self.do_cleanup().context("do cleanup")?; + let rsp = self.do_cleanup().await.context("do cleanup")?; rsp.write_to_writer(&mut std::io::stdout()) .context(Error::FileWrite(format!("write {:?} to stdout", rsp)))?; Ok(()) } - fn do_cleanup(&self) -> Result { + async fn do_cleanup(&self) -> Result { let mut rsp = api::DeleteResponse::new(); rsp.set_exit_status(128 + libc::SIGKILL as u32); let mut exited_time = protobuf::well_known_types::Timestamp::new(); @@ -41,7 +41,9 @@ impl ShimExecutor { info!(sl!(), "remote socket path: {:?}", &file_path); fs::remove_file(file_path).ok(); } - service::ServiceManager::cleanup(&self.args.id).context("cleanup")?; + service::ServiceManager::cleanup(&self.args.id) + .await + .context("cleanup")?; Ok(rsp) } }