runtime-rs: support the functionality of cleanup

Cleanup sandbox resource

Fixes: #4891
Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>

src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs

@@ -14,7 +14,7 @@ pub mod vmm_instance;
 
 use std::sync::Arc;
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 use async_trait::async_trait;
 use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
 use tokio::sync::RwLock;
@@ -128,8 +128,8 @@ impl Hypervisor for Dragonball {
         inner.get_jailer_root().await
     }
 
-    async fn save(&self) -> Result<HypervisorState> {
+    async fn save_state(&self) -> Result<HypervisorState> {
-        Hypervisor::save(self).await
+        self.save().await
     }
 }
 
@@ -140,7 +140,7 @@ impl Persist for Dragonball {
     /// Save a state of the component.
     async fn save(&self) -> Result<Self::State> {
         let inner = self.inner.read().await;
-        inner.save().await
+        inner.save().await.context("save hypervisor state")
     }
     /// Restore a component from a specified state.
     async fn restore(

src/runtime-rs/crates/hypervisor/src/lib.rs

@@ -64,5 +64,5 @@ pub trait Hypervisor: Send + Sync {
     async fn cleanup(&self) -> Result<()>;
     async fn check(&self) -> Result<()>;
     async fn get_jailer_root(&self) -> Result<String>;
-    async fn save(&self) -> Result<HypervisorState>;
+    async fn save_state(&self) -> Result<HypervisorState>;
 }

src/runtime-rs/crates/persist/Cargo.toml

@@ -8,9 +8,9 @@ edition = "2018"
 async-trait = "0.1.48"
 anyhow = "^1.0"
 kata-sys-util = { path = "../../../libs/kata-sys-util"}
+kata-types = { path = "../../../libs/kata-types" }
 libc = "0.2"
 rustc-serialize = "0.3.24"
 serde = { version = "1.0.138", features = ["derive"] }
 serde_json = "1.0.82"
 safe-path = { path = "../../../libs/safe-path"}
-

src/runtime-rs/crates/resource/src/cgroups/mod.rs

@@ -24,6 +24,11 @@ use oci::LinuxResources;
 use persist::sandbox_persist::Persist;
 use tokio::sync::RwLock;
 
+pub struct CgroupArgs {
+    pub sid: String,
+    pub config: TomlConfig,
+}
+
 pub struct CgroupConfig {
     pub path: String,
     pub overhead_path: String,
@@ -228,7 +233,7 @@ impl CgroupsResource {
 #[async_trait]
 impl Persist for CgroupsResource {
     type State = CgroupState;
-    type ConstructorArgs = ();
+    type ConstructorArgs = CgroupArgs;
     /// Save a state of the component.
     async fn save(&self) -> Result<Self::State> {
         Ok(CgroupState {
@@ -239,15 +244,11 @@ impl Persist for CgroupsResource {
     }
     /// Restore a component from a specified state.
     async fn restore(
-        _resource_args: Self::ConstructorArgs,
+        cgroup_args: Self::ConstructorArgs,
         cgroup_state: Self::State,
     ) -> Result<Self> {
         let hier = cgroups_rs::hierarchies::auto();
-        let config = CgroupConfig {
+        let config = CgroupConfig::new(&cgroup_args.sid, &cgroup_args.config)?;
-            path: "".to_string(),
-            overhead_path: "".to_string(),
-            sandbox_cgroup_only: true,
-        };
         let path = cgroup_state.path.unwrap_or_default();
         let cgroup_manager = Cgroup::load(hier, path.as_str());
         Ok(Self {

src/runtime-rs/crates/resource/src/manager_inner.rs

@@ -17,7 +17,7 @@ use oci::LinuxResources;
 use persist::sandbox_persist::Persist;
 
 use crate::{
-    cgroups::CgroupsResource,
+    cgroups::{CgroupArgs, CgroupsResource},
     manager::ManagerArgs,
     network::{self, Network},
     rootfs::{RootFsResource, Rootfs},
@@ -228,8 +228,12 @@ impl Persist for ResourceManagerInner {
         resource_args: Self::ConstructorArgs,
         resource_state: Self::State,
     ) -> Result<Self> {
+        let args = CgroupArgs {
+            sid: resource_args.sid.clone(),
+            config: resource_args.config,
+        };
         Ok(Self {
-            sid: "".to_string(),
+            sid: resource_args.sid,
             agent: resource_args.agent,
             hypervisor: resource_args.hypervisor,
             network: None,
@@ -237,11 +241,11 @@ impl Persist for ResourceManagerInner {
             rootfs_resource: RootFsResource::new(),
             volume_resource: VolumeResource::new(),
             cgroups_resource: CgroupsResource::restore(
-                (),
+                args,
                 resource_state.cgroup_state.unwrap_or_default(),
             )
             .await?,
-            toml_config: Arc::new(resource_args.config),
+            toml_config: Arc::new(TomlConfig::default()),
         })
     }
 }

src/runtime-rs/crates/runtimes/Cargo.toml

@@ -15,7 +15,7 @@ common = { path = "./common" }
 kata-types = { path = "../../../libs/kata-types" }
 logging = { path = "../../../libs/logging"}
 oci = { path = "../../../libs/oci" }
-
+persist = { path = "../persist" }
 # runtime handler
 linux_container = { path = "./linux_container", optional = true }
 virt_container = { path = "./virt_container", optional = true }

src/runtime-rs/crates/runtimes/src/lib.rs

@@ -9,5 +9,5 @@ extern crate slog;
 
 logging::logger_with_subsystem!(sl, "runtimes");
 
-mod manager;
+pub mod manager;
 pub use manager::RuntimeHandlerManager;

src/runtime-rs/crates/runtimes/src/manager.rs

@@ -10,13 +10,16 @@ use anyhow::{anyhow, Context, Result};
 use common::{
     message::Message,
     types::{Request, Response},
-    RuntimeHandler, RuntimeInstance,
+    RuntimeHandler, RuntimeInstance, Sandbox,
 };
 use kata_types::{annotations::Annotation, config::TomlConfig};
-use tokio::sync::{mpsc::Sender, RwLock};
-
 #[cfg(feature = "linux")]
 use linux_container::LinuxContainer;
+use persist::sandbox_persist::Persist;
+use tokio::sync::{mpsc::Sender, RwLock};
+use virt_container::sandbox::SandboxRestoreArgs;
+use virt_container::sandbox::VirtSandbox;
+use virt_container::sandbox_persist::{SandboxState, SandboxTYPE};
 #[cfg(feature = "virt")]
 use virt_container::VirtContainer;
 #[cfg(feature = "wasm")]
@@ -127,8 +130,36 @@ impl RuntimeHandlerManager {
         })
     }
 
-    pub fn cleanup(_id: &str) -> Result<()> {
+    pub async fn cleanup(&self) -> Result<()> {
-        // TODO: load runtime from persist and cleanup
+        let inner = self.inner.read().await;
+        let sender = inner.msg_sender.clone();
+        let sandbox_state = persist::from_disk::<SandboxState>(&inner.id)
+            .context("failed to load the sandbox state")?;
+        let sandbox_args = SandboxRestoreArgs {
+            sid: inner.id.clone(),
+            toml_config: TomlConfig::default(),
+            sender,
+        };
+        match sandbox_state.sandbox_type {
+            SandboxTYPE::VIRTCONTAINER => {
+                let sandbox = VirtSandbox::restore(sandbox_args, sandbox_state)
+                    .await
+                    .context("failed to restore the sandbox")?;
+                sandbox
+                    .cleanup(&inner.id)
+                    .await
+                    .context("failed to cleanup the resource")?;
+            }
+            SandboxTYPE::LINUXCONTAINER => {
+                // TODO :support linux container (https://github.com/kata-containers/kata-containers/issues/4905)
+                return Ok(());
+            }
+            SandboxTYPE::WASMCONTAINER => {
+                // TODO :support wasm container (https://github.com/kata-containers/kata-containers/issues/4906)
+                return Ok(());
+            }
+        }
+
         Ok(())
     }
 

src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs

@@ -21,16 +21,14 @@ use resource::{
     network::{NetworkConfig, NetworkWithNetNsConfig},
     ResourceConfig, ResourceManager,
 };
-use tokio::sync::{
+use tokio::sync::{mpsc::Sender, Mutex, RwLock};
-    mpsc::{channel, Sender},
-    Mutex, RwLock,
-};
 
 use crate::{health_check::HealthCheck, sandbox_persist::SandboxTYPE};
 use persist::{self, sandbox_persist::Persist};
 pub struct SandboxRestoreArgs {
     pub sid: String,
     pub toml_config: TomlConfig,
+    pub sender: Sender<Message>,
 }
 
 #[derive(Clone, Copy, PartialEq, Debug)]
@@ -171,7 +169,12 @@ impl Sandbox for VirtSandbox {
                 .context("get storages for sandbox")?,
             sandbox_pidns: false,
             sandbox_id: id.to_string(),
-            guest_hook_path: "".to_string(),
+            guest_hook_path: self
+                .hypervisor
+                .hypervisor_config()
+                .await
+                .security_info
+                .guest_hook_path,
             kernel_modules: vec![],
         };
 
@@ -252,7 +255,9 @@ impl Sandbox for VirtSandbox {
     }
 
     async fn cleanup(&self, _id: &str) -> Result<()> {
-        // TODO: cleanup
+        self.resource_manager.delete_cgroups().await?;
+        self.hypervisor.cleanup().await?;
+        // TODO: cleanup other snadbox resource
         Ok(())
     }
 }
@@ -267,7 +272,7 @@ impl Persist for VirtSandbox {
         let sandbox_state = crate::sandbox_persist::SandboxState {
             sandbox_type: SandboxTYPE::VIRTCONTAINER,
             resource: Some(self.resource_manager.save().await?),
-            hypervisor: Some(self.hypervisor.save().await?),
+            hypervisor: Some(self.hypervisor.save_state().await?),
         };
         persist::to_disk(&sandbox_state, &self.sid)?;
         Ok(sandbox_state)
@@ -306,10 +311,9 @@ impl Persist for VirtSandbox {
             config,
         };
         let resource_manager = Arc::new(ResourceManager::restore(args, r).await?);
-        let (sender, _receiver) = channel::<Message>(1);
         Ok(Self {
             sid: sid.to_string(),
-            msg_sender: Arc::new(Mutex::new(sender)),
+            msg_sender: Arc::new(Mutex::new(sandbox_args.sender)),
             inner: Arc::new(RwLock::new(SandboxInner::new())),
             agent,
             hypervisor,

src/runtime-rs/crates/service/src/manager.rs

@@ -26,7 +26,6 @@ use tokio::{
 use ttrpc::asynchronous::Server;
 
 use crate::task_service::TaskService;
-
 /// message buffer size
 const MESSAGE_BUFFER_SIZE: usize = 8;
 use persist::KATA_PATH;
@@ -151,7 +150,12 @@ impl ServiceManager {
         Ok(())
     }
 
-    pub fn cleanup(sid: &str) -> Result<()> {
+    pub async fn cleanup(sid: &str) -> Result<()> {
+        let (sender, _receiver) = channel::<Message>(MESSAGE_BUFFER_SIZE);
+        let handler = RuntimeHandlerManager::new(sid, sender)
+            .await
+            .context("new runtime handler")?;
+        handler.cleanup().await?;
         let temp_dir = [KATA_PATH, sid].join("/");
         if std::fs::metadata(temp_dir.as_str()).is_ok() {
             // try to remove dir and skip the result

src/runtime-rs/crates/shim/src/bin/main.rs

@@ -142,7 +142,11 @@ fn real_main() -> Result<()> {
     let action = parse_args(&args).context("parse args")?;
     match action {
         Action::Start(args) => ShimExecutor::new(args).start().context("shim start")?,
-        Action::Delete(args) => ShimExecutor::new(args).delete().context("shim delete")?,
+        Action::Delete(args) => {
+            let mut shim = ShimExecutor::new(args);
+            let rt = get_tokio_runtime().context("get tokio runtime")?;
+            rt.block_on(shim.delete())?
+        }
         Action::Run(args) => {
             // set mnt namespace
             // need setup before other async call

src/runtime-rs/crates/shim/src/shim_delete.rs

@@ -12,15 +12,15 @@ use std::{fs, path::Path};
 use crate::{shim::ShimExecutor, Error};
 
 impl ShimExecutor {
-    pub fn delete(&mut self) -> Result<()> {
+    pub async fn delete(&mut self) -> Result<()> {
         self.args.validate(true).context("validate")?;
-        let rsp = self.do_cleanup().context("do cleanup")?;
+        let rsp = self.do_cleanup().await.context("do cleanup")?;
         rsp.write_to_writer(&mut std::io::stdout())
             .context(Error::FileWrite(format!("write {:?} to stdout", rsp)))?;
         Ok(())
     }
 
-    fn do_cleanup(&self) -> Result<api::DeleteResponse> {
+    async fn do_cleanup(&self) -> Result<api::DeleteResponse> {
         let mut rsp = api::DeleteResponse::new();
         rsp.set_exit_status(128 + libc::SIGKILL as u32);
         let mut exited_time = protobuf::well_known_types::Timestamp::new();
@@ -41,7 +41,9 @@ impl ShimExecutor {
             info!(sl!(), "remote socket path: {:?}", &file_path);
             fs::remove_file(file_path).ok();
         }
-        service::ServiceManager::cleanup(&self.args.id).context("cleanup")?;
+        service::ServiceManager::cleanup(&self.args.id)
+            .await
+            .context("cleanup")?;
         Ok(rsp)
     }
 }