Merge pull request #4658 from openanolis/runtime-rs-persist

runtime-rs: persist file

src/runtime-rs/crates/hypervisor/Cargo.toml

@@ -13,7 +13,9 @@ dbs-utils = "0.1.0"
 go-flag = "0.1.0"
 libc = ">=0.2.39"
 nix = "0.24.1"
+persist = { path = "../persist" }
 seccompiler = "0.2.0"
+serde = { version = "1.0.138", features = ["derive"] }
 serde_json = ">=1.0.9"
 slog = "2.5.2"
 slog-scope = "4.4.0"

src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs

@@ -4,18 +4,21 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use std::{collections::HashSet, fs::create_dir_all, path::PathBuf};
-
+use super::vmm_instance::VmmInstance;
+use crate::{
+    device::Device, hypervisor_persist::HypervisorState, kernel_param::KernelParams, VmmState,
+    HYPERVISOR_DRAGONBALL, VM_ROOTFS_DRIVER_BLK,
+};
 use anyhow::{anyhow, Context, Result};
+use async_trait::async_trait;
 use dragonball::{
     api::v1::{BlockDeviceConfigInfo, BootSourceConfig},
     vm::VmConfigInfo,
 };
 use kata_sys_util::mount;
 use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
-
-use super::{vmm_instance::VmmInstance, RUN_PATH_PREFIX};
-use crate::{device::Device, kernel_param::KernelParams, VmmState, VM_ROOTFS_DRIVER_BLK};
+use persist::{sandbox_persist::Persist, KATA_PATH};
+use std::{collections::HashSet, fs::create_dir_all, path::PathBuf};
 
 const DRAGONBALL_KERNEL: &str = "vmlinux";
 const DRAGONBALL_ROOT_FS: &str = "rootfs";
@@ -137,7 +140,7 @@ impl DragonballInner {
             .map_err(|e| anyhow!("Failed to create dir {} err : {:?}", self.jailer_root, e))?;
 
         // create run dir
-        self.run_dir = [RUN_PATH_PREFIX, self.id.as_str()].join("/");
+        self.run_dir = [KATA_PATH, self.id.as_str()].join("/");
         create_dir_all(self.run_dir.as_str())
             .with_context(|| format!("failed to create dir {}", self.run_dir.as_str()))?;
 
@@ -307,3 +310,43 @@ impl DragonballInner {
         self.config.clone()
     }
 }
+
+#[async_trait]
+impl Persist for DragonballInner {
+    type State = HypervisorState;
+    type ConstructorArgs = ();
+    /// Save a state of the component.
+    async fn save(&self) -> Result<Self::State> {
+        Ok(HypervisorState {
+            hypervisor_type: HYPERVISOR_DRAGONBALL.to_string(),
+            id: self.id.clone(),
+            vm_path: self.vm_path.clone(),
+            jailed: self.jailed,
+            jailer_root: self.jailer_root.clone(),
+            netns: self.netns.clone(),
+            config: self.hypervisor_config(),
+            run_dir: self.run_dir.clone(),
+            cached_block_devices: self.cached_block_devices.clone(),
+            ..Default::default()
+        })
+    }
+    /// Restore a component from a specified state.
+    async fn restore(
+        _hypervisor_args: Self::ConstructorArgs,
+        hypervisor_state: Self::State,
+    ) -> Result<Self> {
+        Ok(DragonballInner {
+            id: hypervisor_state.id,
+            vm_path: hypervisor_state.vm_path,
+            jailed: hypervisor_state.jailed,
+            jailer_root: hypervisor_state.jailer_root,
+            netns: hypervisor_state.netns,
+            config: hypervisor_state.config,
+            state: VmmState::NotReady,
+            vmm_instance: VmmInstance::new(""),
+            run_dir: hypervisor_state.run_dir,
+            pending_devices: vec![],
+            cached_block_devices: hypervisor_state.cached_block_devices,
+        })
+    }
+}

src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs

@@ -13,8 +13,7 @@ use anyhow::{Context, Result};
 
 use super::inner::DragonballInner;
 use crate::{utils, VcpuThreadIds, VmmState};
-
-const KATA_PATH: &str = "/run/kata";
+use persist::KATA_PATH;
 const DEFAULT_HYBRID_VSOCK_NAME: &str = "kata.hvsock";
 
 fn get_vsock_path(root: &str) -> String {

src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs

@@ -7,14 +7,14 @@
 mod inner;
 mod inner_device;
 mod inner_hypervisor;
+use super::HypervisorState;
 use inner::DragonballInner;
+use persist::sandbox_persist::Persist;
 pub mod vmm_instance;
 
-pub const RUN_PATH_PREFIX: &str = "/run/kata";
-
 use std::sync::Arc;
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 use async_trait::async_trait;
 use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
 use tokio::sync::RwLock;
@@ -127,4 +127,29 @@ impl Hypervisor for Dragonball {
         let inner = self.inner.read().await;
         inner.get_jailer_root().await
     }
+
+    async fn save_state(&self) -> Result<HypervisorState> {
+        self.save().await
+    }
+}
+
+#[async_trait]
+impl Persist for Dragonball {
+    type State = HypervisorState;
+    type ConstructorArgs = ();
+    /// Save a state of the component.
+    async fn save(&self) -> Result<Self::State> {
+        let inner = self.inner.read().await;
+        inner.save().await.context("save hypervisor state")
+    }
+    /// Restore a component from a specified state.
+    async fn restore(
+        hypervisor_args: Self::ConstructorArgs,
+        hypervisor_state: Self::State,
+    ) -> Result<Self> {
+        let inner = DragonballInner::restore(hypervisor_args, hypervisor_state).await?;
+        Ok(Self {
+            inner: Arc::new(RwLock::new(inner)),
+        })
+    }
 }

src/runtime-rs/crates/hypervisor/src/hypervisor_persist.rs

@@ -0,0 +1,36 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use crate::HypervisorConfig;
+use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
+
+#[derive(Serialize, Deserialize, Default)]
+pub struct HypervisorState {
+    // Type of hypervisor, E.g. dragonball/qemu/firecracker/acrn.
+    pub hypervisor_type: String,
+    pub pid: Option<i32>,
+    pub uuid: String,
+    // clh sepcific: refer to 'virtcontainers/clh.go:CloudHypervisorState'
+    pub api_socket: String,
+    /// sandbox id
+    pub id: String,
+    /// vm path
+    pub vm_path: String,
+    /// jailed flag
+    pub jailed: bool,
+    /// chroot base for the jailer
+    pub jailer_root: String,
+    /// netns
+    pub netns: Option<String>,
+    /// hypervisor config
+    pub config: HypervisorConfig,
+    /// hypervisor run dir
+    pub run_dir: String,
+    /// cached block device
+    pub cached_block_devices: HashSet<String>,
+    pub virtiofs_daemon_pid: i32,
+}

src/runtime-rs/crates/hypervisor/src/lib.rs

@@ -10,22 +10,24 @@ extern crate slog;
 logging::logger_with_subsystem!(sl, "hypervisor");
 
 pub mod device;
+pub mod hypervisor_persist;
 pub use device::*;
 pub mod dragonball;
 mod kernel_param;
 pub use kernel_param::Param;
 mod utils;
-
 use std::collections::HashMap;
 
 use anyhow::Result;
 use async_trait::async_trait;
+use hypervisor_persist::HypervisorState;
 use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
 
 // Config which driver to use as vm root dev
 const VM_ROOTFS_DRIVER_BLK: &str = "virtio-blk";
 const VM_ROOTFS_DRIVER_PMEM: &str = "virtio-pmem";
 
+pub const HYPERVISOR_DRAGONBALL: &str = "dragonball";
 #[derive(PartialEq)]
 pub(crate) enum VmmState {
     NotReady,
@@ -62,4 +64,5 @@ pub trait Hypervisor: Send + Sync {
     async fn cleanup(&self) -> Result<()>;
     async fn check(&self) -> Result<()>;
     async fn get_jailer_root(&self) -> Result<String>;
+    async fn save_state(&self) -> Result<HypervisorState>;
 }

src/runtime-rs/crates/persist/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "persist"
+version = "0.1.0"
+authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
+edition = "2018"
+
+[dependencies]
+async-trait = "0.1.48"
+anyhow = "^1.0"
+kata-sys-util = { path = "../../../libs/kata-sys-util"}
+kata-types = { path = "../../../libs/kata-types" }
+libc = "0.2"
+rustc-serialize = "0.3.24"
+serde = { version = "1.0.138", features = ["derive"] }
+serde_json = "1.0.82"
+safe-path = { path = "../../../libs/safe-path"}

src/runtime-rs/crates/persist/src/lib.rs

@@ -0,0 +1,85 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+pub mod sandbox_persist;
+use anyhow::{anyhow, Context, Ok, Result};
+use serde::de;
+use std::{fs::File, io::BufReader};
+
+pub const KATA_PATH: &str = "/run/kata";
+pub const PERSIST_FILE: &str = "state.json";
+use kata_sys_util::validate::verify_id;
+use safe_path::scoped_join;
+
+pub fn to_disk<T: serde::Serialize>(value: &T, sid: &str) -> Result<()> {
+    verify_id(sid).context("failed to verify sid")?;
+    let mut path = scoped_join(KATA_PATH, sid)?;
+    if path.exists() {
+        path.push(PERSIST_FILE);
+        let f = File::create(path)
+            .context("failed to create the file")
+            .context("failed to join the path")?;
+        let j = serde_json::to_value(value).context("failed to convert to the json value")?;
+        serde_json::to_writer_pretty(f, &j)?;
+        return Ok(());
+    }
+    return Err(anyhow!("invalid sid {}", sid));
+}
+
+pub fn from_disk<T>(sid: &str) -> Result<T>
+where
+    T: de::DeserializeOwned,
+{
+    verify_id(sid).context("failed to verify sid")?;
+    let mut path = scoped_join(KATA_PATH, sid)?;
+    if path.exists() {
+        path.push(PERSIST_FILE);
+        let file = File::open(path).context("failed to open the file")?;
+        let reader = BufReader::new(file);
+        return serde_json::from_reader(reader).map_err(|e| anyhow!(e.to_string()));
+    }
+    return Err(anyhow!("invalid sid {}", sid));
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{from_disk, to_disk, KATA_PATH};
+    use serde::{Deserialize, Serialize};
+    use std::fs::DirBuilder;
+    use std::{fs, result::Result::Ok};
+    #[test]
+    fn test_to_from_disk() {
+        #[derive(Serialize, Deserialize, Debug)]
+        struct Kata {
+            name: String,
+            key: u8,
+        }
+        let data = Kata {
+            name: "kata".to_string(),
+            key: 1,
+        };
+        // invalid sid
+        assert!(to_disk(&data, "..3").is_err());
+        assert!(to_disk(&data, "../../../3").is_err());
+        assert!(to_disk(&data, "a/b/c").is_err());
+        assert!(to_disk(&data, ".#cdscd.").is_err());
+
+        let sid = "aadede";
+        let sandbox_dir = [KATA_PATH, sid].join("/");
+        if DirBuilder::new()
+            .recursive(true)
+            .create(&sandbox_dir)
+            .is_ok()
+        {
+            assert!(to_disk(&data, sid).is_ok());
+            if let Ok(result) = from_disk::<Kata>(sid) {
+                assert_eq!(result.name, data.name);
+                assert_eq!(result.key, data.key);
+            }
+            assert!(fs::remove_dir_all(&sandbox_dir).is_ok());
+        }
+    }
+}

src/runtime-rs/crates/persist/src/sandbox_persist.rs

@@ -0,0 +1,23 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::Result;
+use async_trait::async_trait;
+
+#[async_trait]
+pub trait Persist
+where
+    Self: Sized,
+{
+    /// The type of the object representing the state of the component.
+    type State;
+    /// The type of the object holding the constructor arguments.
+    type ConstructorArgs;
+    /// Save a state of the component.
+    async fn save(&self) -> Result<Self::State>;
+    /// Restore a component from a specified state.
+    async fn restore(constructor_args: Self::ConstructorArgs, state: Self::State) -> Result<Self>;
+}

src/runtime-rs/crates/resource/Cargo.toml

@@ -18,6 +18,7 @@ nix = "0.24.1"
 rand = "^0.7.2"
 rtnetlink = "0.11.0"
 scopeguard = "1.0.0"
+serde = { version = "1.0.138", features = ["derive"] }
 slog = "2.5.2"
 slog-scope = "4.4.0"
 tokio = { version = "1.8.0", features = ["process"] }
@@ -30,4 +31,5 @@ kata-sys-util = { path = "../../../libs/kata-sys-util" }
 logging = { path = "../../../libs/logging" }
 oci = { path = "../../../libs/oci" }
 actix-rt = "2.7.0"
+persist = { path = "../persist"}
 [features]

src/runtime-rs/crates/resource/src/cgroups/cgroup_persist.rs

@@ -0,0 +1,13 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Default)]
+pub struct CgroupState {
+    pub path: Option<String>,
+    pub overhead_path: Option<String>,
+    pub sandbox_cgroup_only: bool,
+}

src/runtime-rs/crates/resource/src/cgroups/mod.rs

@@ -4,6 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+pub mod cgroup_persist;
 mod utils;
 
 use std::{
@@ -13,13 +14,21 @@ use std::{
 };
 
 use anyhow::{anyhow, Context, Result};
+use async_trait::async_trait;
+use cgroup_persist::CgroupState;
 use cgroups_rs::{cgroup_builder::CgroupBuilder, Cgroup, CgroupPid, CpuResources, Resources};
 use hypervisor::Hypervisor;
 use kata_sys_util::spec::load_oci_spec;
 use kata_types::config::TomlConfig;
 use oci::LinuxResources;
+use persist::sandbox_persist::Persist;
 use tokio::sync::RwLock;
 
+pub struct CgroupArgs {
+    pub sid: String,
+    pub config: TomlConfig,
+}
+
 pub struct CgroupConfig {
     pub path: String,
     pub overhead_path: String,
@@ -48,6 +57,7 @@ pub struct CgroupsResource {
     resources: Arc<RwLock<HashMap<String, Resources>>>,
     cgroup_manager: Cgroup,
     overhead_cgroup_manager: Option<Cgroup>,
+    cgroup_config: CgroupConfig,
 }
 
 impl CgroupsResource {
@@ -91,6 +101,7 @@ impl CgroupsResource {
             cgroup_manager,
             resources: Arc::new(RwLock::new(HashMap::new())),
             overhead_cgroup_manager,
+            cgroup_config: config,
         })
     }
 
@@ -218,3 +229,33 @@ impl CgroupsResource {
         }
     }
 }
+
+#[async_trait]
+impl Persist for CgroupsResource {
+    type State = CgroupState;
+    type ConstructorArgs = CgroupArgs;
+    /// Save a state of the component.
+    async fn save(&self) -> Result<Self::State> {
+        Ok(CgroupState {
+            path: Some(self.cgroup_config.path.clone()),
+            overhead_path: Some(self.cgroup_config.overhead_path.clone()),
+            sandbox_cgroup_only: self.cgroup_config.sandbox_cgroup_only,
+        })
+    }
+    /// Restore a component from a specified state.
+    async fn restore(
+        cgroup_args: Self::ConstructorArgs,
+        cgroup_state: Self::State,
+    ) -> Result<Self> {
+        let hier = cgroups_rs::hierarchies::auto();
+        let config = CgroupConfig::new(&cgroup_args.sid, &cgroup_args.config)?;
+        let path = cgroup_state.path.unwrap_or_default();
+        let cgroup_manager = Cgroup::load(hier, path.as_str());
+        Ok(Self {
+            cgroup_manager,
+            resources: Arc::new(RwLock::new(HashMap::new())),
+            overhead_cgroup_manager: None,
+            cgroup_config: config,
+        })
+    }
+}

src/runtime-rs/crates/resource/src/lib.rs

@@ -16,6 +16,7 @@ pub mod cgroups;
 pub mod manager;
 mod manager_inner;
 pub mod network;
+pub mod resource_persist;
 use network::NetworkConfig;
 pub mod rootfs;
 pub mod share_fs;

src/runtime-rs/crates/resource/src/manager.rs

@@ -4,17 +4,25 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use std::sync::Arc;
-
+use crate::resource_persist::ResourceState;
+use crate::{manager_inner::ResourceManagerInner, rootfs::Rootfs, volume::Volume, ResourceConfig};
 use agent::{Agent, Storage};
 use anyhow::Result;
+use async_trait::async_trait;
 use hypervisor::Hypervisor;
 use kata_types::config::TomlConfig;
 use kata_types::mount::Mount;
 use oci::LinuxResources;
+use persist::sandbox_persist::Persist;
+use std::sync::Arc;
 use tokio::sync::RwLock;
 
-use crate::{manager_inner::ResourceManagerInner, rootfs::Rootfs, volume::Volume, ResourceConfig};
+pub struct ManagerArgs {
+    pub sid: String,
+    pub agent: Arc<dyn Agent>,
+    pub hypervisor: Arc<dyn Hypervisor>,
+    pub config: TomlConfig,
+}
 
 pub struct ResourceManager {
     inner: Arc<RwLock<ResourceManagerInner>>,
@@ -95,3 +103,24 @@ impl ResourceManager {
         inner.delete_cgroups().await
     }
 }
+
+#[async_trait]
+impl Persist for ResourceManager {
+    type State = ResourceState;
+    type ConstructorArgs = ManagerArgs;
+    /// Save a state of the component.
+    async fn save(&self) -> Result<Self::State> {
+        let inner = self.inner.read().await;
+        inner.save().await
+    }
+    /// Restore a component from a specified state.
+    async fn restore(
+        resource_args: Self::ConstructorArgs,
+        resource_state: Self::State,
+    ) -> Result<Self> {
+        let inner = ResourceManagerInner::restore(resource_args, resource_state).await?;
+        Ok(Self {
+            inner: Arc::new(RwLock::new(inner)),
+        })
+    }
+}

src/runtime-rs/crates/resource/src/manager_inner.rs

@@ -6,15 +6,19 @@
 
 use std::sync::Arc;
 
+use crate::resource_persist::ResourceState;
 use agent::{Agent, Storage};
 use anyhow::{Context, Result};
+use async_trait::async_trait;
 use hypervisor::Hypervisor;
 use kata_types::config::TomlConfig;
 use kata_types::mount::Mount;
 use oci::LinuxResources;
+use persist::sandbox_persist::Persist;
 
 use crate::{
-    cgroups::CgroupsResource,
+    cgroups::{CgroupArgs, CgroupsResource},
+    manager::ManagerArgs,
     network::{self, Network},
     rootfs::{RootFsResource, Rootfs},
     share_fs::{self, ShareFs},
@@ -198,3 +202,48 @@ impl ResourceManagerInner {
         self.volume_resource.dump().await;
     }
 }
+
+#[async_trait]
+impl Persist for ResourceManagerInner {
+    type State = ResourceState;
+    type ConstructorArgs = ManagerArgs;
+    /// Save a state of the component.
+    async fn save(&self) -> Result<Self::State> {
+        let mut endpoint_state = vec![];
+        if let Some(network) = &self.network {
+            if let Some(ens) = network.save().await {
+                endpoint_state = ens;
+            }
+        }
+        let cgroup_state = self.cgroups_resource.save().await?;
+        Ok(ResourceState {
+            endpoint: endpoint_state,
+            cgroup_state: Some(cgroup_state),
+        })
+    }
+    /// Restore a component from a specified state.
+    async fn restore(
+        resource_args: Self::ConstructorArgs,
+        resource_state: Self::State,
+    ) -> Result<Self> {
+        let args = CgroupArgs {
+            sid: resource_args.sid.clone(),
+            config: resource_args.config,
+        };
+        Ok(Self {
+            sid: resource_args.sid,
+            agent: resource_args.agent,
+            hypervisor: resource_args.hypervisor,
+            network: None,
+            share_fs: None,
+            rootfs_resource: RootFsResource::new(),
+            volume_resource: VolumeResource::new(),
+            cgroups_resource: CgroupsResource::restore(
+                args,
+                resource_state.cgroup_state.unwrap_or_default(),
+            )
+            .await?,
+            toml_config: Arc::new(TomlConfig::default()),
+        })
+    }
+}

src/runtime-rs/crates/resource/src/network/endpoint/endpoint_persist.rs

@@ -0,0 +1,50 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Clone, Default)]
+pub struct PhysicalEndpointState {
+    pub bdf: String,
+    pub driver: String,
+    pub vendor_id: String,
+    pub device_id: String,
+    pub hard_addr: String,
+}
+
+#[derive(Serialize, Deserialize, Clone, Default)]
+pub struct MacvlanEndpointState {
+    pub if_name: String,
+    pub network_qos: bool,
+}
+
+#[derive(Serialize, Deserialize, Clone, Default)]
+pub struct VlanEndpointState {
+    pub if_name: String,
+    pub network_qos: bool,
+}
+
+#[derive(Serialize, Deserialize, Clone, Default)]
+pub struct VethEndpointState {
+    pub if_name: String,
+    pub network_qos: bool,
+}
+
+#[derive(Serialize, Deserialize, Clone, Default)]
+pub struct IpVlanEndpointState {
+    pub if_name: String,
+    pub network_qos: bool,
+}
+
+#[derive(Serialize, Deserialize, Clone, Default)]
+pub struct EndpointState {
+    pub physical_endpoint: Option<PhysicalEndpointState>,
+    pub veth_endpoint: Option<VethEndpointState>,
+    pub ipvlan_endpoint: Option<IpVlanEndpointState>,
+    pub macvlan_endpoint: Option<MacvlanEndpointState>,
+    pub vlan_endpoint: Option<VlanEndpointState>,
+    // TODO : other endpoint
+}

src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs

@@ -6,6 +6,7 @@
 
 use std::io::{self, Error};
 
+use super::endpoint_persist::{EndpointState, IpVlanEndpointState};
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 
@@ -87,4 +88,14 @@ impl Endpoint for IPVlanEndpoint {
 
         Ok(())
     }
+
+    async fn save(&self) -> Option<EndpointState> {
+        Some(EndpointState {
+            ipvlan_endpoint: Some(IpVlanEndpointState {
+                if_name: self.net_pair.virt_iface.name.clone(),
+                network_qos: self.net_pair.network_qos,
+            }),
+            ..Default::default()
+        })
+    }
 }

src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs

@@ -6,13 +6,13 @@
 
 use std::io::{self, Error};
 
+use super::endpoint_persist::{EndpointState, MacvlanEndpointState};
+use super::Endpoint;
+use crate::network::{utils, NetworkPair};
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 use hypervisor::{device::NetworkConfig, Device, Hypervisor};
 
-use super::Endpoint;
-use crate::network::{utils, NetworkPair};
-
 #[derive(Debug)]
 pub struct MacVlanEndpoint {
     pub(crate) net_pair: NetworkPair,
@@ -81,4 +81,14 @@ impl Endpoint for MacVlanEndpoint {
             .context("remove device")?;
         Ok(())
     }
+
+    async fn save(&self) -> Option<EndpointState> {
+        Some(EndpointState {
+            macvlan_endpoint: Some(MacvlanEndpointState {
+                if_name: self.net_pair.virt_iface.name.clone(),
+                network_qos: self.net_pair.network_qos,
+            }),
+            ..Default::default()
+        })
+    }
 }

src/runtime-rs/crates/resource/src/network/endpoint/mod.rs

@@ -14,16 +14,20 @@ mod vlan_endpoint;
 pub use vlan_endpoint::VlanEndpoint;
 mod macvlan_endpoint;
 pub use macvlan_endpoint::MacVlanEndpoint;
+pub mod endpoint_persist;
 mod endpoints_test;
 
 use anyhow::Result;
 use async_trait::async_trait;
 use hypervisor::Hypervisor;
 
+use super::EndpointState;
+
 #[async_trait]
 pub trait Endpoint: std::fmt::Debug + Send + Sync {
     async fn name(&self) -> String;
     async fn hardware_addr(&self) -> String;
     async fn attach(&self, hypervisor: &dyn Hypervisor) -> Result<()>;
     async fn detach(&self, hypervisor: &dyn Hypervisor) -> Result<()>;
+    async fn save(&self) -> Option<EndpointState>;
 }

src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs

@@ -10,9 +10,9 @@ use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
 use hypervisor::{device, Hypervisor};
 
+use super::endpoint_persist::{EndpointState, PhysicalEndpointState};
 use super::Endpoint;
 use crate::network::utils::{self, link};
-
 pub const SYS_PCI_DEVICES_PATH: &str = "/sys/bus/pci/devices";
 
 #[derive(Debug)]
@@ -141,4 +141,17 @@ impl Endpoint for PhysicalEndpoint {
         })?;
         Ok(())
     }
+
+    async fn save(&self) -> Option<EndpointState> {
+        Some(EndpointState {
+            physical_endpoint: Some(PhysicalEndpointState {
+                bdf: self.bdf.clone(),
+                driver: self.driver.clone(),
+                vendor_id: self.vendor_device_id.vendor_id.clone(),
+                device_id: self.vendor_device_id.device_id.clone(),
+                hard_addr: self.hard_addr.clone(),
+            }),
+            ..Default::default()
+        })
+    }
 }

src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs

@@ -6,13 +6,13 @@
 
 use std::io::{self, Error};
 
+use super::endpoint_persist::{EndpointState, VethEndpointState};
+use super::Endpoint;
+use crate::network::{utils, NetworkPair};
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 use hypervisor::{device::NetworkConfig, Device, Hypervisor};
 
-use super::Endpoint;
-use crate::network::{utils, NetworkPair};
-
 #[derive(Debug)]
 pub struct VethEndpoint {
     net_pair: NetworkPair,
@@ -81,4 +81,13 @@ impl Endpoint for VethEndpoint {
             .context("remove device")?;
         Ok(())
     }
+    async fn save(&self) -> Option<EndpointState> {
+        Some(EndpointState {
+            veth_endpoint: Some(VethEndpointState {
+                if_name: self.net_pair.virt_iface.name.clone(),
+                network_qos: self.net_pair.network_qos,
+            }),
+            ..Default::default()
+        })
+    }
 }

src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs

@@ -9,11 +9,11 @@ use std::io::{self, Error};
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 
+use super::endpoint_persist::{EndpointState, VlanEndpointState};
 use super::Endpoint;
 use crate::network::network_model::TC_FILTER_NET_MODEL_STR;
 use crate::network::{utils, NetworkPair};
 use hypervisor::{device::NetworkConfig, Device, Hypervisor};
-
 #[derive(Debug)]
 pub struct VlanEndpoint {
     pub(crate) net_pair: NetworkPair,
@@ -85,4 +85,14 @@ impl Endpoint for VlanEndpoint {
 
         Ok(())
     }
+
+    async fn save(&self) -> Option<EndpointState> {
+        Some(EndpointState {
+            vlan_endpoint: Some(VlanEndpointState {
+                if_name: self.net_pair.virt_iface.name.clone(),
+                network_qos: self.net_pair.network_qos,
+            }),
+            ..Default::default()
+        })
+    }
 }

src/runtime-rs/crates/resource/src/network/mod.rs

@@ -17,6 +17,7 @@ use network_with_netns::NetworkWithNetns;
 mod network_pair;
 use network_pair::NetworkPair;
 mod utils;
+pub use endpoint::endpoint_persist::EndpointState;
 
 use std::sync::Arc;
 
@@ -35,6 +36,7 @@ pub trait Network: Send + Sync {
     async fn interfaces(&self) -> Result<Vec<agent::Interface>>;
     async fn routes(&self) -> Result<Vec<agent::Route>>;
     async fn neighs(&self) -> Result<Vec<agent::ARPNeighbor>>;
+    async fn save(&self) -> Option<Vec<EndpointState>>;
 }
 
 pub async fn new(config: &NetworkConfig) -> Result<Arc<dyn Network>> {

src/runtime-rs/crates/resource/src/network/network_with_netns.rs

@@ -9,6 +9,7 @@ use std::sync::{
     Arc,
 };
 
+use super::endpoint::endpoint_persist::EndpointState;
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
 use futures::stream::TryStreamExt;
@@ -108,6 +109,17 @@ impl Network for NetworkWithNetns {
         }
         Ok(neighs)
     }
+
+    async fn save(&self) -> Option<Vec<EndpointState>> {
+        let inner = self.inner.read().await;
+        let mut endpoint = vec![];
+        for e in &inner.entity_list {
+            if let Some(state) = e.endpoint.save().await {
+                endpoint.push(state);
+            }
+        }
+        Some(endpoint)
+    }
 }
 
 async fn get_entity_from_netns(config: &NetworkWithNetNsConfig) -> Result<Vec<NetworkEntity>> {

src/runtime-rs/crates/resource/src/resource_persist.rs

@@ -0,0 +1,15 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use crate::network::EndpointState;
+use serde::{Deserialize, Serialize};
+
+use crate::cgroups::cgroup_persist::CgroupState;
+#[derive(Serialize, Deserialize, Default)]
+pub struct ResourceState {
+    pub endpoint: Vec<EndpointState>,
+    pub cgroup_state: Option<CgroupState>,
+}

src/runtime-rs/crates/runtimes/Cargo.toml

@@ -15,7 +15,7 @@ common = { path = "./common" }
 kata-types = { path = "../../../libs/kata-types" }
 logging = { path = "../../../libs/logging"}
 oci = { path = "../../../libs/oci" }
-
+persist = { path = "../persist" }
 # runtime handler
 linux_container = { path = "./linux_container", optional = true }
 virt_container = { path = "./virt_container", optional = true }

src/runtime-rs/crates/runtimes/common/Cargo.toml

@@ -20,7 +20,7 @@ strum = { version = "0.24.0", features = ["derive"] }
 thiserror = "^1.0"
 tokio = { version = "1.8.0", features = ["rt-multi-thread", "process", "fs"] }
 ttrpc = { version = "0.6.1" }
-
+persist = {path = "../../persist"}
 agent = { path = "../../agent" }
 kata-sys-util = { path = "../../../../libs/kata-sys-util" }
 kata-types = { path = "../../../../libs/kata-types" }

src/runtime-rs/crates/runtimes/src/lib.rs

@@ -9,5 +9,5 @@ extern crate slog;
 
 logging::logger_with_subsystem!(sl, "runtimes");
 
-mod manager;
+pub mod manager;
 pub use manager::RuntimeHandlerManager;

src/runtime-rs/crates/runtimes/src/manager.rs

@@ -10,13 +10,16 @@ use anyhow::{anyhow, Context, Result};
 use common::{
     message::Message,
     types::{Request, Response},
-    RuntimeHandler, RuntimeInstance,
+    RuntimeHandler, RuntimeInstance, Sandbox,
 };
 use kata_types::{annotations::Annotation, config::TomlConfig};
-use tokio::sync::{mpsc::Sender, RwLock};
-
 #[cfg(feature = "linux")]
 use linux_container::LinuxContainer;
+use persist::sandbox_persist::Persist;
+use tokio::sync::{mpsc::Sender, RwLock};
+use virt_container::sandbox::SandboxRestoreArgs;
+use virt_container::sandbox::VirtSandbox;
+use virt_container::sandbox_persist::{SandboxState, SandboxTYPE};
 #[cfg(feature = "virt")]
 use virt_container::VirtContainer;
 #[cfg(feature = "wasm")]
@@ -127,8 +130,36 @@ impl RuntimeHandlerManager {
         })
     }
 
-    pub fn cleanup(_id: &str) -> Result<()> {
-        // TODO: load runtime from persist and cleanup
+    pub async fn cleanup(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        let sender = inner.msg_sender.clone();
+        let sandbox_state = persist::from_disk::<SandboxState>(&inner.id)
+            .context("failed to load the sandbox state")?;
+        let sandbox_args = SandboxRestoreArgs {
+            sid: inner.id.clone(),
+            toml_config: TomlConfig::default(),
+            sender,
+        };
+        match sandbox_state.sandbox_type {
+            SandboxTYPE::VIRTCONTAINER => {
+                let sandbox = VirtSandbox::restore(sandbox_args, sandbox_state)
+                    .await
+                    .context("failed to retore the sandbox")?;
+                sandbox
+                    .cleanup(&inner.id)
+                    .await
+                    .context("failed to cleanup the resource")?;
+            }
+            SandboxTYPE::LINUXCONTAINER => {
+                // TODO :support linux container
+                return Ok(());
+            }
+            SandboxTYPE::WASMCONTAINER => {
+                // TODO :support wasm container
+                return Ok(());
+            }
+        }
+
         Ok(())
     }
 

src/runtime-rs/crates/runtimes/virt_container/Cargo.toml

@@ -16,12 +16,13 @@ nix = "0.16.0"
 protobuf = "2.27.0"
 serde = { version = "1.0.100", features = ["derive"] }
 serde_derive = "1.0.27"
-serde_json = "1.0.39"
+serde_json = "1.0.82"
 slog = "2.5.2"
 slog-scope = "4.4.0"
 tokio = { version = "1.8.0" }
 toml = "0.4.2"
 url = "2.1.1"
+async-std = "0.99.5"
 
 agent = { path = "../../agent" }
 common = { path = "../common" }
@@ -30,4 +31,6 @@ kata-sys-util = { path = "../../../../libs/kata-sys-util" }
 kata-types = { path = "../../../../libs/kata-types" }
 logging = { path = "../../../../libs/logging"}
 oci = { path = "../../../../libs/oci" }
+persist = { path = "../../persist"}
 resource = { path = "../../resource" }
+

src/runtime-rs/crates/runtimes/virt_container/src/lib.rs

@@ -12,6 +12,7 @@ logging::logger_with_subsystem!(sl, "virt-container");
 mod container_manager;
 pub mod health_check;
 pub mod sandbox;
+pub mod sandbox_persist;
 
 use std::sync::Arc;
 
@@ -19,13 +20,11 @@ use agent::kata::KataAgent;
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
 use common::{message::Message, RuntimeHandler, RuntimeInstance};
-use hypervisor::{dragonball::Dragonball, Hypervisor};
+use hypervisor::{dragonball::Dragonball, Hypervisor, HYPERVISOR_DRAGONBALL};
 use kata_types::config::{hypervisor::register_hypervisor_plugin, DragonballConfig, TomlConfig};
 use resource::ResourceManager;
 use tokio::sync::mpsc::Sender;
 
-const HYPERVISOR_DRAGONBALL: &str = "dragonball";
-
 unsafe impl Send for VirtContainer {}
 unsafe impl Sync for VirtContainer {}
 pub struct VirtContainer {}

src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs

@@ -6,22 +6,30 @@
 
 use std::sync::Arc;
 
-use agent::{self, Agent};
-use anyhow::{Context, Result};
+use agent::{self, kata::KataAgent, Agent};
+use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
 use common::{
     message::{Action, Message},
     Sandbox,
 };
 use containerd_shim_protos::events::task::TaskOOM;
-use hypervisor::Hypervisor;
+use hypervisor::{dragonball::Dragonball, Hypervisor, HYPERVISOR_DRAGONBALL};
+use kata_types::config::TomlConfig;
 use resource::{
+    manager::ManagerArgs,
     network::{NetworkConfig, NetworkWithNetNsConfig},
     ResourceConfig, ResourceManager,
 };
 use tokio::sync::{mpsc::Sender, Mutex, RwLock};
 
-use crate::health_check::HealthCheck;
+use crate::{health_check::HealthCheck, sandbox_persist::SandboxTYPE};
+use persist::{self, sandbox_persist::Persist};
+pub struct SandboxRestoreArgs {
+    pub sid: String,
+    pub toml_config: TomlConfig,
+    pub sender: Sender<Message>,
+}
 
 #[derive(Clone, Copy, PartialEq, Debug)]
 pub enum SandboxState {
@@ -161,7 +169,12 @@ impl Sandbox for VirtSandbox {
                 .context("get storages for sandbox")?,
             sandbox_pidns: false,
             sandbox_id: id.to_string(),
-            guest_hook_path: "".to_string(),
+            guest_hook_path: self
+                .hypervisor
+                .hypervisor_config()
+                .await
+                .security_info
+                .guest_hook_path,
             kernel_modules: vec![],
         };
 
@@ -205,6 +218,7 @@ impl Sandbox for VirtSandbox {
             }
         });
         self.monitor.start(id, self.agent.clone());
+        self.save().await.context("save state")?;
         Ok(())
     }
 
@@ -241,7 +255,69 @@ impl Sandbox for VirtSandbox {
     }
 
     async fn cleanup(&self, _id: &str) -> Result<()> {
-        // TODO: cleanup
+        self.resource_manager.delete_cgroups().await?;
+        self.hypervisor.cleanup().await?;
+        // TODO: cleanup other snadbox resource
         Ok(())
     }
 }
+
+#[async_trait]
+impl Persist for VirtSandbox {
+    type State = crate::sandbox_persist::SandboxState;
+    type ConstructorArgs = SandboxRestoreArgs;
+    /// Save a state of the component.
+    async fn save(&self) -> Result<Self::State> {
+        let sandbox_state = crate::sandbox_persist::SandboxState {
+            sandbox_type: SandboxTYPE::VIRTCONTAINER,
+            resource: Some(self.resource_manager.save().await?),
+            hypervisor: Some(self.hypervisor.save_state().await?),
+        };
+        persist::to_disk(&sandbox_state, &self.sid)?;
+        Ok(sandbox_state)
+    }
+    /// Restore a component from a specified state.
+    async fn restore(
+        sandbox_args: Self::ConstructorArgs,
+        sandbox_state: Self::State,
+    ) -> Result<Self> {
+        let config = sandbox_args.toml_config;
+        let r = sandbox_state.resource.unwrap_or_default();
+        let h = sandbox_state.hypervisor.unwrap_or_default();
+        let hypervisor = match h.hypervisor_type.as_str() {
+            // TODO support other hypervisors
+            HYPERVISOR_DRAGONBALL => Ok(Arc::new(Dragonball::restore((), h).await?)),
+            _ => Err(anyhow!("Unsupported hypervisor {}", &h.hypervisor_type)),
+        }?;
+        let agent = Arc::new(KataAgent::new(kata_types::config::Agent {
+            debug: true,
+            enable_tracing: false,
+            server_port: 1024,
+            log_port: 1025,
+            dial_timeout_ms: 10,
+            reconnect_timeout_ms: 3_000,
+            request_timeout_ms: 30_000,
+            health_check_request_timeout_ms: 90_000,
+            kernel_modules: Default::default(),
+            container_pipe_size: 0,
+            debug_console_enabled: false,
+        }));
+        let sid = sandbox_args.sid;
+        let args = ManagerArgs {
+            sid: sid.clone(),
+            agent: agent.clone(),
+            hypervisor: hypervisor.clone(),
+            config,
+        };
+        let resource_manager = Arc::new(ResourceManager::restore(args, r).await?);
+        Ok(Self {
+            sid: sid.to_string(),
+            msg_sender: Arc::new(Mutex::new(sandbox_args.sender)),
+            inner: Arc::new(RwLock::new(SandboxInner::new())),
+            agent,
+            hypervisor,
+            resource_manager,
+            monitor: Arc::new(HealthCheck::new(true, false)),
+        })
+    }
+}

src/runtime-rs/crates/runtimes/virt_container/src/sandbox_persist.rs

@@ -0,0 +1,23 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use hypervisor::hypervisor_persist::HypervisorState;
+use resource::resource_persist::ResourceState;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize)]
+pub enum SandboxTYPE {
+    VIRTCONTAINER,
+    LINUXCONTAINER,
+    WASMCONTAINER,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct SandboxState {
+    pub sandbox_type: SandboxTYPE,
+    pub resource: Option<ResourceState>,
+    pub hypervisor: Option<HypervisorState>,
+}

src/runtime-rs/crates/service/Cargo.toml

@@ -16,3 +16,4 @@ common = { path = "../runtimes/common" }
 containerd-shim-protos = { version = "0.2.0", features = ["async"]}
 logging = { path = "../../../libs/logging"}
 runtimes = { path = "../runtimes" }
+persist = { path = "../persist" }

src/runtime-rs/crates/service/src/manager.rs

@@ -26,11 +26,9 @@ use tokio::{
 use ttrpc::asynchronous::Server;
 
 use crate::task_service::TaskService;
-
 /// message buffer size
 const MESSAGE_BUFFER_SIZE: usize = 8;
-
-pub const KATA_PATH: &str = "/run/kata";
+use persist::KATA_PATH;
 
 pub struct ServiceManager {
     receiver: Option<Receiver<Message>>,
@@ -152,7 +150,12 @@ impl ServiceManager {
         Ok(())
     }
 
-    pub fn cleanup(sid: &str) -> Result<()> {
+    pub async fn cleanup(sid: &str) -> Result<()> {
+        let (sender, _receiver) = channel::<Message>(MESSAGE_BUFFER_SIZE);
+        let handler = RuntimeHandlerManager::new(sid, sender)
+            .await
+            .context("new runtime handler")?;
+        handler.cleanup().await?;
         let temp_dir = [KATA_PATH, sid].join("/");
         if std::fs::metadata(temp_dir.as_str()).is_ok() {
             // try to remove dir and skip the result

src/runtime-rs/crates/shim/src/bin/main.rs

@@ -142,7 +142,11 @@ fn real_main() -> Result<()> {
     let action = parse_args(&args).context("parse args")?;
     match action {
         Action::Start(args) => ShimExecutor::new(args).start().context("shim start")?,
-        Action::Delete(args) => ShimExecutor::new(args).delete().context("shim delete")?,
+        Action::Delete(args) => {
+            let mut shim = ShimExecutor::new(args);
+            let rt = get_tokio_runtime().context("get tokio runtime")?;
+            rt.block_on(shim.delete())?
+        }
         Action::Run(args) => {
             // set mnt namespace
             // need setup before other async call

src/runtime-rs/crates/shim/src/shim_delete.rs

@@ -12,15 +12,15 @@ use std::{fs, path::Path};
 use crate::{shim::ShimExecutor, Error};
 
 impl ShimExecutor {
-    pub fn delete(&mut self) -> Result<()> {
+    pub async fn delete(&mut self) -> Result<()> {
         self.args.validate(true).context("validate")?;
-        let rsp = self.do_cleanup().context("do cleanup")?;
+        let rsp = self.do_cleanup().await.context("do cleanup")?;
         rsp.write_to_writer(&mut std::io::stdout())
             .context(Error::FileWrite(format!("write {:?} to stdout", rsp)))?;
         Ok(())
     }
 
-    fn do_cleanup(&self) -> Result<api::DeleteResponse> {
+    async fn do_cleanup(&self) -> Result<api::DeleteResponse> {
         let mut rsp = api::DeleteResponse::new();
         rsp.set_exit_status(128 + libc::SIGKILL as u32);
         let mut exited_time = protobuf::well_known_types::Timestamp::new();
@@ -41,7 +41,9 @@ impl ShimExecutor {
             info!(sl!(), "remote socket path: {:?}", &file_path);
             fs::remove_file(file_path).ok();
         }
-        service::ServiceManager::cleanup(&self.args.id).context("cleanup")?;
+        service::ServiceManager::cleanup(&self.args.id)
+            .await
+            .context("cleanup")?;
         Ok(rsp)
     }
 }