runtime-rs: move pre-start hooks to sandbox_start

In some cases, network endpoints will be configured through Prestart
Hook. So network endpoints may need to be added(hotpluged) after vm
is started and also Prestart Hook is executed.

We move pre-start hook functions' execution to sandbox_start to allow
hooks running between vm_start and netns_scan easily, so that the
lifecycle API can be cleaner.

Signed-off-by: Yushuo <y-shuo@linux.alibaba.com>

docs/design/hooks-handling.md

@@ -49,7 +49,7 @@ The table below summarized when and where those different hooks will be executed
 |---|---|---|---|---|
 | `Prestart(deprecated)` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created. |
 | `CreateRuntime` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created, after `Prestart` hooks. |
-| `CreateContainer` | OCI hook | host runtime namespace | host vmm namespace | After VM is started, before container is created, after `CreateRuntime` hooks. |
+| `CreateContainer` | OCI hook | host runtime namespace | host vmm namespace* | After VM is started, before container is created, after `CreateRuntime` hooks. |
 | `StartContainer` | OCI hook | guest container namespace | guest container namespace | After container is created, before container is started. |
 | `Poststart` | OCI hook | host runtime namespace | host runtime namespace | After container is started, before start operation returns. |
 | `Poststop` | OCI hook | host runtime namespace | host runtime namespace | After container is deleted, before delete operation returns. |
@@ -59,4 +59,5 @@ The table below summarized when and where those different hooks will be executed
 
 + `Hook Path` specifies where hook's path be resolved.
 + `Exec Place` specifies in which namespace those hooks can be executed.
+  + For `CreateContainer` Hooks, OCI requires to run them inside the container namespace while the hook executable path is in the host runtime, which is a non-starter for VM-based containers. So we design to keep them running in the *host vmm namespace.* 
 + `Exec Time` specifies at which time point those hooks can be executed.

src/runtime-rs/crates/runtimes/common/src/sandbox.rs

@@ -9,7 +9,13 @@ use async_trait::async_trait;
 
 #[async_trait]
 pub trait Sandbox: Send + Sync {
-    async fn start(&self, netns: Option<String>, dns: Vec<String>) -> Result<()>;
+    async fn start(
+        &self,
+        netns: Option<String>,
+        dns: Vec<String>,
+        spec: &oci::Spec,
+        state: &oci::State,
+    ) -> Result<()>;
     async fn stop(&self) -> Result<()>;
     async fn cleanup(&self) -> Result<()>;
     async fn shutdown(&self) -> Result<()>;
@@ -17,9 +23,6 @@ pub trait Sandbox: Send + Sync {
     // agent function
     async fn agent_sock(&self) -> Result<String>;
 
-    // hypervisor function
-    async fn get_vmm_master_tid(&self) -> Result<u32>;
-
     // utils
     async fn set_iptables(&self, is_ipv6: bool, data: Vec<u8>) -> Result<Vec<u8>>;
     async fn get_iptables(&self, is_ipv6: bool) -> Result<Vec<u8>>;

src/runtime-rs/crates/runtimes/src/manager.rs

@@ -18,7 +18,6 @@ use hypervisor::Param;
 use kata_types::{
     annotations::Annotation, config::default::DEFAULT_GUEST_DNS_FILE, config::TomlConfig,
 };
-use kata_sys_util::hooks::HookStates;
 
 #[cfg(feature = "linux")]
 use linux_container::LinuxContainer;
@@ -52,6 +51,8 @@ impl RuntimeHandlerManagerInner {
 
     async fn init_runtime_handler(
         &mut self,
+        spec: &oci::Spec,
+        state: &oci::State,
         netns: Option<String>,
         dns: Vec<String>,
         config: Arc<TomlConfig>,
@@ -76,7 +77,7 @@ impl RuntimeHandlerManagerInner {
         // start sandbox
         runtime_instance
             .sandbox
-            .start(netns, dns)
+            .start(netns, dns, spec, state)
             .await
             .context("start sandbox")?;
         self.runtime_instance = Some(Arc::new(runtime_instance));
@@ -128,39 +129,10 @@ impl RuntimeHandlerManagerInner {
         }
 
         let config = load_config(spec, options).context("load config")?;
-        self.init_runtime_handler(netns, dns, Arc::new(config))
+        self.init_runtime_handler(spec, state, netns, dns, Arc::new(config))
             .await
             .context("init runtime handler")?;
 
-        let mut st = state.clone();
-        if let Some(runtime_instance) = self.runtime_instance.clone() {
-            let vmm_master_tid = runtime_instance
-                .sandbox
-                .get_vmm_master_tid()
-                .await
-                .context("get vmm master tid")?;
-            st.pid = vmm_master_tid as i32;
-        }
-
-        // Prestart Hooks [DEPRECATED in newest oci spec]:
-        // * should be run in runtime namespace
-        // * should be run after vm is started, but before container is created
-        //      if Prestart Hook and CreateRuntime Hook are both supported
-        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#prestart
-        if let Some(hooks) = spec.hooks.as_ref() {
-            let mut prestart_hook_states = HookStates::new();
-            prestart_hook_states.execute_hooks(&hooks.prestart, Some(st.clone()))?
-        }
-
-        // CreateRuntime Hooks:
-        // * should be run in runtime namespace
-        // * should be run when creating the runtime
-        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createruntime-hooks
-        if let Some(hooks) = spec.hooks.as_ref() {
-            let mut create_runtime_hook_states = HookStates::new();
-            create_runtime_hook_states.execute_hooks(&hooks.create_runtime, Some(st.clone()))?
-        }
-
         // the sandbox creation can reach here only once and the sandbox is created
         // so we can safely create the shim management socket right now
         // the unwrap here is safe because the runtime handler is correctly created

src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs

@@ -17,6 +17,7 @@ use common::{
 };
 use containerd_shim_protos::events::task::TaskOOM;
 use hypervisor::{dragonball::Dragonball, Hypervisor, HYPERVISOR_DRAGONBALL};
+use kata_sys_util::hooks::HookStates;
 use kata_types::config::{
     default::{DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT},
     TomlConfig,
@@ -117,11 +118,50 @@ impl VirtSandbox {
 
         Ok(resource_configs)
     }
+
+    async fn execute_oci_hook_functions(
+        &self,
+        prestart_hooks: &[oci::Hook],
+        create_runtime_hooks: &[oci::Hook],
+        state: &oci::State,
+    ) -> Result<()> {
+        let mut st = state.clone();
+        // for dragonball, we use vmm_master_tid
+        let vmm_pid = self
+            .hypervisor
+            .get_vmm_master_tid()
+            .await
+            .context("get vmm master tid")?;
+        st.pid = vmm_pid as i32;
+
+        // Prestart Hooks [DEPRECATED in newest oci spec]:
+        // * should be run in runtime namespace
+        // * should be run after vm is started, but before container is created
+        //      if Prestart Hook and CreateRuntime Hook are both supported
+        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#prestart
+        let mut prestart_hook_states = HookStates::new();
+        prestart_hook_states.execute_hooks(prestart_hooks, Some(st.clone()))?;
+
+        // CreateRuntime Hooks:
+        // * should be run in runtime namespace
+        // * should be run when creating the runtime
+        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createruntime-hooks
+        let mut create_runtime_hook_states = HookStates::new();
+        create_runtime_hook_states.execute_hooks(create_runtime_hooks, Some(st.clone()))?;
+
+        Ok(())
+    }
 }
 
 #[async_trait]
 impl Sandbox for VirtSandbox {
-    async fn start(&self, netns: Option<String>, dns: Vec<String>) -> Result<()> {
+    async fn start(
+        &self,
+        netns: Option<String>,
+        dns: Vec<String>,
+        spec: &oci::Spec,
+        state: &oci::State,
+    ) -> Result<()> {
         let id = &self.sid;
 
         // if sandbox running, return
@@ -149,6 +189,17 @@ impl Sandbox for VirtSandbox {
         self.hypervisor.start_vm(10_000).await.context("start vm")?;
         info!(sl!(), "start vm");
 
+        // execute pre-start hook functions, including Prestart Hooks and CreateRuntime Hooks
+        let (prestart_hooks, create_runtime_hooks) = match spec.hooks.as_ref() {
+            Some(hooks) => (hooks.prestart.clone(), hooks.create_runtime.clone()),
+            None => (Vec::new(), Vec::new()),
+        };
+        self.execute_oci_hook_functions(&prestart_hooks, &create_runtime_hooks, state)
+            .await?;
+
+        // TODO: if prestart_hooks is not empty, rescan the network endpoints(rely on hotplug endpoints).
+        // see: https://github.com/kata-containers/kata-containers/issues/6378
+
         // connect agent
         // set agent socket
         let address = self
@@ -278,10 +329,6 @@ impl Sandbox for VirtSandbox {
         self.agent.agent_sock().await
     }
 
-    async fn get_vmm_master_tid(&self) -> Result<u32> {
-        self.hypervisor.get_vmm_master_tid().await
-    }
-
     async fn set_iptables(&self, is_ipv6: bool, data: Vec<u8>) -> Result<Vec<u8>> {
         info!(sl!(), "sb: set_iptables invoked");
         let req = SetIPTablesRequest { is_ipv6, data };