diff --git a/docs/design/hooks-handling.md b/docs/design/hooks-handling.md index 9a5edd907..535c70cc4 100644 --- a/docs/design/hooks-handling.md +++ b/docs/design/hooks-handling.md @@ -49,7 +49,7 @@ The table below summarized when and where those different hooks will be executed |---|---|---|---|---| | `Prestart(deprecated)` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created. | | `CreateRuntime` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created, after `Prestart` hooks. | -| `CreateContainer` | OCI hook | host runtime namespace | host vmm namespace | After VM is started, before container is created, after `CreateRuntime` hooks. | +| `CreateContainer` | OCI hook | host runtime namespace | host vmm namespace* | After VM is started, before container is created, after `CreateRuntime` hooks. | | `StartContainer` | OCI hook | guest container namespace | guest container namespace | After container is created, before container is started. | | `Poststart` | OCI hook | host runtime namespace | host runtime namespace | After container is started, before start operation returns. | | `Poststop` | OCI hook | host runtime namespace | host runtime namespace | After container is deleted, before delete operation returns. | @@ -59,4 +59,5 @@ The table below summarized when and where those different hooks will be executed + `Hook Path` specifies where hook's path be resolved. + `Exec Place` specifies in which namespace those hooks can be executed. + + For `CreateContainer` Hooks, OCI requires to run them inside the container namespace while the hook executable path is in the host runtime, which is a non-starter for VM-based containers. So we design to keep them running in the *host vmm namespace.* + `Exec Time` specifies at which time point those hooks can be executed. \ No newline at end of file diff --git a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs index 56dc95087..3fee8165d 100644 --- a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs @@ -9,7 +9,13 @@ use async_trait::async_trait; #[async_trait] pub trait Sandbox: Send + Sync { - async fn start(&self, netns: Option, dns: Vec) -> Result<()>; + async fn start( + &self, + netns: Option, + dns: Vec, + spec: &oci::Spec, + state: &oci::State, + ) -> Result<()>; async fn stop(&self) -> Result<()>; async fn cleanup(&self) -> Result<()>; async fn shutdown(&self) -> Result<()>; @@ -17,9 +23,6 @@ pub trait Sandbox: Send + Sync { // agent function async fn agent_sock(&self) -> Result; - // hypervisor function - async fn get_vmm_master_tid(&self) -> Result; - // utils async fn set_iptables(&self, is_ipv6: bool, data: Vec) -> Result>; async fn get_iptables(&self, is_ipv6: bool) -> Result>; diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs index ba2b0b0b0..e904718b9 100644 --- a/src/runtime-rs/crates/runtimes/src/manager.rs +++ b/src/runtime-rs/crates/runtimes/src/manager.rs @@ -18,7 +18,6 @@ use hypervisor::Param; use kata_types::{ annotations::Annotation, config::default::DEFAULT_GUEST_DNS_FILE, config::TomlConfig, }; -use kata_sys_util::hooks::HookStates; #[cfg(feature = "linux")] use linux_container::LinuxContainer; @@ -52,6 +51,8 @@ impl RuntimeHandlerManagerInner { async fn init_runtime_handler( &mut self, + spec: &oci::Spec, + state: &oci::State, netns: Option, dns: Vec, config: Arc, @@ -76,7 +77,7 @@ impl RuntimeHandlerManagerInner { // start sandbox runtime_instance .sandbox - .start(netns, dns) + .start(netns, dns, spec, state) .await .context("start sandbox")?; self.runtime_instance = Some(Arc::new(runtime_instance)); @@ -128,39 +129,10 @@ impl RuntimeHandlerManagerInner { } let config = load_config(spec, options).context("load config")?; - self.init_runtime_handler(netns, dns, Arc::new(config)) + self.init_runtime_handler(spec, state, netns, dns, Arc::new(config)) .await .context("init runtime handler")?; - let mut st = state.clone(); - if let Some(runtime_instance) = self.runtime_instance.clone() { - let vmm_master_tid = runtime_instance - .sandbox - .get_vmm_master_tid() - .await - .context("get vmm master tid")?; - st.pid = vmm_master_tid as i32; - } - - // Prestart Hooks [DEPRECATED in newest oci spec]: - // * should be run in runtime namespace - // * should be run after vm is started, but before container is created - // if Prestart Hook and CreateRuntime Hook are both supported - // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#prestart - if let Some(hooks) = spec.hooks.as_ref() { - let mut prestart_hook_states = HookStates::new(); - prestart_hook_states.execute_hooks(&hooks.prestart, Some(st.clone()))? - } - - // CreateRuntime Hooks: - // * should be run in runtime namespace - // * should be run when creating the runtime - // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createruntime-hooks - if let Some(hooks) = spec.hooks.as_ref() { - let mut create_runtime_hook_states = HookStates::new(); - create_runtime_hook_states.execute_hooks(&hooks.create_runtime, Some(st.clone()))? - } - // the sandbox creation can reach here only once and the sandbox is created // so we can safely create the shim management socket right now // the unwrap here is safe because the runtime handler is correctly created diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index 91372c881..0d6e4765e 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -17,6 +17,7 @@ use common::{ }; use containerd_shim_protos::events::task::TaskOOM; use hypervisor::{dragonball::Dragonball, Hypervisor, HYPERVISOR_DRAGONBALL}; +use kata_sys_util::hooks::HookStates; use kata_types::config::{ default::{DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT}, TomlConfig, @@ -117,11 +118,50 @@ impl VirtSandbox { Ok(resource_configs) } + + async fn execute_oci_hook_functions( + &self, + prestart_hooks: &[oci::Hook], + create_runtime_hooks: &[oci::Hook], + state: &oci::State, + ) -> Result<()> { + let mut st = state.clone(); + // for dragonball, we use vmm_master_tid + let vmm_pid = self + .hypervisor + .get_vmm_master_tid() + .await + .context("get vmm master tid")?; + st.pid = vmm_pid as i32; + + // Prestart Hooks [DEPRECATED in newest oci spec]: + // * should be run in runtime namespace + // * should be run after vm is started, but before container is created + // if Prestart Hook and CreateRuntime Hook are both supported + // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#prestart + let mut prestart_hook_states = HookStates::new(); + prestart_hook_states.execute_hooks(prestart_hooks, Some(st.clone()))?; + + // CreateRuntime Hooks: + // * should be run in runtime namespace + // * should be run when creating the runtime + // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createruntime-hooks + let mut create_runtime_hook_states = HookStates::new(); + create_runtime_hook_states.execute_hooks(create_runtime_hooks, Some(st.clone()))?; + + Ok(()) + } } #[async_trait] impl Sandbox for VirtSandbox { - async fn start(&self, netns: Option, dns: Vec) -> Result<()> { + async fn start( + &self, + netns: Option, + dns: Vec, + spec: &oci::Spec, + state: &oci::State, + ) -> Result<()> { let id = &self.sid; // if sandbox running, return @@ -149,6 +189,17 @@ impl Sandbox for VirtSandbox { self.hypervisor.start_vm(10_000).await.context("start vm")?; info!(sl!(), "start vm"); + // execute pre-start hook functions, including Prestart Hooks and CreateRuntime Hooks + let (prestart_hooks, create_runtime_hooks) = match spec.hooks.as_ref() { + Some(hooks) => (hooks.prestart.clone(), hooks.create_runtime.clone()), + None => (Vec::new(), Vec::new()), + }; + self.execute_oci_hook_functions(&prestart_hooks, &create_runtime_hooks, state) + .await?; + + // TODO: if prestart_hooks is not empty, rescan the network endpoints(rely on hotplug endpoints). + // see: https://github.com/kata-containers/kata-containers/issues/6378 + // connect agent // set agent socket let address = self @@ -278,10 +329,6 @@ impl Sandbox for VirtSandbox { self.agent.agent_sock().await } - async fn get_vmm_master_tid(&self) -> Result { - self.hypervisor.get_vmm_master_tid().await - } - async fn set_iptables(&self, is_ipv6: bool, data: Vec) -> Result> { info!(sl!(), "sb: set_iptables invoked"); let req = SetIPTablesRequest { is_ipv6, data };