From 977f281c5c083e6aa5f342b78c35c2783fbf5071 Mon Sep 17 00:00:00 2001 From: Yushuo Date: Mon, 5 Dec 2022 16:49:52 +0800 Subject: [PATCH] runtime-rs: add CreateContainer hook support CreateContainer hook is one kind of OCI hook. In kata, it will be executed after VM is started, before container is created, and after CreateRuntime is executed. The hook path of CreateContainer hook is in host runtime namespace, but it will be executed in host vmm namespace. Fixes: #5787 Signed-off-by: Yushuo --- src/agent/rustjail/src/lib.rs | 3 +++ src/libs/oci/src/lib.rs | 2 ++ src/libs/protocols/protos/oci.proto | 3 +++ src/libs/protocols/src/trans.rs | 6 +++++ .../crates/resource/src/network/mod.rs | 3 ++- .../resource/src/network/utils/netns.rs | 4 +-- .../src/container_manager/manager.rs | 27 ++++++++++++++++++- 7 files changed, 44 insertions(+), 4 deletions(-) diff --git a/src/agent/rustjail/src/lib.rs b/src/agent/rustjail/src/lib.rs index 9ca124b1f..c791e001c 100644 --- a/src/agent/rustjail/src/lib.rs +++ b/src/agent/rustjail/src/lib.rs @@ -156,6 +156,8 @@ fn hooks_grpc_to_oci(h: &grpc::Hooks) -> oci::Hooks { let create_runtime = hook_grpc_to_oci(h.CreateRuntime.as_ref()); + let create_container = hook_grpc_to_oci(h.CreateContainer.as_ref()); + let poststart = hook_grpc_to_oci(h.Poststart.as_ref()); let poststop = hook_grpc_to_oci(h.Poststop.as_ref()); @@ -163,6 +165,7 @@ fn hooks_grpc_to_oci(h: &grpc::Hooks) -> oci::Hooks { oci::Hooks { prestart, create_runtime, + create_container, poststart, poststop, } diff --git a/src/libs/oci/src/lib.rs b/src/libs/oci/src/lib.rs index 7a31662a1..6b0e905d6 100644 --- a/src/libs/oci/src/lib.rs +++ b/src/libs/oci/src/lib.rs @@ -195,6 +195,8 @@ pub struct Hooks { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub create_runtime: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub create_container: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub poststart: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub poststop: Vec, diff --git a/src/libs/protocols/protos/oci.proto b/src/libs/protocols/protos/oci.proto index 43dea0657..e0048beba 100644 --- a/src/libs/protocols/protos/oci.proto +++ b/src/libs/protocols/protos/oci.proto @@ -169,6 +169,9 @@ message Hooks { // Createruntime is a list of hooks to be run during the creation of runtime(sandbox). repeated Hook CreateRuntime = 4 [(gogoproto.nullable) = false]; + + // CreateContainer is a list of hooks to be run after VM is started, and before container is created. + repeated Hook CreateContainer = 5 [(gogoproto.nullable) = false]; } message Hook { diff --git a/src/libs/protocols/src/trans.rs b/src/libs/protocols/src/trans.rs index 3c1aa6f66..79f68ed05 100644 --- a/src/libs/protocols/src/trans.rs +++ b/src/libs/protocols/src/trans.rs @@ -295,6 +295,7 @@ impl From for crate::oci::Hooks { crate::oci::Hooks { Prestart: from_vec(from.prestart), CreateRuntime: from_vec(from.create_runtime), + CreateContainer: from_vec(from.create_container), Poststart: from_vec(from.poststart), Poststop: from_vec(from.poststop), unknown_fields: Default::default(), @@ -979,6 +980,10 @@ impl From for oci::Hooks { for hook in from.take_CreateRuntime().to_vec() { create_runtime.push(hook.into()) } + let mut create_container = Vec::new(); + for hook in from.take_CreateContainer().to_vec() { + create_container.push(hook.into()) + } let mut poststart = Vec::new(); for hook in from.take_Poststart().to_vec() { poststart.push(hook.into()); @@ -990,6 +995,7 @@ impl From for oci::Hooks { oci::Hooks { prestart, create_runtime, + create_container, poststart, poststop, } diff --git a/src/runtime-rs/crates/resource/src/network/mod.rs b/src/runtime-rs/crates/resource/src/network/mod.rs index e572ecc74..a85c2213d 100644 --- a/src/runtime-rs/crates/resource/src/network/mod.rs +++ b/src/runtime-rs/crates/resource/src/network/mod.rs @@ -5,6 +5,7 @@ // mod endpoint; +pub use endpoint::endpoint_persist::EndpointState; pub use endpoint::Endpoint; mod network_entity; mod network_info; @@ -17,7 +18,7 @@ use network_with_netns::NetworkWithNetns; mod network_pair; use network_pair::NetworkPair; mod utils; -pub use endpoint::endpoint_persist::EndpointState; +pub use utils::netns::NetnsGuard; use std::sync::Arc; diff --git a/src/runtime-rs/crates/resource/src/network/utils/netns.rs b/src/runtime-rs/crates/resource/src/network/utils/netns.rs index 07584c641..bb0343dff 100644 --- a/src/runtime-rs/crates/resource/src/network/utils/netns.rs +++ b/src/runtime-rs/crates/resource/src/network/utils/netns.rs @@ -10,12 +10,12 @@ use anyhow::{Context, Result}; use nix::sched::{setns, CloneFlags}; use nix::unistd::{getpid, gettid}; -pub(crate) struct NetnsGuard { +pub struct NetnsGuard { old_netns: Option, } impl NetnsGuard { - pub(crate) fn new(new_netns_path: &str) -> Result { + pub fn new(new_netns_path: &str) -> Result { let old_netns = if !new_netns_path.is_empty() { let current_netns_path = format!("/proc/{}/task/{}/ns/{}", getpid(), gettid(), "net"); let old_netns = File::open(¤t_netns_path) diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs index 326614e9b..53bd36894 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs @@ -20,6 +20,7 @@ use common::{ }; use hypervisor::Hypervisor; use oci::Process as OCIProcess; +use resource::network::NetnsGuard; use resource::ResourceManager; use tokio::sync::RwLock; @@ -60,13 +61,37 @@ impl ContainerManager for VirtContainerManager { async fn create_container(&self, config: ContainerConfig, spec: oci::Spec) -> Result { let container = Container::new( self.pid, - config, + config.clone(), spec.clone(), self.agent.clone(), self.resource_manager.clone(), ) .context("new container")?; + // CreateContainer Hooks: + // * should be run in vmm namespace (hook path in runtime namespace) + // * should be run after the vm is started, before container is created, and after CreateRuntime Hooks + // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createcontainer-hooks + let vmm_master_tid = self.hypervisor.get_vmm_master_tid().await?; + let vmm_netns_path = format!("/proc/{}/task/{}/ns/{}", self.pid, vmm_master_tid, "net"); + let state = oci::State { + version: spec.version.clone(), + id: config.container_id.clone(), + status: oci::ContainerState::Creating, + pid: vmm_master_tid as i32, + bundle: config.bundle.clone(), + annotations: spec.annotations.clone(), + }; + + // new scope, CreateContainer hooks in which will execute in a new network namespace + { + let _netns_guard = NetnsGuard::new(&vmm_netns_path).context("vmm netns guard")?; + if let Some(hooks) = spec.hooks.as_ref() { + let mut create_container_hook_states = HookStates::new(); + create_container_hook_states.execute_hooks(&hooks.create_container, Some(state))?; + } + } + let mut containers = self.containers.write().await; container.create(spec).await.context("create")?; containers.insert(container.container_id.to_string(), container);