From 67972ec48a40434fd7c3883c4538cc1d328ede40 Mon Sep 17 00:00:00 2001 From: Yushuo Date: Thu, 11 May 2023 11:34:05 +0800 Subject: [PATCH] feat(runtime-rs): calculate initial size In this commit, we refactored the logic of static resource management. We defined the sandbox size calculated from PodSandbox's annotation and SingleContainer's spec as initial size, which will always be the sandbox size when booting the VM. The configuration static_sandbox_resource_mgmt controls whether we will modify the sandbox size in the following container operation. Signed-off-by: Yushuo Signed-off-by: Ji-Xinyou --- src/runtime-rs/crates/resource/Cargo.toml | 1 + .../src/cpu_mem/initial_size.rs} | 69 ++++++++++--------- .../crates/resource/src/cpu_mem/mod.rs | 1 + .../crates/resource/src/manager_inner.rs | 2 +- src/runtime-rs/crates/runtimes/src/lib.rs | 1 - src/runtime-rs/crates/runtimes/src/manager.rs | 16 ++--- 6 files changed, 46 insertions(+), 44 deletions(-) rename src/runtime-rs/crates/{runtimes/src/static_resource.rs => resource/src/cpu_mem/initial_size.rs} (82%) diff --git a/src/runtime-rs/crates/resource/Cargo.toml b/src/runtime-rs/crates/resource/Cargo.toml index 9847ce61b1..22ffda48b9 100644 --- a/src/runtime-rs/crates/resource/Cargo.toml +++ b/src/runtime-rs/crates/resource/Cargo.toml @@ -41,4 +41,5 @@ logging = { path = "../../../libs/logging" } oci = { path = "../../../libs/oci" } actix-rt = "2.7.0" persist = { path = "../persist"} + [features] diff --git a/src/runtime-rs/crates/runtimes/src/static_resource.rs b/src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs similarity index 82% rename from src/runtime-rs/crates/runtimes/src/static_resource.rs rename to src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs index 453ce85b3f..53eccc52bd 100644 --- a/src/runtime-rs/crates/runtimes/src/static_resource.rs +++ b/src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs @@ -13,17 +13,16 @@ use kata_types::{ cpu::LinuxContainerCpuResources, k8s::container_type, }; -// static resource that StaticResourceManager needs, this is the spec for the +// initial resource that InitialSizeManager needs, this is the spec for the // sandbox/container's workload #[derive(Clone, Copy, Debug)] -struct StaticResource { +struct InitialSize { vcpu: u32, mem_mb: u32, } -// generate static resource(vcpu and memory in MiB) from spec's information -// used for static resource management -impl TryFrom<&oci::Spec> for StaticResource { +// generate initial resource(vcpu and memory in MiB) from spec's information +impl TryFrom<&oci::Spec> for InitialSize { type Error = anyhow::Error; fn try_from(spec: &oci::Spec) -> Result { let mut vcpu: u32 = 0; @@ -65,31 +64,32 @@ impl TryFrom<&oci::Spec> for StaticResource { } info!( sl!(), - "static resource mgmt result: vcpu={}, mem_mb={}", vcpu, mem_mb + "(from PodSandbox's annotation / SingleContainer's spec) initial size: vcpu={}, mem_mb={}", vcpu, mem_mb ); Ok(Self { vcpu, mem_mb }) } } -// StaticResourceManager is responsible for static resource management +// InitialSizeManager is responsible for initial vcpu/mem management // -// static resource management sizing information is optionally provided, either by +// inital vcpu/mem management sizing information is optionally provided, either by // upper layer runtime (containerd / crio) or by the container spec itself (when it // is a standalone single container such as the one started with *docker run*) // // the sizing information uses three values, cpu quota, cpu period and memory limit, -// and with above values it calculates the # vcpus and memory for the workload and -// add them to default value of the config +// and with above values it calculates the # vcpus and memory for the workload +// +// if the workload # of vcpus and memory is invalid for vmms, we still use default +// value in toml_config #[derive(Clone, Copy, Debug)] -pub struct StaticResourceManager { - resource: StaticResource, +pub struct InitialSizeManager { + resource: InitialSize, } -impl StaticResourceManager { +impl InitialSizeManager { pub fn new(spec: &oci::Spec) -> Result { Ok(Self { - resource: StaticResource::try_from(spec) - .context("failed to construct static resource")?, + resource: InitialSize::try_from(spec).context("failed to construct static resource")?, }) } @@ -100,8 +100,13 @@ impl StaticResourceManager { .hypervisor .get_mut(hypervisor_name) .context("failed to get hypervisor config")?; - hv.cpu_info.default_vcpus += self.resource.vcpu as i32; - hv.memory_info.default_memory += self.resource.mem_mb; + + if self.resource.vcpu > 0 { + hv.cpu_info.default_vcpus = self.resource.vcpu as i32 + } + if self.resource.mem_mb > 0 { + hv.memory_info.default_memory = self.resource.mem_mb; + } Ok(()) } } @@ -151,7 +156,7 @@ mod tests { struct TestData<'a> { desc: &'a str, input: InputData, - result: StaticResource, + result: InitialSize, } fn get_test_data() -> Vec> { @@ -163,7 +168,7 @@ mod tests { quota: None, memory: None, }, - result: StaticResource { vcpu: 0, mem_mb: 0 }, + result: InitialSize { vcpu: 0, mem_mb: 0 }, }, TestData { desc: "normal resource limit", @@ -173,7 +178,7 @@ mod tests { quota: Some(220_000), memory: Some(1024 * 1024 * 512), }, - result: StaticResource { + result: InitialSize { vcpu: 3, mem_mb: 512, }, @@ -183,7 +188,7 @@ mod tests { } #[test] - fn test_static_resource_mgmt_sandbox() { + fn test_initial_size_sandbox() { let tests = get_test_data(); // run tests @@ -210,22 +215,22 @@ mod tests { ..Default::default() }; - let static_resource = StaticResource::try_from(&spec); + let initial_size = InitialSize::try_from(&spec); assert!( - static_resource.is_ok(), + initial_size.is_ok(), "test[{}]: {:?} should be ok", i, d.desc ); - let static_resource = static_resource.unwrap(); + let initial_size = initial_size.unwrap(); assert_eq!( - static_resource.vcpu, d.result.vcpu, + initial_size.vcpu, d.result.vcpu, "test[{}]: {:?} vcpu should be {}", i, d.desc, d.result.vcpu, ); assert_eq!( - static_resource.mem_mb, d.result.mem_mb, + initial_size.mem_mb, d.result.mem_mb, "test[{}]: {:?} memory should be {}", i, d.desc, d.result.mem_mb, ); @@ -233,7 +238,7 @@ mod tests { } #[test] - fn test_static_resource_mgmt_container() { + fn test_initial_size_container() { let tests = get_test_data(); // run tests @@ -261,22 +266,22 @@ mod tests { ..Default::default() }; - let static_resource = StaticResource::try_from(&spec); + let initial_size = InitialSize::try_from(&spec); assert!( - static_resource.is_ok(), + initial_size.is_ok(), "test[{}]: {:?} should be ok", i, d.desc ); - let static_resource = static_resource.unwrap(); + let initial_size = initial_size.unwrap(); assert_eq!( - static_resource.vcpu, d.result.vcpu, + initial_size.vcpu, d.result.vcpu, "test[{}]: {:?} vcpu should be {}", i, d.desc, d.result.vcpu, ); assert_eq!( - static_resource.mem_mb, d.result.mem_mb, + initial_size.mem_mb, d.result.mem_mb, "test[{}]: {:?} memory should be {}", i, d.desc, d.result.mem_mb, ); diff --git a/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs b/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs index 22656753d6..f2984cd1cb 100644 --- a/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs +++ b/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs @@ -5,3 +5,4 @@ // pub mod cpu; +pub mod initial_size; diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index 8c0e06d192..b60925c111 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -317,7 +317,7 @@ impl ResourceManagerInner { sb_bindmnt.cleanup_sandbox_bind_mounts() } } - + pub async fn cleanup(&self) -> Result<()> { // clean up cgroup self.cgroups_resource diff --git a/src/runtime-rs/crates/runtimes/src/lib.rs b/src/runtime-rs/crates/runtimes/src/lib.rs index dffa69697f..867c8ef9e6 100644 --- a/src/runtime-rs/crates/runtimes/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/src/lib.rs @@ -13,4 +13,3 @@ pub mod manager; pub use manager::RuntimeHandlerManager; pub use shim_interface; mod shim_mgmt; -mod static_resource; diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs index 683be36475..d05e1961f9 100644 --- a/src/runtime-rs/crates/runtimes/src/manager.rs +++ b/src/runtime-rs/crates/runtimes/src/manager.rs @@ -21,7 +21,7 @@ use kata_types::{ use linux_container::LinuxContainer; use netns_rs::NetNs; use persist::sandbox_persist::Persist; -use resource::network::generate_netns_name; +use resource::{cpu_mem::initial_size::InitialSizeManager, network::generate_netns_name}; use shim_interface::shim_mgmt::ERR_NO_SHIM_SERVER; use tokio::fs; use tokio::sync::{mpsc::Sender, RwLock}; @@ -35,7 +35,6 @@ use virt_container::{ use wasm_container::WasmContainer; use crate::shim_mgmt::server::MgmtServer; -use crate::static_resource::StaticResourceManager; struct RuntimeHandlerManagerInner { id: String, @@ -423,14 +422,11 @@ fn load_config(spec: &oci::Spec, option: &Option>) -> Result // 2. If this is not a sandbox infrastructure container, but instead a standalone single container (analogous to "docker run..."), // then the container spec itself will contain appropriate sizing information for the entire sandbox (since it is // a single container. - if toml_config.runtime.static_sandbox_resource_mgmt { - info!(sl!(), "static resource management enabled"); - let static_resource_manager = StaticResourceManager::new(spec) - .context("failed to construct static resource manager")?; - static_resource_manager - .setup_config(&mut toml_config) - .context("failed to setup static resource mgmt config")?; - } + let initial_size_manager = + InitialSizeManager::new(spec).context("failed to construct static resource manager")?; + initial_size_manager + .setup_config(&mut toml_config) + .context("failed to setup static resource mgmt config")?; info!(sl!(), "get config content {:?}", &toml_config); Ok(toml_config)