From d428a3f9b962b3bfa9cdc9f3d8f68c4571705490 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Thu, 23 Feb 2023 16:54:08 +0800 Subject: [PATCH 1/5] runtim-rs: get guest memory details get memory block size and guest mem hotplug probe Fixes:#6356 Signed-off-by: Zhongtao Hu --- src/libs/kata-types/src/capabilities.rs | 22 +++++++-- src/runtime-rs/crates/agent/src/kata/agent.rs | 3 +- src/runtime-rs/crates/agent/src/kata/trans.rs | 12 ++++- src/runtime-rs/crates/agent/src/lib.rs | 1 + .../hypervisor/src/ch/inner_hypervisor.rs | 12 +++++ .../crates/hypervisor/src/ch/mod.rs | 17 ++++++- .../crates/hypervisor/src/dragonball/inner.rs | 17 +++++++ .../crates/hypervisor/src/dragonball/mod.rs | 17 ++++++- src/runtime-rs/crates/hypervisor/src/lib.rs | 5 ++- .../crates/hypervisor/src/qemu/inner.rs | 12 +++++ .../crates/hypervisor/src/qemu/mod.rs | 17 ++++++- .../runtimes/virt_container/src/sandbox.rs | 45 ++++++++++++++++++- 12 files changed, 170 insertions(+), 10 deletions(-) diff --git a/src/libs/kata-types/src/capabilities.rs b/src/libs/kata-types/src/capabilities.rs index 7d24ee2c1d..b45bba9421 100644 --- a/src/libs/kata-types/src/capabilities.rs +++ b/src/libs/kata-types/src/capabilities.rs @@ -19,6 +19,8 @@ pub enum CapabilityBits { FsSharingSupport, /// hypervisor supports hybrid-vsock HybridVsockSupport, + /// hypervisor supports memory hotplug probe interface + GuestMemoryHotplugProbe, } /// Capabilities describe a virtcontainers hypervisor capabilities through a bit mask. @@ -47,17 +49,22 @@ impl Capabilities { self.flags = flags; } - /// is_block_device_supported tells if an hypervisor supports block devices. + /// add CapabilityBits + pub fn add(&mut self, flags: CapabilityBits) { + self.flags |= flags; + } + + /// is_block_device_supported tells if the hypervisor supports block devices. pub fn is_block_device_supported(&self) -> bool { self.flags.and(CapabilityBits::BlockDeviceSupport) != 0 } - /// is_block_device_hotplug_supported tells if an hypervisor supports block devices. + /// is_block_device_hotplug_supported tells if the hypervisor supports block devices. pub fn is_block_device_hotplug_supported(&self) -> bool { self.flags.and(CapabilityBits::BlockDeviceHotplugSupport) != 0 } - /// is_multi_queue_supported tells if an hypervisor supports device multi queue support. + /// is_multi_queue_supported tells if the hypervisor supports device multi queue support. pub fn is_multi_queue_supported(&self) -> bool { self.flags.and(CapabilityBits::MultiQueueSupport) != 0 } @@ -71,6 +78,11 @@ impl Capabilities { pub fn is_fs_sharing_supported(&self) -> bool { self.flags.and(CapabilityBits::FsSharingSupport) != 0 } + + /// is_mem_hotplug_probe_supported tells if the hypervisor supports hotplug probe interface + pub fn is_mem_hotplug_probe_supported(&self) -> bool { + self.flags.and(CapabilityBits::GuestMemoryHotplugProbe) != 0 + } } #[cfg(test)] @@ -117,5 +129,9 @@ mod tests { // test set hybrid-vsock support cap.set(CapabilityBits::HybridVsockSupport); assert!(cap.is_hybrid_vsock_supported()); + // test append capabilities + cap.add(CapabilityBits::GuestMemoryHotplugProbe); + assert!(cap.is_mem_hotplug_probe_supported()); + assert!(cap.is_fs_sharing_supported()); } } diff --git a/src/runtime-rs/crates/agent/src/kata/agent.rs b/src/runtime-rs/crates/agent/src/kata/agent.rs index ee3e97bf85..ad5668ef07 100644 --- a/src/runtime-rs/crates/agent/src/kata/agent.rs +++ b/src/runtime-rs/crates/agent/src/kata/agent.rs @@ -122,5 +122,6 @@ impl_agent!( get_volume_stats | crate::VolumeStatsRequest | crate::VolumeStatsResponse | None, resize_volume | crate::ResizeVolumeRequest | crate::Empty | None, online_cpu_mem | crate::OnlineCPUMemRequest | crate::Empty | None, - get_metrics | crate::Empty | crate::MetricsResponse | None + get_metrics | crate::Empty | crate::MetricsResponse | None, + get_guest_details | crate::GetGuestDetailsRequest | crate::GuestDetailsResponse | None ); diff --git a/src/runtime-rs/crates/agent/src/kata/trans.rs b/src/runtime-rs/crates/agent/src/kata/trans.rs index 2190806822..d7b00aba2b 100644 --- a/src/runtime-rs/crates/agent/src/kata/trans.rs +++ b/src/runtime-rs/crates/agent/src/kata/trans.rs @@ -27,7 +27,7 @@ use crate::{ UpdateInterfaceRequest, UpdateRoutesRequest, VersionCheckResponse, VolumeStatsRequest, VolumeStatsResponse, WaitProcessRequest, WriteStreamRequest, }, - OomEventResponse, WaitProcessResponse, WriteStreamResponse, + GetGuestDetailsRequest, OomEventResponse, WaitProcessResponse, WriteStreamResponse, }; fn trans_vec>(from: Vec) -> Vec { @@ -723,6 +723,16 @@ impl From for agent::SetGuestDateTimeRequest { } } +impl From for agent::GuestDetailsRequest { + fn from(from: GetGuestDetailsRequest) -> Self { + Self { + mem_block_size: from.mem_block_size, + mem_hotplug_probe: from.mem_hotplug_probe, + ..Default::default() + } + } +} + impl From for AgentDetails { fn from(src: agent::AgentDetails) -> Self { Self { diff --git a/src/runtime-rs/crates/agent/src/lib.rs b/src/runtime-rs/crates/agent/src/lib.rs index 8dd71e8d7b..7326963ec9 100644 --- a/src/runtime-rs/crates/agent/src/lib.rs +++ b/src/runtime-rs/crates/agent/src/lib.rs @@ -93,4 +93,5 @@ pub trait Agent: AgentManager + HealthService + Send + Sync { async fn set_ip_tables(&self, req: SetIPTablesRequest) -> Result; async fn get_volume_stats(&self, req: VolumeStatsRequest) -> Result; async fn resize_volume(&self, req: ResizeVolumeRequest) -> Result; + async fn get_guest_details(&self, req: GetGuestDetailsRequest) -> Result; } diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs index 6b8b7a0752..85a4ee3fa4 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs @@ -741,6 +741,18 @@ impl CloudHypervisorInner { pub(crate) async fn get_hypervisor_metrics(&self) -> Result { todo!() } + + pub(crate) fn set_capabilities(&mut self, _flag: CapabilityBits) { + todo!() + } + + pub(crate) fn set_guest_memory_block_size(&mut self, _size: u32) { + todo!() + } + + pub(crate) fn guest_memory_block_size_mb(&self) -> u32 { + todo!() + } } // Log all output from the CH process until a shutdown signal is received. diff --git a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs index 6f8b6c269d..328333b9d9 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs @@ -8,7 +8,7 @@ use crate::device::DeviceType; use crate::{Hypervisor, VcpuThreadIds}; use anyhow::{Context, Result}; use async_trait::async_trait; -use kata_types::capabilities::Capabilities; +use kata_types::capabilities::{Capabilities, CapabilityBits}; use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; use persist::sandbox_persist::Persist; use std::sync::Arc; @@ -162,6 +162,21 @@ impl Hypervisor for CloudHypervisor { let inner = self.inner.read().await; inner.get_hypervisor_metrics().await } + + async fn set_capabilities(&self, flag: CapabilityBits) { + let mut inner = self.inner.write().await; + inner.set_capabilities(flag) + } + + async fn set_guest_memory_block_size(&self, size: u32) { + let mut inner = self.inner.write().await; + inner.set_guest_memory_block_size(size); + } + + async fn guest_memory_block_size(&self) -> u32 { + let inner = self.inner.read().await; + inner.guest_memory_block_size_mb() + } } #[async_trait] diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs index 7879f1c44a..51660f1ba1 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -68,6 +68,9 @@ pub struct DragonballInner { /// dragonball capabilities pub(crate) capabilities: Capabilities, + + /// the size of memory block of guest OS + pub(crate) guest_memory_block_size_mb: u32, } impl DragonballInner { @@ -92,6 +95,7 @@ impl DragonballInner { run_dir: "".to_string(), cached_block_devices: Default::default(), capabilities, + guest_memory_block_size_mb: 0, } } @@ -346,6 +350,18 @@ impl DragonballInner { pub fn hypervisor_config(&self) -> HypervisorConfig { self.config.clone() } + + pub(crate) fn set_capabilities(&mut self, flag: CapabilityBits) { + self.capabilities.add(flag); + } + + pub(crate) fn set_guest_memory_block_size(&mut self, size: u32) { + self.guest_memory_block_size_mb = size; + } + + pub(crate) fn get_guest_memory_block_size(&self) -> u32 { + self.guest_memory_block_size_mb + } } #[async_trait] @@ -387,6 +403,7 @@ impl Persist for DragonballInner { pending_devices: vec![], cached_block_devices: hypervisor_state.cached_block_devices, capabilities: Capabilities::new(), + guest_memory_block_size_mb: 0, }) } } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs index f0ed02f706..8a59e350a7 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs @@ -21,7 +21,7 @@ use dragonball::api::v1::{ Backend as DragonballBackend, NetworkInterfaceConfig as DragonballNetworkConfig, VirtioConfig as DragonballVirtioConfig, }; -use kata_types::capabilities::Capabilities; +use kata_types::capabilities::{Capabilities, CapabilityBits}; use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; use tokio::sync::RwLock; use tracing::instrument; @@ -178,6 +178,21 @@ impl Hypervisor for Dragonball { let inner = self.inner.read().await; inner.get_hypervisor_metrics().await } + + async fn set_capabilities(&self, flag: CapabilityBits) { + let mut inner = self.inner.write().await; + inner.set_capabilities(flag) + } + + async fn set_guest_memory_block_size(&self, size: u32) { + let mut inner = self.inner.write().await; + inner.set_guest_memory_block_size(size); + } + + async fn guest_memory_block_size(&self) -> u32 { + let inner = self.inner.read().await; + inner.get_guest_memory_block_size() + } } #[async_trait] diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index 737133de50..3716dff523 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -26,7 +26,7 @@ pub mod ch; use anyhow::Result; use async_trait::async_trait; use hypervisor_persist::HypervisorState; -use kata_types::capabilities::Capabilities; +use kata_types::capabilities::{Capabilities, CapabilityBits}; use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; pub use kata_types::config::hypervisor::HYPERVISOR_NAME_CH; @@ -103,4 +103,7 @@ pub trait Hypervisor: std::fmt::Debug + Send + Sync { async fn save_state(&self) -> Result; async fn capabilities(&self) -> Result; async fn get_hypervisor_metrics(&self) -> Result; + async fn set_capabilities(&self, flag: CapabilityBits); + async fn set_guest_memory_block_size(&self, size: u32); + async fn guest_memory_block_size(&self) -> u32; } diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index e4a3c4e0b0..18096bfc43 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -140,6 +140,18 @@ impl QemuInner { pub(crate) async fn get_hypervisor_metrics(&self) -> Result { todo!() } + + pub(crate) fn set_capabilities(&mut self, _flag: CapabilityBits) { + todo!() + } + + pub(crate) fn set_guest_memory_block_size(&mut self, _size: u32) { + todo!() + } + + pub(crate) fn guest_memory_block_size_mb(&self) -> u32 { + todo!() + } } use crate::device::DeviceType; diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs index 65b4b6e57d..904e582865 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs @@ -10,7 +10,7 @@ use crate::hypervisor_persist::HypervisorState; use crate::Hypervisor; use crate::{HypervisorConfig, VcpuThreadIds}; use inner::QemuInner; -use kata_types::capabilities::Capabilities; +use kata_types::capabilities::{Capabilities, CapabilityBits}; use anyhow::Result; use async_trait::async_trait; @@ -157,4 +157,19 @@ impl Hypervisor for Qemu { let inner = self.inner.read().await; inner.get_hypervisor_metrics().await } + + async fn set_capabilities(&self, flag: CapabilityBits) { + let mut inner = self.inner.write().await; + inner.set_capabilities(flag) + } + + async fn set_guest_memory_block_size(&self, size: u32) { + let mut inner = self.inner.write().await; + inner.set_guest_memory_block_size(size); + } + + async fn guest_memory_block_size(&self) -> u32 { + let inner = self.inner.read().await; + inner.guest_memory_block_size_mb() + } } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index e103f18d70..5e1e999059 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -8,7 +8,9 @@ use std::sync::Arc; use agent::kata::KataAgent; use agent::types::KernelModule; -use agent::{self, Agent, GetIPTablesRequest, SetIPTablesRequest, VolumeStatsRequest}; +use agent::{ + self, Agent, GetGuestDetailsRequest, GetIPTablesRequest, SetIPTablesRequest, VolumeStatsRequest, +}; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use common::message::{Action, Message}; @@ -18,6 +20,7 @@ use hypervisor::VsockConfig; use hypervisor::{dragonball::Dragonball, BlockConfig, Hypervisor, HYPERVISOR_DRAGONBALL}; use hypervisor::{utils::get_hvsock_path, HybridVsockConfig, DEFAULT_GUEST_VSOCK_CID}; use kata_sys_util::hooks::HookStates; +use kata_types::capabilities::CapabilityBits; use kata_types::config::TomlConfig; use persist::{self, sandbox_persist::Persist}; use resource::manager::ManagerArgs; @@ -196,7 +199,41 @@ impl VirtSandbox { // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createruntime-hooks let mut create_runtime_hook_states = HookStates::new(); create_runtime_hook_states.execute_hooks(create_runtime_hooks, Some(st.clone()))?; + Ok(()) + } + // store_guest_details will get the information from the guest OS, like memory block size, agent details and is memory hotplug probe support + async fn store_guest_details(&self) -> Result<()> { + // get the information from agent + let guest_details = self + .agent + .get_guest_details(GetGuestDetailsRequest { + mem_block_size: true, + mem_hotplug_probe: true, + }) + .await + .context("failed to store guest details")?; + + // set memory block size + self.hypervisor + .set_guest_memory_block_size(guest_details.mem_block_size_bytes as u32) + .await; + + // set memory hotplug probe + if guest_details.support_mem_hotplug_probe { + self.hypervisor + .set_capabilities(CapabilityBits::GuestMemoryHotplugProbe) + .await; + } + info!( + sl!(), + "memory block size is {}, memory probe support {}", + self.hypervisor.guest_memory_block_size().await, + self.hypervisor + .capabilities() + .await? + .is_mem_hotplug_probe_supported() + ); Ok(()) } @@ -375,6 +412,12 @@ impl Sandbox for VirtSandbox { .context("create sandbox")?; inner.state = SandboxState::Running; + + // get and store guest details + self.store_guest_details() + .await + .context("failed to store guest details")?; + let agent = self.agent.clone(); let sender = self.msg_sender.clone(); info!(sl!(), "oom watcher start"); From 81e55c424ae438fe78a1f3ccea39b572af67607f Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Wed, 17 May 2023 14:38:55 +0800 Subject: [PATCH 2/5] runtime-rs: add resize_memory trait for hypervisor Fixes: #6875 Signed-off-by: Zhongtao Hu --- src/runtime-rs/crates/hypervisor/Cargo.toml | 2 +- .../hypervisor/src/ch/inner_hypervisor.rs | 12 +++- .../crates/hypervisor/src/ch/mod.rs | 11 +++- .../crates/hypervisor/src/dragonball/inner.rs | 66 ++++++++++++++++++- .../crates/hypervisor/src/dragonball/mod.rs | 13 +++- .../hypervisor/src/dragonball/vmm_instance.rs | 13 ++++ src/runtime-rs/crates/hypervisor/src/lib.rs | 10 +++ .../crates/hypervisor/src/qemu/inner.rs | 10 ++- .../crates/hypervisor/src/qemu/mod.rs | 11 +++- 9 files changed, 138 insertions(+), 10 deletions(-) diff --git a/src/runtime-rs/crates/hypervisor/Cargo.toml b/src/runtime-rs/crates/hypervisor/Cargo.toml index e413791e7b..46879eb256 100644 --- a/src/runtime-rs/crates/hypervisor/Cargo.toml +++ b/src/runtime-rs/crates/hypervisor/Cargo.toml @@ -35,7 +35,7 @@ kata-types = { path = "../../../libs/kata-types" } logging = { path = "../../../libs/logging" } shim-interface = { path = "../../../libs/shim-interface" } -dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "vhost-net", "dbs-upcall"] } +dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "vhost-net", "dbs-upcall","virtio-mem", "virtio-balloon"] } ch-config = { path = "ch-config", optional = true } tests_utils = { path = "../../tests/utils" } diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs index 85a4ee3fa4..59d97ed2c4 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs @@ -8,8 +8,10 @@ use crate::ch::utils::get_api_socket_path; use crate::ch::utils::get_vsock_path; use crate::kernel_param::KernelParams; use crate::utils::{get_jailer_root, get_sandbox_path}; +use crate::MemoryConfig; +use crate::VM_ROOTFS_DRIVER_BLK; +use crate::VM_ROOTFS_DRIVER_PMEM; use crate::{VcpuThreadIds, VmmState}; -use crate::{VM_ROOTFS_DRIVER_BLK, VM_ROOTFS_DRIVER_PMEM}; use anyhow::{anyhow, Context, Result}; use ch_config::ch_api::{ cloud_hypervisor_vm_create, cloud_hypervisor_vm_start, cloud_hypervisor_vmm_ping, @@ -753,6 +755,14 @@ impl CloudHypervisorInner { pub(crate) fn guest_memory_block_size_mb(&self) -> u32 { todo!() } + + pub(crate) fn resize_memory( + &self, + _req_mem_mb: u32, + _curr_mem_mb: u32, + ) -> Result<(u32, MemoryConfig)> { + todo!() + } } // Log all output from the CH process until a shutdown signal is received. diff --git a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs index 328333b9d9..ff6b062b7b 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs @@ -5,7 +5,7 @@ use super::HypervisorState; use crate::device::DeviceType; -use crate::{Hypervisor, VcpuThreadIds}; +use crate::{Hypervisor, MemoryConfig, VcpuThreadIds}; use anyhow::{Context, Result}; use async_trait::async_trait; use kata_types::capabilities::{Capabilities, CapabilityBits}; @@ -177,6 +177,15 @@ impl Hypervisor for CloudHypervisor { let inner = self.inner.read().await; inner.guest_memory_block_size_mb() } + + async fn resize_memory( + &self, + req_mem_mb: u32, + curr_mem_mb: u32, + ) -> Result<(u32, MemoryConfig)> { + let inner = self.inner.read().await; + inner.resize_memory(req_mem_mb, curr_mem_mb) + } } #[async_trait] diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs index 51660f1ba1..3ac87168f5 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -6,13 +6,14 @@ use super::vmm_instance::VmmInstance; use crate::{ - device::DeviceType, hypervisor_persist::HypervisorState, kernel_param::KernelParams, VmmState, - DEV_HUGEPAGES, HUGETLBFS, HUGE_SHMEM, HYPERVISOR_DRAGONBALL, SHMEM, + device::DeviceType, hypervisor_persist::HypervisorState, kernel_param::KernelParams, + MemoryConfig, VmmState, DEV_HUGEPAGES, HUGETLBFS, HUGE_SHMEM, HYPERVISOR_DRAGONBALL, SHMEM, }; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use dragonball::{ api::v1::{BootSourceConfig, VcpuResizeInfo}, + device_manager::{balloon_dev_mgr::BalloonDeviceConfigInfo, mem_dev_mgr::MemDeviceConfigInfo}, vm::VmConfigInfo, }; @@ -26,6 +27,7 @@ use kata_types::{ }; use nix::mount::MsFlags; use persist::sandbox_persist::Persist; +use std::cmp::Ordering; use std::{collections::HashSet, fs::create_dir_all}; const DRAGONBALL_KERNEL: &str = "vmlinux"; @@ -343,6 +345,64 @@ impl DragonballInner { Ok((old_vcpus, new_vcpus)) } + // curr_mem_m size = default + hotplug + pub(crate) fn resize_memory( + &self, + req_mem_mb: u32, + curr_mem_mb: u32, + ) -> Result<(u32, MemoryConfig)> { + let mem_device_to_insert = match req_mem_mb.cmp(&curr_mem_mb) { + Ordering::Greater => { + // We need to insert a new memory device + let add_mem_mb = req_mem_mb - curr_mem_mb; + if self.config.memory_info.enable_virtio_mem { + Some(MemDeviceConfigInfo { + mem_id: format!("mem{}", curr_mem_mb), + size_mib: add_mem_mb as u64, + capacity_mib: add_mem_mb as u64, + multi_region: false, + host_numa_node_id: None, + guest_numa_node_id: None, + use_shared_irq: None, + use_generic_irq: None, + }) + } else { + None + } + } + Ordering::Less => { + // We need to insert a new balloon device to release memory + let balloon_config = BalloonDeviceConfigInfo { + balloon_id: format!("mem{}", curr_mem_mb), + size_mib: (curr_mem_mb - req_mem_mb) as u64, + use_shared_irq: None, + use_generic_irq: None, + f_deflate_on_oom: false, + f_reporting: false, + }; + self.vmm_instance + .insert_balloon_device(balloon_config) + .context("failed to insert balloon device")?; + None + } + Ordering::Equal => None, // Everything is already set up + }; + + // If we have a memory device to insert, do it now + if let Some(mem_config) = mem_device_to_insert { + self.vmm_instance + .insert_mem_device(mem_config) + .context("failed to insert memory device")?; + } + + Ok(( + req_mem_mb, + MemoryConfig { + ..Default::default() + }, + )) + } + pub fn set_hypervisor_config(&mut self, config: HypervisorConfig) { self.config = config; } @@ -359,7 +419,7 @@ impl DragonballInner { self.guest_memory_block_size_mb = size; } - pub(crate) fn get_guest_memory_block_size(&self) -> u32 { + pub(crate) fn guest_memory_block_size_mb(&self) -> u32 { self.guest_memory_block_size_mb } } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs index 8a59e350a7..3af4acc73d 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs @@ -26,7 +26,7 @@ use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; use tokio::sync::RwLock; use tracing::instrument; -use crate::{DeviceType, Hypervisor, NetworkConfig, VcpuThreadIds}; +use crate::{DeviceType, Hypervisor, MemoryConfig, NetworkConfig, VcpuThreadIds}; pub struct Dragonball { inner: Arc>, @@ -191,7 +191,16 @@ impl Hypervisor for Dragonball { async fn guest_memory_block_size(&self) -> u32 { let inner = self.inner.read().await; - inner.get_guest_memory_block_size() + inner.guest_memory_block_size_mb() + } + + async fn resize_memory( + &self, + req_mem_mb: u32, + curr_mem_mb: u32, + ) -> Result<(u32, MemoryConfig)> { + let inner = self.inner.read().await; + inner.resize_memory(req_mem_mb, curr_mem_mb) } } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs index 4311fdf98d..899f4f258c 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs @@ -19,6 +19,7 @@ use dragonball::{ InstanceInfo, InstanceState, NetworkInterfaceConfig, VcpuResizeInfo, VmmAction, VmmActionError, VmmData, VmmRequest, VmmResponse, VmmService, VsockDeviceConfigInfo, }, + device_manager::{balloon_dev_mgr::BalloonDeviceConfigInfo, mem_dev_mgr::MemDeviceConfigInfo}, vm::VmConfigInfo, Vmm, }; @@ -255,6 +256,18 @@ impl VmmInstance { Ok(()) } + pub fn insert_mem_device(&self, cfg: MemDeviceConfigInfo) -> Result<()> { + self.handle_request(Request::Sync(VmmAction::InsertMemDevice(cfg.clone()))) + .with_context(|| format!("Failed to insert memory device : {:?}", cfg))?; + Ok(()) + } + + pub fn insert_balloon_device(&self, cfg: BalloonDeviceConfigInfo) -> Result<()> { + self.handle_request(Request::Sync(VmmAction::InsertBalloonDevice(cfg.clone()))) + .with_context(|| format!("Failed to insert balloon device: {:?}", cfg))?; + Ok(()) + } + pub fn pause(&self) -> Result<()> { todo!() } diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index 3716dff523..9a21530085 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -73,6 +73,14 @@ pub struct VcpuThreadIds { pub vcpus: HashMap, } +#[derive(Debug, Default)] +pub struct MemoryConfig { + pub slot: u32, + pub size_mb: u32, + pub addr: u64, + pub probe: bool, +} + #[async_trait] pub trait Hypervisor: std::fmt::Debug + Send + Sync { // vm manager @@ -106,4 +114,6 @@ pub trait Hypervisor: std::fmt::Debug + Send + Sync { async fn set_capabilities(&self, flag: CapabilityBits); async fn set_guest_memory_block_size(&self, size: u32); async fn guest_memory_block_size(&self) -> u32; + async fn resize_memory(&self, req_mem_mb: u32, curr_mem_mb: u32) + -> Result<(u32, MemoryConfig)>; } diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index 18096bfc43..a2ec9704e3 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -5,7 +5,7 @@ use anyhow::Result; -use crate::{HypervisorConfig, VcpuThreadIds}; +use crate::{HypervisorConfig, MemoryConfig, VcpuThreadIds}; use kata_types::capabilities::{Capabilities, CapabilityBits}; const VSOCK_SCHEME: &str = "vsock"; @@ -152,6 +152,14 @@ impl QemuInner { pub(crate) fn guest_memory_block_size_mb(&self) -> u32 { todo!() } + + pub(crate) fn resize_memory( + &self, + _req_mem_mb: u32, + _curr_mem_mb: u32, + ) -> Result<(u32, MemoryConfig)> { + todo!() + } } use crate::device::DeviceType; diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs index 904e582865..afc6786545 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs @@ -7,7 +7,7 @@ mod inner; use crate::device::DeviceType; use crate::hypervisor_persist::HypervisorState; -use crate::Hypervisor; +use crate::{Hypervisor, MemoryConfig}; use crate::{HypervisorConfig, VcpuThreadIds}; use inner::QemuInner; use kata_types::capabilities::{Capabilities, CapabilityBits}; @@ -172,4 +172,13 @@ impl Hypervisor for Qemu { let inner = self.inner.read().await; inner.guest_memory_block_size_mb() } + + async fn resize_memory( + &self, + req_mem_mb: u32, + curr_mem_mb: u32, + ) -> Result<(u32, MemoryConfig)> { + let inner = self.inner.read().await; + inner.resize_memory(req_mem_mb, curr_mem_mb) + } } From 8d9fd9c067ab97f2f04418aff48d766bb1072407 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Wed, 17 May 2023 16:12:35 +0800 Subject: [PATCH 3/5] runtime-rs: support memory resize Fixes:#6875 Signed-off-by: Zhongtao Hu --- src/runtime-rs/Cargo.lock | 3 + .../hypervisor/src/ch/inner_hypervisor.rs | 6 +- .../crates/hypervisor/src/ch/mod.rs | 8 +- .../crates/hypervisor/src/dragonball/inner.rs | 88 +++++---- .../crates/hypervisor/src/dragonball/mod.rs | 10 +- src/runtime-rs/crates/hypervisor/src/lib.rs | 3 +- .../crates/hypervisor/src/qemu/inner.rs | 6 +- .../crates/hypervisor/src/qemu/mod.rs | 8 +- .../crates/resource/src/cpu_mem/cpu.rs | 23 +-- .../resource/src/cpu_mem/initial_size.rs | 21 ++- .../crates/resource/src/cpu_mem/mem.rs | 170 ++++++++++++++++++ .../crates/resource/src/cpu_mem/mod.rs | 1 + src/runtime-rs/crates/resource/src/manager.rs | 5 +- .../crates/resource/src/manager_inner.rs | 31 +++- .../crates/runtimes/common/Cargo.toml | 1 + .../runtimes/common/src/runtime_handler.rs | 5 +- .../runtimes/linux_container/Cargo.toml | 1 + .../runtimes/linux_container/src/lib.rs | 2 + src/runtime-rs/crates/runtimes/src/manager.rs | 40 ++++- .../crates/runtimes/virt_container/src/lib.rs | 14 +- .../crates/runtimes/wasm_container/Cargo.toml | 1 + .../crates/runtimes/wasm_container/src/lib.rs | 2 + 22 files changed, 336 insertions(+), 113 deletions(-) create mode 100644 src/runtime-rs/crates/resource/src/cpu_mem/mem.rs diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 575f1a0296..af023d7eac 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -508,6 +508,7 @@ dependencies = [ "oci", "persist", "protobuf 3.2.0", + "resource", "serde_json", "slog", "slog-scope", @@ -1761,6 +1762,7 @@ dependencies = [ "async-trait", "common", "kata-types", + "resource", "tokio", ] @@ -4273,6 +4275,7 @@ dependencies = [ "async-trait", "common", "kata-types", + "resource", "tokio", ] diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs index 59d97ed2c4..42f54fc68c 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs @@ -756,11 +756,7 @@ impl CloudHypervisorInner { todo!() } - pub(crate) fn resize_memory( - &self, - _req_mem_mb: u32, - _curr_mem_mb: u32, - ) -> Result<(u32, MemoryConfig)> { + pub(crate) fn resize_memory(&self, _req_mem_mb: u32) -> Result<(u32, MemoryConfig)> { todo!() } } diff --git a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs index ff6b062b7b..31bca4ef34 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs @@ -178,13 +178,9 @@ impl Hypervisor for CloudHypervisor { inner.guest_memory_block_size_mb() } - async fn resize_memory( - &self, - req_mem_mb: u32, - curr_mem_mb: u32, - ) -> Result<(u32, MemoryConfig)> { + async fn resize_memory(&self, req_mem_mb: u32) -> Result<(u32, MemoryConfig)> { let inner = self.inner.read().await; - inner.resize_memory(req_mem_mb, curr_mem_mb) + inner.resize_memory(req_mem_mb) } } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs index 3ac87168f5..55211fdddb 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -73,6 +73,9 @@ pub struct DragonballInner { /// the size of memory block of guest OS pub(crate) guest_memory_block_size_mb: u32, + + /// the hotplug memory size + pub(crate) mem_hotplug_size_mb: u32, } impl DragonballInner { @@ -98,6 +101,7 @@ impl DragonballInner { cached_block_devices: Default::default(), capabilities, guest_memory_block_size_mb: 0, + mem_hotplug_size_mb: 0, } } @@ -345,36 +349,14 @@ impl DragonballInner { Ok((old_vcpus, new_vcpus)) } - // curr_mem_m size = default + hotplug - pub(crate) fn resize_memory( - &self, - req_mem_mb: u32, - curr_mem_mb: u32, - ) -> Result<(u32, MemoryConfig)> { - let mem_device_to_insert = match req_mem_mb.cmp(&curr_mem_mb) { + pub(crate) fn resize_memory(&mut self, req_mem_mb: u32) -> Result<(u32, MemoryConfig)> { + let had_mem_mb = self.config.memory_info.default_memory + self.mem_hotplug_size_mb; + match req_mem_mb.cmp(&had_mem_mb) { Ordering::Greater => { - // We need to insert a new memory device - let add_mem_mb = req_mem_mb - curr_mem_mb; - if self.config.memory_info.enable_virtio_mem { - Some(MemDeviceConfigInfo { - mem_id: format!("mem{}", curr_mem_mb), - size_mib: add_mem_mb as u64, - capacity_mib: add_mem_mb as u64, - multi_region: false, - host_numa_node_id: None, - guest_numa_node_id: None, - use_shared_irq: None, - use_generic_irq: None, - }) - } else { - None - } - } - Ordering::Less => { - // We need to insert a new balloon device to release memory + // clean virtio-ballon device before hotplug memory, resize to 0 let balloon_config = BalloonDeviceConfigInfo { - balloon_id: format!("mem{}", curr_mem_mb), - size_mib: (curr_mem_mb - req_mem_mb) as u64, + balloon_id: "balloon0".to_owned(), + size_mib: 0, use_shared_irq: None, use_generic_irq: None, f_deflate_on_oom: false, @@ -383,17 +365,46 @@ impl DragonballInner { self.vmm_instance .insert_balloon_device(balloon_config) .context("failed to insert balloon device")?; - None - } - Ordering::Equal => None, // Everything is already set up - }; - // If we have a memory device to insert, do it now - if let Some(mem_config) = mem_device_to_insert { - self.vmm_instance - .insert_mem_device(mem_config) - .context("failed to insert memory device")?; - } + // update the hotplug size + self.mem_hotplug_size_mb = req_mem_mb - self.config.memory_info.default_memory; + + // insert a new memory device + let add_mem_mb = req_mem_mb - had_mem_mb; + self.vmm_instance.insert_mem_device(MemDeviceConfigInfo { + mem_id: format!("mem{}", self.mem_hotplug_size_mb), + size_mib: add_mem_mb as u64, + capacity_mib: add_mem_mb as u64, + multi_region: false, + host_numa_node_id: None, + guest_numa_node_id: None, + use_shared_irq: None, + use_generic_irq: None, + })?; + } + Ordering::Less => { + // we only use one balloon device here, and resize it to release memory + // the operation we do here is inserting a new balloon0 device or resizing it + let balloon_config = BalloonDeviceConfigInfo { + balloon_id: "balloon0".to_owned(), + size_mib: (had_mem_mb - req_mem_mb) as u64, + use_shared_irq: None, + use_generic_irq: None, + f_deflate_on_oom: false, + f_reporting: false, + }; + self.vmm_instance + .insert_balloon_device(balloon_config) + .context("failed to insert balloon device")?; + } + Ordering::Equal => { + // Everything is already set up + info!( + sl!(), + "memory size unchanged, no need to do memory resizing" + ); + } + }; Ok(( req_mem_mb, @@ -464,6 +475,7 @@ impl Persist for DragonballInner { cached_block_devices: hypervisor_state.cached_block_devices, capabilities: Capabilities::new(), guest_memory_block_size_mb: 0, + mem_hotplug_size_mb: 0, }) } } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs index 3af4acc73d..cd66ed5b0e 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs @@ -194,13 +194,9 @@ impl Hypervisor for Dragonball { inner.guest_memory_block_size_mb() } - async fn resize_memory( - &self, - req_mem_mb: u32, - curr_mem_mb: u32, - ) -> Result<(u32, MemoryConfig)> { - let inner = self.inner.read().await; - inner.resize_memory(req_mem_mb, curr_mem_mb) + async fn resize_memory(&self, req_mem_mb: u32) -> Result<(u32, MemoryConfig)> { + let mut inner = self.inner.write().await; + inner.resize_memory(req_mem_mb) } } diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index 9a21530085..eb236c2476 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -91,6 +91,7 @@ pub trait Hypervisor: std::fmt::Debug + Send + Sync { async fn save_vm(&self) -> Result<()>; async fn resume_vm(&self) -> Result<()>; async fn resize_vcpu(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)>; // returns (old_vcpus, new_vcpus) + async fn resize_memory(&self, req_mem_mb: u32) -> Result<(u32, MemoryConfig)>; // device manager async fn add_device(&self, device: DeviceType) -> Result; @@ -114,6 +115,4 @@ pub trait Hypervisor: std::fmt::Debug + Send + Sync { async fn set_capabilities(&self, flag: CapabilityBits); async fn set_guest_memory_block_size(&self, size: u32); async fn guest_memory_block_size(&self) -> u32; - async fn resize_memory(&self, req_mem_mb: u32, curr_mem_mb: u32) - -> Result<(u32, MemoryConfig)>; } diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index a2ec9704e3..dfc6d34e4c 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -153,11 +153,7 @@ impl QemuInner { todo!() } - pub(crate) fn resize_memory( - &self, - _req_mem_mb: u32, - _curr_mem_mb: u32, - ) -> Result<(u32, MemoryConfig)> { + pub(crate) fn resize_memory(&self, _req_mem_mb: u32) -> Result<(u32, MemoryConfig)> { todo!() } } diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs index afc6786545..4f5fc485c6 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs @@ -173,12 +173,8 @@ impl Hypervisor for Qemu { inner.guest_memory_block_size_mb() } - async fn resize_memory( - &self, - req_mem_mb: u32, - curr_mem_mb: u32, - ) -> Result<(u32, MemoryConfig)> { + async fn resize_memory(&self, req_mem_mb: u32) -> Result<(u32, MemoryConfig)> { let inner = self.inner.read().await; - inner.resize_memory(req_mem_mb, curr_mem_mb) + inner.resize_memory(req_mem_mb) } } diff --git a/src/runtime-rs/crates/resource/src/cpu_mem/cpu.rs b/src/runtime-rs/crates/resource/src/cpu_mem/cpu.rs index 661e1a5e3a..957dd5667a 100644 --- a/src/runtime-rs/crates/resource/src/cpu_mem/cpu.rs +++ b/src/runtime-rs/crates/resource/src/cpu_mem/cpu.rs @@ -11,7 +11,6 @@ use std::{ sync::Arc, }; -use agent::{Agent, OnlineCPUMemRequest}; use anyhow::{Context, Ok, Result}; use hypervisor::Hypervisor; use kata_types::{config::TomlConfig, cpu::LinuxContainerCpuResources}; @@ -52,7 +51,6 @@ impl CpuResource { linux_cpus: Option<&LinuxCpu>, op: ResourceUpdateOp, hypervisor: &dyn Hypervisor, - agent: &dyn Agent, ) -> Result<()> { self.update_container_cpu_resources(cid, linux_cpus, op) .await @@ -67,13 +65,13 @@ impl CpuResource { } let curr_vcpus = self - .do_update_cpu_resources(vcpu_required, op, hypervisor, agent) + .do_update_cpu_resources(vcpu_required, op, hypervisor) .await?; self.update_current_vcpu(curr_vcpus).await; Ok(()) } - async fn current_vcpu(&self) -> u32 { + pub(crate) async fn current_vcpu(&self) -> u32 { let current_vcpu = self.current_vcpu.read().await; *current_vcpu } @@ -148,7 +146,6 @@ impl CpuResource { new_vcpus: u32, op: ResourceUpdateOp, hypervisor: &dyn Hypervisor, - agent: &dyn Agent, ) -> Result { let old_vcpus = self.current_vcpu().await; @@ -164,25 +161,11 @@ impl CpuResource { // the number of vcpus would not be lower than the default size let new_vcpus = cmp::max(new_vcpus, self.default_vcpu); - let (old, new) = hypervisor + let (_, new) = hypervisor .resize_vcpu(old_vcpus, new_vcpus) .await .context("resize vcpus")?; - if old < new { - let add = new - old; - info!(sl!(), "request to onlineCpuMem with {:?} cpus", add); - - agent - .online_cpu_mem(OnlineCPUMemRequest { - wait: false, - nb_cpus: new, - cpu_only: true, - }) - .await - .context("online vcpus")?; - } - Ok(new) } } diff --git a/src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs b/src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs index bb3af793d0..b3ca4bdad0 100644 --- a/src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs +++ b/src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs @@ -19,6 +19,7 @@ use kata_types::{ struct InitialSize { vcpu: u32, mem_mb: u32, + orig_toml_default_mem: u32, } // generate initial resource(vcpu and memory in MiB) from spec's information @@ -66,7 +67,11 @@ impl TryFrom<&oci::Spec> for InitialSize { sl!(), "(from PodSandbox's annotation / SingleContainer's spec) initial size: vcpu={}, mem_mb={}", vcpu, mem_mb ); - Ok(Self { vcpu, mem_mb }) + Ok(Self { + vcpu, + mem_mb, + orig_toml_default_mem: 0, + }) } } @@ -93,7 +98,7 @@ impl InitialSizeManager { }) } - pub fn setup_config(&self, config: &mut TomlConfig) -> Result<()> { + pub fn setup_config(&mut self, config: &mut TomlConfig) -> Result<()> { // update this data to the hypervisor config for later use by hypervisor let hypervisor_name = &config.runtime.hypervisor_name; let hv = config @@ -104,6 +109,7 @@ impl InitialSizeManager { if self.resource.vcpu > 0 { hv.cpu_info.default_vcpus = self.resource.vcpu as i32 } + self.resource.orig_toml_default_mem = hv.memory_info.default_memory; if self.resource.mem_mb > 0 { // since the memory overhead introduced by kata-agent and system components // will really affect the amount of memory the user can use, so we choose to @@ -114,6 +120,10 @@ impl InitialSizeManager { } Ok(()) } + + pub fn get_orig_toml_default_mem(&self) -> u32 { + self.resource.orig_toml_default_mem + } } fn get_nr_vcpu(resource: &LinuxContainerCpuResources) -> u32 { @@ -173,7 +183,11 @@ mod tests { quota: None, memory: None, }, - result: InitialSize { vcpu: 0, mem_mb: 0 }, + result: InitialSize { + vcpu: 0, + mem_mb: 0, + orig_toml_default_mem: 0, + }, }, TestData { desc: "normal resource limit", @@ -186,6 +200,7 @@ mod tests { result: InitialSize { vcpu: 3, mem_mb: 512, + orig_toml_default_mem: 0, }, }, ] diff --git a/src/runtime-rs/crates/resource/src/cpu_mem/mem.rs b/src/runtime-rs/crates/resource/src/cpu_mem/mem.rs new file mode 100644 index 0000000000..88dbdb12e2 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/cpu_mem/mem.rs @@ -0,0 +1,170 @@ +// Copyright (c) 2019-2023 Alibaba Cloud +// Copyright (c) 2019-2023 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::collections::HashMap; +use std::sync::Arc; + +use anyhow::{Context, Ok, Result}; +use hypervisor::Hypervisor; +use kata_types::config::TomlConfig; +use oci::LinuxResources; +use tokio::sync::RwLock; + +use crate::cpu_mem::initial_size::InitialSizeManager; +use crate::ResourceUpdateOp; + +// MIB_TO_BYTES_SHIFT the number to shift needed to convert MiB to Bytes +pub const MIB_TO_BYTES_SHIFT: i32 = 20; + +#[derive(Default, Debug, Clone)] +pub struct MemResource { + /// Current memory + pub(crate) current_mem: Arc>, + + /// Default memory + pub(crate) orig_toml_default_mem: u32, + + /// MemResource of each container + pub(crate) container_mem_resources: Arc>>, + + /// Use guest swap + pub(crate) use_guest_swap: bool, +} + +impl MemResource { + pub fn new(config: Arc, init_size_manager: InitialSizeManager) -> Result { + let hypervisor_name = config.runtime.hypervisor_name.clone(); + let hypervisor_config = config + .hypervisor + .get(&hypervisor_name) + .context("failed to get hypervisor")?; + + Ok(Self { + current_mem: Arc::new(RwLock::new(hypervisor_config.memory_info.default_memory)), + container_mem_resources: Arc::new(RwLock::new(HashMap::new())), + use_guest_swap: hypervisor_config.memory_info.enable_guest_swap, + orig_toml_default_mem: init_size_manager.get_orig_toml_default_mem(), + }) + } + + pub(crate) async fn update_mem_resources( + &self, + cid: &str, + linux_resources: Option<&LinuxResources>, + op: ResourceUpdateOp, + hypervisor: &dyn Hypervisor, + ) -> Result<()> { + self.update_container_mem_resources(cid, linux_resources, op) + .await + .context("update container memory resources")?; + // the unit here is MB + let (mut mem_sb_mb, need_pod_swap, swap_sb_mb) = self + .total_mems(self.use_guest_swap) + .await + .context("failed to calculate total memory requirement for containers")?; + mem_sb_mb += self.orig_toml_default_mem; + if need_pod_swap { + mem_sb_mb += swap_sb_mb; + } + info!(sl!(), "calculate mem_sb_mb {}", mem_sb_mb); + + let curr_mem = self + .do_update_mem_resource(mem_sb_mb, swap_sb_mb, hypervisor) + .await + .context("failed to update_mem_resource")?; + + self.update_current_mem(curr_mem).await; + Ok(()) + } + + async fn update_current_mem(&self, new_mem: u32) { + let mut current_mem = self.current_mem.write().await; + *current_mem = new_mem; + } + + async fn total_mems(&self, use_guest_swap: bool) -> Result<(u32, bool, u32)> { + // sb stands for sandbox + let mut mem_sandbox = 0; + let mut need_pod_swap = false; + let mut swap_sandbox = 0; + + let resources = self.container_mem_resources.read().await; + + for (_, r) in resources.iter() { + for l in &r.hugepage_limits { + mem_sandbox += l.limit; + } + + if let Some(memory) = &r.memory { + // set current_limit to 0 if memory limit is not set to container + let current_limit = memory.limit.map_or(0, |limit| { + mem_sandbox += limit as u64; + info!(sl!(), "memory sb: {}, memory limit: {}", mem_sandbox, limit); + limit + }); + + if let Some(swappiness) = memory.swappiness { + if swappiness > 0 && use_guest_swap { + if let Some(swap) = memory.swap { + if swap > current_limit { + swap_sandbox = swap.saturating_sub(current_limit); + } + } + // if current_limit is 0, the container will have access to the entire memory available on the host system + // so we add swap for this + else if current_limit == 0 { + need_pod_swap = true; + } else { + swap_sandbox += current_limit; + } + } + } + } + } + + Ok(( + (mem_sandbox >> MIB_TO_BYTES_SHIFT) as u32, + need_pod_swap, + (swap_sandbox >> MIB_TO_BYTES_SHIFT) as u32, + )) + } + + // update container_cpu_resources field + async fn update_container_mem_resources( + &self, + cid: &str, + linux_resources: Option<&LinuxResources>, + op: ResourceUpdateOp, + ) -> Result<()> { + if let Some(r) = linux_resources { + let mut resources = self.container_mem_resources.write().await; + match op { + ResourceUpdateOp::Add | ResourceUpdateOp::Update => { + resources.insert(cid.to_owned(), r.clone()); + } + ResourceUpdateOp::Del => { + resources.remove(cid); + } + } + } + Ok(()) + } + + async fn do_update_mem_resource( + &self, + new_mem: u32, + _swap_sz_mb: u32, + hypervisor: &dyn Hypervisor, + ) -> Result { + info!(sl!(), "requesting vmm to update memory to {:?}", new_mem); + let (new_memory, _mem_config) = hypervisor + .resize_memory(new_mem) + .await + .context("resize memory")?; + + Ok(new_memory) + } +} diff --git a/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs b/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs index f2984cd1cb..191fe16f2a 100644 --- a/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs +++ b/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs @@ -6,3 +6,4 @@ pub mod cpu; pub mod initial_size; +pub mod mem; diff --git a/src/runtime-rs/crates/resource/src/manager.rs b/src/runtime-rs/crates/resource/src/manager.rs index 25cd3301ff..74ae179836 100644 --- a/src/runtime-rs/crates/resource/src/manager.rs +++ b/src/runtime-rs/crates/resource/src/manager.rs @@ -19,6 +19,7 @@ use persist::sandbox_persist::Persist; use tokio::sync::RwLock; use tracing::instrument; +use crate::cpu_mem::initial_size::InitialSizeManager; use crate::network::NetworkConfig; use crate::resource_persist::ResourceState; use crate::ResourceUpdateOp; @@ -47,13 +48,15 @@ impl ResourceManager { agent: Arc, hypervisor: Arc, toml_config: Arc, + init_size_manager: InitialSizeManager, ) -> Result { // Regist resource logger for later use. logging::register_subsystem_logger("runtimes", "resource"); Ok(Self { inner: Arc::new(RwLock::new( - ResourceManagerInner::new(sid, agent, hypervisor, toml_config).await?, + ResourceManagerInner::new(sid, agent, hypervisor, toml_config, init_size_manager) + .await?, )), }) } diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index 592b2f5d4f..38adc90ed1 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -6,7 +6,7 @@ use std::{sync::Arc, thread}; -use agent::{types::Device, Agent, Storage}; +use agent::{types::Device, Agent, OnlineCPUMemRequest, Storage}; use anyhow::{anyhow, Context, Ok, Result}; use async_trait::async_trait; use hypervisor::{ @@ -25,7 +25,7 @@ use tokio::{runtime, sync::RwLock}; use crate::{ cgroups::{CgroupArgs, CgroupsResource}, - cpu_mem::cpu::CpuResource, + cpu_mem::{cpu::CpuResource, initial_size::InitialSizeManager, mem::MemResource}, manager::ManagerArgs, network::{self, Network, NetworkConfig}, resource_persist::ResourceState, @@ -48,6 +48,7 @@ pub(crate) struct ResourceManagerInner { pub volume_resource: VolumeResource, pub cgroups_resource: CgroupsResource, pub cpu_resource: CpuResource, + pub mem_resource: MemResource, } impl ResourceManagerInner { @@ -56,6 +57,7 @@ impl ResourceManagerInner { agent: Arc, hypervisor: Arc, toml_config: Arc, + init_size_manager: InitialSizeManager, ) -> Result { // create device manager let dev_manager = DeviceManager::new(hypervisor.clone()) @@ -64,6 +66,7 @@ impl ResourceManagerInner { let cgroups_resource = CgroupsResource::new(sid, &toml_config)?; let cpu_resource = CpuResource::new(toml_config.clone())?; + let mem_resource = MemResource::new(toml_config.clone(), init_size_manager)?; Ok(Self { sid: sid.to_string(), toml_config, @@ -76,6 +79,7 @@ impl ResourceManagerInner { volume_resource: VolumeResource::new(), cgroups_resource, cpu_resource, + mem_resource, }) } @@ -427,15 +431,23 @@ impl ResourceManagerInner { // if static_sandbox_resource_mgmt, we will not have to update sandbox's cpu or mem resource if !self.toml_config.runtime.static_sandbox_resource_mgmt { + // update cpu self.cpu_resource - .update_cpu_resources( - cid, - linux_cpus, - op, - self.hypervisor.as_ref(), - self.agent.as_ref(), - ) + .update_cpu_resources(cid, linux_cpus, op, self.hypervisor.as_ref()) .await?; + // update memory + self.mem_resource + .update_mem_resources(cid, linux_resources, op, self.hypervisor.as_ref()) + .await?; + + self.agent + .online_cpu_mem(OnlineCPUMemRequest { + wait: false, + nb_cpus: self.cpu_resource.current_vcpu().await, + cpu_only: false, + }) + .await + .context("online vcpus")?; } // we should firstly update the vcpus and mems, and then update the host cgroups @@ -516,6 +528,7 @@ impl Persist for ResourceManagerInner { .await?, toml_config: Arc::new(TomlConfig::default()), cpu_resource: CpuResource::default(), + mem_resource: MemResource::default(), }) } } diff --git a/src/runtime-rs/crates/runtimes/common/Cargo.toml b/src/runtime-rs/crates/runtimes/common/Cargo.toml index a60e1f5f1f..45f7ca490f 100644 --- a/src/runtime-rs/crates/runtimes/common/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/common/Cargo.toml @@ -26,4 +26,5 @@ agent = { path = "../../agent" } kata-sys-util = { path = "../../../../libs/kata-sys-util" } kata-types = { path = "../../../../libs/kata-types" } oci = { path = "../../../../libs/oci" } +resource = { path = "../../resource" } diff --git a/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs b/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs index 80e4149c3b..d86d4e4f8b 100644 --- a/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs +++ b/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs @@ -6,13 +6,13 @@ use std::sync::Arc; +use crate::{message::Message, ContainerManager, Sandbox}; use anyhow::Result; use async_trait::async_trait; use kata_types::config::TomlConfig; +use resource::cpu_mem::initial_size::InitialSizeManager; use tokio::sync::mpsc::Sender; -use crate::{message::Message, ContainerManager, Sandbox}; - #[derive(Clone)] pub struct RuntimeInstance { pub sandbox: Arc, @@ -38,6 +38,7 @@ pub trait RuntimeHandler: Send + Sync { sid: &str, msg_sender: Sender, config: Arc, + init_size_manager: InitialSizeManager, ) -> Result; fn cleanup(&self, id: &str) -> Result<()>; diff --git a/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml b/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml index de3c03ffd0..b258d3634f 100644 --- a/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml @@ -11,3 +11,4 @@ tokio = { version = "1.28.1" } common = { path = "../common" } kata-types = { path = "../../../../libs/kata-types" } +resource = { path = "../../resource" } \ No newline at end of file diff --git a/src/runtime-rs/crates/runtimes/linux_container/src/lib.rs b/src/runtime-rs/crates/runtimes/linux_container/src/lib.rs index 406ccb0b60..70584ca3b2 100644 --- a/src/runtime-rs/crates/runtimes/linux_container/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/linux_container/src/lib.rs @@ -9,6 +9,7 @@ use anyhow::Result; use async_trait::async_trait; use common::{message::Message, RuntimeHandler, RuntimeInstance}; use kata_types::config::TomlConfig; +use resource::cpu_mem::initial_size::InitialSizeManager; use tokio::sync::mpsc::Sender; pub struct LinuxContainer {} @@ -32,6 +33,7 @@ impl RuntimeHandler for LinuxContainer { _sid: &str, _msg_sender: Sender, _config: Arc, + _init_size_manager: InitialSizeManager, ) -> Result { todo!() } diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs index 04ccf6c011..766a02e115 100644 --- a/src/runtime-rs/crates/runtimes/src/manager.rs +++ b/src/runtime-rs/crates/runtimes/src/manager.rs @@ -91,6 +91,7 @@ impl RuntimeHandlerManagerInner { network_env: SandboxNetworkEnv, dns: Vec, config: Arc, + init_size_manager: InitialSizeManager, ) -> Result<()> { info!(sl!(), "new runtime handler {}", &config.runtime.name); let runtime_handler = match config.runtime.name.as_str() { @@ -105,7 +106,12 @@ impl RuntimeHandlerManagerInner { _ => return Err(anyhow!("Unsupported runtime: {}", &config.runtime.name)), }; let runtime_instance = runtime_handler - .new_instance(&self.id, self.msg_sender.clone(), config.clone()) + .new_instance( + &self.id, + self.msg_sender.clone(), + config.clone(), + init_size_manager, + ) .await .context("new runtime instance")?; @@ -160,7 +166,21 @@ impl RuntimeHandlerManagerInner { } } - let config = load_config(spec, options).context("load config")?; + let mut config = load_config(spec, options).context("load config")?; + + // Sandbox sizing information *may* be provided in two scenarios: + // 1. The upper layer runtime (ie, containerd or crio) provide sandbox sizing information as an annotation + // in the 'sandbox container's' spec. This would typically be a scenario where as part of a create sandbox + // request the upper layer runtime receives this information as part of a pod, and makes it available to us + // for sizing purposes. + // 2. If this is not a sandbox infrastructure container, but instead a standalone single container (analogous to "docker run..."), + // then the container spec itself will contain appropriate sizing information for the entire sandbox (since it is + // a single container. + let mut initial_size_manager = + InitialSizeManager::new(spec).context("failed to construct static resource manager")?; + initial_size_manager + .setup_config(&mut config) + .context("failed to setup static resource mgmt config")?; update_component_log_level(&config); @@ -202,10 +222,16 @@ impl RuntimeHandlerManagerInner { netns, network_created, }; - - self.init_runtime_handler(spec, state, network_env, dns, Arc::new(config)) - .await - .context("init runtime handler")?; + self.init_runtime_handler( + spec, + state, + network_env, + dns, + Arc::new(config), + initial_size_manager, + ) + .await + .context("init runtime handler")?; // the sandbox creation can reach here only once and the sandbox is created // so we can safely create the shim management socket right now @@ -507,7 +533,7 @@ fn load_config(spec: &oci::Spec, option: &Option>) -> Result // 2. If this is not a sandbox infrastructure container, but instead a standalone single container (analogous to "docker run..."), // then the container spec itself will contain appropriate sizing information for the entire sandbox (since it is // a single container. - let initial_size_manager = + let mut initial_size_manager = InitialSizeManager::new(spec).context("failed to construct static resource manager")?; initial_size_manager .setup_config(&mut toml_config) diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs b/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs index 295b64fe11..60a3a6eb0e 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs @@ -31,6 +31,7 @@ use hypervisor::ch::CloudHypervisor; #[cfg(feature = "cloud-hypervisor")] use kata_types::config::{hypervisor::HYPERVISOR_NAME_CH, CloudHypervisorConfig}; +use resource::cpu_mem::initial_size::InitialSizeManager; use resource::ResourceManager; use sandbox::VIRTCONTAINER; use tokio::sync::mpsc::Sender; @@ -77,13 +78,22 @@ impl RuntimeHandler for VirtContainer { sid: &str, msg_sender: Sender, config: Arc, + init_size_manager: InitialSizeManager, ) -> Result { let hypervisor = new_hypervisor(&config).await.context("new hypervisor")?; // get uds from hypervisor and get config from toml_config let agent = new_agent(&config).context("new agent")?; - let resource_manager = - Arc::new(ResourceManager::new(sid, agent.clone(), hypervisor.clone(), config).await?); + let resource_manager = Arc::new( + ResourceManager::new( + sid, + agent.clone(), + hypervisor.clone(), + config, + init_size_manager, + ) + .await?, + ); let pid = std::process::id(); let sandbox = sandbox::VirtSandbox::new( diff --git a/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml b/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml index 4f098295a7..fc36796321 100644 --- a/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml @@ -11,3 +11,4 @@ tokio = { version = "1.28.1" } common = { path = "../common" } kata-types = { path = "../../../../libs/kata-types" } +resource = { path = "../../resource" } \ No newline at end of file diff --git a/src/runtime-rs/crates/runtimes/wasm_container/src/lib.rs b/src/runtime-rs/crates/runtimes/wasm_container/src/lib.rs index 77282ac9cf..1a6e0ce430 100644 --- a/src/runtime-rs/crates/runtimes/wasm_container/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/wasm_container/src/lib.rs @@ -9,6 +9,7 @@ use anyhow::Result; use async_trait::async_trait; use common::{message::Message, RuntimeHandler, RuntimeInstance}; use kata_types::config::TomlConfig; +use resource::cpu_mem::initial_size::InitialSizeManager; use tokio::sync::mpsc::Sender; pub struct WasmContainer {} @@ -31,6 +32,7 @@ impl RuntimeHandler for WasmContainer { _sid: &str, _msg_sender: Sender, _config: Arc, + _init_size_manager: InitialSizeManager, ) -> Result { todo!() } From 60394171040c8507de0cef3dda3e223e520985e6 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Wed, 21 Jun 2023 17:51:53 +0800 Subject: [PATCH 4/5] runtime-rs: add default_maxmemory in config file add default_maxmemory in config file Fixes:#6875 Signed-off-by: Zhongtao Hu --- src/libs/kata-types/src/config/hypervisor/mod.rs | 7 +++++++ src/runtime-rs/Makefile | 3 +++ src/runtime-rs/config/configuration-dragonball.toml.in | 7 +++++++ 3 files changed, 17 insertions(+) diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index 69c25200de..fc59e8cd2a 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -653,6 +653,13 @@ pub struct MemoryInfo { /// If swap_in_bytes and memory_limit_in_bytes is not set, the size should be default_memory. #[serde(default)] pub enable_guest_swap: bool, + + /// Default maximum memory in MiB per SB / VM + /// unspecified or == 0 --> will be set to the actual amount of physical RAM + /// > 0 <= amount of physical RAM --> will be set to the specified number + /// > amount of physical RAM --> will be set to the actual amount of physical RAM + #[serde(default)] + pub default_maxmemory: u32, } impl MemoryInfo { diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index fef3ee3728..b48b1436a0 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -124,6 +124,8 @@ DEFMEMSZ := 2048 # - vm template memory # - hugepage memory DEFMEMSLOTS := 10 +# Default maximum memory in MiB +DEFMAXMEMSZ := 0 ##VAR DEFBRIDGES= Default number of bridges DEFBRIDGES := 0 DEFENABLEANNOTATIONS := [\"kernel_params\"] @@ -307,6 +309,7 @@ USER_VARS += DEFMAXVCPUS_ACRN USER_VARS += DEFMAXVCPUS_DB USER_VARS += DEFMEMSZ USER_VARS += DEFMEMSLOTS +USER_VARS += DEFMAXMEMSZ USER_VARS += DEFBRIDGES USER_VARS += DEFNETWORKMODEL_DB USER_VARS += DEFNETWORKMODEL_CLH diff --git a/src/runtime-rs/config/configuration-dragonball.toml.in b/src/runtime-rs/config/configuration-dragonball.toml.in index 4c057af5c4..7f08a7db92 100644 --- a/src/runtime-rs/config/configuration-dragonball.toml.in +++ b/src/runtime-rs/config/configuration-dragonball.toml.in @@ -101,6 +101,13 @@ default_bridges = @DEFBRIDGES@ # If unspecified then it will be set @DEFMEMSZ@ MiB. default_memory = @DEFMEMSZ@ +# Default maximum memory in MiB per SB / VM +# unspecified or == 0 --> will be set to the actual amount of physical RAM +# > 0 <= amount of physical RAM --> will be set to the specified number +# > amount of physical RAM --> will be set to the actual amount of physical RAM +default_maxmemory = @DEFMAXMEMSZ@ + + # Block storage driver to be used for the hypervisor in case the container # rootfs is backed by a block device. DB only supports virtio-blk. block_device_driver = "@DEFBLOCKSTORAGEDRIVER_DB@" From 9a37e77f2a5a238b76adebc1c01f135e2e8c4737 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Wed, 21 Jun 2023 18:10:38 +0800 Subject: [PATCH 5/5] runtime-rs: check the update memory size check the update memory size greater than default max memory size Fixes:#6875 Signed-off-by: Zhongtao Hu --- src/libs/kata-types/Cargo.toml | 3 +- src/libs/kata-types/src/annotations/mod.rs | 4 +- src/libs/kata-types/src/capabilities.rs | 6 +- .../kata-types/src/config/hypervisor/mod.rs | 36 +++--- src/runtime-rs/Cargo.lock | 107 ++++++++++++++++-- .../hypervisor/src/ch/inner_hypervisor.rs | 2 +- .../crates/hypervisor/src/ch/mod.rs | 4 +- .../crates/hypervisor/src/dragonball/inner.rs | 76 +++++++++---- .../crates/hypervisor/src/dragonball/mod.rs | 4 +- src/runtime-rs/crates/hypervisor/src/lib.rs | 2 +- .../crates/hypervisor/src/qemu/inner.rs | 2 +- .../crates/hypervisor/src/qemu/mod.rs | 4 +- .../crates/resource/src/cpu_mem/mem.rs | 69 ++--------- .../crates/resource/src/manager_inner.rs | 2 +- .../runtimes/virt_container/src/sandbox.rs | 2 +- 15 files changed, 201 insertions(+), 122 deletions(-) diff --git a/src/libs/kata-types/Cargo.toml b/src/libs/kata-types/Cargo.toml index 57e21c6731..c139726ccd 100644 --- a/src/libs/kata-types/Cargo.toml +++ b/src/libs/kata-types/Cargo.toml @@ -14,7 +14,7 @@ edition = "2018" bitmask-enum = "2.1.0" anyhow = "1.0" base64 = "0.13.0" -byte-unit = "3.1.4" +byte-unit = "5.0.3" glob = "0.3.0" lazy_static = "1.4.0" num_cpus = "1.13.1" @@ -26,6 +26,7 @@ serde_json = "1.0.73" thiserror = "1.0" toml = "0.5.8" serde-enum-str = "0.4" +sysinfo = "0.29.11" oci = { path = "../oci" } safe-path = { path = "../safe-path" } diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index f888c5cb19..fd316c2e2f 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -705,10 +705,10 @@ impl Annotation { } // Hypervisor Memory related annotations KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY => { - match byte_unit::Byte::from_str(value) { + match byte_unit::Byte::parse_str(value,true) { Ok(mem_bytes) => { let memory_size = mem_bytes - .get_adjusted_unit(byte_unit::ByteUnit::MiB) + .get_adjusted_unit(byte_unit::Unit::MiB) .get_value() as u32; info!(sl!(), "get mem {} from annotations: {}", memory_size, value); diff --git a/src/libs/kata-types/src/capabilities.rs b/src/libs/kata-types/src/capabilities.rs index b45bba9421..485e5bccdc 100644 --- a/src/libs/kata-types/src/capabilities.rs +++ b/src/libs/kata-types/src/capabilities.rs @@ -20,7 +20,7 @@ pub enum CapabilityBits { /// hypervisor supports hybrid-vsock HybridVsockSupport, /// hypervisor supports memory hotplug probe interface - GuestMemoryHotplugProbe, + GuestMemoryProbe, } /// Capabilities describe a virtcontainers hypervisor capabilities through a bit mask. @@ -81,7 +81,7 @@ impl Capabilities { /// is_mem_hotplug_probe_supported tells if the hypervisor supports hotplug probe interface pub fn is_mem_hotplug_probe_supported(&self) -> bool { - self.flags.and(CapabilityBits::GuestMemoryHotplugProbe) != 0 + self.flags.and(CapabilityBits::GuestMemoryProbe) != 0 } } @@ -130,7 +130,7 @@ mod tests { cap.set(CapabilityBits::HybridVsockSupport); assert!(cap.is_hybrid_vsock_supported()); // test append capabilities - cap.add(CapabilityBits::GuestMemoryHotplugProbe); + cap.add(CapabilityBits::GuestMemoryProbe); assert!(cap.is_mem_hotplug_probe_supported()); assert!(cap.is_fs_sharing_supported()); } diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index fc59e8cd2a..d00d2533db 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -22,18 +22,18 @@ //! part and common part. But the Kata 2.0 has adopted a policy to build a superset for all //! hypervisors, so let's contain it... +use super::{default, ConfigOps, ConfigPlugin, TomlConfig}; +use crate::annotations::KATA_ANNO_CFG_HYPERVISOR_PREFIX; +use crate::{eother, resolve_path, sl, validate_path}; +use byte_unit::{Byte, Unit}; +use lazy_static::lazy_static; +use regex::RegexSet; +use serde_enum_str::{Deserialize_enum_str, Serialize_enum_str}; use std::collections::HashMap; use std::io::{self, Result}; use std::path::Path; use std::sync::{Arc, Mutex}; - -use lazy_static::lazy_static; -use regex::RegexSet; -use serde_enum_str::{Deserialize_enum_str, Serialize_enum_str}; - -use super::{default, ConfigOps, ConfigPlugin, TomlConfig}; -use crate::annotations::KATA_ANNO_CFG_HYPERVISOR_PREFIX; -use crate::{eother, resolve_path, sl, validate_path}; +use sysinfo::{System, SystemExt}; mod dragonball; pub use self::dragonball::{DragonballConfig, HYPERVISOR_NAME_DRAGONBALL}; @@ -586,6 +586,13 @@ pub struct MemoryInfo { #[serde(default)] pub default_memory: u32, + /// Default maximum memory in MiB per SB / VM + /// unspecified or == 0 --> will be set to the actual amount of physical RAM + /// > 0 <= amount of physical RAM --> will be set to the specified number + /// > amount of physical RAM --> will be set to the actual amount of physical RAM + #[serde(default)] + pub default_maxmemory: u32, + /// Default memory slots per SB/VM. /// /// This is will determine the times that memory will be hotadded to sandbox/VM. @@ -653,13 +660,6 @@ pub struct MemoryInfo { /// If swap_in_bytes and memory_limit_in_bytes is not set, the size should be default_memory. #[serde(default)] pub enable_guest_swap: bool, - - /// Default maximum memory in MiB per SB / VM - /// unspecified or == 0 --> will be set to the actual amount of physical RAM - /// > 0 <= amount of physical RAM --> will be set to the specified number - /// > amount of physical RAM --> will be set to the actual amount of physical RAM - #[serde(default)] - pub default_maxmemory: u32, } impl MemoryInfo { @@ -669,6 +669,12 @@ impl MemoryInfo { self.file_mem_backend, "Memory backend file {} is invalid: {}" )?; + if self.default_maxmemory == 0 { + let s = System::new_all(); + self.default_maxmemory = Byte::from_u64(s.total_memory()) + .get_adjusted_unit(Unit::MiB) + .get_value() as u32; + } Ok(()) } diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index af023d7eac..8ae740abd9 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -372,12 +372,6 @@ version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" -[[package]] -name = "byte-unit" -version = "3.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415301c9de11005d4b92193c0eb7ac7adc37e5a49e0ac9bed0a42343512744b8" - [[package]] name = "byte-unit" version = "4.0.19" @@ -388,6 +382,17 @@ dependencies = [ "utf8-width", ] +[[package]] +name = "byte-unit" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc40af92e0f7f964b7ab1ebc81315cce78fc484802d534143321c956f58d7be3" +dependencies = [ + "rust_decimal", + "serde", + "utf8-width", +] + [[package]] name = "byteorder" version = "1.4.3" @@ -601,6 +606,30 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "crossbeam-utils", + "memoffset 0.9.0", + "scopeguard", +] + [[package]] name = "crossbeam-utils" version = "0.8.16" @@ -1657,7 +1686,7 @@ dependencies = [ "anyhow", "base64 0.13.1", "bitmask-enum", - "byte-unit 3.1.4", + "byte-unit 5.0.3", "glob", "lazy_static", "num_cpus", @@ -1669,6 +1698,7 @@ dependencies = [ "serde_json", "slog", "slog-scope", + "sysinfo", "thiserror", "toml 0.5.11", ] @@ -1843,6 +1873,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -2023,6 +2062,15 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -2809,6 +2857,26 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -3024,6 +3092,16 @@ dependencies = [ "ordered-multimap", ] +[[package]] +name = "rust_decimal" +version = "1.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06676aec5ccb8fc1da723cc8c0f9a46549f21ebb8753d3915c6c41db1e7f1dc4" +dependencies = [ + "arrayvec", + "num-traits", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -3559,6 +3637,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sysinfo" +version = "0.29.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666" +dependencies = [ + "cfg-if 1.0.0", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "winapi", +] + [[package]] name = "take_mut" version = "0.2.2" diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs index 42f54fc68c..dcc61afa1e 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs @@ -756,7 +756,7 @@ impl CloudHypervisorInner { todo!() } - pub(crate) fn resize_memory(&self, _req_mem_mb: u32) -> Result<(u32, MemoryConfig)> { + pub(crate) fn resize_memory(&self, _new_mem_mb: u32) -> Result<(u32, MemoryConfig)> { todo!() } } diff --git a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs index 31bca4ef34..9fe70e9d0a 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs @@ -178,9 +178,9 @@ impl Hypervisor for CloudHypervisor { inner.guest_memory_block_size_mb() } - async fn resize_memory(&self, req_mem_mb: u32) -> Result<(u32, MemoryConfig)> { + async fn resize_memory(&self, new_mem_mb: u32) -> Result<(u32, MemoryConfig)> { let inner = self.inner.read().await; - inner.resize_memory(req_mem_mb) + inner.resize_memory(new_mem_mb) } } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs index 55211fdddb..28f04296fd 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -32,7 +32,8 @@ use std::{collections::HashSet, fs::create_dir_all}; const DRAGONBALL_KERNEL: &str = "vmlinux"; const DRAGONBALL_ROOT_FS: &str = "rootfs"; - +const BALLOON_DEVICE_ID: &str = "balloon0"; +const MEM_DEVICE_ID: &str = "memmr0"; #[derive(Debug)] pub struct DragonballInner { /// sandbox id @@ -76,6 +77,9 @@ pub struct DragonballInner { /// the hotplug memory size pub(crate) mem_hotplug_size_mb: u32, + + /// the balloon size + pub(crate) balloon_size: u32, } impl DragonballInner { @@ -85,7 +89,8 @@ impl DragonballInner { CapabilityBits::BlockDeviceSupport | CapabilityBits::BlockDeviceHotplugSupport | CapabilityBits::FsSharingSupport - | CapabilityBits::HybridVsockSupport, + | CapabilityBits::HybridVsockSupport + | CapabilityBits::GuestMemoryProbe, ); DragonballInner { id: "".to_string(), @@ -102,6 +107,7 @@ impl DragonballInner { capabilities, guest_memory_block_size_mb: 0, mem_hotplug_size_mb: 0, + balloon_size: 0, } } @@ -349,33 +355,51 @@ impl DragonballInner { Ok((old_vcpus, new_vcpus)) } - pub(crate) fn resize_memory(&mut self, req_mem_mb: u32) -> Result<(u32, MemoryConfig)> { + pub(crate) fn resize_memory(&mut self, new_mem_mb: u32) -> Result<(u32, MemoryConfig)> { + // check the invalid request memory + if new_mem_mb > self.hypervisor_config().memory_info.default_maxmemory { + warn!( + sl!(), + "memory size unchanged, the request memory size {} is greater than the max memory size {}", + new_mem_mb, self.hypervisor_config().memory_info.default_maxmemory + ); + + return Ok(( + 0, + MemoryConfig { + ..Default::default() + }, + )); + } + let had_mem_mb = self.config.memory_info.default_memory + self.mem_hotplug_size_mb; - match req_mem_mb.cmp(&had_mem_mb) { + match new_mem_mb.cmp(&had_mem_mb) { Ordering::Greater => { // clean virtio-ballon device before hotplug memory, resize to 0 - let balloon_config = BalloonDeviceConfigInfo { - balloon_id: "balloon0".to_owned(), - size_mib: 0, - use_shared_irq: None, - use_generic_irq: None, - f_deflate_on_oom: false, - f_reporting: false, - }; - self.vmm_instance - .insert_balloon_device(balloon_config) - .context("failed to insert balloon device")?; + if self.balloon_size > 0 { + let balloon_config = BalloonDeviceConfigInfo { + balloon_id: BALLOON_DEVICE_ID.to_owned(), + size_mib: 0, + use_shared_irq: None, + use_generic_irq: None, + f_deflate_on_oom: false, + f_reporting: false, + }; + self.vmm_instance + .insert_balloon_device(balloon_config) + .context("failed to insert balloon device")?; + } // update the hotplug size - self.mem_hotplug_size_mb = req_mem_mb - self.config.memory_info.default_memory; + self.mem_hotplug_size_mb = new_mem_mb - self.config.memory_info.default_memory; + self.balloon_size = 0; - // insert a new memory device - let add_mem_mb = req_mem_mb - had_mem_mb; + let add_mem_mb = new_mem_mb - had_mem_mb; self.vmm_instance.insert_mem_device(MemDeviceConfigInfo { - mem_id: format!("mem{}", self.mem_hotplug_size_mb), + mem_id: MEM_DEVICE_ID.to_owned(), size_mib: add_mem_mb as u64, - capacity_mib: add_mem_mb as u64, - multi_region: false, + capacity_mib: self.hypervisor_config().memory_info.default_maxmemory as u64, + multi_region: true, host_numa_node_id: None, guest_numa_node_id: None, use_shared_irq: None, @@ -384,15 +408,16 @@ impl DragonballInner { } Ordering::Less => { // we only use one balloon device here, and resize it to release memory - // the operation we do here is inserting a new balloon0 device or resizing it + // the operation we do here is inserting a new BALLOON_DEVICE_ID device or resizing it let balloon_config = BalloonDeviceConfigInfo { - balloon_id: "balloon0".to_owned(), - size_mib: (had_mem_mb - req_mem_mb) as u64, + balloon_id: BALLOON_DEVICE_ID.to_owned(), + size_mib: (had_mem_mb - new_mem_mb) as u64, use_shared_irq: None, use_generic_irq: None, f_deflate_on_oom: false, f_reporting: false, }; + self.balloon_size = had_mem_mb - new_mem_mb; self.vmm_instance .insert_balloon_device(balloon_config) .context("failed to insert balloon device")?; @@ -407,7 +432,7 @@ impl DragonballInner { }; Ok(( - req_mem_mb, + new_mem_mb, MemoryConfig { ..Default::default() }, @@ -476,6 +501,7 @@ impl Persist for DragonballInner { capabilities: Capabilities::new(), guest_memory_block_size_mb: 0, mem_hotplug_size_mb: 0, + balloon_size: 0, }) } } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs index cd66ed5b0e..bec411b88f 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs @@ -194,9 +194,9 @@ impl Hypervisor for Dragonball { inner.guest_memory_block_size_mb() } - async fn resize_memory(&self, req_mem_mb: u32) -> Result<(u32, MemoryConfig)> { + async fn resize_memory(&self, new_mem_mb: u32) -> Result<(u32, MemoryConfig)> { let mut inner = self.inner.write().await; - inner.resize_memory(req_mem_mb) + inner.resize_memory(new_mem_mb) } } diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index eb236c2476..24987447d2 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -91,7 +91,7 @@ pub trait Hypervisor: std::fmt::Debug + Send + Sync { async fn save_vm(&self) -> Result<()>; async fn resume_vm(&self) -> Result<()>; async fn resize_vcpu(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)>; // returns (old_vcpus, new_vcpus) - async fn resize_memory(&self, req_mem_mb: u32) -> Result<(u32, MemoryConfig)>; + async fn resize_memory(&self, new_mem_mb: u32) -> Result<(u32, MemoryConfig)>; // device manager async fn add_device(&self, device: DeviceType) -> Result; diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index dfc6d34e4c..4cbf2673f9 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -153,7 +153,7 @@ impl QemuInner { todo!() } - pub(crate) fn resize_memory(&self, _req_mem_mb: u32) -> Result<(u32, MemoryConfig)> { + pub(crate) fn resize_memory(&self, _new_mem_mb: u32) -> Result<(u32, MemoryConfig)> { todo!() } } diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs index 4f5fc485c6..3bede9dae2 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs @@ -173,8 +173,8 @@ impl Hypervisor for Qemu { inner.guest_memory_block_size_mb() } - async fn resize_memory(&self, req_mem_mb: u32) -> Result<(u32, MemoryConfig)> { + async fn resize_memory(&self, new_mem_mb: u32) -> Result<(u32, MemoryConfig)> { let inner = self.inner.read().await; - inner.resize_memory(req_mem_mb) + inner.resize_memory(new_mem_mb) } } diff --git a/src/runtime-rs/crates/resource/src/cpu_mem/mem.rs b/src/runtime-rs/crates/resource/src/cpu_mem/mem.rs index 88dbdb12e2..64942c907d 100644 --- a/src/runtime-rs/crates/resource/src/cpu_mem/mem.rs +++ b/src/runtime-rs/crates/resource/src/cpu_mem/mem.rs @@ -9,7 +9,6 @@ use std::sync::Arc; use anyhow::{Context, Ok, Result}; use hypervisor::Hypervisor; -use kata_types::config::TomlConfig; use oci::LinuxResources; use tokio::sync::RwLock; @@ -21,31 +20,17 @@ pub const MIB_TO_BYTES_SHIFT: i32 = 20; #[derive(Default, Debug, Clone)] pub struct MemResource { - /// Current memory - pub(crate) current_mem: Arc>, - /// Default memory pub(crate) orig_toml_default_mem: u32, /// MemResource of each container pub(crate) container_mem_resources: Arc>>, - - /// Use guest swap - pub(crate) use_guest_swap: bool, } impl MemResource { - pub fn new(config: Arc, init_size_manager: InitialSizeManager) -> Result { - let hypervisor_name = config.runtime.hypervisor_name.clone(); - let hypervisor_config = config - .hypervisor - .get(&hypervisor_name) - .context("failed to get hypervisor")?; - + pub fn new(init_size_manager: InitialSizeManager) -> Result { Ok(Self { - current_mem: Arc::new(RwLock::new(hypervisor_config.memory_info.default_memory)), container_mem_resources: Arc::new(RwLock::new(HashMap::new())), - use_guest_swap: hypervisor_config.memory_info.enable_guest_swap, orig_toml_default_mem: init_size_manager.get_orig_toml_default_mem(), }) } @@ -61,36 +46,23 @@ impl MemResource { .await .context("update container memory resources")?; // the unit here is MB - let (mut mem_sb_mb, need_pod_swap, swap_sb_mb) = self - .total_mems(self.use_guest_swap) + let mut mem_sb_mb = self + .total_mems() .await .context("failed to calculate total memory requirement for containers")?; mem_sb_mb += self.orig_toml_default_mem; - if need_pod_swap { - mem_sb_mb += swap_sb_mb; - } info!(sl!(), "calculate mem_sb_mb {}", mem_sb_mb); - let curr_mem = self - .do_update_mem_resource(mem_sb_mb, swap_sb_mb, hypervisor) + let _curr_mem = self + .do_update_mem_resource(mem_sb_mb, hypervisor) .await .context("failed to update_mem_resource")?; - self.update_current_mem(curr_mem).await; Ok(()) } - async fn update_current_mem(&self, new_mem: u32) { - let mut current_mem = self.current_mem.write().await; - *current_mem = new_mem; - } - - async fn total_mems(&self, use_guest_swap: bool) -> Result<(u32, bool, u32)> { - // sb stands for sandbox + async fn total_mems(&self) -> Result { let mut mem_sandbox = 0; - let mut need_pod_swap = false; - let mut swap_sandbox = 0; - let resources = self.container_mem_resources.read().await; for (_, r) in resources.iter() { @@ -100,36 +72,17 @@ impl MemResource { if let Some(memory) = &r.memory { // set current_limit to 0 if memory limit is not set to container - let current_limit = memory.limit.map_or(0, |limit| { + let _current_limit = memory.limit.map_or(0, |limit| { mem_sandbox += limit as u64; info!(sl!(), "memory sb: {}, memory limit: {}", mem_sandbox, limit); limit }); - - if let Some(swappiness) = memory.swappiness { - if swappiness > 0 && use_guest_swap { - if let Some(swap) = memory.swap { - if swap > current_limit { - swap_sandbox = swap.saturating_sub(current_limit); - } - } - // if current_limit is 0, the container will have access to the entire memory available on the host system - // so we add swap for this - else if current_limit == 0 { - need_pod_swap = true; - } else { - swap_sandbox += current_limit; - } - } - } + // TODO support memory guest swap + // https://github.com/kata-containers/kata-containers/issues/7293 } } - Ok(( - (mem_sandbox >> MIB_TO_BYTES_SHIFT) as u32, - need_pod_swap, - (swap_sandbox >> MIB_TO_BYTES_SHIFT) as u32, - )) + Ok((mem_sandbox >> MIB_TO_BYTES_SHIFT) as u32) } // update container_cpu_resources field @@ -156,10 +109,10 @@ impl MemResource { async fn do_update_mem_resource( &self, new_mem: u32, - _swap_sz_mb: u32, hypervisor: &dyn Hypervisor, ) -> Result { info!(sl!(), "requesting vmm to update memory to {:?}", new_mem); + let (new_memory, _mem_config) = hypervisor .resize_memory(new_mem) .await diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index 38adc90ed1..f266b7966a 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -66,7 +66,7 @@ impl ResourceManagerInner { let cgroups_resource = CgroupsResource::new(sid, &toml_config)?; let cpu_resource = CpuResource::new(toml_config.clone())?; - let mem_resource = MemResource::new(toml_config.clone(), init_size_manager)?; + let mem_resource = MemResource::new(init_size_manager)?; Ok(Self { sid: sid.to_string(), toml_config, diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index 5e1e999059..59ad1c08b7 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -222,7 +222,7 @@ impl VirtSandbox { // set memory hotplug probe if guest_details.support_mem_hotplug_probe { self.hypervisor - .set_capabilities(CapabilityBits::GuestMemoryHotplugProbe) + .set_capabilities(CapabilityBits::GuestMemoryProbe) .await; } info!(