mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-07-16 08:26:16 +00:00
Merge pull request #6876 from openanolis/memory_hotlug
runtime-rs: support Memory hotplug
This commit is contained in:
Zhongtao Hu
GitHub
commit
0f80dc636c
@ -14,7 +14,7 @@ edition = "2018"
|
||||
bitmask-enum = "2.1.0"
|
||||
anyhow = "1.0"
|
||||
base64 = "0.13.0"
|
||||
byte-unit = "3.1.4"
|
||||
byte-unit = "5.0.3"
|
||||
glob = "0.3.0"
|
||||
lazy_static = "1.4.0"
|
||||
num_cpus = "1.13.1"
|
||||
@ -26,6 +26,7 @@ serde_json = "1.0.73"
|
||||
thiserror = "1.0"
|
||||
toml = "0.5.8"
|
||||
serde-enum-str = "0.4"
|
||||
sysinfo = "0.29.11"
|
||||
|
||||
oci = { path = "../oci" }
|
||||
safe-path = { path = "../safe-path" }
|
||||
|
||||
@ -705,10 +705,10 @@ impl Annotation {
|
||||
}
|
||||
// Hypervisor Memory related annotations
|
||||
KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY => {
|
||||
match byte_unit::Byte::from_str(value) {
|
||||
match byte_unit::Byte::parse_str(value,true) {
|
||||
Ok(mem_bytes) => {
|
||||
let memory_size = mem_bytes
|
||||
.get_adjusted_unit(byte_unit::ByteUnit::MiB)
|
||||
.get_adjusted_unit(byte_unit::Unit::MiB)
|
||||
.get_value()
|
||||
as u32;
|
||||
info!(sl!(), "get mem {} from annotations: {}", memory_size, value);
|
||||
|
||||
@ -19,6 +19,8 @@ pub enum CapabilityBits {
|
||||
FsSharingSupport,
|
||||
/// hypervisor supports hybrid-vsock
|
||||
HybridVsockSupport,
|
||||
/// hypervisor supports memory hotplug probe interface
|
||||
GuestMemoryProbe,
|
||||
}
|
||||
|
||||
/// Capabilities describe a virtcontainers hypervisor capabilities through a bit mask.
|
||||
@ -47,17 +49,22 @@ impl Capabilities {
|
||||
self.flags = flags;
|
||||
}
|
||||
|
||||
/// is_block_device_supported tells if an hypervisor supports block devices.
|
||||
/// add CapabilityBits
|
||||
pub fn add(&mut self, flags: CapabilityBits) {
|
||||
self.flags |= flags;
|
||||
}
|
||||
|
||||
/// is_block_device_supported tells if the hypervisor supports block devices.
|
||||
pub fn is_block_device_supported(&self) -> bool {
|
||||
self.flags.and(CapabilityBits::BlockDeviceSupport) != 0
|
||||
}
|
||||
|
||||
/// is_block_device_hotplug_supported tells if an hypervisor supports block devices.
|
||||
/// is_block_device_hotplug_supported tells if the hypervisor supports block devices.
|
||||
pub fn is_block_device_hotplug_supported(&self) -> bool {
|
||||
self.flags.and(CapabilityBits::BlockDeviceHotplugSupport) != 0
|
||||
}
|
||||
|
||||
/// is_multi_queue_supported tells if an hypervisor supports device multi queue support.
|
||||
/// is_multi_queue_supported tells if the hypervisor supports device multi queue support.
|
||||
pub fn is_multi_queue_supported(&self) -> bool {
|
||||
self.flags.and(CapabilityBits::MultiQueueSupport) != 0
|
||||
}
|
||||
@ -71,6 +78,11 @@ impl Capabilities {
|
||||
pub fn is_fs_sharing_supported(&self) -> bool {
|
||||
self.flags.and(CapabilityBits::FsSharingSupport) != 0
|
||||
}
|
||||
|
||||
/// is_mem_hotplug_probe_supported tells if the hypervisor supports hotplug probe interface
|
||||
pub fn is_mem_hotplug_probe_supported(&self) -> bool {
|
||||
self.flags.and(CapabilityBits::GuestMemoryProbe) != 0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@ -117,5 +129,9 @@ mod tests {
|
||||
// test set hybrid-vsock support
|
||||
cap.set(CapabilityBits::HybridVsockSupport);
|
||||
assert!(cap.is_hybrid_vsock_supported());
|
||||
// test append capabilities
|
||||
cap.add(CapabilityBits::GuestMemoryProbe);
|
||||
assert!(cap.is_mem_hotplug_probe_supported());
|
||||
assert!(cap.is_fs_sharing_supported());
|
||||
}
|
||||
}
|
||||
|
||||
@ -22,18 +22,18 @@
|
||||
//! part and common part. But the Kata 2.0 has adopted a policy to build a superset for all
|
||||
//! hypervisors, so let's contain it...
|
||||
|
||||
use super::{default, ConfigOps, ConfigPlugin, TomlConfig};
|
||||
use crate::annotations::KATA_ANNO_CFG_HYPERVISOR_PREFIX;
|
||||
use crate::{eother, resolve_path, sl, validate_path};
|
||||
use byte_unit::{Byte, Unit};
|
||||
use lazy_static::lazy_static;
|
||||
use regex::RegexSet;
|
||||
use serde_enum_str::{Deserialize_enum_str, Serialize_enum_str};
|
||||
use std::collections::HashMap;
|
||||
use std::io::{self, Result};
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use lazy_static::lazy_static;
|
||||
use regex::RegexSet;
|
||||
use serde_enum_str::{Deserialize_enum_str, Serialize_enum_str};
|
||||
|
||||
use super::{default, ConfigOps, ConfigPlugin, TomlConfig};
|
||||
use crate::annotations::KATA_ANNO_CFG_HYPERVISOR_PREFIX;
|
||||
use crate::{eother, resolve_path, sl, validate_path};
|
||||
use sysinfo::{System, SystemExt};
|
||||
|
||||
mod dragonball;
|
||||
pub use self::dragonball::{DragonballConfig, HYPERVISOR_NAME_DRAGONBALL};
|
||||
@ -586,6 +586,13 @@ pub struct MemoryInfo {
|
||||
#[serde(default)]
|
||||
pub default_memory: u32,
|
||||
|
||||
/// Default maximum memory in MiB per SB / VM
|
||||
/// unspecified or == 0 --> will be set to the actual amount of physical RAM
|
||||
/// > 0 <= amount of physical RAM --> will be set to the specified number
|
||||
/// > amount of physical RAM --> will be set to the actual amount of physical RAM
|
||||
#[serde(default)]
|
||||
pub default_maxmemory: u32,
|
||||
|
||||
/// Default memory slots per SB/VM.
|
||||
///
|
||||
/// This is will determine the times that memory will be hotadded to sandbox/VM.
|
||||
@ -662,6 +669,12 @@ impl MemoryInfo {
|
||||
self.file_mem_backend,
|
||||
"Memory backend file {} is invalid: {}"
|
||||
)?;
|
||||
if self.default_maxmemory == 0 {
|
||||
let s = System::new_all();
|
||||
self.default_maxmemory = Byte::from_u64(s.total_memory())
|
||||
.get_adjusted_unit(Unit::MiB)
|
||||
.get_value() as u32;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
110
src/runtime-rs/Cargo.lock
generated
110
src/runtime-rs/Cargo.lock
generated
@ -372,12 +372,6 @@ version = "3.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
|
||||
|
||||
[[package]]
|
||||
name = "byte-unit"
|
||||
version = "3.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "415301c9de11005d4b92193c0eb7ac7adc37e5a49e0ac9bed0a42343512744b8"
|
||||
|
||||
[[package]]
|
||||
name = "byte-unit"
|
||||
version = "4.0.19"
|
||||
@ -388,6 +382,17 @@ dependencies = [
|
||||
"utf8-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "byte-unit"
|
||||
version = "5.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bc40af92e0f7f964b7ab1ebc81315cce78fc484802d534143321c956f58d7be3"
|
||||
dependencies = [
|
||||
"rust_decimal",
|
||||
"serde",
|
||||
"utf8-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.4.3"
|
||||
@ -508,6 +513,7 @@ dependencies = [
|
||||
"oci",
|
||||
"persist",
|
||||
"protobuf 3.2.0",
|
||||
"resource",
|
||||
"serde_json",
|
||||
"slog",
|
||||
"slog-scope",
|
||||
@ -600,6 +606,30 @@ dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if 1.0.0",
|
||||
"crossbeam-utils",
|
||||
"memoffset 0.9.0",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.16"
|
||||
@ -1656,7 +1686,7 @@ dependencies = [
|
||||
"anyhow",
|
||||
"base64 0.13.1",
|
||||
"bitmask-enum",
|
||||
"byte-unit 3.1.4",
|
||||
"byte-unit 5.0.3",
|
||||
"glob",
|
||||
"lazy_static",
|
||||
"num_cpus",
|
||||
@ -1668,6 +1698,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"slog",
|
||||
"slog-scope",
|
||||
"sysinfo",
|
||||
"thiserror",
|
||||
"toml 0.5.11",
|
||||
]
|
||||
@ -1761,6 +1792,7 @@ dependencies = [
|
||||
"async-trait",
|
||||
"common",
|
||||
"kata-types",
|
||||
"resource",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
@ -1841,6 +1873,15 @@ dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mime"
|
||||
version = "0.3.17"
|
||||
@ -2021,6 +2062,15 @@ dependencies = [
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ntapi"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nu-ansi-term"
|
||||
version = "0.46.0"
|
||||
@ -2807,6 +2857,26 @@ dependencies = [
|
||||
"rand_core 0.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1"
|
||||
dependencies = [
|
||||
"either",
|
||||
"rayon-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed"
|
||||
dependencies = [
|
||||
"crossbeam-deque",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rdrand"
|
||||
version = "0.4.0"
|
||||
@ -3022,6 +3092,16 @@ dependencies = [
|
||||
"ordered-multimap",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust_decimal"
|
||||
version = "1.33.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06676aec5ccb8fc1da723cc8c0f9a46549f21ebb8753d3915c6c41db1e7f1dc4"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.23"
|
||||
@ -3557,6 +3637,21 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sysinfo"
|
||||
version = "0.29.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
"ntapi",
|
||||
"once_cell",
|
||||
"rayon",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "take_mut"
|
||||
version = "0.2.2"
|
||||
@ -4273,6 +4368,7 @@ dependencies = [
|
||||
"async-trait",
|
||||
"common",
|
||||
"kata-types",
|
||||
"resource",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
|
||||
@ -124,6 +124,8 @@ DEFMEMSZ := 2048
|
||||
# - vm template memory
|
||||
# - hugepage memory
|
||||
DEFMEMSLOTS := 10
|
||||
# Default maximum memory in MiB
|
||||
DEFMAXMEMSZ := 0
|
||||
##VAR DEFBRIDGES=<number> Default number of bridges
|
||||
DEFBRIDGES := 0
|
||||
DEFENABLEANNOTATIONS := [\"kernel_params\"]
|
||||
@ -307,6 +309,7 @@ USER_VARS += DEFMAXVCPUS_ACRN
|
||||
USER_VARS += DEFMAXVCPUS_DB
|
||||
USER_VARS += DEFMEMSZ
|
||||
USER_VARS += DEFMEMSLOTS
|
||||
USER_VARS += DEFMAXMEMSZ
|
||||
USER_VARS += DEFBRIDGES
|
||||
USER_VARS += DEFNETWORKMODEL_DB
|
||||
USER_VARS += DEFNETWORKMODEL_CLH
|
||||
|
||||
@ -101,6 +101,13 @@ default_bridges = @DEFBRIDGES@
|
||||
# If unspecified then it will be set @DEFMEMSZ@ MiB.
|
||||
default_memory = @DEFMEMSZ@
|
||||
|
||||
# Default maximum memory in MiB per SB / VM
|
||||
# unspecified or == 0 --> will be set to the actual amount of physical RAM
|
||||
# > 0 <= amount of physical RAM --> will be set to the specified number
|
||||
# > amount of physical RAM --> will be set to the actual amount of physical RAM
|
||||
default_maxmemory = @DEFMAXMEMSZ@
|
||||
|
||||
|
||||
# Block storage driver to be used for the hypervisor in case the container
|
||||
# rootfs is backed by a block device. DB only supports virtio-blk.
|
||||
block_device_driver = "@DEFBLOCKSTORAGEDRIVER_DB@"
|
||||
|
||||
@ -122,5 +122,6 @@ impl_agent!(
|
||||
get_volume_stats | crate::VolumeStatsRequest | crate::VolumeStatsResponse | None,
|
||||
resize_volume | crate::ResizeVolumeRequest | crate::Empty | None,
|
||||
online_cpu_mem | crate::OnlineCPUMemRequest | crate::Empty | None,
|
||||
get_metrics | crate::Empty | crate::MetricsResponse | None
|
||||
get_metrics | crate::Empty | crate::MetricsResponse | None,
|
||||
get_guest_details | crate::GetGuestDetailsRequest | crate::GuestDetailsResponse | None
|
||||
);
|
||||
|
||||
@ -27,7 +27,7 @@ use crate::{
|
||||
UpdateInterfaceRequest, UpdateRoutesRequest, VersionCheckResponse, VolumeStatsRequest,
|
||||
VolumeStatsResponse, WaitProcessRequest, WriteStreamRequest,
|
||||
},
|
||||
OomEventResponse, WaitProcessResponse, WriteStreamResponse,
|
||||
GetGuestDetailsRequest, OomEventResponse, WaitProcessResponse, WriteStreamResponse,
|
||||
};
|
||||
|
||||
fn trans_vec<F: Sized + Clone, T: From<F>>(from: Vec<F>) -> Vec<T> {
|
||||
@ -723,6 +723,16 @@ impl From<SetGuestDateTimeRequest> for agent::SetGuestDateTimeRequest {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<GetGuestDetailsRequest> for agent::GuestDetailsRequest {
|
||||
fn from(from: GetGuestDetailsRequest) -> Self {
|
||||
Self {
|
||||
mem_block_size: from.mem_block_size,
|
||||
mem_hotplug_probe: from.mem_hotplug_probe,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<agent::AgentDetails> for AgentDetails {
|
||||
fn from(src: agent::AgentDetails) -> Self {
|
||||
Self {
|
||||
|
||||
@ -93,4 +93,5 @@ pub trait Agent: AgentManager + HealthService + Send + Sync {
|
||||
async fn set_ip_tables(&self, req: SetIPTablesRequest) -> Result<SetIPTablesResponse>;
|
||||
async fn get_volume_stats(&self, req: VolumeStatsRequest) -> Result<VolumeStatsResponse>;
|
||||
async fn resize_volume(&self, req: ResizeVolumeRequest) -> Result<Empty>;
|
||||
async fn get_guest_details(&self, req: GetGuestDetailsRequest) -> Result<GuestDetailsResponse>;
|
||||
}
|
||||
|
||||
@ -35,7 +35,7 @@ kata-types = { path = "../../../libs/kata-types" }
|
||||
logging = { path = "../../../libs/logging" }
|
||||
shim-interface = { path = "../../../libs/shim-interface" }
|
||||
|
||||
dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "vhost-net", "dbs-upcall"] }
|
||||
dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "vhost-net", "dbs-upcall","virtio-mem", "virtio-balloon"] }
|
||||
|
||||
ch-config = { path = "ch-config", optional = true }
|
||||
tests_utils = { path = "../../tests/utils" }
|
||||
|
||||
@ -8,8 +8,10 @@ use crate::ch::utils::get_api_socket_path;
|
||||
use crate::ch::utils::get_vsock_path;
|
||||
use crate::kernel_param::KernelParams;
|
||||
use crate::utils::{get_jailer_root, get_sandbox_path};
|
||||
use crate::MemoryConfig;
|
||||
use crate::VM_ROOTFS_DRIVER_BLK;
|
||||
use crate::VM_ROOTFS_DRIVER_PMEM;
|
||||
use crate::{VcpuThreadIds, VmmState};
|
||||
use crate::{VM_ROOTFS_DRIVER_BLK, VM_ROOTFS_DRIVER_PMEM};
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use ch_config::ch_api::{
|
||||
cloud_hypervisor_vm_create, cloud_hypervisor_vm_start, cloud_hypervisor_vmm_ping,
|
||||
@ -741,6 +743,22 @@ impl CloudHypervisorInner {
|
||||
pub(crate) async fn get_hypervisor_metrics(&self) -> Result<String> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
pub(crate) fn set_capabilities(&mut self, _flag: CapabilityBits) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
pub(crate) fn set_guest_memory_block_size(&mut self, _size: u32) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
pub(crate) fn guest_memory_block_size_mb(&self) -> u32 {
|
||||
todo!()
|
||||
}
|
||||
|
||||
pub(crate) fn resize_memory(&self, _new_mem_mb: u32) -> Result<(u32, MemoryConfig)> {
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
|
||||
// Log all output from the CH process until a shutdown signal is received.
|
||||
|
||||
@ -5,10 +5,10 @@
|
||||
|
||||
use super::HypervisorState;
|
||||
use crate::device::DeviceType;
|
||||
use crate::{Hypervisor, VcpuThreadIds};
|
||||
use crate::{Hypervisor, MemoryConfig, VcpuThreadIds};
|
||||
use anyhow::{Context, Result};
|
||||
use async_trait::async_trait;
|
||||
use kata_types::capabilities::Capabilities;
|
||||
use kata_types::capabilities::{Capabilities, CapabilityBits};
|
||||
use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
|
||||
use persist::sandbox_persist::Persist;
|
||||
use std::sync::Arc;
|
||||
@ -162,6 +162,26 @@ impl Hypervisor for CloudHypervisor {
|
||||
let inner = self.inner.read().await;
|
||||
inner.get_hypervisor_metrics().await
|
||||
}
|
||||
|
||||
async fn set_capabilities(&self, flag: CapabilityBits) {
|
||||
let mut inner = self.inner.write().await;
|
||||
inner.set_capabilities(flag)
|
||||
}
|
||||
|
||||
async fn set_guest_memory_block_size(&self, size: u32) {
|
||||
let mut inner = self.inner.write().await;
|
||||
inner.set_guest_memory_block_size(size);
|
||||
}
|
||||
|
||||
async fn guest_memory_block_size(&self) -> u32 {
|
||||
let inner = self.inner.read().await;
|
||||
inner.guest_memory_block_size_mb()
|
||||
}
|
||||
|
||||
async fn resize_memory(&self, new_mem_mb: u32) -> Result<(u32, MemoryConfig)> {
|
||||
let inner = self.inner.read().await;
|
||||
inner.resize_memory(new_mem_mb)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
||||
@ -6,13 +6,14 @@
|
||||
|
||||
use super::vmm_instance::VmmInstance;
|
||||
use crate::{
|
||||
device::DeviceType, hypervisor_persist::HypervisorState, kernel_param::KernelParams, VmmState,
|
||||
DEV_HUGEPAGES, HUGETLBFS, HUGE_SHMEM, HYPERVISOR_DRAGONBALL, SHMEM,
|
||||
device::DeviceType, hypervisor_persist::HypervisorState, kernel_param::KernelParams,
|
||||
MemoryConfig, VmmState, DEV_HUGEPAGES, HUGETLBFS, HUGE_SHMEM, HYPERVISOR_DRAGONBALL, SHMEM,
|
||||
};
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use async_trait::async_trait;
|
||||
use dragonball::{
|
||||
api::v1::{BootSourceConfig, VcpuResizeInfo},
|
||||
device_manager::{balloon_dev_mgr::BalloonDeviceConfigInfo, mem_dev_mgr::MemDeviceConfigInfo},
|
||||
vm::VmConfigInfo,
|
||||
};
|
||||
|
||||
@ -26,11 +27,13 @@ use kata_types::{
|
||||
};
|
||||
use nix::mount::MsFlags;
|
||||
use persist::sandbox_persist::Persist;
|
||||
use std::cmp::Ordering;
|
||||
use std::{collections::HashSet, fs::create_dir_all};
|
||||
|
||||
const DRAGONBALL_KERNEL: &str = "vmlinux";
|
||||
const DRAGONBALL_ROOT_FS: &str = "rootfs";
|
||||
|
||||
const BALLOON_DEVICE_ID: &str = "balloon0";
|
||||
const MEM_DEVICE_ID: &str = "memmr0";
|
||||
#[derive(Debug)]
|
||||
pub struct DragonballInner {
|
||||
/// sandbox id
|
||||
@ -68,6 +71,15 @@ pub struct DragonballInner {
|
||||
|
||||
/// dragonball capabilities
|
||||
pub(crate) capabilities: Capabilities,
|
||||
|
||||
/// the size of memory block of guest OS
|
||||
pub(crate) guest_memory_block_size_mb: u32,
|
||||
|
||||
/// the hotplug memory size
|
||||
pub(crate) mem_hotplug_size_mb: u32,
|
||||
|
||||
/// the balloon size
|
||||
pub(crate) balloon_size: u32,
|
||||
}
|
||||
|
||||
impl DragonballInner {
|
||||
@ -77,7 +89,8 @@ impl DragonballInner {
|
||||
CapabilityBits::BlockDeviceSupport
|
||||
| CapabilityBits::BlockDeviceHotplugSupport
|
||||
| CapabilityBits::FsSharingSupport
|
||||
| CapabilityBits::HybridVsockSupport,
|
||||
| CapabilityBits::HybridVsockSupport
|
||||
| CapabilityBits::GuestMemoryProbe,
|
||||
);
|
||||
DragonballInner {
|
||||
id: "".to_string(),
|
||||
@ -92,6 +105,9 @@ impl DragonballInner {
|
||||
run_dir: "".to_string(),
|
||||
cached_block_devices: Default::default(),
|
||||
capabilities,
|
||||
guest_memory_block_size_mb: 0,
|
||||
mem_hotplug_size_mb: 0,
|
||||
balloon_size: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@ -339,6 +355,90 @@ impl DragonballInner {
|
||||
Ok((old_vcpus, new_vcpus))
|
||||
}
|
||||
|
||||
pub(crate) fn resize_memory(&mut self, new_mem_mb: u32) -> Result<(u32, MemoryConfig)> {
|
||||
// check the invalid request memory
|
||||
if new_mem_mb > self.hypervisor_config().memory_info.default_maxmemory {
|
||||
warn!(
|
||||
sl!(),
|
||||
"memory size unchanged, the request memory size {} is greater than the max memory size {}",
|
||||
new_mem_mb, self.hypervisor_config().memory_info.default_maxmemory
|
||||
);
|
||||
|
||||
return Ok((
|
||||
0,
|
||||
MemoryConfig {
|
||||
..Default::default()
|
||||
},
|
||||
));
|
||||
}
|
||||
|
||||
let had_mem_mb = self.config.memory_info.default_memory + self.mem_hotplug_size_mb;
|
||||
match new_mem_mb.cmp(&had_mem_mb) {
|
||||
Ordering::Greater => {
|
||||
// clean virtio-ballon device before hotplug memory, resize to 0
|
||||
if self.balloon_size > 0 {
|
||||
let balloon_config = BalloonDeviceConfigInfo {
|
||||
balloon_id: BALLOON_DEVICE_ID.to_owned(),
|
||||
size_mib: 0,
|
||||
use_shared_irq: None,
|
||||
use_generic_irq: None,
|
||||
f_deflate_on_oom: false,
|
||||
f_reporting: false,
|
||||
};
|
||||
self.vmm_instance
|
||||
.insert_balloon_device(balloon_config)
|
||||
.context("failed to insert balloon device")?;
|
||||
}
|
||||
|
||||
// update the hotplug size
|
||||
self.mem_hotplug_size_mb = new_mem_mb - self.config.memory_info.default_memory;
|
||||
self.balloon_size = 0;
|
||||
|
||||
let add_mem_mb = new_mem_mb - had_mem_mb;
|
||||
self.vmm_instance.insert_mem_device(MemDeviceConfigInfo {
|
||||
mem_id: MEM_DEVICE_ID.to_owned(),
|
||||
size_mib: add_mem_mb as u64,
|
||||
capacity_mib: self.hypervisor_config().memory_info.default_maxmemory as u64,
|
||||
multi_region: true,
|
||||
host_numa_node_id: None,
|
||||
guest_numa_node_id: None,
|
||||
use_shared_irq: None,
|
||||
use_generic_irq: None,
|
||||
})?;
|
||||
}
|
||||
Ordering::Less => {
|
||||
// we only use one balloon device here, and resize it to release memory
|
||||
// the operation we do here is inserting a new BALLOON_DEVICE_ID device or resizing it
|
||||
let balloon_config = BalloonDeviceConfigInfo {
|
||||
balloon_id: BALLOON_DEVICE_ID.to_owned(),
|
||||
size_mib: (had_mem_mb - new_mem_mb) as u64,
|
||||
use_shared_irq: None,
|
||||
use_generic_irq: None,
|
||||
f_deflate_on_oom: false,
|
||||
f_reporting: false,
|
||||
};
|
||||
self.balloon_size = had_mem_mb - new_mem_mb;
|
||||
self.vmm_instance
|
||||
.insert_balloon_device(balloon_config)
|
||||
.context("failed to insert balloon device")?;
|
||||
}
|
||||
Ordering::Equal => {
|
||||
// Everything is already set up
|
||||
info!(
|
||||
sl!(),
|
||||
"memory size unchanged, no need to do memory resizing"
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
Ok((
|
||||
new_mem_mb,
|
||||
MemoryConfig {
|
||||
..Default::default()
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub fn set_hypervisor_config(&mut self, config: HypervisorConfig) {
|
||||
self.config = config;
|
||||
}
|
||||
@ -346,6 +446,18 @@ impl DragonballInner {
|
||||
pub fn hypervisor_config(&self) -> HypervisorConfig {
|
||||
self.config.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn set_capabilities(&mut self, flag: CapabilityBits) {
|
||||
self.capabilities.add(flag);
|
||||
}
|
||||
|
||||
pub(crate) fn set_guest_memory_block_size(&mut self, size: u32) {
|
||||
self.guest_memory_block_size_mb = size;
|
||||
}
|
||||
|
||||
pub(crate) fn guest_memory_block_size_mb(&self) -> u32 {
|
||||
self.guest_memory_block_size_mb
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@ -387,6 +499,9 @@ impl Persist for DragonballInner {
|
||||
pending_devices: vec![],
|
||||
cached_block_devices: hypervisor_state.cached_block_devices,
|
||||
capabilities: Capabilities::new(),
|
||||
guest_memory_block_size_mb: 0,
|
||||
mem_hotplug_size_mb: 0,
|
||||
balloon_size: 0,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@ -21,12 +21,12 @@ use dragonball::api::v1::{
|
||||
Backend as DragonballBackend, NetworkInterfaceConfig as DragonballNetworkConfig,
|
||||
VirtioConfig as DragonballVirtioConfig,
|
||||
};
|
||||
use kata_types::capabilities::Capabilities;
|
||||
use kata_types::capabilities::{Capabilities, CapabilityBits};
|
||||
use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::instrument;
|
||||
|
||||
use crate::{DeviceType, Hypervisor, NetworkConfig, VcpuThreadIds};
|
||||
use crate::{DeviceType, Hypervisor, MemoryConfig, NetworkConfig, VcpuThreadIds};
|
||||
|
||||
pub struct Dragonball {
|
||||
inner: Arc<RwLock<DragonballInner>>,
|
||||
@ -178,6 +178,26 @@ impl Hypervisor for Dragonball {
|
||||
let inner = self.inner.read().await;
|
||||
inner.get_hypervisor_metrics().await
|
||||
}
|
||||
|
||||
async fn set_capabilities(&self, flag: CapabilityBits) {
|
||||
let mut inner = self.inner.write().await;
|
||||
inner.set_capabilities(flag)
|
||||
}
|
||||
|
||||
async fn set_guest_memory_block_size(&self, size: u32) {
|
||||
let mut inner = self.inner.write().await;
|
||||
inner.set_guest_memory_block_size(size);
|
||||
}
|
||||
|
||||
async fn guest_memory_block_size(&self) -> u32 {
|
||||
let inner = self.inner.read().await;
|
||||
inner.guest_memory_block_size_mb()
|
||||
}
|
||||
|
||||
async fn resize_memory(&self, new_mem_mb: u32) -> Result<(u32, MemoryConfig)> {
|
||||
let mut inner = self.inner.write().await;
|
||||
inner.resize_memory(new_mem_mb)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
||||
@ -19,6 +19,7 @@ use dragonball::{
|
||||
InstanceInfo, InstanceState, NetworkInterfaceConfig, VcpuResizeInfo, VmmAction,
|
||||
VmmActionError, VmmData, VmmRequest, VmmResponse, VmmService, VsockDeviceConfigInfo,
|
||||
},
|
||||
device_manager::{balloon_dev_mgr::BalloonDeviceConfigInfo, mem_dev_mgr::MemDeviceConfigInfo},
|
||||
vm::VmConfigInfo,
|
||||
Vmm,
|
||||
};
|
||||
@ -255,6 +256,18 @@ impl VmmInstance {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn insert_mem_device(&self, cfg: MemDeviceConfigInfo) -> Result<()> {
|
||||
self.handle_request(Request::Sync(VmmAction::InsertMemDevice(cfg.clone())))
|
||||
.with_context(|| format!("Failed to insert memory device : {:?}", cfg))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn insert_balloon_device(&self, cfg: BalloonDeviceConfigInfo) -> Result<()> {
|
||||
self.handle_request(Request::Sync(VmmAction::InsertBalloonDevice(cfg.clone())))
|
||||
.with_context(|| format!("Failed to insert balloon device: {:?}", cfg))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn pause(&self) -> Result<()> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
@ -26,7 +26,7 @@ pub mod ch;
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use hypervisor_persist::HypervisorState;
|
||||
use kata_types::capabilities::Capabilities;
|
||||
use kata_types::capabilities::{Capabilities, CapabilityBits};
|
||||
use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
|
||||
|
||||
pub use kata_types::config::hypervisor::HYPERVISOR_NAME_CH;
|
||||
@ -73,6 +73,14 @@ pub struct VcpuThreadIds {
|
||||
pub vcpus: HashMap<u32, u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct MemoryConfig {
|
||||
pub slot: u32,
|
||||
pub size_mb: u32,
|
||||
pub addr: u64,
|
||||
pub probe: bool,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait Hypervisor: std::fmt::Debug + Send + Sync {
|
||||
// vm manager
|
||||
@ -83,6 +91,7 @@ pub trait Hypervisor: std::fmt::Debug + Send + Sync {
|
||||
async fn save_vm(&self) -> Result<()>;
|
||||
async fn resume_vm(&self) -> Result<()>;
|
||||
async fn resize_vcpu(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)>; // returns (old_vcpus, new_vcpus)
|
||||
async fn resize_memory(&self, new_mem_mb: u32) -> Result<(u32, MemoryConfig)>;
|
||||
|
||||
// device manager
|
||||
async fn add_device(&self, device: DeviceType) -> Result<DeviceType>;
|
||||
@ -103,4 +112,7 @@ pub trait Hypervisor: std::fmt::Debug + Send + Sync {
|
||||
async fn save_state(&self) -> Result<HypervisorState>;
|
||||
async fn capabilities(&self) -> Result<Capabilities>;
|
||||
async fn get_hypervisor_metrics(&self) -> Result<String>;
|
||||
async fn set_capabilities(&self, flag: CapabilityBits);
|
||||
async fn set_guest_memory_block_size(&self, size: u32);
|
||||
async fn guest_memory_block_size(&self) -> u32;
|
||||
}
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
use crate::{HypervisorConfig, VcpuThreadIds};
|
||||
use crate::{HypervisorConfig, MemoryConfig, VcpuThreadIds};
|
||||
use kata_types::capabilities::{Capabilities, CapabilityBits};
|
||||
|
||||
const VSOCK_SCHEME: &str = "vsock";
|
||||
@ -140,6 +140,22 @@ impl QemuInner {
|
||||
pub(crate) async fn get_hypervisor_metrics(&self) -> Result<String> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
pub(crate) fn set_capabilities(&mut self, _flag: CapabilityBits) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
pub(crate) fn set_guest_memory_block_size(&mut self, _size: u32) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
pub(crate) fn guest_memory_block_size_mb(&self) -> u32 {
|
||||
todo!()
|
||||
}
|
||||
|
||||
pub(crate) fn resize_memory(&self, _new_mem_mb: u32) -> Result<(u32, MemoryConfig)> {
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
|
||||
use crate::device::DeviceType;
|
||||
|
||||
@ -7,10 +7,10 @@ mod inner;
|
||||
|
||||
use crate::device::DeviceType;
|
||||
use crate::hypervisor_persist::HypervisorState;
|
||||
use crate::Hypervisor;
|
||||
use crate::{Hypervisor, MemoryConfig};
|
||||
use crate::{HypervisorConfig, VcpuThreadIds};
|
||||
use inner::QemuInner;
|
||||
use kata_types::capabilities::Capabilities;
|
||||
use kata_types::capabilities::{Capabilities, CapabilityBits};
|
||||
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
@ -157,4 +157,24 @@ impl Hypervisor for Qemu {
|
||||
let inner = self.inner.read().await;
|
||||
inner.get_hypervisor_metrics().await
|
||||
}
|
||||
|
||||
async fn set_capabilities(&self, flag: CapabilityBits) {
|
||||
let mut inner = self.inner.write().await;
|
||||
inner.set_capabilities(flag)
|
||||
}
|
||||
|
||||
async fn set_guest_memory_block_size(&self, size: u32) {
|
||||
let mut inner = self.inner.write().await;
|
||||
inner.set_guest_memory_block_size(size);
|
||||
}
|
||||
|
||||
async fn guest_memory_block_size(&self) -> u32 {
|
||||
let inner = self.inner.read().await;
|
||||
inner.guest_memory_block_size_mb()
|
||||
}
|
||||
|
||||
async fn resize_memory(&self, new_mem_mb: u32) -> Result<(u32, MemoryConfig)> {
|
||||
let inner = self.inner.read().await;
|
||||
inner.resize_memory(new_mem_mb)
|
||||
}
|
||||
}
|
||||
|
||||
@ -11,7 +11,6 @@ use std::{
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use agent::{Agent, OnlineCPUMemRequest};
|
||||
use anyhow::{Context, Ok, Result};
|
||||
use hypervisor::Hypervisor;
|
||||
use kata_types::{config::TomlConfig, cpu::LinuxContainerCpuResources};
|
||||
@ -52,7 +51,6 @@ impl CpuResource {
|
||||
linux_cpus: Option<&LinuxCpu>,
|
||||
op: ResourceUpdateOp,
|
||||
hypervisor: &dyn Hypervisor,
|
||||
agent: &dyn Agent,
|
||||
) -> Result<()> {
|
||||
self.update_container_cpu_resources(cid, linux_cpus, op)
|
||||
.await
|
||||
@ -67,13 +65,13 @@ impl CpuResource {
|
||||
}
|
||||
|
||||
let curr_vcpus = self
|
||||
.do_update_cpu_resources(vcpu_required, op, hypervisor, agent)
|
||||
.do_update_cpu_resources(vcpu_required, op, hypervisor)
|
||||
.await?;
|
||||
self.update_current_vcpu(curr_vcpus).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn current_vcpu(&self) -> u32 {
|
||||
pub(crate) async fn current_vcpu(&self) -> u32 {
|
||||
let current_vcpu = self.current_vcpu.read().await;
|
||||
*current_vcpu
|
||||
}
|
||||
@ -148,7 +146,6 @@ impl CpuResource {
|
||||
new_vcpus: u32,
|
||||
op: ResourceUpdateOp,
|
||||
hypervisor: &dyn Hypervisor,
|
||||
agent: &dyn Agent,
|
||||
) -> Result<u32> {
|
||||
let old_vcpus = self.current_vcpu().await;
|
||||
|
||||
@ -164,25 +161,11 @@ impl CpuResource {
|
||||
// the number of vcpus would not be lower than the default size
|
||||
let new_vcpus = cmp::max(new_vcpus, self.default_vcpu);
|
||||
|
||||
let (old, new) = hypervisor
|
||||
let (_, new) = hypervisor
|
||||
.resize_vcpu(old_vcpus, new_vcpus)
|
||||
.await
|
||||
.context("resize vcpus")?;
|
||||
|
||||
if old < new {
|
||||
let add = new - old;
|
||||
info!(sl!(), "request to onlineCpuMem with {:?} cpus", add);
|
||||
|
||||
agent
|
||||
.online_cpu_mem(OnlineCPUMemRequest {
|
||||
wait: false,
|
||||
nb_cpus: new,
|
||||
cpu_only: true,
|
||||
})
|
||||
.await
|
||||
.context("online vcpus")?;
|
||||
}
|
||||
|
||||
Ok(new)
|
||||
}
|
||||
}
|
||||
|
||||
@ -19,6 +19,7 @@ use kata_types::{
|
||||
struct InitialSize {
|
||||
vcpu: u32,
|
||||
mem_mb: u32,
|
||||
orig_toml_default_mem: u32,
|
||||
}
|
||||
|
||||
// generate initial resource(vcpu and memory in MiB) from spec's information
|
||||
@ -66,7 +67,11 @@ impl TryFrom<&oci::Spec> for InitialSize {
|
||||
sl!(),
|
||||
"(from PodSandbox's annotation / SingleContainer's spec) initial size: vcpu={}, mem_mb={}", vcpu, mem_mb
|
||||
);
|
||||
Ok(Self { vcpu, mem_mb })
|
||||
Ok(Self {
|
||||
vcpu,
|
||||
mem_mb,
|
||||
orig_toml_default_mem: 0,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@ -93,7 +98,7 @@ impl InitialSizeManager {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn setup_config(&self, config: &mut TomlConfig) -> Result<()> {
|
||||
pub fn setup_config(&mut self, config: &mut TomlConfig) -> Result<()> {
|
||||
// update this data to the hypervisor config for later use by hypervisor
|
||||
let hypervisor_name = &config.runtime.hypervisor_name;
|
||||
let hv = config
|
||||
@ -104,6 +109,7 @@ impl InitialSizeManager {
|
||||
if self.resource.vcpu > 0 {
|
||||
hv.cpu_info.default_vcpus = self.resource.vcpu as i32
|
||||
}
|
||||
self.resource.orig_toml_default_mem = hv.memory_info.default_memory;
|
||||
if self.resource.mem_mb > 0 {
|
||||
// since the memory overhead introduced by kata-agent and system components
|
||||
// will really affect the amount of memory the user can use, so we choose to
|
||||
@ -114,6 +120,10 @@ impl InitialSizeManager {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_orig_toml_default_mem(&self) -> u32 {
|
||||
self.resource.orig_toml_default_mem
|
||||
}
|
||||
}
|
||||
|
||||
fn get_nr_vcpu(resource: &LinuxContainerCpuResources) -> u32 {
|
||||
@ -173,7 +183,11 @@ mod tests {
|
||||
quota: None,
|
||||
memory: None,
|
||||
},
|
||||
result: InitialSize { vcpu: 0, mem_mb: 0 },
|
||||
result: InitialSize {
|
||||
vcpu: 0,
|
||||
mem_mb: 0,
|
||||
orig_toml_default_mem: 0,
|
||||
},
|
||||
},
|
||||
TestData {
|
||||
desc: "normal resource limit",
|
||||
@ -186,6 +200,7 @@ mod tests {
|
||||
result: InitialSize {
|
||||
vcpu: 3,
|
||||
mem_mb: 512,
|
||||
orig_toml_default_mem: 0,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
123
src/runtime-rs/crates/resource/src/cpu_mem/mem.rs
Normal file
123
src/runtime-rs/crates/resource/src/cpu_mem/mem.rs
Normal file
@ -0,0 +1,123 @@
|
||||
// Copyright (c) 2019-2023 Alibaba Cloud
|
||||
// Copyright (c) 2019-2023 Ant Group
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context, Ok, Result};
|
||||
use hypervisor::Hypervisor;
|
||||
use oci::LinuxResources;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use crate::cpu_mem::initial_size::InitialSizeManager;
|
||||
use crate::ResourceUpdateOp;
|
||||
|
||||
// MIB_TO_BYTES_SHIFT the number to shift needed to convert MiB to Bytes
|
||||
pub const MIB_TO_BYTES_SHIFT: i32 = 20;
|
||||
|
||||
#[derive(Default, Debug, Clone)]
|
||||
pub struct MemResource {
|
||||
/// Default memory
|
||||
pub(crate) orig_toml_default_mem: u32,
|
||||
|
||||
/// MemResource of each container
|
||||
pub(crate) container_mem_resources: Arc<RwLock<HashMap<String, LinuxResources>>>,
|
||||
}
|
||||
|
||||
impl MemResource {
|
||||
pub fn new(init_size_manager: InitialSizeManager) -> Result<Self> {
|
||||
Ok(Self {
|
||||
container_mem_resources: Arc::new(RwLock::new(HashMap::new())),
|
||||
orig_toml_default_mem: init_size_manager.get_orig_toml_default_mem(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn update_mem_resources(
|
||||
&self,
|
||||
cid: &str,
|
||||
linux_resources: Option<&LinuxResources>,
|
||||
op: ResourceUpdateOp,
|
||||
hypervisor: &dyn Hypervisor,
|
||||
) -> Result<()> {
|
||||
self.update_container_mem_resources(cid, linux_resources, op)
|
||||
.await
|
||||
.context("update container memory resources")?;
|
||||
// the unit here is MB
|
||||
let mut mem_sb_mb = self
|
||||
.total_mems()
|
||||
.await
|
||||
.context("failed to calculate total memory requirement for containers")?;
|
||||
mem_sb_mb += self.orig_toml_default_mem;
|
||||
info!(sl!(), "calculate mem_sb_mb {}", mem_sb_mb);
|
||||
|
||||
let _curr_mem = self
|
||||
.do_update_mem_resource(mem_sb_mb, hypervisor)
|
||||
.await
|
||||
.context("failed to update_mem_resource")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn total_mems(&self) -> Result<u32> {
|
||||
let mut mem_sandbox = 0;
|
||||
let resources = self.container_mem_resources.read().await;
|
||||
|
||||
for (_, r) in resources.iter() {
|
||||
for l in &r.hugepage_limits {
|
||||
mem_sandbox += l.limit;
|
||||
}
|
||||
|
||||
if let Some(memory) = &r.memory {
|
||||
// set current_limit to 0 if memory limit is not set to container
|
||||
let _current_limit = memory.limit.map_or(0, |limit| {
|
||||
mem_sandbox += limit as u64;
|
||||
info!(sl!(), "memory sb: {}, memory limit: {}", mem_sandbox, limit);
|
||||
limit
|
||||
});
|
||||
// TODO support memory guest swap
|
||||
// https://github.com/kata-containers/kata-containers/issues/7293
|
||||
}
|
||||
}
|
||||
|
||||
Ok((mem_sandbox >> MIB_TO_BYTES_SHIFT) as u32)
|
||||
}
|
||||
|
||||
// update container_cpu_resources field
|
||||
async fn update_container_mem_resources(
|
||||
&self,
|
||||
cid: &str,
|
||||
linux_resources: Option<&LinuxResources>,
|
||||
op: ResourceUpdateOp,
|
||||
) -> Result<()> {
|
||||
if let Some(r) = linux_resources {
|
||||
let mut resources = self.container_mem_resources.write().await;
|
||||
match op {
|
||||
ResourceUpdateOp::Add | ResourceUpdateOp::Update => {
|
||||
resources.insert(cid.to_owned(), r.clone());
|
||||
}
|
||||
ResourceUpdateOp::Del => {
|
||||
resources.remove(cid);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn do_update_mem_resource(
|
||||
&self,
|
||||
new_mem: u32,
|
||||
hypervisor: &dyn Hypervisor,
|
||||
) -> Result<u32> {
|
||||
info!(sl!(), "requesting vmm to update memory to {:?}", new_mem);
|
||||
|
||||
let (new_memory, _mem_config) = hypervisor
|
||||
.resize_memory(new_mem)
|
||||
.await
|
||||
.context("resize memory")?;
|
||||
|
||||
Ok(new_memory)
|
||||
}
|
||||
}
|
||||
@ -6,3 +6,4 @@
|
||||
|
||||
pub mod cpu;
|
||||
pub mod initial_size;
|
||||
pub mod mem;
|
||||
|
||||
@ -19,6 +19,7 @@ use persist::sandbox_persist::Persist;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::instrument;
|
||||
|
||||
use crate::cpu_mem::initial_size::InitialSizeManager;
|
||||
use crate::network::NetworkConfig;
|
||||
use crate::resource_persist::ResourceState;
|
||||
use crate::ResourceUpdateOp;
|
||||
@ -47,13 +48,15 @@ impl ResourceManager {
|
||||
agent: Arc<dyn Agent>,
|
||||
hypervisor: Arc<dyn Hypervisor>,
|
||||
toml_config: Arc<TomlConfig>,
|
||||
init_size_manager: InitialSizeManager,
|
||||
) -> Result<Self> {
|
||||
// Regist resource logger for later use.
|
||||
logging::register_subsystem_logger("runtimes", "resource");
|
||||
|
||||
Ok(Self {
|
||||
inner: Arc::new(RwLock::new(
|
||||
ResourceManagerInner::new(sid, agent, hypervisor, toml_config).await?,
|
||||
ResourceManagerInner::new(sid, agent, hypervisor, toml_config, init_size_manager)
|
||||
.await?,
|
||||
)),
|
||||
})
|
||||
}
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
|
||||
use std::{sync::Arc, thread};
|
||||
|
||||
use agent::{types::Device, Agent, Storage};
|
||||
use agent::{types::Device, Agent, OnlineCPUMemRequest, Storage};
|
||||
use anyhow::{anyhow, Context, Ok, Result};
|
||||
use async_trait::async_trait;
|
||||
use hypervisor::{
|
||||
@ -25,7 +25,7 @@ use tokio::{runtime, sync::RwLock};
|
||||
|
||||
use crate::{
|
||||
cgroups::{CgroupArgs, CgroupsResource},
|
||||
cpu_mem::cpu::CpuResource,
|
||||
cpu_mem::{cpu::CpuResource, initial_size::InitialSizeManager, mem::MemResource},
|
||||
manager::ManagerArgs,
|
||||
network::{self, Network, NetworkConfig},
|
||||
resource_persist::ResourceState,
|
||||
@ -48,6 +48,7 @@ pub(crate) struct ResourceManagerInner {
|
||||
pub volume_resource: VolumeResource,
|
||||
pub cgroups_resource: CgroupsResource,
|
||||
pub cpu_resource: CpuResource,
|
||||
pub mem_resource: MemResource,
|
||||
}
|
||||
|
||||
impl ResourceManagerInner {
|
||||
@ -56,6 +57,7 @@ impl ResourceManagerInner {
|
||||
agent: Arc<dyn Agent>,
|
||||
hypervisor: Arc<dyn Hypervisor>,
|
||||
toml_config: Arc<TomlConfig>,
|
||||
init_size_manager: InitialSizeManager,
|
||||
) -> Result<Self> {
|
||||
// create device manager
|
||||
let dev_manager = DeviceManager::new(hypervisor.clone())
|
||||
@ -64,6 +66,7 @@ impl ResourceManagerInner {
|
||||
|
||||
let cgroups_resource = CgroupsResource::new(sid, &toml_config)?;
|
||||
let cpu_resource = CpuResource::new(toml_config.clone())?;
|
||||
let mem_resource = MemResource::new(init_size_manager)?;
|
||||
Ok(Self {
|
||||
sid: sid.to_string(),
|
||||
toml_config,
|
||||
@ -76,6 +79,7 @@ impl ResourceManagerInner {
|
||||
volume_resource: VolumeResource::new(),
|
||||
cgroups_resource,
|
||||
cpu_resource,
|
||||
mem_resource,
|
||||
})
|
||||
}
|
||||
|
||||
@ -427,15 +431,23 @@ impl ResourceManagerInner {
|
||||
|
||||
// if static_sandbox_resource_mgmt, we will not have to update sandbox's cpu or mem resource
|
||||
if !self.toml_config.runtime.static_sandbox_resource_mgmt {
|
||||
// update cpu
|
||||
self.cpu_resource
|
||||
.update_cpu_resources(
|
||||
cid,
|
||||
linux_cpus,
|
||||
op,
|
||||
self.hypervisor.as_ref(),
|
||||
self.agent.as_ref(),
|
||||
)
|
||||
.update_cpu_resources(cid, linux_cpus, op, self.hypervisor.as_ref())
|
||||
.await?;
|
||||
// update memory
|
||||
self.mem_resource
|
||||
.update_mem_resources(cid, linux_resources, op, self.hypervisor.as_ref())
|
||||
.await?;
|
||||
|
||||
self.agent
|
||||
.online_cpu_mem(OnlineCPUMemRequest {
|
||||
wait: false,
|
||||
nb_cpus: self.cpu_resource.current_vcpu().await,
|
||||
cpu_only: false,
|
||||
})
|
||||
.await
|
||||
.context("online vcpus")?;
|
||||
}
|
||||
|
||||
// we should firstly update the vcpus and mems, and then update the host cgroups
|
||||
@ -516,6 +528,7 @@ impl Persist for ResourceManagerInner {
|
||||
.await?,
|
||||
toml_config: Arc::new(TomlConfig::default()),
|
||||
cpu_resource: CpuResource::default(),
|
||||
mem_resource: MemResource::default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@ -26,4 +26,5 @@ agent = { path = "../../agent" }
|
||||
kata-sys-util = { path = "../../../../libs/kata-sys-util" }
|
||||
kata-types = { path = "../../../../libs/kata-types" }
|
||||
oci = { path = "../../../../libs/oci" }
|
||||
resource = { path = "../../resource" }
|
||||
|
||||
|
||||
@ -6,13 +6,13 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::{message::Message, ContainerManager, Sandbox};
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use kata_types::config::TomlConfig;
|
||||
use resource::cpu_mem::initial_size::InitialSizeManager;
|
||||
use tokio::sync::mpsc::Sender;
|
||||
|
||||
use crate::{message::Message, ContainerManager, Sandbox};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct RuntimeInstance {
|
||||
pub sandbox: Arc<dyn Sandbox>,
|
||||
@ -38,6 +38,7 @@ pub trait RuntimeHandler: Send + Sync {
|
||||
sid: &str,
|
||||
msg_sender: Sender<Message>,
|
||||
config: Arc<TomlConfig>,
|
||||
init_size_manager: InitialSizeManager,
|
||||
) -> Result<RuntimeInstance>;
|
||||
|
||||
fn cleanup(&self, id: &str) -> Result<()>;
|
||||
|
||||
@ -11,3 +11,4 @@ tokio = { version = "1.28.1" }
|
||||
|
||||
common = { path = "../common" }
|
||||
kata-types = { path = "../../../../libs/kata-types" }
|
||||
resource = { path = "../../resource" }
|
||||
@ -9,6 +9,7 @@ use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use common::{message::Message, RuntimeHandler, RuntimeInstance};
|
||||
use kata_types::config::TomlConfig;
|
||||
use resource::cpu_mem::initial_size::InitialSizeManager;
|
||||
use tokio::sync::mpsc::Sender;
|
||||
|
||||
pub struct LinuxContainer {}
|
||||
@ -32,6 +33,7 @@ impl RuntimeHandler for LinuxContainer {
|
||||
_sid: &str,
|
||||
_msg_sender: Sender<Message>,
|
||||
_config: Arc<TomlConfig>,
|
||||
_init_size_manager: InitialSizeManager,
|
||||
) -> Result<RuntimeInstance> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
@ -91,6 +91,7 @@ impl RuntimeHandlerManagerInner {
|
||||
network_env: SandboxNetworkEnv,
|
||||
dns: Vec<String>,
|
||||
config: Arc<TomlConfig>,
|
||||
init_size_manager: InitialSizeManager,
|
||||
) -> Result<()> {
|
||||
info!(sl!(), "new runtime handler {}", &config.runtime.name);
|
||||
let runtime_handler = match config.runtime.name.as_str() {
|
||||
@ -105,7 +106,12 @@ impl RuntimeHandlerManagerInner {
|
||||
_ => return Err(anyhow!("Unsupported runtime: {}", &config.runtime.name)),
|
||||
};
|
||||
let runtime_instance = runtime_handler
|
||||
.new_instance(&self.id, self.msg_sender.clone(), config.clone())
|
||||
.new_instance(
|
||||
&self.id,
|
||||
self.msg_sender.clone(),
|
||||
config.clone(),
|
||||
init_size_manager,
|
||||
)
|
||||
.await
|
||||
.context("new runtime instance")?;
|
||||
|
||||
@ -160,7 +166,21 @@ impl RuntimeHandlerManagerInner {
|
||||
}
|
||||
}
|
||||
|
||||
let config = load_config(spec, options).context("load config")?;
|
||||
let mut config = load_config(spec, options).context("load config")?;
|
||||
|
||||
// Sandbox sizing information *may* be provided in two scenarios:
|
||||
// 1. The upper layer runtime (ie, containerd or crio) provide sandbox sizing information as an annotation
|
||||
// in the 'sandbox container's' spec. This would typically be a scenario where as part of a create sandbox
|
||||
// request the upper layer runtime receives this information as part of a pod, and makes it available to us
|
||||
// for sizing purposes.
|
||||
// 2. If this is not a sandbox infrastructure container, but instead a standalone single container (analogous to "docker run..."),
|
||||
// then the container spec itself will contain appropriate sizing information for the entire sandbox (since it is
|
||||
// a single container.
|
||||
let mut initial_size_manager =
|
||||
InitialSizeManager::new(spec).context("failed to construct static resource manager")?;
|
||||
initial_size_manager
|
||||
.setup_config(&mut config)
|
||||
.context("failed to setup static resource mgmt config")?;
|
||||
|
||||
update_component_log_level(&config);
|
||||
|
||||
@ -202,10 +222,16 @@ impl RuntimeHandlerManagerInner {
|
||||
netns,
|
||||
network_created,
|
||||
};
|
||||
|
||||
self.init_runtime_handler(spec, state, network_env, dns, Arc::new(config))
|
||||
.await
|
||||
.context("init runtime handler")?;
|
||||
self.init_runtime_handler(
|
||||
spec,
|
||||
state,
|
||||
network_env,
|
||||
dns,
|
||||
Arc::new(config),
|
||||
initial_size_manager,
|
||||
)
|
||||
.await
|
||||
.context("init runtime handler")?;
|
||||
|
||||
// the sandbox creation can reach here only once and the sandbox is created
|
||||
// so we can safely create the shim management socket right now
|
||||
@ -507,7 +533,7 @@ fn load_config(spec: &oci::Spec, option: &Option<Vec<u8>>) -> Result<TomlConfig>
|
||||
// 2. If this is not a sandbox infrastructure container, but instead a standalone single container (analogous to "docker run..."),
|
||||
// then the container spec itself will contain appropriate sizing information for the entire sandbox (since it is
|
||||
// a single container.
|
||||
let initial_size_manager =
|
||||
let mut initial_size_manager =
|
||||
InitialSizeManager::new(spec).context("failed to construct static resource manager")?;
|
||||
initial_size_manager
|
||||
.setup_config(&mut toml_config)
|
||||
|
||||
@ -31,6 +31,7 @@ use hypervisor::ch::CloudHypervisor;
|
||||
#[cfg(feature = "cloud-hypervisor")]
|
||||
use kata_types::config::{hypervisor::HYPERVISOR_NAME_CH, CloudHypervisorConfig};
|
||||
|
||||
use resource::cpu_mem::initial_size::InitialSizeManager;
|
||||
use resource::ResourceManager;
|
||||
use sandbox::VIRTCONTAINER;
|
||||
use tokio::sync::mpsc::Sender;
|
||||
@ -77,13 +78,22 @@ impl RuntimeHandler for VirtContainer {
|
||||
sid: &str,
|
||||
msg_sender: Sender<Message>,
|
||||
config: Arc<TomlConfig>,
|
||||
init_size_manager: InitialSizeManager,
|
||||
) -> Result<RuntimeInstance> {
|
||||
let hypervisor = new_hypervisor(&config).await.context("new hypervisor")?;
|
||||
|
||||
// get uds from hypervisor and get config from toml_config
|
||||
let agent = new_agent(&config).context("new agent")?;
|
||||
let resource_manager =
|
||||
Arc::new(ResourceManager::new(sid, agent.clone(), hypervisor.clone(), config).await?);
|
||||
let resource_manager = Arc::new(
|
||||
ResourceManager::new(
|
||||
sid,
|
||||
agent.clone(),
|
||||
hypervisor.clone(),
|
||||
config,
|
||||
init_size_manager,
|
||||
)
|
||||
.await?,
|
||||
);
|
||||
let pid = std::process::id();
|
||||
|
||||
let sandbox = sandbox::VirtSandbox::new(
|
||||
|
||||
@ -8,7 +8,9 @@ use std::sync::Arc;
|
||||
|
||||
use agent::kata::KataAgent;
|
||||
use agent::types::KernelModule;
|
||||
use agent::{self, Agent, GetIPTablesRequest, SetIPTablesRequest, VolumeStatsRequest};
|
||||
use agent::{
|
||||
self, Agent, GetGuestDetailsRequest, GetIPTablesRequest, SetIPTablesRequest, VolumeStatsRequest,
|
||||
};
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use async_trait::async_trait;
|
||||
use common::message::{Action, Message};
|
||||
@ -18,6 +20,7 @@ use hypervisor::VsockConfig;
|
||||
use hypervisor::{dragonball::Dragonball, BlockConfig, Hypervisor, HYPERVISOR_DRAGONBALL};
|
||||
use hypervisor::{utils::get_hvsock_path, HybridVsockConfig, DEFAULT_GUEST_VSOCK_CID};
|
||||
use kata_sys_util::hooks::HookStates;
|
||||
use kata_types::capabilities::CapabilityBits;
|
||||
use kata_types::config::TomlConfig;
|
||||
use persist::{self, sandbox_persist::Persist};
|
||||
use resource::manager::ManagerArgs;
|
||||
@ -196,7 +199,41 @@ impl VirtSandbox {
|
||||
// * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createruntime-hooks
|
||||
let mut create_runtime_hook_states = HookStates::new();
|
||||
create_runtime_hook_states.execute_hooks(create_runtime_hooks, Some(st.clone()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// store_guest_details will get the information from the guest OS, like memory block size, agent details and is memory hotplug probe support
|
||||
async fn store_guest_details(&self) -> Result<()> {
|
||||
// get the information from agent
|
||||
let guest_details = self
|
||||
.agent
|
||||
.get_guest_details(GetGuestDetailsRequest {
|
||||
mem_block_size: true,
|
||||
mem_hotplug_probe: true,
|
||||
})
|
||||
.await
|
||||
.context("failed to store guest details")?;
|
||||
|
||||
// set memory block size
|
||||
self.hypervisor
|
||||
.set_guest_memory_block_size(guest_details.mem_block_size_bytes as u32)
|
||||
.await;
|
||||
|
||||
// set memory hotplug probe
|
||||
if guest_details.support_mem_hotplug_probe {
|
||||
self.hypervisor
|
||||
.set_capabilities(CapabilityBits::GuestMemoryProbe)
|
||||
.await;
|
||||
}
|
||||
info!(
|
||||
sl!(),
|
||||
"memory block size is {}, memory probe support {}",
|
||||
self.hypervisor.guest_memory_block_size().await,
|
||||
self.hypervisor
|
||||
.capabilities()
|
||||
.await?
|
||||
.is_mem_hotplug_probe_supported()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@ -375,6 +412,12 @@ impl Sandbox for VirtSandbox {
|
||||
.context("create sandbox")?;
|
||||
|
||||
inner.state = SandboxState::Running;
|
||||
|
||||
// get and store guest details
|
||||
self.store_guest_details()
|
||||
.await
|
||||
.context("failed to store guest details")?;
|
||||
|
||||
let agent = self.agent.clone();
|
||||
let sender = self.msg_sender.clone();
|
||||
info!(sl!(), "oom watcher start");
|
||||
|
||||
@ -11,3 +11,4 @@ tokio = { version = "1.28.1" }
|
||||
|
||||
common = { path = "../common" }
|
||||
kata-types = { path = "../../../../libs/kata-types" }
|
||||
resource = { path = "../../resource" }
|
||||
@ -9,6 +9,7 @@ use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use common::{message::Message, RuntimeHandler, RuntimeInstance};
|
||||
use kata_types::config::TomlConfig;
|
||||
use resource::cpu_mem::initial_size::InitialSizeManager;
|
||||
use tokio::sync::mpsc::Sender;
|
||||
pub struct WasmContainer {}
|
||||
|
||||
@ -31,6 +32,7 @@ impl RuntimeHandler for WasmContainer {
|
||||
_sid: &str,
|
||||
_msg_sender: Sender<Message>,
|
||||
_config: Arc<TomlConfig>,
|
||||
_init_size_manager: InitialSizeManager,
|
||||
) -> Result<RuntimeInstance> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user