diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index f40bed3b1f..eaab7eb82f 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -641,6 +641,13 @@ pub struct CpuInfo { /// - `> number of physical cores`: Set to actual number of physical cores #[serde(default)] pub default_vcpus: f32, + /// vCPU overhead to be added when sandbox/container CPU limits are provided. + /// + /// This value is used by runtime-rs static sandbox sizing as: + /// - if no CPU limits are provided: use `default_vcpus` + /// - if CPU limits are provided: use `overhead_vcpus + workload_vcpus` + #[serde(default)] + pub overhead_vcpus: f32, /// Default maximum number of vCPUs per SB/VM: /// - Unspecified or `0`: Set to actual number of physical cores or @@ -973,6 +980,14 @@ pub struct MemoryInfo { /// Default memory size in MiB for SB/VM. #[serde(default)] pub default_memory: u32, + /// Memory overhead in MiB to be added when sandbox/container memory + /// limits are provided. + /// + /// This value is used by runtime-rs static sandbox sizing as: + /// - if no memory limits are provided: use `default_memory` + /// - if memory limits are provided: use `overhead_memory + workload_memory` + #[serde(default)] + pub overhead_memory: u32, /// Default maximum memory in MiB per SB/VM: /// - Unspecified or `0`: Set to actual physical RAM @@ -1974,11 +1989,13 @@ mod tests { input: &mut CpuInfo { cpu_features: "".to_string(), default_vcpus: 0.0, + overhead_vcpus: 0.0, default_maxvcpus: 0, }, output: CpuInfo { cpu_features: "".to_string(), default_vcpus, + overhead_vcpus: 0.0, default_maxvcpus: node_cpus as u32, }, }, @@ -1987,11 +2004,13 @@ mod tests { input: &mut CpuInfo { cpu_features: "a,b,c".to_string(), default_vcpus: 9999999.0, + overhead_vcpus: 0.0, default_maxvcpus: 9999999, }, output: CpuInfo { cpu_features: "a,b,c".to_string(), default_vcpus: node_cpus, + overhead_vcpus: 0.0, default_maxvcpus: node_cpus as u32, }, }, @@ -2000,14 +2019,31 @@ mod tests { input: &mut CpuInfo { cpu_features: "a, b ,c".to_string(), default_vcpus: -1.0, + overhead_vcpus: 0.0, default_maxvcpus: 1, }, output: CpuInfo { cpu_features: "a,b,c".to_string(), default_vcpus: 1.0, + overhead_vcpus: 0.0, default_maxvcpus: 1, }, }, + TestData { + desc: "overhead_vcpus explicitly set keeps value", + input: &mut CpuInfo { + cpu_features: "x, y".to_string(), + default_vcpus: 0.0, + overhead_vcpus: 0.5, + default_maxvcpus: 2, + }, + output: CpuInfo { + cpu_features: "x,y".to_string(), + default_vcpus, + overhead_vcpus: 0.5, + default_maxvcpus: 2, + }, + }, ]; for tc in tests.iter_mut() { @@ -2029,9 +2065,30 @@ mod tests { "test[{}] default_maxvcpus", tc.desc ); + assert_eq!( + tc.input.overhead_vcpus, tc.output.overhead_vcpus, + "test[{}] overhead_vcpus", + tc.desc + ); } } + #[test] + fn test_memory_info_adjust_config_keeps_explicit_overhead_memory() { + let mut mem = MemoryInfo { + default_memory: 1024, + overhead_memory: 512, + default_maxmemory: 4096, + ..Default::default() + }; + + mem.adjust_config().unwrap(); + + assert_eq!(mem.overhead_memory, 512); + assert_eq!(mem.default_memory, 1024); + assert_eq!(mem.default_maxmemory, 4096); + } + #[cfg(all(target_arch = "powerpc64", target_endian = "little"))] use rstest::rstest; diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index edcdcf2ee3..aa79ae33a2 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -161,6 +161,22 @@ DEFVCPUS := 1 DEFMAXVCPUS := 0 ##VAR DEFMEMSZ= Default memory size in MiB DEFMEMSZ := 2048 +##VAR DEFOVERHEADVCPUS_QEMU= vCPU overhead for qemu runtimes +DEFOVERHEADVCPUS_QEMU := 0.2 +##VAR DEFOVERHEADMEMSZ_QEMU= Memory overhead (MiB) for qemu runtimes +DEFOVERHEADMEMSZ_QEMU := 32 +##VAR DEFOVERHEADVCPUS_CLH= vCPU overhead for clh runtimes +DEFOVERHEADVCPUS_CLH := 0.2 +##VAR DEFOVERHEADMEMSZ_CLH= Memory overhead (MiB) for clh runtimes +DEFOVERHEADMEMSZ_CLH := 32 +##VAR DEFOVERHEADVCPUS_DB= vCPU overhead for dragonball runtimes +DEFOVERHEADVCPUS_DB := 0.2 +##VAR DEFOVERHEADMEMSZ_DB= Memory overhead (MiB) for dragonball runtimes +DEFOVERHEADMEMSZ_DB := 32 +##VAR DEFOVERHEADVCPUS_TEE= vCPU overhead for TEE runtimes +DEFOVERHEADVCPUS_TEE := 0.4 +##VAR DEFOVERHEADMEMSZ_TEE= Memory overhead (MiB) for SNP/TDX runtimes +DEFOVERHEADMEMSZ_TEE := 128 ##VAR DEFMEMSLOTS= Default memory slots # Cases to consider : # - nvdimm rootfs image @@ -452,6 +468,8 @@ endif KERNELVERITYPARAMS_NV ?= DEFAULTVCPUS_NV := 1 DEFAULTMEMORY_NV := 8192 + DEFOVERHEADVCPUS_NV := 0.5 + DEFOVERHEADMEMSZ_NV := 512 DEFAULTTIMEOUT_NV := 1200 DEFAULTLAUNCHPROCESSTIMEOUT_NV := 15 DEFAULTPCIEROOTPORT_NV := 8 @@ -672,6 +690,14 @@ USER_VARS += SHAREDIR USER_VARS += SYSCONFDIR USER_VARS += DEFVCPUS USER_VARS += DEFVCPUS_QEMU +USER_VARS += DEFOVERHEADVCPUS_QEMU +USER_VARS += DEFOVERHEADMEMSZ_QEMU +USER_VARS += DEFOVERHEADVCPUS_CLH +USER_VARS += DEFOVERHEADMEMSZ_CLH +USER_VARS += DEFOVERHEADVCPUS_TEE +USER_VARS += DEFOVERHEADVCPUS_DB +USER_VARS += DEFOVERHEADMEMSZ_DB +USER_VARS += DEFOVERHEADMEMSZ_TEE USER_VARS += DEFMAXVCPUS USER_VARS += DEFMAXVCPUS_DB USER_VARS += DEFMAXVCPUS_QEMU @@ -760,6 +786,8 @@ USER_VARS += KERNELPARAMS_CONFIDENTIAL_NV USER_VARS += KERNELVERITYPARAMS_NV USER_VARS += DEFAULTVCPUS_NV USER_VARS += DEFAULTMEMORY_NV +USER_VARS += DEFOVERHEADVCPUS_NV +USER_VARS += DEFOVERHEADMEMSZ_NV USER_VARS += DEFAULTTIMEOUT_NV USER_VARS += DEFAULTLAUNCHPROCESSTIMEOUT_NV USER_VARS += DEFAULTPCIEROOTPORT_NV diff --git a/src/runtime-rs/config/configuration-clh-azure-runtime-rs.toml.in b/src/runtime-rs/config/configuration-clh-azure-runtime-rs.toml.in index 308ad7bbd9..668f5e527d 100644 --- a/src/runtime-rs/config/configuration-clh-azure-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-clh-azure-runtime-rs.toml.in @@ -65,6 +65,15 @@ kernel_params = "@KERNELPARAMS@" # > number of physical cores --> will be set to the actual number of physical cores default_vcpus = @DEFVCPUS@ +# Guest-side vCPU overhead budget (fractional) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_vcpus = requested_vcpus + overhead_vcpus +# (rounded up at boot). If a workload limit is set on another dimension (for example +# memory) but CPU is missing, requested_vcpus is treated as 0 and vm_vcpus equals +# overhead_vcpus (minimum 1 at boot). When no workload limits are present, +# default_vcpus is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_vcpus = @DEFOVERHEADVCPUS_CLH@ + # Default maximum number of vCPUs per SB/VM: # unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number # of vCPUs supported by KVM if that number is exceeded @@ -85,6 +94,14 @@ default_maxvcpus = @DEFMAXVCPUS@ # If unspecified then it will be set @DEFMEMSZ@ MiB. default_memory = @DEFMEMSZ@ +# Guest-side memory overhead budget (MiB) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_memory = requested_memory + overhead_memory. +# If a workload limit is set on another dimension (for example CPU) but memory is +# missing, requested_memory is treated as 0, so vm_memory equals overhead_memory. +# When no workload limits are present, default_memory is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_memory = @DEFOVERHEADMEMSZ_CLH@ + # Shared file system type: # - virtio-fs # - virtio-fs-nydus diff --git a/src/runtime-rs/config/configuration-clh-runtime-rs.toml.in b/src/runtime-rs/config/configuration-clh-runtime-rs.toml.in index c34e95b152..842f77fcd9 100644 --- a/src/runtime-rs/config/configuration-clh-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-clh-runtime-rs.toml.in @@ -65,6 +65,15 @@ kernel_params = "@KERNELPARAMS@" # > number of physical cores --> will be set to the actual number of physical cores default_vcpus = @DEFVCPUS@ +# Guest-side vCPU overhead budget (fractional) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_vcpus = requested_vcpus + overhead_vcpus +# (rounded up at boot). If a workload limit is set on another dimension (for example +# memory) but CPU is missing, requested_vcpus is treated as 0 and vm_vcpus equals +# overhead_vcpus (minimum 1 at boot). When no workload limits are present, +# default_vcpus is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_vcpus = @DEFOVERHEADVCPUS_CLH@ + # Default maximum number of vCPUs per SB/VM: # unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number # of vCPUs supported by KVM if that number is exceeded @@ -85,6 +94,14 @@ default_maxvcpus = @DEFMAXVCPUS@ # If unspecified then it will be set @DEFMEMSZ@ MiB. default_memory = @DEFMEMSZ@ +# Guest-side memory overhead budget (MiB) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_memory = requested_memory + overhead_memory. +# If a workload limit is set on another dimension (for example CPU) but memory is +# missing, requested_memory is treated as 0, so vm_memory equals overhead_memory. +# When no workload limits are present, default_memory is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_memory = @DEFOVERHEADMEMSZ_CLH@ + # Shared file system type: # - virtio-fs # - virtio-fs-nydus diff --git a/src/runtime-rs/config/configuration-dragonball.toml.in b/src/runtime-rs/config/configuration-dragonball.toml.in index 44e5c903f2..bc13e8427c 100644 --- a/src/runtime-rs/config/configuration-dragonball.toml.in +++ b/src/runtime-rs/config/configuration-dragonball.toml.in @@ -68,6 +68,15 @@ firmware = "@FIRMWAREPATH@" # > number of physical cores --> will be set to the actual number of physical cores default_vcpus = @DEFVCPUS@ +# Guest-side vCPU overhead budget (fractional) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_vcpus = requested_vcpus + overhead_vcpus +# (rounded up at boot). If a workload limit is set on another dimension (for example +# memory) but CPU is missing, requested_vcpus is treated as 0 and vm_vcpus equals +# overhead_vcpus (minimum 1 at boot). When no workload limits are present, +# default_vcpus is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_vcpus = @DEFOVERHEADVCPUS_DB@ + # Default maximum number of vCPUs per SB/VM: # unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number @@ -112,6 +121,14 @@ reclaim_guest_freed_memory = false # If unspecified then it will be set @DEFMEMSZ@ MiB. default_memory = @DEFMEMSZ@ +# Guest-side memory overhead budget (MiB) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_memory = requested_memory + overhead_memory. +# If a workload limit is set on another dimension (for example CPU) but memory is +# missing, requested_memory is treated as 0, so vm_memory equals overhead_memory. +# When no workload limits are present, default_memory is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_memory = @DEFOVERHEADMEMSZ_DB@ + # Default maximum memory in MiB per SB / VM # unspecified or == 0 --> will be set to the actual amount of physical RAM # > 0 <= amount of physical RAM --> will be set to the specified number diff --git a/src/runtime-rs/config/configuration-qemu-coco-dev-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-coco-dev-runtime-rs.toml.in index 437cd740c4..3002a8254d 100644 --- a/src/runtime-rs/config/configuration-qemu-coco-dev-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-coco-dev-runtime-rs.toml.in @@ -107,6 +107,15 @@ cpu_features = "@CPUFEATURES@" # > number of physical cores --> will be set to the actual number of physical cores default_vcpus = @DEFVCPUS_QEMU@ +# Guest-side vCPU overhead budget (fractional) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_vcpus = requested_vcpus + overhead_vcpus +# (rounded up at boot). If a workload limit is set on another dimension (for example +# memory) but CPU is missing, requested_vcpus is treated as 0 and vm_vcpus equals +# overhead_vcpus (minimum 1 at boot). When no workload limits are present, +# default_vcpus is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_vcpus = @DEFOVERHEADVCPUS_TEE@ + # Default maximum number of vCPUs per SB/VM: # unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number # of vCPUs supported by KVM if that number is exceeded @@ -149,6 +158,14 @@ reclaim_guest_freed_memory = false # Default memory size in MiB for SB/VM. # If unspecified then it will be set @DEFMEMSZ@ MiB. default_memory = @DEFMEMSZ@ + +# Guest-side memory overhead budget (MiB) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_memory = requested_memory + overhead_memory. +# If a workload limit is set on another dimension (for example CPU) but memory is +# missing, requested_memory is treated as 0, so vm_memory equals overhead_memory. +# When no workload limits are present, default_memory is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_memory = @DEFOVERHEADMEMSZ_TEE@ # # Default memory slots per SB/VM. # If unspecified then it will be set @DEFMEMSLOTS@. diff --git a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in index 3738301bcd..ff1785ad14 100644 --- a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in @@ -99,6 +99,15 @@ cpu_features = "@CPUFEATURES@" # > number of physical cores --> will be set to the actual number of physical cores default_vcpus = @DEFAULTVCPUS_NV@ +# Guest-side vCPU overhead budget (fractional) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_vcpus = requested_vcpus + overhead_vcpus +# (rounded up at boot). If a workload limit is set on another dimension (for example +# memory) but CPU is missing, requested_vcpus is treated as 0 and vm_vcpus equals +# overhead_vcpus (minimum 1 at boot). When no workload limits are present, +# default_vcpus is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_vcpus = @DEFOVERHEADVCPUS_NV@ + # Default maximum number of vCPUs per SB/VM: # unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number # of vCPUs supported by KVM if that number is exceeded @@ -141,6 +150,14 @@ reclaim_guest_freed_memory = false # Default memory size in MiB for SB/VM. # If unspecified then it will be set @DEFMEMSZ@ MiB. default_memory = @DEFAULTMEMORY_NV@ + +# Guest-side memory overhead budget (MiB) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_memory = requested_memory + overhead_memory. +# If a workload limit is set on another dimension (for example CPU) but memory is +# missing, requested_memory is treated as 0, so vm_memory equals overhead_memory. +# When no workload limits are present, default_memory is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_memory = @DEFOVERHEADMEMSZ_NV@ # # Default memory slots per SB/VM. # If unspecified then it will be set @DEFMEMSLOTS@. diff --git a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-snp-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-snp-runtime-rs.toml.in index 06f7a2e9f9..82814e2bf0 100644 --- a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-snp-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-snp-runtime-rs.toml.in @@ -140,6 +140,15 @@ cpu_features = "@CPUFEATURES@" # > number of physical cores --> will be set to the actual number of physical cores default_vcpus = @DEFAULTVCPUS_NV@ +# Guest-side vCPU overhead budget (fractional) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_vcpus = requested_vcpus + overhead_vcpus +# (rounded up at boot). If a workload limit is set on another dimension (for example +# memory) but CPU is missing, requested_vcpus is treated as 0 and vm_vcpus equals +# overhead_vcpus (minimum 1 at boot). When no workload limits are present, +# default_vcpus is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_vcpus = @DEFOVERHEADVCPUS_NV@ + # Default maximum number of vCPUs per SB/VM: # unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number # of vCPUs supported by KVM if that number is exceeded @@ -182,6 +191,14 @@ reclaim_guest_freed_memory = false # Default memory size in MiB for SB/VM. # If unspecified then it will be set @DEFMEMSZ@ MiB. default_memory = @DEFAULTMEMORY_NV@ + +# Guest-side memory overhead budget (MiB) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_memory = requested_memory + overhead_memory. +# If a workload limit is set on another dimension (for example CPU) but memory is +# missing, requested_memory is treated as 0, so vm_memory equals overhead_memory. +# When no workload limits are present, default_memory is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_memory = @DEFOVERHEADMEMSZ_NV@ # # Default memory slots per SB/VM. # If unspecified then it will be set @DEFMEMSLOTS@. diff --git a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-tdx-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-tdx-runtime-rs.toml.in index 9ae7041cc5..1df79b54b9 100644 --- a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-tdx-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-tdx-runtime-rs.toml.in @@ -116,6 +116,15 @@ cpu_features = "@CPUFEATURES@" # > number of physical cores --> will be set to the actual number of physical cores default_vcpus = @DEFAULTVCPUS_NV@ +# Guest-side vCPU overhead budget (fractional) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_vcpus = requested_vcpus + overhead_vcpus +# (rounded up at boot). If a workload limit is set on another dimension (for example +# memory) but CPU is missing, requested_vcpus is treated as 0 and vm_vcpus equals +# overhead_vcpus (minimum 1 at boot). When no workload limits are present, +# default_vcpus is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_vcpus = @DEFOVERHEADVCPUS_NV@ + # Default maximum number of vCPUs per SB/VM: # unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number # of vCPUs supported by KVM if that number is exceeded @@ -158,6 +167,14 @@ reclaim_guest_freed_memory = false # Default memory size in MiB for SB/VM. # If unspecified then it will be set @DEFMEMSZ@ MiB. default_memory = @DEFAULTMEMORY_NV@ + +# Guest-side memory overhead budget (MiB) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_memory = requested_memory + overhead_memory. +# If a workload limit is set on another dimension (for example CPU) but memory is +# missing, requested_memory is treated as 0, so vm_memory equals overhead_memory. +# When no workload limits are present, default_memory is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_memory = @DEFOVERHEADMEMSZ_NV@ # # Default memory slots per SB/VM. # If unspecified then it will be set @DEFMEMSLOTS@. diff --git a/src/runtime-rs/config/configuration-qemu-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-runtime-rs.toml.in index 47adda3409..897ae5166d 100644 --- a/src/runtime-rs/config/configuration-qemu-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-runtime-rs.toml.in @@ -86,6 +86,15 @@ cpu_features = "@CPUFEATURES@" # > number of physical cores --> will be set to the actual number of physical cores default_vcpus = @DEFVCPUS_QEMU@ +# Guest-side vCPU overhead budget (fractional) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_vcpus = requested_vcpus + overhead_vcpus +# (rounded up at boot). If a workload limit is set on another dimension (for example +# memory) but CPU is missing, requested_vcpus is treated as 0 and vm_vcpus equals +# overhead_vcpus (minimum 1 at boot). When no workload limits are present, +# default_vcpus is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_vcpus = @DEFOVERHEADVCPUS_QEMU@ + # Default maximum number of vCPUs per SB/VM: # unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number # of vCPUs supported by KVM if that number is exceeded @@ -128,6 +137,14 @@ reclaim_guest_freed_memory = false # Default memory size in MiB for SB/VM. # If unspecified then it will be set @DEFMEMSZ@ MiB. default_memory = @DEFMEMSZ@ + +# Guest-side memory overhead budget (MiB) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_memory = requested_memory + overhead_memory. +# If a workload limit is set on another dimension (for example CPU) but memory is +# missing, requested_memory is treated as 0, so vm_memory equals overhead_memory. +# When no workload limits are present, default_memory is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_memory = @DEFOVERHEADMEMSZ_QEMU@ # # Default memory slots per SB/VM. # If unspecified then it will be set @DEFMEMSLOTS@. diff --git a/src/runtime-rs/config/configuration-qemu-se-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-se-runtime-rs.toml.in index 7b3d1649e4..21b8d27560 100644 --- a/src/runtime-rs/config/configuration-qemu-se-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-se-runtime-rs.toml.in @@ -95,6 +95,15 @@ cpu_features = "@CPUFEATURES@" # > number of physical cores --> will be set to the actual number of physical cores default_vcpus = @DEFVCPUS_QEMU@ +# Guest-side vCPU overhead budget (fractional) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_vcpus = requested_vcpus + overhead_vcpus +# (rounded up at boot). If a workload limit is set on another dimension (for example +# memory) but CPU is missing, requested_vcpus is treated as 0 and vm_vcpus equals +# overhead_vcpus (minimum 1 at boot). When no workload limits are present, +# default_vcpus is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_vcpus = @DEFOVERHEADVCPUS_TEE@ + # Default maximum number of vCPUs per SB/VM: # unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number # of vCPUs supported by KVM if that number is exceeded @@ -127,6 +136,14 @@ default_bridges = @DEFBRIDGES@ # Default memory size in MiB for SB/VM. # If unspecified then it will be set @DEFMEMSZ@ MiB. default_memory = @DEFMEMSZ@ + +# Guest-side memory overhead budget (MiB) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_memory = requested_memory + overhead_memory. +# If a workload limit is set on another dimension (for example CPU) but memory is +# missing, requested_memory is treated as 0, so vm_memory equals overhead_memory. +# When no workload limits are present, default_memory is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_memory = @DEFOVERHEADMEMSZ_TEE@ # # Default memory slots per SB/VM. # If unspecified then it will be set @DEFMEMSLOTS@. diff --git a/src/runtime-rs/config/configuration-qemu-snp-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-snp-runtime-rs.toml.in index de39c6a424..2770269b00 100644 --- a/src/runtime-rs/config/configuration-qemu-snp-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-snp-runtime-rs.toml.in @@ -133,6 +133,15 @@ cpu_features = "@CPUFEATURES@" # > number of physical cores --> will be set to the actual number of physical cores default_vcpus = @DEFVCPUS_QEMU@ +# Guest-side vCPU overhead budget (fractional) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_vcpus = requested_vcpus + overhead_vcpus +# (rounded up at boot). If a workload limit is set on another dimension (for example +# memory) but CPU is missing, requested_vcpus is treated as 0 and vm_vcpus equals +# overhead_vcpus (minimum 1 at boot). When no workload limits are present, +# default_vcpus is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_vcpus = @DEFOVERHEADVCPUS_TEE@ + # Default maximum number of vCPUs per SB/VM: # unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number # of vCPUs supported by KVM if that number is exceeded @@ -166,6 +175,14 @@ default_bridges = @DEFBRIDGES@ # If unspecified then it will be set @DEFMEMSZ@ MiB. default_memory = @DEFMEMSZ@ +# Guest-side memory overhead budget (MiB) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_memory = requested_memory + overhead_memory. +# If a workload limit is set on another dimension (for example CPU) but memory is +# missing, requested_memory is treated as 0, so vm_memory equals overhead_memory. +# When no workload limits are present, default_memory is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_memory = @DEFOVERHEADMEMSZ_TEE@ + # # Default memory slots per SB/VM. # If unspecified then it will be set @DEFMEMSLOTS@. diff --git a/src/runtime-rs/config/configuration-qemu-tdx-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-tdx-runtime-rs.toml.in index ccf5b4da37..d09e7583c4 100644 --- a/src/runtime-rs/config/configuration-qemu-tdx-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-tdx-runtime-rs.toml.in @@ -111,6 +111,15 @@ cpu_features = "@CPUFEATURES@" # > number of physical cores --> will be set to the actual number of physical cores default_vcpus = 1 +# Guest-side vCPU overhead budget (fractional) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_vcpus = requested_vcpus + overhead_vcpus +# (rounded up at boot). If a workload limit is set on another dimension (for example +# memory) but CPU is missing, requested_vcpus is treated as 0 and vm_vcpus equals +# overhead_vcpus (minimum 1 at boot). When no workload limits are present, +# default_vcpus is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_vcpus = @DEFOVERHEADVCPUS_TEE@ + # Default maximum number of vCPUs per SB/VM: # unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number # of vCPUs supported by KVM if that number is exceeded @@ -143,6 +152,14 @@ default_bridges = @DEFBRIDGES@ # Default memory size in MiB for SB/VM. # If unspecified then it will be set @DEFMEMSZ@ MiB. default_memory = @DEFMEMSZ@ + +# Guest-side memory overhead budget (MiB) used with static_sandbox_resource_mgmt. +# When workload limits are present, vm_memory = requested_memory + overhead_memory. +# If a workload limit is set on another dimension (for example CPU) but memory is +# missing, requested_memory is treated as 0, so vm_memory equals overhead_memory. +# When no workload limits are present, default_memory is used instead. +# See docs/how-to/how-to-size-sandbox-overhead-runtime-rs.md +overhead_memory = @DEFOVERHEADMEMSZ_TEE@ # # Default memory slots per SB/VM. # If unspecified then it will be set @DEFMEMSLOTS@. diff --git a/src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs b/src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs index b886d9faae..934207d37c 100644 --- a/src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs +++ b/src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs @@ -6,7 +6,7 @@ use std::{collections::HashMap, convert::TryFrom}; -use anyhow::{Context, Result}; +use anyhow::{ensure, Context, Result}; use kata_types::{ annotations::Annotation, config::TomlConfig, container::ContainerType, cpu::LinuxContainerCpuResources, k8s::container_type, @@ -159,28 +159,36 @@ impl InitialSizeManager { .get_mut(hypervisor_name) .context("failed to get hypervisor config")?; - if self.resource.vcpu > 0.0 { - info!(sl!(), "resource with vcpu {}", self.resource.vcpu); - if config.runtime.static_sandbox_resource_mgmt { - hv.cpu_info.default_vcpus += self.resource.vcpu; - } - } - - if config.runtime.static_sandbox_resource_mgmt { - let new_vcpus_ceil = hv.cpu_info.default_vcpus.ceil() as u32; - hv.cpu_info.default_maxvcpus = new_vcpus_ceil; - } - self.resource.orig_toml_default_mem = hv.memory_info.default_memory; - if self.resource.mem_mb > 0 { - info!(sl!(), "resource with memory {}", self.resource.mem_mb); - if config.runtime.static_sandbox_resource_mgmt { - hv.memory_info.default_memory += self.resource.mem_mb; - if hv.memory_info.default_maxmemory < hv.memory_info.default_memory { - hv.memory_info.default_maxmemory = hv.memory_info.default_memory; - } - } + + // Non-static mode keeps configured defaults unchanged. + if !config.runtime.static_sandbox_resource_mgmt { + validate_non_zero_sandbox_memory(hypervisor_name, hv.memory_info.default_memory)?; + return Ok(()); } + + if self.resource.vcpu > 0.0 || self.resource.mem_mb > 0 { + if self.resource.vcpu > 0.0 { + info!(sl!(), "resource with vcpu {}", self.resource.vcpu); + } + if self.resource.mem_mb > 0 { + info!(sl!(), "resource with memory {}", self.resource.mem_mb); + } + + hv.cpu_info.default_vcpus = + (hv.cpu_info.overhead_vcpus + self.resource.vcpu).max(1.0); + + hv.memory_info.default_memory = + hv.memory_info.overhead_memory + self.resource.mem_mb; + hv.memory_info.default_maxmemory = hv + .memory_info + .default_maxmemory + .max(hv.memory_info.default_memory); + } + + hv.cpu_info.default_maxvcpus = hv.cpu_info.default_vcpus.ceil() as u32; + + validate_non_zero_sandbox_memory(hypervisor_name, hv.memory_info.default_memory)?; Ok(()) } @@ -189,6 +197,15 @@ impl InitialSizeManager { } } +fn validate_non_zero_sandbox_memory(hypervisor_name: &str, memory_mib: u32) -> Result<()> { + ensure!( + memory_mib > 0, + "computed sandbox memory is 0 MiB for hypervisor '{}'; set a non-zero memory limit or configure non-zero default_memory/overhead_memory", + hypervisor_name + ); + Ok(()) +} + fn get_nr_vcpu(resource: &LinuxContainerCpuResources) -> f32 { if let Some(v) = resource.get_vcpus() { v as f32 @@ -227,6 +244,7 @@ mod tests { use super::*; use kata_types::annotations::cri_containerd; use oci_spec::runtime::{LinuxBuilder, LinuxMemory, LinuxMemoryBuilder, LinuxResourcesBuilder}; + use rstest::rstest; use std::collections::HashMap; #[derive(Clone)] struct InputData { @@ -398,8 +416,10 @@ mod tests { fn make_config( default_vcpus: f32, + overhead_vcpus: f32, default_maxvcpus: u32, default_memory: u32, + overhead_memory: u32, default_maxmemory: u32, static_sandbox_resource_mgmt: bool, ) -> TomlConfig { @@ -411,8 +431,10 @@ mod tests { .insert("qemu".to_owned(), Hypervisor::default()); config.hypervisor.entry("qemu".to_owned()).and_modify(|hv| { hv.cpu_info.default_vcpus = default_vcpus; + hv.cpu_info.overhead_vcpus = overhead_vcpus; hv.cpu_info.default_maxvcpus = default_maxvcpus; hv.memory_info.default_memory = default_memory; + hv.memory_info.overhead_memory = overhead_memory; hv.memory_info.default_maxmemory = default_maxmemory; }); config.runtime.hypervisor_name = "qemu".to_owned(); @@ -422,7 +444,7 @@ mod tests { #[test] fn test_setup_config_static_applies_vcpu_and_memory() { - let mut config = make_config(1.0, 4, 256, 4096, true); + let mut config = make_config(1.0, 0.5, 4, 256, 128, 4096, true); let mut mgr = InitialSizeManager { resource: InitialSize { vcpu: 1.2, @@ -433,13 +455,13 @@ mod tests { mgr.setup_config(&mut config).unwrap(); let hv = config.hypervisor.get("qemu").unwrap(); - assert_eq!(hv.cpu_info.default_vcpus, 2.2); - assert_eq!(hv.memory_info.default_memory, 768); + assert_eq!(hv.cpu_info.default_vcpus, 1.7); + assert_eq!(hv.memory_info.default_memory, 640); } #[test] fn test_setup_config_non_static_does_not_apply() { - let mut config = make_config(1.0, 4, 256, 4096, false); + let mut config = make_config(1.0, 0.5, 4, 256, 128, 4096, false); let mut mgr = InitialSizeManager { resource: InitialSize { vcpu: 1.2, @@ -456,7 +478,7 @@ mod tests { #[test] fn test_setup_config_clamps_maxvcpus() { - let mut config = make_config(1.0, 2, 256, 4096, true); + let mut config = make_config(1.0, 1.0, 2, 256, 128, 4096, true); let mut mgr = InitialSizeManager { resource: InitialSize { vcpu: 2.5, @@ -473,7 +495,7 @@ mod tests { #[test] fn test_setup_config_static_reduces_maxvcpus_to_static_total() { - let mut config = make_config(1.0, 8, 256, 4096, true); + let mut config = make_config(1.0, 0.5, 8, 256, 128, 4096, true); let mut mgr = InitialSizeManager { resource: InitialSize { vcpu: 1.2, @@ -484,13 +506,13 @@ mod tests { mgr.setup_config(&mut config).unwrap(); let hv = config.hypervisor.get("qemu").unwrap(); - assert_eq!(hv.cpu_info.default_vcpus, 2.2); - assert_eq!(hv.cpu_info.default_maxvcpus, 3); + assert_eq!(hv.cpu_info.default_vcpus, 1.7); + assert_eq!(hv.cpu_info.default_maxvcpus, 2); } #[test] fn test_setup_config_clamps_maxmemory() { - let mut config = make_config(1.0, 4, 256, 300, true); + let mut config = make_config(1.0, 0.5, 4, 256, 128, 300, true); let mut mgr = InitialSizeManager { resource: InitialSize { vcpu: 0.0, @@ -501,13 +523,13 @@ mod tests { mgr.setup_config(&mut config).unwrap(); let hv = config.hypervisor.get("qemu").unwrap(); - assert_eq!(hv.memory_info.default_memory, 768); - assert_eq!(hv.memory_info.default_maxmemory, 768); + assert_eq!(hv.memory_info.default_memory, 640); + assert_eq!(hv.memory_info.default_maxmemory, 640); } #[test] fn test_setup_config_preserves_orig_toml_default_mem() { - let mut config = make_config(1.0, 4, 256, 4096, true); + let mut config = make_config(1.0, 0.5, 4, 256, 128, 4096, true); let mut mgr = InitialSizeManager { resource: InitialSize { vcpu: 0.0, @@ -551,4 +573,77 @@ mod tests { assert!((mgr.resource.vcpu - 1.2).abs() < VCPU_TOLERANCE); assert_eq!(mgr.resource.mem_mb, 256); } + + #[test] + fn test_setup_config_static_without_limits_uses_toml_defaults() { + let mut config = make_config(2.0, 0.5, 8, 512, 128, 4096, true); + let mut mgr = InitialSizeManager { + resource: InitialSize { + vcpu: 0.0, + mem_mb: 0, + orig_toml_default_mem: 0, + }, + }; + + mgr.setup_config(&mut config).unwrap(); + let hv = config.hypervisor.get("qemu").unwrap(); + assert_eq!(hv.cpu_info.default_vcpus, 2.0); + assert_eq!(hv.memory_info.default_memory, 512); + } + + #[test] + fn test_setup_config_static_errors_on_zero_memory() { + let mut config = make_config(1.0, 0.5, 8, 1024, 0, 4096, true); + let mut mgr = InitialSizeManager { + resource: InitialSize { + vcpu: 1.0, + mem_mb: 0, + orig_toml_default_mem: 0, + }, + }; + + let err = mgr.setup_config(&mut config).unwrap_err().to_string(); + assert!(err.contains("computed sandbox memory is 0 MiB")); + assert!(err.contains("default_memory/overhead_memory")); + } + + #[rstest] + #[case::both_limits(3.0, 0.75, 1024, 256, 1.25, 1024, 2.0, 1280)] + #[case::cpu_only_limit(3.0, 0.5, 1024, 128, 1.5, 0, 2.0, 128)] + #[case::memory_only_limit(3.0, 0.5, 1024, 128, 0.0, 512, 1.0, 640)] + #[case::both_limits_zero_overhead(3.0, 0.0, 1024, 0, 1.25, 1024, 1.25, 1024)] + #[case::memory_only_zero_overhead(3.0, 0.0, 1024, 0, 0.0, 512, 1.0, 512)] + fn test_setup_config_static_requested_vs_defaults( + #[case] default_vcpus: f32, + #[case] overhead_vcpus: f32, + #[case] default_memory: u32, + #[case] overhead_memory: u32, + #[case] requested_vcpus: f32, + #[case] requested_mem_mb: u32, + #[case] expected_default_vcpus: f32, + #[case] expected_default_memory: u32, + ) { + let mut config = make_config( + default_vcpus, + overhead_vcpus, + 8, + default_memory, + overhead_memory, + 4096, + true, + ); + let mut mgr = InitialSizeManager { + resource: InitialSize { + vcpu: requested_vcpus, + mem_mb: requested_mem_mb, + orig_toml_default_mem: 0, + }, + }; + + mgr.setup_config(&mut config).unwrap(); + let hv = config.hypervisor.get("qemu").unwrap(); + + assert_eq!(hv.cpu_info.default_vcpus, expected_default_vcpus); + assert_eq!(hv.memory_info.default_memory, expected_default_memory); + } } diff --git a/tests/integration/kubernetes/runtimeclass_workloads/pod-guest-pull-in-trusted-storage.yaml.in b/tests/integration/kubernetes/runtimeclass_workloads/pod-guest-pull-in-trusted-storage.yaml.in index 8bc921b17f..a1fcb53393 100644 --- a/tests/integration/kubernetes/runtimeclass_workloads/pod-guest-pull-in-trusted-storage.yaml.in +++ b/tests/integration/kubernetes/runtimeclass_workloads/pod-guest-pull-in-trusted-storage.yaml.in @@ -31,6 +31,7 @@ spec: resources: limits: cpu: "2" + memory: "2Gi" volumeDevices: - devicePath: /dev/trusted_store name: trusted-storage