From 9af9844bc7331d0a37d0159f299ab3fb0e443669 Mon Sep 17 00:00:00 2001 From: Manuel Huber Date: Thu, 2 Jan 2025 22:49:49 +0000 Subject: [PATCH] runtime: Allocate default workload vcpus - similar to the static_sandbox_default_workload_mem option, assign a default number of vcpus to the VM when no limits are given, 1 vcpu in this case - similar to commit c7b8ee9, do not allocate additional vcpus when limits are provided Signed-off-by: Manuel Huber --- src/runtime/Makefile | 6 ++++-- src/runtime/config/configuration-clh.toml.in | 7 ++++++- src/runtime/pkg/katautils/config-settings.go.in | 2 +- src/runtime/pkg/katautils/config.go | 2 ++ src/runtime/pkg/oci/utils.go | 12 ++++++++++-- src/runtime/virtcontainers/hypervisor.go | 2 +- src/runtime/virtcontainers/sandbox.go | 2 ++ .../node-builder/azure-linux/package_build.sh | 2 +- 8 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index f36209c4b5..49635b15d5 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -198,7 +198,7 @@ STRATOVIRTPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTCMD) STRATOVIRTVALIDHYPERVISORPATHS := [\"$(STRATOVIRTPATH)\"] # Default number of vCPUs -DEFVCPUS := 1 +DEFVCPUS ?= 1 # Default maximum number of vCPUs DEFMAXVCPUS := 0 # Default memory size in MiB @@ -274,8 +274,9 @@ DEFSANDBOXCGROUPONLY ?= false DEFSTATICRESOURCEMGMT ?= false DEFSTATICRESOURCEMGMT_TEE = true -# Default memory for use for workloads within the sandbox if no specific workload memory value is requested +# Default memory and vcpus for use for workloads within the sandbox if no specific workload values are requested DEFSTATICSANDBOXWORKLOADMEM ?= 2048 +DEFSTATICSANDBOXWORKLOADVCPUS ?= 1 DEFDISABLEIMAGENVDIMM ?= false @@ -750,6 +751,7 @@ USER_VARS += DEFSTATICRESOURCEMGMT_FC USER_VARS += DEFSTATICRESOURCEMGMT_STRATOVIRT USER_VARS += DEFSTATICRESOURCEMGMT_TEE USER_VARS += DEFSTATICSANDBOXWORKLOADMEM +USER_VARS += DEFSTATICSANDBOXWORKLOADVCPUS USER_VARS += DEFBINDMOUNTS USER_VARS += DEFCREATECONTAINERTIMEOUT USER_VARS += DEFDANCONF diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index 8fa876499e..44d0934225 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -95,7 +95,7 @@ kernel_params = "@KERNELPARAMS@" # < 0 --> will be set to the actual number of physical cores # > 0 <= number of physical cores --> will be set to the specified number # > number of physical cores --> will be set to the actual number of physical cores -default_vcpus = 1 +default_vcpus = @DEFVCPUS@ # Default maximum number of vCPUs per SB/VM: # unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number @@ -437,6 +437,11 @@ static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_CLH@ # default amount of memory available within the sandbox. static_sandbox_default_workload_mem=@DEFSTATICSANDBOXWORKLOADMEM@ +# If set, the runtime will use the value as the default number of vcpus for the sandbox when no workload vcpu request is passed +# down to the shim via the OCI when static sandbox resource management is enabled. With this, we ensure that workloads have a proper +# default amount of vcpus available within the sandbox. +static_sandbox_default_workload_vcpus=@DEFSTATICSANDBOXWORKLOADVCPUS@ + # If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path. # This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory. # If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts` diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in index d7680dd1bd..c921d48ca6 100644 --- a/src/runtime/pkg/katautils/config-settings.go.in +++ b/src/runtime/pkg/katautils/config-settings.go.in @@ -59,7 +59,7 @@ const defaultKernelParams = "" const defaultMachineType = "q35" const defaultQgsPort = 4050 -const defaultVCPUCount uint32 = 1 +const defaultVCPUCount uint32 = 0 const defaultMaxVCPUCount uint32 = 0 const defaultMemSize uint32 = 2048 // MiB const defaultMemSlots uint32 = 10 diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 3894cd9c69..357915917b 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -190,6 +190,7 @@ type runtime struct { SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` StaticSandboxResourceMgmt bool `toml:"static_sandbox_resource_mgmt"` StaticSandboxWorkloadDefaultMem uint32 `toml:"static_sandbox_default_workload_mem"` + StaticSandboxWorkloadDefaultVcpus float32 `toml:"static_sandbox_default_workload_vcpus"` EnablePprof bool `toml:"enable_pprof"` DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"` CreateContainerTimeout uint64 `toml:"create_container_timeout"` @@ -1564,6 +1565,7 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat config.GuestSeLinuxLabel = tomlConf.Runtime.GuestSeLinuxLabel config.StaticSandboxResourceMgmt = tomlConf.Runtime.StaticSandboxResourceMgmt config.StaticSandboxWorkloadDefaultMem = tomlConf.Runtime.StaticSandboxWorkloadDefaultMem + config.StaticSandboxWorkloadDefaultVcpus = tomlConf.Runtime.StaticSandboxWorkloadDefaultVcpus config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs config.EnablePprof = tomlConf.Runtime.EnablePprof diff --git a/src/runtime/pkg/oci/utils.go b/src/runtime/pkg/oci/utils.go index b32f8fe602..aad04053d2 100644 --- a/src/runtime/pkg/oci/utils.go +++ b/src/runtime/pkg/oci/utils.go @@ -156,6 +156,9 @@ type RuntimeConfig struct { // Memory to allocate for workloads within the sandbox when workload memory is unspecified StaticSandboxWorkloadDefaultMem uint32 + // vcpus to allocate for workloads within the sandbox when workload vcpus is unspecified + StaticSandboxWorkloadDefaultVcpus float32 + // Determines if create a netns for hypervisor process DisableNewNetNs bool @@ -1141,6 +1144,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st StaticWorkloadDefaultMem: runtime.StaticSandboxWorkloadDefaultMem, + StaticWorkloadDefaultVcpus: runtime.StaticSandboxWorkloadDefaultVcpus, + ShmSize: shmSize, VfioMode: runtime.VfioMode, @@ -1171,11 +1176,14 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st // with the base number of CPU/memory (which is equal to the default CPU/memory specified for the runtime // configuration or annotations) as well as any specified workload resources. if sandboxConfig.StaticResourceMgmt { - // If no Limits are set in pod config, use StaticWorkloadDefaultMem to ensure the containers generally - // have a reasonable amount of memory available + // If no Limits are set in pod config, use StaticWorkloadDefaultMem/Vcpus to ensure the containers generally + // have a reasonable amount of resources available if sandboxConfig.SandboxResources.WorkloadMemMB == 0 { sandboxConfig.SandboxResources.WorkloadMemMB = sandboxConfig.StaticWorkloadDefaultMem } + if sandboxConfig.SandboxResources.WorkloadCPUs == 0 { + sandboxConfig.SandboxResources.WorkloadCPUs = sandboxConfig.StaticWorkloadDefaultVcpus + } sandboxConfig.SandboxResources.BaseCPUs = sandboxConfig.HypervisorConfig.NumVCPUsF sandboxConfig.SandboxResources.BaseMemMB = sandboxConfig.HypervisorConfig.MemorySize diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 0231660b83..6eff153f25 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -62,7 +62,7 @@ const ( procCPUInfo = "/proc/cpuinfo" - defaultVCPUs = float32(1) + defaultVCPUs = float32(0) // 2 GiB defaultMemSzMiB = 2048 diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 0d391323f8..76c47b23f0 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -163,6 +163,8 @@ type SandboxConfig struct { StaticWorkloadDefaultMem uint32 + StaticWorkloadDefaultVcpus float32 + // Memory to allocate for workloads within the sandbox when workload memory is unspecified ShmSize uint64 diff --git a/tools/osbuilder/node-builder/azure-linux/package_build.sh b/tools/osbuilder/node-builder/azure-linux/package_build.sh index ce53530f28..fb93eec197 100755 --- a/tools/osbuilder/node-builder/azure-linux/package_build.sh +++ b/tools/osbuilder/node-builder/azure-linux/package_build.sh @@ -21,7 +21,7 @@ source "${common_file}" # these options ensure we produce the proper CLH config file runtime_make_flags="SKIP_GO_VERSION_CHECK=1 QEMUCMD= FCCMD= ACRNCMD= STRATOVIRTCMD= DEFAULT_HYPERVISOR=cloud-hypervisor - DEFMEMSZ=0 DEFSTATICSANDBOXWORKLOADMEM=512 DEFVIRTIOFSDAEMON=${VIRTIOFSD_BINARY_LOCATION} PREFIX=${INSTALL_PATH_PREFIX}" + DEFMEMSZ=0 DEFSTATICSANDBOXWORKLOADMEM=512 DEFVCPUS=0 DEFSTATICSANDBOXWORKLOADVCPUS=1 DEFVIRTIOFSDAEMON=${VIRTIOFSD_BINARY_LOCATION} PREFIX=${INSTALL_PATH_PREFIX}" # - for vanilla Kata we use the kernel binary. For ConfPods we use IGVM, so no need to provide kernel path. # - for vanilla Kata we explicitly set DEFSTATICRESOURCEMGMT_CLH. For ConfPods,