diff --git a/Makefile b/Makefile index a9055ec236..a27ec3127c 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,8 @@ PROXYPATH := $(PKGLIBEXECDIR)/$(PROXYCMD) # Default number of vCPUs DEFVCPUS := 1 +# Default maximum number of vCPUs +DEFMAXVCPUS := 0 # Default memory size in MiB DEFMEMSZ := 2048 #Default number of bridges @@ -169,6 +171,7 @@ USER_VARS += SHAREDIR USER_VARS += SHIMPATH USER_VARS += SYSCONFDIR USER_VARS += DEFVCPUS +USER_VARS += DEFMAXVCPUS USER_VARS += DEFMEMSZ USER_VARS += DEFBRIDGES USER_VARS += DEFNETWORKMODEL @@ -262,6 +265,7 @@ const defaultMachineType = "$(MACHINETYPE)" const defaultRootDirectory = "$(PKGRUNDIR)" const defaultVCPUCount uint32 = $(DEFVCPUS) +const defaultMaxVCPUCount uint32 = $(DEFMAXVCPUS) const defaultMemSize uint32 = $(DEFMEMSZ) // MiB const defaultBridgesCount uint32 = $(DEFBRIDGES) const defaultInterNetworkingModel = "$(DEFNETWORKMODEL)" @@ -347,6 +351,7 @@ $(GENERATED_FILES): %: %.in Makefile VERSION -e "s|@MACHINETYPE@|$(MACHINETYPE)|g" \ -e "s|@SHIMPATH@|$(SHIMPATH)|g" \ -e "s|@DEFVCPUS@|$(DEFVCPUS)|g" \ + -e "s|@DEFMAXVCPUS@|$(DEFMAXVCPUS)|g" \ -e "s|@DEFMEMSZ@|$(DEFMEMSZ)|g" \ -e "s|@DEFBRIDGES@|$(DEFBRIDGES)|g" \ -e "s|@DEFNETWORKMODEL@|$(DEFNETWORKMODEL)|g" \ diff --git a/cli/config.go b/cli/config.go index 9950d0ae40..40d64a788c 100644 --- a/cli/config.go +++ b/cli/config.go @@ -74,6 +74,7 @@ type hypervisor struct { KernelParams string `toml:"kernel_params"` MachineType string `toml:"machine_type"` DefaultVCPUs int32 `toml:"default_vcpus"` + DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` DefaultMemSz uint32 `toml:"default_memory"` DefaultBridges uint32 `toml:"default_bridges"` Msize9p uint32 `toml:"msize_9p"` @@ -202,6 +203,25 @@ func (h hypervisor) defaultVCPUs() uint32 { return uint32(h.DefaultVCPUs) } +func (h hypervisor) defaultMaxVCPUs() uint32 { + numcpus := uint32(goruntime.NumCPU()) + maxvcpus := vc.MaxQemuVCPUs() + reqVCPUs := h.DefaultMaxVCPUs + + //don't exceed the number of physical CPUs. If a default is not provided, use the + // numbers of physical CPUs + if reqVCPUs >= numcpus || reqVCPUs == 0 { + reqVCPUs = numcpus + } + + // Don't exceed the maximum number of vCPUs supported by hypervisor + if reqVCPUs > maxvcpus { + return maxvcpus + } + + return reqVCPUs +} + func (h hypervisor) defaultMemSz() uint32 { if h.DefaultMemSz < 8 { return defaultMemSize // MiB @@ -313,6 +333,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), HypervisorMachineType: machineType, DefaultVCPUs: h.defaultVCPUs(), + DefaultMaxVCPUs: h.defaultMaxVCPUs(), DefaultMemSz: h.defaultMemSz(), DefaultBridges: h.defaultBridges(), DisableBlockDeviceUse: h.DisableBlockDeviceUse, @@ -418,6 +439,7 @@ func loadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat MachineAccelerators: defaultMachineAccelerators, HypervisorMachineType: defaultMachineType, DefaultVCPUs: defaultVCPUCount, + DefaultMaxVCPUs: defaultMaxVCPUCount, DefaultMemSz: defaultMemSize, DefaultBridges: defaultBridgesCount, MemPrealloc: defaultEnableMemPrealloc, diff --git a/cli/config/configuration.toml.in b/cli/config/configuration.toml.in index 4b310aa208..b6e91a31ac 100644 --- a/cli/config/configuration.toml.in +++ b/cli/config/configuration.toml.in @@ -45,6 +45,21 @@ machine_accelerators="@MACHINEACCELERATORS@" # > number of physical cores --> will be set to the actual number of physical cores default_vcpus = 1 +# Default maximum number of vCPUs per SB/VM: +# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when +# the actual number of physical cores is greater than it. +# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU +# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs +# can be added to a SB/VM, but the memory footprint will be big. Another example, with +# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of +# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable, +# unless you know what are you doing. +default_maxvcpus = @DEFMAXVCPUS@ # Bridges can be used to hot plug devices. # Limitations: diff --git a/cli/config_test.go b/cli/config_test.go index fc24137a42..5dbdf7524d 100644 --- a/cli/config_test.go +++ b/cli/config_test.go @@ -44,6 +44,7 @@ func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath image = "` + imagePath + `" machine_type = "` + machineType + `" default_vcpus = ` + strconv.FormatUint(uint64(defaultVCPUCount), 10) + ` + default_maxvcpus = ` + strconv.FormatUint(uint64(defaultMaxVCPUCount), 10) + ` default_memory = ` + strconv.FormatUint(uint64(defaultMemSize), 10) + ` disable_block_device_use = ` + strconv.FormatBool(disableBlock) + ` enable_iothreads = ` + strconv.FormatBool(enableIOThreads) + ` @@ -129,6 +130,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), HypervisorMachineType: machineType, DefaultVCPUs: defaultVCPUCount, + DefaultMaxVCPUs: uint32(goruntime.NumCPU()), DefaultMemSz: defaultMemSize, DisableBlockDeviceUse: disableBlockDevice, BlockDeviceDriver: defaultBlockDeviceDriver, @@ -513,6 +515,7 @@ func TestMinimalRuntimeConfig(t *testing.T) { InitrdPath: defaultInitrdPath, HypervisorMachineType: defaultMachineType, DefaultVCPUs: defaultVCPUCount, + DefaultMaxVCPUs: defaultMaxVCPUCount, DefaultMemSz: defaultMemSize, DisableBlockDeviceUse: defaultDisableBlockDeviceUse, DefaultBridges: defaultBridgesCount, @@ -658,10 +661,13 @@ func TestNewShimConfig(t *testing.T) { func TestHypervisorDefaults(t *testing.T) { assert := assert.New(t) + numCPUs := goruntime.NumCPU() + h := hypervisor{} assert.Equal(h.machineType(), defaultMachineType, "default hypervisor machine type wrong") assert.Equal(h.defaultVCPUs(), defaultVCPUCount, "default vCPU number is wrong") + assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong") assert.Equal(h.defaultMemSz(), defaultMemSize, "default memory size is wrong") machineType := "foo" @@ -670,15 +676,24 @@ func TestHypervisorDefaults(t *testing.T) { // auto inferring h.DefaultVCPUs = -1 - assert.Equal(h.defaultVCPUs(), uint32(goruntime.NumCPU()), "default vCPU number is wrong") + assert.Equal(h.defaultVCPUs(), uint32(numCPUs), "default vCPU number is wrong") h.DefaultVCPUs = 2 assert.Equal(h.defaultVCPUs(), uint32(2), "default vCPU number is wrong") - numCPUs := goruntime.NumCPU() h.DefaultVCPUs = int32(numCPUs) + 1 assert.Equal(h.defaultVCPUs(), uint32(numCPUs), "default vCPU number is wrong") + h.DefaultMaxVCPUs = 2 + assert.Equal(h.defaultMaxVCPUs(), uint32(h.DefaultMaxVCPUs), "default max vCPU number is wrong") + + h.DefaultMaxVCPUs = uint32(numCPUs) + 1 + assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong") + + maxvcpus := vc.MaxQemuVCPUs() + h.DefaultMaxVCPUs = uint32(maxvcpus) + 1 + assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong") + h.DefaultMemSz = 1024 assert.Equal(h.defaultMemSz(), uint32(1024), "default memory size is wrong") } diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 2e0907ca88..0d6631192f 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -41,7 +41,7 @@ const ( ) // In some architectures the maximum number of vCPUs depends on the number of physical cores. -var defaultMaxQemuVCPUs = maxQemuVCPUs() +var defaultMaxQemuVCPUs = MaxQemuVCPUs() // deviceType describes a virtualized device type. type deviceType int diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 95ae5c8260..b233f0925b 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -122,6 +122,9 @@ func (q *qemu) kernelParameters() string { // use default parameters params = append(params, defaultKernelParameters...) + // set the maximum number of vCPUs + params = append(params, Param{"nr_cpus", fmt.Sprintf("%d", q.config.DefaultMaxVCPUs)}) + // add the params specified by the provided config. As the kernel // honours the last parameter value set and since the config-provided // params are added here, they will take priority over the defaults. @@ -197,7 +200,7 @@ func (q *qemu) init(sandbox *Sandbox) error { } func (q *qemu) cpuTopology() govmmQemu.SMP { - return q.arch.cpuTopology(q.config.DefaultVCPUs) + return q.arch.cpuTopology(q.config.DefaultVCPUs, q.config.DefaultMaxVCPUs) } func (q *qemu) memoryTopology(sandboxConfig SandboxConfig) (govmmQemu.Memory, error) { diff --git a/virtcontainers/qemu_amd64.go b/virtcontainers/qemu_amd64.go index 2e3406a921..6099ade1e7 100644 --- a/virtcontainers/qemu_amd64.go +++ b/virtcontainers/qemu_amd64.go @@ -71,8 +71,8 @@ var supportedQemuMachines = []govmmQemu.Machine{ }, } -// returns the maximum number of vCPUs supported -func maxQemuVCPUs() uint32 { +// MaxQemuVCPUs returns the maximum number of vCPUs supported +func MaxQemuVCPUs() uint32 { return uint32(240) } diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index 28a06d2a00..795c63cc5f 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -42,7 +42,7 @@ type qemuArch interface { bridges(number uint32) []Bridge // cpuTopology returns the CPU topology for the given amount of vcpus - cpuTopology(vcpus uint32) govmmQemu.SMP + cpuTopology(vcpus, maxvcpus uint32) govmmQemu.SMP // cpuModel returns the CPU model for the machine type cpuModel() string @@ -219,13 +219,13 @@ func (q *qemuArchBase) bridges(number uint32) []Bridge { return bridges } -func (q *qemuArchBase) cpuTopology(vcpus uint32) govmmQemu.SMP { +func (q *qemuArchBase) cpuTopology(vcpus, maxvcpus uint32) govmmQemu.SMP { smp := govmmQemu.SMP{ CPUs: vcpus, Sockets: vcpus, Cores: defaultCores, Threads: defaultThreads, - MaxCPUs: defaultMaxQemuVCPUs, + MaxCPUs: maxvcpus, } return smp diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go index 24100d1e84..906d354cc6 100644 --- a/virtcontainers/qemu_arch_base_test.go +++ b/virtcontainers/qemu_arch_base_test.go @@ -169,7 +169,7 @@ func TestQemuArchBaseCPUTopology(t *testing.T) { MaxCPUs: defaultMaxQemuVCPUs, } - smp := qemuArchBase.cpuTopology(vcpus) + smp := qemuArchBase.cpuTopology(vcpus, defaultMaxQemuVCPUs) assert.Equal(expectedSMP, smp) } diff --git a/virtcontainers/qemu_arm64.go b/virtcontainers/qemu_arm64.go index 2b80440b5f..42ae767189 100644 --- a/virtcontainers/qemu_arm64.go +++ b/virtcontainers/qemu_arm64.go @@ -42,8 +42,8 @@ var supportedQemuMachines = []govmmQemu.Machine{ }, } -// returns the maximum number of vCPUs supported -func maxQemuVCPUs() uint32 { +// MaxQemuVCPUs returns the maximum number of vCPUs supported +func MaxQemuVCPUs() uint32 { return uint32(runtime.NumCPU()) } diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index 14751db5e1..4470e8b3ac 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -52,7 +52,7 @@ func testQemuKernelParameters(t *testing.T, kernelParams []Param, expected strin } func TestQemuKernelParameters(t *testing.T) { - expectedOut := "panic=1 initcall_debug foo=foo bar=bar" + expectedOut := fmt.Sprintf("panic=1 initcall_debug nr_cpus=%d foo=foo bar=bar", MaxQemuVCPUs()) params := []Param{ { Key: "foo", @@ -128,7 +128,8 @@ func TestQemuCPUTopology(t *testing.T) { q := &qemu{ arch: &qemuArchBase{}, config: HypervisorConfig{ - DefaultVCPUs: uint32(vcpus), + DefaultVCPUs: uint32(vcpus), + DefaultMaxVCPUs: uint32(vcpus), }, } @@ -137,7 +138,7 @@ func TestQemuCPUTopology(t *testing.T) { Sockets: uint32(vcpus), Cores: defaultCores, Threads: defaultThreads, - MaxCPUs: defaultMaxQemuVCPUs, + MaxCPUs: uint32(vcpus), } smp := q.cpuTopology()