From b28d0274ffba82d5119ec2c96de42e00724972cd Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Wed, 24 Nov 2021 18:48:57 +0100 Subject: [PATCH] virtcontainers: Make max vCPU config less QEMU specific Even though it's still actually defined as the QEMU upper bound, it's now abstracted away through govmm. Signed-off-by: Samuel Ortiz --- src/runtime/pkg/katautils/config.go | 3 ++- src/runtime/pkg/katautils/config_test.go | 3 ++- src/runtime/pkg/oci/utils.go | 5 +++-- src/runtime/virtcontainers/hypervisor.go | 7 ++++--- src/runtime/virtcontainers/hypervisor_test.go | 2 +- src/runtime/virtcontainers/qemu_amd64.go | 5 ----- .../virtcontainers/qemu_arch_base_test.go | 6 +++--- src/runtime/virtcontainers/qemu_arm64.go | 18 ------------------ src/runtime/virtcontainers/qemu_arm64_test.go | 5 +++-- src/runtime/virtcontainers/qemu_ppc64le.go | 5 ----- src/runtime/virtcontainers/qemu_s390x.go | 8 -------- src/runtime/virtcontainers/qemu_test.go | 5 +++-- 12 files changed, 21 insertions(+), 51 deletions(-) diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 06f6a6df02..e699c0c557 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -15,6 +15,7 @@ import ( "strings" "github.com/BurntSushi/toml" + "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci" @@ -343,7 +344,7 @@ func (h hypervisor) defaultVCPUs() uint32 { func (h hypervisor) defaultMaxVCPUs() uint32 { numcpus := uint32(goruntime.NumCPU()) - maxvcpus := vc.MaxQemuVCPUs() + maxvcpus := govmm.MaxVCPUs() reqVCPUs := h.DefaultMaxVCPUs //don't exceed the number of physical CPUs. If a default is not provided, use the diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index 5bfd612507..74a776745a 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -17,6 +17,7 @@ import ( "syscall" "testing" + "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" @@ -917,7 +918,7 @@ func TestHypervisorDefaults(t *testing.T) { h.DefaultMaxVCPUs = numCPUs + 1 assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong") - maxvcpus := vc.MaxQemuVCPUs() + maxvcpus := govmm.MaxVCPUs() h.DefaultMaxVCPUs = maxvcpus + 1 assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong") diff --git a/src/runtime/pkg/oci/utils.go b/src/runtime/pkg/oci/utils.go index a7b3860ef7..66172dcba9 100644 --- a/src/runtime/pkg/oci/utils.go +++ b/src/runtime/pkg/oci/utils.go @@ -23,6 +23,7 @@ import ( "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/resource" + "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" @@ -637,8 +638,8 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e return fmt.Errorf("Number of cpus %d in annotation default_maxvcpus is greater than the number of CPUs %d on the system", max, numCPUs) } - if sbConfig.HypervisorType == vc.QemuHypervisor && max > vc.MaxQemuVCPUs() { - return fmt.Errorf("Number of cpus %d in annotation default_maxvcpus is greater than max no of CPUs %d supported for qemu", max, vc.MaxQemuVCPUs()) + if sbConfig.HypervisorType == vc.QemuHypervisor && max > govmm.MaxVCPUs() { + return fmt.Errorf("Number of cpus %d in annotation default_maxvcpus is greater than max no of CPUs %d supported for qemu", max, govmm.MaxVCPUs()) } sbConfig.HypervisorConfig.DefaultMaxVCPUs = max return nil diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 1753420164..140f0b0069 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" + "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" @@ -76,7 +77,7 @@ var ( ) // In some architectures the maximum number of vCPUs depends on the number of physical cores. -var defaultMaxQemuVCPUs = MaxQemuVCPUs() +var defaultMaxVCPUs = govmm.MaxVCPUs() // agnostic list of kernel root parameters for NVDIMM var commonNvdimmKernelRootParams = []Param{ //nolint: unused, deadcode, varcheck @@ -574,8 +575,8 @@ func (conf *HypervisorConfig) Valid() error { conf.BlockDeviceDriver = config.VirtioBlockCCW } - if conf.DefaultMaxVCPUs == 0 || conf.DefaultMaxVCPUs > defaultMaxQemuVCPUs { - conf.DefaultMaxVCPUs = defaultMaxQemuVCPUs + if conf.DefaultMaxVCPUs == 0 || conf.DefaultMaxVCPUs > defaultMaxVCPUs { + conf.DefaultMaxVCPUs = defaultMaxVCPUs } if conf.Msize9p == 0 && conf.SharedFS != config.VirtioFS { diff --git a/src/runtime/virtcontainers/hypervisor_test.go b/src/runtime/virtcontainers/hypervisor_test.go index f8bbbd330d..86aefde697 100644 --- a/src/runtime/virtcontainers/hypervisor_test.go +++ b/src/runtime/virtcontainers/hypervisor_test.go @@ -185,7 +185,7 @@ func TestHypervisorConfigDefaults(t *testing.T) { MemorySize: defaultMemSzMiB, DefaultBridges: defaultBridges, BlockDeviceDriver: defaultBlockDriver, - DefaultMaxVCPUs: defaultMaxQemuVCPUs, + DefaultMaxVCPUs: defaultMaxVCPUs, Msize9p: defaultMsize9p, } diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go index 7570659615..fe0650dbcc 100644 --- a/src/runtime/virtcontainers/qemu_amd64.go +++ b/src/runtime/virtcontainers/qemu_amd64.go @@ -81,11 +81,6 @@ var supportedQemuMachines = []govmmQemu.Machine{ }, } -// MaxQemuVCPUs returns the maximum number of vCPUs supported -func MaxQemuVCPUs() uint32 { - return uint32(240) -} - func newQemuArch(config HypervisorConfig) (qemuArch, error) { machineType := config.HypervisorMachineType if machineType == "" { diff --git a/src/runtime/virtcontainers/qemu_arch_base_test.go b/src/runtime/virtcontainers/qemu_arch_base_test.go index b4b8086859..5611c95390 100644 --- a/src/runtime/virtcontainers/qemu_arch_base_test.go +++ b/src/runtime/virtcontainers/qemu_arch_base_test.go @@ -173,13 +173,13 @@ func TestQemuArchBaseCPUTopology(t *testing.T) { expectedSMP := govmmQemu.SMP{ CPUs: vcpus, - Sockets: defaultMaxQemuVCPUs, + Sockets: defaultMaxVCPUs, Cores: defaultCores, Threads: defaultThreads, - MaxCPUs: defaultMaxQemuVCPUs, + MaxCPUs: defaultMaxVCPUs, } - smp := qemuArchBase.cpuTopology(vcpus, defaultMaxQemuVCPUs) + smp := qemuArchBase.cpuTopology(vcpus, defaultMaxVCPUs) assert.Equal(expectedSMP, smp) } diff --git a/src/runtime/virtcontainers/qemu_arm64.go b/src/runtime/virtcontainers/qemu_arm64.go index ee07b3241c..250e709f67 100644 --- a/src/runtime/virtcontainers/qemu_arm64.go +++ b/src/runtime/virtcontainers/qemu_arm64.go @@ -29,8 +29,6 @@ const qmpMigrationWaitTimeout = 10 * time.Second const defaultQemuMachineOptions = "usb=off,accel=kvm,gic-version=host" -var defaultGICVersion = uint32(3) - var kernelParams = []Param{ {"console", "hvc0"}, {"console", "hvc1"}, @@ -42,22 +40,6 @@ var supportedQemuMachine = govmmQemu.Machine{ Options: defaultQemuMachineOptions, } -//In qemu, maximum number of vCPUs depends on the GIC version, or on how -//many redistributors we can fit into the memory map. -//related codes are under github.com/qemu/qemu/hw/arm/virt.c(Line 135 and 1306 in stable-2.11) -//for now, qemu only supports v2 and v3, we treat v4 as v3 based on -//backward compatibility. -var gicList = map[uint32]uint32{ - uint32(2): uint32(8), - uint32(3): uint32(123), - uint32(4): uint32(123), -} - -// MaxQemuVCPUs returns the maximum number of vCPUs supported -func MaxQemuVCPUs() uint32 { - return gicList[defaultGICVersion] -} - func newQemuArch(config HypervisorConfig) (qemuArch, error) { machineType := config.HypervisorMachineType if machineType == "" { diff --git a/src/runtime/virtcontainers/qemu_arm64_test.go b/src/runtime/virtcontainers/qemu_arm64_test.go index 2766ae44a7..fb43f00b19 100644 --- a/src/runtime/virtcontainers/qemu_arm64_test.go +++ b/src/runtime/virtcontainers/qemu_arm64_test.go @@ -11,6 +11,7 @@ import ( "os" "testing" + "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" "github.com/stretchr/testify/assert" ) @@ -54,10 +55,10 @@ func TestQemuArm64MemoryTopology(t *testing.T) { assert.Equal(expectedMemory, m) } -func TestMaxQemuVCPUs(t *testing.T) { +func TestMaxVCPUs(t *testing.T) { assert := assert.New(t) - vCPUs := MaxQemuVCPUs() + vCPUs := govmm.MaxVCPUs() assert.Equal(uint32(123), vCPUs) } diff --git a/src/runtime/virtcontainers/qemu_ppc64le.go b/src/runtime/virtcontainers/qemu_ppc64le.go index eef94bde97..a1154fe33f 100644 --- a/src/runtime/virtcontainers/qemu_ppc64le.go +++ b/src/runtime/virtcontainers/qemu_ppc64le.go @@ -54,11 +54,6 @@ func (q *qemuPPC64le) Logger() *logrus.Entry { return hvLogger.WithField("subsystem", "qemuPPC64le") } -// MaxQemuVCPUs returns the maximum number of vCPUs supported -func MaxQemuVCPUs() uint32 { - return uint32(128) -} - func newQemuArch(config HypervisorConfig) (qemuArch, error) { machineType := config.HypervisorMachineType if machineType == "" { diff --git a/src/runtime/virtcontainers/qemu_s390x.go b/src/runtime/virtcontainers/qemu_s390x.go index fa15c759f0..131c683850 100644 --- a/src/runtime/virtcontainers/qemu_s390x.go +++ b/src/runtime/virtcontainers/qemu_s390x.go @@ -47,14 +47,6 @@ var supportedQemuMachine = govmmQemu.Machine{ Options: defaultQemuMachineOptions, } -// MaxQemuVCPUs returns the maximum number of vCPUs supported -func MaxQemuVCPUs() uint32 { - // Max number of virtual Cpu defined in qemu. See - // https://github.com/qemu/qemu/blob/80422b00196a7af4c6efb628fae0ad8b644e98af/target/s390x/cpu.h#L55 - // #define S390_MAX_CPUS 248 - return uint32(248) -} - func newQemuArch(config HypervisorConfig) (qemuArch, error) { machineType := config.HypervisorMachineType if machineType == "" { diff --git a/src/runtime/virtcontainers/qemu_test.go b/src/runtime/virtcontainers/qemu_test.go index 5feecaabed..b1f9aab5e6 100644 --- a/src/runtime/virtcontainers/qemu_test.go +++ b/src/runtime/virtcontainers/qemu_test.go @@ -12,6 +12,7 @@ import ( "path/filepath" "testing" + "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" @@ -30,7 +31,7 @@ func newQemuConfig() HypervisorConfig { MemorySize: defaultMemSzMiB, DefaultBridges: defaultBridges, BlockDeviceDriver: defaultBlockDriver, - DefaultMaxVCPUs: defaultMaxQemuVCPUs, + DefaultMaxVCPUs: defaultMaxVCPUs, Msize9p: defaultMsize9p, } } @@ -54,7 +55,7 @@ func testQemuKernelParameters(t *testing.T, kernelParams []Param, expected strin } func TestQemuKernelParameters(t *testing.T) { - expectedOut := fmt.Sprintf("panic=1 nr_cpus=%d foo=foo bar=bar", MaxQemuVCPUs()) + expectedOut := fmt.Sprintf("panic=1 nr_cpus=%d foo=foo bar=bar", govmm.MaxVCPUs()) params := []Param{ { Key: "foo",