Merge pull request #3615 from sameo/topic/hypervisor

Make the hypervisor framework not Linux specific
This commit is contained in:
Eric Ernst 2022-02-23 16:02:00 -08:00 committed by GitHub
commit c6cc038364
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 253 additions and 143 deletions

View File

@ -0,0 +1,12 @@
//
// Copyright (c) 2018 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
//
package govmm
// MaxVCPUs returns the maximum number of vCPUs supported
func MaxVCPUs() uint32 {
return uint32(240)
}

View File

@ -0,0 +1,25 @@
//
// Copyright (c) 2018 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
//
package govmm
//In qemu, maximum number of vCPUs depends on the GIC version, or on how
//many redistributors we can fit into the memory map.
//related codes are under github.com/qemu/qemu/hw/arm/virt.c(Line 135 and 1306 in stable-2.11)
//for now, qemu only supports v2 and v3, we treat v4 as v3 based on
//backward compatibility.
var gicList = map[uint32]uint32{
uint32(2): uint32(8),
uint32(3): uint32(123),
uint32(4): uint32(123),
}
var defaultGICVersion = uint32(3)
// MaxVCPUs returns the maximum number of vCPUs supported
func MaxVCPUs() uint32 {
return gicList[defaultGICVersion]
}

View File

@ -0,0 +1,12 @@
//
// Copyright (c) 2018 IBM
//
// SPDX-License-Identifier: Apache-2.0
//
package qemu
// MaxVCPUs returns the maximum number of vCPUs supported
func MaxVCPUs() uint32 {
return uint32(128)
}

View File

@ -0,0 +1,15 @@
//
// Copyright (c) 2018 IBM
//
// SPDX-License-Identifier: Apache-2.0
//
package govmm
// MaxVCPUs returns the maximum number of vCPUs supported
func MaxVCPUs() uint32 {
// Max number of virtual Cpu defined in qemu. See
// https://github.com/qemu/qemu/blob/80422b00196a7af4c6efb628fae0ad8b644e98af/target/s390x/cpu.h#L55
// #define S390_MAX_CPUS 248
return uint32(248)
}

View File

@ -15,6 +15,7 @@ import (
"strings"
"github.com/BurntSushi/toml"
"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
@ -343,7 +344,7 @@ func (h hypervisor) defaultVCPUs() uint32 {
func (h hypervisor) defaultMaxVCPUs() uint32 {
numcpus := uint32(goruntime.NumCPU())
maxvcpus := vc.MaxQemuVCPUs()
maxvcpus := govmm.MaxVCPUs()
reqVCPUs := h.DefaultMaxVCPUs
//don't exceed the number of physical CPUs. If a default is not provided, use the

View File

@ -17,6 +17,7 @@ import (
"syscall"
"testing"
"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
@ -919,7 +920,7 @@ func TestHypervisorDefaults(t *testing.T) {
h.DefaultMaxVCPUs = numCPUs + 1
assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong")
maxvcpus := vc.MaxQemuVCPUs()
maxvcpus := govmm.MaxVCPUs()
h.DefaultMaxVCPUs = maxvcpus + 1
assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong")

View File

@ -23,6 +23,7 @@ import (
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/api/resource"
"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
@ -637,8 +638,8 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
return fmt.Errorf("Number of cpus %d in annotation default_maxvcpus is greater than the number of CPUs %d on the system", max, numCPUs)
}
if sbConfig.HypervisorType == vc.QemuHypervisor && max > vc.MaxQemuVCPUs() {
return fmt.Errorf("Number of cpus %d in annotation default_maxvcpus is greater than max no of CPUs %d supported for qemu", max, vc.MaxQemuVCPUs())
if sbConfig.HypervisorType == vc.QemuHypervisor && max > govmm.MaxVCPUs() {
return fmt.Errorf("Number of cpus %d in annotation default_maxvcpus is greater than max no of CPUs %d supported for qemu", max, govmm.MaxVCPUs())
}
sbConfig.HypervisorConfig.DefaultMaxVCPUs = max
return nil

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2019 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2019 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2019 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2019 Ericsson Eurolab Deutschland GmbH
//
// SPDX-License-Identifier: Apache-2.0

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2019 Ericsson Eurolab Deutschland G.m.b.H.
//
// SPDX-License-Identifier: Apache-2.0

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2018 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2020 Ant Group
//
// SPDX-License-Identifier: Apache-2.0

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2019 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0

View File

@ -14,10 +14,12 @@ import (
"strconv"
"strings"
"github.com/pkg/errors"
"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
"github.com/sirupsen/logrus"
)
@ -69,14 +71,17 @@ const (
// MinHypervisorMemory is the minimum memory required for a VM.
MinHypervisorMemory = 256
defaultMsize9p = 8192
)
var (
hvLogger = logrus.WithField("source", "virtcontainers/hypervisor")
hvLogger = logrus.WithField("source", "virtcontainers/hypervisor")
noGuestMemHotplugErr error = errors.New("guest memory hotplug not supported")
)
// In some architectures the maximum number of vCPUs depends on the number of physical cores.
var defaultMaxQemuVCPUs = MaxQemuVCPUs()
var defaultMaxVCPUs = govmm.MaxVCPUs()
// agnostic list of kernel root parameters for NVDIMM
var commonNvdimmKernelRootParams = []Param{ //nolint: unused, deadcode, varcheck
@ -192,25 +197,6 @@ func (hType *HypervisorType) String() string {
}
}
// NewHypervisor returns an hypervisor from a hypervisor type.
func NewHypervisor(hType HypervisorType) (Hypervisor, error) {
switch hType {
case QemuHypervisor:
return &qemu{}, nil
case FirecrackerHypervisor:
return &firecracker{}, nil
case AcrnHypervisor:
return &Acrn{}, nil
case ClhHypervisor:
return &cloudHypervisor{}, nil
case MockHypervisor:
return &mockHypervisor{}, nil
default:
return nil, fmt.Errorf("Unknown hypervisor type %s", hType)
}
}
// GetHypervisorSocketTemplate returns the full "template" path to the
// hypervisor socket. If the specified hypervisor doesn't use a socket,
// an empty string is returned.
@ -574,8 +560,8 @@ func (conf *HypervisorConfig) Valid() error {
conf.BlockDeviceDriver = config.VirtioBlockCCW
}
if conf.DefaultMaxVCPUs == 0 || conf.DefaultMaxVCPUs > defaultMaxQemuVCPUs {
conf.DefaultMaxVCPUs = defaultMaxQemuVCPUs
if conf.DefaultMaxVCPUs == 0 || conf.DefaultMaxVCPUs > defaultMaxVCPUs {
conf.DefaultMaxVCPUs = defaultMaxVCPUs
}
if conf.Msize9p == 0 && conf.SharedFS != config.VirtioFS {
@ -894,17 +880,55 @@ func GetHypervisorPid(h Hypervisor) int {
return pids[0]
}
func generateVMSocket(id string, vmStogarePath string) (interface{}, error) {
vhostFd, contextID, err := utils.FindContextID()
if err != nil {
return nil, err
}
// Kind of guest protection
type guestProtection uint8
return types.VSock{
VhostFd: vhostFd,
ContextID: contextID,
Port: uint32(vSockPort),
}, nil
const (
noneProtection guestProtection = iota
//Intel Trust Domain Extensions
//https://software.intel.com/content/www/us/en/develop/articles/intel-trust-domain-extensions.html
// Exclude from lint checking for it won't be used on arm64 code
tdxProtection
// AMD Secure Encrypted Virtualization
// https://developer.amd.com/sev/
// Exclude from lint checking for it won't be used on arm64 code
sevProtection
// IBM POWER 9 Protected Execution Facility
// https://www.kernel.org/doc/html/latest/powerpc/ultravisor.html
// Exclude from lint checking for it won't be used on arm64 code
pefProtection
// IBM Secure Execution (IBM Z & LinuxONE)
// https://www.kernel.org/doc/html/latest/virt/kvm/s390-pv.html
// Exclude from lint checking for it won't be used on arm64 code
seProtection
)
var guestProtectionStr = [...]string{
noneProtection: "none",
pefProtection: "pef",
seProtection: "se",
sevProtection: "sev",
tdxProtection: "tdx",
}
func (gp guestProtection) String() string {
return guestProtectionStr[gp]
}
func genericAvailableGuestProtections() (protections []string) {
return
}
func AvailableGuestProtections() (protections []string) {
gp, err := availableGuestProtection()
if err != nil || gp == noneProtection {
return genericAvailableGuestProtections()
}
return []string{gp.String()}
}
// hypervisor is the virtcontainers hypervisor interface.

View File

@ -0,0 +1,45 @@
// Copyright (c) 2021 Apple Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
package virtcontainers
import (
"fmt"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
)
func generateVMSocket(id string, vmStogarePath string) (interface{}, error) {
vhostFd, contextID, err := utils.FindContextID()
if err != nil {
return nil, err
}
return types.VSock{
VhostFd: vhostFd,
ContextID: contextID,
Port: uint32(vSockPort),
}, nil
}
// NewHypervisor returns an hypervisor from a hypervisor type.
func NewHypervisor(hType HypervisorType) (Hypervisor, error) {
switch hType {
case QemuHypervisor:
return &qemu{}, nil
case FirecrackerHypervisor:
return &firecracker{}, nil
case AcrnHypervisor:
return &Acrn{}, nil
case ClhHypervisor:
return &cloudHypervisor{}, nil
case MockHypervisor:
return &mockHypervisor{}, nil
default:
return nil, fmt.Errorf("Unknown hypervisor type %s", hType)
}
}

View File

@ -7,6 +7,14 @@ package virtcontainers
import "os"
const (
tdxSysFirmwareDir = "/sys/firmware/tdx_seam/"
tdxCPUFlag = "tdx"
sevKvmParameterPath = "/sys/module/kvm_amd/parameters/sev"
)
// Implementation of this function is architecture specific
func availableGuestProtection() (guestProtection, error) {
flags, err := CPUFlags(procCPUInfo)

View File

@ -4,7 +4,7 @@
package virtcontainers
//Returns pefProtection if the firmware directory exists
// Guest protection is not supported on ARM64.
func availableGuestProtection() (guestProtection, error) {
return noneProtection, nil
}

View File

@ -185,7 +185,7 @@ func TestHypervisorConfigDefaults(t *testing.T) {
MemorySize: defaultMemSzMiB,
DefaultBridges: defaultBridges,
BlockDeviceDriver: defaultBlockDriver,
DefaultMaxVCPUs: defaultMaxQemuVCPUs,
DefaultMaxVCPUs: defaultMaxVCPUs,
Msize9p: defaultMsize9p,
}

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2016 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
@ -128,8 +131,6 @@ const (
qemuStopSandboxTimeoutSecs = 15
)
var noGuestMemHotplugErr error = errors.New("guest memory hotplug not supported")
// agnostic list of kernel parameters
var defaultKernelParameters = []Param{
{"panic", "1"},

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2018 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
@ -36,12 +39,6 @@ const (
defaultQemuMachineOptions = "accel=kvm,kernel_irqchip=on"
qmpMigrationWaitTimeout = 5 * time.Second
tdxSysFirmwareDir = "/sys/firmware/tdx_seam/"
tdxCPUFlag = "tdx"
sevKvmParameterPath = "/sys/module/kvm_amd/parameters/sev"
)
var qemuPaths = map[string]string{
@ -81,11 +78,6 @@ var supportedQemuMachines = []govmmQemu.Machine{
},
}
// MaxQemuVCPUs returns the maximum number of vCPUs supported
func MaxQemuVCPUs() uint32 {
return uint32(240)
}
func newQemuArch(config HypervisorConfig) (qemuArch, error) {
machineType := config.HypervisorMachineType
if machineType == "" {

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2018 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2018 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
@ -150,57 +153,6 @@ type qemuArch interface {
appendProtectionDevice(devices []govmmQemu.Device, firmware, firmwareVolume string) ([]govmmQemu.Device, string, error)
}
// Kind of guest protection
type guestProtection uint8
const (
noneProtection guestProtection = iota
//Intel Trust Domain Extensions
//https://software.intel.com/content/www/us/en/develop/articles/intel-trust-domain-extensions.html
// Exclude from lint checking for it won't be used on arm64 code
tdxProtection
// AMD Secure Encrypted Virtualization
// https://developer.amd.com/sev/
// Exclude from lint checking for it won't be used on arm64 code
sevProtection
// IBM POWER 9 Protected Execution Facility
// https://www.kernel.org/doc/html/latest/powerpc/ultravisor.html
// Exclude from lint checking for it won't be used on arm64 code
pefProtection
// IBM Secure Execution (IBM Z & LinuxONE)
// https://www.kernel.org/doc/html/latest/virt/kvm/s390-pv.html
// Exclude from lint checking for it won't be used on arm64 code
seProtection
)
var guestProtectionStr = [...]string{
noneProtection: "none",
pefProtection: "pef",
seProtection: "se",
sevProtection: "sev",
tdxProtection: "tdx",
}
func (gp guestProtection) String() string {
return guestProtectionStr[gp]
}
func genericAvailableGuestProtections() (protections []string) {
return
}
func AvailableGuestProtections() (protections []string) {
gp, err := availableGuestProtection()
if err != nil || gp == noneProtection {
return genericAvailableGuestProtections()
}
return []string{gp.String()}
}
type qemuArchBase struct {
qemuExePath string
qemuMachine govmmQemu.Machine
@ -226,7 +178,6 @@ const (
defaultBridgeBus = "pcie.0"
defaultPCBridgeBus = "pci.0"
maxDevIDSize = 31
defaultMsize9p = 8192
pcieRootPortPrefix = "rp"
)

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2018 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
@ -173,13 +176,13 @@ func TestQemuArchBaseCPUTopology(t *testing.T) {
expectedSMP := govmmQemu.SMP{
CPUs: vcpus,
Sockets: defaultMaxQemuVCPUs,
Sockets: defaultMaxVCPUs,
Cores: defaultCores,
Threads: defaultThreads,
MaxCPUs: defaultMaxQemuVCPUs,
MaxCPUs: defaultMaxVCPUs,
}
smp := qemuArchBase.cpuTopology(vcpus, defaultMaxQemuVCPUs)
smp := qemuArchBase.cpuTopology(vcpus, defaultMaxVCPUs)
assert.Equal(expectedSMP, smp)
}

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2018 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
@ -29,8 +32,6 @@ const qmpMigrationWaitTimeout = 10 * time.Second
const defaultQemuMachineOptions = "usb=off,accel=kvm,gic-version=host"
var defaultGICVersion = uint32(3)
var kernelParams = []Param{
{"console", "hvc0"},
{"console", "hvc1"},
@ -42,22 +43,6 @@ var supportedQemuMachine = govmmQemu.Machine{
Options: defaultQemuMachineOptions,
}
//In qemu, maximum number of vCPUs depends on the GIC version, or on how
//many redistributors we can fit into the memory map.
//related codes are under github.com/qemu/qemu/hw/arm/virt.c(Line 135 and 1306 in stable-2.11)
//for now, qemu only supports v2 and v3, we treat v4 as v3 based on
//backward compatibility.
var gicList = map[uint32]uint32{
uint32(2): uint32(8),
uint32(3): uint32(123),
uint32(4): uint32(123),
}
// MaxQemuVCPUs returns the maximum number of vCPUs supported
func MaxQemuVCPUs() uint32 {
return gicList[defaultGICVersion]
}
func newQemuArch(config HypervisorConfig) (qemuArch, error) {
machineType := config.HypervisorMachineType
if machineType == "" {

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2018 IBM
//
// SPDX-License-Identifier: Apache-2.0
@ -11,6 +14,7 @@ import (
"os"
"testing"
"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
"github.com/stretchr/testify/assert"
)
@ -54,10 +58,10 @@ func TestQemuArm64MemoryTopology(t *testing.T) {
assert.Equal(expectedMemory, m)
}
func TestMaxQemuVCPUs(t *testing.T) {
func TestMaxVCPUs(t *testing.T) {
assert := assert.New(t)
vCPUs := MaxQemuVCPUs()
vCPUs := govmm.MaxVCPUs()
assert.Equal(uint32(123), vCPUs)
}

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2018 IBM
//
// SPDX-License-Identifier: Apache-2.0
@ -54,11 +57,6 @@ func (q *qemuPPC64le) Logger() *logrus.Entry {
return hvLogger.WithField("subsystem", "qemuPPC64le")
}
// MaxQemuVCPUs returns the maximum number of vCPUs supported
func MaxQemuVCPUs() uint32 {
return uint32(128)
}
func newQemuArch(config HypervisorConfig) (qemuArch, error) {
machineType := config.HypervisorMachineType
if machineType == "" {

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2018 IBM
//
// SPDX-License-Identifier: Apache-2.0

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2018 IBM
//
// SPDX-License-Identifier: Apache-2.0
@ -47,14 +50,6 @@ var supportedQemuMachine = govmmQemu.Machine{
Options: defaultQemuMachineOptions,
}
// MaxQemuVCPUs returns the maximum number of vCPUs supported
func MaxQemuVCPUs() uint32 {
// Max number of virtual Cpu defined in qemu. See
// https://github.com/qemu/qemu/blob/80422b00196a7af4c6efb628fae0ad8b644e98af/target/s390x/cpu.h#L55
// #define S390_MAX_CPUS 248
return uint32(248)
}
func newQemuArch(config HypervisorConfig) (qemuArch, error) {
machineType := config.HypervisorMachineType
if machineType == "" {

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2018 IBM
//
// SPDX-License-Identifier: Apache-2.0

View File

@ -1,3 +1,6 @@
//go:build linux
// +build linux
//
// Copyright (c) 2016 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
@ -12,6 +15,7 @@ import (
"path/filepath"
"testing"
"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist"
@ -30,7 +34,7 @@ func newQemuConfig() HypervisorConfig {
MemorySize: defaultMemSzMiB,
DefaultBridges: defaultBridges,
BlockDeviceDriver: defaultBlockDriver,
DefaultMaxVCPUs: defaultMaxQemuVCPUs,
DefaultMaxVCPUs: defaultMaxVCPUs,
Msize9p: defaultMsize9p,
}
}
@ -54,7 +58,7 @@ func testQemuKernelParameters(t *testing.T, kernelParams []Param, expected strin
}
func TestQemuKernelParameters(t *testing.T) {
expectedOut := fmt.Sprintf("panic=1 nr_cpus=%d foo=foo bar=bar", MaxQemuVCPUs())
expectedOut := fmt.Sprintf("panic=1 nr_cpus=%d foo=foo bar=bar", govmm.MaxVCPUs())
params := []Param{
{
Key: "foo",