diff --git a/Makefile b/Makefile index 800ab89841..6caced7c3a 100644 --- a/Makefile +++ b/Makefile @@ -81,6 +81,7 @@ BINDIR := $(EXEC_PREFIX)/bin NEMUBINDIR := $(PREFIXDEPS)/bin QEMUBINDIR := $(PREFIXDEPS)/bin FCBINDIR := $(PREFIXDEPS)/bin +ACRNBINDIR := $(PREFIXDEPS)/bin VIRTIOFSDBINDIR := $(PREFIXDEPS)/bin SYSCONFDIR := /etc LOCALSTATEDIR := /var @@ -113,6 +114,7 @@ FIRMWAREPATH := # Name of default configuration file the runtime will use. CONFIG_FILE = configuration.toml +HYPERVISOR_ACRN = acrn HYPERVISOR_FC = firecracker HYPERVISOR_NEMU = nemu HYPERVISOR_QEMU = qemu @@ -121,7 +123,7 @@ HYPERVISOR_QEMU = qemu DEFAULT_HYPERVISOR = $(HYPERVISOR_QEMU) # List of hypervisors this build system can generate configuration for. -HYPERVISORS := $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_NEMU) +HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_NEMU) QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD) @@ -129,6 +131,9 @@ NEMUPATH := $(NEMUBINDIR)/$(NEMUCMD) FCPATH = $(FCBINDIR)/$(FCCMD) +ACRNPATH := $(ACRNBINDIR)/$(ACRNCMD) +ACRNCTLPATH := $(ACRNBINDIR)/$(ACRNCTLCMD) + SHIMCMD := $(BIN_PREFIX)-shim SHIMPATH := $(PKGLIBEXECDIR)/$(SHIMCMD) @@ -198,7 +203,7 @@ VERSION := ${shell cat ./VERSION} # List of configuration files to build and install CONFIGS = -CONFIG_PATHS = +CONFIG_PATHS = SYSCONFIG_PATHS = # List of hypervisors known for the current architecture @@ -276,6 +281,28 @@ ifneq (,$(FCCMD)) KERNELPATH_FC = $(KERNELDIR)/$(KERNEL_NAME_FC) endif +ifneq (,$(ACRNCMD)) + KNOWN_HYPERVISORS += $(HYPERVISOR_ACRN) + + CONFIG_FILE_ACRN = configuration-acrn.toml + CONFIG_ACRN = $(CLI_DIR)/config/$(CONFIG_FILE_ACRN) + CONFIG_ACRN_IN = $(CONFIG_ACRN).in + + CONFIG_PATH_ACRN = $(abspath $(CONFDIR)/$(CONFIG_FILE_ACRN)) + CONFIG_PATHS += $(CONFIG_PATH_ACRN) + + SYSCONFIG_ACRN = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_ACRN)) + SYSCONFIG_PATHS += $(SYSCONFIG_ACRN) + + CONFIGS += $(CONFIG_ACRN) + + # acrn-specific options (all should be suffixed by "_ACRN") + DEFBLOCKSTORAGEDRIVER_ACRN := virtio-blk + DEFNETWORKMODEL_ACRN := bridged + KERNEL_NAME_ACRN = $(call MAKE_KERNEL_NAME,$(KERNELTYPE)) + KERNELPATH_ACRN = $(KERNELDIR)/$(KERNEL_NAME_ACRN) +endif + ifeq (,$(KNOWN_HYPERVISORS)) $(error "ERROR: No hypervisors known for architecture $(ARCH) (looked for: $(HYPERVISORS))") endif @@ -300,6 +327,10 @@ ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_NEMU)) DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_NEMU) endif +ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_ACRN)) + DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_ACRN) +endif + CONFDIR := $(DEFAULTSDIR)/$(PROJECT_DIR) SYSCONFDIR := $(SYSCONFDIR)/$(PROJECT_DIR) @@ -318,6 +349,10 @@ USER_VARS += BINDIR USER_VARS += CONFIG_PATH USER_VARS += DESTDIR USER_VARS += DEFAULT_HYPERVISOR +USER_VARS += ACRNCMD +USER_VARS += ACRNCTLCMD +USER_VARS += ACRNPATH +USER_VARS += ACRNCTLPATH USER_VARS += FCCMD USER_VARS += FCPATH USER_VARS += NEMUCMD @@ -331,6 +366,7 @@ USER_VARS += MACHINETYPE USER_VARS += KERNELDIR USER_VARS += KERNELTYPE USER_VARS += KERNELTYPE_FC +USER_VARS += KERNELTYPE_ACRN USER_VARS += FIRMWAREPATH USER_VARS += FIRMWAREPATH_NEMU USER_VARS += MACHINEACCELERATORS @@ -359,12 +395,14 @@ USER_VARS += DEFMAXVCPUS USER_VARS += DEFMEMSZ USER_VARS += DEFMEMSLOTS USER_VARS += DEFBRIDGES +USER_VARS += DEFNETWORKMODEL_ACRN USER_VARS += DEFNETWORKMODEL_FC USER_VARS += DEFNETWORKMODEL_QEMU USER_VARS += DEFNETWORKMODEL_NEMU USER_VARS += DEFDISABLEGUESTSECCOMP USER_VARS += DEFAULTEXPFEATURES USER_VARS += DEFDISABLEBLOCK +USER_VARS += DEFBLOCKSTORAGEDRIVER_ACRN USER_VARS += DEFBLOCKSTORAGEDRIVER_FC USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU USER_VARS += DEFBLOCKSTORAGEDRIVER_NEMU @@ -472,14 +510,18 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit $(QUIET_GENERATE)$(SED) \ -e "s|@COMMIT@|$(shell cat .git-commit)|g" \ -e "s|@VERSION@|$(VERSION)|g" \ + -e "s|@CONFIG_ACRN_IN@|$(CONFIG_ACRN_IN)|g" \ -e "s|@CONFIG_QEMU_IN@|$(CONFIG_QEMU_IN)|g" \ -e "s|@CONFIG_NEMU_IN@|$(CONFIG_NEMU_IN)|g" \ -e "s|@CONFIG_FC_IN@|$(CONFIG_FC_IN)|g" \ -e "s|@CONFIG_PATH@|$(CONFIG_PATH)|g" \ -e "s|@FCPATH@|$(FCPATH)|g" \ -e "s|@NEMUPATH@|$(NEMUPATH)|g" \ + -e "s|@ACRNPATH@|$(ACRNPATH)|g" \ + -e "s|@ACRNCTLPATH@|$(ACRNCTLPATH)|g" \ -e "s|@SYSCONFIG@|$(SYSCONFIG)|g" \ -e "s|@IMAGEPATH@|$(IMAGEPATH)|g" \ + -e "s|@KERNELPATH_ACRN@|$(KERNELPATH_ACRN)|g" \ -e "s|@KERNELPATH_FC@|$(KERNELPATH_FC)|g" \ -e "s|@KERNELPATH@|$(KERNELPATH)|g" \ -e "s|@INITRDPATH@|$(INITRDPATH)|g" \ @@ -507,12 +549,14 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@DEFMEMSZ@|$(DEFMEMSZ)|g" \ -e "s|@DEFMEMSLOTS@|$(DEFMEMSLOTS)|g" \ -e "s|@DEFBRIDGES@|$(DEFBRIDGES)|g" \ + -e "s|@DEFNETWORKMODEL_ACRN@|$(DEFNETWORKMODEL_ACRN)|g" \ -e "s|@DEFNETWORKMODEL_FC@|$(DEFNETWORKMODEL_FC)|g" \ -e "s|@DEFNETWORKMODEL_QEMU@|$(DEFNETWORKMODEL_QEMU)|g" \ -e "s|@DEFNETWORKMODEL_NEMU@|$(DEFNETWORKMODEL_NEMU)|g" \ -e "s|@DEFDISABLEGUESTSECCOMP@|$(DEFDISABLEGUESTSECCOMP)|g" \ -e "s|@DEFAULTEXPFEATURES@|$(DEFAULTEXPFEATURES)|g" \ -e "s|@DEFDISABLEBLOCK@|$(DEFDISABLEBLOCK)|g" \ + -e "s|@DEFBLOCKSTORAGEDRIVER_ACRN@|$(DEFBLOCKSTORAGEDRIVER_ACRN)|g" \ -e "s|@DEFBLOCKSTORAGEDRIVER_FC@|$(DEFBLOCKSTORAGEDRIVER_FC)|g" \ -e "s|@DEFBLOCKSTORAGEDRIVER_QEMU@|$(DEFBLOCKSTORAGEDRIVER_QEMU)|g" \ -e "s|@DEFBLOCKSTORAGEDRIVER_NEMU@|$(DEFBLOCKSTORAGEDRIVER_NEMU)|g" \ @@ -680,6 +724,9 @@ ifneq (,$(findstring $(HYPERVISOR_NEMU),$(KNOWN_HYPERVISORS))) endif ifneq (,$(findstring $(HYPERVISOR_FC),$(KNOWN_HYPERVISORS))) @printf "\t$(HYPERVISOR_FC) hypervisor path (FCPATH) : %s\n" $(abspath $(FCPATH)) +endif +ifneq (,$(findstring $(HYPERVISOR_ACRN),$(KNOWN_HYPERVISORS))) + @printf "\t$(HYPERVISOR_ACRN) hypervisor path (ACRNPATH) : %s\n" $(abspath $(ACRNPATH)) endif @printf "\tassets path (PKGDATADIR) : %s\n" $(abspath $(PKGDATADIR)) @printf "\tproxy+shim path (PKGLIBEXECDIR) : %s\n" $(abspath $(PKGLIBEXECDIR)) diff --git a/arch/amd64-options.mk b/arch/amd64-options.mk index 78db6662c6..a038b71e4f 100644 --- a/arch/amd64-options.mk +++ b/arch/amd64-options.mk @@ -16,3 +16,7 @@ FCCMD := firecracker # NEMU binary name NEMUCMD := nemu-system-x86_64 + +#ACRN binary name +ACRNCMD := acrn-dm +ACRNCTLCMD := acrnctl diff --git a/cli/config/configuration-acrn.toml.in b/cli/config/configuration-acrn.toml.in new file mode 100644 index 0000000000..cbb2cc1740 --- /dev/null +++ b/cli/config/configuration-acrn.toml.in @@ -0,0 +1,235 @@ +# Copyright (c) 2017-2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +# XXX: WARNING: this file is auto-generated. +# XXX: +# XXX: Source file: "@CONFIG_ACRN_IN@" +# XXX: Project: +# XXX: Name: @PROJECT_NAME@ +# XXX: Type: @PROJECT_TYPE@ + +[hypervisor.acrn] +path = "@ACRNPATH@" +ctlpath = "@ACRNCTLPATH@" +kernel = "@KERNELPATH_ACRN@" +image = "@IMAGEPATH@" + +# Optional space-separated list of options to pass to the guest kernel. +# For example, use `kernel_params = "vsyscall=emulate"` if you are having +# trouble running pre-2.15 glibc. +# +# WARNING: - any parameter specified here will take priority over the default +# parameter value of the same name used to start the virtual machine. +# Do not set values here unless you understand the impact of doing so as you +# may stop the virtual machine from booting. +# To see the list of default parameters, enable hypervisor debug, create a +# container and look for 'default-kernel-parameters' log entries. +kernel_params = "@KERNELPARAMS@" + +# Path to the firmware. +# If you want that acrn uses the default firmware leave this option empty +firmware = "@FIRMWAREPATH@" + +# Default number of vCPUs per SB/VM: +# unspecified or 0 --> will be set to @DEFVCPUS@ +# < 0 --> will be set to the actual number of physical cores +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores +default_vcpus = 1 + +# Default maximum number of vCPUs per SB/VM: +# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when +# the actual number of physical cores is greater than it. +# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU +# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs +# can be added to a SB/VM, but the memory footprint will be big. Another example, with +# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of +# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable, +# unless you know what are you doing. +default_maxvcpus = @DEFMAXVCPUS@ + +# Bridges can be used to hot plug devices. +# Limitations: +# * Currently only pci bridges are supported +# * Until 30 devices per bridge can be hot plugged. +# * Until 5 PCI bridges can be cold plugged per VM. +# This limitation could be a bug in the kernel +# Default number of bridges per SB/VM: +# unspecified or 0 --> will be set to @DEFBRIDGES@ +# > 1 <= 5 --> will be set to the specified number +# > 5 --> will be set to 5 +default_bridges = @DEFBRIDGES@ + +# Default memory size in MiB for SB/VM. +# If unspecified then it will be set @DEFMEMSZ@ MiB. +default_memory = @DEFMEMSZ@ + +# Block storage driver to be used for the hypervisor in case the container +# rootfs is backed by a block device. ACRN only supports virtio-blk. +block_device_driver = "@DEFBLOCKSTORAGEDRIVER_ACRN@" + +# This option changes the default hypervisor and kernel parameters +# to enable debug output where available. This extra output is added +# to the proxy logs, but only when proxy debug is also enabled. +# +# Default false +#enable_debug = true + +# Disable the customizations done in the runtime when it detects +# that it is running on top a VMM. This will result in the runtime +# behaving as it would when running on bare metal. +# +#disable_nesting_checks = true + +# If host doesn't support vhost_net, set to true. Thus we won't create vhost fds for nics. +# Default false +#disable_vhost_net = true + +# Path to OCI hook binaries in the *guest rootfs*. +# This does not affect host-side hooks which must instead be added to +# the OCI spec passed to the runtime. +# +# You can create a rootfs with hooks by customizing the osbuilder scripts: +# https://github.com/kata-containers/osbuilder +# +# Hooks must be stored in a subdirectory of guest_hook_path according to their +# hook type, i.e. "guest_hook_path/{prestart,postart,poststop}". +# The agent will scan these directories for executable files and add them, in +# lexicographical order, to the lifecycle of the guest container. +# Hooks are executed in the runtime namespace of the guest. See the official documentation: +# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks +# Warnings will be logged if any error is encountered will scanning for hooks, +# but it will not abort container execution. +#guest_hook_path = "/usr/share/oci/hooks" + +[proxy.@PROJECT_TYPE@] +path = "@PROXYPATH@" + +# If enabled, proxy messages will be sent to the system log +# (default: disabled) +#enable_debug = true + +[shim.@PROJECT_TYPE@] +path = "@SHIMPATH@" + +# If enabled, shim messages will be sent to the system log +# (default: disabled) +#enable_debug = true + +# If enabled, the shim will create opentracing.io traces and spans. +# (See https://www.jaegertracing.io/docs/getting-started). +# +# Note: By default, the shim runs in a separate network namespace. Therefore, +# to allow it to send trace details to the Jaeger agent running on the host, +# it is necessary to set 'disable_new_netns=true' so that it runs in the host +# network namespace. +# +# (default: disabled) +#enable_tracing = true + +[agent.@PROJECT_TYPE@] +# If enabled, make the agent display debug-level messages. +# (default: disabled) +#enable_debug = true + +# Enable agent tracing. +# +# If enabled, the default trace mode is "dynamic" and the +# default trace type is "isolated". The trace mode and type are set +# explicity with the `trace_type=` and `trace_mode=` options. +# +# Notes: +# +# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly +# setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing` +# will NOT activate agent tracing. +# +# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for +# full details. +# +# (default: disabled) +#enable_tracing = true +# +#trace_mode = "dynamic" +#trace_type = "isolated" + +[netmon] +# If enabled, the network monitoring process gets started when the +# sandbox is created. This allows for the detection of some additional +# network being added to the existing network namespace, after the +# sandbox has been created. +# (default: disabled) +#enable_netmon = true + +# Specify the path to the netmon binary. +path = "@NETMONPATH@" + +# If enabled, netmon messages will be sent to the system log +# (default: disabled) +#enable_debug = true + +[runtime] +# If enabled, the runtime will log additional debug messages to the +# system log +# (default: disabled) +#enable_debug = true +# +# Internetworking model +# Determines how the VM should be connected to the +# the container network interface +# Options: +# +# - bridged +# Uses a linux bridge to interconnect the container interface to +# the VM. Works for most cases except macvlan and ipvlan. +# +# - macvtap +# Used when the Container network interface can be bridged using +# macvtap. +# +# - none +# Used when customize network. Only creates a tap device. No veth pair. +# +# - tcfilter +# Uses tc filter rules to redirect traffic from the network interface +# provided by plugin to a tap interface connected to the VM. +# +internetworking_model="@DEFNETWORKMODEL_ACRN@" + +# disable guest seccomp +# Determines whether container seccomp profiles are passed to the virtual +# machine and applied by the kata agent. If set to true, seccomp is not applied +# within the guest +# (default: true) +disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ + +# If enabled, the runtime will create opentracing.io traces and spans. +# (See https://www.jaegertracing.io/docs/getting-started). +# (default: disabled) +#enable_tracing = true + +# If enabled, the runtime will not create a network namespace for shim and hypervisor processes. +# This option may have some potential impacts to your host. It should only be used when you know what you're doing. +# `disable_new_netns` conflicts with `enable_netmon` +# `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only +# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge +# (like OVS) directly. +# If you are using docker, `disable_new_netns` only works with `docker run --net=none` +# (default: false) +#disable_new_netns = true + +# Enabled experimental feature list, format: ["a", "b"]. +# Experimental features are features not stable enough for production, +# They may break compatibility, and are prepared for a big version bump. +# Supported experimental features: +# 1. "newstore": new persist storage driver which breaks backward compatibility, +# expected to move out of experimental in 2.0.0. +# (default: []) +experimental=@DEFAULTEXPFEATURES@ diff --git a/cli/kata-check.go b/cli/kata-check.go index 28c23b6dd7..068c743738 100644 --- a/cli/kata-check.go +++ b/cli/kata-check.go @@ -16,6 +16,7 @@ const int ioctl_KVM_CHECK_EXTENSION = KVM_CHECK_EXTENSION; import "C" import ( + "errors" "fmt" "os" "os/exec" @@ -26,6 +27,7 @@ import ( "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/pkg/oci" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) @@ -311,7 +313,12 @@ var kataCheckCLICommand = cli.Command{ span, _ := katautils.Trace(ctx, "kata-check") defer span.Finish() - err = setCPUtype() + runtimeConfig, ok := context.App.Metadata["runtimeConfig"].(oci.RuntimeConfig) + if !ok { + return errors.New("kata-check: cannot determine runtime config") + } + + err = setCPUtype(runtimeConfig.HypervisorType) if err != nil { return err } @@ -332,7 +339,7 @@ var kataCheckCLICommand = cli.Command{ kataLog.Info(successMessageCapable) if os.Geteuid() == 0 { - err = archHostCanCreateVMContainer() + err = archHostCanCreateVMContainer(runtimeConfig.HypervisorType) if err != nil { return err } diff --git a/cli/kata-check_amd64.go b/cli/kata-check_amd64.go index 21e210e891..a36a3ec16a 100644 --- a/cli/kata-check_amd64.go +++ b/cli/kata-check_amd64.go @@ -7,11 +7,13 @@ package main import ( "fmt" - "github.com/sirupsen/logrus" "io/ioutil" "strings" + "syscall" + "unsafe" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/sirupsen/logrus" ) const ( @@ -24,6 +26,17 @@ const ( msgKernelVirtio = "Host kernel accelerator for virtio" msgKernelVirtioNet = "Host kernel accelerator for virtio network" msgKernelVirtioVhostVsock = "Host Support for Linux VM Sockets" + cpuFlagVMX = "vmx" + cpuFlagLM = "lm" + cpuFlagSVM = "svm" + cpuFlagSSE4_1 = "sse4_1" + kernelModvhm = "vhm_dev" + kernelModvhost = "vhost" + kernelModvhostnet = "vhost_net" + kernelModvhostvsock = "vhost_vsock" + kernelModkvm = "kvm" + kernelModkvmintel = "kvm_intel" + kernelModkvmamd = "kvm_amd" ) // CPU types @@ -33,6 +46,28 @@ const ( cpuTypeUnknown = -1 ) +const acrnDevice = "/dev/acrn_vhm" + +// ioctl_ACRN_CREATE_VM is the IOCTL to create VM in ACRN. +// Current Linux mainstream kernel doesn't have support for ACRN. +// Due to this several macros are not defined in Linux headers. +// Until the support is available, directly use the value instead +// of macros. +//https://github.com/kata-containers/runtime/issues/1784 +const ioctl_ACRN_CREATE_VM = 0x43000010 //nolint +const ioctl_ACRN_DESTROY_VM = 0x43000011 //nolint + +type acrn_create_vm struct { //nolint + vmid uint16 //nolint + reserved0 uint16 //nolint + vcpu_num uint16 //nolint + reserved1 uint16 //nolint + uuid [16]uint8 + vm_flag uint64 //nolint + req_buf uint64 //nolint + reserved2 [16]uint8 //nolint +} + // cpuType save the CPU type var cpuType int @@ -49,7 +84,7 @@ var archRequiredCPUAttribs map[string]string // required module parameters. var archRequiredKernelModules map[string]kernelModule -func setCPUtype() error { +func setCPUtype(hypervisorType vc.HypervisorType) error { cpuType = getCPUtype() if cpuType == cpuTypeUnknown { @@ -66,64 +101,88 @@ func setCPUtype() error { "unrestricted_guest": "Y", } } - archRequiredCPUFlags = map[string]string{ - "vmx": "Virtualization support", - "lm": "64Bit CPU", - "sse4_1": "SSE4.1", - } - archRequiredCPUAttribs = map[string]string{ - archGenuineIntel: "Intel Architecture CPU", - } - archRequiredKernelModules = map[string]kernelModule{ - "kvm": { - desc: msgKernelVM, - required: true, - }, - "kvm_intel": { - desc: "Intel KVM", - parameters: kvmIntelParams, - required: true, - }, - "vhost": { - desc: msgKernelVirtio, - required: true, - }, - "vhost_net": { - desc: msgKernelVirtioNet, - required: true, - }, - "vhost_vsock": { - desc: msgKernelVirtioVhostVsock, - required: false, - }, + + switch hypervisorType { + case "firecracker": + fallthrough + case "qemu": + archRequiredCPUFlags = map[string]string{ + cpuFlagVMX: "Virtualization support", + cpuFlagLM: "64Bit CPU", + cpuFlagSSE4_1: "SSE4.1", + } + archRequiredCPUAttribs = map[string]string{ + archGenuineIntel: "Intel Architecture CPU", + } + archRequiredKernelModules = map[string]kernelModule{ + kernelModkvm: { + desc: msgKernelVM, + }, + kernelModkvmintel: { + desc: "Intel KVM", + parameters: kvmIntelParams, + }, + kernelModvhost: { + desc: msgKernelVirtio, + }, + kernelModvhostnet: { + desc: msgKernelVirtioNet, + }, + kernelModvhostvsock: { + desc: msgKernelVirtioVhostVsock, + required: false, + }, + } + case "acrn": + archRequiredCPUFlags = map[string]string{ + cpuFlagLM: "64Bit CPU", + cpuFlagSSE4_1: "SSE4.1", + } + archRequiredCPUAttribs = map[string]string{ + archGenuineIntel: "Intel Architecture CPU", + } + archRequiredKernelModules = map[string]kernelModule{ + kernelModvhm: { + desc: "Intel ACRN", + }, + kernelModvhost: { + desc: msgKernelVirtio, + }, + kernelModvhostnet: { + desc: msgKernelVirtioNet, + }, + } + default: + return fmt.Errorf("setCPUtype: Unknown hypervisor type %s", hypervisorType) } + } else if cpuType == cpuTypeAMD { archRequiredCPUFlags = map[string]string{ - "svm": "Virtualization support", - "lm": "64Bit CPU", - "sse4_1": "SSE4.1", + cpuFlagSVM: "Virtualization support", + cpuFlagLM: "64Bit CPU", + cpuFlagSSE4_1: "SSE4.1", } archRequiredCPUAttribs = map[string]string{ archAuthenticAMD: "AMD Architecture CPU", } archRequiredKernelModules = map[string]kernelModule{ - "kvm": { + kernelModkvm: { desc: msgKernelVM, required: true, }, - "kvm_amd": { + kernelModkvmamd: { desc: "AMD KVM", required: true, }, - "vhost": { + kernelModvhost: { desc: msgKernelVirtio, required: true, }, - "vhost_net": { + kernelModvhostnet: { desc: msgKernelVirtioNet, required: true, }, - "vhost_vsock": { + kernelModvhostvsock: { desc: msgKernelVirtioVhostVsock, required: false, }, @@ -155,8 +214,72 @@ func kvmIsUsable() error { return genericKvmIsUsable() } -func archHostCanCreateVMContainer() error { - return kvmIsUsable() +// acrnIsUsable determines if it will be possible to create a full virtual machine +// by creating a minimal VM and then deleting it. +func acrnIsUsable() error { + flags := syscall.O_RDWR | syscall.O_CLOEXEC + + f, err := syscall.Open(acrnDevice, flags, 0) + if err != nil { + return err + } + defer syscall.Close(f) + + fieldLogger := kataLog.WithField("check-type", "full") + + fieldLogger.WithField("device", acrnDevice).Info("device available") + + createVM := acrn_create_vm{ + uuid: [16]uint8{ + 0xd2, 0x79, 0x54, 0x38, 0x25, 0xd6, 0x11, 0xe8, + 0x86, 0x4e, 0xcb, 0x7a, 0x18, 0xb3, 0x46, 0x43, + }, + } + + ret, _, errno := syscall.Syscall(syscall.SYS_IOCTL, + uintptr(f), + uintptr(ioctl_ACRN_CREATE_VM), + uintptr(unsafe.Pointer(&createVM))) + if ret != 0 || errno != 0 { + if errno == syscall.EBUSY { + fieldLogger.WithField("reason", "another hypervisor running").Error("cannot create VM") + } + fieldLogger.WithFields(logrus.Fields{ + "ret": ret, + "errno": errno, + }).Info("Create VM Error") + return errno + } + + ret, _, errno = syscall.Syscall(syscall.SYS_IOCTL, + uintptr(f), + uintptr(ioctl_ACRN_DESTROY_VM), + 0) + if ret != 0 || errno != 0 { + fieldLogger.WithFields(logrus.Fields{ + "ret": ret, + "errno": errno, + }).Info("Destroy VM Error") + return errno + } + + fieldLogger.WithField("feature", "create-vm").Info("feature available") + + return nil +} + +func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { + + switch hypervisorType { + case "qemu": + fallthrough + case "firecracker": + return kvmIsUsable() + case "acrn": + return acrnIsUsable() + default: + return fmt.Errorf("archHostCanCreateVMContainer: Unknown hypervisor type %s", hypervisorType) + } } // hostIsVMContainerCapable checks to see if the host is theoretically capable diff --git a/cli/kata-check_amd64_test.go b/cli/kata-check_amd64_test.go index bf2bbedd02..2fcba9ae29 100644 --- a/cli/kata-check_amd64_test.go +++ b/cli/kata-check_amd64_test.go @@ -55,6 +55,9 @@ func TestCCCheckCLIFunction(t *testing.T) { } defer os.RemoveAll(dir) + _, config, err := makeRuntimeConfig(dir) + assert.NoError(err) + savedSysModuleDir := sysModuleDir savedProcCPUInfo := procCPUInfo @@ -108,6 +111,7 @@ func TestCCCheckCLIFunction(t *testing.T) { ctx := createCLIContext(nil) ctx.App.Name = "foo" + ctx.App.Metadata["runtimeConfig"] = config // create buffer to save logger output buf := &bytes.Buffer{} @@ -514,6 +518,10 @@ foo : bar func TestSetCPUtype(t *testing.T) { assert := assert.New(t) + tmpdir, err := ioutil.TempDir("", "") + assert.NoError(err) + defer os.RemoveAll(tmpdir) + savedArchRequiredCPUFlags := archRequiredCPUFlags savedArchRequiredCPUAttribs := archRequiredCPUAttribs savedArchRequiredKernelModules := archRequiredKernelModules @@ -528,7 +536,10 @@ func TestSetCPUtype(t *testing.T) { archRequiredCPUAttribs = map[string]string{} archRequiredKernelModules = map[string]kernelModule{} - setCPUtype() + _, config, err := makeRuntimeConfig(tmpdir) + assert.NoError(err) + + setCPUtype(config.HypervisorType) assert.NotEmpty(archRequiredCPUFlags) assert.NotEmpty(archRequiredCPUAttribs) diff --git a/cli/kata-check_arm64.go b/cli/kata-check_arm64.go index 5f835fcaba..959f312a45 100644 --- a/cli/kata-check_arm64.go +++ b/cli/kata-check_arm64.go @@ -8,6 +8,7 @@ package main import ( "fmt" + vc "github.com/kata-containers/runtime/virtcontainers" "github.com/sirupsen/logrus" ) @@ -56,7 +57,7 @@ var archRequiredKVMExtensions = map[string]kvmExtension{ }, } -func setCPUtype() error { +func setCPUtype(hypervisorType vc.HypervisorType) error { return nil } @@ -84,7 +85,7 @@ func checkKVMExtensions() error { return nil } -func archHostCanCreateVMContainer() error { +func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { if err := kvmIsUsable(); err != nil { return err } diff --git a/cli/kata-check_arm64_test.go b/cli/kata-check_arm64_test.go index 433c0996fd..3c0588f29e 100644 --- a/cli/kata-check_arm64_test.go +++ b/cli/kata-check_arm64_test.go @@ -36,6 +36,9 @@ func TestCCCheckCLIFunction(t *testing.T) { } defer os.RemoveAll(dir) + _, config, err := makeRuntimeConfig(dir) + assert.NoError(err) + savedSysModuleDir := sysModuleDir savedProcCPUInfo := procCPUInfo @@ -78,6 +81,7 @@ func TestCCCheckCLIFunction(t *testing.T) { ctx := createCLIContext(nil) ctx.App.Name = "foo" + ctx.App.Metadata["runtimeConfig"] = config // create buffer to save logger output buf := &bytes.Buffer{} diff --git a/cli/kata-check_generic_test.go b/cli/kata-check_generic_test.go index 5332d4e892..1441268308 100644 --- a/cli/kata-check_generic_test.go +++ b/cli/kata-check_generic_test.go @@ -8,6 +8,8 @@ package main import ( + "io/ioutil" + "os" "testing" "github.com/stretchr/testify/assert" @@ -16,6 +18,10 @@ import ( func testSetCPUTypeGeneric(t *testing.T) { assert := assert.New(t) + tmpdir, err := ioutil.TempDir("", "") + assert.NoError(err) + defer os.RemoveAll(tmpdir) + savedArchRequiredCPUFlags := archRequiredCPUFlags savedArchRequiredCPUAttribs := archRequiredCPUAttribs savedArchRequiredKernelModules := archRequiredKernelModules @@ -30,7 +36,10 @@ func testSetCPUTypeGeneric(t *testing.T) { assert.Empty(archRequiredCPUAttribs) assert.NotEmpty(archRequiredKernelModules) - setCPUtype() + _, config, err := makeRuntimeConfig(tmpdir) + assert.NoError(err) + + setCPUtype(config.HypervisorType) assert.Equal(archRequiredCPUFlags, savedArchRequiredCPUFlags) assert.Equal(archRequiredCPUAttribs, savedArchRequiredCPUAttribs) diff --git a/cli/kata-check_ppc64le.go b/cli/kata-check_ppc64le.go index 4d7ec3494b..b57c971ced 100644 --- a/cli/kata-check_ppc64le.go +++ b/cli/kata-check_ppc64le.go @@ -8,12 +8,13 @@ package main import ( "fmt" "os/exec" + "regexp" + "strconv" "strings" "github.com/kata-containers/runtime/pkg/katautils" + vc "github.com/kata-containers/runtime/virtcontainers" "github.com/sirupsen/logrus" - "regexp" - "strconv" ) const ( @@ -55,11 +56,11 @@ var archRequiredKernelModules = map[string]kernelModule{ }, } -func setCPUtype() error { +func setCPUtype(hypervisorType vc.HypervisorType) error { return nil } -func archHostCanCreateVMContainer() error { +func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { return kvmIsUsable() } diff --git a/cli/kata-check_ppc64le_test.go b/cli/kata-check_ppc64le_test.go index 29859c30cb..d49ec445f5 100644 --- a/cli/kata-check_ppc64le_test.go +++ b/cli/kata-check_ppc64le_test.go @@ -53,6 +53,9 @@ func TestCCCheckCLIFunction(t *testing.T) { } defer os.RemoveAll(dir) + _, config, err := makeRuntimeConfig(dir) + assert.NoError(err) + savedSysModuleDir := sysModuleDir savedProcCPUInfo := procCPUInfo @@ -98,6 +101,7 @@ func TestCCCheckCLIFunction(t *testing.T) { ctx := createCLIContext(nil) ctx.App.Name = "foo" + ctx.App.Metadata["runtimeConfig"] = config // create buffer to save logger output buf := &bytes.Buffer{} diff --git a/cli/kata-check_s390x.go b/cli/kata-check_s390x.go index 1e5f42d503..0c67f1520a 100644 --- a/cli/kata-check_s390x.go +++ b/cli/kata-check_s390x.go @@ -7,8 +7,10 @@ package main import ( "fmt" - "github.com/sirupsen/logrus" "strings" + + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/sirupsen/logrus" ) const ( @@ -42,7 +44,7 @@ var archRequiredKernelModules = map[string]kernelModule{ }, } -func setCPUtype() error { +func setCPUtype(hypervisorType vc.HypervisorType) error { return nil } @@ -52,7 +54,7 @@ func kvmIsUsable() error { return genericKvmIsUsable() } -func archHostCanCreateVMContainer() error { +func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { return kvmIsUsable() } diff --git a/cli/kata-check_s390x_test.go b/cli/kata-check_s390x_test.go index c8bc129ea4..108d6e1e8e 100644 --- a/cli/kata-check_s390x_test.go +++ b/cli/kata-check_s390x_test.go @@ -53,6 +53,9 @@ func TestCCCheckCLIFunction(t *testing.T) { } defer os.RemoveAll(dir) + _, config, err := makeRuntimeConfig(dir) + assert.NoError(err) + savedSysModuleDir := sysModuleDir savedProcCPUInfo := procCPUInfo @@ -97,6 +100,7 @@ func TestCCCheckCLIFunction(t *testing.T) { ctx := createCLIContext(nil) ctx.App.Name = "foo" + ctx.App.Metadata["runtimeConfig"] = config // create buffer to save logger output buf := &bytes.Buffer{} diff --git a/cli/kata-check_test.go b/cli/kata-check_test.go index 29767a9260..87605ef123 100644 --- a/cli/kata-check_test.go +++ b/cli/kata-check_test.go @@ -659,6 +659,9 @@ func TestCheckCLIFunctionFail(t *testing.T) { } defer os.RemoveAll(dir) + _, config, err := makeRuntimeConfig(dir) + assert.NoError(err) + oldProcCPUInfo := procCPUInfo // doesn't exist @@ -670,6 +673,7 @@ func TestCheckCLIFunctionFail(t *testing.T) { ctx := createCLIContext(nil) ctx.App.Name = "foo" + ctx.App.Metadata["runtimeConfig"] = config fn, ok := kataCheckCLICommand.Action.(func(context *cli.Context) error) assert.True(ok) diff --git a/cli/kata-env.go b/cli/kata-env.go index c8610daa4c..f9c1e9d808 100644 --- a/cli/kata-env.go +++ b/cli/kata-env.go @@ -9,9 +9,8 @@ import ( "encoding/json" "errors" "os" - "strings" - runtim "runtime" + "strings" "github.com/BurntSushi/toml" "github.com/kata-containers/runtime/pkg/katautils" @@ -358,7 +357,7 @@ func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo { } func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err error) { - err = setCPUtype() + err = setCPUtype(config.HypervisorType) if err != nil { return EnvInfo{}, err } diff --git a/cli/main.go b/cli/main.go index a021a0a775..72c9165e7d 100644 --- a/cli/main.go +++ b/cli/main.go @@ -254,54 +254,54 @@ func beforeSubcommands(c *cli.Context) error { handleShowConfig(c) - if userWantsUsage(c) || (c.NArg() == 1 && (c.Args()[0] == checkCmd)) { + if userWantsUsage(c) { // No setup required if the user just - // wants to see the usage statement or are - // running a command that does not manipulate - // containers. + // wants to see the usage statement. return nil } - if path := c.GlobalString("log"); path != "" { - f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640) - if err != nil { - return err - } - kataLog.Logger.Out = f - } - - switch c.GlobalString("log-format") { - case "text": - // retain logrus's default. - case "json": - kataLog.Logger.Formatter = new(logrus.JSONFormatter) - default: - return fmt.Errorf("unknown log-format %q", c.GlobalString("log-format")) - } - + ignoreLogging := false var traceRootSpan string - // Add the name of the sub-command to each log entry for easier - // debugging. - cmdName := c.Args().First() - if c.App.Command(cmdName) != nil { - kataLog = kataLog.WithField("command", cmdName) + subCmdIsCheckCmd := (c.NArg() == 1 && (c.Args()[0] == checkCmd)) + if !subCmdIsCheckCmd { + if path := c.GlobalString("log"); path != "" { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640) + if err != nil { + return err + } + kataLog.Logger.Out = f + } - // Name for the root span (used for tracing) now the - // sub-command name is known. - traceRootSpan = name + " " + cmdName - } + switch c.GlobalString("log-format") { + case "text": + // retain logrus's default. + case "json": + kataLog.Logger.Formatter = new(logrus.JSONFormatter) + default: + return fmt.Errorf("unknown log-format %q", c.GlobalString("log-format")) + } - // Since a context is required, pass a new (throw-away) one - we - // cannot use the main context as tracing hasn't been enabled yet - // (meaning any spans created at this point will be silently ignored). - setExternalLoggers(context.Background(), kataLog) + // Add the name of the sub-command to each log entry for easier + // debugging. + cmdName := c.Args().First() + if c.App.Command(cmdName) != nil { + kataLog = kataLog.WithField("command", cmdName) - ignoreLogging := false + // Name for the root span (used for tracing) now the + // sub-command name is known. + traceRootSpan = name + " " + cmdName + } - if c.NArg() == 1 && c.Args()[0] == envCmd { - // simply report the logging setup - ignoreLogging = true + // Since a context is required, pass a new (throw-away) one - we + // cannot use the main context as tracing hasn't been enabled yet + // (meaning any spans created at this point will be silently ignored). + setExternalLoggers(context.Background(), kataLog) + + if c.NArg() == 1 && c.Args()[0] == envCmd { + // simply report the logging setup + ignoreLogging = true + } } configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreLogging, false) @@ -309,19 +309,21 @@ func beforeSubcommands(c *cli.Context) error { fatal(err) } - debug = runtimeConfig.Debug - crashOnError = runtimeConfig.Debug + if !subCmdIsCheckCmd { + debug = runtimeConfig.Debug + crashOnError = runtimeConfig.Debug - if traceRootSpan != "" { - // Create the tracer. - // - // Note: no spans are created until the command-line has been parsed. - // This delays collection of trace data slightly but benefits the user by - // ensuring the first span is the name of the sub-command being - // invoked from the command-line. - err = setupTracing(c, traceRootSpan) - if err != nil { - return err + if traceRootSpan != "" { + // Create the tracer. + // + // Note: no spans are created until the command-line has been parsed. + // This delays collection of trace data slightly but benefits the user by + // ensuring the first span is the name of the sub-command being + // invoked from the command-line. + err = setupTracing(c, traceRootSpan) + if err != nil { + return err + } } } diff --git a/pkg/katautils/config-settings.go b/pkg/katautils/config-settings.go index 7c13339552..ac3c44c994 100644 --- a/pkg/katautils/config-settings.go +++ b/pkg/katautils/config-settings.go @@ -9,6 +9,7 @@ package katautils var defaultHypervisorPath = "/usr/bin/qemu-lite-system-x86_64" +var defaultHypervisorCtlPath = "/usr/bin/acrnctl" var defaultImagePath = "/usr/share/kata-containers/kata-containers.img" var defaultKernelPath = "/usr/share/kata-containers/vmlinuz.container" var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img" diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index 64ee309855..a7faacf31f 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -50,6 +50,7 @@ const ( // supported hypervisor component types firecrackerHypervisorTableType = "firecracker" qemuHypervisorTableType = "qemu" + acrnHypervisorTableType = "acrn" // supported proxy component types kataProxyTableType = "kata" @@ -84,6 +85,7 @@ type factory struct { type hypervisor struct { Path string `toml:"path"` Kernel string `toml:"kernel"` + CtlPath string `toml:"ctlpath"` Initrd string `toml:"initrd"` Image string `toml:"image"` Firmware string `toml:"firmware"` @@ -163,6 +165,16 @@ func (h hypervisor) path() (string, error) { return ResolvePath(p) } +func (h hypervisor) ctlpath() (string, error) { + p := h.CtlPath + + if h.CtlPath == "" { + p = defaultHypervisorCtlPath + } + + return ResolvePath(p) +} + func (h hypervisor) kernel() (string, error) { p := h.Kernel @@ -602,6 +614,67 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { }, nil } +func newAcrnHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { + hypervisor, err := h.path() + if err != nil { + return vc.HypervisorConfig{}, err + } + + hypervisorctl, err := h.ctlpath() + if err != nil { + return vc.HypervisorConfig{}, err + } + + kernel, err := h.kernel() + if err != nil { + return vc.HypervisorConfig{}, err + } + + image, err := h.image() + if err != nil { + return vc.HypervisorConfig{}, err + } + + if image == "" { + return vc.HypervisorConfig{}, + errors.New("image must be defined in the configuration file") + } + + firmware, err := h.firmware() + if err != nil { + return vc.HypervisorConfig{}, err + } + + kernelParams := h.kernelParams() + + blockDriver, err := h.blockDeviceDriver() + if err != nil { + return vc.HypervisorConfig{}, err + } + + return vc.HypervisorConfig{ + HypervisorPath: hypervisor, + KernelPath: kernel, + ImagePath: image, + HypervisorCtlPath: hypervisorctl, + FirmwarePath: firmware, + KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), + NumVCPUs: h.defaultVCPUs(), + DefaultMaxVCPUs: h.defaultMaxVCPUs(), + MemorySize: h.defaultMemSz(), + MemSlots: h.defaultMemSlots(), + EntropySource: h.GetEntropySource(), + DefaultBridges: h.defaultBridges(), + HugePages: h.HugePages, + Mlock: !h.Swap, + Debug: h.Debug, + DisableNestingChecks: h.DisableNestingChecks, + BlockDeviceDriver: blockDriver, + DisableVhostNet: h.DisableVhostNet, + GuestHookPath: h.guestHookPath(), + }, nil +} + func newFactoryConfig(f factory) (oci.FactoryConfig, error) { if f.TemplatePath == "" { f.TemplatePath = defaultTemplatePath @@ -642,11 +715,15 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi case qemuHypervisorTableType: config.HypervisorType = vc.QemuHypervisor hConfig, err = newQemuHypervisorConfig(hypervisor) + case acrnHypervisorTableType: + config.HypervisorType = vc.AcrnHypervisor + hConfig, err = newAcrnHypervisorConfig(hypervisor) } if err != nil { return fmt.Errorf("%v: %v", configPath, err) } + config.HypervisorConfig = hConfig } diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go new file mode 100644 index 0000000000..aaf401b867 --- /dev/null +++ b/virtcontainers/acrn.go @@ -0,0 +1,621 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/store" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/kata-containers/runtime/virtcontainers/utils" + opentracing "github.com/opentracing/opentracing-go" + "github.com/sirupsen/logrus" +) + +// AcrnState keeps Acrn's state +type AcrnState struct { + UUID string +} + +// AcrnInfo keeps PID of the hypervisor +type AcrnInfo struct { + PID int +} + +// acrn is an Hypervisor interface implementation for the Linux acrn hypervisor. +type acrn struct { + id string + + store *store.VCStore + config HypervisorConfig + acrnConfig Config + state AcrnState + info AcrnInfo + arch acrnArch + ctx context.Context +} + +const ( + acrnConsoleSocket = "console.sock" + acrnStopSandboxTimeoutSecs = 15 +) + +// agnostic list of kernel parameters +var acrnDefaultKernelParameters = []Param{ + {"panic", "1"}, +} + +func (a *acrn) kernelParameters() string { + // get a list of arch kernel parameters + params := a.arch.kernelParameters(a.config.Debug) + + // use default parameters + params = append(params, acrnDefaultKernelParameters...) + + // set the maximum number of vCPUs + params = append(params, Param{"maxcpus", fmt.Sprintf("%d", a.config.NumVCPUs)}) + + // add the params specified by the provided config. As the kernel + // honours the last parameter value set and since the config-provided + // params are added here, they will take priority over the defaults. + params = append(params, a.config.KernelParams...) + + paramsStr := SerializeParams(params, "=") + + return strings.Join(paramsStr, " ") +} + +// Adds all capabilities supported by acrn implementation of hypervisor interface +func (a *acrn) capabilities() types.Capabilities { + span, _ := a.trace("capabilities") + defer span.Finish() + + return a.arch.capabilities() +} + +func (a *acrn) hypervisorConfig() HypervisorConfig { + return a.config +} + +// get the acrn binary path +func (a *acrn) acrnPath() (string, error) { + p, err := a.config.HypervisorAssetPath() + if err != nil { + return "", err + } + + if p == "" { + p, err = a.arch.acrnPath() + if err != nil { + return "", err + } + } + + if _, err = os.Stat(p); os.IsNotExist(err) { + return "", fmt.Errorf("acrn path (%s) does not exist", p) + } + + return p, nil +} + +// get the ACRNCTL binary path +func (a *acrn) acrnctlPath() (string, error) { + ctlpath, err := a.config.HypervisorCtlAssetPath() + if err != nil { + return "", err + } + + if ctlpath == "" { + ctlpath, err = a.arch.acrnctlPath() + if err != nil { + return "", err + } + } + + if _, err = os.Stat(ctlpath); os.IsNotExist(err) { + return "", fmt.Errorf("acrnctl path (%s) does not exist", ctlpath) + } + + return ctlpath, nil +} + +// Logger returns a logrus logger appropriate for logging acrn messages +func (a *acrn) Logger() *logrus.Entry { + return virtLog.WithField("subsystem", "acrn") +} + +func (a *acrn) trace(name string) (opentracing.Span, context.Context) { + if a.ctx == nil { + a.Logger().WithField("type", "bug").Error("trace called before context set") + a.ctx = context.Background() + } + + span, ctx := opentracing.StartSpanFromContext(a.ctx, name) + + span.SetTag("subsystem", "hypervisor") + span.SetTag("type", "acrn") + + return span, ctx +} + +func (a *acrn) memoryTopology() (Memory, error) { + memMb := uint64(a.config.MemorySize) + + return a.arch.memoryTopology(memMb), nil +} + +func (a *acrn) appendImage(devices []Device, imagePath string) ([]Device, error) { + if imagePath == "" { + return nil, fmt.Errorf("Image path is empty: %s", imagePath) + } + + // Get sandbox and increment the globalIndex. + // This is to make sure the VM rootfs occupies + // the first Index which is /dev/vda. + sandbox, err := globalSandboxList.lookupSandbox(a.id) + if sandbox == nil && err != nil { + return nil, err + } + sandbox.GetAndSetSandboxBlockIndex() + + devices, err = a.arch.appendImage(devices, imagePath) + if err != nil { + return nil, err + } + + return devices, nil +} + +func (a *acrn) buildDevices(imagePath string) ([]Device, error) { + var devices []Device + + if imagePath == "" { + return nil, fmt.Errorf("Image Path should not be empty: %s", imagePath) + } + + console, err := a.getSandboxConsole(a.id) + if err != nil { + return nil, err + } + + // Add bridges before any other devices. This way we make sure that + // bridge gets the first available PCI address. + devices = a.arch.appendBridges(devices) + + //Add LPC device to the list of other devices. + devices = a.arch.appendLPC(devices) + + devices = a.arch.appendConsole(devices, console) + + devices, err = a.appendImage(devices, imagePath) + if err != nil { + return nil, err + } + + // Create virtio blk devices with dummy backend as a place + // holder for container rootfs (as acrn doesn't support hot-plug). + // Once the container rootfs is known, replace the dummy backend + // with actual path (using block rescan feature in acrn) + devices, err = a.createDummyVirtioBlkDev(devices) + if err != nil { + return nil, err + } + + return devices, nil +} + +// setup sets the Acrn structure up. +func (a *acrn) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error { + span, _ := a.trace("setup") + defer span.Finish() + + err := hypervisorConfig.valid() + if err != nil { + return err + } + + a.id = id + a.store = vcStore + a.config = *hypervisorConfig + a.arch = newAcrnArch(a.config) + if err = a.store.Load(store.Hypervisor, &a.state); err != nil { + // acrn currently supports only known UUIDs for security + // reasons (FuSa). When launching VM, only these pre-defined + // UUID should be used else VM launch will fail. acrn team is + // working on a solution to expose these pre-defined UUIDs + // to Kata in order for it to launch VMs successfully. + // Until this support is available, Kata is limited to + // launch 1 VM (using the default UUID). + // https://github.com/kata-containers/runtime/issues/1785 + + // The path might already exist, but in case of VM templating, + // we have to create it since the sandbox has not created it yet. + if err = os.MkdirAll(store.SandboxRuntimeRootPath(id), store.DirMode); err != nil { + return err + } + + if err = a.store.Store(store.Hypervisor, a.state); err != nil { + return err + } + } + + if err = a.store.Load(store.Hypervisor, &a.info); err != nil { + a.Logger().WithField("function", "setup").WithError(err).Info("No info could be fetched") + } + + return nil +} + +func (a *acrn) createDummyVirtioBlkDev(devices []Device) ([]Device, error) { + span, _ := a.trace("createDummyVirtioBlkDev") + defer span.Finish() + + // Since acrn doesn't support hot-plug, dummy virtio-blk + // devices are added and later replaced with container-rootfs. + // Starting from driveIndex 1, as 0 is allocated for VM rootfs. + for driveIndex := 1; driveIndex <= AcrnBlkDevPoolSz; driveIndex++ { + drive := config.BlockDrive{ + File: "nodisk", + Index: driveIndex, + } + + devices = a.arch.appendBlockDevice(devices, drive) + } + + return devices, nil +} + +// createSandbox is the Hypervisor sandbox creation. +func (a *acrn) createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, store *store.VCStore) error { + // Save the tracing context + a.ctx = ctx + + span, _ := a.trace("createSandbox") + defer span.Finish() + + if err := a.setup(id, hypervisorConfig, store); err != nil { + return err + } + + memory, err := a.memoryTopology() + if err != nil { + return err + } + + kernelPath, err := a.config.KernelAssetPath() + if err != nil { + return err + } + + imagePath, err := a.config.ImageAssetPath() + if err != nil { + return err + } + + kernel := Kernel{ + Path: kernelPath, + ImagePath: imagePath, + Params: a.kernelParameters(), + } + + // Disabling UUID check until the below is fixed. + // https://github.com/kata-containers/runtime/issues/1785 + + devices, err := a.buildDevices(imagePath) + if err != nil { + return err + } + + acrnPath, err := a.acrnPath() + if err != nil { + return err + } + + acrnctlPath, err := a.acrnctlPath() + if err != nil { + return err + } + + acrnConfig := Config{ + ACPIVirt: true, + Path: acrnPath, + CtlPath: acrnctlPath, + Memory: memory, + NumCPU: a.config.NumVCPUs, + Devices: devices, + Kernel: kernel, + Name: fmt.Sprintf("sandbox-%s", a.id), + } + + a.acrnConfig = acrnConfig + + return nil +} + +// startSandbox will start the Sandbox's VM. +func (a *acrn) startSandbox(timeoutSecs int) error { + span, _ := a.trace("startSandbox") + defer span.Finish() + + if a.config.Debug { + params := a.arch.kernelParameters(a.config.Debug) + strParams := SerializeParams(params, "=") + formatted := strings.Join(strParams, " ") + + // The name of this field matches a similar one generated by + // the runtime and allows users to identify which parameters + // are set here, which come from the runtime and which are set + // by the user. + a.Logger().WithField("default-kernel-parameters", formatted).Debug() + } + + vmPath := filepath.Join(store.RunVMStoragePath, a.id) + err := os.MkdirAll(vmPath, store.DirMode) + if err != nil { + return err + } + defer func() { + if err != nil { + if err := os.RemoveAll(vmPath); err != nil { + a.Logger().WithError(err).Error("Failed to clean up vm directory") + } + } + }() + + var strErr string + var PID int + PID, strErr, err = LaunchAcrn(a.acrnConfig, virtLog.WithField("subsystem", "acrn-dm")) + if err != nil { + return fmt.Errorf("%s", strErr) + } + a.info.PID = PID + + if err = a.waitSandbox(timeoutSecs); err != nil { + a.Logger().WithField("acrn wait failed:", err).Debug() + return err + } + + //Store VMM information + return a.store.Store(store.Hypervisor, a.info) + +} + +// waitSandbox will wait for the Sandbox's VM to be up and running. +func (a *acrn) waitSandbox(timeoutSecs int) error { + span, _ := a.trace("waitSandbox") + defer span.Finish() + + if timeoutSecs < 0 { + return fmt.Errorf("Invalid timeout %ds", timeoutSecs) + } + + time.Sleep(time.Duration(timeoutSecs) * time.Second) + + return nil +} + +// stopSandbox will stop the Sandbox's VM. +func (a *acrn) stopSandbox() (err error) { + span, _ := a.trace("stopSandbox") + defer span.Finish() + + a.Logger().Info("Stopping acrn VM") + + defer func() { + if err != nil { + a.Logger().Info("stopSandbox failed") + } else { + a.Logger().Info("acrn VM stopped") + } + }() + + pid := a.info.PID + + // Check if VM process is running, in case it is not, let's + // return from here. + if err = syscall.Kill(pid, syscall.Signal(0)); err != nil { + a.Logger().Info("acrn VM already stopped") + return nil + } + + // Send signal to the VM process to try to stop it properly + if err = syscall.Kill(pid, syscall.SIGINT); err != nil { + a.Logger().Info("Sending signal to stop acrn VM failed") + return err + } + + // Wait for the VM process to terminate + tInit := time.Now() + for { + if err = syscall.Kill(pid, syscall.Signal(0)); err != nil { + a.Logger().Info("acrn VM stopped after sending signal") + return nil + } + + if time.Since(tInit).Seconds() >= acrnStopSandboxTimeoutSecs { + a.Logger().Warnf("VM still running after waiting %ds", acrnStopSandboxTimeoutSecs) + break + } + + // Let's avoid to run a too busy loop + time.Sleep(time.Duration(50) * time.Millisecond) + } + + // Let's try with a hammer now, a SIGKILL should get rid of the + // VM process. + return syscall.Kill(pid, syscall.SIGKILL) + +} + +func (a *acrn) updateBlockDevice(drive *config.BlockDrive) error { + var err error + if drive.File == "" || drive.Index >= AcrnBlkDevPoolSz { + return fmt.Errorf("Empty filepath or invalid drive index, Dive ID:%s, Drive Index:%d", + drive.ID, drive.Index) + } + + slot := AcrnBlkdDevSlot[drive.Index] + + //Explicitly set PCIAddr to NULL, so that VirtPath can be used + drive.PCIAddr = "" + + args := []string{"blkrescan", a.acrnConfig.Name, fmt.Sprintf("%d,%s", slot, drive.File)} + + a.Logger().WithFields(logrus.Fields{ + "drive": drive, + "path": a.config.HypervisorCtlPath, + }).Info("updateBlockDevice with acrnctl path") + cmd := exec.Command(a.config.HypervisorCtlPath, args...) + if err := cmd.Run(); err != nil { + a.Logger().WithError(err).Error("updating Block device with newFile path") + } + + return err +} + +func (a *acrn) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) { + span, _ := a.trace("hotplugAddDevice") + defer span.Finish() + + switch devType { + case blockDev: + //The drive placeholder has to exist prior to Update + return nil, a.updateBlockDevice(devInfo.(*config.BlockDrive)) + default: + return nil, fmt.Errorf("hotplugAddDevice: unsupported device: devInfo:%v, deviceType%v", + devInfo, devType) + } +} + +func (a *acrn) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { + span, _ := a.trace("hotplugRemoveDevice") + defer span.Finish() + + // Not supported. return success + + return nil, nil +} + +func (a *acrn) pauseSandbox() error { + span, _ := a.trace("pauseSandbox") + defer span.Finish() + + // Not supported. return success + + return nil +} + +func (a *acrn) resumeSandbox() error { + span, _ := a.trace("resumeSandbox") + defer span.Finish() + + // Not supported. return success + + return nil +} + +// addDevice will add extra devices to Acrn command line. +func (a *acrn) addDevice(devInfo interface{}, devType deviceType) error { + var err error + span, _ := a.trace("addDevice") + defer span.Finish() + + switch v := devInfo.(type) { + case types.Volume: + // Not supported. return success + err = nil + case types.Socket: + a.acrnConfig.Devices = a.arch.appendSocket(a.acrnConfig.Devices, v) + case kataVSOCK: + // Not supported. return success + err = nil + case Endpoint: + a.acrnConfig.Devices = a.arch.appendNetwork(a.acrnConfig.Devices, v) + case config.BlockDrive: + a.acrnConfig.Devices = a.arch.appendBlockDevice(a.acrnConfig.Devices, v) + case config.VhostUserDeviceAttrs: + // Not supported. return success + err = nil + case config.VFIODev: + // Not supported. return success + err = nil + default: + err = nil + a.Logger().WithField("unknown-device-type", devInfo).Error("Adding device") + } + + return err +} + +// getSandboxConsole builds the path of the console where we can read +// logs coming from the sandbox. +func (a *acrn) getSandboxConsole(id string) (string, error) { + span, _ := a.trace("getSandboxConsole") + defer span.Finish() + + return utils.BuildSocketPath(store.RunVMStoragePath, id, acrnConsoleSocket) +} + +func (a *acrn) saveSandbox() error { + a.Logger().Info("save sandbox") + + // Not supported. return success + + return nil +} + +func (a *acrn) disconnect() { + span, _ := a.trace("disconnect") + defer span.Finish() + + // Not supported. +} + +func (a *acrn) getThreadIDs() (vcpuThreadIDs, error) { + span, _ := a.trace("getThreadIDs") + defer span.Finish() + + // Not supported. return success + //Just allocating an empty map + + return vcpuThreadIDs{}, nil +} + +func (a *acrn) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { + return 0, memoryDevice{}, nil +} + +func (a *acrn) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { + return 0, 0, nil +} + +func (a *acrn) cleanup() error { + span, _ := a.trace("cleanup") + defer span.Finish() + + return nil +} + +func (a *acrn) pid() int { + return a.info.PID +} + +func (a *acrn) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, store *store.VCStore, j []byte) error { + return errors.New("acrn is not supported by VM cache") +} + +func (a *acrn) toGrpc() ([]byte, error) { + return nil, errors.New("acrn is not supported by VM cache") +} diff --git a/virtcontainers/acrn_arch_base.go b/virtcontainers/acrn_arch_base.go new file mode 100644 index 0000000000..a9882576ad --- /dev/null +++ b/virtcontainers/acrn_arch_base.go @@ -0,0 +1,772 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "strings" + + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/sirupsen/logrus" +) + +type acrnArch interface { + + // acrnPath returns the path to the acrn binary + acrnPath() (string, error) + + // acrnctlPath returns the path to the acrnctl binary + acrnctlPath() (string, error) + + // kernelParameters returns the kernel parameters + // if debug is true then kernel debug parameters are included + kernelParameters(debug bool) []Param + + //capabilities returns the capabilities supported by acrn + capabilities() types.Capabilities + + // memoryTopology returns the memory topology using the given amount of memoryMb and hostMemoryMb + memoryTopology(memMb uint64) Memory + + // appendConsole appends a console to devices + appendConsole(devices []Device, path string) []Device + + // appendImage appends an image to devices + appendImage(devices []Device, path string) ([]Device, error) + + // appendBridges appends bridges to devices + appendBridges(devices []Device) []Device + + // appendLPC appends LPC to devices + // UART device emulated by the acrn-dm is connected to the system by the LPC bus + appendLPC(devices []Device) []Device + + // appendSocket appends a socket to devices + appendSocket(devices []Device, socket types.Socket) []Device + + // appendNetwork appends a endpoint device to devices + appendNetwork(devices []Device, endpoint Endpoint) []Device + + // appendBlockDevice appends a block drive to devices + appendBlockDevice(devices []Device, drive config.BlockDrive) []Device + + // handleImagePath handles the Hypervisor Config image path + handleImagePath(config HypervisorConfig) +} + +type acrnArchBase struct { + path string + ctlpath string + kernelParamsNonDebug []Param + kernelParamsDebug []Param + kernelParams []Param +} + +const acrnPath = "/usr/bin/acrn-dm" +const acrnctlPath = "/usr/bin/acrnctl" + +// acrn GVT-g slot is harded code to 2 as there is +// no simple way to pass arguments of PCI slots from +// device model (acrn-dm) to ACRNGT module. +const acrnGVTgReservedSlot = 2 + +const acrnLPCDev = "lpc" +const acrnHostBridge = "hostbridge" + +var baselogger *logrus.Entry + +// AcrnBlkDevPoolSz defines the number of dummy virtio-blk +// device that will be created for hot-plugging container +// rootfs. Since acrn doesn't support hot-plug, dummy virtio-blk +// devices are added and later replaced with container-rootfs. +var AcrnBlkDevPoolSz = 8 + +// AcrnBlkdDevSlot array provides translation between +// the vitio-blk device index and slot it is currently +// attached. +// Allocating extra 1 to accommodate for VM rootfs +// which is at driveIndex 0 +var AcrnBlkdDevSlot = make([]int, AcrnBlkDevPoolSz+1) + +// acrnKernelParamsNonDebug is a list of the default kernel +// parameters that will be used in standard (non-debug) mode. +var acrnKernelParamsNonDebug = []Param{ + {"quiet", ""}, +} + +// acrnKernelParamsSystemdNonDebug is a list of the default systemd related +// kernel parameters that will be used in standard (non-debug) mode. +var acrnKernelParamsSystemdNonDebug = []Param{ + {"systemd.show_status", "false"}, +} + +// acrnKernelParamsDebug is a list of the default kernel +// parameters that will be used in debug mode (as much boot output as +// possible). +var acrnKernelParamsDebug = []Param{ + {"debug", ""}, +} + +// acrnKernelParamsSystemdDebug is a list of the default systemd related kernel +// parameters that will be used in debug mode (as much boot output as +// possible). +var acrnKernelParamsSystemdDebug = []Param{ + {"systemd.show_status", "true"}, + {"systemd.log_level", "debug"}, + {"systemd.log_target", "kmsg"}, + {"printk.devkmsg", "on"}, +} + +var acrnKernelRootParams = []Param{ + {"root", "/dev/vda1 rw rootwait"}, +} + +var acrnKernelParams = []Param{ + {"tsc", "reliable"}, + {"no_timer_check", ""}, + {"nohpet", ""}, + {"console", "tty0"}, + {"console", "ttyS0"}, + {"console", "hvc0"}, + {"log_buf_len", "16M"}, + {"consoleblank", "0"}, + {"iommu", "off"}, + {"i915.avail_planes_per_pipe", "0x070F00"}, + {"i915.enable_hangcheck", "0"}, + {"i915.nuclear_pageflip", "1"}, + {"i915.enable_guc_loading", "0"}, + {"i915.enable_guc_submission", "0"}, + {"i915.enable_guc", "0"}, +} + +// Device is the acrn device interface. +type Device interface { + Valid() bool + AcrnParams(slot int, config *Config) []string +} + +// ConsoleDeviceBackend is the character device backend for acrn +type ConsoleDeviceBackend string + +const ( + + // Socket creates a 2 way stream socket (TCP or Unix). + Socket ConsoleDeviceBackend = "socket" + + // Stdio sends traffic from the guest to acrn's standard output. + Stdio ConsoleDeviceBackend = "console" + + // File backend only supports console output to a file (no input). + File ConsoleDeviceBackend = "file" + + // TTY is an alias for Serial. + TTY ConsoleDeviceBackend = "tty" + + // PTY creates a new pseudo-terminal on the host and connect to it. + PTY ConsoleDeviceBackend = "pty" +) + +// BEPortType marks the port as console port or virtio-serial port +type BEPortType int + +const ( + // SerialBE marks the port as serial port + SerialBE BEPortType = iota + + //ConsoleBE marks the port as console port (append @) + ConsoleBE +) + +// ConsoleDevice represents a acrn console device. +type ConsoleDevice struct { + // Name of the socket + Name string + + //Backend device used for virtio-console + Backend ConsoleDeviceBackend + + // PortType marks the port as serial or console port (@) + PortType BEPortType + + //Path to virtio-console backend (can be omitted for pty, tty, stdio) + Path string +} + +// NetDeviceType is a acrn networking device type. +type NetDeviceType string + +const ( + // TAP is a TAP networking device type. + TAP NetDeviceType = "tap" + + // MACVTAP is a macvtap networking device type. + MACVTAP NetDeviceType = "macvtap" +) + +// NetDevice represents a guest networking device +type NetDevice struct { + // Type is the netdev type (e.g. tap). + Type NetDeviceType + + // IfName is the interface name + IFName string + + //MACAddress is the networking device interface MAC address + MACAddress string +} + +// BlockDevice represents a acrn block device. +type BlockDevice struct { + + // mem path to block device + FilePath string + + //BlkIndex - Blk index corresponding to slot + Index int +} + +// BridgeDevice represents a acrn bridge device like pci-bridge, pxb, etc. +type BridgeDevice struct { + + // Function is PCI function. Func can be from 0 to 7 + Function int + + // Emul is a string describing the type of PCI device e.g. virtio-net + Emul string + + // Config is an optional string, depending on the device, that can be + // used for configuration + Config string +} + +// LPCDevice represents a acrn LPC device +type LPCDevice struct { + + // Function is PCI function. Func can be from 0 to 7 + Function int + + // Emul is a string describing the type of PCI device e.g. virtio-net + Emul string +} + +// Memory is the guest memory configuration structure. +type Memory struct { + // Size is the amount of memory made available to the guest. + // It should be suffixed with M or G for sizes in megabytes or + // gigabytes respectively. + Size string +} + +// Kernel is the guest kernel configuration structure. +type Kernel struct { + // Path is the guest kernel path on the host filesystem. + Path string + + // InitrdPath is the guest initrd path on the host filesystem. + ImagePath string + + // Params is the kernel parameters string. + Params string +} + +// Config is the acrn configuration structure. +// It allows for passing custom settings and parameters to the acrn-dm API. +type Config struct { + + // Path is the acrn binary path. + Path string + + // Path is the acrn binary path. + CtlPath string + + // Name is the acrn guest name + Name string + + // UUID is the acrn process UUID. + UUID string + + // Devices is a list of devices for acrn to create and drive. + Devices []Device + + // Kernel is the guest kernel configuration. + Kernel Kernel + + // Memory is the guest memory configuration. + Memory Memory + + acrnParams []string + + // ACPI virtualization support + ACPIVirt bool + + // NumCPU is the number of CPUs for guest + NumCPU uint32 +} + +// MaxAcrnVCPUs returns the maximum number of vCPUs supported +func MaxAcrnVCPUs() uint32 { + return uint32(8) +} + +func newAcrnArch(config HypervisorConfig) acrnArch { + a := &acrnArchBase{ + path: acrnPath, + ctlpath: acrnctlPath, + kernelParamsNonDebug: acrnKernelParamsNonDebug, + kernelParamsDebug: acrnKernelParamsDebug, + kernelParams: acrnKernelParams, + } + + a.handleImagePath(config) + return a +} + +func (a *acrnArchBase) acrnPath() (string, error) { + p := a.path + return p, nil +} + +func (a *acrnArchBase) acrnctlPath() (string, error) { + ctlpath := a.ctlpath + return ctlpath, nil +} + +func (a *acrnArchBase) kernelParameters(debug bool) []Param { + params := a.kernelParams + + if debug { + params = append(params, a.kernelParamsDebug...) + } else { + params = append(params, a.kernelParamsNonDebug...) + } + + return params +} + +func (a *acrnArchBase) memoryTopology(memoryMb uint64) Memory { + mem := fmt.Sprintf("%dM", memoryMb) + memory := Memory{ + Size: mem, + } + + return memory +} + +func (a *acrnArchBase) capabilities() types.Capabilities { + var caps types.Capabilities + + // For devicemapper disable support for filesystem sharing + caps.SetFsSharingUnsupported() + caps.SetBlockDeviceSupport() + caps.SetBlockDeviceHotplugSupport() + + return caps +} + +// Valid returns true if the CharDevice structure is valid and complete. +func (cdev ConsoleDevice) Valid() bool { + if cdev.Backend != "tty" && cdev.Backend != "pty" && + cdev.Backend != "console" && cdev.Backend != "socket" && + cdev.Backend != "file" { + return false + } else if cdev.PortType != ConsoleBE && cdev.PortType != SerialBE { + return false + } else if cdev.Path == "" { + return false + } else { + return true + } +} + +// AcrnParams returns the acrn parameters built out of this console device. +func (cdev ConsoleDevice) AcrnParams(slot int, config *Config) []string { + var acrnParams []string + var deviceParams []string + + acrnParams = append(acrnParams, "-s") + deviceParams = append(deviceParams, fmt.Sprintf("%d,virtio-console,", slot)) + + if cdev.PortType == ConsoleBE { + deviceParams = append(deviceParams, "@") + } + + switch cdev.Backend { + case "pty": + deviceParams = append(deviceParams, "pty:pty_port") + case "tty": + deviceParams = append(deviceParams, fmt.Sprintf("tty:tty_port=%s", cdev.Path)) + case "socket": + deviceParams = append(deviceParams, fmt.Sprintf("socket:%s=%s", cdev.Name, cdev.Path)) + case "file": + deviceParams = append(deviceParams, fmt.Sprintf("file:file_port=%s", cdev.Path)) + case "stdio": + deviceParams = append(deviceParams, "stdio:stdio_port") + default: + // do nothing. Error should be already caught + } + + acrnParams = append(acrnParams, strings.Join(deviceParams, "")) + return acrnParams +} + +// AcrnNetdevParam converts to the acrn type to string +func (netdev NetDevice) AcrnNetdevParam() []string { + var deviceParams []string + + switch netdev.Type { + case TAP: + deviceParams = append(deviceParams, netdev.IFName) + deviceParams = append(deviceParams, fmt.Sprintf(",mac=%s", netdev.MACAddress)) + case MACVTAP: + deviceParams = append(deviceParams, netdev.IFName) + default: + deviceParams = append(deviceParams, netdev.IFName) + + } + + return deviceParams +} + +// Valid returns true if the NetDevice structure is valid and complete. +func (netdev NetDevice) Valid() bool { + if netdev.IFName == "" { + return false + } else if netdev.MACAddress == "" { + return false + } else if netdev.Type != TAP && netdev.Type != MACVTAP { + return false + } else { + return true + } +} + +// AcrnParams returns the acrn parameters built out of this network device. +func (netdev NetDevice) AcrnParams(slot int, config *Config) []string { + var acrnParams []string + + acrnParams = append(acrnParams, "-s") + acrnParams = append(acrnParams, fmt.Sprintf("%d,virtio-net,%s", slot, strings.Join(netdev.AcrnNetdevParam(), ""))) + + return acrnParams +} + +// Valid returns true if the BlockDevice structure is valid and complete. +func (blkdev BlockDevice) Valid() bool { + return blkdev.FilePath != "" +} + +// AcrnParams returns the acrn parameters built out of this block device. +func (blkdev BlockDevice) AcrnParams(slot int, config *Config) []string { + var acrnParams []string + + device := "virtio-blk" + acrnParams = append(acrnParams, "-s") + acrnParams = append(acrnParams, fmt.Sprintf("%d,%s,%s", + slot, device, blkdev.FilePath)) + + // Update the global array (BlkIndex<->slot) + // Used to identify slots for the hot-plugged virtio-blk devices + if blkdev.Index <= AcrnBlkDevPoolSz { + AcrnBlkdDevSlot[blkdev.Index] = slot + } else { + baselogger.WithFields(logrus.Fields{ + "device": device, + "index": blkdev.Index, + }).Info("Invalid index device") + } + + return acrnParams +} + +// Valid returns true if the BridgeDevice structure is valid and complete. +func (bridgeDev BridgeDevice) Valid() bool { + if bridgeDev.Function != 0 || bridgeDev.Emul != acrnHostBridge { + return false + } + return true +} + +// AcrnParams returns the acrn parameters built out of this bridge device. +func (bridgeDev BridgeDevice) AcrnParams(slot int, config *Config) []string { + var acrnParams []string + + acrnParams = append(acrnParams, "-s") + acrnParams = append(acrnParams, fmt.Sprintf("%d:%d,%s", slot, + bridgeDev.Function, bridgeDev.Emul)) + + return acrnParams +} + +// Valid returns true if the BridgeDevice structure is valid and complete. +func (lpcDev LPCDevice) Valid() bool { + return lpcDev.Emul == acrnLPCDev +} + +// AcrnParams returns the acrn parameters built out of this bridge device. +func (lpcDev LPCDevice) AcrnParams(slot int, config *Config) []string { + var acrnParams []string + var deviceParams []string + + acrnParams = append(acrnParams, "-s") + acrnParams = append(acrnParams, fmt.Sprintf("%d:%d,%s", slot, + lpcDev.Function, lpcDev.Emul)) + + //define UART port + deviceParams = append(deviceParams, "-l") + deviceParams = append(deviceParams, "com1,stdio") + acrnParams = append(acrnParams, strings.Join(deviceParams, "")) + + return acrnParams +} + +func (config *Config) appendName() { + if config.Name != "" { + config.acrnParams = append(config.acrnParams, config.Name) + } +} + +func (config *Config) appendDevices() { + slot := 0 + for _, d := range config.Devices { + if !d.Valid() { + continue + } + + if slot == acrnGVTgReservedSlot { + slot++ /*Slot 2 is assigned for GVT-g in acrn, so skip 2 */ + baselogger.Info("Slot 2 is assigned for GVT-g in acrn, so skipping this slot") + + } + config.acrnParams = append(config.acrnParams, d.AcrnParams(slot, config)...) + slot++ + } +} + +func (config *Config) appendUUID() { + if config.UUID != "" { + config.acrnParams = append(config.acrnParams, "-U") + config.acrnParams = append(config.acrnParams, config.UUID) + } +} + +func (config *Config) appendACPI() { + if config.ACPIVirt { + config.acrnParams = append(config.acrnParams, "-A") + } +} + +func (config *Config) appendMemory() { + if config.Memory.Size != "" { + config.acrnParams = append(config.acrnParams, "-m") + config.acrnParams = append(config.acrnParams, config.Memory.Size) + } +} + +func (config *Config) appendCPUs() { + if config.NumCPU != 0 { + config.acrnParams = append(config.acrnParams, "-c") + config.acrnParams = append(config.acrnParams, fmt.Sprintf("%d", config.NumCPU)) + } + +} + +func (config *Config) appendKernel() { + if config.Kernel.Path == "" { + return + } + config.acrnParams = append(config.acrnParams, "-k") + config.acrnParams = append(config.acrnParams, config.Kernel.Path) + + if config.Kernel.Params == "" { + return + } + config.acrnParams = append(config.acrnParams, "-B") + config.acrnParams = append(config.acrnParams, config.Kernel.Params) +} + +// LaunchAcrn can be used to launch a new acrn instance. +// +// The Config parameter contains a set of acrn parameters and settings. +// +// This function writes its log output via logger parameter. +func LaunchAcrn(config Config, logger *logrus.Entry) (int, string, error) { + baselogger = logger + config.appendUUID() + config.appendACPI() + config.appendMemory() + config.appendCPUs() + config.appendDevices() + config.appendKernel() + config.appendName() + + return LaunchCustomAcrn(context.Background(), config.Path, config.acrnParams, logger) +} + +// LaunchCustomAcrn can be used to launch a new acrn instance. +// +// The path parameter is used to pass the acrn executable path. +// +// params is a slice of options to pass to acrn-dm +// +// This function writes its log output via logger parameter. +func LaunchCustomAcrn(ctx context.Context, path string, params []string, + logger *logrus.Entry) (int, string, error) { + + errStr := "" + + if path == "" { + path = "acrn-dm" + } + + /* #nosec */ + cmd := exec.CommandContext(ctx, path, params...) + + var stderr bytes.Buffer + cmd.Stderr = &stderr + logger.WithFields(logrus.Fields{ + "Path": path, + "Params": params, + }).Info("launching acrn with:") + + err := cmd.Start() + if err != nil { + logger.Errorf("Unable to launch %s: %v", path, err) + errStr = stderr.String() + logger.Errorf("%s", errStr) + } + return cmd.Process.Pid, errStr, err +} + +func (a *acrnArchBase) appendImage(devices []Device, path string) ([]Device, error) { + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil, err + } + + ImgBlkdevice := BlockDevice{ + FilePath: path, + Index: 0, + } + + devices = append(devices, ImgBlkdevice) + + return devices, nil +} + +// appendBridges appends to devices the given bridges +func (a *acrnArchBase) appendBridges(devices []Device) []Device { + devices = append(devices, + BridgeDevice{ + Function: 0, + Emul: acrnHostBridge, + Config: "", + }, + ) + + return devices +} + +// appendBridges appends to devices the given bridges +func (a *acrnArchBase) appendLPC(devices []Device) []Device { + devices = append(devices, + LPCDevice{ + Function: 0, + Emul: acrnLPCDev, + }, + ) + + return devices +} + +func (a *acrnArchBase) appendConsole(devices []Device, path string) []Device { + console := ConsoleDevice{ + Name: "console0", + Backend: Socket, + PortType: ConsoleBE, + Path: path, + } + + devices = append(devices, console) + return devices +} + +func (a *acrnArchBase) appendSocket(devices []Device, socket types.Socket) []Device { + serailsocket := ConsoleDevice{ + Name: socket.Name, + Backend: Socket, + PortType: SerialBE, + Path: socket.HostPath, + } + + devices = append(devices, serailsocket) + return devices +} + +func networkModelToAcrnType(model NetInterworkingModel) NetDeviceType { + switch model { + case NetXConnectBridgedModel: + return TAP + case NetXConnectMacVtapModel: + return MACVTAP + default: + //TAP should work for most other cases + return TAP + } +} + +func (a *acrnArchBase) appendNetwork(devices []Device, endpoint Endpoint) []Device { + switch ep := endpoint.(type) { + case *VethEndpoint: + netPair := ep.NetworkPair() + devices = append(devices, + NetDevice{ + Type: networkModelToAcrnType(netPair.NetInterworkingModel), + IFName: netPair.TAPIface.Name, + MACAddress: netPair.TAPIface.HardAddr, + }, + ) + case *MacvtapEndpoint: + devices = append(devices, + NetDevice{ + Type: MACVTAP, + IFName: ep.Name(), + MACAddress: ep.HardwareAddr(), + }, + ) + default: + // Return devices as is for unsupported endpoint. + baselogger.WithField("Endpoint", endpoint).Error("Unsupported N/W Endpoint") + } + + return devices +} + +func (a *acrnArchBase) appendBlockDevice(devices []Device, drive config.BlockDrive) []Device { + if drive.File == "" { + return devices + } + + devices = append(devices, + BlockDevice{ + FilePath: drive.File, + Index: drive.Index, + }, + ) + + return devices +} + +func (a *acrnArchBase) handleImagePath(config HypervisorConfig) { + if config.ImagePath != "" { + a.kernelParams = append(a.kernelParams, acrnKernelRootParams...) + a.kernelParamsNonDebug = append(a.kernelParamsNonDebug, acrnKernelParamsSystemdNonDebug...) + a.kernelParamsDebug = append(a.kernelParamsDebug, acrnKernelParamsSystemdDebug...) + } +} diff --git a/virtcontainers/acrn_arch_base_test.go b/virtcontainers/acrn_arch_base_test.go new file mode 100644 index 0000000000..1ba57ac637 --- /dev/null +++ b/virtcontainers/acrn_arch_base_test.go @@ -0,0 +1,299 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "fmt" + "io/ioutil" + "net" + "path/filepath" + "testing" + + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/store" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/stretchr/testify/assert" +) + +const ( + acrnArchBaseAcrnPath = "/usr/bin/acrn" + acrnArchBaseAcrnCtlPath = "/usr/bin/acrnctl" +) + +var acrnArchBaseKernelParamsNonDebug = []Param{ + {"quiet", ""}, +} + +var acrnArchBaseKernelParamsDebug = []Param{ + {"debug", ""}, +} + +var acrnArchBaseKernelParams = []Param{ + {"root", "/dev/vda"}, +} + +func newAcrnArchBase() *acrnArchBase { + return &acrnArchBase{ + path: acrnArchBaseAcrnPath, + ctlpath: acrnArchBaseAcrnCtlPath, + kernelParamsNonDebug: acrnArchBaseKernelParamsNonDebug, + kernelParamsDebug: acrnArchBaseKernelParamsDebug, + kernelParams: acrnArchBaseKernelParams, + } +} + +func TestAcrnArchBaseAcrnPaths(t *testing.T) { + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + p, err := acrnArchBase.acrnPath() + assert.NoError(err) + assert.Equal(p, acrnArchBaseAcrnPath) + + ctlp, err := acrnArchBase.acrnctlPath() + assert.NoError(err) + assert.Equal(ctlp, acrnArchBaseAcrnCtlPath) +} + +func TestAcrnArchBaseKernelParameters(t *testing.T) { + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + // with debug params + expectedParams := acrnArchBaseKernelParams + debugParams := acrnArchBaseKernelParamsDebug + expectedParams = append(expectedParams, debugParams...) + p := acrnArchBase.kernelParameters(true) + assert.Equal(expectedParams, p) + + // with non-debug params + expectedParams = acrnArchBaseKernelParams + nonDebugParams := acrnArchBaseKernelParamsNonDebug + expectedParams = append(expectedParams, nonDebugParams...) + p = acrnArchBase.kernelParameters(false) + assert.Equal(expectedParams, p) +} + +func TestAcrnArchBaseCapabilities(t *testing.T) { + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + c := acrnArchBase.capabilities() + assert.True(c.IsBlockDeviceSupported()) + assert.True(c.IsBlockDeviceHotplugSupported()) + assert.False(c.IsFsSharingSupported()) +} + +func TestAcrnArchBaseMemoryTopology(t *testing.T) { + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + mem := uint64(8192) + + expectedMemory := Memory{ + Size: fmt.Sprintf("%dM", mem), + } + + m := acrnArchBase.memoryTopology(mem) + assert.Equal(expectedMemory, m) +} + +func TestAcrnArchBaseAppendConsoles(t *testing.T) { + var devices []Device + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + path := filepath.Join(store.SandboxRuntimeRootPath(sandboxID), consoleSocket) + + expectedOut := []Device{ + ConsoleDevice{ + Name: "console0", + Backend: Socket, + PortType: ConsoleBE, + Path: path, + }, + } + + devices = acrnArchBase.appendConsole(devices, path) + assert.Equal(expectedOut, devices) +} + +func TestAcrnArchBaseAppendImage(t *testing.T) { + var devices []Device + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + image, err := ioutil.TempFile("", "img") + assert.NoError(err) + err = image.Close() + assert.NoError(err) + + devices, err = acrnArchBase.appendImage(devices, image.Name()) + assert.NoError(err) + assert.Len(devices, 1) + + expectedOut := []Device{ + BlockDevice{ + FilePath: image.Name(), + Index: 0, + }, + } + + assert.Equal(expectedOut, devices) +} + +func TestAcrnArchBaseAppendBridges(t *testing.T) { + function := 0 + emul := acrnHostBridge + config := "" + + var devices []Device + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + devices = acrnArchBase.appendBridges(devices) + assert.Len(devices, 1) + + expectedOut := []Device{ + BridgeDevice{ + Function: function, + Emul: emul, + Config: config, + }, + } + + assert.Equal(expectedOut, devices) +} + +func TestAcrnArchBaseAppendLpcDevice(t *testing.T) { + function := 0 + emul := acrnLPCDev + + var devices []Device + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + devices = acrnArchBase.appendLPC(devices) + assert.Len(devices, 1) + + expectedOut := []Device{ + LPCDevice{ + Function: function, + Emul: emul, + }, + } + + assert.Equal(expectedOut, devices) +} + +func testAcrnArchBaseAppend(t *testing.T, structure interface{}, expected []Device) { + var devices []Device + var err error + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + switch s := structure.(type) { + case types.Socket: + devices = acrnArchBase.appendSocket(devices, s) + case config.BlockDrive: + devices = acrnArchBase.appendBlockDevice(devices, s) + } + + assert.NoError(err) + assert.Equal(devices, expected) +} + +func TestAcrnArchBaseAppendSocket(t *testing.T) { + name := "archserial.test" + hostPath := "/tmp/archserial.sock" + + expectedOut := []Device{ + ConsoleDevice{ + Name: name, + Backend: Socket, + PortType: SerialBE, + Path: hostPath, + }, + } + + socket := types.Socket{ + HostPath: hostPath, + Name: name, + } + + testAcrnArchBaseAppend(t, socket, expectedOut) +} + +func TestAcrnArchBaseAppendBlockDevice(t *testing.T) { + path := "/tmp/archtest.img" + index := 5 + + expectedOut := []Device{ + BlockDevice{ + FilePath: path, + Index: index, + }, + } + + drive := config.BlockDrive{ + File: path, + Index: index, + } + + testAcrnArchBaseAppend(t, drive, expectedOut) +} + +func TestAcrnArchBaseAppendNetwork(t *testing.T) { + var devices []Device + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} + + vethEp := &VethEndpoint{ + NetPair: NetworkInterfacePair{ + TapInterface: TapInterface{ + ID: "uniqueTestID0", + Name: "br0_kata", + TAPIface: NetworkInterface{ + Name: "tap0_kata", + }, + }, + VirtIface: NetworkInterface{ + Name: "eth0", + HardAddr: macAddr.String(), + }, + NetInterworkingModel: DefaultNetInterworkingModel, + }, + EndpointType: VethEndpointType, + } + + macvtapEp := &MacvtapEndpoint{ + EndpointType: MacvtapEndpointType, + EndpointProperties: NetworkInfo{ + Iface: NetlinkIface{ + Type: "macvtap", + }, + }, + } + + expectedOut := []Device{ + NetDevice{ + Type: TAP, + IFName: vethEp.NetPair.TAPIface.Name, + MACAddress: vethEp.NetPair.TAPIface.HardAddr, + }, + NetDevice{ + Type: MACVTAP, + IFName: macvtapEp.Name(), + MACAddress: macvtapEp.HardwareAddr(), + }, + } + + devices = acrnArchBase.appendNetwork(devices, vethEp) + devices = acrnArchBase.appendNetwork(devices, macvtapEp) + assert.Equal(expectedOut, devices) +} diff --git a/virtcontainers/acrn_test.go b/virtcontainers/acrn_test.go new file mode 100644 index 0000000000..c57f682409 --- /dev/null +++ b/virtcontainers/acrn_test.go @@ -0,0 +1,285 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "context" + "fmt" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/store" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/stretchr/testify/assert" +) + +func newAcrnConfig() HypervisorConfig { + return HypervisorConfig{ + KernelPath: testAcrnKernelPath, + ImagePath: testAcrnImagePath, + HypervisorPath: testAcrnPath, + HypervisorCtlPath: testAcrnCtlPath, + NumVCPUs: defaultVCPUs, + MemorySize: defaultMemSzMiB, + BlockDeviceDriver: config.VirtioBlock, + DefaultBridges: defaultBridges, + DefaultMaxVCPUs: MaxAcrnVCPUs(), + // Adding this here, as hypervisorconfig.valid() + // forcefully adds it even when 9pfs is not supported + Msize9p: defaultMsize9p, + } +} + +func testAcrnKernelParameters(t *testing.T, kernelParams []Param, debug bool) { + acrnConfig := newAcrnConfig() + acrnConfig.KernelParams = kernelParams + + if debug == true { + acrnConfig.Debug = true + } + + a := &acrn{ + config: acrnConfig, + arch: &acrnArchBase{}, + } + + expected := fmt.Sprintf("panic=1 maxcpus=%d foo=foo bar=bar", a.config.NumVCPUs) + + params := a.kernelParameters() + if params != expected { + t.Fatalf("Got: %v, Expecting: %v", params, expected) + } +} + +func TestAcrnKernelParameters(t *testing.T) { + params := []Param{ + { + Key: "foo", + Value: "foo", + }, + { + Key: "bar", + Value: "bar", + }, + } + + testAcrnKernelParameters(t, params, true) + testAcrnKernelParameters(t, params, false) +} + +func TestAcrnCapabilities(t *testing.T) { + a := &acrn{ + ctx: context.Background(), + arch: &acrnArchBase{}, + } + + caps := a.capabilities() + if !caps.IsBlockDeviceSupported() { + t.Fatal("Block device should be supported") + } + + if !caps.IsBlockDeviceHotplugSupported() { + t.Fatal("Block device hotplug should be supported") + } +} + +func testAcrnAddDevice(t *testing.T, devInfo interface{}, devType deviceType, expected []Device) { + a := &acrn{ + ctx: context.Background(), + arch: &acrnArchBase{}, + } + + err := a.addDevice(devInfo, devType) + if err != nil { + t.Fatal(err) + } + + if reflect.DeepEqual(a.acrnConfig.Devices, expected) == false { + t.Fatalf("Got %v\nExpecting %v", a.acrnConfig.Devices, expected) + } +} + +func TestAcrnAddDeviceSerialPortDev(t *testing.T) { + name := "serial.test" + hostPath := "/tmp/serial.sock" + + expectedOut := []Device{ + ConsoleDevice{ + Name: name, + Backend: Socket, + PortType: SerialBE, + Path: hostPath, + }, + } + + socket := types.Socket{ + HostPath: hostPath, + Name: name, + } + + testAcrnAddDevice(t, socket, serialPortDev, expectedOut) +} + +func TestAcrnAddDeviceBlockDev(t *testing.T) { + path := "/tmp/test.img" + index := 1 + + expectedOut := []Device{ + BlockDevice{ + FilePath: path, + Index: index, + }, + } + + drive := config.BlockDrive{ + File: path, + Index: index, + } + + testAcrnAddDevice(t, drive, blockDev, expectedOut) +} + +func TestAcrnHotplugUnsupportedDeviceType(t *testing.T) { + assert := assert.New(t) + + acrnConfig := newAcrnConfig() + a := &acrn{ + ctx: context.Background(), + id: "acrnTest", + config: acrnConfig, + } + + _, err := a.hotplugAddDevice(&memoryDevice{0, 128, uint64(0), false}, fsDev) + assert.Error(err) +} + +func TestAcrnUpdateBlockDeviceInvalidPath(t *testing.T) { + assert := assert.New(t) + + path := "" + index := 1 + + acrnConfig := newAcrnConfig() + a := &acrn{ + ctx: context.Background(), + id: "acrnBlkTest", + config: acrnConfig, + } + + drive := &config.BlockDrive{ + File: path, + Index: index, + } + + err := a.updateBlockDevice(drive) + assert.Error(err) +} + +func TestAcrnUpdateBlockDeviceInvalidIdx(t *testing.T) { + assert := assert.New(t) + + path := "/tmp/test.img" + index := AcrnBlkDevPoolSz + 1 + + acrnConfig := newAcrnConfig() + a := &acrn{ + ctx: context.Background(), + id: "acrnBlkTest", + config: acrnConfig, + } + + drive := &config.BlockDrive{ + File: path, + Index: index, + } + + err := a.updateBlockDevice(drive) + assert.Error(err) +} + +func TestAcrnGetSandboxConsole(t *testing.T) { + a := &acrn{ + ctx: context.Background(), + } + sandboxID := "testSandboxID" + expected := filepath.Join(store.RunVMStoragePath, sandboxID, consoleSocket) + + result, err := a.getSandboxConsole(sandboxID) + if err != nil { + t.Fatal(err) + } + + if result != expected { + t.Fatalf("Got %s\nExpecting %s", result, expected) + } +} + +func TestAcrnCreateSandbox(t *testing.T) { + acrnConfig := newAcrnConfig() + a := &acrn{} + + sandbox := &Sandbox{ + ctx: context.Background(), + id: "testSandbox", + config: &SandboxConfig{ + HypervisorConfig: acrnConfig, + }, + } + + vcStore, err := store.NewVCSandboxStore(sandbox.ctx, sandbox.id) + if err != nil { + t.Fatal(err) + } + sandbox.store = vcStore + + if err = globalSandboxList.addSandbox(sandbox); err != nil { + t.Fatalf("Could not add sandbox to global list: %v", err) + } + + defer globalSandboxList.removeSandbox(sandbox.id) + + // Create the hypervisor fake binary + testAcrnPath := filepath.Join(testDir, testHypervisor) + _, err = os.Create(testAcrnPath) + if err != nil { + t.Fatalf("Could not create hypervisor file %s: %v", testAcrnPath, err) + } + + if err := a.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil { + t.Fatal(err) + } + + if reflect.DeepEqual(acrnConfig, a.config) == false { + t.Fatalf("Got %v\nExpecting %v", a.config, acrnConfig) + } +} + +func TestAcrnMemoryTopology(t *testing.T) { + mem := uint32(1000) + + a := &acrn{ + arch: &acrnArchBase{}, + config: HypervisorConfig{ + MemorySize: mem, + }, + } + + expectedOut := Memory{ + Size: fmt.Sprintf("%dM", mem), + } + + memory, err := a.memoryTopology() + if err != nil { + t.Fatal(err) + } + + if reflect.DeepEqual(memory, expectedOut) == false { + t.Fatalf("Got %v\nExpecting %v", memory, expectedOut) + } +} diff --git a/virtcontainers/device/api/interface.go b/virtcontainers/device/api/interface.go index 828049c6f6..f4060c046a 100644 --- a/virtcontainers/device/api/interface.go +++ b/virtcontainers/device/api/interface.go @@ -7,10 +7,9 @@ package api import ( - "github.com/sirupsen/logrus" - "github.com/kata-containers/runtime/virtcontainers/device/config" persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" + "github.com/sirupsen/logrus" ) var devLogger = logrus.WithField("subsystem", "device") @@ -36,6 +35,7 @@ type DeviceReceiver interface { // this is only for virtio-blk and virtio-scsi support GetAndSetSandboxBlockIndex() (int, error) DecrementSandboxBlockIndex() error + GetHypervisorType() string // this is for appending device to hypervisor boot params AppendDevice(Device) error diff --git a/virtcontainers/device/api/mockDeviceReceiver.go b/virtcontainers/device/api/mockDeviceReceiver.go index c080d072d9..e7d937deda 100644 --- a/virtcontainers/device/api/mockDeviceReceiver.go +++ b/virtcontainers/device/api/mockDeviceReceiver.go @@ -36,3 +36,8 @@ func (mockDC *MockDeviceReceiver) DecrementSandboxBlockIndex() error { func (mockDC *MockDeviceReceiver) AppendDevice(Device) error { return nil } + +// GetHypervisorType is used for getting Hypervisor name currently used. +func (mockDC *MockDeviceReceiver) GetHypervisorType() string { + return "" +} diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 9ad51ff061..15ce20e93f 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -31,6 +31,9 @@ const ( // QemuHypervisor is the QEMU hypervisor. QemuHypervisor HypervisorType = "qemu" + // AcrnHypervisor is the ACRN hypervisor. + AcrnHypervisor HypervisorType = "acrn" + // MockHypervisor is a mock hypervisor for testing purposes MockHypervisor HypervisorType = "mock" ) @@ -118,6 +121,9 @@ func (hType *HypervisorType) Set(value string) error { case "firecracker": *hType = FirecrackerHypervisor return nil + case "acrn": + *hType = AcrnHypervisor + return nil case "mock": *hType = MockHypervisor return nil @@ -133,6 +139,8 @@ func (hType *HypervisorType) String() string { return string(QemuHypervisor) case FirecrackerHypervisor: return string(FirecrackerHypervisor) + case AcrnHypervisor: + return string(AcrnHypervisor) case MockHypervisor: return string(MockHypervisor) default: @@ -147,6 +155,8 @@ func newHypervisor(hType HypervisorType) (hypervisor, error) { return &qemu{}, nil case FirecrackerHypervisor: return &firecracker{}, nil + case AcrnHypervisor: + return &acrn{}, nil case MockHypervisor: return &mockHypervisor{}, nil default: @@ -212,6 +222,9 @@ type HypervisorConfig struct { // HypervisorPath is the hypervisor executable host path. HypervisorPath string + // HypervisorCtlPath is the hypervisor ctl executable host path. + HypervisorCtlPath string + // BlockDeviceDriver specifies the driver to be used for block device // either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver BlockDeviceDriver string @@ -430,6 +443,8 @@ func (conf *HypervisorConfig) assetPath(t types.AssetType) (string, error) { return conf.InitrdPath, nil case types.HypervisorAsset: return conf.HypervisorPath, nil + case types.HypervisorCtlAsset: + return conf.HypervisorCtlPath, nil case types.FirmwareAsset: return conf.FirmwarePath, nil default: @@ -477,6 +492,11 @@ func (conf *HypervisorConfig) HypervisorAssetPath() (string, error) { return conf.assetPath(types.HypervisorAsset) } +// HypervisorCtlAssetPath returns the VM hypervisor ctl path +func (conf *HypervisorConfig) HypervisorCtlAssetPath() (string, error) { + return conf.assetPath(types.HypervisorCtlAsset) +} + // CustomHypervisorAsset returns true if the hypervisor asset is a custom one, false otherwise. func (conf *HypervisorConfig) CustomHypervisorAsset() bool { return conf.isCustomAsset(types.HypervisorAsset) diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 61e739d172..d11f65c217 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -19,6 +19,7 @@ import ( "syscall" "time" + "github.com/gogo/protobuf/proto" aTypes "github.com/kata-containers/agent/pkg/types" kataclient "github.com/kata-containers/agent/protocols/client" "github.com/kata-containers/agent/protocols/grpc" @@ -30,10 +31,8 @@ import ( "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" - opentracing "github.com/opentracing/opentracing-go" - - "github.com/gogo/protobuf/proto" "github.com/opencontainers/runtime-spec/specs-go" + opentracing "github.com/opentracing/opentracing-go" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" "golang.org/x/net/context" @@ -1084,7 +1083,12 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat rootfs.Source = blockDrive.VirtPath } else if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock { rootfs.Driver = kataBlkDevType - rootfs.Source = blockDrive.PCIAddr + if blockDrive.PCIAddr == "" { + rootfs.Source = blockDrive.VirtPath + } else { + rootfs.Source = blockDrive.PCIAddr + } + } else { rootfs.Driver = kataSCSIDevType rootfs.Source = blockDrive.SCSIAddr diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 8be04e9c05..e44e103e70 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -15,12 +15,6 @@ import ( "syscall" "github.com/containernetworking/plugins/pkg/ns" - specs "github.com/opencontainers/runtime-spec/specs-go" - opentracing "github.com/opentracing/opentracing-go" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/vishvananda/netlink" - "github.com/kata-containers/agent/protocols/grpc" "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" @@ -34,6 +28,11 @@ import ( "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" + specs "github.com/opencontainers/runtime-spec/specs-go" + opentracing "github.com/opentracing/opentracing-go" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" ) const ( @@ -1842,3 +1841,9 @@ func (s *Sandbox) calculateSandboxCPUs() uint32 { } return utils.CalculateVCpusFromMilliCpus(mCPU) } + +// GetHypervisorType is used for getting Hypervisor name currently used. +// Sandbox implement DeviceReceiver interface from device/api/interface.go +func (s *Sandbox) GetHypervisorType() string { + return string(s.config.HypervisorType) +} diff --git a/virtcontainers/types/asset.go b/virtcontainers/types/asset.go index dd9bab3b7c..7c9afd374c 100644 --- a/virtcontainers/types/asset.go +++ b/virtcontainers/types/asset.go @@ -49,6 +49,9 @@ const ( // HypervisorAsset is an hypervisor asset. HypervisorAsset AssetType = "hypervisor" + // HypervisorCtlAsset is an hypervisor control asset. + HypervisorCtlAsset AssetType = "hypervisorctl" + // FirmwareAsset is a firmware asset. FirmwareAsset AssetType = "firmware" ) diff --git a/virtcontainers/virtcontainers_test.go b/virtcontainers/virtcontainers_test.go index 424a83ed6d..21c499d524 100644 --- a/virtcontainers/virtcontainers_test.go +++ b/virtcontainers/virtcontainers_test.go @@ -25,6 +25,7 @@ const testKernel = "kernel" const testInitrd = "initrd" const testImage = "image" const testHypervisor = "hypervisor" +const testHypervisorCtl = "hypervisorctl" const testBundle = "bundle" const testDisabledAsNonRoot = "Test disabled as requires root privileges" @@ -41,6 +42,10 @@ var testQemuKernelPath = "" var testQemuInitrdPath = "" var testQemuImagePath = "" var testQemuPath = "" +var testAcrnKernelPath = "" +var testAcrnImagePath = "" +var testAcrnPath = "" +var testAcrnCtlPath = "" var testHyperstartCtlSocket = "" var testHyperstartTtySocket = "" @@ -67,6 +72,18 @@ func setup() { } } +func setupAcrn() { + os.Mkdir(filepath.Join(testDir, testBundle), store.DirMode) + + for _, filename := range []string{testAcrnKernelPath, testAcrnImagePath, testAcrnPath, testAcrnCtlPath} { + _, err := os.Create(filename) + if err != nil { + fmt.Printf("Could not recreate %s:%v", filename, err) + os.Exit(1) + } + } +} + // TestMain is the common main function used by ALL the test functions // for this package. func TestMain(m *testing.M) { @@ -102,6 +119,13 @@ func TestMain(m *testing.M) { setup() + testAcrnKernelPath = filepath.Join(testDir, testKernel) + testAcrnImagePath = filepath.Join(testDir, testImage) + testAcrnPath = filepath.Join(testDir, testHypervisor) + testAcrnCtlPath = filepath.Join(testDir, testHypervisorCtl) + + setupAcrn() + // allow the tests to run without affecting the host system. store.ConfigStoragePath = filepath.Join(testDir, store.StoragePathSuffix, "config") store.RunStoragePath = filepath.Join(testDir, store.StoragePathSuffix, "run")