From 01713d59cb634b07720a11a12907399584dba5e1 Mon Sep 17 00:00:00 2001 From: Johan Kuijpers Date: Tue, 5 Nov 2019 15:52:59 +0100 Subject: [PATCH] runtime: added cloud hypervisor driver Initial release of cloud hypervisor driver for kata-runtime Fixes: #2046 Signed-off-by: Johan Kuijpers --- Makefile | 45 +- arch/amd64-options.mk | 3 + cli/config/configuration-clh.toml.in | 212 ++++ cli/kata-check_amd64.go | 4 + pkg/katautils/config.go | 90 ++ pkg/katautils/config_test.go | 60 ++ virtcontainers/clh.go | 1294 +++++++++++++++++++++++++ virtcontainers/hypervisor.go | 10 + virtcontainers/virtcontainers_test.go | 1 + 9 files changed, 1718 insertions(+), 1 deletion(-) create mode 100644 cli/config/configuration-clh.toml.in create mode 100644 virtcontainers/clh.go diff --git a/Makefile b/Makefile index 5baf7d5e1e..a06549f9af 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,7 @@ endif PREFIXDEPS := $(PREFIX) BINDIR := $(EXEC_PREFIX)/bin QEMUBINDIR := $(PREFIXDEPS)/bin +CLHBINDIR := $(PREFIXDEPS)/bin FCBINDIR := $(PREFIXDEPS)/bin ACRNBINDIR := $(PREFIXDEPS)/bin VIRTIOFSDBINDIR := $(PREFIXDEPS)/bin @@ -117,18 +118,21 @@ HYPERVISOR_ACRN = acrn HYPERVISOR_FC = firecracker JAILER_FC = jailer HYPERVISOR_QEMU = qemu +HYPERVISOR_CLH = cloud-hypervisor HYPERVISOR_QEMU_VIRTIOFS = qemu-virtiofs # Determines which hypervisor is specified in $(CONFIG_FILE). DEFAULT_HYPERVISOR = $(HYPERVISOR_QEMU) # List of hypervisors this build system can generate configuration for. -HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_QEMU_VIRTIOFS) +HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_QEMU_VIRTIOFS) $(HYPERVISOR_CLH) QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD) QEMUVIRTIOFSPATH := $(QEMUBINDIR)/$(QEMUVIRTIOFSCMD) +CLHPATH := $(CLHBINDIR)/$(CLHCMD) + FCPATH = $(FCBINDIR)/$(FCCMD) FCJAILERPATH = $(FCBINDIR)/$(FCJAILERCMD) @@ -258,6 +262,30 @@ ifneq (,$(QEMUVIRTIOFSCMD)) KERNELVIRTIOFSPATH = $(KERNELDIR)/$(KERNELNAMEVIRTIOFS) endif +ifneq (,$(CLHCMD)) + KNOWN_HYPERVISORS += $(HYPERVISOR_CLH) + + CONFIG_FILE_CLH = configuration-clh.toml + CONFIG_CLH = $(CLI_DIR)/config/$(CONFIG_FILE_CLH) + CONFIG_CLH_IN = $(CONFIG_CLH).in + + CONFIG_PATH_CLH = $(abspath $(CONFDIR)/$(CONFIG_FILE_CLH)) + CONFIG_PATHS += $(CONFIG_PATH_CLH) + + SYSCONFIG_CLH = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_CLH)) + SYSCONFIG_PATHS += $(SYSCONFIG_CLH) + + CONFIGS += $(CONFIG_CLH) + + # CLH-specific options (all should be suffixed by "_CLH") + # currently, huge pages are required for virtiofsd support + DEFENABLEHUGEPAGES_CLH := true + DEFNETWORKMODEL_CLH := tcfilter + KERNELTYPE_CLH = uncompressed + KERNEL_NAME_CLH = $(call MAKE_KERNEL_NAME,$(KERNELTYPE_CLH)) + KERNELPATH_CLH = $(KERNELDIR)/$(KERNEL_NAME_CLH) +endif + ifneq (,$(FCCMD)) KNOWN_HYPERVISORS += $(HYPERVISOR_FC) @@ -332,6 +360,10 @@ ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_ACRN)) DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_ACRN) endif +ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_CLH)) + DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_CLH) +endif + CONFDIR := $(DEFAULTSDIR)/$(PROJECT_DIR) SYSCONFDIR := $(SYSCONFDIR)/$(PROJECT_DIR) @@ -367,8 +399,10 @@ USER_VARS += KERNELDIR USER_VARS += KERNELTYPE USER_VARS += KERNELTYPE_FC USER_VARS += KERNELTYPE_ACRN +USER_VARS += KERNELTYPE_CLH USER_VARS += FIRMWAREPATH USER_VARS += MACHINEACCELERATORS +USER_VARS += DEFMACHINETYPE_CLH USER_VARS += KERNELPARAMS USER_VARS += LIBEXECDIR USER_VARS += LOCALSTATEDIR @@ -519,21 +553,26 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@CONFIG_ACRN_IN@|$(CONFIG_ACRN_IN)|g" \ -e "s|@CONFIG_QEMU_IN@|$(CONFIG_QEMU_IN)|g" \ -e "s|@CONFIG_QEMU_VIRTIOFS_IN@|$(CONFIG_QEMU_VIRTIOFS_IN)|g" \ + -e "s|@CONFIG_CLH_IN@|$(CONFIG_CLH_IN)|g" \ -e "s|@CONFIG_FC_IN@|$(CONFIG_FC_IN)|g" \ -e "s|@CONFIG_PATH@|$(CONFIG_PATH)|g" \ -e "s|@FCPATH@|$(FCPATH)|g" \ -e "s|@FCJAILERPATH@|$(FCJAILERPATH)|g" \ -e "s|@ACRNPATH@|$(ACRNPATH)|g" \ -e "s|@ACRNCTLPATH@|$(ACRNCTLPATH)|g" \ + -e "s|@CLHPATH@|$(CLHPATH)|g" \ -e "s|@SYSCONFIG@|$(SYSCONFIG)|g" \ -e "s|@IMAGEPATH@|$(IMAGEPATH)|g" \ -e "s|@KERNELPATH_ACRN@|$(KERNELPATH_ACRN)|g" \ -e "s|@KERNELPATH_FC@|$(KERNELPATH_FC)|g" \ + -e "s|@KERNELPATH_CLH@|$(KERNELPATH_CLH)|g" \ -e "s|@KERNELPATH@|$(KERNELPATH)|g" \ -e "s|@KERNELVIRTIOFSPATH@|$(KERNELVIRTIOFSPATH)|g" \ -e "s|@INITRDPATH@|$(INITRDPATH)|g" \ -e "s|@FIRMWAREPATH@|$(FIRMWAREPATH)|g" \ -e "s|@MACHINEACCELERATORS@|$(MACHINEACCELERATORS)|g" \ + -e "s|@FIRMWAREPATH_CLH@|$(FIRMWAREPATH_CLH)|g" \ + -e "s|@DEFMACHINETYPE_CLH@|$(DEFMACHINETYPE_CLH)|g" \ -e "s|@KERNELPARAMS@|$(KERNELPARAMS)|g" \ -e "s|@LOCALSTATEDIR@|$(LOCALSTATEDIR)|g" \ -e "s|@PKGLIBEXECDIR@|$(PKGLIBEXECDIR)|g" \ @@ -557,6 +596,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@DEFMEMSLOTS@|$(DEFMEMSLOTS)|g" \ -e "s|@DEFBRIDGES@|$(DEFBRIDGES)|g" \ -e "s|@DEFNETWORKMODEL_ACRN@|$(DEFNETWORKMODEL_ACRN)|g" \ + -e "s|@DEFNETWORKMODEL_CLH@|$(DEFNETWORKMODEL_CLH)|g" \ -e "s|@DEFNETWORKMODEL_FC@|$(DEFNETWORKMODEL_FC)|g" \ -e "s|@DEFNETWORKMODEL_QEMU@|$(DEFNETWORKMODEL_QEMU)|g" \ -e "s|@DEFDISABLEGUESTSECCOMP@|$(DEFDISABLEGUESTSECCOMP)|g" \ @@ -729,6 +769,9 @@ endif ifneq (,$(findstring $(HYPERVISOR_QEMU_VIRTIOFS),$(KNOWN_HYPERVISORS))) @printf "\t$(HYPERVISOR_QEMU_VIRTIOFS) hypervisor path (QEMUVIRTIOFSPATH) : %s\n" $(abspath $(QEMUVIRTIOFSPATH)) endif +ifneq (,$(findstring $(HYPERVISOR_CLH),$(KNOWN_HYPERVISORS))) + @printf "\t$(HYPERVISOR_CLH) hypervisor path (CLHPATH) : %s\n" $(abspath $(CLHPATH)) +endif ifneq (,$(findstring $(HYPERVISOR_FC),$(KNOWN_HYPERVISORS))) @printf "\t$(HYPERVISOR_FC) hypervisor path (FCPATH) : %s\n" $(abspath $(FCPATH)) endif diff --git a/arch/amd64-options.mk b/arch/amd64-options.mk index 797372f607..7bfc5ff97f 100644 --- a/arch/amd64-options.mk +++ b/arch/amd64-options.mk @@ -22,3 +22,6 @@ FCJAILERCMD := jailer #ACRN binary name ACRNCMD := acrn-dm ACRNCTLCMD := acrnctl + +# cloud-hypervisor binary name +CLHCMD := cloud-hypervisor diff --git a/cli/config/configuration-clh.toml.in b/cli/config/configuration-clh.toml.in new file mode 100644 index 0000000000..a63b6c8753 --- /dev/null +++ b/cli/config/configuration-clh.toml.in @@ -0,0 +1,212 @@ +# Copyright (c) 2019 Ericsson Eurolab Deutschland GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +# XXX: WARNING: this file is auto-generated. +# XXX: +# XXX: Source file: "@CONFIG_CLH_IN@" +# XXX: Project: +# XXX: Name: @PROJECT_NAME@ +# XXX: Type: @PROJECT_TYPE@ + +[hypervisor.clh] +path = "@CLHPATH@" +kernel = "@KERNELPATH_CLH@" +image = "@IMAGEPATH@" + +# Optional space-separated list of options to pass to the guest kernel. +# For example, use `kernel_params = "vsyscall=emulate"` if you are having +# trouble running pre-2.15 glibc. +# +# WARNING: - any parameter specified here will take priority over the default +# parameter value of the same name used to start the virtual machine. +# Do not set values here unless you understand the impact of doing so as you +# may stop the virtual machine from booting. +# To see the list of default parameters, enable hypervisor debug, create a +# container and look for 'default-kernel-parameters' log entries. +kernel_params = "@KERNELPARAMS@" + +# Default number of vCPUs per SB/VM: +# unspecified or 0 --> will be set to @DEFVCPUS@ +# < 0 --> will be set to the actual number of physical cores +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores +default_vcpus = 1 + +# Default maximum number of vCPUs per SB/VM: +# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when +# the actual number of physical cores is greater than it. +# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU +# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs +# can be added to a SB/VM, but the memory footprint will be big. Another example, with +# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of +# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable, +# unless you know what are you doing. +default_maxvcpus = @DEFMAXVCPUS@ + +# Default memory size in MiB for SB/VM. +# If unspecified then it will be set @DEFMEMSZ@ MiB. +default_memory = @DEFMEMSZ@ + +# Default memory slots per SB/VM. +# If unspecified then it will be set @DEFMEMSLOTS@. +# This is will determine the times that memory will be hotadded to sandbox/VM. +#memory_slots = @DEFMEMSLOTS@ + +# Path to vhost-user-fs daemon. +virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" + +# cloud-hypervisor prefers virtiofs caching (dax) for performance reasons +virtio_fs_cache = "always" + +# This option changes the default hypervisor and kernel parameters +# to enable debug output where available. This extra output is added +# to the proxy logs, but only when proxy debug is also enabled. +# +# Default false +# enable_debug = true + +[proxy.@PROJECT_TYPE@] +path = "@PROXYPATH@" + +# If enabled, proxy messages will be sent to the system log +# (default: disabled) +#enable_debug = true + +[shim.@PROJECT_TYPE@] +path = "@SHIMPATH@" + +# If enabled, shim messages will be sent to the system log +# (default: disabled) +#enable_debug = true + +# If enabled, the shim will create opentracing.io traces and spans. +# (See https://www.jaegertracing.io/docs/getting-started). +# +# Note: By default, the shim runs in a separate network namespace. Therefore, +# to allow it to send trace details to the Jaeger agent running on the host, +# it is necessary to set 'disable_new_netns=true' so that it runs in the host +# network namespace. +# +# (default: disabled) +#enable_tracing = true + + +[agent.@PROJECT_TYPE@] +# If enabled, make the agent display debug-level messages. +# (default: disabled) +#enable_debug = true + +# Enable agent tracing. +# +# If enabled, the default trace mode is "dynamic" and the +# default trace type is "isolated". The trace mode and type are set +# explicity with the `trace_type=` and `trace_mode=` options. +# +# Notes: +# +# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly +# setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing` +# will NOT activate agent tracing. +# +# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for +# full details. +# +# (default: disabled) +#enable_tracing = true +# +#trace_mode = "dynamic" +#trace_type = "isolated" + + +[netmon] +# If enabled, the network monitoring process gets started when the +# sandbox is created. This allows for the detection of some additional +# network being added to the existing network namespace, after the +# sandbox has been created. +# (default: disabled) +#enable_netmon = true + +# Specify the path to the netmon binary. +path = "@NETMONPATH@" + +# If enabled, netmon messages will be sent to the system log +# (default: disabled) +#enable_debug = true + + +[runtime] +# If enabled, the runtime will log additional debug messages to the +# system log +# (default: disabled) +#enable_debug = true +# +# Internetworking model +# Determines how the VM should be connected to the +# the container network interface +# Options: +# +# - bridged (Deprecated) +# Uses a linux bridge to interconnect the container interface to +# the VM. Works for most cases except macvlan and ipvlan. +# ***NOTE: This feature has been deprecated with plans to remove this +# feature in the future. Please use other network models listed below. +# +# +# - macvtap +# Used when the Container network interface can be bridged using +# macvtap. +# +# - none +# Used when customize network. Only creates a tap device. No veth pair. +# +# - tcfilter +# Uses tc filter rules to redirect traffic from the network interface +# provided by plugin to a tap interface connected to the VM. +# +internetworking_model="@DEFNETWORKMODEL_CLH@" + +# disable guest seccomp +# Determines whether container seccomp profiles are passed to the virtual +# machine and applied by the kata agent. If set to true, seccomp is not applied +# within the guest +# (default: true) +disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ + +# If enabled, the runtime will create opentracing.io traces and spans. +# (See https://www.jaegertracing.io/docs/getting-started). +# (default: disabled) +#enable_tracing = true + +# If enabled, the runtime will not create a network namespace for shim and hypervisor processes. +# This option may have some potential impacts to your host. It should only be used when you know what you're doing. +# `disable_new_netns` conflicts with `enable_netmon` +# `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only +# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge +# (like OVS) directly. +# If you are using docker, `disable_new_netns` only works with `docker run --net=none` +# (default: false) +#disable_new_netns = true + +# if enabled, the runtime will add all the kata processes inside one dedicated cgroup. +# The container cgroups in the host are not created, just one single cgroup per sandbox. +# The sandbox cgroup is not constrained by the runtime +# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox. +# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation. +# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType +sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ + +# Enabled experimental feature list, format: ["a", "b"]. +# Experimental features are features not stable enough for production, +# They may break compatibility, and are prepared for a big version bump. +# Supported experimental features: +# 1. "newstore": new persist storage driver which breaks backward compatibility, +# expected to move out of experimental in 2.0.0. +# (default: []) +experimental=@DEFAULTEXPFEATURES@ diff --git a/cli/kata-check_amd64.go b/cli/kata-check_amd64.go index 3962e09e88..1464b90908 100644 --- a/cli/kata-check_amd64.go +++ b/cli/kata-check_amd64.go @@ -107,6 +107,8 @@ func setCPUtype(hypervisorType vc.HypervisorType) error { switch hypervisorType { case "firecracker": fallthrough + case "clh": + fallthrough case "qemu": archRequiredCPUFlags = map[string]string{ cpuFlagVMX: "Virtualization support", @@ -291,6 +293,8 @@ func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { switch hypervisorType { case "qemu": fallthrough + case "clh": + fallthrough case "firecracker": return kvmIsUsable() case "acrn": diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index 386f9df5a8..153f0fa988 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -49,6 +49,7 @@ var ( const ( // supported hypervisor component types firecrackerHypervisorTableType = "firecracker" + clhHypervisorTableType = "clh" qemuHypervisorTableType = "qemu" acrnHypervisorTableType = "acrn" @@ -701,6 +702,92 @@ func newAcrnHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { }, nil } +func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { + hypervisor, err := h.path() + if err != nil { + return vc.HypervisorConfig{}, err + } + + kernel, err := h.kernel() + if err != nil { + return vc.HypervisorConfig{}, err + } + + initrd, image, err := h.getInitrdAndImage() + if err != nil { + return vc.HypervisorConfig{}, err + } + + if initrd != "" { + return vc.HypervisorConfig{}, + errors.New("having an initrd defined in the configuration file is not supported") + } + + if image == "" { + return vc.HypervisorConfig{}, + errors.New("image must be defined in the configuration file") + } + + firmware, err := h.firmware() + if err != nil { + return vc.HypervisorConfig{}, err + } + + machineAccelerators := h.machineAccelerators() + kernelParams := h.kernelParams() + machineType := h.machineType() + + blockDriver, err := h.blockDeviceDriver() + if err != nil { + return vc.HypervisorConfig{}, err + } + + sharedFS := config.VirtioFS + + if h.VirtioFSDaemon == "" { + return vc.HypervisorConfig{}, + errors.New("virtio-fs daemon path is missing in configuration file") + } + + return vc.HypervisorConfig{ + HypervisorPath: hypervisor, + KernelPath: kernel, + InitrdPath: initrd, + ImagePath: image, + FirmwarePath: firmware, + MachineAccelerators: machineAccelerators, + KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), + HypervisorMachineType: machineType, + NumVCPUs: h.defaultVCPUs(), + DefaultMaxVCPUs: h.defaultMaxVCPUs(), + MemorySize: h.defaultMemSz(), + MemSlots: h.defaultMemSlots(), + MemOffset: h.defaultMemOffset(), + EntropySource: h.GetEntropySource(), + DefaultBridges: h.defaultBridges(), + DisableBlockDeviceUse: h.DisableBlockDeviceUse, + SharedFS: sharedFS, + VirtioFSDaemon: h.VirtioFSDaemon, + VirtioFSCacheSize: h.VirtioFSCacheSize, + VirtioFSCache: h.VirtioFSCache, + MemPrealloc: h.MemPrealloc, + HugePages: h.HugePages, + FileBackedMemRootDir: h.FileBackedMemRootDir, + Mlock: !h.Swap, + Debug: h.Debug, + DisableNestingChecks: h.DisableNestingChecks, + BlockDeviceDriver: blockDriver, + BlockDeviceCacheSet: h.BlockDeviceCacheSet, + BlockDeviceCacheDirect: h.BlockDeviceCacheDirect, + BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush, + EnableIOThreads: h.EnableIOThreads, + Msize9p: h.msize9p(), + HotplugVFIOOnRootBus: h.HotplugVFIOOnRootBus, + DisableVhostNet: true, + UseVSock: true, + }, nil +} + func newFactoryConfig(f factory) (oci.FactoryConfig, error) { if f.TemplatePath == "" { f.TemplatePath = defaultTemplatePath @@ -744,6 +831,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi case acrnHypervisorTableType: config.HypervisorType = vc.AcrnHypervisor hConfig, err = newAcrnHypervisorConfig(hypervisor) + case clhHypervisorTableType: + config.HypervisorType = vc.ClhHypervisor + hConfig, err = newClhHypervisorConfig(hypervisor) } if err != nil { diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go index 3ad78a6bc0..edf41826c3 100644 --- a/pkg/katautils/config_test.go +++ b/pkg/katautils/config_test.go @@ -885,6 +885,66 @@ func TestNewQemuHypervisorConfigImageAndInitrd(t *testing.T) { assert.Error(err) } +func TestNewClhHypervisorConfig(t *testing.T) { + + assert := assert.New(t) + + tmpdir, err := ioutil.TempDir(testDir, "") + assert.NoError(err) + defer os.RemoveAll(tmpdir) + + hypervisorPath := path.Join(tmpdir, "hypervisor") + kernelPath := path.Join(tmpdir, "kernel") + imagePath := path.Join(tmpdir, "image") + virtioFsDaemon := path.Join(tmpdir, "virtiofsd") + + for _, file := range []string{imagePath, hypervisorPath, kernelPath, virtioFsDaemon} { + err = createEmptyFile(file) + assert.NoError(err) + } + + hypervisor := hypervisor{ + Path: hypervisorPath, + Kernel: kernelPath, + Image: imagePath, + VirtioFSDaemon: virtioFsDaemon, + VirtioFSCache: "always", + } + config, err := newClhHypervisorConfig(hypervisor) + if err != nil { + t.Fatal(err) + } + + if config.HypervisorPath != hypervisor.Path { + t.Errorf("Expected hypervisor path %v, got %v", hypervisor.Path, config.HypervisorPath) + } + + if config.KernelPath != hypervisor.Kernel { + t.Errorf("Expected kernel path %v, got %v", hypervisor.Kernel, config.KernelPath) + } + + if config.ImagePath != hypervisor.Image { + t.Errorf("Expected image path %v, got %v", hypervisor.Image, config.ImagePath) + } + + if config.ImagePath != hypervisor.Image { + t.Errorf("Expected image path %v, got %v", hypervisor.Image, config.ImagePath) + } + + if config.UseVSock != true { + t.Errorf("Expected UseVSock %v, got %v", true, config.UseVSock) + } + + if config.DisableVhostNet != true { + t.Errorf("Expected DisableVhostNet %v, got %v", true, config.DisableVhostNet) + } + + if config.VirtioFSCache != "always" { + t.Errorf("Expected VirtioFSCache %v, got %v", true, config.VirtioFSCache) + } + +} + func TestNewShimConfig(t *testing.T) { dir, err := ioutil.TempDir(testDir, "shim-config-") if err != nil { diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go new file mode 100644 index 0000000000..2e05418c01 --- /dev/null +++ b/virtcontainers/clh.go @@ -0,0 +1,1294 @@ +// Copyright (c) 2019 Ericsson Eurolab Deutschland GmbH +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "bufio" + "context" + "fmt" + "io/ioutil" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" + opentracing "github.com/opentracing/opentracing-go" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/store" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/kata-containers/runtime/virtcontainers/utils" +) + +// +// Constants and type definitions related to cloud hypervisor +// + +type clhState uint8 + +const ( + clhNotReady clhState = iota + clhReady +) + +const ( + clhTimeout = 10 + clhSocket = "clh.sock" + clhAPISocket = "clh-api.sock" + clhLogFile = "clh.log" + virtioFsSocket = "virtiofsd.sock" + clhSerial = "serial-tty.log" + supportedMajorVersion = 0 + supportedMinorVersion = 3 + defaultClhPath = "/usr/local/bin/cloud-hypervisor" + virtioFsCacheAlways = "always" +) + +type CloudHypervisorVersion struct { + Major int + Minor int + Revision int +} + +// +// Cloud hypervisor state +// +type CloudHypervisorState struct { + state clhState + PID int + VirtiofsdPID int +} + +func (s *CloudHypervisorState) reset() { + s.PID = 0 + s.VirtiofsdPID = 0 + s.state = clhNotReady +} + +type cloudHypervisor struct { + id string + state CloudHypervisorState + store *store.VCStore + config HypervisorConfig + ctx context.Context + socketPath string + version CloudHypervisorVersion + cliBuilder *DefaultCLIBuilder +} + +var clhKernelParams = []Param{ + + {"root", "/dev/vda1"}, + {"panic", "1"}, // upon kernel panic wait 1 second before reboot + {"no_timer_check", ""}, // do not check broken timer IRQ resources + {"noreplace-smp", ""}, // do not replace SMP instructions + {"agent.log_vport", fmt.Sprintf("%d", vSockLogsPort)}, // tell the agent where to send the logs +} + +var clhDebugKernelParams = []Param{ + + {"console", "ttyS0,115200n8"}, // enable serial console + {"systemd.log_level", "debug"}, // enable systemd debug output + {"systemd.log_target", "console"}, // send loggng to the console + {"initcall_debug", "1"}, // print init call timing information to the console +} + +//########################################################### +// +// hypervisor interface implementation for cloud-hypervisor +// +//########################################################### + +func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error { + clh.ctx = ctx + + span, _ := clh.trace("createSandbox") + defer span.Finish() + + err := hypervisorConfig.valid() + if err != nil { + return err + } + + clh.id = id + clh.store = vcStore + clh.config = *hypervisorConfig + clh.state.state = clhNotReady + + err = clh.getAvailableVersion() + if err != nil { + return err + } + + if clh.version.Major < supportedMajorVersion && clh.version.Minor < supportedMinorVersion { + errorMessage := fmt.Sprintf("Unsupported version: cloud-hypervisor %d.%d not supported by this driver version (%d.%d)", + clh.version.Major, + clh.version.Minor, + supportedMajorVersion, + supportedMinorVersion) + return errors.New(errorMessage) + } + + clh.cliBuilder = &DefaultCLIBuilder{} + + socketPath, err := clh.vsockSocketPath(id) + if err != nil { + clh.Logger().Info("Invalid socket path for cloud-hypervisor") + return nil + } + clh.socketPath = socketPath + + clh.Logger().WithField("function", "createSandbox").Info("creating Sandbox") + + // No need to return an error from there since there might be nothing + // to fetch if this is the first time the hypervisor is created. + if err := clh.store.Load(store.Hypervisor, &clh.state); err != nil { + clh.Logger().WithField("function", "createSandbox").WithError(err).Info("No info could be fetched") + } + + // Set initial memomory size of the cloud hypervisor + clh.cliBuilder.SetMemory(&CLIMemory{ + memorySize: clh.config.MemorySize, + backingFile: "/dev/shm", + }) + // Set initial amount of cpu's for the cloud hypervisor + clh.cliBuilder.SetCpus(&CLICpus{ + cpus: clh.config.NumVCPUs, + }) + + // Add the kernel path + kernelPath, err := clh.config.KernelAssetPath() + if err != nil { + return err + } + clh.cliBuilder.SetKernel(&CLIKernel{ + path: kernelPath, + }) + + // First take the default parameters defined by this driver + clh.cliBuilder.AddKernelParameters(clhKernelParams) + + // Followed by extra debug parameters if debug enabled in configuration file + if clh.config.Debug { + clh.cliBuilder.AddKernelParameters(clhDebugKernelParams) + } + + // Followed by extra debug parameters defined in the configuration file + clh.cliBuilder.AddKernelParameters(clh.config.KernelParams) + + // set random device generator to hypervisor + clh.cliBuilder.SetRng(&CLIRng{ + src: clh.config.EntropySource, + iommu: false, + }) + + // Add the hybrid vsock device to hypervisor + clh.cliBuilder.SetVsock(&CLIVsock{ + cid: 3, + socketPath: clh.socketPath, + iommu: false, + }) + + // set the initial root/boot disk of hypervisor + imagePath, err := clh.config.ImageAssetPath() + if err != nil { + return err + } + + if imagePath != "" { + clh.cliBuilder.SetDisk(&CLIDisk{ + path: imagePath, + iommu: false, + }) + } + + // set the virtio-fs to the hypervisor + vfsdSockPath, err := clh.virtioFsSocketPath(clh.id) + if err != nil { + return err + } + if clh.config.VirtioFSCache == virtioFsCacheAlways { + clh.cliBuilder.SetFs(&CLIFs{ + tag: "kataShared", + socketPath: vfsdSockPath, + queues: 1, + queueSize: 512, + dax: true, + }) + } else { + clh.cliBuilder.SetFs(&CLIFs{ + tag: "kataShared", + socketPath: vfsdSockPath, + queues: 1, + queueSize: 512, + }) + } + + // set the serial console to the cloud hypervisor + if clh.config.Debug { + serialPath, err := clh.serialPath(clh.id) + if err != nil { + return err + } + clh.cliBuilder.SetSerial(&CLISerialConsole{ + consoleType: cctFILE, + filePath: serialPath, + }) + logFilePath, err := clh.logFilePath(clh.id) + if err != nil { + return err + } + clh.cliBuilder.SetLogFile(&CLILogFile{ + path: logFilePath, + }) + } + + clh.cliBuilder.SetConsole(&CLIConsole{ + consoleType: cctOFF, + }) + + // Move the API endpoint socket location for the + // by default enabled api endpoint + apiSocketPath, err := clh.apiSocketPath(id) + if err != nil { + clh.Logger().Info("Invalid api socket path for cloud-hypervisor") + return nil + } + clh.cliBuilder.SetAPISocket(&CLIAPISocket{ + socketPath: apiSocketPath, + }) + + return nil +} + +func (clh *cloudHypervisor) startSandbox(timeout int) error { + span, _ := clh.trace("startSandbox") + defer span.Finish() + + clh.Logger().WithField("function", "startSandbox").Info("starting Sandbox") + + vmPath := filepath.Join(store.RunVMStoragePath(), clh.id) + err := os.MkdirAll(vmPath, store.DirMode) + if err != nil { + return err + } + + if clh.config.SharedFS == config.VirtioFS { + clh.Logger().WithField("function", "startSandbox").Info("Starting virtiofsd") + _, err = clh.setupVirtiofsd(timeout) + if err != nil { + return err + } + if err = clh.storeState(); err != nil { + return err + } + } else { + return errors.New("cloud-hypervisor only supports virtio based file sharing") + } + + var strErr string + strErr, pid, err := clh.LaunchClh() + if err != nil { + return fmt.Errorf("failed to launch cloud-hypervisor: %s, error messages from log: %s", err, strErr) + } + if err := clh.waitVMM(clhTimeout); err != nil { + clh.Logger().WithField("error", err).Warn("cloud-hypervisor init failed") + clh.shutdownVirtiofsd() + return err + } + + clh.state.PID = pid + clh.state.state = clhReady + clh.storeState() + + return nil +} + +func (clh *cloudHypervisor) getSandboxConsole(id string) (string, error) { + clh.Logger().WithField("function", "getSandboxConsole").WithField("id", id).Info("Get Sandbox Console") + return "", nil +} + +func (clh *cloudHypervisor) disconnect() { + clh.Logger().WithField("function", "disconnect").Info("Disconnecting Sandbox Console") +} + +func (clh *cloudHypervisor) getThreadIDs() (vcpuThreadIDs, error) { + + clh.Logger().WithField("function", "getThreadIDs").Info("get thread ID's") + + var vcpuInfo vcpuThreadIDs + + vcpuInfo.vcpus = make(map[int]int) + + return vcpuInfo, nil +} + +func (clh *cloudHypervisor) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) { + clh.Logger().WithField("function", "hotplugAddDevice").Warn("hotplug add device not supported") + return nil, nil +} + +func (clh *cloudHypervisor) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { + clh.Logger().WithField("function", "hotplugRemoveDevice").Warn("hotplug remove device not supported") + return nil, nil +} + +func (clh *cloudHypervisor) hypervisorConfig() HypervisorConfig { + return clh.config +} + +func (clh *cloudHypervisor) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { + clh.Logger().WithField("function", "resizeMemory").Warn("not supported") + return 0, memoryDevice{}, nil +} + +func (clh *cloudHypervisor) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { + clh.Logger().WithField("function", "resizeVCPUs").Warn("not supported") + return 0, 0, nil +} + +func (clh *cloudHypervisor) cleanup() error { + clh.Logger().WithField("function", "cleanup").Info("cleanup") + return nil +} + +func (clh *cloudHypervisor) pauseSandbox() error { + clh.Logger().WithField("function", "pauseSandbox").Info("Pause Sandbox") + return nil +} + +func (clh *cloudHypervisor) saveSandbox() error { + clh.Logger().WithField("function", "saveSandboxC").Info("Save Sandbox") + return nil +} + +func (clh *cloudHypervisor) resumeSandbox() error { + clh.Logger().WithField("function", "resumeSandbox").Info("Resume Sandbox") + return nil +} + +// stopSandbox will stop the Sandbox's VM. +func (clh *cloudHypervisor) stopSandbox() (err error) { + span, _ := clh.trace("stopSandbox") + defer span.Finish() + clh.Logger().WithField("function", "stopSandbox").Info("Stop Sandbox") + return clh.terminate() +} + +func (clh *cloudHypervisor) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, store *store.VCStore, j []byte) error { + return errors.New("cloudHypervisor is not supported by VM cache") +} + +func (clh *cloudHypervisor) toGrpc() ([]byte, error) { + return nil, errors.New("cloudHypervisor is not supported by VM cache") +} + +func (clh *cloudHypervisor) save() (s persistapi.HypervisorState) { + s.Pid = clh.state.PID + s.Type = string(ClhHypervisor) + return +} + +func (clh *cloudHypervisor) load(s persistapi.HypervisorState) { + clh.state.PID = s.Pid + clh.state.VirtiofsdPID = s.VirtiofsdPid +} + +func (clh *cloudHypervisor) check() error { + return nil +} + +func (clh *cloudHypervisor) getPids() []int { + + var pids []int + pids = append(pids, clh.state.PID) + + return pids +} + +//########################################################################### +// +// Local helper methods related to the hypervisor interface implementation +// +//########################################################################### + +func (clh *cloudHypervisor) addDevice(devInfo interface{}, devType deviceType) error { + span, _ := clh.trace("addDevice") + defer span.Finish() + + var err error + + switch v := devInfo.(type) { + case Endpoint: + clh.Logger().WithField("function", "addDevice").Infof("Adding Endpoint of type %v", v) + clh.cliBuilder.AddNet(CLINet{ + device: v.Name(), + mac: v.HardwareAddr(), + }) + + default: + clh.Logger().WithField("function", "addDevice").Warnf("Add device of type %v is not supported.", v) + } + + return err +} + +func (clh *cloudHypervisor) Logger() *log.Entry { + return virtLog.WithField("subsystem", "cloudHypervisor") +} + +func (clh *cloudHypervisor) capabilities() types.Capabilities { + span, _ := clh.trace("capabilities") + defer span.Finish() + + clh.Logger().WithField("function", "capabilities").Info("get Capabilities") + var caps types.Capabilities + return caps + +} + +func (clh *cloudHypervisor) trace(name string) (opentracing.Span, context.Context) { + + if clh.ctx == nil { + clh.Logger().WithField("type", "bug").Error("trace called before context set") + clh.ctx = context.Background() + } + + span, ctx := opentracing.StartSpanFromContext(clh.ctx, name) + + span.SetTag("subsystem", "cloudHypervisor") + span.SetTag("type", "clh") + + return span, ctx +} + +func (clh *cloudHypervisor) terminate() (err error) { + span, _ := clh.trace("terminate") + defer span.Finish() + + defer func() { + if err != nil { + clh.Logger().Info("Terminate Cloud Hypervisor failed") + } else { + clh.Logger().Info("Cloud Hypervisor stopped") + clh.reset() + clh.Logger().Debug("removing virtiofsd and vm sockets") + path, err := clh.virtioFsSocketPath(clh.id) + if err == nil { + rerr := os.Remove(path) + if rerr != nil { + clh.Logger().WithField("path", path).Warn("removing virtiofsd socket failed") + } + } + path, err = clh.vsockSocketPath(clh.id) + if err == nil { + rerr := os.Remove(path) + if rerr != nil { + clh.Logger().WithField("path", path).Warn("removing vm socket failed") + } + } + } + }() + + pid := clh.state.PID + if pid == 0 { + clh.Logger().WithField("PID", pid).Info("Skipping kill cloud hypervisor. invalid pid") + return nil + } + clh.Logger().WithField("PID", pid).Info("Stopping Cloud Hypervisor") + + // Check if VM process is running, in case it is not, let's + // return from here. + if err = syscall.Kill(pid, syscall.Signal(0)); err != nil { + return nil + } + + // Send a SIGTERM to the VM process to try to stop it properly + if err = syscall.Kill(pid, syscall.SIGTERM); err != nil { + return err + } + + // Wait for the VM process to terminate + tInit := time.Now() + for { + if err = syscall.Kill(pid, syscall.Signal(0)); err != nil { + return nil + } + + if time.Since(tInit).Seconds() >= fcStopSandboxTimeout { + clh.Logger().Warnf("VM still running after waiting %ds", fcStopSandboxTimeout) + break + } + + // Let's avoid to run a too busy loop + time.Sleep(time.Duration(50) * time.Millisecond) + } + + // Let's try with a hammer now, a SIGKILL should get rid of the + // VM process. + return syscall.Kill(pid, syscall.SIGKILL) +} + +func (clh *cloudHypervisor) reset() { + clh.state.reset() + clh.storeState() +} + +func (clh *cloudHypervisor) generateSocket(id string, useVsock bool) (interface{}, error) { + if !useVsock { + return nil, fmt.Errorf("Can't generate socket path for cloud-hypervisor: vsocks is disabled") + } + + udsPath, err := clh.vsockSocketPath(id) + if err != nil { + clh.Logger().Info("Can't generate socket path for cloud-hypervisor") + return types.HybridVSock{}, err + } + clh.Logger().WithField("function", "generateSocket").Infof("Using hybrid vsock %s:%d", udsPath, vSockPort) + clh.socketPath = udsPath + return types.HybridVSock{ + UdsPath: udsPath, + Port: uint32(vSockPort), + }, nil +} + +func (clh *cloudHypervisor) setupVirtiofsd(timeout int) (remain int, err error) { + + sockPath, perr := clh.virtioFsSocketPath(clh.id) + if perr != nil { + return 0, perr + } + + theArgs, err := clh.virtiofsdArgs(sockPath) + if err != nil { + return 0, err + } + + clh.Logger().WithField("path", clh.config.VirtioFSDaemon).Info() + clh.Logger().WithField("args", strings.Join(theArgs, " ")).Info() + + cmd := exec.Command(clh.config.VirtioFSDaemon, theArgs...) + stderr, err := cmd.StderrPipe() + if err != nil { + return 0, err + } + + if err = cmd.Start(); err != nil { + return 0, err + } + defer func() { + if err != nil { + clh.state.VirtiofsdPID = 0 + cmd.Process.Kill() + } else { + clh.state.VirtiofsdPID = cmd.Process.Pid + + } + clh.storeState() + }() + + // Wait for socket to become available + sockReady := make(chan error, 1) + timeStart := time.Now() + go func() { + scanner := bufio.NewScanner(stderr) + var sent bool + for scanner.Scan() { + if clh.config.Debug { + clh.Logger().WithField("source", "virtiofsd").Debug(scanner.Text()) + } + if !sent && strings.Contains(scanner.Text(), "Waiting for vhost-user socket connection...") { + sockReady <- nil + sent = true + } + } + if !sent { + if err := scanner.Err(); err != nil { + sockReady <- err + } else { + sockReady <- fmt.Errorf("virtiofsd did not announce socket connection") + } + } + clh.Logger().Info("virtiofsd quits") + // Wait to release resources of virtiofsd process + cmd.Process.Wait() + + }() + + return clh.waitVirtiofsd(timeStart, timeout, sockReady, + fmt.Sprintf("virtiofsd (pid=%d) socket %s", cmd.Process.Pid, sockPath)) +} + +func (clh *cloudHypervisor) waitVirtiofsd(start time.Time, timeout int, ready chan error, errMsg string) (int, error) { + var err error + + timeoutDuration := time.Duration(timeout) * time.Second + select { + case err = <-ready: + case <-time.After(timeoutDuration): + err = fmt.Errorf("timed out waiting for %s", errMsg) + } + if err != nil { + return 0, err + } + + // Now reduce timeout by the elapsed time + elapsed := time.Since(start) + if elapsed < timeoutDuration { + timeout = timeout - int(elapsed.Seconds()) + } else { + timeout = 0 + } + return timeout, nil +} + +func (clh *cloudHypervisor) virtiofsdArgs(sockPath string) ([]string, error) { + + sourcePath := filepath.Join(kataHostSharedDir(), clh.id) + if _, err := os.Stat(sourcePath); os.IsNotExist(err) { + os.MkdirAll(sourcePath, os.ModePerm) + } + + args := []string{ + "-f", + "-o", "vhost_user_socket=" + sockPath, + "-o", "source=" + sourcePath, + "-o", "cache=" + clh.config.VirtioFSCache} + + if len(clh.config.VirtioFSExtraArgs) != 0 { + args = append(args, clh.config.VirtioFSExtraArgs...) + } + return args, nil +} + +func (clh *cloudHypervisor) shutdownVirtiofsd() (err error) { + + err = syscall.Kill(-clh.state.VirtiofsdPID, syscall.SIGKILL) + + if err != nil { + clh.state.VirtiofsdPID = 0 + clh.storeState() + } + return err + +} + +func (clh *cloudHypervisor) virtioFsSocketPath(id string) (string, error) { + return utils.BuildSocketPath(store.RunVMStoragePath(), id, virtioFsSocket) +} + +func (clh *cloudHypervisor) vsockSocketPath(id string) (string, error) { + return utils.BuildSocketPath(store.RunVMStoragePath(), id, clhSocket) +} + +func (clh *cloudHypervisor) serialPath(id string) (string, error) { + return utils.BuildSocketPath(store.RunVMStoragePath(), id, clhSerial) +} + +func (clh *cloudHypervisor) apiSocketPath(id string) (string, error) { + return utils.BuildSocketPath(store.RunVMStoragePath(), id, clhAPISocket) +} + +func (clh *cloudHypervisor) logFilePath(id string) (string, error) { + return utils.BuildSocketPath(store.RunVMStoragePath(), id, clhLogFile) +} + +func (clh *cloudHypervisor) storeState() error { + if clh.store != nil { + if err := clh.store.Store(store.Hypervisor, clh.state); err != nil { + return err + } + } + return nil +} + +func (clh *cloudHypervisor) waitVMM(timeout int) error { + + var err error + timeoutDuration := time.Duration(timeout) * time.Second + + sockReady := make(chan error, 1) + go func() { + udsPath, err := clh.vsockSocketPath(clh.id) + if err != nil { + sockReady <- err + } + + for { + addr, err := net.ResolveUnixAddr("unix", udsPath) + if err != nil { + sockReady <- err + } + conn, err := net.DialUnix("unix", nil, addr) + + if err != nil { + time.Sleep(50 * time.Millisecond) + } else { + conn.Close() + sockReady <- nil + + break + } + } + }() + + select { + case err = <-sockReady: + case <-time.After(timeoutDuration): + err = fmt.Errorf("timed out waiting for cloud-hypervisor vsock") + } + + time.Sleep(1000 * time.Millisecond) + return err +} + +func (clh *cloudHypervisor) clhPath() (string, error) { + p, err := clh.config.HypervisorAssetPath() + if err != nil { + return "", err + } + + if p == "" { + p = defaultClhPath + } + + if _, err = os.Stat(p); os.IsNotExist(err) { + return "", fmt.Errorf("Cloud-Hypervisor path (%s) does not exist", p) + } + + return p, nil +} + +func (clh *cloudHypervisor) getAvailableVersion() error { + + clhPath, err := clh.clhPath() + if err != nil { + return err + } + + cmd := exec.Command(clhPath, "--version") + out, err := cmd.CombinedOutput() + if err != nil { + return err + } + + words := strings.Fields(string(out)) + if len(words) != 2 { + return errors.New("Failed to parse cloud-hypervisor version response. Illegal length") + } + versionSplit := strings.SplitN(words[1], ".", -1) + if len(versionSplit) != 3 { + return errors.New("Failed to parse cloud-hypervisor version field. Illegal length") + } + + major, err := strconv.ParseUint(versionSplit[0], 10, 64) + if err != nil { + return err + } + minor, err := strconv.ParseUint(versionSplit[1], 10, 64) + if err != nil { + return err + } + revision, err := strconv.ParseUint(versionSplit[2], 10, 64) + if err != nil { + return err + } + clh.version = CloudHypervisorVersion{ + Major: int(major), + Minor: int(minor), + Revision: int(revision), + } + return nil +} + +func (clh *cloudHypervisor) LaunchClh() (string, int, error) { + + errStr := "" + + clhPath, err := clh.clhPath() + if err != nil { + return "", -1, err + } + director := &CommandLineDirector{} + + cli, err := director.Build(clh.cliBuilder) + if err != nil { + return "", -1, err + } + + clh.Logger().WithField("path", clhPath).Info() + clh.Logger().WithField("args", strings.Join(cli.args, " ")).Info() + + cmd := exec.Command(clhPath, cli.args...) + cmd.Stderr = ioutil.Discard + + if clh.config.Debug { + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, "RUST_BACKTRACE=FULL") + } + + if err := cmd.Start(); err != nil { + fmt.Println("Error starting cloudHypervisor", err) + if cmd.Process != nil { + cmd.Process.Kill() + } + return errStr, 0, err + } + + return errStr, cmd.Process.Pid, nil +} + +//########################################################################### +// +// Cloud-hypervisor CLI builder +// +//########################################################################### + +const ( + cctOFF string = "off" + cctFILE string = "file" +) + +const ( + cscApisocket string = "--api-socket" + cscCmdline string = "--cmdline" + cscConsole string = "--console" + cscCpus string = "--cpus" + cscDisk string = "--disk" + cscFs string = "--fs" + cscKernel string = "--kernel" + cscLogFile string = "--log-file" + cscMemory string = "--memory" + cscNet string = "--net" + cscRng string = "--rng" + cscSerial string = "--serial" + cscVsock string = "--vsock" +) + +type CommandLineBuilder interface { + AddKernelParameters(cmdline []Param) + SetConsole(console *CLIConsole) + SetCpus(cpus *CLICpus) + SetDisk(disk *CLIDisk) + SetFs(fs *CLIFs) + SetKernel(kernel *CLIKernel) + SetMemory(memory *CLIMemory) + AddNet(net CLINet) + SetRng(rng *CLIRng) + SetSerial(serial *CLISerialConsole) + SetVsock(vsock *CLIVsock) + SetAPISocket(apiSocket *CLIAPISocket) + SetLogFile(logFile *CLILogFile) + GetCommandLine() (*CommandLine, error) +} + +type CLIOption interface { + Build(cmdline *CommandLine) +} + +type CommandLine struct { + args []string +} + +//********************************** +// The (virtio) Console +//********************************** +type CLIConsole struct { + consoleType string + filePath string + iommu bool +} + +func (o *CLIConsole) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscConsole) + + consoleArg := "" + if o.consoleType == cctFILE { + consoleArg = o.consoleType + "=" + o.filePath + if o.iommu { + consoleArg += ",iommu=on" + } else { + consoleArg += ",iommu=off" + } + } else { + consoleArg = o.consoleType + } + + cmdline.args = append(cmdline.args, consoleArg) +} + +//********************************** +// The serial port +//********************************** +type CLISerialConsole struct { + consoleType string + filePath string +} + +func (o *CLISerialConsole) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscSerial) + if o.consoleType == cctFILE { + cmdline.args = append(cmdline.args, o.consoleType+"="+o.filePath) + } else { + cmdline.args = append(cmdline.args, o.consoleType) + } + +} + +//********************************** +// The API socket +//********************************** +type CLIAPISocket struct { + socketPath string +} + +func (o *CLIAPISocket) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscApisocket) + if o.socketPath != "" { + cmdline.args = append(cmdline.args, o.socketPath) + } +} + +//********************************** +// The amount of memory in Mb +//********************************** +type CLIMemory struct { + memorySize uint32 + backingFile string +} + +func (o *CLIMemory) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscMemory) + if o.backingFile == "" { + cmdline.args = append(cmdline.args, "size="+strconv.FormatUint(uint64(o.memorySize), 10)+"M") + } else { + cmdline.args = append(cmdline.args, "size="+strconv.FormatUint(uint64(o.memorySize), 10)+"M,file="+o.backingFile) + } + +} + +//********************************** +// The number of CPU's +//********************************** +type CLICpus struct { + cpus uint32 +} + +func (o *CLICpus) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscCpus) + cmdline.args = append(cmdline.args, strconv.FormatUint(uint64(o.cpus), 10)) + +} + +//********************************** +// The Path to the kernel image +//********************************** +type CLIKernel struct { + path string +} + +func (o *CLIKernel) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscKernel) + cmdline.args = append(cmdline.args, o.path) + +} + +//**************************************** +// The Path to the root (boot) disk image +//**************************************** +type CLIDisk struct { + path string + iommu bool +} + +func (o *CLIDisk) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscDisk) + if o.iommu { + cmdline.args = append(cmdline.args, "path="+o.path+",iommu=on") + } else { + cmdline.args = append(cmdline.args, "path="+o.path+",iommu=off") + } + +} + +//**************************************** +// The random device +//**************************************** +type CLIRng struct { + src string + iommu bool +} + +func (o *CLIRng) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscRng) + if o.iommu { + cmdline.args = append(cmdline.args, "src="+o.src+",iommu=on") + } else { + cmdline.args = append(cmdline.args, "src="+o.src+",iommu=off") + } + +} + +//**************************************** +// The VSock socket +//**************************************** +type CLIVsock struct { + socketPath string + cid uint32 + iommu bool +} + +func (o *CLIVsock) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscVsock) + if o.iommu { + cmdline.args = append(cmdline.args, "cid="+strconv.FormatUint(uint64(o.cid), 10)+",sock="+o.socketPath+",iommu=on") + } else { + cmdline.args = append(cmdline.args, "cid="+strconv.FormatUint(uint64(o.cid), 10)+",sock="+o.socketPath+",iommu=off") + } +} + +//**************************************** +// The shard (virtio) file system +//**************************************** +type CLIFs struct { + tag string + socketPath string + queues uint32 + queueSize uint32 + dax bool +} + +func (o *CLIFs) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscFs) + + fsarg := "tag=" + o.tag + ",sock=" + o.socketPath + if o.dax { + fsarg += ",dax=on" + } else { + fsarg += ",num_queues=" + strconv.FormatUint(uint64(o.queues), 10) + ",queue_size=" + strconv.FormatUint(uint64(o.queueSize), 10) + } + cmdline.args = append(cmdline.args, fsarg) +} + +//**************************************** +// The net (nic) +//**************************************** +type CLINet struct { + device string + mac string + iommu bool +} + +type CLINets struct { + networks []CLINet +} + +func (o *CLINets) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscNet) + + networks := "" + netIndex := 1 + for _, net := range o.networks { + tapName := "tap" + strconv.FormatUint(uint64(netIndex), 10) + netIndex++ + if net.iommu { + networks += "tap=" + tapName + ",mac=" + net.mac + ",iommu=on" + } else { + networks += "tap=" + tapName + ",mac=" + net.mac + } + } + cmdline.args = append(cmdline.args, networks) +} + +//**************************************** +// The log file +//**************************************** +type CLILogFile struct { + path string +} + +func (o *CLILogFile) Build(cmdline *CommandLine) { + + if o.path != "" { + cmdline.args = append(cmdline.args, cscLogFile) + cmdline.args = append(cmdline.args, o.path) + } +} + +//**************************************** +// The kernel command line +//**************************************** +type CLICmdline struct { + params []Param +} + +func (o *CLICmdline) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscCmdline) + + var paramBuilder strings.Builder + for _, p := range o.params { + paramBuilder.WriteString(p.Key) + if len(p.Value) > 0 { + + paramBuilder.WriteString("=") + paramBuilder.WriteString(p.Value) + } + paramBuilder.WriteString(" ") + } + cmdline.args = append(cmdline.args, strings.TrimSpace(paramBuilder.String())) + +} + +//********************************** +// The Default Builder +//********************************** +type DefaultCLIBuilder struct { + console *CLIConsole + serial *CLISerialConsole + apiSocket *CLIAPISocket + cpus *CLICpus + memory *CLIMemory + kernel *CLIKernel + disk *CLIDisk + fs *CLIFs + rng *CLIRng + logFile *CLILogFile + vsock *CLIVsock + cmdline *CLICmdline + nets *CLINets +} + +func (d *DefaultCLIBuilder) AddKernelParameters(params []Param) { + + if d.cmdline == nil { + d.cmdline = &CLICmdline{} + } + d.cmdline.params = append(d.cmdline.params, params...) +} + +func (d *DefaultCLIBuilder) SetConsole(console *CLIConsole) { + d.console = console +} + +func (d *DefaultCLIBuilder) SetCpus(cpus *CLICpus) { + d.cpus = cpus +} + +func (d *DefaultCLIBuilder) SetDisk(disk *CLIDisk) { + d.disk = disk +} + +func (d *DefaultCLIBuilder) SetFs(fs *CLIFs) { + d.fs = fs +} + +func (d *DefaultCLIBuilder) SetKernel(kernel *CLIKernel) { + d.kernel = kernel +} + +func (d *DefaultCLIBuilder) SetMemory(memory *CLIMemory) { + d.memory = memory +} + +func (d *DefaultCLIBuilder) AddNet(net CLINet) { + if d.nets == nil { + d.nets = &CLINets{} + } + d.nets.networks = append(d.nets.networks, net) +} + +func (d *DefaultCLIBuilder) SetRng(rng *CLIRng) { + d.rng = rng +} + +func (d *DefaultCLIBuilder) SetSerial(serial *CLISerialConsole) { + d.serial = serial +} + +func (d *DefaultCLIBuilder) SetVsock(vsock *CLIVsock) { + d.vsock = vsock +} + +func (d *DefaultCLIBuilder) SetAPISocket(apiSocket *CLIAPISocket) { + d.apiSocket = apiSocket +} + +func (d *DefaultCLIBuilder) SetLogFile(logFile *CLILogFile) { + d.logFile = logFile +} + +func (d *DefaultCLIBuilder) GetCommandLine() (*CommandLine, error) { + + cmdLine := &CommandLine{} + + if d.serial != nil { + d.serial.Build(cmdLine) + } + + if d.console != nil { + d.console.Build(cmdLine) + } + + if d.logFile != nil { + d.logFile.Build(cmdLine) + } + + if d.cpus != nil { + d.cpus.Build(cmdLine) + } + if d.memory != nil { + d.memory.Build(cmdLine) + } + if d.disk != nil { + d.disk.Build(cmdLine) + } + if d.rng != nil { + d.rng.Build(cmdLine) + } + if d.vsock != nil { + d.vsock.Build(cmdLine) + } + if d.fs != nil { + d.fs.Build(cmdLine) + } + if d.kernel != nil { + d.kernel.Build(cmdLine) + } + if d.nets != nil { + d.nets.Build(cmdLine) + } + if d.cmdline != nil { + d.cmdline.Build(cmdLine) + } + + return cmdLine, nil +} + +type CommandLineDirector struct{} + +func (s *CommandLineDirector) Build(builder CommandLineBuilder) (*CommandLine, error) { + return builder.GetCommandLine() +} diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index cdb2767a66..82bf44b654 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -42,6 +42,9 @@ const ( // AcrnHypervisor is the ACRN hypervisor. AcrnHypervisor HypervisorType = "acrn" + // ClhHypervisor is the ICH hypervisor. + ClhHypervisor HypervisorType = "clh" + // MockHypervisor is a mock hypervisor for testing purposes MockHypervisor HypervisorType = "mock" ) @@ -152,6 +155,9 @@ func (hType *HypervisorType) Set(value string) error { case "acrn": *hType = AcrnHypervisor return nil + case "clh": + *hType = ClhHypervisor + return nil case "mock": *hType = MockHypervisor return nil @@ -169,6 +175,8 @@ func (hType *HypervisorType) String() string { return string(FirecrackerHypervisor) case AcrnHypervisor: return string(AcrnHypervisor) + case ClhHypervisor: + return string(ClhHypervisor) case MockHypervisor: return string(MockHypervisor) default: @@ -185,6 +193,8 @@ func newHypervisor(hType HypervisorType) (hypervisor, error) { return &firecracker{}, nil case AcrnHypervisor: return &Acrn{}, nil + case ClhHypervisor: + return &cloudHypervisor{}, nil case MockHypervisor: return &mockHypervisor{}, nil default: diff --git a/virtcontainers/virtcontainers_test.go b/virtcontainers/virtcontainers_test.go index 7b0ccbda68..4514250ba3 100644 --- a/virtcontainers/virtcontainers_test.go +++ b/virtcontainers/virtcontainers_test.go @@ -46,6 +46,7 @@ var testAcrnKernelPath = "" var testAcrnImagePath = "" var testAcrnPath = "" var testAcrnCtlPath = "" + var testHyperstartCtlSocket = "" var testHyperstartTtySocket = ""