diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go new file mode 100644 index 0000000000..aaf401b867 --- /dev/null +++ b/virtcontainers/acrn.go @@ -0,0 +1,621 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/store" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/kata-containers/runtime/virtcontainers/utils" + opentracing "github.com/opentracing/opentracing-go" + "github.com/sirupsen/logrus" +) + +// AcrnState keeps Acrn's state +type AcrnState struct { + UUID string +} + +// AcrnInfo keeps PID of the hypervisor +type AcrnInfo struct { + PID int +} + +// acrn is an Hypervisor interface implementation for the Linux acrn hypervisor. +type acrn struct { + id string + + store *store.VCStore + config HypervisorConfig + acrnConfig Config + state AcrnState + info AcrnInfo + arch acrnArch + ctx context.Context +} + +const ( + acrnConsoleSocket = "console.sock" + acrnStopSandboxTimeoutSecs = 15 +) + +// agnostic list of kernel parameters +var acrnDefaultKernelParameters = []Param{ + {"panic", "1"}, +} + +func (a *acrn) kernelParameters() string { + // get a list of arch kernel parameters + params := a.arch.kernelParameters(a.config.Debug) + + // use default parameters + params = append(params, acrnDefaultKernelParameters...) + + // set the maximum number of vCPUs + params = append(params, Param{"maxcpus", fmt.Sprintf("%d", a.config.NumVCPUs)}) + + // add the params specified by the provided config. As the kernel + // honours the last parameter value set and since the config-provided + // params are added here, they will take priority over the defaults. + params = append(params, a.config.KernelParams...) + + paramsStr := SerializeParams(params, "=") + + return strings.Join(paramsStr, " ") +} + +// Adds all capabilities supported by acrn implementation of hypervisor interface +func (a *acrn) capabilities() types.Capabilities { + span, _ := a.trace("capabilities") + defer span.Finish() + + return a.arch.capabilities() +} + +func (a *acrn) hypervisorConfig() HypervisorConfig { + return a.config +} + +// get the acrn binary path +func (a *acrn) acrnPath() (string, error) { + p, err := a.config.HypervisorAssetPath() + if err != nil { + return "", err + } + + if p == "" { + p, err = a.arch.acrnPath() + if err != nil { + return "", err + } + } + + if _, err = os.Stat(p); os.IsNotExist(err) { + return "", fmt.Errorf("acrn path (%s) does not exist", p) + } + + return p, nil +} + +// get the ACRNCTL binary path +func (a *acrn) acrnctlPath() (string, error) { + ctlpath, err := a.config.HypervisorCtlAssetPath() + if err != nil { + return "", err + } + + if ctlpath == "" { + ctlpath, err = a.arch.acrnctlPath() + if err != nil { + return "", err + } + } + + if _, err = os.Stat(ctlpath); os.IsNotExist(err) { + return "", fmt.Errorf("acrnctl path (%s) does not exist", ctlpath) + } + + return ctlpath, nil +} + +// Logger returns a logrus logger appropriate for logging acrn messages +func (a *acrn) Logger() *logrus.Entry { + return virtLog.WithField("subsystem", "acrn") +} + +func (a *acrn) trace(name string) (opentracing.Span, context.Context) { + if a.ctx == nil { + a.Logger().WithField("type", "bug").Error("trace called before context set") + a.ctx = context.Background() + } + + span, ctx := opentracing.StartSpanFromContext(a.ctx, name) + + span.SetTag("subsystem", "hypervisor") + span.SetTag("type", "acrn") + + return span, ctx +} + +func (a *acrn) memoryTopology() (Memory, error) { + memMb := uint64(a.config.MemorySize) + + return a.arch.memoryTopology(memMb), nil +} + +func (a *acrn) appendImage(devices []Device, imagePath string) ([]Device, error) { + if imagePath == "" { + return nil, fmt.Errorf("Image path is empty: %s", imagePath) + } + + // Get sandbox and increment the globalIndex. + // This is to make sure the VM rootfs occupies + // the first Index which is /dev/vda. + sandbox, err := globalSandboxList.lookupSandbox(a.id) + if sandbox == nil && err != nil { + return nil, err + } + sandbox.GetAndSetSandboxBlockIndex() + + devices, err = a.arch.appendImage(devices, imagePath) + if err != nil { + return nil, err + } + + return devices, nil +} + +func (a *acrn) buildDevices(imagePath string) ([]Device, error) { + var devices []Device + + if imagePath == "" { + return nil, fmt.Errorf("Image Path should not be empty: %s", imagePath) + } + + console, err := a.getSandboxConsole(a.id) + if err != nil { + return nil, err + } + + // Add bridges before any other devices. This way we make sure that + // bridge gets the first available PCI address. + devices = a.arch.appendBridges(devices) + + //Add LPC device to the list of other devices. + devices = a.arch.appendLPC(devices) + + devices = a.arch.appendConsole(devices, console) + + devices, err = a.appendImage(devices, imagePath) + if err != nil { + return nil, err + } + + // Create virtio blk devices with dummy backend as a place + // holder for container rootfs (as acrn doesn't support hot-plug). + // Once the container rootfs is known, replace the dummy backend + // with actual path (using block rescan feature in acrn) + devices, err = a.createDummyVirtioBlkDev(devices) + if err != nil { + return nil, err + } + + return devices, nil +} + +// setup sets the Acrn structure up. +func (a *acrn) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error { + span, _ := a.trace("setup") + defer span.Finish() + + err := hypervisorConfig.valid() + if err != nil { + return err + } + + a.id = id + a.store = vcStore + a.config = *hypervisorConfig + a.arch = newAcrnArch(a.config) + if err = a.store.Load(store.Hypervisor, &a.state); err != nil { + // acrn currently supports only known UUIDs for security + // reasons (FuSa). When launching VM, only these pre-defined + // UUID should be used else VM launch will fail. acrn team is + // working on a solution to expose these pre-defined UUIDs + // to Kata in order for it to launch VMs successfully. + // Until this support is available, Kata is limited to + // launch 1 VM (using the default UUID). + // https://github.com/kata-containers/runtime/issues/1785 + + // The path might already exist, but in case of VM templating, + // we have to create it since the sandbox has not created it yet. + if err = os.MkdirAll(store.SandboxRuntimeRootPath(id), store.DirMode); err != nil { + return err + } + + if err = a.store.Store(store.Hypervisor, a.state); err != nil { + return err + } + } + + if err = a.store.Load(store.Hypervisor, &a.info); err != nil { + a.Logger().WithField("function", "setup").WithError(err).Info("No info could be fetched") + } + + return nil +} + +func (a *acrn) createDummyVirtioBlkDev(devices []Device) ([]Device, error) { + span, _ := a.trace("createDummyVirtioBlkDev") + defer span.Finish() + + // Since acrn doesn't support hot-plug, dummy virtio-blk + // devices are added and later replaced with container-rootfs. + // Starting from driveIndex 1, as 0 is allocated for VM rootfs. + for driveIndex := 1; driveIndex <= AcrnBlkDevPoolSz; driveIndex++ { + drive := config.BlockDrive{ + File: "nodisk", + Index: driveIndex, + } + + devices = a.arch.appendBlockDevice(devices, drive) + } + + return devices, nil +} + +// createSandbox is the Hypervisor sandbox creation. +func (a *acrn) createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, store *store.VCStore) error { + // Save the tracing context + a.ctx = ctx + + span, _ := a.trace("createSandbox") + defer span.Finish() + + if err := a.setup(id, hypervisorConfig, store); err != nil { + return err + } + + memory, err := a.memoryTopology() + if err != nil { + return err + } + + kernelPath, err := a.config.KernelAssetPath() + if err != nil { + return err + } + + imagePath, err := a.config.ImageAssetPath() + if err != nil { + return err + } + + kernel := Kernel{ + Path: kernelPath, + ImagePath: imagePath, + Params: a.kernelParameters(), + } + + // Disabling UUID check until the below is fixed. + // https://github.com/kata-containers/runtime/issues/1785 + + devices, err := a.buildDevices(imagePath) + if err != nil { + return err + } + + acrnPath, err := a.acrnPath() + if err != nil { + return err + } + + acrnctlPath, err := a.acrnctlPath() + if err != nil { + return err + } + + acrnConfig := Config{ + ACPIVirt: true, + Path: acrnPath, + CtlPath: acrnctlPath, + Memory: memory, + NumCPU: a.config.NumVCPUs, + Devices: devices, + Kernel: kernel, + Name: fmt.Sprintf("sandbox-%s", a.id), + } + + a.acrnConfig = acrnConfig + + return nil +} + +// startSandbox will start the Sandbox's VM. +func (a *acrn) startSandbox(timeoutSecs int) error { + span, _ := a.trace("startSandbox") + defer span.Finish() + + if a.config.Debug { + params := a.arch.kernelParameters(a.config.Debug) + strParams := SerializeParams(params, "=") + formatted := strings.Join(strParams, " ") + + // The name of this field matches a similar one generated by + // the runtime and allows users to identify which parameters + // are set here, which come from the runtime and which are set + // by the user. + a.Logger().WithField("default-kernel-parameters", formatted).Debug() + } + + vmPath := filepath.Join(store.RunVMStoragePath, a.id) + err := os.MkdirAll(vmPath, store.DirMode) + if err != nil { + return err + } + defer func() { + if err != nil { + if err := os.RemoveAll(vmPath); err != nil { + a.Logger().WithError(err).Error("Failed to clean up vm directory") + } + } + }() + + var strErr string + var PID int + PID, strErr, err = LaunchAcrn(a.acrnConfig, virtLog.WithField("subsystem", "acrn-dm")) + if err != nil { + return fmt.Errorf("%s", strErr) + } + a.info.PID = PID + + if err = a.waitSandbox(timeoutSecs); err != nil { + a.Logger().WithField("acrn wait failed:", err).Debug() + return err + } + + //Store VMM information + return a.store.Store(store.Hypervisor, a.info) + +} + +// waitSandbox will wait for the Sandbox's VM to be up and running. +func (a *acrn) waitSandbox(timeoutSecs int) error { + span, _ := a.trace("waitSandbox") + defer span.Finish() + + if timeoutSecs < 0 { + return fmt.Errorf("Invalid timeout %ds", timeoutSecs) + } + + time.Sleep(time.Duration(timeoutSecs) * time.Second) + + return nil +} + +// stopSandbox will stop the Sandbox's VM. +func (a *acrn) stopSandbox() (err error) { + span, _ := a.trace("stopSandbox") + defer span.Finish() + + a.Logger().Info("Stopping acrn VM") + + defer func() { + if err != nil { + a.Logger().Info("stopSandbox failed") + } else { + a.Logger().Info("acrn VM stopped") + } + }() + + pid := a.info.PID + + // Check if VM process is running, in case it is not, let's + // return from here. + if err = syscall.Kill(pid, syscall.Signal(0)); err != nil { + a.Logger().Info("acrn VM already stopped") + return nil + } + + // Send signal to the VM process to try to stop it properly + if err = syscall.Kill(pid, syscall.SIGINT); err != nil { + a.Logger().Info("Sending signal to stop acrn VM failed") + return err + } + + // Wait for the VM process to terminate + tInit := time.Now() + for { + if err = syscall.Kill(pid, syscall.Signal(0)); err != nil { + a.Logger().Info("acrn VM stopped after sending signal") + return nil + } + + if time.Since(tInit).Seconds() >= acrnStopSandboxTimeoutSecs { + a.Logger().Warnf("VM still running after waiting %ds", acrnStopSandboxTimeoutSecs) + break + } + + // Let's avoid to run a too busy loop + time.Sleep(time.Duration(50) * time.Millisecond) + } + + // Let's try with a hammer now, a SIGKILL should get rid of the + // VM process. + return syscall.Kill(pid, syscall.SIGKILL) + +} + +func (a *acrn) updateBlockDevice(drive *config.BlockDrive) error { + var err error + if drive.File == "" || drive.Index >= AcrnBlkDevPoolSz { + return fmt.Errorf("Empty filepath or invalid drive index, Dive ID:%s, Drive Index:%d", + drive.ID, drive.Index) + } + + slot := AcrnBlkdDevSlot[drive.Index] + + //Explicitly set PCIAddr to NULL, so that VirtPath can be used + drive.PCIAddr = "" + + args := []string{"blkrescan", a.acrnConfig.Name, fmt.Sprintf("%d,%s", slot, drive.File)} + + a.Logger().WithFields(logrus.Fields{ + "drive": drive, + "path": a.config.HypervisorCtlPath, + }).Info("updateBlockDevice with acrnctl path") + cmd := exec.Command(a.config.HypervisorCtlPath, args...) + if err := cmd.Run(); err != nil { + a.Logger().WithError(err).Error("updating Block device with newFile path") + } + + return err +} + +func (a *acrn) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) { + span, _ := a.trace("hotplugAddDevice") + defer span.Finish() + + switch devType { + case blockDev: + //The drive placeholder has to exist prior to Update + return nil, a.updateBlockDevice(devInfo.(*config.BlockDrive)) + default: + return nil, fmt.Errorf("hotplugAddDevice: unsupported device: devInfo:%v, deviceType%v", + devInfo, devType) + } +} + +func (a *acrn) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { + span, _ := a.trace("hotplugRemoveDevice") + defer span.Finish() + + // Not supported. return success + + return nil, nil +} + +func (a *acrn) pauseSandbox() error { + span, _ := a.trace("pauseSandbox") + defer span.Finish() + + // Not supported. return success + + return nil +} + +func (a *acrn) resumeSandbox() error { + span, _ := a.trace("resumeSandbox") + defer span.Finish() + + // Not supported. return success + + return nil +} + +// addDevice will add extra devices to Acrn command line. +func (a *acrn) addDevice(devInfo interface{}, devType deviceType) error { + var err error + span, _ := a.trace("addDevice") + defer span.Finish() + + switch v := devInfo.(type) { + case types.Volume: + // Not supported. return success + err = nil + case types.Socket: + a.acrnConfig.Devices = a.arch.appendSocket(a.acrnConfig.Devices, v) + case kataVSOCK: + // Not supported. return success + err = nil + case Endpoint: + a.acrnConfig.Devices = a.arch.appendNetwork(a.acrnConfig.Devices, v) + case config.BlockDrive: + a.acrnConfig.Devices = a.arch.appendBlockDevice(a.acrnConfig.Devices, v) + case config.VhostUserDeviceAttrs: + // Not supported. return success + err = nil + case config.VFIODev: + // Not supported. return success + err = nil + default: + err = nil + a.Logger().WithField("unknown-device-type", devInfo).Error("Adding device") + } + + return err +} + +// getSandboxConsole builds the path of the console where we can read +// logs coming from the sandbox. +func (a *acrn) getSandboxConsole(id string) (string, error) { + span, _ := a.trace("getSandboxConsole") + defer span.Finish() + + return utils.BuildSocketPath(store.RunVMStoragePath, id, acrnConsoleSocket) +} + +func (a *acrn) saveSandbox() error { + a.Logger().Info("save sandbox") + + // Not supported. return success + + return nil +} + +func (a *acrn) disconnect() { + span, _ := a.trace("disconnect") + defer span.Finish() + + // Not supported. +} + +func (a *acrn) getThreadIDs() (vcpuThreadIDs, error) { + span, _ := a.trace("getThreadIDs") + defer span.Finish() + + // Not supported. return success + //Just allocating an empty map + + return vcpuThreadIDs{}, nil +} + +func (a *acrn) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { + return 0, memoryDevice{}, nil +} + +func (a *acrn) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { + return 0, 0, nil +} + +func (a *acrn) cleanup() error { + span, _ := a.trace("cleanup") + defer span.Finish() + + return nil +} + +func (a *acrn) pid() int { + return a.info.PID +} + +func (a *acrn) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, store *store.VCStore, j []byte) error { + return errors.New("acrn is not supported by VM cache") +} + +func (a *acrn) toGrpc() ([]byte, error) { + return nil, errors.New("acrn is not supported by VM cache") +} diff --git a/virtcontainers/acrn_arch_base.go b/virtcontainers/acrn_arch_base.go new file mode 100644 index 0000000000..a9882576ad --- /dev/null +++ b/virtcontainers/acrn_arch_base.go @@ -0,0 +1,772 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "strings" + + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/sirupsen/logrus" +) + +type acrnArch interface { + + // acrnPath returns the path to the acrn binary + acrnPath() (string, error) + + // acrnctlPath returns the path to the acrnctl binary + acrnctlPath() (string, error) + + // kernelParameters returns the kernel parameters + // if debug is true then kernel debug parameters are included + kernelParameters(debug bool) []Param + + //capabilities returns the capabilities supported by acrn + capabilities() types.Capabilities + + // memoryTopology returns the memory topology using the given amount of memoryMb and hostMemoryMb + memoryTopology(memMb uint64) Memory + + // appendConsole appends a console to devices + appendConsole(devices []Device, path string) []Device + + // appendImage appends an image to devices + appendImage(devices []Device, path string) ([]Device, error) + + // appendBridges appends bridges to devices + appendBridges(devices []Device) []Device + + // appendLPC appends LPC to devices + // UART device emulated by the acrn-dm is connected to the system by the LPC bus + appendLPC(devices []Device) []Device + + // appendSocket appends a socket to devices + appendSocket(devices []Device, socket types.Socket) []Device + + // appendNetwork appends a endpoint device to devices + appendNetwork(devices []Device, endpoint Endpoint) []Device + + // appendBlockDevice appends a block drive to devices + appendBlockDevice(devices []Device, drive config.BlockDrive) []Device + + // handleImagePath handles the Hypervisor Config image path + handleImagePath(config HypervisorConfig) +} + +type acrnArchBase struct { + path string + ctlpath string + kernelParamsNonDebug []Param + kernelParamsDebug []Param + kernelParams []Param +} + +const acrnPath = "/usr/bin/acrn-dm" +const acrnctlPath = "/usr/bin/acrnctl" + +// acrn GVT-g slot is harded code to 2 as there is +// no simple way to pass arguments of PCI slots from +// device model (acrn-dm) to ACRNGT module. +const acrnGVTgReservedSlot = 2 + +const acrnLPCDev = "lpc" +const acrnHostBridge = "hostbridge" + +var baselogger *logrus.Entry + +// AcrnBlkDevPoolSz defines the number of dummy virtio-blk +// device that will be created for hot-plugging container +// rootfs. Since acrn doesn't support hot-plug, dummy virtio-blk +// devices are added and later replaced with container-rootfs. +var AcrnBlkDevPoolSz = 8 + +// AcrnBlkdDevSlot array provides translation between +// the vitio-blk device index and slot it is currently +// attached. +// Allocating extra 1 to accommodate for VM rootfs +// which is at driveIndex 0 +var AcrnBlkdDevSlot = make([]int, AcrnBlkDevPoolSz+1) + +// acrnKernelParamsNonDebug is a list of the default kernel +// parameters that will be used in standard (non-debug) mode. +var acrnKernelParamsNonDebug = []Param{ + {"quiet", ""}, +} + +// acrnKernelParamsSystemdNonDebug is a list of the default systemd related +// kernel parameters that will be used in standard (non-debug) mode. +var acrnKernelParamsSystemdNonDebug = []Param{ + {"systemd.show_status", "false"}, +} + +// acrnKernelParamsDebug is a list of the default kernel +// parameters that will be used in debug mode (as much boot output as +// possible). +var acrnKernelParamsDebug = []Param{ + {"debug", ""}, +} + +// acrnKernelParamsSystemdDebug is a list of the default systemd related kernel +// parameters that will be used in debug mode (as much boot output as +// possible). +var acrnKernelParamsSystemdDebug = []Param{ + {"systemd.show_status", "true"}, + {"systemd.log_level", "debug"}, + {"systemd.log_target", "kmsg"}, + {"printk.devkmsg", "on"}, +} + +var acrnKernelRootParams = []Param{ + {"root", "/dev/vda1 rw rootwait"}, +} + +var acrnKernelParams = []Param{ + {"tsc", "reliable"}, + {"no_timer_check", ""}, + {"nohpet", ""}, + {"console", "tty0"}, + {"console", "ttyS0"}, + {"console", "hvc0"}, + {"log_buf_len", "16M"}, + {"consoleblank", "0"}, + {"iommu", "off"}, + {"i915.avail_planes_per_pipe", "0x070F00"}, + {"i915.enable_hangcheck", "0"}, + {"i915.nuclear_pageflip", "1"}, + {"i915.enable_guc_loading", "0"}, + {"i915.enable_guc_submission", "0"}, + {"i915.enable_guc", "0"}, +} + +// Device is the acrn device interface. +type Device interface { + Valid() bool + AcrnParams(slot int, config *Config) []string +} + +// ConsoleDeviceBackend is the character device backend for acrn +type ConsoleDeviceBackend string + +const ( + + // Socket creates a 2 way stream socket (TCP or Unix). + Socket ConsoleDeviceBackend = "socket" + + // Stdio sends traffic from the guest to acrn's standard output. + Stdio ConsoleDeviceBackend = "console" + + // File backend only supports console output to a file (no input). + File ConsoleDeviceBackend = "file" + + // TTY is an alias for Serial. + TTY ConsoleDeviceBackend = "tty" + + // PTY creates a new pseudo-terminal on the host and connect to it. + PTY ConsoleDeviceBackend = "pty" +) + +// BEPortType marks the port as console port or virtio-serial port +type BEPortType int + +const ( + // SerialBE marks the port as serial port + SerialBE BEPortType = iota + + //ConsoleBE marks the port as console port (append @) + ConsoleBE +) + +// ConsoleDevice represents a acrn console device. +type ConsoleDevice struct { + // Name of the socket + Name string + + //Backend device used for virtio-console + Backend ConsoleDeviceBackend + + // PortType marks the port as serial or console port (@) + PortType BEPortType + + //Path to virtio-console backend (can be omitted for pty, tty, stdio) + Path string +} + +// NetDeviceType is a acrn networking device type. +type NetDeviceType string + +const ( + // TAP is a TAP networking device type. + TAP NetDeviceType = "tap" + + // MACVTAP is a macvtap networking device type. + MACVTAP NetDeviceType = "macvtap" +) + +// NetDevice represents a guest networking device +type NetDevice struct { + // Type is the netdev type (e.g. tap). + Type NetDeviceType + + // IfName is the interface name + IFName string + + //MACAddress is the networking device interface MAC address + MACAddress string +} + +// BlockDevice represents a acrn block device. +type BlockDevice struct { + + // mem path to block device + FilePath string + + //BlkIndex - Blk index corresponding to slot + Index int +} + +// BridgeDevice represents a acrn bridge device like pci-bridge, pxb, etc. +type BridgeDevice struct { + + // Function is PCI function. Func can be from 0 to 7 + Function int + + // Emul is a string describing the type of PCI device e.g. virtio-net + Emul string + + // Config is an optional string, depending on the device, that can be + // used for configuration + Config string +} + +// LPCDevice represents a acrn LPC device +type LPCDevice struct { + + // Function is PCI function. Func can be from 0 to 7 + Function int + + // Emul is a string describing the type of PCI device e.g. virtio-net + Emul string +} + +// Memory is the guest memory configuration structure. +type Memory struct { + // Size is the amount of memory made available to the guest. + // It should be suffixed with M or G for sizes in megabytes or + // gigabytes respectively. + Size string +} + +// Kernel is the guest kernel configuration structure. +type Kernel struct { + // Path is the guest kernel path on the host filesystem. + Path string + + // InitrdPath is the guest initrd path on the host filesystem. + ImagePath string + + // Params is the kernel parameters string. + Params string +} + +// Config is the acrn configuration structure. +// It allows for passing custom settings and parameters to the acrn-dm API. +type Config struct { + + // Path is the acrn binary path. + Path string + + // Path is the acrn binary path. + CtlPath string + + // Name is the acrn guest name + Name string + + // UUID is the acrn process UUID. + UUID string + + // Devices is a list of devices for acrn to create and drive. + Devices []Device + + // Kernel is the guest kernel configuration. + Kernel Kernel + + // Memory is the guest memory configuration. + Memory Memory + + acrnParams []string + + // ACPI virtualization support + ACPIVirt bool + + // NumCPU is the number of CPUs for guest + NumCPU uint32 +} + +// MaxAcrnVCPUs returns the maximum number of vCPUs supported +func MaxAcrnVCPUs() uint32 { + return uint32(8) +} + +func newAcrnArch(config HypervisorConfig) acrnArch { + a := &acrnArchBase{ + path: acrnPath, + ctlpath: acrnctlPath, + kernelParamsNonDebug: acrnKernelParamsNonDebug, + kernelParamsDebug: acrnKernelParamsDebug, + kernelParams: acrnKernelParams, + } + + a.handleImagePath(config) + return a +} + +func (a *acrnArchBase) acrnPath() (string, error) { + p := a.path + return p, nil +} + +func (a *acrnArchBase) acrnctlPath() (string, error) { + ctlpath := a.ctlpath + return ctlpath, nil +} + +func (a *acrnArchBase) kernelParameters(debug bool) []Param { + params := a.kernelParams + + if debug { + params = append(params, a.kernelParamsDebug...) + } else { + params = append(params, a.kernelParamsNonDebug...) + } + + return params +} + +func (a *acrnArchBase) memoryTopology(memoryMb uint64) Memory { + mem := fmt.Sprintf("%dM", memoryMb) + memory := Memory{ + Size: mem, + } + + return memory +} + +func (a *acrnArchBase) capabilities() types.Capabilities { + var caps types.Capabilities + + // For devicemapper disable support for filesystem sharing + caps.SetFsSharingUnsupported() + caps.SetBlockDeviceSupport() + caps.SetBlockDeviceHotplugSupport() + + return caps +} + +// Valid returns true if the CharDevice structure is valid and complete. +func (cdev ConsoleDevice) Valid() bool { + if cdev.Backend != "tty" && cdev.Backend != "pty" && + cdev.Backend != "console" && cdev.Backend != "socket" && + cdev.Backend != "file" { + return false + } else if cdev.PortType != ConsoleBE && cdev.PortType != SerialBE { + return false + } else if cdev.Path == "" { + return false + } else { + return true + } +} + +// AcrnParams returns the acrn parameters built out of this console device. +func (cdev ConsoleDevice) AcrnParams(slot int, config *Config) []string { + var acrnParams []string + var deviceParams []string + + acrnParams = append(acrnParams, "-s") + deviceParams = append(deviceParams, fmt.Sprintf("%d,virtio-console,", slot)) + + if cdev.PortType == ConsoleBE { + deviceParams = append(deviceParams, "@") + } + + switch cdev.Backend { + case "pty": + deviceParams = append(deviceParams, "pty:pty_port") + case "tty": + deviceParams = append(deviceParams, fmt.Sprintf("tty:tty_port=%s", cdev.Path)) + case "socket": + deviceParams = append(deviceParams, fmt.Sprintf("socket:%s=%s", cdev.Name, cdev.Path)) + case "file": + deviceParams = append(deviceParams, fmt.Sprintf("file:file_port=%s", cdev.Path)) + case "stdio": + deviceParams = append(deviceParams, "stdio:stdio_port") + default: + // do nothing. Error should be already caught + } + + acrnParams = append(acrnParams, strings.Join(deviceParams, "")) + return acrnParams +} + +// AcrnNetdevParam converts to the acrn type to string +func (netdev NetDevice) AcrnNetdevParam() []string { + var deviceParams []string + + switch netdev.Type { + case TAP: + deviceParams = append(deviceParams, netdev.IFName) + deviceParams = append(deviceParams, fmt.Sprintf(",mac=%s", netdev.MACAddress)) + case MACVTAP: + deviceParams = append(deviceParams, netdev.IFName) + default: + deviceParams = append(deviceParams, netdev.IFName) + + } + + return deviceParams +} + +// Valid returns true if the NetDevice structure is valid and complete. +func (netdev NetDevice) Valid() bool { + if netdev.IFName == "" { + return false + } else if netdev.MACAddress == "" { + return false + } else if netdev.Type != TAP && netdev.Type != MACVTAP { + return false + } else { + return true + } +} + +// AcrnParams returns the acrn parameters built out of this network device. +func (netdev NetDevice) AcrnParams(slot int, config *Config) []string { + var acrnParams []string + + acrnParams = append(acrnParams, "-s") + acrnParams = append(acrnParams, fmt.Sprintf("%d,virtio-net,%s", slot, strings.Join(netdev.AcrnNetdevParam(), ""))) + + return acrnParams +} + +// Valid returns true if the BlockDevice structure is valid and complete. +func (blkdev BlockDevice) Valid() bool { + return blkdev.FilePath != "" +} + +// AcrnParams returns the acrn parameters built out of this block device. +func (blkdev BlockDevice) AcrnParams(slot int, config *Config) []string { + var acrnParams []string + + device := "virtio-blk" + acrnParams = append(acrnParams, "-s") + acrnParams = append(acrnParams, fmt.Sprintf("%d,%s,%s", + slot, device, blkdev.FilePath)) + + // Update the global array (BlkIndex<->slot) + // Used to identify slots for the hot-plugged virtio-blk devices + if blkdev.Index <= AcrnBlkDevPoolSz { + AcrnBlkdDevSlot[blkdev.Index] = slot + } else { + baselogger.WithFields(logrus.Fields{ + "device": device, + "index": blkdev.Index, + }).Info("Invalid index device") + } + + return acrnParams +} + +// Valid returns true if the BridgeDevice structure is valid and complete. +func (bridgeDev BridgeDevice) Valid() bool { + if bridgeDev.Function != 0 || bridgeDev.Emul != acrnHostBridge { + return false + } + return true +} + +// AcrnParams returns the acrn parameters built out of this bridge device. +func (bridgeDev BridgeDevice) AcrnParams(slot int, config *Config) []string { + var acrnParams []string + + acrnParams = append(acrnParams, "-s") + acrnParams = append(acrnParams, fmt.Sprintf("%d:%d,%s", slot, + bridgeDev.Function, bridgeDev.Emul)) + + return acrnParams +} + +// Valid returns true if the BridgeDevice structure is valid and complete. +func (lpcDev LPCDevice) Valid() bool { + return lpcDev.Emul == acrnLPCDev +} + +// AcrnParams returns the acrn parameters built out of this bridge device. +func (lpcDev LPCDevice) AcrnParams(slot int, config *Config) []string { + var acrnParams []string + var deviceParams []string + + acrnParams = append(acrnParams, "-s") + acrnParams = append(acrnParams, fmt.Sprintf("%d:%d,%s", slot, + lpcDev.Function, lpcDev.Emul)) + + //define UART port + deviceParams = append(deviceParams, "-l") + deviceParams = append(deviceParams, "com1,stdio") + acrnParams = append(acrnParams, strings.Join(deviceParams, "")) + + return acrnParams +} + +func (config *Config) appendName() { + if config.Name != "" { + config.acrnParams = append(config.acrnParams, config.Name) + } +} + +func (config *Config) appendDevices() { + slot := 0 + for _, d := range config.Devices { + if !d.Valid() { + continue + } + + if slot == acrnGVTgReservedSlot { + slot++ /*Slot 2 is assigned for GVT-g in acrn, so skip 2 */ + baselogger.Info("Slot 2 is assigned for GVT-g in acrn, so skipping this slot") + + } + config.acrnParams = append(config.acrnParams, d.AcrnParams(slot, config)...) + slot++ + } +} + +func (config *Config) appendUUID() { + if config.UUID != "" { + config.acrnParams = append(config.acrnParams, "-U") + config.acrnParams = append(config.acrnParams, config.UUID) + } +} + +func (config *Config) appendACPI() { + if config.ACPIVirt { + config.acrnParams = append(config.acrnParams, "-A") + } +} + +func (config *Config) appendMemory() { + if config.Memory.Size != "" { + config.acrnParams = append(config.acrnParams, "-m") + config.acrnParams = append(config.acrnParams, config.Memory.Size) + } +} + +func (config *Config) appendCPUs() { + if config.NumCPU != 0 { + config.acrnParams = append(config.acrnParams, "-c") + config.acrnParams = append(config.acrnParams, fmt.Sprintf("%d", config.NumCPU)) + } + +} + +func (config *Config) appendKernel() { + if config.Kernel.Path == "" { + return + } + config.acrnParams = append(config.acrnParams, "-k") + config.acrnParams = append(config.acrnParams, config.Kernel.Path) + + if config.Kernel.Params == "" { + return + } + config.acrnParams = append(config.acrnParams, "-B") + config.acrnParams = append(config.acrnParams, config.Kernel.Params) +} + +// LaunchAcrn can be used to launch a new acrn instance. +// +// The Config parameter contains a set of acrn parameters and settings. +// +// This function writes its log output via logger parameter. +func LaunchAcrn(config Config, logger *logrus.Entry) (int, string, error) { + baselogger = logger + config.appendUUID() + config.appendACPI() + config.appendMemory() + config.appendCPUs() + config.appendDevices() + config.appendKernel() + config.appendName() + + return LaunchCustomAcrn(context.Background(), config.Path, config.acrnParams, logger) +} + +// LaunchCustomAcrn can be used to launch a new acrn instance. +// +// The path parameter is used to pass the acrn executable path. +// +// params is a slice of options to pass to acrn-dm +// +// This function writes its log output via logger parameter. +func LaunchCustomAcrn(ctx context.Context, path string, params []string, + logger *logrus.Entry) (int, string, error) { + + errStr := "" + + if path == "" { + path = "acrn-dm" + } + + /* #nosec */ + cmd := exec.CommandContext(ctx, path, params...) + + var stderr bytes.Buffer + cmd.Stderr = &stderr + logger.WithFields(logrus.Fields{ + "Path": path, + "Params": params, + }).Info("launching acrn with:") + + err := cmd.Start() + if err != nil { + logger.Errorf("Unable to launch %s: %v", path, err) + errStr = stderr.String() + logger.Errorf("%s", errStr) + } + return cmd.Process.Pid, errStr, err +} + +func (a *acrnArchBase) appendImage(devices []Device, path string) ([]Device, error) { + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil, err + } + + ImgBlkdevice := BlockDevice{ + FilePath: path, + Index: 0, + } + + devices = append(devices, ImgBlkdevice) + + return devices, nil +} + +// appendBridges appends to devices the given bridges +func (a *acrnArchBase) appendBridges(devices []Device) []Device { + devices = append(devices, + BridgeDevice{ + Function: 0, + Emul: acrnHostBridge, + Config: "", + }, + ) + + return devices +} + +// appendBridges appends to devices the given bridges +func (a *acrnArchBase) appendLPC(devices []Device) []Device { + devices = append(devices, + LPCDevice{ + Function: 0, + Emul: acrnLPCDev, + }, + ) + + return devices +} + +func (a *acrnArchBase) appendConsole(devices []Device, path string) []Device { + console := ConsoleDevice{ + Name: "console0", + Backend: Socket, + PortType: ConsoleBE, + Path: path, + } + + devices = append(devices, console) + return devices +} + +func (a *acrnArchBase) appendSocket(devices []Device, socket types.Socket) []Device { + serailsocket := ConsoleDevice{ + Name: socket.Name, + Backend: Socket, + PortType: SerialBE, + Path: socket.HostPath, + } + + devices = append(devices, serailsocket) + return devices +} + +func networkModelToAcrnType(model NetInterworkingModel) NetDeviceType { + switch model { + case NetXConnectBridgedModel: + return TAP + case NetXConnectMacVtapModel: + return MACVTAP + default: + //TAP should work for most other cases + return TAP + } +} + +func (a *acrnArchBase) appendNetwork(devices []Device, endpoint Endpoint) []Device { + switch ep := endpoint.(type) { + case *VethEndpoint: + netPair := ep.NetworkPair() + devices = append(devices, + NetDevice{ + Type: networkModelToAcrnType(netPair.NetInterworkingModel), + IFName: netPair.TAPIface.Name, + MACAddress: netPair.TAPIface.HardAddr, + }, + ) + case *MacvtapEndpoint: + devices = append(devices, + NetDevice{ + Type: MACVTAP, + IFName: ep.Name(), + MACAddress: ep.HardwareAddr(), + }, + ) + default: + // Return devices as is for unsupported endpoint. + baselogger.WithField("Endpoint", endpoint).Error("Unsupported N/W Endpoint") + } + + return devices +} + +func (a *acrnArchBase) appendBlockDevice(devices []Device, drive config.BlockDrive) []Device { + if drive.File == "" { + return devices + } + + devices = append(devices, + BlockDevice{ + FilePath: drive.File, + Index: drive.Index, + }, + ) + + return devices +} + +func (a *acrnArchBase) handleImagePath(config HypervisorConfig) { + if config.ImagePath != "" { + a.kernelParams = append(a.kernelParams, acrnKernelRootParams...) + a.kernelParamsNonDebug = append(a.kernelParamsNonDebug, acrnKernelParamsSystemdNonDebug...) + a.kernelParamsDebug = append(a.kernelParamsDebug, acrnKernelParamsSystemdDebug...) + } +} diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 91b340748e..15ce20e93f 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -121,6 +121,9 @@ func (hType *HypervisorType) Set(value string) error { case "firecracker": *hType = FirecrackerHypervisor return nil + case "acrn": + *hType = AcrnHypervisor + return nil case "mock": *hType = MockHypervisor return nil @@ -136,6 +139,8 @@ func (hType *HypervisorType) String() string { return string(QemuHypervisor) case FirecrackerHypervisor: return string(FirecrackerHypervisor) + case AcrnHypervisor: + return string(AcrnHypervisor) case MockHypervisor: return string(MockHypervisor) default: @@ -150,6 +155,8 @@ func newHypervisor(hType HypervisorType) (hypervisor, error) { return &qemu{}, nil case FirecrackerHypervisor: return &firecracker{}, nil + case AcrnHypervisor: + return &acrn{}, nil case MockHypervisor: return &mockHypervisor{}, nil default: @@ -436,6 +443,8 @@ func (conf *HypervisorConfig) assetPath(t types.AssetType) (string, error) { return conf.InitrdPath, nil case types.HypervisorAsset: return conf.HypervisorPath, nil + case types.HypervisorCtlAsset: + return conf.HypervisorCtlPath, nil case types.FirmwareAsset: return conf.FirmwarePath, nil default: @@ -483,6 +492,11 @@ func (conf *HypervisorConfig) HypervisorAssetPath() (string, error) { return conf.assetPath(types.HypervisorAsset) } +// HypervisorCtlAssetPath returns the VM hypervisor ctl path +func (conf *HypervisorConfig) HypervisorCtlAssetPath() (string, error) { + return conf.assetPath(types.HypervisorCtlAsset) +} + // CustomHypervisorAsset returns true if the hypervisor asset is a custom one, false otherwise. func (conf *HypervisorConfig) CustomHypervisorAsset() bool { return conf.isCustomAsset(types.HypervisorAsset) diff --git a/virtcontainers/types/asset.go b/virtcontainers/types/asset.go index dd9bab3b7c..7c9afd374c 100644 --- a/virtcontainers/types/asset.go +++ b/virtcontainers/types/asset.go @@ -49,6 +49,9 @@ const ( // HypervisorAsset is an hypervisor asset. HypervisorAsset AssetType = "hypervisor" + // HypervisorCtlAsset is an hypervisor control asset. + HypervisorCtlAsset AssetType = "hypervisorctl" + // FirmwareAsset is a firmware asset. FirmwareAsset AssetType = "firmware" )