virtcontainers: add vm factory support

Add vm factory support per design in the VM Factory plugin section.
The vm factory controls how a new vm is created:

1. direct: vm is created directly
2. template: vm is created via vm template. A template vm is pre-created
   and saved. Later vm is just a clone of the template vm so that they
   readonly share a portion of initial memory (including kernel, initramfs
   and the kata agent). CPU and memory are hot plugged when necessary.
3. cache: vm is created via vm caches. A set of cached vm are pre-created
   and maintained alive. New vms are created by just picking a cached vm.
   CPU and memory are hot plugged when necessary.

Fixes: #303

Signed-off-by: Peng Tao <bergwolf@gmail.com>

virtcontainers/factory.go

@@ -0,0 +1,15 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package virtcontainers
+
+// Factory controls how a new VM is created.
+type Factory interface {
+	// GetVM gets a new VM from the factory.
+	GetVM(config VMConfig) (*VM, error)
+
+	// CloseFactory closes and cleans up the factory.
+	CloseFactory()
+}

virtcontainers/factory/base/base.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package base
+
+import vc "github.com/kata-containers/runtime/virtcontainers"
+
+// FactoryBase is vm factory's internal base factory interfaces.
+// The difference between FactoryBase and Factory is that the Factory
+// also handles vm config validation/comparison and possible CPU/memory
+// hotplugs. It's better to do it at the factory level instead of doing
+// the same work in each of the factory implementations.
+type FactoryBase interface {
+	// Config returns base factory config.
+	Config() vc.VMConfig
+
+	// GetBaseVM returns a paused VM created by the base factory.
+	GetBaseVM() (*vc.VM, error)
+
+	// CloseFactory closes the base factory.
+	CloseFactory()
+}

virtcontainers/factory/cache/cache.go

@@ -0,0 +1,83 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// cache implements base vm factory on top of other base vm factory.
+
+package cache
+
+import (
+	"fmt"
+	"sync"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/factory/base"
+)
+
+type cache struct {
+	base base.FactoryBase
+
+	cacheCh   chan *vc.VM
+	closed    chan<- int
+	wg        sync.WaitGroup
+	closeOnce sync.Once
+}
+
+// New creates a new cached vm factory.
+func New(count uint, b base.FactoryBase) base.FactoryBase {
+	if count < 1 {
+		return b
+	}
+
+	cacheCh := make(chan *vc.VM)
+	closed := make(chan int, count)
+	c := cache{base: b, cacheCh: cacheCh, closed: closed}
+	for i := 0; i < int(count); i++ {
+		c.wg.Add(1)
+		go func() {
+			for {
+				vm, err := b.GetBaseVM()
+				if err != nil {
+					c.wg.Done()
+					c.CloseFactory()
+					return
+				}
+
+				select {
+				case cacheCh <- vm:
+				case <-closed:
+					vm.Stop()
+					c.wg.Done()
+					return
+				}
+			}
+		}()
+	}
+	return &c
+}
+
+// Config returns cache vm factory's base factory config.
+func (c *cache) Config() vc.VMConfig {
+	return c.base.Config()
+}
+
+// GetBaseVM returns a base VM from cache factory's base factory.
+func (c *cache) GetBaseVM() (*vc.VM, error) {
+	vm, ok := <-c.cacheCh
+	if ok {
+		return vm, nil
+	}
+	return nil, fmt.Errorf("cache factory is closed")
+}
+
+// CloseFactory closes the cache factory.
+func (c *cache) CloseFactory() {
+	c.closeOnce.Do(func() {
+		for len(c.closed) < cap(c.closed) { // send sufficient closed signal
+			c.closed <- 0
+		}
+		c.wg.Wait()
+		close(c.cacheCh)
+		c.base.CloseFactory()
+	})
+}

virtcontainers/factory/direct/direct.go

@@ -0,0 +1,46 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// direct implements base vm factory without vm templating.
+
+package direct
+
+import (
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/factory/base"
+)
+
+type direct struct {
+	config vc.VMConfig
+}
+
+// New returns a new direct vm factory.
+func New(config vc.VMConfig) base.FactoryBase {
+	return &direct{config}
+}
+
+// Config returns the direct factory's configuration.
+func (d *direct) Config() vc.VMConfig {
+	return d.config
+}
+
+// GetBaseVM create a new VM directly.
+func (d *direct) GetBaseVM() (*vc.VM, error) {
+	vm, err := vc.NewVM(d.config)
+	if err != nil {
+		return nil, err
+	}
+
+	err = vm.Pause()
+	if err != nil {
+		vm.Stop()
+		return nil, err
+	}
+
+	return vm, nil
+}
+
+// CloseFactory closes the direct vm factory.
+func (d *direct) CloseFactory() {
+}

virtcontainers/factory/factory.go

@@ -0,0 +1,176 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package factory
+
+import (
+	"fmt"
+	"reflect"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/factory/base"
+	"github.com/kata-containers/runtime/virtcontainers/factory/cache"
+	"github.com/kata-containers/runtime/virtcontainers/factory/direct"
+	"github.com/kata-containers/runtime/virtcontainers/factory/template"
+	"github.com/sirupsen/logrus"
+)
+
+var factoryLogger = logrus.FieldLogger(logrus.New())
+
+// Config is a collection of VM factory configurations.
+type Config struct {
+	Template bool
+	Cache    uint
+
+	VMConfig vc.VMConfig
+}
+
+func (f *Config) validate() error {
+	return f.VMConfig.Valid()
+}
+
+type factory struct {
+	base base.FactoryBase
+}
+
+// NewFactory returns a working factory.
+func NewFactory(config Config, fetchOnly bool) (vc.Factory, error) {
+	err := config.validate()
+	if err != nil {
+		return nil, err
+	}
+
+	if fetchOnly && config.Cache > 0 {
+		return nil, fmt.Errorf("cache factory does not support fetch")
+	}
+
+	var b base.FactoryBase
+	if config.Template {
+		if fetchOnly {
+			b, err = template.Fetch(config.VMConfig)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			b = template.New(config.VMConfig)
+		}
+	} else {
+		b = direct.New(config.VMConfig)
+	}
+
+	if config.Cache > 0 {
+		b = cache.New(config.Cache, b)
+	}
+
+	return &factory{b}, nil
+}
+
+func (f *factory) log() *logrus.Entry {
+	return factoryLogger.WithField("subsystem", "factory")
+}
+
+func resetHypervisorConfig(config *vc.HypervisorConfig) {
+	config.DefaultVCPUs = 0
+	config.DefaultMemSz = 0
+	config.BootToBeTemplate = false
+	config.BootFromTemplate = false
+	config.MemoryPath = ""
+	config.DevicesStatePath = ""
+}
+
+// It's important that baseConfig and newConfig are passed by value!
+func checkVMConfig(config1, config2 vc.VMConfig) error {
+	if config1.HypervisorType != config2.HypervisorType {
+		return fmt.Errorf("hypervisor type does not match: %s vs. %s", config1.HypervisorType, config2.HypervisorType)
+	}
+
+	if config1.AgentType != config2.AgentType {
+		return fmt.Errorf("agent type does not match: %s vs. %s", config1.AgentType, config2.AgentType)
+	}
+
+	// check hypervisor config details
+	resetHypervisorConfig(&config1.HypervisorConfig)
+	resetHypervisorConfig(&config2.HypervisorConfig)
+
+	if !reflect.DeepEqual(config1, config2) {
+		return fmt.Errorf("hypervisor config does not match, base: %+v. new: %+v", config1, config2)
+	}
+
+	return nil
+}
+
+func (f *factory) checkConfig(config vc.VMConfig) error {
+	baseConfig := f.base.Config()
+
+	return checkVMConfig(config, baseConfig)
+}
+
+// GetVM returns a working blank VM created by the factory.
+func (f *factory) GetVM(config vc.VMConfig) (*vc.VM, error) {
+	hypervisorConfig := config.HypervisorConfig
+	err := config.Valid()
+	if err != nil {
+		f.log().WithError(err).Error("invalid hypervisor config")
+		return nil, err
+	}
+
+	err = f.checkConfig(config)
+	if err != nil {
+		f.log().WithError(err).Info("fallback to direct factory vm")
+		return direct.New(config).GetBaseVM()
+	}
+
+	f.log().Info("get base VM")
+	vm, err := f.base.GetBaseVM()
+	if err != nil {
+		f.log().WithError(err).Error("failed to get base VM")
+		return nil, err
+	}
+
+	// cleanup upon error
+	defer func() {
+		if err != nil {
+			f.log().WithError(err).Error("clean up vm")
+			vm.Stop()
+		}
+	}()
+
+	err = vm.Resume()
+	if err != nil {
+		return nil, err
+	}
+
+	online := false
+	baseConfig := f.base.Config().HypervisorConfig
+	if baseConfig.DefaultVCPUs < hypervisorConfig.DefaultVCPUs {
+		err = vm.AddCPUs(hypervisorConfig.DefaultVCPUs - baseConfig.DefaultVCPUs)
+		if err != nil {
+			return nil, err
+		}
+		online = true
+	}
+
+	if baseConfig.DefaultMemSz < hypervisorConfig.DefaultMemSz {
+		err = vm.AddMemory(hypervisorConfig.DefaultMemSz - baseConfig.DefaultMemSz)
+		if err != nil {
+			return nil, err
+		}
+		online = true
+	}
+
+	if online {
+		err = vm.OnlineCPUMemory()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return vm, nil
+}
+
+// CloseFactory closes the factory.
+func (f *factory) CloseFactory() {
+	f.base.CloseFactory()
+}

virtcontainers/factory/factory_test.go

@@ -0,0 +1,49 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package factory
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+)
+
+func TestNewFactory(t *testing.T) {
+	var config Config
+
+	assert := assert.New(t)
+
+	_, err := NewFactory(config, true)
+	assert.Error(err)
+	_, err = NewFactory(config, false)
+	assert.Error(err)
+
+	config.VMConfig = vc.VMConfig{
+		HypervisorType: vc.MockHypervisor,
+		AgentType:      vc.NoopAgentType,
+	}
+
+	_, err = NewFactory(config, false)
+	assert.Error(err)
+
+	testDir, err := ioutil.TempDir("", "vmfactory-tmp-")
+	assert.Nil(err)
+
+	config.VMConfig.HypervisorConfig = vc.HypervisorConfig{
+		KernelPath: testDir,
+		ImagePath:  testDir,
+	}
+
+	_, err = NewFactory(config, false)
+	assert.Nil(err)
+
+	config.Cache = 10
+	_, err = NewFactory(config, true)
+	assert.Error(err)
+}

virtcontainers/factory/template/template.go

@@ -0,0 +1,143 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// template implements base vm factory with vm templating.
+
+package template
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+	"time"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/factory/base"
+	"github.com/kata-containers/runtime/virtcontainers/factory/direct"
+)
+
+type template struct {
+	statePath string
+	config    vc.VMConfig
+}
+
+// Fetch finds and returns a pre-built template factory.
+// TODO: save template metadata and fetch from storage.
+func Fetch(config vc.VMConfig) (base.FactoryBase, error) {
+	statePath := vc.RunVMStoragePath + "/template"
+	t := &template{statePath, config}
+
+	err := t.checkTemplateVM()
+	if err != nil {
+		return nil, err
+	}
+
+	return t, nil
+}
+
+// New creates a new VM template factory.
+func New(config vc.VMConfig) base.FactoryBase {
+	statePath := vc.RunVMStoragePath + "/template"
+	t := &template{statePath, config}
+
+	err := t.prepareTemplateFiles()
+	if err != nil {
+		// fallback to direct factory if template is not supported.
+		return direct.New(config)
+	}
+
+	err = t.createTemplateVM()
+	if err != nil {
+		// fallback to direct factory if template is not supported.
+		return direct.New(config)
+	}
+
+	return t
+}
+
+// Config returns template factory's configuration.
+func (t *template) Config() vc.VMConfig {
+	return t.config
+}
+
+// GetBaseVM creates a new paused VM from the template VM.
+func (t *template) GetBaseVM() (*vc.VM, error) {
+	return t.createFromTemplateVM()
+}
+
+// CloseFactory cleans up the template VM.
+func (t *template) CloseFactory() {
+	syscall.Unmount(t.statePath, 0)
+	os.RemoveAll(t.statePath)
+}
+
+func (t *template) prepareTemplateFiles() error {
+	// create and mount tmpfs for the shared memory file
+	err := os.MkdirAll(t.statePath, 0700)
+	if err != nil {
+		return err
+	}
+	flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV)
+	opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.DefaultMemSz+8)
+	if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil {
+		return err
+	}
+	f, err := os.Create(t.statePath + "/memory")
+	if err != nil {
+		return err
+	}
+	f.Close()
+
+	return nil
+}
+
+func (t *template) createTemplateVM() error {
+	// create the template vm
+	config := t.config
+	config.HypervisorConfig.BootToBeTemplate = true
+	config.HypervisorConfig.BootFromTemplate = false
+	config.HypervisorConfig.MemoryPath = t.statePath + "/memory"
+	config.HypervisorConfig.DevicesStatePath = t.statePath + "/state"
+
+	vm, err := vc.NewVM(config)
+	if err != nil {
+		return err
+	}
+	defer vm.Stop()
+
+	err = vm.Pause()
+	if err != nil {
+		return err
+	}
+
+	err = vm.Save()
+	if err != nil {
+		return err
+	}
+
+	// qemu QMP does not wait for migration to finish...
+	time.Sleep(1 * time.Second)
+
+	return nil
+}
+
+func (t *template) createFromTemplateVM() (*vc.VM, error) {
+	config := t.config
+	config.HypervisorConfig.BootToBeTemplate = false
+	config.HypervisorConfig.BootFromTemplate = true
+	config.HypervisorConfig.MemoryPath = t.statePath + "/memory"
+	config.HypervisorConfig.DevicesStatePath = t.statePath + "/state"
+
+	return vc.NewVM(config)
+}
+
+func (t *template) checkTemplateVM() error {
+	_, err := os.Stat(t.statePath + "/memory")
+	if err != nil {
+		return err
+	}
+
+	_, err = os.Stat(t.statePath + "/state")
+	return err
+}