diff --git a/virtcontainers/factory.go b/virtcontainers/factory.go new file mode 100644 index 0000000000..ad4f223114 --- /dev/null +++ b/virtcontainers/factory.go @@ -0,0 +1,15 @@ +// Copyright (c) 2018 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +// Factory controls how a new VM is created. +type Factory interface { + // GetVM gets a new VM from the factory. + GetVM(config VMConfig) (*VM, error) + + // CloseFactory closes and cleans up the factory. + CloseFactory() +} diff --git a/virtcontainers/factory/base/base.go b/virtcontainers/factory/base/base.go new file mode 100644 index 0000000000..449bdddf62 --- /dev/null +++ b/virtcontainers/factory/base/base.go @@ -0,0 +1,24 @@ +// Copyright (c) 2018 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package base + +import vc "github.com/kata-containers/runtime/virtcontainers" + +// FactoryBase is vm factory's internal base factory interfaces. +// The difference between FactoryBase and Factory is that the Factory +// also handles vm config validation/comparison and possible CPU/memory +// hotplugs. It's better to do it at the factory level instead of doing +// the same work in each of the factory implementations. +type FactoryBase interface { + // Config returns base factory config. + Config() vc.VMConfig + + // GetBaseVM returns a paused VM created by the base factory. + GetBaseVM() (*vc.VM, error) + + // CloseFactory closes the base factory. + CloseFactory() +} diff --git a/virtcontainers/factory/cache/cache.go b/virtcontainers/factory/cache/cache.go new file mode 100644 index 0000000000..f008617d64 --- /dev/null +++ b/virtcontainers/factory/cache/cache.go @@ -0,0 +1,83 @@ +// Copyright (c) 2018 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// cache implements base vm factory on top of other base vm factory. + +package cache + +import ( + "fmt" + "sync" + + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/factory/base" +) + +type cache struct { + base base.FactoryBase + + cacheCh chan *vc.VM + closed chan<- int + wg sync.WaitGroup + closeOnce sync.Once +} + +// New creates a new cached vm factory. +func New(count uint, b base.FactoryBase) base.FactoryBase { + if count < 1 { + return b + } + + cacheCh := make(chan *vc.VM) + closed := make(chan int, count) + c := cache{base: b, cacheCh: cacheCh, closed: closed} + for i := 0; i < int(count); i++ { + c.wg.Add(1) + go func() { + for { + vm, err := b.GetBaseVM() + if err != nil { + c.wg.Done() + c.CloseFactory() + return + } + + select { + case cacheCh <- vm: + case <-closed: + vm.Stop() + c.wg.Done() + return + } + } + }() + } + return &c +} + +// Config returns cache vm factory's base factory config. +func (c *cache) Config() vc.VMConfig { + return c.base.Config() +} + +// GetBaseVM returns a base VM from cache factory's base factory. +func (c *cache) GetBaseVM() (*vc.VM, error) { + vm, ok := <-c.cacheCh + if ok { + return vm, nil + } + return nil, fmt.Errorf("cache factory is closed") +} + +// CloseFactory closes the cache factory. +func (c *cache) CloseFactory() { + c.closeOnce.Do(func() { + for len(c.closed) < cap(c.closed) { // send sufficient closed signal + c.closed <- 0 + } + c.wg.Wait() + close(c.cacheCh) + c.base.CloseFactory() + }) +} diff --git a/virtcontainers/factory/direct/direct.go b/virtcontainers/factory/direct/direct.go new file mode 100644 index 0000000000..db1f6c8546 --- /dev/null +++ b/virtcontainers/factory/direct/direct.go @@ -0,0 +1,46 @@ +// Copyright (c) 2018 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// direct implements base vm factory without vm templating. + +package direct + +import ( + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/factory/base" +) + +type direct struct { + config vc.VMConfig +} + +// New returns a new direct vm factory. +func New(config vc.VMConfig) base.FactoryBase { + return &direct{config} +} + +// Config returns the direct factory's configuration. +func (d *direct) Config() vc.VMConfig { + return d.config +} + +// GetBaseVM create a new VM directly. +func (d *direct) GetBaseVM() (*vc.VM, error) { + vm, err := vc.NewVM(d.config) + if err != nil { + return nil, err + } + + err = vm.Pause() + if err != nil { + vm.Stop() + return nil, err + } + + return vm, nil +} + +// CloseFactory closes the direct vm factory. +func (d *direct) CloseFactory() { +} diff --git a/virtcontainers/factory/factory.go b/virtcontainers/factory/factory.go new file mode 100644 index 0000000000..0efc783646 --- /dev/null +++ b/virtcontainers/factory/factory.go @@ -0,0 +1,176 @@ +// Copyright (c) 2018 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package factory + +import ( + "fmt" + "reflect" + + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/factory/base" + "github.com/kata-containers/runtime/virtcontainers/factory/cache" + "github.com/kata-containers/runtime/virtcontainers/factory/direct" + "github.com/kata-containers/runtime/virtcontainers/factory/template" + "github.com/sirupsen/logrus" +) + +var factoryLogger = logrus.FieldLogger(logrus.New()) + +// Config is a collection of VM factory configurations. +type Config struct { + Template bool + Cache uint + + VMConfig vc.VMConfig +} + +func (f *Config) validate() error { + return f.VMConfig.Valid() +} + +type factory struct { + base base.FactoryBase +} + +// NewFactory returns a working factory. +func NewFactory(config Config, fetchOnly bool) (vc.Factory, error) { + err := config.validate() + if err != nil { + return nil, err + } + + if fetchOnly && config.Cache > 0 { + return nil, fmt.Errorf("cache factory does not support fetch") + } + + var b base.FactoryBase + if config.Template { + if fetchOnly { + b, err = template.Fetch(config.VMConfig) + if err != nil { + return nil, err + } + } else { + b = template.New(config.VMConfig) + } + } else { + b = direct.New(config.VMConfig) + } + + if config.Cache > 0 { + b = cache.New(config.Cache, b) + } + + return &factory{b}, nil +} + +func (f *factory) log() *logrus.Entry { + return factoryLogger.WithField("subsystem", "factory") +} + +func resetHypervisorConfig(config *vc.HypervisorConfig) { + config.DefaultVCPUs = 0 + config.DefaultMemSz = 0 + config.BootToBeTemplate = false + config.BootFromTemplate = false + config.MemoryPath = "" + config.DevicesStatePath = "" +} + +// It's important that baseConfig and newConfig are passed by value! +func checkVMConfig(config1, config2 vc.VMConfig) error { + if config1.HypervisorType != config2.HypervisorType { + return fmt.Errorf("hypervisor type does not match: %s vs. %s", config1.HypervisorType, config2.HypervisorType) + } + + if config1.AgentType != config2.AgentType { + return fmt.Errorf("agent type does not match: %s vs. %s", config1.AgentType, config2.AgentType) + } + + // check hypervisor config details + resetHypervisorConfig(&config1.HypervisorConfig) + resetHypervisorConfig(&config2.HypervisorConfig) + + if !reflect.DeepEqual(config1, config2) { + return fmt.Errorf("hypervisor config does not match, base: %+v. new: %+v", config1, config2) + } + + return nil +} + +func (f *factory) checkConfig(config vc.VMConfig) error { + baseConfig := f.base.Config() + + return checkVMConfig(config, baseConfig) +} + +// GetVM returns a working blank VM created by the factory. +func (f *factory) GetVM(config vc.VMConfig) (*vc.VM, error) { + hypervisorConfig := config.HypervisorConfig + err := config.Valid() + if err != nil { + f.log().WithError(err).Error("invalid hypervisor config") + return nil, err + } + + err = f.checkConfig(config) + if err != nil { + f.log().WithError(err).Info("fallback to direct factory vm") + return direct.New(config).GetBaseVM() + } + + f.log().Info("get base VM") + vm, err := f.base.GetBaseVM() + if err != nil { + f.log().WithError(err).Error("failed to get base VM") + return nil, err + } + + // cleanup upon error + defer func() { + if err != nil { + f.log().WithError(err).Error("clean up vm") + vm.Stop() + } + }() + + err = vm.Resume() + if err != nil { + return nil, err + } + + online := false + baseConfig := f.base.Config().HypervisorConfig + if baseConfig.DefaultVCPUs < hypervisorConfig.DefaultVCPUs { + err = vm.AddCPUs(hypervisorConfig.DefaultVCPUs - baseConfig.DefaultVCPUs) + if err != nil { + return nil, err + } + online = true + } + + if baseConfig.DefaultMemSz < hypervisorConfig.DefaultMemSz { + err = vm.AddMemory(hypervisorConfig.DefaultMemSz - baseConfig.DefaultMemSz) + if err != nil { + return nil, err + } + online = true + } + + if online { + err = vm.OnlineCPUMemory() + if err != nil { + return nil, err + } + } + + return vm, nil +} + +// CloseFactory closes the factory. +func (f *factory) CloseFactory() { + f.base.CloseFactory() +} diff --git a/virtcontainers/factory/factory_test.go b/virtcontainers/factory/factory_test.go new file mode 100644 index 0000000000..ee2612843a --- /dev/null +++ b/virtcontainers/factory/factory_test.go @@ -0,0 +1,49 @@ +// Copyright (c) 2018 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package factory + +import ( + "io/ioutil" + "testing" + + "github.com/stretchr/testify/assert" + + vc "github.com/kata-containers/runtime/virtcontainers" +) + +func TestNewFactory(t *testing.T) { + var config Config + + assert := assert.New(t) + + _, err := NewFactory(config, true) + assert.Error(err) + _, err = NewFactory(config, false) + assert.Error(err) + + config.VMConfig = vc.VMConfig{ + HypervisorType: vc.MockHypervisor, + AgentType: vc.NoopAgentType, + } + + _, err = NewFactory(config, false) + assert.Error(err) + + testDir, err := ioutil.TempDir("", "vmfactory-tmp-") + assert.Nil(err) + + config.VMConfig.HypervisorConfig = vc.HypervisorConfig{ + KernelPath: testDir, + ImagePath: testDir, + } + + _, err = NewFactory(config, false) + assert.Nil(err) + + config.Cache = 10 + _, err = NewFactory(config, true) + assert.Error(err) +} diff --git a/virtcontainers/factory/template/template.go b/virtcontainers/factory/template/template.go new file mode 100644 index 0000000000..55dddc67cf --- /dev/null +++ b/virtcontainers/factory/template/template.go @@ -0,0 +1,143 @@ +// Copyright (c) 2018 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// template implements base vm factory with vm templating. + +package template + +import ( + "fmt" + "os" + "syscall" + "time" + + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/factory/base" + "github.com/kata-containers/runtime/virtcontainers/factory/direct" +) + +type template struct { + statePath string + config vc.VMConfig +} + +// Fetch finds and returns a pre-built template factory. +// TODO: save template metadata and fetch from storage. +func Fetch(config vc.VMConfig) (base.FactoryBase, error) { + statePath := vc.RunVMStoragePath + "/template" + t := &template{statePath, config} + + err := t.checkTemplateVM() + if err != nil { + return nil, err + } + + return t, nil +} + +// New creates a new VM template factory. +func New(config vc.VMConfig) base.FactoryBase { + statePath := vc.RunVMStoragePath + "/template" + t := &template{statePath, config} + + err := t.prepareTemplateFiles() + if err != nil { + // fallback to direct factory if template is not supported. + return direct.New(config) + } + + err = t.createTemplateVM() + if err != nil { + // fallback to direct factory if template is not supported. + return direct.New(config) + } + + return t +} + +// Config returns template factory's configuration. +func (t *template) Config() vc.VMConfig { + return t.config +} + +// GetBaseVM creates a new paused VM from the template VM. +func (t *template) GetBaseVM() (*vc.VM, error) { + return t.createFromTemplateVM() +} + +// CloseFactory cleans up the template VM. +func (t *template) CloseFactory() { + syscall.Unmount(t.statePath, 0) + os.RemoveAll(t.statePath) +} + +func (t *template) prepareTemplateFiles() error { + // create and mount tmpfs for the shared memory file + err := os.MkdirAll(t.statePath, 0700) + if err != nil { + return err + } + flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV) + opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.DefaultMemSz+8) + if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil { + return err + } + f, err := os.Create(t.statePath + "/memory") + if err != nil { + return err + } + f.Close() + + return nil +} + +func (t *template) createTemplateVM() error { + // create the template vm + config := t.config + config.HypervisorConfig.BootToBeTemplate = true + config.HypervisorConfig.BootFromTemplate = false + config.HypervisorConfig.MemoryPath = t.statePath + "/memory" + config.HypervisorConfig.DevicesStatePath = t.statePath + "/state" + + vm, err := vc.NewVM(config) + if err != nil { + return err + } + defer vm.Stop() + + err = vm.Pause() + if err != nil { + return err + } + + err = vm.Save() + if err != nil { + return err + } + + // qemu QMP does not wait for migration to finish... + time.Sleep(1 * time.Second) + + return nil +} + +func (t *template) createFromTemplateVM() (*vc.VM, error) { + config := t.config + config.HypervisorConfig.BootToBeTemplate = false + config.HypervisorConfig.BootFromTemplate = true + config.HypervisorConfig.MemoryPath = t.statePath + "/memory" + config.HypervisorConfig.DevicesStatePath = t.statePath + "/state" + + return vc.NewVM(config) +} + +func (t *template) checkTemplateVM() error { + _, err := os.Stat(t.statePath + "/memory") + if err != nil { + return err + } + + _, err = os.Stat(t.statePath + "/state") + return err +}