Merge pull request #684 from bergwolf/vm-proxy

factory: start proxy before assign vm to a sandbox
This commit is contained in:
Peng Tao 2018-09-17 16:37:46 +08:00 committed by GitHub
commit a5e82c1d4d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 732 additions and 248 deletions

View File

@ -145,6 +145,12 @@ type agent interface {
// start the proxy // start the proxy
startProxy(sandbox *Sandbox) error startProxy(sandbox *Sandbox) error
// set to use an existing proxy
setProxy(sandbox *Sandbox, proxy proxy, pid int, url string) error
// get agent url
getAgentURL() (string, error)
// createSandbox will tell the agent to perform necessary setup for a Sandbox. // createSandbox will tell the agent to perform necessary setup for a Sandbox.
createSandbox(sandbox *Sandbox) error createSandbox(sandbox *Sandbox) error

View File

@ -5,28 +5,27 @@
package virtcontainers package virtcontainers
import ( import "os/exec"
"os/exec"
)
type ccProxy struct { type ccProxy struct {
} }
// start is the proxy start implementation for ccProxy. // start is the proxy start implementation for ccProxy.
func (p *ccProxy) start(sandbox *Sandbox, params proxyParams) (int, string, error) { func (p *ccProxy) start(params proxyParams) (int, string, error) {
config, err := newProxyConfig(sandbox.config) if err := validateProxyParams(params); err != nil {
if err != nil {
return -1, "", err return -1, "", err
} }
params.logger.Info("Starting cc proxy")
// construct the socket path the proxy instance will use // construct the socket path the proxy instance will use
proxyURL, err := defaultProxyURL(sandbox, SocketTypeUNIX) proxyURL, err := defaultProxyURL(params.id, SocketTypeUNIX)
if err != nil { if err != nil {
return -1, "", err return -1, "", err
} }
args := []string{config.Path, "-uri", proxyURL} args := []string{params.path, "-uri", proxyURL}
if config.Debug { if params.debug {
args = append(args, "-log", "debug") args = append(args, "-log", "debug")
} }
@ -38,7 +37,7 @@ func (p *ccProxy) start(sandbox *Sandbox, params proxyParams) (int, string, erro
return cmd.Process.Pid, proxyURL, nil return cmd.Process.Pid, proxyURL, nil
} }
func (p *ccProxy) stop(sandbox *Sandbox, pid int) error { func (p *ccProxy) stop(pid int) error {
return nil return nil
} }

View File

@ -7,10 +7,22 @@ package virtcontainers
import ( import (
"testing" "testing"
"github.com/stretchr/testify/assert"
) )
func TestCCProxyStart(t *testing.T) { func TestCCProxyStart(t *testing.T) {
proxy := &ccProxy{} proxy := &ccProxy{}
testProxyStart(t, nil, proxy, CCProxyType) testProxyStart(t, nil, proxy)
}
func TestCCProxy(t *testing.T) {
proxy := &ccProxy{}
assert := assert.New(t)
err := proxy.stop(0)
assert.Nil(err)
assert.False(proxy.consoleWatched())
} }

View File

@ -21,7 +21,7 @@ type FactoryBase interface {
Config() vc.VMConfig Config() vc.VMConfig
// GetBaseVM returns a paused VM created by the base factory. // GetBaseVM returns a paused VM created by the base factory.
GetBaseVM(ctx context.Context) (*vc.VM, error) GetBaseVM(ctx context.Context, config vc.VMConfig) (*vc.VM, error)
// CloseFactory closes the base factory. // CloseFactory closes the base factory.
CloseFactory(ctx context.Context) CloseFactory(ctx context.Context)

View File

@ -37,7 +37,7 @@ func New(ctx context.Context, count uint, b base.FactoryBase) base.FactoryBase {
c.wg.Add(1) c.wg.Add(1)
go func() { go func() {
for { for {
vm, err := b.GetBaseVM(ctx) vm, err := b.GetBaseVM(ctx, c.Config())
if err != nil { if err != nil {
c.wg.Done() c.wg.Done()
c.CloseFactory(ctx) c.CloseFactory(ctx)
@ -63,7 +63,7 @@ func (c *cache) Config() vc.VMConfig {
} }
// GetBaseVM returns a base VM from cache factory's base factory. // GetBaseVM returns a base VM from cache factory's base factory.
func (c *cache) GetBaseVM(ctx context.Context) (*vc.VM, error) { func (c *cache) GetBaseVM(ctx context.Context, config vc.VMConfig) (*vc.VM, error) {
vm, ok := <-c.cacheCh vm, ok := <-c.cacheCh
if ok { if ok {
return vm, nil return vm, nil

View File

@ -27,6 +27,7 @@ func TestTemplateFactory(t *testing.T) {
vmConfig := vc.VMConfig{ vmConfig := vc.VMConfig{
HypervisorType: vc.MockHypervisor, HypervisorType: vc.MockHypervisor,
AgentType: vc.NoopAgentType, AgentType: vc.NoopAgentType,
ProxyType: vc.NoopProxyType,
HypervisorConfig: hyperConfig, HypervisorConfig: hyperConfig,
} }
@ -39,7 +40,7 @@ func TestTemplateFactory(t *testing.T) {
assert.Equal(f.Config(), vmConfig) assert.Equal(f.Config(), vmConfig)
// GetBaseVM // GetBaseVM
_, err := f.GetBaseVM(ctx) _, err := f.GetBaseVM(ctx, vmConfig)
assert.Nil(err) assert.Nil(err)
// CloseFactory // CloseFactory

View File

@ -28,8 +28,8 @@ func (d *direct) Config() vc.VMConfig {
} }
// GetBaseVM create a new VM directly. // GetBaseVM create a new VM directly.
func (d *direct) GetBaseVM(ctx context.Context) (*vc.VM, error) { func (d *direct) GetBaseVM(ctx context.Context, config vc.VMConfig) (*vc.VM, error) {
vm, err := vc.NewVM(ctx, d.config) vm, err := vc.NewVM(ctx, config)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -26,6 +26,7 @@ func TestTemplateFactory(t *testing.T) {
vmConfig := vc.VMConfig{ vmConfig := vc.VMConfig{
HypervisorType: vc.MockHypervisor, HypervisorType: vc.MockHypervisor,
AgentType: vc.NoopAgentType, AgentType: vc.NoopAgentType,
ProxyType: vc.NoopProxyType,
HypervisorConfig: hyperConfig, HypervisorConfig: hyperConfig,
} }
@ -38,7 +39,7 @@ func TestTemplateFactory(t *testing.T) {
assert.Equal(f.Config(), vmConfig) assert.Equal(f.Config(), vmConfig)
// GetBaseVM // GetBaseVM
_, err := f.GetBaseVM(ctx) _, err := f.GetBaseVM(ctx, vmConfig)
assert.Nil(err) assert.Nil(err)
// CloseFactory // CloseFactory

View File

@ -29,10 +29,6 @@ type Config struct {
VMConfig vc.VMConfig VMConfig vc.VMConfig
} }
func (f *Config) validate() error {
return f.VMConfig.Valid()
}
type factory struct { type factory struct {
base base.FactoryBase base base.FactoryBase
} }
@ -50,7 +46,7 @@ func NewFactory(ctx context.Context, config Config, fetchOnly bool) (vc.Factory,
span, _ := trace(ctx, "NewFactory") span, _ := trace(ctx, "NewFactory")
defer span.Finish() defer span.Finish()
err := config.validate() err := config.VMConfig.Valid()
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -93,13 +89,15 @@ func (f *factory) log() *logrus.Entry {
return factoryLogger.WithField("subsystem", "factory") return factoryLogger.WithField("subsystem", "factory")
} }
func resetHypervisorConfig(config *vc.HypervisorConfig) { func resetHypervisorConfig(config *vc.VMConfig) {
config.NumVCPUs = 0 config.HypervisorConfig.NumVCPUs = 0
config.MemorySize = 0 config.HypervisorConfig.MemorySize = 0
config.BootToBeTemplate = false config.HypervisorConfig.BootToBeTemplate = false
config.BootFromTemplate = false config.HypervisorConfig.BootFromTemplate = false
config.MemoryPath = "" config.HypervisorConfig.MemoryPath = ""
config.DevicesStatePath = "" config.HypervisorConfig.DevicesStatePath = ""
config.ProxyType = vc.NoopProxyType
config.ProxyConfig = vc.ProxyConfig{}
} }
// It's important that baseConfig and newConfig are passed by value! // It's important that baseConfig and newConfig are passed by value!
@ -113,8 +111,8 @@ func checkVMConfig(config1, config2 vc.VMConfig) error {
} }
// check hypervisor config details // check hypervisor config details
resetHypervisorConfig(&config1.HypervisorConfig) resetHypervisorConfig(&config1)
resetHypervisorConfig(&config2.HypervisorConfig) resetHypervisorConfig(&config2)
if !reflect.DeepEqual(config1, config2) { if !reflect.DeepEqual(config1, config2) {
return fmt.Errorf("hypervisor config does not match, base: %+v. new: %+v", config1, config2) return fmt.Errorf("hypervisor config does not match, base: %+v. new: %+v", config1, config2)
@ -129,13 +127,25 @@ func (f *factory) checkConfig(config vc.VMConfig) error {
return checkVMConfig(config, baseConfig) return checkVMConfig(config, baseConfig)
} }
func (f *factory) validateNewVMConfig(config vc.VMConfig) error {
if len(config.AgentType.String()) == 0 {
return fmt.Errorf("Missing agent type")
}
if len(config.ProxyType.String()) == 0 {
return fmt.Errorf("Missing proxy type")
}
return config.Valid()
}
// GetVM returns a working blank VM created by the factory. // GetVM returns a working blank VM created by the factory.
func (f *factory) GetVM(ctx context.Context, config vc.VMConfig) (*vc.VM, error) { func (f *factory) GetVM(ctx context.Context, config vc.VMConfig) (*vc.VM, error) {
span, _ := trace(ctx, "GetVM") span, _ := trace(ctx, "GetVM")
defer span.Finish() defer span.Finish()
hypervisorConfig := config.HypervisorConfig hypervisorConfig := config.HypervisorConfig
err := config.Valid() err := f.validateNewVMConfig(config)
if err != nil { if err != nil {
f.log().WithError(err).Error("invalid hypervisor config") f.log().WithError(err).Error("invalid hypervisor config")
return nil, err return nil, err
@ -144,11 +154,11 @@ func (f *factory) GetVM(ctx context.Context, config vc.VMConfig) (*vc.VM, error)
err = f.checkConfig(config) err = f.checkConfig(config)
if err != nil { if err != nil {
f.log().WithError(err).Info("fallback to direct factory vm") f.log().WithError(err).Info("fallback to direct factory vm")
return direct.New(ctx, config).GetBaseVM(ctx) return direct.New(ctx, config).GetBaseVM(ctx, config)
} }
f.log().Info("get base VM") f.log().Info("get base VM")
vm, err := f.base.GetBaseVM(ctx) vm, err := f.base.GetBaseVM(ctx, config)
if err != nil { if err != nil {
f.log().WithError(err).Error("failed to get base VM") f.log().WithError(err).Error("failed to get base VM")
return nil, err return nil, err

View File

@ -94,23 +94,27 @@ func TestFactorySetLogger(t *testing.T) {
func TestVMConfigValid(t *testing.T) { func TestVMConfigValid(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
config := Config{}
err := config.validate()
assert.Error(err)
testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") testDir, _ := ioutil.TempDir("", "vmfactory-tmp-")
config.VMConfig = vc.VMConfig{ config := vc.VMConfig{
HypervisorType: vc.MockHypervisor, HypervisorType: vc.MockHypervisor,
AgentType: vc.NoopAgentType,
HypervisorConfig: vc.HypervisorConfig{ HypervisorConfig: vc.HypervisorConfig{
KernelPath: testDir, KernelPath: testDir,
ImagePath: testDir, ImagePath: testDir,
}, },
} }
err = config.validate() f := factory{}
err := f.validateNewVMConfig(config)
assert.NotNil(err)
config.AgentType = vc.NoopAgentType
err = f.validateNewVMConfig(config)
assert.NotNil(err)
config.ProxyType = vc.NoopProxyType
err = f.validateNewVMConfig(config)
assert.Nil(err) assert.Nil(err)
} }
@ -165,10 +169,14 @@ func TestFactoryGetVM(t *testing.T) {
} }
vmConfig := vc.VMConfig{ vmConfig := vc.VMConfig{
HypervisorType: vc.MockHypervisor, HypervisorType: vc.MockHypervisor,
AgentType: vc.NoopAgentType,
HypervisorConfig: hyperConfig, HypervisorConfig: hyperConfig,
AgentType: vc.NoopAgentType,
ProxyType: vc.NoopProxyType,
} }
err := vmConfig.Valid()
assert.Nil(err)
ctx := context.Background() ctx := context.Background()
// direct factory // direct factory

View File

@ -23,6 +23,10 @@ type template struct {
config vc.VMConfig config vc.VMConfig
} }
var templateProxyType = vc.KataBuiltInProxyType
var templateWaitForAgent = 2 * time.Second
var templateWaitForMigration = 1 * time.Second
// Fetch finds and returns a pre-built template factory. // Fetch finds and returns a pre-built template factory.
// TODO: save template metadata and fetch from storage. // TODO: save template metadata and fetch from storage.
func Fetch(config vc.VMConfig) (base.FactoryBase, error) { func Fetch(config vc.VMConfig) (base.FactoryBase, error) {
@ -63,8 +67,8 @@ func (t *template) Config() vc.VMConfig {
} }
// GetBaseVM creates a new paused VM from the template VM. // GetBaseVM creates a new paused VM from the template VM.
func (t *template) GetBaseVM(ctx context.Context) (*vc.VM, error) { func (t *template) GetBaseVM(ctx context.Context, config vc.VMConfig) (*vc.VM, error) {
return t.createFromTemplateVM(ctx) return t.createFromTemplateVM(ctx, config)
} }
// CloseFactory cleans up the template VM. // CloseFactory cleans up the template VM.
@ -100,6 +104,8 @@ func (t *template) createTemplateVM(ctx context.Context) error {
config.HypervisorConfig.BootFromTemplate = false config.HypervisorConfig.BootFromTemplate = false
config.HypervisorConfig.MemoryPath = t.statePath + "/memory" config.HypervisorConfig.MemoryPath = t.statePath + "/memory"
config.HypervisorConfig.DevicesStatePath = t.statePath + "/state" config.HypervisorConfig.DevicesStatePath = t.statePath + "/state"
// template vm uses builtin proxy
config.ProxyType = templateProxyType
vm, err := vc.NewVM(ctx, config) vm, err := vc.NewVM(ctx, config)
if err != nil { if err != nil {
@ -107,28 +113,41 @@ func (t *template) createTemplateVM(ctx context.Context) error {
} }
defer vm.Stop() defer vm.Stop()
err = vm.Pause() if err = vm.Disconnect(); err != nil {
if err != nil {
return err return err
} }
err = vm.Save() // Sleep a bit to let the agent grpc server clean up
if err != nil { // When we close connection to the agent, it needs sometime to cleanup
// and restart listening on the communication( serial or vsock) port.
// That time can be saved if we sleep a bit to wait for the agent to
// come around and start listening again. The sleep is only done when
// creating new vm templates and saves time for every new vm that are
// created from template, so it worth the invest.
time.Sleep(templateWaitForAgent)
if err = vm.Pause(); err != nil {
return err
}
if err = vm.Save(); err != nil {
return err return err
} }
// qemu QMP does not wait for migration to finish... // qemu QMP does not wait for migration to finish...
time.Sleep(1 * time.Second) time.Sleep(templateWaitForMigration)
return nil return nil
} }
func (t *template) createFromTemplateVM(ctx context.Context) (*vc.VM, error) { func (t *template) createFromTemplateVM(ctx context.Context, c vc.VMConfig) (*vc.VM, error) {
config := t.config config := t.config
config.HypervisorConfig.BootToBeTemplate = false config.HypervisorConfig.BootToBeTemplate = false
config.HypervisorConfig.BootFromTemplate = true config.HypervisorConfig.BootFromTemplate = true
config.HypervisorConfig.MemoryPath = t.statePath + "/memory" config.HypervisorConfig.MemoryPath = t.statePath + "/memory"
config.HypervisorConfig.DevicesStatePath = t.statePath + "/state" config.HypervisorConfig.DevicesStatePath = t.statePath + "/state"
config.ProxyType = c.ProxyType
config.ProxyConfig = c.ProxyConfig
return vc.NewVM(ctx, config) return vc.NewVM(ctx, config)
} }

View File

@ -10,6 +10,7 @@ import (
"io/ioutil" "io/ioutil"
"os" "os"
"testing" "testing"
"time"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@ -19,6 +20,9 @@ import (
func TestTemplateFactory(t *testing.T) { func TestTemplateFactory(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
templateWaitForMigration = 1 * time.Microsecond
templateWaitForAgent = 1 * time.Microsecond
testDir, _ := ioutil.TempDir("", "vmfactory-tmp-") testDir, _ := ioutil.TempDir("", "vmfactory-tmp-")
hyperConfig := vc.HypervisorConfig{ hyperConfig := vc.HypervisorConfig{
KernelPath: testDir, KernelPath: testDir,
@ -26,10 +30,14 @@ func TestTemplateFactory(t *testing.T) {
} }
vmConfig := vc.VMConfig{ vmConfig := vc.VMConfig{
HypervisorType: vc.MockHypervisor, HypervisorType: vc.MockHypervisor,
AgentType: vc.NoopAgentType,
HypervisorConfig: hyperConfig, HypervisorConfig: hyperConfig,
AgentType: vc.NoopAgentType,
ProxyType: vc.NoopProxyType,
} }
err := vmConfig.Valid()
assert.Nil(err)
ctx := context.Background() ctx := context.Background()
// New // New
@ -39,7 +47,7 @@ func TestTemplateFactory(t *testing.T) {
assert.Equal(f.Config(), vmConfig) assert.Equal(f.Config(), vmConfig)
// GetBaseVM // GetBaseVM
_, err := f.GetBaseVM(ctx) _, err = f.GetBaseVM(ctx, vmConfig)
assert.Nil(err) assert.Nil(err)
// Fetch // Fetch
@ -48,6 +56,8 @@ func TestTemplateFactory(t *testing.T) {
config: vmConfig, config: vmConfig,
} }
assert.Equal(tt.Config(), vmConfig)
err = tt.checkTemplateVM() err = tt.checkTemplateVM()
assert.Error(err) assert.Error(err)
@ -62,9 +72,22 @@ func TestTemplateFactory(t *testing.T) {
assert.Nil(err) assert.Nil(err)
err = tt.createTemplateVM(ctx) err = tt.createTemplateVM(ctx)
assert.Error(err)
templateProxyType = vc.NoopProxyType
_, err = tt.GetBaseVM(ctx, vmConfig)
assert.Nil(err) assert.Nil(err)
_, err = tt.GetBaseVM(ctx) _, err = f.GetBaseVM(ctx, vmConfig)
assert.Nil(err)
err = tt.createTemplateVM(ctx)
assert.Nil(err)
_, err = tt.GetBaseVM(ctx, vmConfig)
assert.Nil(err)
_, err = f.GetBaseVM(ctx, vmConfig)
assert.Nil(err) assert.Nil(err)
// CloseFactory // CloseFactory

View File

@ -389,16 +389,34 @@ func (h *hyper) startProxy(sandbox *Sandbox) error {
return nil return nil
} }
if h.state.URL != "" {
h.Logger().WithFields(logrus.Fields{
"sandbox": sandbox.id,
"proxy-pid": h.state.ProxyPid,
"proxy-url": h.state.URL,
}).Infof("proxy already started")
return nil
}
// Start the proxy here // Start the proxy here
pid, uri, err := h.proxy.start(sandbox, proxyParams{}) pid, uri, err := h.proxy.start(proxyParams{
id: sandbox.id,
path: sandbox.config.ProxyConfig.Path,
debug: sandbox.config.ProxyConfig.Debug,
logger: h.Logger(),
})
if err != nil { if err != nil {
return err return err
} }
defer func() {
if err != nil {
h.proxy.stop(pid)
}
}()
// Fill agent state with proxy information, and store them. // Fill agent state with proxy information, and store them.
h.state.ProxyPid = pid if err = h.setProxy(sandbox, h.proxy, pid, uri); err != nil {
h.state.URL = uri
if err := sandbox.storage.storeAgentState(sandbox.id, h.state); err != nil {
return err return err
} }
@ -461,7 +479,18 @@ func (h *hyper) stopSandbox(sandbox *Sandbox) error {
return err return err
} }
return h.proxy.stop(sandbox, h.state.ProxyPid) if err := h.proxy.stop(h.state.ProxyPid); err != nil {
return err
}
h.state.ProxyPid = -1
h.state.URL = ""
if err := sandbox.storage.storeAgentState(sandbox.id, h.state); err != nil {
// ignore error
h.Logger().WithError(err).WithField("sandbox", sandbox.id).Error("failed to clean up agent state")
}
return nil
} }
// handleBlockVolumes handles volumes that are block device files, by // handleBlockVolumes handles volumes that are block device files, by
@ -932,3 +961,29 @@ func (h *hyper) reseedRNG(data []byte) error {
// hyperstart-agent does not support reseeding // hyperstart-agent does not support reseeding
return nil return nil
} }
func (h *hyper) getAgentURL() (string, error) {
// hyperstart-agent does not support getting agent url
return "", nil
}
func (h *hyper) setProxy(sandbox *Sandbox, proxy proxy, pid int, url string) error {
if url == "" {
return fmt.Errorf("invalid empty proxy url")
}
if h.state.URL != "" && h.state.URL != url {
h.proxy.stop(h.state.ProxyPid)
}
h.proxy = proxy
h.state.ProxyPid = pid
h.state.URL = url
if sandbox != nil {
if err := sandbox.storage.storeAgentState(sandbox.id, h.state); err != nil {
return err
}
}
return nil
}

View File

@ -238,3 +238,26 @@ func TestHyperListRoutes(t *testing.T) {
_, err := h.listRoutes() _, err := h.listRoutes()
assert.Nil(err) assert.Nil(err)
} }
func TestHyperSetProxy(t *testing.T) {
assert := assert.New(t)
h := &hyper{}
p := &ccProxy{}
s := &Sandbox{storage: &filesystem{}}
err := h.setProxy(s, p, 0, "")
assert.Error(err)
err = h.setProxy(s, p, 0, "foobar")
assert.Error(err)
}
func TestHyperGetAgentUrl(t *testing.T) {
assert := assert.New(t)
h := &hyper{}
url, err := h.getAgentURL()
assert.Nil(err)
assert.Empty(url)
}

View File

@ -471,19 +471,37 @@ func (k *kataAgent) startProxy(sandbox *Sandbox) error {
return nil return nil
} }
if k.state.URL != "" {
k.Logger().WithFields(logrus.Fields{
"sandbox": sandbox.id,
"proxy-pid": k.state.ProxyPid,
"proxy-url": k.state.URL,
}).Infof("proxy already started")
return nil
}
// Get agent socket path to provide it to the proxy. // Get agent socket path to provide it to the proxy.
agentURL, err := k.agentURL() agentURL, err := k.agentURL()
if err != nil { if err != nil {
return err return err
} }
consoleURL, err := sandbox.hypervisor.getSandboxConsole(sandbox.id)
if err != nil {
return err
}
proxyParams := proxyParams{ proxyParams := proxyParams{
id: sandbox.id,
path: sandbox.config.ProxyConfig.Path,
agentURL: agentURL, agentURL: agentURL,
consoleURL: consoleURL,
logger: k.Logger().WithField("sandbox", sandbox.id), logger: k.Logger().WithField("sandbox", sandbox.id),
debug: sandbox.config.ProxyConfig.Debug,
} }
// Start the proxy here // Start the proxy here
pid, uri, err := k.proxy.start(sandbox, proxyParams) pid, uri, err := k.proxy.start(proxyParams)
if err != nil { if err != nil {
return err return err
} }
@ -491,15 +509,13 @@ func (k *kataAgent) startProxy(sandbox *Sandbox) error {
// If error occurs after kata-proxy process start, // If error occurs after kata-proxy process start,
// then rollback to kill kata-proxy process // then rollback to kill kata-proxy process
defer func() { defer func() {
if err != nil && pid > 0 { if err != nil {
k.proxy.stop(sandbox, pid) k.proxy.stop(pid)
} }
}() }()
// Fill agent state with proxy information, and store them. // Fill agent state with proxy information, and store them.
k.state.ProxyPid = pid if err = k.setProxy(sandbox, k.proxy, pid, uri); err != nil {
k.state.URL = uri
if err = sandbox.storage.storeAgentState(sandbox.id, k.state); err != nil {
return err return err
} }
@ -512,6 +528,35 @@ func (k *kataAgent) startProxy(sandbox *Sandbox) error {
return nil return nil
} }
func (k *kataAgent) getAgentURL() (string, error) {
return k.agentURL()
}
func (k *kataAgent) setProxy(sandbox *Sandbox, proxy proxy, pid int, url string) error {
if url == "" {
var err error
if url, err = k.agentURL(); err != nil {
return err
}
}
// Are we setting the same proxy again?
if k.proxy != nil && k.state.URL != "" && k.state.URL != url {
k.proxy.stop(k.state.ProxyPid)
}
k.proxy = proxy
k.state.ProxyPid = pid
k.state.URL = url
if sandbox != nil {
if err := sandbox.storage.storeAgentState(sandbox.id, k.state); err != nil {
return err
}
}
return nil
}
func (k *kataAgent) startSandbox(sandbox *Sandbox) error { func (k *kataAgent) startSandbox(sandbox *Sandbox) error {
span, _ := k.trace("startSandbox") span, _ := k.trace("startSandbox")
defer span.Finish() defer span.Finish()
@ -602,7 +647,19 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
return err return err
} }
return k.proxy.stop(sandbox, k.state.ProxyPid) if err := k.proxy.stop(k.state.ProxyPid); err != nil {
return err
}
// clean up agent state
k.state.ProxyPid = -1
k.state.URL = ""
if err := sandbox.storage.storeAgentState(sandbox.id, k.state); err != nil {
// ignore error
k.Logger().WithError(err).WithField("sandbox", sandbox.id).Error("failed to clean up agent state")
}
return nil
} }
func (k *kataAgent) cleanupSandbox(sandbox *Sandbox) error { func (k *kataAgent) cleanupSandbox(sandbox *Sandbox) error {

View File

@ -795,3 +795,35 @@ func TestAgentNetworkOperation(t *testing.T) {
_, err = k.listRoutes() _, err = k.listRoutes()
assert.Nil(err) assert.Nil(err)
} }
func TestKataAgentSetProxy(t *testing.T) {
assert := assert.New(t)
k := &kataAgent{}
p := &kataBuiltInProxy{}
s := &Sandbox{storage: &filesystem{}}
err := k.setProxy(s, p, 0, "")
assert.Error(err)
err = k.setProxy(s, p, 0, "foobar")
assert.Error(err)
}
func TestKataGetAgentUrl(t *testing.T) {
assert := assert.New(t)
k := &kataAgent{}
err := k.generateVMSocket("foobar", KataAgentConfig{})
assert.Nil(err)
url, err := k.getAgentURL()
assert.Nil(err)
assert.NotEmpty(url)
err = k.generateVMSocket("foobar", KataAgentConfig{UseVSock: true})
assert.Nil(err)
url, err = k.getAgentURL()
assert.Nil(err)
assert.NotEmpty(url)
}

View File

@ -14,6 +14,8 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
var buildinProxyConsoleProto = consoleProtoUnix
// This is a kata builtin proxy implementation of the proxy interface. Kata proxy // This is a kata builtin proxy implementation of the proxy interface. Kata proxy
// functionality is implemented inside the virtcontainers library. // functionality is implemented inside the virtcontainers library.
type kataBuiltInProxy struct { type kataBuiltInProxy struct {
@ -26,22 +28,36 @@ func (p *kataBuiltInProxy) consoleWatched() bool {
return p.conn != nil return p.conn != nil
} }
func (p *kataBuiltInProxy) validateParams(params proxyParams) error {
if len(params.id) == 0 || len(params.agentURL) == 0 || len(params.consoleURL) == 0 {
return fmt.Errorf("Invalid proxy parameters %+v", params)
}
if params.logger == nil {
return fmt.Errorf("Invalid proxy parameter: proxy logger is not set")
}
return nil
}
// start is the proxy start implementation for kata builtin proxy. // start is the proxy start implementation for kata builtin proxy.
// It starts the console watcher for the guest. // It starts the console watcher for the guest.
// It returns agentURL to let agent connect directly. // It returns agentURL to let agent connect directly.
func (p *kataBuiltInProxy) start(sandbox *Sandbox, params proxyParams) (int, string, error) { func (p *kataBuiltInProxy) start(params proxyParams) (int, string, error) {
if p.consoleWatched() { if err := p.validateParams(params); err != nil {
return -1, "", fmt.Errorf("kata builtin proxy running for sandbox %s", p.sandboxID)
}
p.sandboxID = sandbox.id
console, err := sandbox.hypervisor.getSandboxConsole(sandbox.id)
if err != nil {
return -1, "", err return -1, "", err
} }
err = p.watchConsole(consoleProtoUnix, console, params.logger) if p.consoleWatched() {
return -1, "", fmt.Errorf("kata builtin proxy running for sandbox %s", params.id)
}
params.logger.Info("Starting builtin kata proxy")
p.sandboxID = params.id
err := p.watchConsole(buildinProxyConsoleProto, params.consoleURL, params.logger)
if err != nil { if err != nil {
p.sandboxID = ""
return -1, "", err return -1, "", err
} }
@ -49,7 +65,7 @@ func (p *kataBuiltInProxy) start(sandbox *Sandbox, params proxyParams) (int, str
} }
// stop is the proxy stop implementation for kata builtin proxy. // stop is the proxy stop implementation for kata builtin proxy.
func (p *kataBuiltInProxy) stop(sandbox *Sandbox, pid int) error { func (p *kataBuiltInProxy) stop(pid int) error {
if p.conn != nil { if p.conn != nil {
p.conn.Close() p.conn.Close()
p.conn = nil p.conn = nil

View File

@ -0,0 +1,50 @@
// Copyright (c) 2018 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
package virtcontainers
import (
"testing"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
func TestKataBuiltinProxy(t *testing.T) {
assert := assert.New(t)
p := kataBuiltInProxy{}
params := proxyParams{}
err := p.validateParams(params)
assert.NotNil(err)
params.id = "foobarproxy"
err = p.validateParams(params)
assert.NotNil(err)
params.agentURL = "foobaragent"
err = p.validateParams(params)
assert.NotNil(err)
params.consoleURL = "foobarconsole"
err = p.validateParams(params)
assert.NotNil(err)
params.logger = logrus.WithField("proxy", params.id)
err = p.validateParams(params)
assert.Nil(err)
buildinProxyConsoleProto = "foobarproto"
_, _, err = p.start(params)
assert.NotNil(err)
assert.Empty(p.sandboxID)
err = p.stop(0)
assert.Nil(err)
assert.False(p.consoleWatched())
}

View File

@ -6,7 +6,6 @@
package virtcontainers package virtcontainers
import ( import (
"fmt"
"os/exec" "os/exec"
"syscall" "syscall"
) )
@ -23,42 +22,28 @@ func (p *kataProxy) consoleWatched() bool {
} }
// start is kataProxy start implementation for proxy interface. // start is kataProxy start implementation for proxy interface.
func (p *kataProxy) start(sandbox *Sandbox, params proxyParams) (int, string, error) { func (p *kataProxy) start(params proxyParams) (int, string, error) {
sandbox.Logger().Info("Starting regular Kata proxy rather than built-in") if err := validateProxyParams(params); err != nil {
if sandbox.agent == nil {
return -1, "", fmt.Errorf("No agent")
}
if params.agentURL == "" {
return -1, "", fmt.Errorf("AgentURL cannot be empty")
}
config, err := newProxyConfig(sandbox.config)
if err != nil {
return -1, "", err return -1, "", err
} }
params.logger.Info("Starting regular Kata proxy rather than built-in")
// construct the socket path the proxy instance will use // construct the socket path the proxy instance will use
proxyURL, err := defaultProxyURL(sandbox, SocketTypeUNIX) proxyURL, err := defaultProxyURL(params.id, SocketTypeUNIX)
if err != nil { if err != nil {
return -1, "", err return -1, "", err
} }
args := []string{ args := []string{
config.Path, params.path,
"-listen-socket", proxyURL, "-listen-socket", proxyURL,
"-mux-socket", params.agentURL, "-mux-socket", params.agentURL,
"-sandbox", sandbox.ID(), "-sandbox", params.id,
} }
if config.Debug { if params.debug {
args = append(args, "-log", "debug") args = append(args, "-log", "debug", "-agent-logs-socket", params.consoleURL)
console, err := sandbox.hypervisor.getSandboxConsole(sandbox.id)
if err != nil {
return -1, "", err
}
args = append(args, "-agent-logs-socket", console)
} }
cmd := exec.Command(args[0], args[1:]...) cmd := exec.Command(args[0], args[1:]...)
@ -70,7 +55,7 @@ func (p *kataProxy) start(sandbox *Sandbox, params proxyParams) (int, string, er
} }
// stop is kataProxy stop implementation for proxy interface. // stop is kataProxy stop implementation for proxy interface.
func (p *kataProxy) stop(sandbox *Sandbox, pid int) error { func (p *kataProxy) stop(pid int) error {
// Signal the proxy with SIGTERM. // Signal the proxy with SIGTERM.
return syscall.Kill(pid, syscall.SIGTERM) return syscall.Kill(pid, syscall.SIGTERM)
} }

View File

@ -13,5 +13,5 @@ func TestKataProxyStart(t *testing.T) {
agent := &kataAgent{} agent := &kataAgent{}
proxy := &kataProxy{} proxy := &kataProxy{}
testProxyStart(t, agent, proxy, KataProxyType) testProxyStart(t, agent, proxy)
} }

View File

@ -23,8 +23,13 @@ type noProxy struct {
} }
// start is noProxy start implementation for proxy interface. // start is noProxy start implementation for proxy interface.
func (p *noProxy) start(sandbox *Sandbox, params proxyParams) (int, string, error) { func (p *noProxy) start(params proxyParams) (int, string, error) {
sandbox.Logger().Info("No proxy started because of no-proxy implementation") if params.logger == nil {
return -1, "", fmt.Errorf("proxy logger is not set")
}
params.logger.Info("No proxy started because of no-proxy implementation")
if params.agentURL == "" { if params.agentURL == "" {
return -1, "", fmt.Errorf("AgentURL cannot be empty") return -1, "", fmt.Errorf("AgentURL cannot be empty")
} }
@ -33,7 +38,7 @@ func (p *noProxy) start(sandbox *Sandbox, params proxyParams) (int, string, erro
} }
// stop is noProxy stop implementation for proxy interface. // stop is noProxy stop implementation for proxy interface.
func (p *noProxy) stop(sandbox *Sandbox, pid int) error { func (p *noProxy) stop(pid int) error {
return nil return nil
} }

View File

@ -7,34 +7,30 @@ package virtcontainers
import ( import (
"testing" "testing"
"github.com/stretchr/testify/assert"
) )
func TestNoProxyStart(t *testing.T) { func TestNoProxyStart(t *testing.T) {
sandbox := &Sandbox{
agent: newAgent(NoopAgentType),
}
p := &noProxy{} p := &noProxy{}
assert := assert.New(t)
agentURL := "agentURL" agentURL := "agentURL"
pid, vmURL, err := p.start(sandbox, proxyParams{agentURL: agentURL}) _, _, err := p.start(proxyParams{
if err != nil { agentURL: agentURL,
t.Fatal(err) })
} assert.NotNil(err)
if vmURL != agentURL { pid, vmURL, err := p.start(proxyParams{
t.Fatalf("Got URL %q, expecting %q", vmURL, agentURL) agentURL: agentURL,
} logger: testDefaultLogger,
})
assert.Nil(err)
assert.Equal(vmURL, agentURL)
assert.Equal(pid, 0)
if pid != 0 { err = p.stop(0)
t.Fatal("Failure since returned PID should be 0") assert.Nil(err)
}
} assert.False(p.consoleWatched())
func TestNoProxyStop(t *testing.T) {
p := &noProxy{}
if err := p.stop(&Sandbox{}, 0); err != nil {
t.Fatal(err)
}
} }

View File

@ -187,3 +187,13 @@ func (n *noopAgent) getSharePath(id string) string {
func (n *noopAgent) reseedRNG(data []byte) error { func (n *noopAgent) reseedRNG(data []byte) error {
return nil return nil
} }
// getAgentURL is the Noop agent url getter. It returns nothing.
func (n *noopAgent) getAgentURL() (string, error) {
return "", nil
}
// setProxy is the Noop agent proxy setter. It does nothing.
func (n *noopAgent) setProxy(sandbox *Sandbox, proxy proxy, pid int, url string) error {
return nil
}

View File

@ -9,6 +9,8 @@ package virtcontainers
import ( import (
"context" "context"
"testing" "testing"
"github.com/stretchr/testify/assert"
) )
func testCreateNoopContainer() (*Sandbox, *Container, error) { func testCreateNoopContainer() (*Sandbox, *Container, error) {
@ -251,3 +253,22 @@ func TestNoopAgentListRoutes(t *testing.T) {
t.Fatal("listRoutes failed") t.Fatal("listRoutes failed")
} }
} }
func TestNoopAgentRSetProxy(t *testing.T) {
n := &noopAgent{}
p := &noopProxy{}
s := &Sandbox{}
err := n.setProxy(s, p, 0, "")
if err != nil {
t.Fatal("set proxy failed")
}
}
func TestNoopGetAgentUrl(t *testing.T) {
assert := assert.New(t)
n := &noopAgent{}
url, err := n.getAgentURL()
assert.Nil(err)
assert.Empty(url)
}

View File

@ -13,13 +13,13 @@ var noopProxyURL = "noopProxyURL"
// register is the proxy start implementation for testing purpose. // register is the proxy start implementation for testing purpose.
// It does nothing. // It does nothing.
func (p *noopProxy) start(sandbox *Sandbox, params proxyParams) (int, string, error) { func (p *noopProxy) start(params proxyParams) (int, string, error) {
return 0, noopProxyURL, nil return 0, noopProxyURL, nil
} }
// stop is the proxy stop implementation for testing purpose. // stop is the proxy stop implementation for testing purpose.
// It does nothing. // It does nothing.
func (p *noopProxy) stop(sandbox *Sandbox, pid int) error { func (p *noopProxy) stop(pid int) error {
return nil return nil
} }

View File

@ -0,0 +1,26 @@
// Copyright (c) 2018 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
package virtcontainers
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestNoopProxy(t *testing.T) {
n := &noopProxy{}
assert := assert.New(t)
_, url, err := n.start(proxyParams{})
assert.Nil(err)
assert.Equal(url, noopProxyURL)
err = n.stop(0)
assert.Nil(err)
assert.False(n.consoleWatched())
}

View File

@ -9,7 +9,6 @@ import (
"fmt" "fmt"
"path/filepath" "path/filepath"
"github.com/mitchellh/mapstructure"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
@ -23,8 +22,12 @@ type ProxyConfig struct {
// proxyParams is the structure providing specific parameters needed // proxyParams is the structure providing specific parameters needed
// for the execution of the proxy binary. // for the execution of the proxy binary.
type proxyParams struct { type proxyParams struct {
id string
path string
agentURL string agentURL string
consoleURL string
logger *logrus.Entry logger *logrus.Entry
debug bool
} }
// ProxyType describes a proxy type. // ProxyType describes a proxy type.
@ -120,34 +123,30 @@ func newProxy(pType ProxyType) (proxy, error) {
} }
} }
// newProxyConfig returns a proxy config from a generic SandboxConfig handler, func validateProxyParams(p proxyParams) error {
// after it properly checked the configuration was valid. if len(p.path) == 0 || len(p.id) == 0 || len(p.agentURL) == 0 || len(p.consoleURL) == 0 {
func newProxyConfig(sandboxConfig *SandboxConfig) (ProxyConfig, error) { return fmt.Errorf("Invalid proxy parameters %+v", p)
if sandboxConfig == nil {
return ProxyConfig{}, fmt.Errorf("Sandbox config cannot be nil")
} }
var config ProxyConfig if p.logger == nil {
switch sandboxConfig.ProxyType { return fmt.Errorf("Invalid proxy parameter: proxy logger is not set")
case KataProxyType:
fallthrough
case CCProxyType:
if err := mapstructure.Decode(sandboxConfig.ProxyConfig, &config); err != nil {
return ProxyConfig{}, err
}
} }
if config.Path == "" { return nil
return ProxyConfig{}, fmt.Errorf("Proxy path cannot be empty")
}
return config, nil
} }
func defaultProxyURL(sandbox *Sandbox, socketType string) (string, error) { func validateProxyConfig(proxyConfig ProxyConfig) error {
if len(proxyConfig.Path) == 0 {
return fmt.Errorf("Proxy path cannot be empty")
}
return nil
}
func defaultProxyURL(id, socketType string) (string, error) {
switch socketType { switch socketType {
case SocketTypeUNIX: case SocketTypeUNIX:
socketPath := filepath.Join(runStoragePath, sandbox.id, "proxy.sock") socketPath := filepath.Join(runStoragePath, id, "proxy.sock")
return fmt.Sprintf("unix://%s", socketPath), nil return fmt.Sprintf("unix://%s", socketPath), nil
case SocketTypeVSOCK: case SocketTypeVSOCK:
// TODO Build the VSOCK default URL // TODO Build the VSOCK default URL
@ -163,13 +162,13 @@ func isProxyBuiltIn(pType ProxyType) bool {
// proxy is the virtcontainers proxy interface. // proxy is the virtcontainers proxy interface.
type proxy interface { type proxy interface {
// start launches a proxy instance for the specified sandbox, returning // start launches a proxy instance with specified parameters, returning
// the PID of the process and the URL used to connect to it. // the PID of the process and the URL used to connect to it.
start(sandbox *Sandbox, params proxyParams) (int, string, error) start(params proxyParams) (int, string, error)
// stop terminates a proxy instance after all communications with the // stop terminates a proxy instance after all communications with the
// agent inside the VM have been properly stopped. // agent inside the VM have been properly stopped.
stop(sandbox *Sandbox, pid int) error stop(pid int) error
//check if the proxy has watched the vm console. //check if the proxy has watched the vm console.
consoleWatched() bool consoleWatched() bool

View File

@ -13,9 +13,12 @@ import (
"reflect" "reflect"
"testing" "testing"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
var testDefaultLogger = logrus.WithField("proxy", "test")
func testSetProxyType(t *testing.T, value string, expected ProxyType) { func testSetProxyType(t *testing.T, value string, expected ProxyType) {
var proxyType ProxyType var proxyType ProxyType
@ -154,15 +157,15 @@ func TestNewProxyFromUnknownProxyType(t *testing.T) {
} }
} }
func testNewProxyConfigFromSandboxConfig(t *testing.T, sandboxConfig SandboxConfig, expected ProxyConfig) { func testNewProxyFromSandboxConfig(t *testing.T, sandboxConfig SandboxConfig) {
result, err := newProxyConfig(&sandboxConfig) if _, err := newProxy(sandboxConfig.ProxyType); err != nil {
if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if reflect.DeepEqual(result, expected) == false { if err := validateProxyConfig(sandboxConfig.ProxyConfig); err != nil {
t.Fatalf("Got %+v\nExpecting %+v", result, expected) t.Fatal(err)
} }
} }
var testProxyPath = "proxy-path" var testProxyPath = "proxy-path"
@ -177,7 +180,7 @@ func TestNewProxyConfigFromCCProxySandboxConfig(t *testing.T) {
ProxyConfig: proxyConfig, ProxyConfig: proxyConfig,
} }
testNewProxyConfigFromSandboxConfig(t, sandboxConfig, proxyConfig) testNewProxyFromSandboxConfig(t, sandboxConfig)
} }
func TestNewProxyConfigFromKataProxySandboxConfig(t *testing.T) { func TestNewProxyConfigFromKataProxySandboxConfig(t *testing.T) {
@ -190,22 +193,11 @@ func TestNewProxyConfigFromKataProxySandboxConfig(t *testing.T) {
ProxyConfig: proxyConfig, ProxyConfig: proxyConfig,
} }
testNewProxyConfigFromSandboxConfig(t, sandboxConfig, proxyConfig) testNewProxyFromSandboxConfig(t, sandboxConfig)
}
func TestNewProxyConfigNilSandboxConfigFailure(t *testing.T) {
if _, err := newProxyConfig(nil); err == nil {
t.Fatal("Should fail because SandboxConfig provided is nil")
}
} }
func TestNewProxyConfigNoPathFailure(t *testing.T) { func TestNewProxyConfigNoPathFailure(t *testing.T) {
sandboxConfig := &SandboxConfig{ if err := validateProxyConfig(ProxyConfig{}); err == nil {
ProxyType: CCProxyType,
ProxyConfig: ProxyConfig{},
}
if _, err := newProxyConfig(sandboxConfig); err == nil {
t.Fatal("Should fail because ProxyConfig has no Path") t.Fatal("Should fail because ProxyConfig has no Path")
} }
} }
@ -217,7 +209,7 @@ func testDefaultProxyURL(expectedURL string, socketType string, sandboxID string
id: sandboxID, id: sandboxID,
} }
url, err := defaultProxyURL(sandbox, socketType) url, err := defaultProxyURL(sandbox.id, socketType)
if err != nil { if err != nil {
return err return err
} }
@ -253,7 +245,7 @@ func TestDefaultProxyURLUnknown(t *testing.T) {
} }
} }
func testProxyStart(t *testing.T, agent agent, proxy proxy, proxyType ProxyType) { func testProxyStart(t *testing.T, agent agent, proxy proxy) {
assert := assert.New(t) assert := assert.New(t)
assert.NotNil(proxy) assert.NotNil(proxy)
@ -263,7 +255,6 @@ func testProxyStart(t *testing.T, agent agent, proxy proxy, proxyType ProxyType)
defer os.RemoveAll(tmpdir) defer os.RemoveAll(tmpdir)
type testData struct { type testData struct {
sandbox *Sandbox
params proxyParams params proxyParams
expectedURI string expectedURI string
expectError bool expectError bool
@ -274,60 +265,43 @@ func testProxyStart(t *testing.T, agent agent, proxy proxy, proxyType ProxyType)
expectedURI := fmt.Sprintf("unix://%s", expectedSocketPath) expectedURI := fmt.Sprintf("unix://%s", expectedSocketPath)
data := []testData{ data := []testData{
{&Sandbox{}, proxyParams{}, "", true}, {proxyParams{}, "", true},
{ {
&Sandbox{ // no path
config: &SandboxConfig{
ProxyType: "invalid",
},
},
proxyParams{},
"", true,
},
{
&Sandbox{
config: &SandboxConfig{
ProxyType: proxyType,
// invalid - no path
ProxyConfig: ProxyConfig{},
},
},
proxyParams{},
"", true,
},
{
&Sandbox{
config: &SandboxConfig{
ProxyType: proxyType,
ProxyConfig: ProxyConfig{
Path: invalidPath,
},
},
},
proxyParams{},
"", true,
},
{
&Sandbox{
id: testSandboxID,
agent: agent,
config: &SandboxConfig{
ProxyType: proxyType,
ProxyConfig: ProxyConfig{
Path: "echo",
},
},
},
proxyParams{ proxyParams{
id: "foobar",
agentURL: "agentURL", agentURL: "agentURL",
consoleURL: "consoleURL",
logger: testDefaultLogger,
},
"", true,
},
{
// invalid path
proxyParams{
id: "foobar",
path: invalidPath,
agentURL: "agentURL",
consoleURL: "consoleURL",
logger: testDefaultLogger,
},
"", true,
},
{
// good case
proxyParams{
id: testSandboxID,
path: "echo",
agentURL: "agentURL",
consoleURL: "consoleURL",
logger: testDefaultLogger,
}, },
expectedURI, false, expectedURI, false,
}, },
} }
for _, d := range data { for _, d := range data {
pid, uri, err := proxy.start(d.sandbox, d.params) pid, uri, err := proxy.start(d.params)
if d.expectError { if d.expectError {
assert.Error(err) assert.Error(err)
continue continue
@ -338,3 +312,43 @@ func testProxyStart(t *testing.T, agent agent, proxy proxy, proxyType ProxyType)
assert.Equal(d.expectedURI, uri) assert.Equal(d.expectedURI, uri)
} }
} }
func TestValidateProxyConfig(t *testing.T) {
assert := assert.New(t)
config := ProxyConfig{}
err := validateProxyConfig(config)
assert.Error(err)
config.Path = "foobar"
err = validateProxyConfig(config)
assert.Nil(err)
}
func TestValidateProxyParams(t *testing.T) {
assert := assert.New(t)
p := proxyParams{}
err := validateProxyParams(p)
assert.Error(err)
p.path = "foobar"
err = validateProxyParams(p)
assert.Error(err)
p.id = "foobar1"
err = validateProxyParams(p)
assert.Error(err)
p.agentURL = "foobar2"
err = validateProxyParams(p)
assert.Error(err)
p.consoleURL = "foobar3"
err = validateProxyParams(p)
assert.Error(err)
p.logger = &logrus.Entry{}
err = validateProxyParams(p)
assert.Nil(err)
}

View File

@ -1151,6 +1151,8 @@ func (s *Sandbox) startVM() error {
HypervisorConfig: s.config.HypervisorConfig, HypervisorConfig: s.config.HypervisorConfig,
AgentType: s.config.AgentType, AgentType: s.config.AgentType,
AgentConfig: s.config.AgentConfig, AgentConfig: s.config.AgentConfig,
ProxyType: s.config.ProxyType,
ProxyConfig: s.config.ProxyConfig,
}) })
if err != nil { if err != nil {
return err return err
@ -1640,9 +1642,9 @@ func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType)
if _, err := s.hypervisor.hotplugAddDevice(dev, vfioDev); err != nil { if _, err := s.hypervisor.hotplugAddDevice(dev, vfioDev); err != nil {
s.Logger(). s.Logger().
WithFields(logrus.Fields{ WithFields(logrus.Fields{
"sandboxid": s.id, "sandbox": s.id,
"vfio device ID": dev.ID, "vfio-device-ID": dev.ID,
"vfio device BDF": dev.BDF, "vfio-device-BDF": dev.BDF,
}).WithError(err).Error("failed to hotplug VFIO device") }).WithError(err).Error("failed to hotplug VFIO device")
return err return err
} }
@ -1677,9 +1679,9 @@ func (s *Sandbox) HotplugRemoveDevice(device api.Device, devType config.DeviceTy
if _, err := s.hypervisor.hotplugRemoveDevice(dev, vfioDev); err != nil { if _, err := s.hypervisor.hotplugRemoveDevice(dev, vfioDev); err != nil {
s.Logger().WithError(err). s.Logger().WithError(err).
WithFields(logrus.Fields{ WithFields(logrus.Fields{
"sandboxid": s.id, "sandbox": s.id,
"vfio device ID": dev.ID, "vfio-device-ID": dev.ID,
"vfio device BDF": dev.BDF, "vfio-device-BDF": dev.BDF,
}).Error("failed to hot unplug VFIO device") }).Error("failed to hot unplug VFIO device")
return err return err
} }

View File

@ -21,6 +21,10 @@ type VM struct {
hypervisor hypervisor hypervisor hypervisor
agent agent agent agent
proxy proxy
proxyPid int
proxyURL string
cpu uint32 cpu uint32
memory uint32 memory uint32
@ -34,6 +38,9 @@ type VMConfig struct {
AgentType AgentType AgentType AgentType
AgentConfig interface{} AgentConfig interface{}
ProxyType ProxyType
ProxyConfig ProxyConfig
} }
// Valid check VMConfig validity. // Valid check VMConfig validity.
@ -41,8 +48,56 @@ func (c *VMConfig) Valid() error {
return c.HypervisorConfig.valid() return c.HypervisorConfig.valid()
} }
func setupProxy(h hypervisor, agent agent, config VMConfig, id string) (int, string, proxy, error) {
consoleURL, err := h.getSandboxConsole(id)
if err != nil {
return -1, "", nil, err
}
agentURL, err := agent.getAgentURL()
if err != nil {
return -1, "", nil, err
}
// default to kata builtin proxy
proxyType := config.ProxyType
if len(proxyType.String()) == 0 {
proxyType = KataBuiltInProxyType
}
proxy, err := newProxy(proxyType)
if err != nil {
return -1, "", nil, err
}
proxyParams := proxyParams{
id: id,
path: config.ProxyConfig.Path,
agentURL: agentURL,
consoleURL: consoleURL,
logger: virtLog.WithField("vm", id),
debug: config.ProxyConfig.Debug,
}
pid, url, err := proxy.start(proxyParams)
if err != nil {
virtLog.WithFields(logrus.Fields{
"vm": id,
"proxy type": config.ProxyType,
"params": proxyParams,
}).WithError(err).Error("failed to start proxy")
return -1, "", nil, err
}
return pid, url, proxy, nil
}
// NewVM creates a new VM based on provided VMConfig. // NewVM creates a new VM based on provided VMConfig.
func NewVM(ctx context.Context, config VMConfig) (*VM, error) { func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
var (
proxy proxy
pid int
url string
)
// 1. setup hypervisor
hypervisor, err := newHypervisor(config.HypervisorType) hypervisor, err := newHypervisor(config.HypervisorType)
if err != nil { if err != nil {
return nil, err return nil, err
@ -54,11 +109,11 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
id := uuid.Generate().String() id := uuid.Generate().String()
virtLog.WithField("vm id", id).WithField("config", config).Info("create new vm") virtLog.WithField("vm", id).WithField("config", config).Info("create new vm")
defer func() { defer func() {
if err != nil { if err != nil {
virtLog.WithField("vm id", id).WithError(err).Error("failed to create new vm") virtLog.WithField("vm", id).WithError(err).Error("failed to create new vm")
} }
}() }()
@ -70,37 +125,48 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
return nil, err return nil, err
} }
// 2. setup agent
agent := newAgent(config.AgentType) agent := newAgent(config.AgentType)
agentConfig := newAgentConfig(config.AgentType, config.AgentConfig)
// do not keep connection for temp agent
if c, ok := agentConfig.(KataAgentConfig); ok {
c.LongLiveConn = false
}
vmSharePath := buildVMSharePath(id) vmSharePath := buildVMSharePath(id)
err = agent.configure(hypervisor, id, vmSharePath, true, agentConfig) err = agent.configure(hypervisor, id, vmSharePath, isProxyBuiltIn(config.ProxyType), config.AgentConfig)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// 3. boot up guest vm
if err = hypervisor.startSandbox(); err != nil { if err = hypervisor.startSandbox(); err != nil {
return nil, err return nil, err
} }
if err = hypervisor.waitSandbox(vmStartTimeout); err != nil {
return nil, err
}
defer func() { defer func() {
if err != nil { if err != nil {
virtLog.WithField("vm id", id).WithError(err).Info("clean up vm") virtLog.WithField("vm", id).WithError(err).Info("clean up vm")
hypervisor.stopSandbox() hypervisor.stopSandbox()
} }
}() }()
// VMs booted from template are paused, do not check // 4. setup proxy
if !config.HypervisorConfig.BootFromTemplate { pid, url, proxy, err = setupProxy(hypervisor, agent, config, id)
err = hypervisor.waitSandbox(vmStartTimeout)
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer func() {
if err != nil {
virtLog.WithField("vm", id).WithError(err).Info("clean up proxy")
proxy.stop(pid)
}
}()
if err = agent.setProxy(nil, proxy, pid, url); err != nil {
return nil, err
}
virtLog.WithField("vm id", id).Info("check agent status") // 5. check agent aliveness
// VMs booted from template are paused, do not check
if !config.HypervisorConfig.BootFromTemplate {
virtLog.WithField("vm", id).Info("check agent status")
err = agent.check() err = agent.check()
if err != nil { if err != nil {
return nil, err return nil, err
@ -111,6 +177,9 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
id: id, id: id,
hypervisor: hypervisor, hypervisor: hypervisor,
agent: agent, agent: agent,
proxy: proxy,
proxyPid: pid,
proxyURL: url,
cpu: config.HypervisorConfig.NumVCPUs, cpu: config.HypervisorConfig.NumVCPUs,
memory: config.HypervisorConfig.MemorySize, memory: config.HypervisorConfig.MemorySize,
}, nil }, nil
@ -121,7 +190,7 @@ func buildVMSharePath(id string) string {
} }
func (v *VM) logger() logrus.FieldLogger { func (v *VM) logger() logrus.FieldLogger {
return virtLog.WithField("vm id", v.id) return virtLog.WithField("vm", v.id)
} }
// Pause pauses a VM. // Pause pauses a VM.
@ -148,9 +217,24 @@ func (v *VM) Start() error {
return v.hypervisor.startSandbox() return v.hypervisor.startSandbox()
} }
// Disconnect agent and proxy connections to a VM
func (v *VM) Disconnect() error {
v.logger().Info("kill vm")
if err := v.agent.disconnect(); err != nil {
v.logger().WithError(err).Error("failed to disconnect agent")
}
if err := v.proxy.stop(v.proxyPid); err != nil {
v.logger().WithError(err).Error("failed to stop proxy")
}
return nil
}
// Stop stops a VM process. // Stop stops a VM process.
func (v *VM) Stop() error { func (v *VM) Stop() error {
v.logger().Info("kill vm") v.logger().Info("kill vm")
return v.hypervisor.stopSandbox() return v.hypervisor.stopSandbox()
} }
@ -227,8 +311,14 @@ func (v *VM) assignSandbox(s *Sandbox) error {
"vmSockDir": vmSockDir, "vmSockDir": vmSockDir,
"sbSharePath": sbSharePath, "sbSharePath": sbSharePath,
"sbSockDir": sbSockDir, "sbSockDir": sbSockDir,
"proxy-pid": v.proxyPid,
"proxy-url": v.proxyURL,
}).Infof("assign vm to sandbox %s", s.id) }).Infof("assign vm to sandbox %s", s.id)
if err := s.agent.setProxy(s, v.proxy, v.proxyPid, v.proxyURL); err != nil {
return err
}
// First make sure the symlinks do not exist // First make sure the symlinks do not exist
os.RemoveAll(sbSharePath) os.RemoveAll(sbSharePath)
os.RemoveAll(sbSockDir) os.RemoveAll(sbSockDir)

View File

@ -20,6 +20,7 @@ func TestNewVM(t *testing.T) {
config := VMConfig{ config := VMConfig{
HypervisorType: MockHypervisor, HypervisorType: MockHypervisor,
AgentType: NoopAgentType, AgentType: NoopAgentType,
ProxyType: NoopProxyType,
} }
hyperConfig := HypervisorConfig{ hyperConfig := HypervisorConfig{
KernelPath: testDir, KernelPath: testDir,
@ -43,6 +44,8 @@ func TestNewVM(t *testing.T) {
assert.Nil(err) assert.Nil(err)
err = vm.Start() err = vm.Start()
assert.Nil(err) assert.Nil(err)
err = vm.Disconnect()
assert.Nil(err)
err = vm.Save() err = vm.Save()
assert.Nil(err) assert.Nil(err)
err = vm.Stop() err = vm.Stop()
@ -86,3 +89,24 @@ func TestVMConfigValid(t *testing.T) {
err = config.Valid() err = config.Valid()
assert.Nil(err) assert.Nil(err)
} }
func TestSetupProxy(t *testing.T) {
assert := assert.New(t)
config := VMConfig{
HypervisorType: MockHypervisor,
AgentType: NoopAgentType,
}
hypervisor := &mockHypervisor{}
agent := &noopAgent{}
// wrong proxy type
config.ProxyType = "invalidProxyType"
_, _, _, err := setupProxy(hypervisor, agent, config, "foobar")
assert.NotNil(err)
config.ProxyType = NoopProxyType
_, _, _, err = setupProxy(hypervisor, agent, config, "foobar")
assert.Nil(err)
}