qemu: prepare for vm templating support

1. support qemu migration save operation
2. setup vm templating parameters per hypervisor config
3. create vm storage path when it does not exist. This can happen when
an empty guest is created without a sandbox.

Signed-off-by: Peng Tao <bergwolf@gmail.com>

virtcontainers/hypervisor.go

@@ -216,6 +216,38 @@ type HypervisorConfig struct {
 
 	// Msize9p is used as the msize for 9p shares
 	Msize9p uint32
+
+	// BootToBeTemplate used to indicate if the VM is created to be a template VM
+	BootToBeTemplate bool
+
+	// BootFromTemplate used to indicate if the VM should be created from a template VM
+	BootFromTemplate bool
+
+	// MemoryPath is the memory file path of VM memory. Used when either BootToBeTemplate or
+	// BootFromTemplate is true.
+	MemoryPath string
+
+	// DevicesStatePath is the VM device state file path. Used when either BootToBeTemplate or
+	// BootFromTemplate is true.
+	DevicesStatePath string
+}
+
+func (conf *HypervisorConfig) checkTemplateConfig() error {
+	if conf.BootToBeTemplate && conf.BootFromTemplate {
+		return fmt.Errorf("Cannot set both 'to be' and 'from' vm tempate")
+	}
+
+	if conf.BootToBeTemplate || conf.BootFromTemplate {
+		if conf.MemoryPath == "" {
+			return fmt.Errorf("Missing MemoryPath for vm template")
+		}
+
+		if conf.BootFromTemplate && conf.DevicesStatePath == "" {
+			return fmt.Errorf("Missing DevicesStatePath to load from vm template")
+		}
+	}
+
+	return nil
 }
 
 func (conf *HypervisorConfig) valid() error {
@@ -227,6 +259,10 @@ func (conf *HypervisorConfig) valid() error {
 		return fmt.Errorf("Missing image and initrd path")
 	}
 
+	if err := conf.checkTemplateConfig(); err != nil {
+		return err
+	}
+
 	if conf.DefaultVCPUs == 0 {
 		conf.DefaultVCPUs = defaultVCPUs
 	}
@@ -505,6 +541,7 @@ type hypervisor interface {
 	waitSandbox(timeout int) error
 	stopSandbox() error
 	pauseSandbox() error
+	saveSandbox() error
 	resumeSandbox() error
 	addDevice(devInfo interface{}, devType deviceType) error
 	hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error)

virtcontainers/hypervisor_test.go

@@ -157,6 +157,30 @@ func TestHypervisorConfigIsValid(t *testing.T) {
 	testHypervisorConfigValid(t, hypervisorConfig, true)
 }
 
+func TestHypervisorConfigValidTemplateConfig(t *testing.T) {
+	hypervisorConfig := &HypervisorConfig{
+		KernelPath:       fmt.Sprintf("%s/%s", testDir, testKernel),
+		ImagePath:        fmt.Sprintf("%s/%s", testDir, testImage),
+		HypervisorPath:   fmt.Sprintf("%s/%s", testDir, testHypervisor),
+		BootToBeTemplate: true,
+		BootFromTemplate: true,
+	}
+	testHypervisorConfigValid(t, hypervisorConfig, false)
+
+	hypervisorConfig.BootToBeTemplate = false
+	testHypervisorConfigValid(t, hypervisorConfig, false)
+	hypervisorConfig.MemoryPath = "foobar"
+	testHypervisorConfigValid(t, hypervisorConfig, false)
+	hypervisorConfig.DevicesStatePath = "foobar"
+	testHypervisorConfigValid(t, hypervisorConfig, true)
+
+	hypervisorConfig.BootFromTemplate = false
+	hypervisorConfig.BootToBeTemplate = true
+	testHypervisorConfigValid(t, hypervisorConfig, true)
+	hypervisorConfig.MemoryPath = ""
+	testHypervisorConfigValid(t, hypervisorConfig, false)
+}
+
 func TestHypervisorConfigDefaults(t *testing.T) {
 	hypervisorConfig := &HypervisorConfig{
 		KernelPath:     fmt.Sprintf("%s/%s", testDir, testKernel),

virtcontainers/mock_hypervisor.go

@@ -46,6 +46,10 @@ func (m *mockHypervisor) resumeSandbox() error {
 	return nil
 }
 
+func (m *mockHypervisor) saveSandbox() error {
+	return nil
+}
+
 func (m *mockHypervisor) addDevice(devInfo interface{}, devType deviceType) error {
 	return nil
 }

virtcontainers/mock_hypervisor_test.go

@@ -96,3 +96,11 @@ func TestMockHypervisorGetSandboxConsole(t *testing.T) {
 		t.Fatalf("Got %s\nExpecting %s", result, expected)
 	}
 }
+
+func TestMockHypervisorSaveSandbox(t *testing.T) {
+	var m *mockHypervisor
+
+	if err := m.saveSandbox(); err != nil {
+		t.Fatal(err)
+	}
+}

virtcontainers/qemu.go

@@ -64,11 +64,16 @@ type qemu struct {
 	arch qemuArch
 }
 
-const qmpCapErrMsg = "Failed to negoatiate QMP capabilities"
+const (
+	consoleSocket = "console.sock"
+	qmpSocket     = "qmp.sock"
 
-const qmpSocket = "qmp.sock"
+	qmpCapErrMsg                      = "Failed to negoatiate QMP capabilities"
+	qmpCapMigrationBypassSharedMemory = "bypass-shared-memory"
+	qmpExecCatCmd                     = "exec:cat"
 
-const consoleSocket = "console.sock"
+	scsiControllerID = "scsi0"
+)
 
 var qemuMajorVersion int
 var qemuMinorVersion int
@@ -86,10 +91,6 @@ const (
 	removeDevice
 )
 
-const (
-	scsiControllerID = "scsi0"
-)
-
 type qmpLogger struct {
 	logger *logrus.Entry
 }
@@ -191,6 +192,12 @@ func (q *qemu) init(id string, hypervisorConfig *HypervisorConfig, vmConfig Reso
 		q.Logger().Debug("Creating UUID")
 		q.state.UUID = uuid.Generate().String()
 
+		// The path might already exist, but in case of VM templating,
+		// we have to create it since the sandbox has not created it yet.
+		if err = os.MkdirAll(filepath.Join(runStoragePath, id), dirMode); err != nil {
+			return err
+		}
+
 		if err = q.storage.storeHypervisorState(q.id, q.state); err != nil {
 			return err
 		}
@@ -242,7 +249,7 @@ func (q *qemu) memoryTopology() (govmmQemu.Memory, error) {
 }
 
 func (q *qemu) qmpSocketPath(id string) (string, error) {
-	return utils.BuildSocketPath(runStoragePath, id, qmpSocket)
+	return utils.BuildSocketPath(RunVMStoragePath, id, qmpSocket)
 }
 
 func (q *qemu) getQemuMachine() (govmmQemu.Machine, error) {
@@ -334,6 +341,26 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
 
 }
 
+func (q *qemu) setupTemplate(knobs *govmmQemu.Knobs, memory *govmmQemu.Memory) govmmQemu.Incoming {
+	incoming := govmmQemu.Incoming{}
+
+	if q.config.BootToBeTemplate || q.config.BootFromTemplate {
+		knobs.FileBackedMem = true
+		memory.Path = q.config.MemoryPath
+
+		if q.config.BootToBeTemplate {
+			knobs.FileBackedMemShared = true
+		}
+
+		if q.config.BootFromTemplate {
+			incoming.MigrationType = govmmQemu.MigrationExec
+			incoming.Exec = "cat " + q.config.DevicesStatePath
+		}
+	}
+
+	return incoming
+}
+
 // createSandbox is the Hypervisor sandbox creation implementation for govmmQemu.
 func (q *qemu) createSandbox() error {
 	machine, err := q.getQemuMachine()
@@ -379,6 +406,8 @@ func (q *qemu) createSandbox() error {
 		Params:     params,
 	}
 
+	incoming := q.setupTemplate(&knobs, &memory)
+
 	rtc := govmmQemu.RTC{
 		Base:     "utc",
 		DriftFix: "slew",
@@ -439,6 +468,7 @@ func (q *qemu) createSandbox() error {
 		RTC:         rtc,
 		QMPSockets:  qmpSockets,
 		Knobs:       knobs,
+		Incoming:    incoming,
 		VGA:         "none",
 		GlobalParam: "kvm-pit.lost_tick_policy=discard",
 		Bios:        firmwarePath,
@@ -545,7 +575,18 @@ func (q *qemu) stopSandbox() error {
 		return err
 	}
 
-	return qmp.ExecuteQuit(q.qmpMonitorCh.ctx)
+	err = qmp.ExecuteQuit(q.qmpMonitorCh.ctx)
+	if err != nil {
+		q.Logger().WithError(err).Error("Fail to execute qmp QUIT")
+		return err
+	}
+
+	err = os.RemoveAll(RunVMStoragePath + q.id)
+	if err != nil {
+		q.Logger().WithError(err).Error("Fail to clean up vm directory")
+	}
+
+	return nil
 }
 
 func (q *qemu) togglePauseSandbox(pause bool) error {
@@ -987,7 +1028,59 @@ func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error {
 // getSandboxConsole builds the path of the console where we can read
 // logs coming from the sandbox.
 func (q *qemu) getSandboxConsole(id string) (string, error) {
-	return utils.BuildSocketPath(runStoragePath, id, consoleSocket)
+	return utils.BuildSocketPath(RunVMStoragePath, id, consoleSocket)
+}
+
+func (q *qemu) saveSandbox() error {
+	defer func(qemu *qemu) {
+		if q.qmpMonitorCh.qmp != nil {
+			q.qmpMonitorCh.qmp.Shutdown()
+		}
+	}(q)
+
+	q.Logger().Info("save sandbox")
+
+	cfg := govmmQemu.QMPConfig{Logger: newQMPLogger()}
+
+	// Auto-closed by QMPStart().
+	disconnectCh := make(chan struct{})
+
+	qmp, _, err := govmmQemu.QMPStart(q.qmpMonitorCh.ctx, q.qmpMonitorCh.path, cfg, disconnectCh)
+	if err != nil {
+		q.Logger().WithError(err).Error("Failed to connect to QEMU instance")
+		return err
+	}
+
+	q.qmpMonitorCh.qmp = qmp
+
+	err = qmp.ExecuteQMPCapabilities(q.qmpMonitorCh.ctx)
+	if err != nil {
+		q.Logger().WithError(err).Error(qmpCapErrMsg)
+		return err
+	}
+
+	// BootToBeTemplate sets the VM to be a template that other VMs can clone from. We would want to
+	// bypass shared memory when saving the VM to a local file through migration exec.
+	if q.config.BootToBeTemplate {
+		err = q.qmpMonitorCh.qmp.ExecSetMigrationCaps(q.qmpMonitorCh.ctx, []map[string]interface{}{
+			{
+				"capability": qmpCapMigrationBypassSharedMemory,
+				"state":      true,
+			},
+		})
+		if err != nil {
+			q.Logger().WithError(err).Error("set migration bypass shared memory")
+			return err
+		}
+	}
+
+	err = q.qmpMonitorCh.qmp.ExecSetMigrateArguments(q.qmpMonitorCh.ctx, fmt.Sprintf("%s>%s", qmpExecCatCmd, q.config.DevicesStatePath))
+	if err != nil {
+		q.Logger().WithError(err).Error("exec migration")
+		return err
+	}
+
+	return nil
 }
 
 // genericAppendBridges appends to devices the given bridges

virtcontainers/qemu_test.go

@@ -117,8 +117,8 @@ func TestQemuInitMissingParentDirFail(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err := q.init(sandbox.id, &sandbox.config.HypervisorConfig, sandbox.config.VMConfig, sandbox.storage); err == nil {
-		t.Fatal("Qemu init() expected to fail because of missing parent directory for storage")
+	if err := q.init(sandbox.id, &sandbox.config.HypervisorConfig, sandbox.config.VMConfig, sandbox.storage); err != nil {
+		t.Fatalf("Qemu init() is not expected to fail because of missing parent directory for storage: %v", err)
 	}
 }
 
@@ -249,7 +249,7 @@ func TestQemuAddDeviceSerialPortDev(t *testing.T) {
 func TestQemuGetSandboxConsole(t *testing.T) {
 	q := &qemu{}
 	sandboxID := "testSandboxID"
-	expected := filepath.Join(runStoragePath, sandboxID, consoleSocket)
+	expected := filepath.Join(RunVMStoragePath, sandboxID, consoleSocket)
 
 	result, err := q.getSandboxConsole(sandboxID)
 	if err != nil {