Merge pull request #18 from bergwolf/templating

Add APIs to enable vm templating

qemu/qemu.go

@@ -1126,6 +1126,10 @@ type Memory struct {
 	// MaxMem is the maximum amount of memory that can be made available
 	// to the guest through e.g. hot pluggable memory.
 	MaxMem string
+
+	// Path is the file path of the memory device. It points to a local
+	// file path used by FileBackedMem.
+	Path string
 }
 
 // Kernel is the guest kernel configuration structure.
@@ -1167,10 +1171,20 @@ type Knobs struct {
 	// MemPrealloc will allocate all the RAM upfront
 	MemPrealloc bool
 
+	// FileBackedMem requires Memory.Size and Memory.Path of the VM to
+	// be set.
+	FileBackedMem bool
+
+	// FileBackedMemShared will set the FileBackedMem device as shared.
+	FileBackedMemShared bool
+
 	// Mlock will control locking of memory
 	// Only active when Realtime is set to true
 	Mlock bool
 
+	// Stopped will not start guest CPU at startup
+	Stopped bool
+
 	// Realtime will enable realtime QEMU
 	Realtime bool
 }
@@ -1180,6 +1194,24 @@ type IOThread struct {
 	ID string
 }
 
+const (
+	// MigrationFD is the migration incoming type based on open file descriptor.
+	// Skip default 0 so that it must be set on purpose.
+	MigrationFD = 1
+	// MigrationExec is the migration incoming type based on commands.
+	MigrationExec = 2
+)
+
+// Incoming controls migration source preparation
+type Incoming struct {
+	// Possible values are MigrationFD, MigrationExec
+	MigrationType int
+	// Only valid if MigrationType == MigrationFD
+	FD *os.File
+	// Only valid if MigrationType == MigrationExec
+	Exec string
+}
+
 // Config is the qemu configuration structure.
 // It allows for passing custom settings and parameters to the qemu API.
 type Config struct {
@@ -1231,6 +1263,9 @@ type Config struct {
 	// Bios is the -bios parameter
 	Bios string
 
+	// Incoming controls migration source preparation
+	Incoming Incoming
+
 	// fds is a list of open file descriptors to be passed to the spawned qemu process
 	fds []*os.File
 
@@ -1433,23 +1468,7 @@ func (config *Config) appendKernel() {
 	}
 }
 
-func (config *Config) appendKnobs() {
+func (config *Config) appendMemoryKnobs() {
-	if config.Knobs.NoUserConfig == true {
-		config.qemuParams = append(config.qemuParams, "-no-user-config")
-	}
-
-	if config.Knobs.NoDefaults == true {
-		config.qemuParams = append(config.qemuParams, "-nodefaults")
-	}
-
-	if config.Knobs.NoGraphic == true {
-		config.qemuParams = append(config.qemuParams, "-nographic")
-	}
-
-	if config.Knobs.Daemonize == true {
-		config.qemuParams = append(config.qemuParams, "-daemonize")
-	}
-
 	if config.Knobs.HugePages == true {
 		if config.Memory.Size != "" {
 			dimmName := "dimm1"
@@ -1474,7 +1493,42 @@ func (config *Config) appendKnobs() {
 			config.qemuParams = append(config.qemuParams, "-device")
 			config.qemuParams = append(config.qemuParams, deviceMemParam)
 		}
+	} else if config.Knobs.FileBackedMem == true {
+		if config.Memory.Size != "" && config.Memory.Path != "" {
+			dimmName := "dimm1"
+			objMemParam := "memory-backend-file,id=" + dimmName + ",size=" + config.Memory.Size + ",mem-path=" + config.Memory.Path
+			if config.Knobs.FileBackedMemShared == true {
+				objMemParam += ",share=on"
 			}
+			numaMemParam := "node,memdev=" + dimmName
+
+			config.qemuParams = append(config.qemuParams, "-object")
+			config.qemuParams = append(config.qemuParams, objMemParam)
+
+			config.qemuParams = append(config.qemuParams, "-numa")
+			config.qemuParams = append(config.qemuParams, numaMemParam)
+		}
+	}
+}
+
+func (config *Config) appendKnobs() {
+	if config.Knobs.NoUserConfig == true {
+		config.qemuParams = append(config.qemuParams, "-no-user-config")
+	}
+
+	if config.Knobs.NoDefaults == true {
+		config.qemuParams = append(config.qemuParams, "-nodefaults")
+	}
+
+	if config.Knobs.NoGraphic == true {
+		config.qemuParams = append(config.qemuParams, "-nographic")
+	}
+
+	if config.Knobs.Daemonize == true {
+		config.qemuParams = append(config.qemuParams, "-daemonize")
+	}
+
+	config.appendMemoryKnobs()
 
 	if config.Knobs.Realtime == true {
 		config.qemuParams = append(config.qemuParams, "-realtime")
@@ -1495,6 +1549,10 @@ func (config *Config) appendKnobs() {
 			config.qemuParams = append(config.qemuParams, "mlock=off")
 		}
 	}
+
+	if config.Knobs.Stopped == true {
+		config.qemuParams = append(config.qemuParams, "-S")
+	}
 }
 
 func (config *Config) appendBios() {
@@ -1513,6 +1571,20 @@ func (config *Config) appendIOThreads() {
 	}
 }
 
+func (config *Config) appendIncoming() {
+	var uri string
+	switch config.Incoming.MigrationType {
+	case MigrationExec:
+		uri = fmt.Sprintf("exec:%s", config.Incoming.Exec)
+	case MigrationFD:
+		chFDs := config.appendFDs([]*os.File{config.Incoming.FD})
+		uri = fmt.Sprintf("fd:%d", chFDs[0])
+	default:
+		return
+	}
+	config.qemuParams = append(config.qemuParams, "-S", "-incoming", uri)
+}
+
 // LaunchQemu can be used to launch a new qemu instance.
 //
 // The Config parameter contains a set of qemu parameters and settings.
@@ -1537,6 +1609,7 @@ func LaunchQemu(config Config, logger QMPLog) (string, error) {
 	config.appendKernel()
 	config.appendBios()
 	config.appendIOThreads()
+	config.appendIncoming()
 
 	if err := config.appendCPUs(); err != nil {
 		return "", err

qemu/qemu_test.go

@@ -28,7 +28,12 @@ const volumeUUID = "67d86208-b46c-4465-9018-e14187d4010"
 
 func testAppend(structure interface{}, expected string, t *testing.T) {
 	var config Config
+	testConfigAppend(&config, structure, expected, t)
 
+	return
+}
+
+func testConfigAppend(config *Config, structure interface{}, expected string, t *testing.T) {
 	switch s := structure.(type) {
 	case Machine:
 		config.Machine = s
@@ -71,6 +76,9 @@ func testAppend(structure interface{}, expected string, t *testing.T) {
 	case IOThread:
 		config.IOThreads = []IOThread{s}
 		config.appendIOThreads()
+	case Incoming:
+		config.Incoming = s
+		config.appendIncoming()
 	}
 
 	result := strings.Join(config.qemuParams, " ")
@@ -388,15 +396,18 @@ func TestAppendEmptyDevice(t *testing.T) {
 }
 
 func TestAppendKnobsAllTrue(t *testing.T) {
-	var knobsString = "-no-user-config -nodefaults -nographic -daemonize -realtime mlock=on"
+	var knobsString = "-no-user-config -nodefaults -nographic -daemonize -realtime mlock=on -S"
 	knobs := Knobs{
 		NoUserConfig:        true,
 		NoDefaults:          true,
 		NoGraphic:           true,
 		Daemonize:           true,
 		MemPrealloc:         true,
+		FileBackedMem:       true,
+		FileBackedMemShared: true,
 		Realtime:            true,
 		Mlock:               true,
+		Stopped:             true,
 	}
 
 	testAppend(knobs, knobsString, t)
@@ -409,13 +420,107 @@ func TestAppendKnobsAllFalse(t *testing.T) {
 		NoDefaults:          false,
 		NoGraphic:           false,
 		MemPrealloc:         false,
+		FileBackedMem:       false,
+		FileBackedMemShared: false,
 		Realtime:            false,
 		Mlock:               false,
+		Stopped:             false,
 	}
 
 	testAppend(knobs, knobsString, t)
 }
 
+func TestAppendMemoryHugePages(t *testing.T) {
+	conf := &Config{
+		Memory: Memory{
+			Size:   "1G",
+			Slots:  8,
+			MaxMem: "3G",
+			Path:   "foobar",
+		},
+	}
+	memString := "-m 1G,slots=8,maxmem=3G"
+	testConfigAppend(conf, conf.Memory, memString, t)
+
+	knobs := Knobs{
+		HugePages:           true,
+		MemPrealloc:         true,
+		FileBackedMem:       true,
+		FileBackedMemShared: true,
+	}
+	knobsString := "-object memory-backend-file,id=dimm1,size=1G,mem-path=/dev/hugepages,share=on,prealloc=on -numa node,memdev=dimm1"
+	mlockFalseString := "-realtime mlock=off"
+
+	testConfigAppend(conf, knobs, memString+" "+knobsString+" "+mlockFalseString, t)
+}
+
+func TestAppendMemoryMemPrealloc(t *testing.T) {
+	conf := &Config{
+		Memory: Memory{
+			Size:   "1G",
+			Slots:  8,
+			MaxMem: "3G",
+			Path:   "foobar",
+		},
+	}
+	memString := "-m 1G,slots=8,maxmem=3G"
+	testConfigAppend(conf, conf.Memory, memString, t)
+
+	knobs := Knobs{
+		MemPrealloc:         true,
+		FileBackedMem:       true,
+		FileBackedMemShared: true,
+	}
+	knobsString := "-object memory-backend-ram,id=dimm1,size=1G,prealloc=on -device pc-dimm,id=dimm1,memdev=dimm1"
+	mlockFalseString := "-realtime mlock=off"
+
+	testConfigAppend(conf, knobs, memString+" "+knobsString+" "+mlockFalseString, t)
+}
+
+func TestAppendMemoryFileBackedMemShared(t *testing.T) {
+	conf := &Config{
+		Memory: Memory{
+			Size:   "1G",
+			Slots:  8,
+			MaxMem: "3G",
+			Path:   "foobar",
+		},
+	}
+	memString := "-m 1G,slots=8,maxmem=3G"
+	testConfigAppend(conf, conf.Memory, memString, t)
+
+	knobs := Knobs{
+		FileBackedMem:       true,
+		FileBackedMemShared: true,
+	}
+	knobsString := "-object memory-backend-file,id=dimm1,size=1G,mem-path=foobar,share=on -numa node,memdev=dimm1"
+	mlockFalseString := "-realtime mlock=off"
+
+	testConfigAppend(conf, knobs, memString+" "+knobsString+" "+mlockFalseString, t)
+}
+
+func TestAppendMemoryFileBackedMem(t *testing.T) {
+	conf := &Config{
+		Memory: Memory{
+			Size:   "1G",
+			Slots:  8,
+			MaxMem: "3G",
+			Path:   "foobar",
+		},
+	}
+	memString := "-m 1G,slots=8,maxmem=3G"
+	testConfigAppend(conf, conf.Memory, memString, t)
+
+	knobs := Knobs{
+		FileBackedMem:       true,
+		FileBackedMemShared: false,
+	}
+	knobsString := "-object memory-backend-file,id=dimm1,size=1G,mem-path=foobar -numa node,memdev=dimm1"
+	mlockFalseString := "-realtime mlock=off"
+
+	testConfigAppend(conf, knobs, memString+" "+knobsString+" "+mlockFalseString, t)
+}
+
 var kernelString = "-kernel /opt/vmlinux.container -initrd /opt/initrd.container -append root=/dev/pmem0p1 rootflags=dax,data=ordered,errors=remount-ro rw rootfstype=ext4 tsc=reliable"
 
 func TestAppendKernel(t *testing.T) {
@@ -435,6 +540,7 @@ func TestAppendMemory(t *testing.T) {
 		Size:   "2G",
 		Slots:  2,
 		MaxMem: "3G",
+		Path:   "",
 	}
 
 	testAppend(memory, memoryString, t)
@@ -556,3 +662,25 @@ func TestAppendIOThread(t *testing.T) {
 
 	testAppend(ioThread, ioThreadString, t)
 }
+
+var incomingStringFD = "-S -incoming fd:3"
+
+func TestAppendIncomingFD(t *testing.T) {
+	source := Incoming{
+		MigrationType: MigrationFD,
+		FD:            os.Stdout,
+	}
+
+	testAppend(source, incomingStringFD, t)
+}
+
+var incomingStringExec = "-S -incoming exec:test migration cmd"
+
+func TestAppendIncomingExec(t *testing.T) {
+	source := Incoming{
+		MigrationType: MigrationExec,
+		Exec:          "test migration cmd",
+	}
+
+	testAppend(source, incomingStringExec, t)
+}

qemu/qmp.go

@@ -828,3 +828,56 @@ func (q *QMP) ExecuteQueryHotpluggableCPUs(ctx context.Context) ([]HotpluggableC
 
 	return cpus, nil
 }
+
+// ExecSetMigrationCaps sets migration capabilities
+func (q *QMP) ExecSetMigrationCaps(ctx context.Context, caps []map[string]interface{}) error {
+	args := map[string]interface{}{
+		"capabilities": caps,
+	}
+
+	return q.executeCommand(ctx, "migrate-set-capabilities", args, nil)
+}
+
+// ExecSetMigrateArguments sets the command line used for migration
+func (q *QMP) ExecSetMigrateArguments(ctx context.Context, url string) error {
+	args := map[string]interface{}{
+		"uri": url,
+	}
+
+	return q.executeCommand(ctx, "migrate", args, nil)
+}
+
+// ExecHotplugMemory adds size of MiB memory to the guest
+func (q *QMP) ExecHotplugMemory(ctx context.Context, qomtype, id, mempath string, size int) error {
+	args := map[string]interface{}{
+		"qom-type": qomtype,
+		"id":       id,
+		"props":    map[string]interface{}{"size": uint64(size) << 20},
+	}
+	if mempath != "" {
+		args["mem-path"] = mempath
+	}
+	err := q.executeCommand(ctx, "object-add", args, nil)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err != nil {
+			q.cfg.Logger.Errorf("Unable to hotplug memory device: %v", err)
+			err = q.executeCommand(ctx, "object-del", map[string]interface{}{"id": id}, nil)
+			if err != nil {
+				q.cfg.Logger.Warningf("Unable to clean up memory object: %v", err)
+			}
+		}
+	}()
+
+	args = map[string]interface{}{
+		"driver": "pc-dimm",
+		"id":     "dimm" + id,
+		"memdev": id,
+	}
+	err = q.executeCommand(ctx, "device_add", args, nil)
+
+	return err
+}

qemu/qmp_test.go

@@ -887,3 +887,61 @@ func TestQMPExecuteQueryHotpluggableCPUs(t *testing.T) {
 	q.Shutdown()
 	<-disconnectedCh
 }
+
+// Checks that migrate capabilities can be set
+func TestExecSetMigrationCaps(t *testing.T) {
+	connectedCh := make(chan *QMPVersion)
+	disconnectedCh := make(chan struct{})
+	buf := newQMPTestCommandBuffer(t)
+	buf.AddCommand("migrate-set-capabilities", nil, "return", nil)
+	cfg := QMPConfig{Logger: qmpTestLogger{}}
+	q := startQMPLoop(buf, cfg, connectedCh, disconnectedCh)
+	checkVersion(t, connectedCh)
+	caps := []map[string]interface{}{
+		{
+			"capability": "bypass-shared-memory",
+			"state":      true,
+		},
+	}
+	err := q.ExecSetMigrationCaps(context.Background(), caps)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v\n", err)
+	}
+	q.Shutdown()
+	<-disconnectedCh
+}
+
+// Checks that migrate arguments can be set
+func TestExecSetMigrateArguments(t *testing.T) {
+	connectedCh := make(chan *QMPVersion)
+	disconnectedCh := make(chan struct{})
+	buf := newQMPTestCommandBuffer(t)
+	buf.AddCommand("migrate", nil, "return", nil)
+	cfg := QMPConfig{Logger: qmpTestLogger{}}
+	q := startQMPLoop(buf, cfg, connectedCh, disconnectedCh)
+	checkVersion(t, connectedCh)
+	err := q.ExecSetMigrateArguments(context.Background(), "exec:foobar")
+	if err != nil {
+		t.Fatalf("Unexpected error: %v\n", err)
+	}
+	q.Shutdown()
+	<-disconnectedCh
+}
+
+// Checks hotplug memory
+func TestExecHotplugMemory(t *testing.T) {
+	connectedCh := make(chan *QMPVersion)
+	disconnectedCh := make(chan struct{})
+	buf := newQMPTestCommandBuffer(t)
+	buf.AddCommand("object-add", nil, "return", nil)
+	buf.AddCommand("device_add", nil, "return", nil)
+	cfg := QMPConfig{Logger: qmpTestLogger{}}
+	q := startQMPLoop(buf, cfg, connectedCh, disconnectedCh)
+	checkVersion(t, connectedCh)
+	err := q.ExecHotplugMemory(context.Background(), "memory-backend-ram", "mem0", "", 128)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v\n", err)
+	}
+	q.Shutdown()
+	<-disconnectedCh
+}