diff --git a/qemu/qemu.go b/qemu/qemu.go index f2f4f5b8b2..41381d80ee 100644 --- a/qemu/qemu.go +++ b/qemu/qemu.go @@ -1126,6 +1126,10 @@ type Memory struct { // MaxMem is the maximum amount of memory that can be made available // to the guest through e.g. hot pluggable memory. MaxMem string + + // Path is the file path of the memory device. It points to a local + // file path used by FileBackedMem. + Path string } // Kernel is the guest kernel configuration structure. @@ -1167,10 +1171,20 @@ type Knobs struct { // MemPrealloc will allocate all the RAM upfront MemPrealloc bool + // FileBackedMem requires Memory.Size and Memory.Path of the VM to + // be set. + FileBackedMem bool + + // FileBackedMemShared will set the FileBackedMem device as shared. + FileBackedMemShared bool + // Mlock will control locking of memory // Only active when Realtime is set to true Mlock bool + // Stopped will not start guest CPU at startup + Stopped bool + // Realtime will enable realtime QEMU Realtime bool } @@ -1180,6 +1194,24 @@ type IOThread struct { ID string } +const ( + // MigrationFD is the migration incoming type based on open file descriptor. + // Skip default 0 so that it must be set on purpose. + MigrationFD = 1 + // MigrationExec is the migration incoming type based on commands. + MigrationExec = 2 +) + +// Incoming controls migration source preparation +type Incoming struct { + // Possible values are MigrationFD, MigrationExec + MigrationType int + // Only valid if MigrationType == MigrationFD + FD *os.File + // Only valid if MigrationType == MigrationExec + Exec string +} + // Config is the qemu configuration structure. // It allows for passing custom settings and parameters to the qemu API. type Config struct { @@ -1231,6 +1263,9 @@ type Config struct { // Bios is the -bios parameter Bios string + // Incoming controls migration source preparation + Incoming Incoming + // fds is a list of open file descriptors to be passed to the spawned qemu process fds []*os.File @@ -1433,23 +1468,7 @@ func (config *Config) appendKernel() { } } -func (config *Config) appendKnobs() { - if config.Knobs.NoUserConfig == true { - config.qemuParams = append(config.qemuParams, "-no-user-config") - } - - if config.Knobs.NoDefaults == true { - config.qemuParams = append(config.qemuParams, "-nodefaults") - } - - if config.Knobs.NoGraphic == true { - config.qemuParams = append(config.qemuParams, "-nographic") - } - - if config.Knobs.Daemonize == true { - config.qemuParams = append(config.qemuParams, "-daemonize") - } - +func (config *Config) appendMemoryKnobs() { if config.Knobs.HugePages == true { if config.Memory.Size != "" { dimmName := "dimm1" @@ -1474,7 +1493,42 @@ func (config *Config) appendKnobs() { config.qemuParams = append(config.qemuParams, "-device") config.qemuParams = append(config.qemuParams, deviceMemParam) } + } else if config.Knobs.FileBackedMem == true { + if config.Memory.Size != "" && config.Memory.Path != "" { + dimmName := "dimm1" + objMemParam := "memory-backend-file,id=" + dimmName + ",size=" + config.Memory.Size + ",mem-path=" + config.Memory.Path + if config.Knobs.FileBackedMemShared == true { + objMemParam += ",share=on" + } + numaMemParam := "node,memdev=" + dimmName + + config.qemuParams = append(config.qemuParams, "-object") + config.qemuParams = append(config.qemuParams, objMemParam) + + config.qemuParams = append(config.qemuParams, "-numa") + config.qemuParams = append(config.qemuParams, numaMemParam) + } } +} + +func (config *Config) appendKnobs() { + if config.Knobs.NoUserConfig == true { + config.qemuParams = append(config.qemuParams, "-no-user-config") + } + + if config.Knobs.NoDefaults == true { + config.qemuParams = append(config.qemuParams, "-nodefaults") + } + + if config.Knobs.NoGraphic == true { + config.qemuParams = append(config.qemuParams, "-nographic") + } + + if config.Knobs.Daemonize == true { + config.qemuParams = append(config.qemuParams, "-daemonize") + } + + config.appendMemoryKnobs() if config.Knobs.Realtime == true { config.qemuParams = append(config.qemuParams, "-realtime") @@ -1495,6 +1549,10 @@ func (config *Config) appendKnobs() { config.qemuParams = append(config.qemuParams, "mlock=off") } } + + if config.Knobs.Stopped == true { + config.qemuParams = append(config.qemuParams, "-S") + } } func (config *Config) appendBios() { @@ -1513,6 +1571,20 @@ func (config *Config) appendIOThreads() { } } +func (config *Config) appendIncoming() { + var uri string + switch config.Incoming.MigrationType { + case MigrationExec: + uri = fmt.Sprintf("exec:%s", config.Incoming.Exec) + case MigrationFD: + chFDs := config.appendFDs([]*os.File{config.Incoming.FD}) + uri = fmt.Sprintf("fd:%d", chFDs[0]) + default: + return + } + config.qemuParams = append(config.qemuParams, "-S", "-incoming", uri) +} + // LaunchQemu can be used to launch a new qemu instance. // // The Config parameter contains a set of qemu parameters and settings. @@ -1537,6 +1609,7 @@ func LaunchQemu(config Config, logger QMPLog) (string, error) { config.appendKernel() config.appendBios() config.appendIOThreads() + config.appendIncoming() if err := config.appendCPUs(); err != nil { return "", err diff --git a/qemu/qemu_test.go b/qemu/qemu_test.go index 895fe0f15c..14637910d0 100644 --- a/qemu/qemu_test.go +++ b/qemu/qemu_test.go @@ -28,7 +28,12 @@ const volumeUUID = "67d86208-b46c-4465-9018-e14187d4010" func testAppend(structure interface{}, expected string, t *testing.T) { var config Config + testConfigAppend(&config, structure, expected, t) + return +} + +func testConfigAppend(config *Config, structure interface{}, expected string, t *testing.T) { switch s := structure.(type) { case Machine: config.Machine = s @@ -71,6 +76,9 @@ func testAppend(structure interface{}, expected string, t *testing.T) { case IOThread: config.IOThreads = []IOThread{s} config.appendIOThreads() + case Incoming: + config.Incoming = s + config.appendIncoming() } result := strings.Join(config.qemuParams, " ") @@ -388,15 +396,18 @@ func TestAppendEmptyDevice(t *testing.T) { } func TestAppendKnobsAllTrue(t *testing.T) { - var knobsString = "-no-user-config -nodefaults -nographic -daemonize -realtime mlock=on" + var knobsString = "-no-user-config -nodefaults -nographic -daemonize -realtime mlock=on -S" knobs := Knobs{ - NoUserConfig: true, - NoDefaults: true, - NoGraphic: true, - Daemonize: true, - MemPrealloc: true, - Realtime: true, - Mlock: true, + NoUserConfig: true, + NoDefaults: true, + NoGraphic: true, + Daemonize: true, + MemPrealloc: true, + FileBackedMem: true, + FileBackedMemShared: true, + Realtime: true, + Mlock: true, + Stopped: true, } testAppend(knobs, knobsString, t) @@ -405,17 +416,111 @@ func TestAppendKnobsAllTrue(t *testing.T) { func TestAppendKnobsAllFalse(t *testing.T) { var knobsString = "-realtime mlock=off" knobs := Knobs{ - NoUserConfig: false, - NoDefaults: false, - NoGraphic: false, - MemPrealloc: false, - Realtime: false, - Mlock: false, + NoUserConfig: false, + NoDefaults: false, + NoGraphic: false, + MemPrealloc: false, + FileBackedMem: false, + FileBackedMemShared: false, + Realtime: false, + Mlock: false, + Stopped: false, } testAppend(knobs, knobsString, t) } +func TestAppendMemoryHugePages(t *testing.T) { + conf := &Config{ + Memory: Memory{ + Size: "1G", + Slots: 8, + MaxMem: "3G", + Path: "foobar", + }, + } + memString := "-m 1G,slots=8,maxmem=3G" + testConfigAppend(conf, conf.Memory, memString, t) + + knobs := Knobs{ + HugePages: true, + MemPrealloc: true, + FileBackedMem: true, + FileBackedMemShared: true, + } + knobsString := "-object memory-backend-file,id=dimm1,size=1G,mem-path=/dev/hugepages,share=on,prealloc=on -numa node,memdev=dimm1" + mlockFalseString := "-realtime mlock=off" + + testConfigAppend(conf, knobs, memString+" "+knobsString+" "+mlockFalseString, t) +} + +func TestAppendMemoryMemPrealloc(t *testing.T) { + conf := &Config{ + Memory: Memory{ + Size: "1G", + Slots: 8, + MaxMem: "3G", + Path: "foobar", + }, + } + memString := "-m 1G,slots=8,maxmem=3G" + testConfigAppend(conf, conf.Memory, memString, t) + + knobs := Knobs{ + MemPrealloc: true, + FileBackedMem: true, + FileBackedMemShared: true, + } + knobsString := "-object memory-backend-ram,id=dimm1,size=1G,prealloc=on -device pc-dimm,id=dimm1,memdev=dimm1" + mlockFalseString := "-realtime mlock=off" + + testConfigAppend(conf, knobs, memString+" "+knobsString+" "+mlockFalseString, t) +} + +func TestAppendMemoryFileBackedMemShared(t *testing.T) { + conf := &Config{ + Memory: Memory{ + Size: "1G", + Slots: 8, + MaxMem: "3G", + Path: "foobar", + }, + } + memString := "-m 1G,slots=8,maxmem=3G" + testConfigAppend(conf, conf.Memory, memString, t) + + knobs := Knobs{ + FileBackedMem: true, + FileBackedMemShared: true, + } + knobsString := "-object memory-backend-file,id=dimm1,size=1G,mem-path=foobar,share=on -numa node,memdev=dimm1" + mlockFalseString := "-realtime mlock=off" + + testConfigAppend(conf, knobs, memString+" "+knobsString+" "+mlockFalseString, t) +} + +func TestAppendMemoryFileBackedMem(t *testing.T) { + conf := &Config{ + Memory: Memory{ + Size: "1G", + Slots: 8, + MaxMem: "3G", + Path: "foobar", + }, + } + memString := "-m 1G,slots=8,maxmem=3G" + testConfigAppend(conf, conf.Memory, memString, t) + + knobs := Knobs{ + FileBackedMem: true, + FileBackedMemShared: false, + } + knobsString := "-object memory-backend-file,id=dimm1,size=1G,mem-path=foobar -numa node,memdev=dimm1" + mlockFalseString := "-realtime mlock=off" + + testConfigAppend(conf, knobs, memString+" "+knobsString+" "+mlockFalseString, t) +} + var kernelString = "-kernel /opt/vmlinux.container -initrd /opt/initrd.container -append root=/dev/pmem0p1 rootflags=dax,data=ordered,errors=remount-ro rw rootfstype=ext4 tsc=reliable" func TestAppendKernel(t *testing.T) { @@ -435,6 +540,7 @@ func TestAppendMemory(t *testing.T) { Size: "2G", Slots: 2, MaxMem: "3G", + Path: "", } testAppend(memory, memoryString, t) @@ -556,3 +662,25 @@ func TestAppendIOThread(t *testing.T) { testAppend(ioThread, ioThreadString, t) } + +var incomingStringFD = "-S -incoming fd:3" + +func TestAppendIncomingFD(t *testing.T) { + source := Incoming{ + MigrationType: MigrationFD, + FD: os.Stdout, + } + + testAppend(source, incomingStringFD, t) +} + +var incomingStringExec = "-S -incoming exec:test migration cmd" + +func TestAppendIncomingExec(t *testing.T) { + source := Incoming{ + MigrationType: MigrationExec, + Exec: "test migration cmd", + } + + testAppend(source, incomingStringExec, t) +} diff --git a/qemu/qmp.go b/qemu/qmp.go index 8888ad4d4a..37334e99e2 100644 --- a/qemu/qmp.go +++ b/qemu/qmp.go @@ -828,3 +828,56 @@ func (q *QMP) ExecuteQueryHotpluggableCPUs(ctx context.Context) ([]HotpluggableC return cpus, nil } + +// ExecSetMigrationCaps sets migration capabilities +func (q *QMP) ExecSetMigrationCaps(ctx context.Context, caps []map[string]interface{}) error { + args := map[string]interface{}{ + "capabilities": caps, + } + + return q.executeCommand(ctx, "migrate-set-capabilities", args, nil) +} + +// ExecSetMigrateArguments sets the command line used for migration +func (q *QMP) ExecSetMigrateArguments(ctx context.Context, url string) error { + args := map[string]interface{}{ + "uri": url, + } + + return q.executeCommand(ctx, "migrate", args, nil) +} + +// ExecHotplugMemory adds size of MiB memory to the guest +func (q *QMP) ExecHotplugMemory(ctx context.Context, qomtype, id, mempath string, size int) error { + args := map[string]interface{}{ + "qom-type": qomtype, + "id": id, + "props": map[string]interface{}{"size": uint64(size) << 20}, + } + if mempath != "" { + args["mem-path"] = mempath + } + err := q.executeCommand(ctx, "object-add", args, nil) + if err != nil { + return err + } + + defer func() { + if err != nil { + q.cfg.Logger.Errorf("Unable to hotplug memory device: %v", err) + err = q.executeCommand(ctx, "object-del", map[string]interface{}{"id": id}, nil) + if err != nil { + q.cfg.Logger.Warningf("Unable to clean up memory object: %v", err) + } + } + }() + + args = map[string]interface{}{ + "driver": "pc-dimm", + "id": "dimm" + id, + "memdev": id, + } + err = q.executeCommand(ctx, "device_add", args, nil) + + return err +} diff --git a/qemu/qmp_test.go b/qemu/qmp_test.go index 85850fcb5b..f5b472d741 100644 --- a/qemu/qmp_test.go +++ b/qemu/qmp_test.go @@ -887,3 +887,61 @@ func TestQMPExecuteQueryHotpluggableCPUs(t *testing.T) { q.Shutdown() <-disconnectedCh } + +// Checks that migrate capabilities can be set +func TestExecSetMigrationCaps(t *testing.T) { + connectedCh := make(chan *QMPVersion) + disconnectedCh := make(chan struct{}) + buf := newQMPTestCommandBuffer(t) + buf.AddCommand("migrate-set-capabilities", nil, "return", nil) + cfg := QMPConfig{Logger: qmpTestLogger{}} + q := startQMPLoop(buf, cfg, connectedCh, disconnectedCh) + checkVersion(t, connectedCh) + caps := []map[string]interface{}{ + { + "capability": "bypass-shared-memory", + "state": true, + }, + } + err := q.ExecSetMigrationCaps(context.Background(), caps) + if err != nil { + t.Fatalf("Unexpected error: %v\n", err) + } + q.Shutdown() + <-disconnectedCh +} + +// Checks that migrate arguments can be set +func TestExecSetMigrateArguments(t *testing.T) { + connectedCh := make(chan *QMPVersion) + disconnectedCh := make(chan struct{}) + buf := newQMPTestCommandBuffer(t) + buf.AddCommand("migrate", nil, "return", nil) + cfg := QMPConfig{Logger: qmpTestLogger{}} + q := startQMPLoop(buf, cfg, connectedCh, disconnectedCh) + checkVersion(t, connectedCh) + err := q.ExecSetMigrateArguments(context.Background(), "exec:foobar") + if err != nil { + t.Fatalf("Unexpected error: %v\n", err) + } + q.Shutdown() + <-disconnectedCh +} + +// Checks hotplug memory +func TestExecHotplugMemory(t *testing.T) { + connectedCh := make(chan *QMPVersion) + disconnectedCh := make(chan struct{}) + buf := newQMPTestCommandBuffer(t) + buf.AddCommand("object-add", nil, "return", nil) + buf.AddCommand("device_add", nil, "return", nil) + cfg := QMPConfig{Logger: qmpTestLogger{}} + q := startQMPLoop(buf, cfg, connectedCh, disconnectedCh) + checkVersion(t, connectedCh) + err := q.ExecHotplugMemory(context.Background(), "memory-backend-ram", "mem0", "", 128) + if err != nil { + t.Fatalf("Unexpected error: %v\n", err) + } + q.Shutdown() + <-disconnectedCh +}