From 3e4989db429f231eda7bf86990708906ec733cab Mon Sep 17 00:00:00 2001 From: Ganesh Maharaj Mahalingam Date: Tue, 18 Jun 2019 10:59:37 -0700 Subject: [PATCH 1/2] vendor: update govmm and match code update govmm to support virtiofs memory hotplug Commits: 0c900f5 Allow sharing of memory backend file f695ddf qemu: add migration incoming defer support f0f18dd qmp: add virtio-blk multiqueue 7d3deea qemu: Add a virtio-blk-pci device driver support Fixes: #1745 Signed-off-by: Ganesh Maharaj Mahalingam --- Gopkg.lock | 4 ++-- Gopkg.toml | 2 +- vendor/github.com/intel/govmm/qemu/qemu.go | 7 ++++++ .../intel/govmm/qemu/qemu_arch_base.go | 1 + vendor/github.com/intel/govmm/qemu/qmp.go | 23 +++++++++++++++---- virtcontainers/qemu.go | 4 ++-- 6 files changed, 32 insertions(+), 9 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 4eb1537d07..5f5803834d 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -379,11 +379,11 @@ revision = "2f1d1f20f75d5404f53b9edf6b53ed5505508675" [[projects]] - digest = "1:2690f7d938dd074d30aa60849f26bcb9f5dd3ad88220a1f24c895c0df63fd1ae" + digest = "1:8e8b93094815f3bbc3792eaf78ce5cf45e0376b8ba0c92f3d58714d7a34f5d0b" name = "github.com/intel/govmm" packages = ["qemu"] pruneopts = "NUT" - revision = "b3e7a9e78463a10f2a19e1a966c76a3afb215781" + revision = "9f389cb319af066f210b28dc8846d679878fe9f2" [[projects]] digest = "1:36dfd4701e98a9d8371dd3053e32d4f29e82b07bcc9e655db82138f9273bcb0f" diff --git a/Gopkg.toml b/Gopkg.toml index c69b407332..f21960b887 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -48,7 +48,7 @@ [[constraint]] name = "github.com/intel/govmm" - revision = "b3e7a9e78463a10f2a19e1a966c76a3afb215781" + revision = "9f389cb319af066f210b28dc8846d679878fe9f2" [[constraint]] name = "github.com/kata-containers/agent" diff --git a/vendor/github.com/intel/govmm/qemu/qemu.go b/vendor/github.com/intel/govmm/qemu/qemu.go index 16e268d686..f281d4ecba 100644 --- a/vendor/github.com/intel/govmm/qemu/qemu.go +++ b/vendor/github.com/intel/govmm/qemu/qemu.go @@ -71,6 +71,9 @@ const ( // VirtioBlock is the block device driver. VirtioBlock DeviceDriver = "virtio-blk" + // VirtioBlockPCI is a pci bus block device driver + VirtioBlockPCI DeviceDriver = "virtio-blk-pci" + // Console is the console device driver. Console DeviceDriver = "virtconsole" @@ -1394,6 +1397,8 @@ const ( MigrationFD = 1 // MigrationExec is the migration incoming type based on commands. MigrationExec = 2 + // MigrationDefer is the defer incoming type + MigrationDefer = 3 ) // Incoming controls migration source preparation @@ -1776,6 +1781,8 @@ func (config *Config) appendIncoming() { case MigrationFD: chFDs := config.appendFDs([]*os.File{config.Incoming.FD}) uri = fmt.Sprintf("fd:%d", chFDs[0]) + case MigrationDefer: + uri = "defer" default: return } diff --git a/vendor/github.com/intel/govmm/qemu/qemu_arch_base.go b/vendor/github.com/intel/govmm/qemu/qemu_arch_base.go index 521741fa6a..ca9d78e73f 100644 --- a/vendor/github.com/intel/govmm/qemu/qemu_arch_base.go +++ b/vendor/github.com/intel/govmm/qemu/qemu_arch_base.go @@ -47,6 +47,7 @@ var isVirtioPCI = map[DeviceDriver]bool{ VirtioNetPCI: true, VirtioSerial: true, VirtioBlock: true, + VirtioBlockPCI: true, Console: false, VirtioSerialPort: false, VHostVSock: true, diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go index 1e2542fb6d..359e236e69 100644 --- a/vendor/github.com/intel/govmm/qemu/qmp.go +++ b/vendor/github.com/intel/govmm/qemu/qmp.go @@ -25,6 +25,7 @@ import ( "io" "net" "os" + "strconv" "syscall" "time" @@ -1058,12 +1059,12 @@ func (q *QMP) ExecuteDeviceDel(ctx context.Context, devID string) error { // ExecutePCIDeviceAdd is the PCI version of ExecuteDeviceAdd. This function can be used // to hot plug PCI devices on PCI(E) bridges, unlike ExecuteDeviceAdd this function receive the -// device address on its parent bus. bus is optional. shared denotes if the drive can be shared -// allowing it to be passed more than once. +// device address on its parent bus. bus is optional. queues specifies the number of queues of +// a block device. shared denotes if the drive can be shared allowing it to be passed more than once. // disableModern indicates if virtio version 1.0 should be replaced by the // former version 0.9, as there is a KVM bug that occurs when using virtio // 1.0 in nested environments. -func (q *QMP) ExecutePCIDeviceAdd(ctx context.Context, blockdevID, devID, driver, addr, bus, romfile string, shared, disableModern bool) error { +func (q *QMP) ExecutePCIDeviceAdd(ctx context.Context, blockdevID, devID, driver, addr, bus, romfile string, queues int, shared, disableModern bool) error { args := map[string]interface{}{ "id": devID, "driver": driver, @@ -1076,6 +1077,9 @@ func (q *QMP) ExecutePCIDeviceAdd(ctx context.Context, blockdevID, devID, driver if shared && (q.version.Major > 2 || (q.version.Major == 2 && q.version.Minor >= 10)) { args["share-rw"] = "on" } + if queues > 0 { + args["num-queues"] = strconv.Itoa(queues) + } if isVirtioPCI[DeviceDriver(driver)] { args["romfile"] = romfile @@ -1281,7 +1285,7 @@ func (q *QMP) ExecQueryCpusFast(ctx context.Context) ([]CPUInfoFast, error) { } // ExecHotplugMemory adds size of MiB memory to the guest -func (q *QMP) ExecHotplugMemory(ctx context.Context, qomtype, id, mempath string, size int) error { +func (q *QMP) ExecHotplugMemory(ctx context.Context, qomtype, id, mempath string, size int, share bool) error { props := map[string]interface{}{"size": uint64(size) << 20} args := map[string]interface{}{ "qom-type": qomtype, @@ -1291,6 +1295,9 @@ func (q *QMP) ExecHotplugMemory(ctx context.Context, qomtype, id, mempath string if mempath != "" { props["mem-path"] = mempath } + if share { + props["share"] = true + } err := q.executeCommand(ctx, "object-add", args, nil) if err != nil { return err @@ -1453,3 +1460,11 @@ func (q *QMP) ExecuteQueryMigration(ctx context.Context) (MigrationStatus, error return status, nil } + +// ExecuteMigrationIncoming start migration from incoming uri. +func (q *QMP) ExecuteMigrationIncoming(ctx context.Context, uri string) error { + args := map[string]interface{}{ + "uri": uri, + } + return q.executeCommand(ctx, "migrate-incoming", args, nil) +} diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 2f1fce6249..cc84b2e386 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -953,7 +953,7 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev // PCI address is in the format bridge-addr/device-addr eg. "03/02" drive.PCIAddr = fmt.Sprintf("%02x", bridge.Addr) + "/" + addr - if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, true, q.arch.runNested()); err != nil { + if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, q.arch.runNested()); err != nil { return err } } else { @@ -1363,7 +1363,7 @@ func (q *qemu) hotplugAddMemory(memDev *memoryDevice) (int, error) { } memDev.slot = maxSlot + 1 } - err = q.qmpMonitorCh.qmp.ExecHotplugMemory(q.qmpMonitorCh.ctx, "memory-backend-ram", "mem"+strconv.Itoa(memDev.slot), "", memDev.sizeMB) + err = q.qmpMonitorCh.qmp.ExecHotplugMemory(q.qmpMonitorCh.ctx, "memory-backend-ram", "mem"+strconv.Itoa(memDev.slot), "", memDev.sizeMB, false) if err != nil { q.Logger().WithError(err).Error("hotplug memory") return 0, err From 050f8e97157e96cae42986632f191eebab25ad7b Mon Sep 17 00:00:00 2001 From: Ganesh Maharaj Mahalingam Date: Tue, 9 Jul 2019 08:51:23 -0700 Subject: [PATCH 2/2] runtime: Disable disable-modern for virtio QMP add History: the previous version of kvm (unknown) and qemu-lite had an issue using virtio 1.0 version when it came to device hotplug, which led to the team to disable 1.0 version of virtio for hotplug (set disable-modern=on). Please check https://github.com/containers/virtcontainers/commit/e99f6b2931c4d28570f7437d9935fbc902636f0b for further info. We have since moved to QEMU4.0 and probably a later version of kvm as default across all distros. This change is to move to virtio 1.0 for hotplugging devices. Fixes: #1870 Signed-off-by: Ganesh Maharaj Mahalingam --- virtcontainers/qemu.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index cc84b2e386..727bdd5469 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -40,6 +40,11 @@ import ( // such as SeaBIOS or OVMF for instance, to handle this directly. const romFile = "" +// disable-modern is a option to QEMU that will fall back to using 0.9 version +// of virtio. Since moving to QEMU4.0, we can start using virtio 1.0 version. +// Default value is false. +const defaultDisableModern = false + type qmpChannel struct { ctx context.Context path string @@ -953,7 +958,7 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev // PCI address is in the format bridge-addr/device-addr eg. "03/02" drive.PCIAddr = fmt.Sprintf("%02x", bridge.Addr) + "/" + addr - if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, q.arch.runNested()); err != nil { + if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil { return err } } else { @@ -968,7 +973,7 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev return err } - if err = q.qmpMonitorCh.qmp.ExecuteSCSIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, bus, romFile, scsiID, lun, true, q.arch.runNested()); err != nil { + if err = q.qmpMonitorCh.qmp.ExecuteSCSIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, bus, romFile, scsiID, lun, true, defaultDisableModern); err != nil { return err } } @@ -1119,7 +1124,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) error { if machine.Type == QemuCCWVirtio { return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, int(q.config.NumVCPUs)) } - return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, romFile, int(q.config.NumVCPUs), q.arch.runNested()) + return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, romFile, int(q.config.NumVCPUs), defaultDisableModern) } if err := q.removeDeviceFromBridge(tap.ID); err != nil {