From ac6b9c53d22c74629ec351d60134cf863f52ab40 Mon Sep 17 00:00:00 2001 From: Marcel Apfelbaum Date: Mon, 21 Jun 2021 20:13:13 +0300 Subject: [PATCH] runtime: Hot-plug virtio-mem device on PCI bridge Currently the virtio-mem device is hotplugged on the root bus. This doesn't work for PCIe machines like q35. Hotplug the virtio-mem device into the pci bridge instead. Fixes #1953 Signed-off-by: Marcel Apfelbaum --- src/runtime/go.mod | 2 +- src/runtime/go.sum | 2 ++ .../kata-containers/govmm/qemu/qmp.go | 12 ++++++++++-- src/runtime/vendor/modules.txt | 4 +--- src/runtime/virtcontainers/qemu.go | 18 +++++++++++++++--- 5 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/runtime/go.mod b/src/runtime/go.mod index aa52037ac3..1116fbd960 100644 --- a/src/runtime/go.mod +++ b/src/runtime/go.mod @@ -31,7 +31,7 @@ require ( github.com/gogo/protobuf v1.3.1 github.com/hashicorp/go-multierror v1.0.0 github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9 - github.com/kata-containers/govmm v0.0.0-20210520142420-eb57f004d89f + github.com/kata-containers/govmm v0.0.0-20210622075516-263136e69ac8 github.com/mdlayher/vsock v0.0.0-20191108225356-d9c65923cb8f github.com/opencontainers/image-spec v1.0.1 // indirect github.com/opencontainers/runc v1.0.0-rc9.0.20200102164712-2b52db75279c diff --git a/src/runtime/go.sum b/src/runtime/go.sum index 1193119bd7..a8ef2c61c9 100644 --- a/src/runtime/go.sum +++ b/src/runtime/go.sum @@ -274,6 +274,8 @@ github.com/kata-containers/govmm v0.0.0-20210428163604-f0e9a35308ee h1:M4N7AdSHg github.com/kata-containers/govmm v0.0.0-20210428163604-f0e9a35308ee/go.mod h1:VmAHbsL5lLfzHW/MNL96NVLF840DNEV5i683kISgFKk= github.com/kata-containers/govmm v0.0.0-20210520142420-eb57f004d89f h1:jXMZY7GIz5kSv3/Rdiesg1WMvgXJKNOk3KxwxgNWAVk= github.com/kata-containers/govmm v0.0.0-20210520142420-eb57f004d89f/go.mod h1:VmAHbsL5lLfzHW/MNL96NVLF840DNEV5i683kISgFKk= +github.com/kata-containers/govmm v0.0.0-20210622075516-263136e69ac8 h1:ZQ/qgoFsgmvk7AikgW1pI4dFaXfkUEVn+KK63ju3vgU= +github.com/kata-containers/govmm v0.0.0-20210622075516-263136e69ac8/go.mod h1:VmAHbsL5lLfzHW/MNL96NVLF840DNEV5i683kISgFKk= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= diff --git a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go index 97e924559c..23c288dc7d 100644 --- a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go +++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go @@ -1386,7 +1386,7 @@ func (q *QMP) ExecQueryCpusFast(ctx context.Context) ([]CPUInfoFast, error) { } // ExecMemdevAdd adds size of MiB memory device to the guest -func (q *QMP) ExecMemdevAdd(ctx context.Context, qomtype, id, mempath string, size int, share bool, driver, driverID string) error { +func (q *QMP) ExecMemdevAdd(ctx context.Context, qomtype, id, mempath string, size int, share bool, driver, driverID, addr, bus string) error { props := map[string]interface{}{"size": uint64(size) << 20} args := map[string]interface{}{ "qom-type": qomtype, @@ -1419,6 +1419,14 @@ func (q *QMP) ExecMemdevAdd(ctx context.Context, qomtype, id, mempath string, si "id": driverID, "memdev": id, } + + if bus != "" { + args["bus"] = bus + } + if addr != "" { + args["addr"] = addr + } + err = q.executeCommand(ctx, "device_add", args, nil) return err @@ -1426,7 +1434,7 @@ func (q *QMP) ExecMemdevAdd(ctx context.Context, qomtype, id, mempath string, si // ExecHotplugMemory adds size of MiB memory to the guest func (q *QMP) ExecHotplugMemory(ctx context.Context, qomtype, id, mempath string, size int, share bool) error { - return q.ExecMemdevAdd(ctx, qomtype, id, mempath, size, share, "pc-dimm", "dimm"+id) + return q.ExecMemdevAdd(ctx, qomtype, id, mempath, size, share, "pc-dimm", "dimm"+id, "", "") } // ExecuteNVDIMMDeviceAdd adds a block device to a QEMU instance using diff --git a/src/runtime/vendor/modules.txt b/src/runtime/vendor/modules.txt index cef609ea5e..9648172339 100644 --- a/src/runtime/vendor/modules.txt +++ b/src/runtime/vendor/modules.txt @@ -143,8 +143,6 @@ github.com/coreos/go-systemd/dbus github.com/cri-o/cri-o/pkg/annotations # github.com/davecgh/go-spew v1.1.1 github.com/davecgh/go-spew/spew -# github.com/dlespiau/covertool v0.0.0-20180314162135-b0c4c6d0583a -## explicit # github.com/docker/distribution v2.7.1+incompatible ## explicit github.com/docker/distribution/digestset @@ -224,7 +222,7 @@ github.com/hashicorp/go-multierror # github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9 ## explicit github.com/intel-go/cpuid -# github.com/kata-containers/govmm v0.0.0-20210520142420-eb57f004d89f +# github.com/kata-containers/govmm v0.0.0-20210622075516-263136e69ac8 ## explicit github.com/kata-containers/govmm/qemu # github.com/konsorten/go-windows-terminal-sequences v1.0.1 diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 89a8ce13d6..edfb02d131 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -708,7 +708,7 @@ func (q *qemu) getMemArgs() (bool, string, string, error) { return share, target, memoryBack, nil } -func (q *qemu) setupVirtioMem() error { +func (q *qemu) setupVirtioMem(ctx context.Context) error { maxMem, err := q.hostMemMB() if err != nil { return err @@ -724,7 +724,19 @@ func (q *qemu) setupVirtioMem() error { if err = q.qmpSetup(); err != nil { return err } - err = q.qmpMonitorCh.qmp.ExecMemdevAdd(q.qmpMonitorCh.ctx, memoryBack, "virtiomem", target, sizeMB, share, "virtio-mem-pci", "virtiomem0") + + addr, bridge, err := q.arch.addDeviceToBridge(ctx, "virtiomem-dev", types.PCI) + if err != nil { + return err + } + + defer func() { + if err != nil { + q.arch.removeDeviceFromBridge("virtiomem-dev") + } + }() + + err = q.qmpMonitorCh.qmp.ExecMemdevAdd(q.qmpMonitorCh.ctx, memoryBack, "virtiomem", target, sizeMB, share, "virtio-mem-pci", "virtiomem0", addr, bridge.ID) if err == nil { q.config.VirtioMem = true q.Logger().Infof("Setup %dMB virtio-mem-pci success", sizeMB) @@ -832,7 +844,7 @@ func (q *qemu) startSandbox(ctx context.Context, timeout int) error { } if q.config.VirtioMem { - err = q.setupVirtioMem() + err = q.setupVirtioMem(ctx) } return err