From 67ac4f45858057672b4d0994fa07116db72cdd63 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Wed, 31 Mar 2021 13:22:25 +0200 Subject: [PATCH 1/2] runtime: update GoVMM for memory backend support Update GoVMM to get memory backend support for non-DIMM setups. This is necessary for virtio-fs on s390x. Signed-off-by: Jakob Naucke --- src/runtime/go.mod | 2 +- src/runtime/go.sum | 2 + .../kata-containers/govmm/qemu/qemu.go | 55 ++++++++++++++++--- src/runtime/vendor/modules.txt | 2 +- 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/src/runtime/go.mod b/src/runtime/go.mod index 22d69514c5..df9c2d2228 100644 --- a/src/runtime/go.mod +++ b/src/runtime/go.mod @@ -31,7 +31,7 @@ require ( github.com/gogo/googleapis v1.4.0 // indirect github.com/gogo/protobuf v1.3.1 github.com/hashicorp/go-multierror v1.0.0 - github.com/kata-containers/govmm v0.0.0-20210112013750-7d320e8f5dca + github.com/kata-containers/govmm v0.0.0-20210329205824-7fbc685865d2 github.com/mdlayher/vsock v0.0.0-20191108225356-d9c65923cb8f github.com/opencontainers/image-spec v1.0.1 // indirect github.com/opencontainers/runc v1.0.0-rc9.0.20200102164712-2b52db75279c diff --git a/src/runtime/go.sum b/src/runtime/go.sum index 1937048b4e..46e7d2b003 100644 --- a/src/runtime/go.sum +++ b/src/runtime/go.sum @@ -276,6 +276,8 @@ github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0t github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kata-containers/govmm v0.0.0-20210112013750-7d320e8f5dca h1:UdXFthwasAPnmv37gLJUEFsW9FaabYA+mM6FoSi8kzU= github.com/kata-containers/govmm v0.0.0-20210112013750-7d320e8f5dca/go.mod h1:VmAHbsL5lLfzHW/MNL96NVLF840DNEV5i683kISgFKk= +github.com/kata-containers/govmm v0.0.0-20210329205824-7fbc685865d2 h1:oDJsQ5avmW8PFUkSJdsuaGL3SR4/YQRno51GNGl+IDU= +github.com/kata-containers/govmm v0.0.0-20210329205824-7fbc685865d2/go.mod h1:VmAHbsL5lLfzHW/MNL96NVLF840DNEV5i683kISgFKk= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= diff --git a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qemu.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qemu.go index 9cfe39cbd1..53ac898407 100644 --- a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qemu.go +++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qemu.go @@ -131,6 +131,9 @@ const ( // PCIeRootPort is a PCIe Root Port, the PCIe device should be hotplugged to this port. PCIeRootPort DeviceDriver = "pcie-root-port" + + // Loader is the Loader device driver. + Loader DeviceDriver = "loader" ) func isDimmSupported(config *Config) bool { @@ -552,7 +555,7 @@ func (cdev CharDevice) QemuParams(config *Config) []string { cdevParams = append(cdevParams, string(cdev.Backend)) cdevParams = append(cdevParams, fmt.Sprintf(",id=%s", cdev.ID)) if cdev.Backend == Socket { - cdevParams = append(cdevParams, fmt.Sprintf(",path=%s,server,nowait", cdev.Path)) + cdevParams = append(cdevParams, fmt.Sprintf(",path=%s,server=on,wait=off", cdev.Path)) } else { cdevParams = append(cdevParams, fmt.Sprintf(",path=%s", cdev.Path)) } @@ -1138,6 +1141,40 @@ func (dev PVPanicDevice) QemuParams(config *Config) []string { return []string{"-device", "pvpanic"} } +// LoaderDevice represents a qemu loader device. +type LoaderDevice struct { + File string + ID string +} + +// Valid returns true if there is a valid structure defined for LoaderDevice +func (dev LoaderDevice) Valid() bool { + if dev.File == "" { + return false + } + + if dev.ID == "" { + return false + } + + return true +} + +// QemuParams returns the qemu parameters built out of this loader device. +func (dev LoaderDevice) QemuParams(config *Config) []string { + var qemuParams []string + var devParams []string + + devParams = append(devParams, "loader") + devParams = append(devParams, fmt.Sprintf("file=%s", dev.File)) + devParams = append(devParams, fmt.Sprintf("id=%s", dev.ID)) + + qemuParams = append(qemuParams, "-device") + qemuParams = append(qemuParams, strings.Join(devParams, ",")) + + return qemuParams +} + // VhostUserDevice represents a qemu vhost-user device meant to be passed // in to the guest type VhostUserDevice struct { @@ -2385,9 +2422,9 @@ func (config *Config) appendQMPSockets() { qmpParams := append([]string{}, fmt.Sprintf("%s:", q.Type)) qmpParams = append(qmpParams, q.Name) if q.Server { - qmpParams = append(qmpParams, ",server") + qmpParams = append(qmpParams, ",server=on") if q.NoWait { - qmpParams = append(qmpParams, ",nowait") + qmpParams = append(qmpParams, ",wait=off") } } @@ -2528,9 +2565,6 @@ func (config *Config) appendMemoryKnobs() { if config.Memory.Size == "" { return } - if !isDimmSupported(config) { - return - } var objMemParam, numaMemParam string dimmName := "dimm1" if config.Knobs.HugePages { @@ -2553,8 +2587,13 @@ func (config *Config) appendMemoryKnobs() { config.qemuParams = append(config.qemuParams, "-object") config.qemuParams = append(config.qemuParams, objMemParam) - config.qemuParams = append(config.qemuParams, "-numa") - config.qemuParams = append(config.qemuParams, numaMemParam) + if isDimmSupported(config) { + config.qemuParams = append(config.qemuParams, "-numa") + config.qemuParams = append(config.qemuParams, numaMemParam) + } else { + config.qemuParams = append(config.qemuParams, "-machine") + config.qemuParams = append(config.qemuParams, "memory-backend="+dimmName) + } } func (config *Config) appendKnobs() { diff --git a/src/runtime/vendor/modules.txt b/src/runtime/vendor/modules.txt index 8bdde57b7e..aceaac2a95 100644 --- a/src/runtime/vendor/modules.txt +++ b/src/runtime/vendor/modules.txt @@ -222,7 +222,7 @@ github.com/hashicorp/errwrap # github.com/hashicorp/go-multierror v1.0.0 ## explicit github.com/hashicorp/go-multierror -# github.com/kata-containers/govmm v0.0.0-20210112013750-7d320e8f5dca +# github.com/kata-containers/govmm v0.0.0-20210329205824-7fbc685865d2 ## explicit github.com/kata-containers/govmm/qemu # github.com/konsorten/go-windows-terminal-sequences v1.0.1 From 7f6091133302f3038361b37bfdb3cc39f057154e Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Wed, 31 Mar 2021 13:22:35 +0200 Subject: [PATCH 2/2] virtcontainers: Allow s390x appendVhostUserDevice Remove the prohibition of vhost-user devices on s390x, which are by now supported (e.g. vhost-user-fs-ccw). As a consequence, appendVhostUserDevice no longer needs an error in its signature. This enables virtio-fs support on s390x. Fixes: #1469 Signed-off-by: Jakob Naucke --- src/runtime/virtcontainers/qemu.go | 4 ++-- src/runtime/virtcontainers/qemu_arch_base.go | 6 +++--- src/runtime/virtcontainers/qemu_arch_base_test.go | 2 +- src/runtime/virtcontainers/qemu_s390x.go | 6 ------ src/runtime/virtcontainers/qemu_s390x_test.go | 14 -------------- 5 files changed, 6 insertions(+), 26 deletions(-) diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 98f9a844ee..46a6bf9b48 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -1879,7 +1879,7 @@ func (q *qemu) addDevice(ctx context.Context, devInfo interface{}, devType devic vhostDev.SocketPath = sockPath vhostDev.DevID = id - q.qemuConfig.Devices, err = q.arch.appendVhostUserDevice(q.qemuConfig.Devices, vhostDev) + q.qemuConfig.Devices = q.arch.appendVhostUserDevice(q.qemuConfig.Devices, vhostDev) } else { q.Logger().WithField("volume-type", "virtio-9p").Info("adding volume") q.qemuConfig.Devices, err = q.arch.append9PVolume(ctx, q.qemuConfig.Devices, v) @@ -1894,7 +1894,7 @@ func (q *qemu) addDevice(ctx context.Context, devInfo interface{}, devType devic case config.BlockDrive: q.qemuConfig.Devices, err = q.arch.appendBlockDevice(ctx, q.qemuConfig.Devices, v) case config.VhostUserDeviceAttrs: - q.qemuConfig.Devices, err = q.arch.appendVhostUserDevice(q.qemuConfig.Devices, v) + q.qemuConfig.Devices = q.arch.appendVhostUserDevice(q.qemuConfig.Devices, v) case config.VFIODev: q.qemuConfig.Devices = q.arch.appendVFIODevice(q.qemuConfig.Devices, v) default: diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index ae0ce3a13e..aa8e4d59fd 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -96,7 +96,7 @@ type qemuArch interface { appendBlockDevice(ctx context.Context, devices []govmmQemu.Device, drive config.BlockDrive) ([]govmmQemu.Device, error) // appendVhostUserDevice appends a vhost user device to devices - appendVhostUserDevice(devices []govmmQemu.Device, drive config.VhostUserDeviceAttrs) ([]govmmQemu.Device, error) + appendVhostUserDevice(devices []govmmQemu.Device, drive config.VhostUserDeviceAttrs) []govmmQemu.Device // appendVFIODevice appends a VFIO device to devices appendVFIODevice(devices []govmmQemu.Device, vfioDevice config.VFIODev) []govmmQemu.Device @@ -633,7 +633,7 @@ func (q *qemuArchBase) appendBlockDevice(_ context.Context, devices []govmmQemu. return devices, nil } -func (q *qemuArchBase) appendVhostUserDevice(devices []govmmQemu.Device, attr config.VhostUserDeviceAttrs) ([]govmmQemu.Device, error) { +func (q *qemuArchBase) appendVhostUserDevice(devices []govmmQemu.Device, attr config.VhostUserDeviceAttrs) []govmmQemu.Device { qemuVhostUserDevice := govmmQemu.VhostUserDevice{} switch attr.Type { @@ -658,7 +658,7 @@ func (q *qemuArchBase) appendVhostUserDevice(devices []govmmQemu.Device, attr co devices = append(devices, qemuVhostUserDevice) - return devices, nil + return devices } func (q *qemuArchBase) appendVFIODevice(devices []govmmQemu.Device, vfioDev config.VFIODev) []govmmQemu.Device { diff --git a/src/runtime/virtcontainers/qemu_arch_base_test.go b/src/runtime/virtcontainers/qemu_arch_base_test.go index d4aba54fac..f134a15ae3 100644 --- a/src/runtime/virtcontainers/qemu_arch_base_test.go +++ b/src/runtime/virtcontainers/qemu_arch_base_test.go @@ -224,7 +224,7 @@ func testQemuArchBaseAppend(t *testing.T, structure interface{}, expected []govm case config.VFIODev: devices = qemuArchBase.appendVFIODevice(devices, s) case config.VhostUserDeviceAttrs: - devices, err = qemuArchBase.appendVhostUserDevice(devices, s) + devices = qemuArchBase.appendVhostUserDevice(devices, s) } assert.NoError(err) diff --git a/src/runtime/virtcontainers/qemu_s390x.go b/src/runtime/virtcontainers/qemu_s390x.go index 01968b6b1f..6a4c78264f 100644 --- a/src/runtime/virtcontainers/qemu_s390x.go +++ b/src/runtime/virtcontainers/qemu_s390x.go @@ -152,12 +152,6 @@ func (q *qemuS390x) appendCCWBlockDevice(ctx context.Context, devices []govmmQem return devices, nil } -// appendVhostUserDevice throws an error if vhost devices are tried to be used. -// See issue https://github.com/kata-containers/runtime/issues/659 -func (q *qemuS390x) appendVhostUserDevice(devices []govmmQemu.Device, attr config.VhostUserDeviceAttrs) ([]govmmQemu.Device, error) { - return nil, fmt.Errorf("No vhost-user devices supported on s390x") -} - // supportGuestMemoryHotplug return false for s390x architecture. The pc-dimm backend device for s390x // is not support. PC-DIMM is not listed in the devices supported by qemu-system-s390x -device help func (q *qemuS390x) supportGuestMemoryHotplug() bool { diff --git a/src/runtime/virtcontainers/qemu_s390x_test.go b/src/runtime/virtcontainers/qemu_s390x_test.go index 70ddf3ff21..3355680bb1 100644 --- a/src/runtime/virtcontainers/qemu_s390x_test.go +++ b/src/runtime/virtcontainers/qemu_s390x_test.go @@ -52,17 +52,3 @@ func TestQemuS390xMemoryTopology(t *testing.T) { m := s390x.memoryTopology(mem, hostMem, slots) assert.Equal(expectedMemory, m) } - -func TestQemuS390xAppendVhostUserDevice(t *testing.T) { - macAddress := "00:11:22:33:44:55:66" - qemu := qemuS390x{} - assert := assert.New(t) - - vhostUserDevice := config.VhostUserDeviceAttrs{ - Type: config.VhostUserNet, - MacAddress: macAddress, - } - - _, err := qemu.appendVhostUserDevice(nil, vhostUserDevice) - assert.Error(err) -}