diff --git a/virtcontainers/factory/template/template.go b/virtcontainers/factory/template/template.go index c7a2b23789..7b5ab9a14d 100644 --- a/virtcontainers/factory/template/template.go +++ b/virtcontainers/factory/template/template.go @@ -96,7 +96,7 @@ func (t *template) prepareTemplateFiles() error { return err } flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV) - opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+8) + opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize) if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil { t.close() return err diff --git a/virtcontainers/factory/template/template_amd64.go b/virtcontainers/factory/template/template_amd64.go new file mode 100644 index 0000000000..2bc875f82c --- /dev/null +++ b/virtcontainers/factory/template/template_amd64.go @@ -0,0 +1,14 @@ +// Copyright (c) 2019 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// template implements base vm factory with vm templating. + +package template + +// templateDeviceStateSize denotes device state size when +// mount tmpfs. +// when bypass-shared-memory is not support like arm64, +// creating template will occupy more space. That's why we +// put it here. +const templateDeviceStateSize = 8 diff --git a/virtcontainers/factory/template/template_arm64.go b/virtcontainers/factory/template/template_arm64.go new file mode 100644 index 0000000000..185e29eb1b --- /dev/null +++ b/virtcontainers/factory/template/template_arm64.go @@ -0,0 +1,14 @@ +// Copyright (c) 2019 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// template implements base vm factory with vm templating. + +package template + +// templateDeviceStateSize denotes device state size when +// mount tmpfs. +// when bypass-shared-memory is not support like arm64, +// creating template will occupy more space. That's why we +// put it here. +const templateDeviceStateSize = 300 diff --git a/virtcontainers/factory/template/template_ppc64le.go b/virtcontainers/factory/template/template_ppc64le.go new file mode 100644 index 0000000000..2bc875f82c --- /dev/null +++ b/virtcontainers/factory/template/template_ppc64le.go @@ -0,0 +1,14 @@ +// Copyright (c) 2019 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// template implements base vm factory with vm templating. + +package template + +// templateDeviceStateSize denotes device state size when +// mount tmpfs. +// when bypass-shared-memory is not support like arm64, +// creating template will occupy more space. That's why we +// put it here. +const templateDeviceStateSize = 8 diff --git a/virtcontainers/factory/template/template_s390x.go b/virtcontainers/factory/template/template_s390x.go new file mode 100644 index 0000000000..2bc875f82c --- /dev/null +++ b/virtcontainers/factory/template/template_s390x.go @@ -0,0 +1,14 @@ +// Copyright (c) 2019 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// template implements base vm factory with vm templating. + +package template + +// templateDeviceStateSize denotes device state size when +// mount tmpfs. +// when bypass-shared-memory is not support like arm64, +// creating template will occupy more space. That's why we +// put it here. +const templateDeviceStateSize = 8 diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index cde61e20a6..cea8d76232 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -89,10 +89,8 @@ const ( consoleSocket = "console.sock" qmpSocket = "qmp.sock" - qmpCapErrMsg = "Failed to negoatiate QMP capabilities" - qmpCapMigrationBypassSharedMemory = "bypass-shared-memory" - qmpExecCatCmd = "exec:cat" - qmpMigrationWaitTimeout = 5 * time.Second + qmpCapErrMsg = "Failed to negoatiate QMP capabilities" + qmpExecCatCmd = "exec:cat" scsiControllerID = "scsi0" rngID = "rng0" @@ -1331,12 +1329,7 @@ func (q *qemu) saveSandbox() error { // BootToBeTemplate sets the VM to be a template that other VMs can clone from. We would want to // bypass shared memory when saving the VM to a local file through migration exec. if q.config.BootToBeTemplate { - err = q.qmpMonitorCh.qmp.ExecSetMigrationCaps(q.qmpMonitorCh.ctx, []map[string]interface{}{ - { - "capability": qmpCapMigrationBypassSharedMemory, - "state": true, - }, - }) + err := q.arch.setBypassSharedMemoryMigrationCaps(q.qmpMonitorCh.ctx, q.qmpMonitorCh.qmp) if err != nil { q.Logger().WithError(err).Error("set migration bypass shared memory") return err diff --git a/virtcontainers/qemu_amd64.go b/virtcontainers/qemu_amd64.go index e09a66ab09..4cddaae1c8 100644 --- a/virtcontainers/qemu_amd64.go +++ b/virtcontainers/qemu_amd64.go @@ -7,6 +7,7 @@ package virtcontainers import ( "os" + "time" "github.com/kata-containers/runtime/virtcontainers/types" @@ -24,6 +25,10 @@ const defaultQemuMachineType = QemuPC const defaultQemuMachineOptions = "accel=kvm,kernel_irqchip,nvdimm" +const qmpCapMigrationBypassSharedMemory = "bypass-shared-memory" + +const qmpMigrationWaitTimeout = 5 * time.Second + var qemuPaths = map[string]string{ QemuPCLite: "/usr/bin/qemu-lite-system-x86_64", QemuPC: defaultQemuPath, diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index 4200ea22cb..bf145b43a2 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -6,6 +6,7 @@ package virtcontainers import ( + "context" "encoding/hex" "fmt" "os" @@ -100,6 +101,9 @@ type qemuArch interface { // supportGuestMemoryHotplug returns if the guest supports memory hotplug supportGuestMemoryHotplug() bool + + // setBypassSharedMemoryMigrationCaps set bypass-shared-memory capability for migration + setBypassSharedMemoryMigrationCaps(context.Context, *govmmQemu.QMP) error } type qemuArchBase struct { @@ -570,3 +574,13 @@ func (q *qemuArchBase) handleImagePath(config HypervisorConfig) { func (q *qemuArchBase) supportGuestMemoryHotplug() bool { return true } + +func (q *qemuArchBase) setBypassSharedMemoryMigrationCaps(ctx context.Context, qmp *govmmQemu.QMP) error { + err := qmp.ExecSetMigrationCaps(ctx, []map[string]interface{}{ + { + "capability": qmpCapMigrationBypassSharedMemory, + "state": true, + }, + }) + return err +} diff --git a/virtcontainers/qemu_arm64.go b/virtcontainers/qemu_arm64.go index 79e37ed98f..81c0f2af81 100644 --- a/virtcontainers/qemu_arm64.go +++ b/virtcontainers/qemu_arm64.go @@ -6,10 +6,12 @@ package virtcontainers import ( + "context" "io/ioutil" "os" "runtime" "strings" + "time" govmmQemu "github.com/intel/govmm/qemu" "github.com/kata-containers/runtime/virtcontainers/types" @@ -25,6 +27,10 @@ const defaultQemuPath = "/usr/bin/qemu-system-aarch64" const defaultQemuMachineType = QemuVirt +const qmpMigrationWaitTimeout = 10 * time.Second + +const qmpCapMigrationBypassSharedMemory = "bypass-shared-memory" + var defaultQemuMachineOptions = "usb=off,accel=kvm,nvdimm,gic-version=" + getGuestGICVersion() var qemuPaths = map[string]string{ @@ -189,3 +195,8 @@ func (q *qemuArm64) appendImage(devices []govmmQemu.Device, path string) ([]govm return devices, nil } + +func (q *qemuArm64) setBypassSharedMemoryMigrationCaps(_ context.Context, _ *govmmQemu.QMP) error { + // bypass-shared-memory not support in arm64 for now + return nil +} diff --git a/virtcontainers/qemu_ppc64le.go b/virtcontainers/qemu_ppc64le.go index cc77f683b2..6ef4015731 100644 --- a/virtcontainers/qemu_ppc64le.go +++ b/virtcontainers/qemu_ppc64le.go @@ -8,6 +8,7 @@ package virtcontainers import ( "encoding/hex" "os" + "time" govmmQemu "github.com/intel/govmm/qemu" deviceConfig "github.com/kata-containers/runtime/virtcontainers/device/config" @@ -29,6 +30,10 @@ const defaultQemuMachineOptions = "accel=kvm,usb=off" const defaultMemMaxPPC64le = 32256 // Restrict MemMax to 32Gb on PPC64le +const qmpCapMigrationBypassSharedMemory = "bypass-shared-memory" + +const qmpMigrationWaitTimeout = 5 * time.Second + var qemuPaths = map[string]string{ QemuPseries: defaultQemuPath, } diff --git a/virtcontainers/qemu_s390x.go b/virtcontainers/qemu_s390x.go index a9c9970b81..ad4086159f 100644 --- a/virtcontainers/qemu_s390x.go +++ b/virtcontainers/qemu_s390x.go @@ -7,6 +7,7 @@ package virtcontainers import ( "fmt" + "time" govmmQemu "github.com/intel/govmm/qemu" "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/types" @@ -25,6 +26,10 @@ const defaultQemuMachineOptions = "accel=kvm" const virtioSerialCCW = "virtio-serial-ccw" +const qmpCapMigrationBypassSharedMemory = "bypass-shared-memory" + +const qmpMigrationWaitTimeout = 5 * time.Second + var qemuPaths = map[string]string{ QemuCCWVirtio: defaultQemuPath, }