Template: enable template for arm64

Now, template feature can't work on arm64. When using bypass-shared-memory
cap to saving sandbox as a template, new sandbox cloning from the template
will fail. From initial investigation, it maybe a qemu issue. Further
research should be done.
To enable template on arm64, this patch adds a switch to offer the capability
to open or close the bypass-shared-memory cap.
While closing bypass-shared-memory cap, saving vm will occupy more space
and consume more time. So increase 300M for mount size and bump the time
waiting for migration to 10 seconds.

Fixes: #1336

Signed-off-by: Jianyong Wu  <jianyong.wu@arm.com>

virtcontainers/factory/template/template.go

@@ -90,7 +90,7 @@ func (t *template) prepareTemplateFiles() error {
 		return err
 	}
 	flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV)
-	opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+8)
+	opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize)
 	if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil {
 		t.close()
 		return err

virtcontainers/factory/template/template_amd64.go

@@ -0,0 +1,14 @@
+// Copyright (c) 2019 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// template implements base vm factory with vm templating.
+
+package template
+
+// templateDeviceStateSize denotes device state size when
+// mount tmpfs.
+// when bypass-shared-memory is not support like arm64,
+// creating template will occupy more space. That's why we
+// put it here.
+const templateDeviceStateSize = 8

virtcontainers/factory/template/template_arm64.go

@@ -0,0 +1,14 @@
+// Copyright (c) 2019 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// template implements base vm factory with vm templating.
+
+package template
+
+// templateDeviceStateSize denotes device state size when
+// mount tmpfs.
+// when bypass-shared-memory is not support like arm64,
+// creating template will occupy more space. That's why we
+// put it here.
+const templateDeviceStateSize = 300

virtcontainers/factory/template/template_ppc64le.go

@@ -0,0 +1,14 @@
+// Copyright (c) 2019 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// template implements base vm factory with vm templating.
+
+package template
+
+// templateDeviceStateSize denotes device state size when
+// mount tmpfs.
+// when bypass-shared-memory is not support like arm64,
+// creating template will occupy more space. That's why we
+// put it here.
+const templateDeviceStateSize = 8

virtcontainers/factory/template/template_s390x.go

@@ -0,0 +1,14 @@
+// Copyright (c) 2019 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// template implements base vm factory with vm templating.
+
+package template
+
+// templateDeviceStateSize denotes device state size when
+// mount tmpfs.
+// when bypass-shared-memory is not support like arm64,
+// creating template will occupy more space. That's why we
+// put it here.
+const templateDeviceStateSize = 8

virtcontainers/qemu.go

@@ -89,10 +89,8 @@ const (
 	consoleSocket = "console.sock"
 	qmpSocket     = "qmp.sock"
 
-	qmpCapErrMsg                      = "Failed to negoatiate QMP capabilities"
-	qmpCapMigrationBypassSharedMemory = "bypass-shared-memory"
-	qmpExecCatCmd                     = "exec:cat"
-	qmpMigrationWaitTimeout           = 5 * time.Second
+	qmpCapErrMsg  = "Failed to negoatiate QMP capabilities"
+	qmpExecCatCmd = "exec:cat"
 
 	scsiControllerID  = "scsi0"
 	rngID             = "rng0"
@@ -1320,12 +1318,7 @@ func (q *qemu) saveSandbox() error {
 	// BootToBeTemplate sets the VM to be a template that other VMs can clone from. We would want to
 	// bypass shared memory when saving the VM to a local file through migration exec.
 	if q.config.BootToBeTemplate {
-		err = q.qmpMonitorCh.qmp.ExecSetMigrationCaps(q.qmpMonitorCh.ctx, []map[string]interface{}{
-			{
-				"capability": qmpCapMigrationBypassSharedMemory,
-				"state":      true,
-			},
-		})
+		err := q.arch.setBypassSharedMemoryMigrationCaps(q.qmpMonitorCh.ctx, q.qmpMonitorCh.qmp)
 		if err != nil {
 			q.Logger().WithError(err).Error("set migration bypass shared memory")
 			return err

virtcontainers/qemu_amd64.go

@@ -7,6 +7,7 @@ package virtcontainers
 
 import (
 	"os"
+	"time"
 
 	"github.com/kata-containers/runtime/virtcontainers/types"
 
@@ -24,6 +25,10 @@ const defaultQemuMachineType = QemuPC
 
 const defaultQemuMachineOptions = "accel=kvm,kernel_irqchip,nvdimm"
 
+const qmpCapMigrationBypassSharedMemory = "bypass-shared-memory"
+
+const qmpMigrationWaitTimeout = 5 * time.Second
+
 var qemuPaths = map[string]string{
 	QemuPCLite: "/usr/bin/qemu-lite-system-x86_64",
 	QemuPC:     defaultQemuPath,

virtcontainers/qemu_arch_base.go

@@ -6,6 +6,7 @@
 package virtcontainers
 
 import (
+	"context"
 	"encoding/hex"
 	"fmt"
 	"os"
@@ -100,6 +101,9 @@ type qemuArch interface {
 
 	// supportGuestMemoryHotplug returns if the guest supports memory hotplug
 	supportGuestMemoryHotplug() bool
+
+	// setBypassSharedMemoryMigrationCaps set bypass-shared-memory capability for migration
+	setBypassSharedMemoryMigrationCaps(context.Context, *govmmQemu.QMP) error
 }
 
 type qemuArchBase struct {
@@ -570,3 +574,13 @@ func (q *qemuArchBase) handleImagePath(config HypervisorConfig) {
 func (q *qemuArchBase) supportGuestMemoryHotplug() bool {
 	return true
 }
+
+func (q *qemuArchBase) setBypassSharedMemoryMigrationCaps(ctx context.Context, qmp *govmmQemu.QMP) error {
+	err := qmp.ExecSetMigrationCaps(ctx, []map[string]interface{}{
+		{
+			"capability": qmpCapMigrationBypassSharedMemory,
+			"state":      true,
+		},
+	})
+	return err
+}

virtcontainers/qemu_arm64.go

@@ -6,10 +6,12 @@
 package virtcontainers
 
 import (
+	"context"
 	"io/ioutil"
 	"os"
 	"runtime"
 	"strings"
+	"time"
 
 	govmmQemu "github.com/intel/govmm/qemu"
 	"github.com/kata-containers/runtime/virtcontainers/types"
@@ -25,6 +27,10 @@ const defaultQemuPath = "/usr/bin/qemu-system-aarch64"
 
 const defaultQemuMachineType = QemuVirt
 
+const qmpMigrationWaitTimeout = 10 * time.Second
+
+const qmpCapMigrationBypassSharedMemory = "bypass-shared-memory"
+
 var defaultQemuMachineOptions = "usb=off,accel=kvm,nvdimm,gic-version=" + getGuestGICVersion()
 
 var qemuPaths = map[string]string{
@@ -189,3 +195,8 @@ func (q *qemuArm64) appendImage(devices []govmmQemu.Device, path string) ([]govm
 
 	return devices, nil
 }
+
+func (q *qemuArm64) setBypassSharedMemoryMigrationCaps(_ context.Context, _ *govmmQemu.QMP) error {
+	// bypass-shared-memory not support in arm64 for now
+	return nil
+}

virtcontainers/qemu_ppc64le.go

@@ -8,6 +8,7 @@ package virtcontainers
 import (
 	"encoding/hex"
 	"os"
+	"time"
 
 	govmmQemu "github.com/intel/govmm/qemu"
 	deviceConfig "github.com/kata-containers/runtime/virtcontainers/device/config"
@@ -29,6 +30,10 @@ const defaultQemuMachineOptions = "accel=kvm,usb=off"
 
 const defaultMemMaxPPC64le = 32256 // Restrict MemMax to 32Gb on PPC64le
 
+const qmpCapMigrationBypassSharedMemory = "bypass-shared-memory"
+
+const qmpMigrationWaitTimeout = 5 * time.Second
+
 var qemuPaths = map[string]string{
 	QemuPseries: defaultQemuPath,
 }

virtcontainers/qemu_s390x.go

@@ -7,6 +7,7 @@ package virtcontainers
 
 import (
 	"fmt"
+	"time"
 	govmmQemu "github.com/intel/govmm/qemu"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/types"
@@ -25,6 +26,10 @@ const defaultQemuMachineOptions = "accel=kvm"
 
 const virtioSerialCCW = "virtio-serial-ccw"
 
+const qmpCapMigrationBypassSharedMemory = "bypass-shared-memory"
+
+const qmpMigrationWaitTimeout = 5 * time.Second
+
 var qemuPaths = map[string]string{
 	QemuCCWVirtio: defaultQemuPath,
 }