memory: Add new option memory_offset

This value will be plused to max memory of hypervisor.
It is the memory address space for the NVDIMM devie.
If set block storage driver (block_device_driver) to "nvdimm",
should set memory_offset to the size of block device.

Signed-off-by: Hui Zhu <teawater@hyper.sh>

cli/config/configuration.toml.in

@@ -82,6 +82,13 @@ default_memory = @DEFMEMSZ@
 # This is will determine the times that memory will be hotadded to sandbox/VM.
 #memory_slots = @DEFMEMSLOTS@
 
+# The size in MiB will be plused to max memory of hypervisor.
+# It is the memory address space for the NVDIMM devie.
+# If set block storage driver (block_device_driver) to "nvdimm",
+# should set memory_offset to the size of block device.
+# Default 0
+#memory_offset = 0
+
 # Disable block device from being used for a container's rootfs.
 # In case of a storage driver like devicemapper where a container's 
 # root file system is backed by a block device, the block device is passed

pkg/katautils/config-settings.go

@@ -24,6 +24,7 @@ const defaultVCPUCount uint32 = 1
 const defaultMaxVCPUCount uint32 = 0
 const defaultMemSize uint32 = 2048 // MiB
 const defaultMemSlots uint32 = 10
+const defaultMemOffset uint32 = 0 // MiB
 const defaultBridgesCount uint32 = 1
 const defaultInterNetworkingModel = "macvtap"
 const defaultDisableBlockDeviceUse bool = false

pkg/katautils/config.go

@@ -98,6 +98,7 @@ type hypervisor struct {
 	DefaultMaxVCPUs         uint32 `toml:"default_maxvcpus"`
 	MemorySize              uint32 `toml:"default_memory"`
 	MemSlots                uint32 `toml:"memory_slots"`
+	MemOffset               uint32 `toml:"memory_offset"`
 	DefaultBridges          uint32 `toml:"default_bridges"`
 	Msize9p                 uint32 `toml:"msize_9p"`
 	DisableBlockDeviceUse   bool   `toml:"disable_block_device_use"`
@@ -281,6 +282,15 @@ func (h hypervisor) defaultMemSlots() uint32 {
 	return slots
 }
 
+func (h hypervisor) defaultMemOffset() uint32 {
+	offset := h.MemOffset
+	if offset == 0 {
+		offset = defaultMemOffset
+	}
+
+	return offset
+}
+
 func (h hypervisor) defaultBridges() uint32 {
 	if h.DefaultBridges == 0 {
 		return defaultBridgesCount
@@ -514,6 +524,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		DefaultMaxVCPUs:         h.defaultMaxVCPUs(),
 		MemorySize:              h.defaultMemSz(),
 		MemSlots:                h.defaultMemSlots(),
+		MemOffset:               h.defaultMemOffset(),
 		EntropySource:           h.GetEntropySource(),
 		DefaultBridges:          h.defaultBridges(),
 		DisableBlockDeviceUse:   h.DisableBlockDeviceUse,
@@ -677,6 +688,7 @@ func initConfig() (config oci.RuntimeConfig, err error) {
 		NumVCPUs:                defaultVCPUCount,
 		DefaultMaxVCPUs:         defaultMaxVCPUCount,
 		MemorySize:              defaultMemSize,
+		MemOffset:               defaultMemOffset,
 		DefaultBridges:          defaultBridgesCount,
 		MemPrealloc:             defaultEnableMemPrealloc,
 		HugePages:               defaultEnableHugePages,

virtcontainers/hypervisor.go

@@ -169,6 +169,9 @@ type HypervisorConfig struct {
 	// MemSlots specifies default memory slots the VM.
 	MemSlots uint32
 
+	// MemOffset specifies memory space for nvdimm device
+	MemOffset uint32
+
 	// KernelParams are additional guest kernel parameters.
 	KernelParams []Param
 

virtcontainers/qemu.go

@@ -1435,12 +1435,11 @@ func genericBridges(number uint32, machineType string) []Bridge {
 	return bridges
 }
 
-func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8) govmmQemu.Memory {
+func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8, memoryOffset uint32) govmmQemu.Memory {
-	// NVDIMM device needs memory space 1024MB
+	// image NVDIMM device needs memory space 1024MB
 	// See https://github.com/clearcontainers/runtime/issues/380
-	memoryOffset := 1024
+	memoryOffset += 1024
 
-	// add 1G memory space for nvdimm device (vm guest image)
 	memMax := fmt.Sprintf("%dM", hostMemoryMb+uint64(memoryOffset))
 
 	mem := fmt.Sprintf("%dM", memoryMb)

virtcontainers/qemu_amd64.go

@@ -87,6 +87,7 @@ func newQemuArch(config HypervisorConfig) qemuArch {
 	q := &qemuAmd64{
 		qemuArchBase{
 			machineType:           machineType,
+			memoryOffset:          config.MemOffset,
 			qemuPaths:             qemuPaths,
 			supportedQemuMachines: supportedQemuMachines,
 			kernelParamsNonDebug:  kernelParamsNonDebug,
@@ -96,6 +97,7 @@ func newQemuArch(config HypervisorConfig) qemuArch {
 	}
 
 	q.handleImagePath(config)
+
 	return q
 }
 
@@ -126,7 +128,7 @@ func (q *qemuAmd64) cpuModel() string {
 }
 
 func (q *qemuAmd64) memoryTopology(memoryMb, hostMemoryMb uint64, slots uint8) govmmQemu.Memory {
-	return genericMemoryTopology(memoryMb, hostMemoryMb, slots)
+	return genericMemoryTopology(memoryMb, hostMemoryMb, slots, q.memoryOffset)
 }
 
 func (q *qemuAmd64) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) {

virtcontainers/qemu_arch_base.go

@@ -103,6 +103,7 @@ type qemuArch interface {
 
 type qemuArchBase struct {
 	machineType           string
+	memoryOffset          uint32
 	nestedRun             bool
 	vhost                 bool
 	networkIndex          int

virtcontainers/qemu_arm64.go

@@ -136,6 +136,7 @@ func newQemuArch(config HypervisorConfig) qemuArch {
 	q := &qemuArm64{
 		qemuArchBase{
 			machineType:           machineType,
+			memoryOffset:          config.MemOffset,
 			qemuPaths:             qemuPaths,
 			supportedQemuMachines: supportedQemuMachines,
 			kernelParamsNonDebug:  kernelParamsNonDebug,

virtcontainers/qemu_ppc64le.go

@@ -74,6 +74,7 @@ func newQemuArch(config HypervisorConfig) qemuArch {
 	q := &qemuPPC64le{
 		qemuArchBase{
 			machineType:           machineType,
+			memoryOffset:          config.MemOffset,
 			qemuPaths:             qemuPaths,
 			supportedQemuMachines: supportedQemuMachines,
 			kernelParamsNonDebug:  kernelParamsNonDebug,
@@ -83,6 +84,9 @@ func newQemuArch(config HypervisorConfig) qemuArch {
 	}
 
 	q.handleImagePath(config)
+
+	q.memoryOffset = config.MemOffset
+
 	return q
 }
 
@@ -121,7 +125,7 @@ func (q *qemuPPC64le) memoryTopology(memoryMb, hostMemoryMb uint64, slots uint8)
 		hostMemoryMb = defaultMemMaxPPC64le
 	}
 
-	return genericMemoryTopology(memoryMb, hostMemoryMb, slots)
+	return genericMemoryTopology(memoryMb, hostMemoryMb, slots, q.memoryOffset)
 }
 
 func (q *qemuPPC64le) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) {

virtcontainers/qemu_s390x.go

@@ -61,6 +61,7 @@ func newQemuArch(config HypervisorConfig) qemuArch {
 	q := &qemuS390x{
 		qemuArchBase{
 			machineType:           machineType,
+			memoryOffset:          config.MemOffset,
 			qemuPaths:             qemuPaths,
 			supportedQemuMachines: supportedQemuMachines,
 			kernelParamsNonDebug:  kernelParamsNonDebug,