diff --git a/cli/config/configuration.toml.in b/cli/config/configuration.toml.in index 8efd4bd9d6..e9062928ce 100644 --- a/cli/config/configuration.toml.in +++ b/cli/config/configuration.toml.in @@ -82,6 +82,13 @@ default_memory = @DEFMEMSZ@ # This is will determine the times that memory will be hotadded to sandbox/VM. #memory_slots = @DEFMEMSLOTS@ +# The size in MiB will be plused to max memory of hypervisor. +# It is the memory address space for the NVDIMM devie. +# If set block storage driver (block_device_driver) to "nvdimm", +# should set memory_offset to the size of block device. +# Default 0 +#memory_offset = 0 + # Disable block device from being used for a container's rootfs. # In case of a storage driver like devicemapper where a container's # root file system is backed by a block device, the block device is passed diff --git a/pkg/katautils/config-settings.go b/pkg/katautils/config-settings.go index 8dcf6ac370..b85efcb5f3 100644 --- a/pkg/katautils/config-settings.go +++ b/pkg/katautils/config-settings.go @@ -24,6 +24,7 @@ const defaultVCPUCount uint32 = 1 const defaultMaxVCPUCount uint32 = 0 const defaultMemSize uint32 = 2048 // MiB const defaultMemSlots uint32 = 10 +const defaultMemOffset uint32 = 0 // MiB const defaultBridgesCount uint32 = 1 const defaultInterNetworkingModel = "macvtap" const defaultDisableBlockDeviceUse bool = false diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index 3bd697dec7..00912bdf09 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -98,6 +98,7 @@ type hypervisor struct { DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` MemorySize uint32 `toml:"default_memory"` MemSlots uint32 `toml:"memory_slots"` + MemOffset uint32 `toml:"memory_offset"` DefaultBridges uint32 `toml:"default_bridges"` Msize9p uint32 `toml:"msize_9p"` DisableBlockDeviceUse bool `toml:"disable_block_device_use"` @@ -281,6 +282,15 @@ func (h hypervisor) defaultMemSlots() uint32 { return slots } +func (h hypervisor) defaultMemOffset() uint32 { + offset := h.MemOffset + if offset == 0 { + offset = defaultMemOffset + } + + return offset +} + func (h hypervisor) defaultBridges() uint32 { if h.DefaultBridges == 0 { return defaultBridgesCount @@ -514,6 +524,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { DefaultMaxVCPUs: h.defaultMaxVCPUs(), MemorySize: h.defaultMemSz(), MemSlots: h.defaultMemSlots(), + MemOffset: h.defaultMemOffset(), EntropySource: h.GetEntropySource(), DefaultBridges: h.defaultBridges(), DisableBlockDeviceUse: h.DisableBlockDeviceUse, @@ -677,6 +688,7 @@ func initConfig() (config oci.RuntimeConfig, err error) { NumVCPUs: defaultVCPUCount, DefaultMaxVCPUs: defaultMaxVCPUCount, MemorySize: defaultMemSize, + MemOffset: defaultMemOffset, DefaultBridges: defaultBridgesCount, MemPrealloc: defaultEnableMemPrealloc, HugePages: defaultEnableHugePages, diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index b218b01694..bff95df32f 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -169,6 +169,9 @@ type HypervisorConfig struct { // MemSlots specifies default memory slots the VM. MemSlots uint32 + // MemOffset specifies memory space for nvdimm device + MemOffset uint32 + // KernelParams are additional guest kernel parameters. KernelParams []Param diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 536e76dfeb..9c79c20b69 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -1435,12 +1435,11 @@ func genericBridges(number uint32, machineType string) []Bridge { return bridges } -func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8) govmmQemu.Memory { - // NVDIMM device needs memory space 1024MB +func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8, memoryOffset uint32) govmmQemu.Memory { + // image NVDIMM device needs memory space 1024MB // See https://github.com/clearcontainers/runtime/issues/380 - memoryOffset := 1024 + memoryOffset += 1024 - // add 1G memory space for nvdimm device (vm guest image) memMax := fmt.Sprintf("%dM", hostMemoryMb+uint64(memoryOffset)) mem := fmt.Sprintf("%dM", memoryMb) diff --git a/virtcontainers/qemu_amd64.go b/virtcontainers/qemu_amd64.go index 8dfddde932..7e3fc9d3de 100644 --- a/virtcontainers/qemu_amd64.go +++ b/virtcontainers/qemu_amd64.go @@ -87,6 +87,7 @@ func newQemuArch(config HypervisorConfig) qemuArch { q := &qemuAmd64{ qemuArchBase{ machineType: machineType, + memoryOffset: config.MemOffset, qemuPaths: qemuPaths, supportedQemuMachines: supportedQemuMachines, kernelParamsNonDebug: kernelParamsNonDebug, @@ -96,6 +97,7 @@ func newQemuArch(config HypervisorConfig) qemuArch { } q.handleImagePath(config) + return q } @@ -126,7 +128,7 @@ func (q *qemuAmd64) cpuModel() string { } func (q *qemuAmd64) memoryTopology(memoryMb, hostMemoryMb uint64, slots uint8) govmmQemu.Memory { - return genericMemoryTopology(memoryMb, hostMemoryMb, slots) + return genericMemoryTopology(memoryMb, hostMemoryMb, slots, q.memoryOffset) } func (q *qemuAmd64) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) { diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index 53e952410c..7c098f7b53 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -103,6 +103,7 @@ type qemuArch interface { type qemuArchBase struct { machineType string + memoryOffset uint32 nestedRun bool vhost bool networkIndex int diff --git a/virtcontainers/qemu_arm64.go b/virtcontainers/qemu_arm64.go index 64fb285822..bc72b0fa79 100644 --- a/virtcontainers/qemu_arm64.go +++ b/virtcontainers/qemu_arm64.go @@ -136,6 +136,7 @@ func newQemuArch(config HypervisorConfig) qemuArch { q := &qemuArm64{ qemuArchBase{ machineType: machineType, + memoryOffset: config.MemOffset, qemuPaths: qemuPaths, supportedQemuMachines: supportedQemuMachines, kernelParamsNonDebug: kernelParamsNonDebug, diff --git a/virtcontainers/qemu_ppc64le.go b/virtcontainers/qemu_ppc64le.go index 05d37ce755..ca69e2d831 100644 --- a/virtcontainers/qemu_ppc64le.go +++ b/virtcontainers/qemu_ppc64le.go @@ -74,6 +74,7 @@ func newQemuArch(config HypervisorConfig) qemuArch { q := &qemuPPC64le{ qemuArchBase{ machineType: machineType, + memoryOffset: config.MemOffset, qemuPaths: qemuPaths, supportedQemuMachines: supportedQemuMachines, kernelParamsNonDebug: kernelParamsNonDebug, @@ -83,6 +84,9 @@ func newQemuArch(config HypervisorConfig) qemuArch { } q.handleImagePath(config) + + q.memoryOffset = config.MemOffset + return q } @@ -121,7 +125,7 @@ func (q *qemuPPC64le) memoryTopology(memoryMb, hostMemoryMb uint64, slots uint8) hostMemoryMb = defaultMemMaxPPC64le } - return genericMemoryTopology(memoryMb, hostMemoryMb, slots) + return genericMemoryTopology(memoryMb, hostMemoryMb, slots, q.memoryOffset) } func (q *qemuPPC64le) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) { diff --git a/virtcontainers/qemu_s390x.go b/virtcontainers/qemu_s390x.go index 893b2dd62c..cd87d681e6 100644 --- a/virtcontainers/qemu_s390x.go +++ b/virtcontainers/qemu_s390x.go @@ -61,6 +61,7 @@ func newQemuArch(config HypervisorConfig) qemuArch { q := &qemuS390x{ qemuArchBase{ machineType: machineType, + memoryOffset: config.MemOffset, qemuPaths: qemuPaths, supportedQemuMachines: supportedQemuMachines, kernelParamsNonDebug: kernelParamsNonDebug,