From afdc9604249336a0715b6c28717bb628e535db2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 27 Jun 2022 15:18:27 +0200 Subject: [PATCH 1/4] hypervisor: Add default_maxmemory configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's add a `default_maxmemory` configuration, which allows the admins to set the maximum amount of memory to be used by a VM, considering the initial amount + whatever ends up being hotplugged via the pod limits. By default this value is 0 (zero), and it means that the whole physical RAM is the limit. Fixes: #4516 Signed-off-by: Fabiano Fidêncio --- src/runtime/go.mod | 1 + src/runtime/go.sum | 2 + src/runtime/pkg/katatestutils/utils.go | 1 + src/runtime/pkg/katautils/config.go | 20 +++++++ src/runtime/pkg/katautils/config_test.go | 4 ++ .../vendor/github.com/pbnjay/memory/LICENSE | 29 +++++++++ .../vendor/github.com/pbnjay/memory/README.md | 41 +++++++++++++ .../vendor/github.com/pbnjay/memory/doc.go | 24 ++++++++ .../vendor/github.com/pbnjay/memory/go.mod | 3 + .../github.com/pbnjay/memory/memory_bsd.go | 19 ++++++ .../github.com/pbnjay/memory/memory_darwin.go | 49 +++++++++++++++ .../github.com/pbnjay/memory/memory_linux.go | 29 +++++++++ .../pbnjay/memory/memory_windows.go | 60 +++++++++++++++++++ .../github.com/pbnjay/memory/memsysctl.go | 21 +++++++ .../vendor/github.com/pbnjay/memory/stub.go | 10 ++++ src/runtime/vendor/modules.txt | 3 + src/runtime/virtcontainers/hypervisor.go | 3 + 17 files changed, 319 insertions(+) create mode 100644 src/runtime/vendor/github.com/pbnjay/memory/LICENSE create mode 100644 src/runtime/vendor/github.com/pbnjay/memory/README.md create mode 100644 src/runtime/vendor/github.com/pbnjay/memory/doc.go create mode 100644 src/runtime/vendor/github.com/pbnjay/memory/go.mod create mode 100644 src/runtime/vendor/github.com/pbnjay/memory/memory_bsd.go create mode 100644 src/runtime/vendor/github.com/pbnjay/memory/memory_darwin.go create mode 100644 src/runtime/vendor/github.com/pbnjay/memory/memory_linux.go create mode 100644 src/runtime/vendor/github.com/pbnjay/memory/memory_windows.go create mode 100644 src/runtime/vendor/github.com/pbnjay/memory/memsysctl.go create mode 100644 src/runtime/vendor/github.com/pbnjay/memory/stub.go diff --git a/src/runtime/go.mod b/src/runtime/go.mod index 14f0f3ecf3..36a2f618b9 100644 --- a/src/runtime/go.mod +++ b/src/runtime/go.mod @@ -33,6 +33,7 @@ require ( github.com/opencontainers/runc v1.1.2 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 github.com/opencontainers/selinux v1.10.1 + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.1 github.com/prometheus/client_model v0.2.0 diff --git a/src/runtime/go.sum b/src/runtime/go.sum index 86868038e9..63870b2325 100644 --- a/src/runtime/go.sum +++ b/src/runtime/go.sum @@ -767,6 +767,8 @@ github.com/opencontainers/selinux v1.10.1 h1:09LIPVRP3uuZGQvgR+SgMSNBd1Eb3vlRbGq github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= diff --git a/src/runtime/pkg/katatestutils/utils.go b/src/runtime/pkg/katatestutils/utils.go index 527c9cfbc7..5676f4451c 100644 --- a/src/runtime/pkg/katatestutils/utils.go +++ b/src/runtime/pkg/katatestutils/utils.go @@ -226,6 +226,7 @@ type RuntimeConfigOptions struct { DefaultVCPUCount uint32 DefaultMaxVCPUCount uint32 DefaultMemSize uint32 + DefaultMaxMemorySize uint64 DefaultMsize9p uint32 DisableBlock bool EnableIOThreads bool diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index dbdfdac303..c0ffb99267 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -24,6 +24,7 @@ import ( vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" + "github.com/pbnjay/memory" "github.com/sirupsen/logrus" ) @@ -121,6 +122,7 @@ type hypervisor struct { DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` MemorySize uint32 `toml:"default_memory"` MemSlots uint32 `toml:"memory_slots"` + DefaultMaxMemorySize uint64 `toml:"default_maxmemory"` DefaultBridges uint32 `toml:"default_bridges"` Msize9p uint32 `toml:"msize_9p"` PCIeRootPort uint32 `toml:"pcie_root_port"` @@ -400,6 +402,20 @@ func (h hypervisor) defaultMemOffset() uint64 { return offset } +func (h hypervisor) defaultMaxMemSz() uint64 { + hostMemory := memory.TotalMemory() / 1024 / 1024 //MiB + + if h.DefaultMaxMemorySize == 0 { + return hostMemory + } + + if h.DefaultMaxMemorySize > hostMemory { + return hostMemory + } + + return h.DefaultMaxMemorySize +} + func (h hypervisor) defaultBridges() uint32 { if h.DefaultBridges == 0 { return defaultBridgesCount @@ -635,6 +651,7 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { DefaultMaxVCPUs: h.defaultMaxVCPUs(), MemorySize: h.defaultMemSz(), MemSlots: h.defaultMemSlots(), + DefaultMaxMemorySize: h.defaultMaxMemSz(), EntropySource: h.GetEntropySource(), EntropySourceList: h.EntropySourceList, DefaultBridges: h.defaultBridges(), @@ -736,6 +753,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { MemorySize: h.defaultMemSz(), MemSlots: h.defaultMemSlots(), MemOffset: h.defaultMemOffset(), + DefaultMaxMemorySize: h.defaultMaxMemSz(), VirtioMem: h.VirtioMem, EntropySource: h.GetEntropySource(), EntropySourceList: h.EntropySourceList, @@ -833,6 +851,7 @@ func newAcrnHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { DefaultMaxVCPUs: h.defaultMaxVCPUs(), MemorySize: h.defaultMemSz(), MemSlots: h.defaultMemSlots(), + DefaultMaxMemorySize: h.defaultMaxMemSz(), EntropySource: h.GetEntropySource(), EntropySourceList: h.EntropySourceList, DefaultBridges: h.defaultBridges(), @@ -910,6 +929,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { MemorySize: h.defaultMemSz(), MemSlots: h.defaultMemSlots(), MemOffset: h.defaultMemOffset(), + DefaultMaxMemorySize: h.defaultMaxMemSz(), VirtioMem: h.VirtioMem, EntropySource: h.GetEntropySource(), EntropySourceList: h.EntropySourceList, diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index 0bcac2137f..65b541e73c 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -22,6 +22,7 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/pkg/oci" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" + "github.com/pbnjay/memory" "github.com/stretchr/testify/assert" ) @@ -85,6 +86,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf sharedFS := "virtio-9p" virtioFSdaemon := path.Join(dir, "virtiofsd") epcSize := int64(0) + maxMemory := uint64(memory.TotalMemory() / 1024 / 1024) configFileOptions := ktu.RuntimeConfigOptions{ Hypervisor: "qemu", @@ -104,6 +106,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf DefaultVCPUCount: defaultVCPUCount, DefaultMaxVCPUCount: defaultMaxVCPUCount, DefaultMemSize: defaultMemSize, + DefaultMaxMemorySize: maxMemory, DefaultMsize9p: defaultMsize9p, HypervisorDebug: hypervisorDebug, RuntimeDebug: runtimeDebug, @@ -153,6 +156,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf NumVCPUs: defaultVCPUCount, DefaultMaxVCPUs: getCurrentCpuNum(), MemorySize: defaultMemSize, + DefaultMaxMemorySize: maxMemory, DisableBlockDeviceUse: disableBlockDevice, BlockDeviceDriver: defaultBlockDeviceDriver, DefaultBridges: defaultBridgesCount, diff --git a/src/runtime/vendor/github.com/pbnjay/memory/LICENSE b/src/runtime/vendor/github.com/pbnjay/memory/LICENSE new file mode 100644 index 0000000000..63ca4a6d24 --- /dev/null +++ b/src/runtime/vendor/github.com/pbnjay/memory/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2017, Jeremy Jay +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/runtime/vendor/github.com/pbnjay/memory/README.md b/src/runtime/vendor/github.com/pbnjay/memory/README.md new file mode 100644 index 0000000000..e98f261a0f --- /dev/null +++ b/src/runtime/vendor/github.com/pbnjay/memory/README.md @@ -0,0 +1,41 @@ +# memory + +Package `memory` provides two methods reporting total physical system memory +accessible to the kernel, and free memory available to the running application. + +This package has no external dependency besides the standard library and default operating system tools. + +Documentation: +[![GoDoc](https://godoc.org/github.com/pbnjay/memory?status.svg)](https://godoc.org/github.com/pbnjay/memory) + +This is useful for dynamic code to minimize thrashing and other contention, similar to the stdlib `runtime.NumCPU` +See some history of the proposal at https://github.com/golang/go/issues/21816 + + +## Example + +```go +fmt.Printf("Total system memory: %d\n", memory.TotalMemory()) +fmt.Printf("Free memory: %d\n", memory.FreeMemory()) +``` + + +## Testing + +Tested/working on: + - macOS 10.12.6 (16G29), 10.15.7 (19H2) + - Windows 10 1511 (10586.1045) + - Linux RHEL (3.10.0-327.3.1.el7.x86_64) + - Raspberry Pi 3 (ARMv8) on Raspbian, ODROID-C1+ (ARMv7) on Ubuntu, C.H.I.P + (ARMv7). + - Amazon Linux 2 aarch64 (m6a.large, 4.14.203-156.332.amzn2.aarch64) + +Tested on virtual machines: + - Windows 7 SP1 386 + - Debian stretch 386 + - NetBSD 7.1 amd64 + 386 + - OpenBSD 6.1 amd64 + 386 + - FreeBSD 11.1 amd64 + 386 + - DragonFly BSD 4.8.1 amd64 + +If you have access to untested systems please test and report success/bugs. diff --git a/src/runtime/vendor/github.com/pbnjay/memory/doc.go b/src/runtime/vendor/github.com/pbnjay/memory/doc.go new file mode 100644 index 0000000000..4e4f984c0f --- /dev/null +++ b/src/runtime/vendor/github.com/pbnjay/memory/doc.go @@ -0,0 +1,24 @@ +// Package memory provides a single method reporting total system memory +// accessible to the kernel. +package memory + +// TotalMemory returns the total accessible system memory in bytes. +// +// The total accessible memory is installed physical memory size minus reserved +// areas for the kernel and hardware, if such reservations are reported by +// the operating system. +// +// If accessible memory size could not be determined, then 0 is returned. +func TotalMemory() uint64 { + return sysTotalMemory() +} + +// FreeMemory returns the total free system memory in bytes. +// +// The total free memory is installed physical memory size minus reserved +// areas for other applications running on the same system. +// +// If free memory size could not be determined, then 0 is returned. +func FreeMemory() uint64 { + return sysFreeMemory() +} diff --git a/src/runtime/vendor/github.com/pbnjay/memory/go.mod b/src/runtime/vendor/github.com/pbnjay/memory/go.mod new file mode 100644 index 0000000000..5965765912 --- /dev/null +++ b/src/runtime/vendor/github.com/pbnjay/memory/go.mod @@ -0,0 +1,3 @@ +module github.com/pbnjay/memory + +go 1.16 diff --git a/src/runtime/vendor/github.com/pbnjay/memory/memory_bsd.go b/src/runtime/vendor/github.com/pbnjay/memory/memory_bsd.go new file mode 100644 index 0000000000..49d808a9e7 --- /dev/null +++ b/src/runtime/vendor/github.com/pbnjay/memory/memory_bsd.go @@ -0,0 +1,19 @@ +// +build freebsd openbsd dragonfly netbsd + +package memory + +func sysTotalMemory() uint64 { + s, err := sysctlUint64("hw.physmem") + if err != nil { + return 0 + } + return s +} + +func sysFreeMemory() uint64 { + s, err := sysctlUint64("hw.usermem") + if err != nil { + return 0 + } + return s +} diff --git a/src/runtime/vendor/github.com/pbnjay/memory/memory_darwin.go b/src/runtime/vendor/github.com/pbnjay/memory/memory_darwin.go new file mode 100644 index 0000000000..a3f4576990 --- /dev/null +++ b/src/runtime/vendor/github.com/pbnjay/memory/memory_darwin.go @@ -0,0 +1,49 @@ +// +build darwin + +package memory + +import ( + "os/exec" + "regexp" + "strconv" +) + +func sysTotalMemory() uint64 { + s, err := sysctlUint64("hw.memsize") + if err != nil { + return 0 + } + return s +} + +func sysFreeMemory() uint64 { + cmd := exec.Command("vm_stat") + outBytes, err := cmd.Output() + if err != nil { + return 0 + } + + rePageSize := regexp.MustCompile("page size of ([0-9]*) bytes") + reFreePages := regexp.MustCompile("Pages free: *([0-9]*)\\.") + + // default: page size of 4096 bytes + matches := rePageSize.FindSubmatchIndex(outBytes) + pageSize := uint64(4096) + if len(matches) == 4 { + pageSize, err = strconv.ParseUint(string(outBytes[matches[2]:matches[3]]), 10, 64) + if err != nil { + return 0 + } + } + + // ex: Pages free: 1126961. + matches = reFreePages.FindSubmatchIndex(outBytes) + freePages := uint64(0) + if len(matches) == 4 { + freePages, err = strconv.ParseUint(string(outBytes[matches[2]:matches[3]]), 10, 64) + if err != nil { + return 0 + } + } + return freePages * pageSize +} diff --git a/src/runtime/vendor/github.com/pbnjay/memory/memory_linux.go b/src/runtime/vendor/github.com/pbnjay/memory/memory_linux.go new file mode 100644 index 0000000000..3d07711ce5 --- /dev/null +++ b/src/runtime/vendor/github.com/pbnjay/memory/memory_linux.go @@ -0,0 +1,29 @@ +// +build linux + +package memory + +import "syscall" + +func sysTotalMemory() uint64 { + in := &syscall.Sysinfo_t{} + err := syscall.Sysinfo(in) + if err != nil { + return 0 + } + // If this is a 32-bit system, then these fields are + // uint32 instead of uint64. + // So we always convert to uint64 to match signature. + return uint64(in.Totalram) * uint64(in.Unit) +} + +func sysFreeMemory() uint64 { + in := &syscall.Sysinfo_t{} + err := syscall.Sysinfo(in) + if err != nil { + return 0 + } + // If this is a 32-bit system, then these fields are + // uint32 instead of uint64. + // So we always convert to uint64 to match signature. + return uint64(in.Freeram) * uint64(in.Unit) +} diff --git a/src/runtime/vendor/github.com/pbnjay/memory/memory_windows.go b/src/runtime/vendor/github.com/pbnjay/memory/memory_windows.go new file mode 100644 index 0000000000..c8500cc6f3 --- /dev/null +++ b/src/runtime/vendor/github.com/pbnjay/memory/memory_windows.go @@ -0,0 +1,60 @@ +// +build windows + +package memory + +import ( + "syscall" + "unsafe" +) + +// omitting a few fields for brevity... +// https://msdn.microsoft.com/en-us/library/windows/desktop/aa366589(v=vs.85).aspx +type memStatusEx struct { + dwLength uint32 + dwMemoryLoad uint32 + ullTotalPhys uint64 + ullAvailPhys uint64 + unused [5]uint64 +} + +func sysTotalMemory() uint64 { + kernel32, err := syscall.LoadDLL("kernel32.dll") + if err != nil { + return 0 + } + // GetPhysicallyInstalledSystemMemory is simpler, but broken on + // older versions of windows (and uses this under the hood anyway). + globalMemoryStatusEx, err := kernel32.FindProc("GlobalMemoryStatusEx") + if err != nil { + return 0 + } + msx := &memStatusEx{ + dwLength: 64, + } + r, _, _ := globalMemoryStatusEx.Call(uintptr(unsafe.Pointer(msx))) + if r == 0 { + return 0 + } + return msx.ullTotalPhys +} + +func sysFreeMemory() uint64 { + kernel32, err := syscall.LoadDLL("kernel32.dll") + if err != nil { + return 0 + } + // GetPhysicallyInstalledSystemMemory is simpler, but broken on + // older versions of windows (and uses this under the hood anyway). + globalMemoryStatusEx, err := kernel32.FindProc("GlobalMemoryStatusEx") + if err != nil { + return 0 + } + msx := &memStatusEx{ + dwLength: 64, + } + r, _, _ := globalMemoryStatusEx.Call(uintptr(unsafe.Pointer(msx))) + if r == 0 { + return 0 + } + return msx.ullAvailPhys +} diff --git a/src/runtime/vendor/github.com/pbnjay/memory/memsysctl.go b/src/runtime/vendor/github.com/pbnjay/memory/memsysctl.go new file mode 100644 index 0000000000..438d9eff8e --- /dev/null +++ b/src/runtime/vendor/github.com/pbnjay/memory/memsysctl.go @@ -0,0 +1,21 @@ +// +build darwin freebsd openbsd dragonfly netbsd + +package memory + +import ( + "syscall" + "unsafe" +) + +func sysctlUint64(name string) (uint64, error) { + s, err := syscall.Sysctl(name) + if err != nil { + return 0, err + } + // hack because the string conversion above drops a \0 + b := []byte(s) + if len(b) < 8 { + b = append(b, 0) + } + return *(*uint64)(unsafe.Pointer(&b[0])), nil +} diff --git a/src/runtime/vendor/github.com/pbnjay/memory/stub.go b/src/runtime/vendor/github.com/pbnjay/memory/stub.go new file mode 100644 index 0000000000..f29473ba08 --- /dev/null +++ b/src/runtime/vendor/github.com/pbnjay/memory/stub.go @@ -0,0 +1,10 @@ +// +build !linux,!darwin,!windows,!freebsd,!dragonfly,!netbsd,!openbsd + +package memory + +func sysTotalMemory() uint64 { + return 0 +} +func sysFreeMemory() uint64 { + return 0 +} diff --git a/src/runtime/vendor/modules.txt b/src/runtime/vendor/modules.txt index 8c27d91df4..fb015d1c20 100644 --- a/src/runtime/vendor/modules.txt +++ b/src/runtime/vendor/modules.txt @@ -259,6 +259,9 @@ github.com/opencontainers/selinux/go-selinux github.com/opencontainers/selinux/go-selinux/label github.com/opencontainers/selinux/pkg/pwalk github.com/opencontainers/selinux/pkg/pwalkdir +# github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 +## explicit +github.com/pbnjay/memory # github.com/pkg/errors v0.9.1 ## explicit github.com/pkg/errors diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 119c4667c3..e39885dac8 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -441,6 +441,9 @@ type HypervisorConfig struct { // DefaultMem specifies default memory size in MiB for the VM. MemorySize uint32 + // DefaultMaxMemorySize specifies the maximum amount of RAM in MiB for the VM. + DefaultMaxMemorySize uint64 + // DefaultBridges specifies default number of bridges for the VM. // Bridges can be used to hot plug devices DefaultBridges uint32 From 58ff2bd5c9b11cce8b62470afc633eeb87a3ef51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 27 Jun 2022 22:09:21 +0200 Subject: [PATCH 2/4] clh,qemu: Adapt to using default_maxmemory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's adapt Cloud Hypervisor's and QEMU's code to properly behave to the newly added `default_maxmemory` config. While implementing this, a change of behaviour (or a bug fix, depending on how you see it) has been introduced as if a pod requests more memory than the amount avaiable in the host, instead of failing to start the pod, we simply hotplug the maximum amount of memory available, mimicing better the runc behaviour. Fixes: #4516 Signed-off-by: Fabiano Fidêncio --- src/runtime/virtcontainers/clh.go | 12 +++++----- src/runtime/virtcontainers/qemu.go | 29 ++++++------------------- src/runtime/virtcontainers/qemu_test.go | 11 +++++----- 3 files changed, 20 insertions(+), 32 deletions(-) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 6984b73be5..801d53a9bd 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -480,12 +480,9 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net // Enable hugepages if needed clh.vmconfig.Memory.Hugepages = func(b bool) *bool { return &b }(clh.config.HugePages) if !clh.config.ConfidentialGuest { - hostMemKb, err := GetHostMemorySizeKb(procMemInfo) - if err != nil { - return nil - } + hotplugSize := clh.config.DefaultMaxMemorySize // OpenAPI only supports int64 values - clh.vmconfig.Memory.HotplugSize = func(i int64) *int64 { return &i }(int64((utils.MemUnit(hostMemKb) * utils.KiB).ToBytes())) + clh.vmconfig.Memory.HotplugSize = func(i int64) *int64 { return &i }(int64((utils.MemUnit(hotplugSize) * utils.MiB).ToBytes())) } // Set initial amount of cpu's for the virtual machine clh.vmconfig.Cpus = chclient.NewCpusConfig(int32(clh.config.NumVCPUs), int32(clh.config.DefaultMaxVCPUs)) @@ -882,6 +879,11 @@ func (clh *cloudHypervisor) ResizeMemory(ctx context.Context, reqMemMB uint32, m return 0, MemoryDevice{}, err } + maxHotplugSize := utils.MemUnit(*info.Config.Memory.HotplugSize) * utils.Byte + if reqMemMB > uint32(maxHotplugSize.ToMiB()) { + reqMemMB = uint32(maxHotplugSize.ToMiB()) + } + currentMem := utils.MemUnit(info.Config.Memory.Size) * utils.Byte newMem := utils.MemUnit(reqMemMB) * utils.MiB diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index c8e48953f6..f2c501ac3a 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -315,11 +315,7 @@ func (q *qemu) hostMemMB() (uint64, error) { } func (q *qemu) memoryTopology() (govmmQemu.Memory, error) { - hostMemMb, err := q.hostMemMB() - if err != nil { - return govmmQemu.Memory{}, err - } - + hostMemMb := q.config.DefaultMaxMemorySize memMb := uint64(q.config.MemorySize) return q.arch.memoryTopology(memMb, hostMemMb, uint8(q.config.MemSlots)), nil @@ -779,12 +775,8 @@ func (q *qemu) getMemArgs() (bool, string, string, error) { } func (q *qemu) setupVirtioMem(ctx context.Context) error { - maxMem, err := q.hostMemMB() - if err != nil { - return err - } // backend memory size must be multiple of 4Mib - sizeMB := (int(maxMem) - int(q.config.MemorySize)) >> 2 << 2 + sizeMB := (int(q.config.DefaultMaxMemorySize) - int(q.config.MemorySize)) >> 2 << 2 share, target, memoryBack, err := q.getMemArgs() if err != nil { @@ -1970,8 +1962,6 @@ func (q *qemu) hotplugMemory(memDev *MemoryDevice, op Operation) (int, error) { return 0, err } - currentMemory := int(q.config.MemorySize) + q.state.HotpluggedMemory - if memDev.SizeMB == 0 { memLog.Debug("hotplug is not required") return 0, nil @@ -1985,17 +1975,7 @@ func (q *qemu) hotplugMemory(memDev *MemoryDevice, op Operation) (int, error) { return 0, nil case AddDevice: memLog.WithField("operation", "add").Debugf("Requested to add memory: %d MB", memDev.SizeMB) - maxMem, err := q.hostMemMB() - if err != nil { - return 0, err - } - // Don't exceed the maximum amount of memory - if currentMemory+memDev.SizeMB > int(maxMem) { - // Fixme: return a typed error - return 0, fmt.Errorf("Unable to hotplug %d MiB memory, the SB has %d MiB and the maximum amount is %d MiB", - memDev.SizeMB, currentMemory, maxMem) - } memoryAdded, err := q.hotplugAddMemory(memDev) if err != nil { return memoryAdded, err @@ -2231,6 +2211,11 @@ func (q *qemu) ResizeMemory(ctx context.Context, reqMemMB uint32, memoryBlockSiz case currentMemory < reqMemMB: //hotplug addMemMB := reqMemMB - currentMemory + + if currentMemory+addMemMB > uint32(q.config.DefaultMaxMemorySize) { + addMemMB = uint32(q.config.DefaultMaxMemorySize) - currentMemory + } + memHotplugMB, err := calcHotplugMemMiBSize(addMemMB, memoryBlockSizeMB) if err != nil { return currentMemory, MemoryDevice{}, err diff --git a/src/runtime/virtcontainers/qemu_test.go b/src/runtime/virtcontainers/qemu_test.go index b50d73a917..b932189e8d 100644 --- a/src/runtime/virtcontainers/qemu_test.go +++ b/src/runtime/virtcontainers/qemu_test.go @@ -21,6 +21,7 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" + "github.com/pbnjay/memory" "github.com/pkg/errors" "github.com/stretchr/testify/assert" ) @@ -172,20 +173,20 @@ func TestQemuCPUTopology(t *testing.T) { func TestQemuMemoryTopology(t *testing.T) { mem := uint32(1000) + maxMem := memory.TotalMemory() / 1024 / 1024 //MiB slots := uint32(8) assert := assert.New(t) q := &qemu{ arch: &qemuArchBase{}, config: HypervisorConfig{ - MemorySize: mem, - MemSlots: slots, + MemorySize: mem, + MemSlots: slots, + DefaultMaxMemorySize: maxMem, }, } - hostMemKb, err := GetHostMemorySizeKb(procMemInfo) - assert.NoError(err) - memMax := fmt.Sprintf("%dM", int(float64(hostMemKb)/1024)) + memMax := fmt.Sprintf("%dM", int(maxMem)) expectedOut := govmmQemu.Memory{ Size: fmt.Sprintf("%dM", mem), From 0939f5181bfa6ba59f52d15b94c9f62a3911046a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 27 Jun 2022 15:02:00 +0200 Subject: [PATCH 3/4] config: Expose default_maxmemory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expose the newly added `default_maxmemory` to the project's Makefile and to the configuration files. Fixes: #4516 Signed-off-by: Fabiano Fidêncio --- src/runtime/Makefile | 3 +++ src/runtime/config/configuration-clh.toml.in | 6 ++++++ src/runtime/config/configuration-fc.toml.in | 7 +++++++ src/runtime/config/configuration-qemu.toml.in | 6 ++++++ 4 files changed, 22 insertions(+) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 757d0a48a9..fa07b87deb 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -158,6 +158,8 @@ DEFMEMSZ := 2048 # - vm template memory # - hugepage memory DEFMEMSLOTS := 10 +# Default maximum memory in MiB +DEFMAXMEMSZ := 0 #Default number of bridges DEFBRIDGES := 1 DEFENABLEANNOTATIONS := [\"enable_iommu\"] @@ -442,6 +444,7 @@ USER_VARS += DEFMAXVCPUS USER_VARS += DEFMAXVCPUS_ACRN USER_VARS += DEFMEMSZ USER_VARS += DEFMEMSLOTS +USER_VARS += DEFMAXMEMSZ USER_VARS += DEFBRIDGES USER_VARS += DEFNETWORKMODEL_ACRN USER_VARS += DEFNETWORKMODEL_CLH diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index 41d4ae4930..5d2d9c2f10 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -105,6 +105,12 @@ default_memory = @DEFMEMSZ@ # This is will determine the times that memory will be hotadded to sandbox/VM. #memory_slots = @DEFMEMSLOTS@ +# Default maximum memory in MiB per SB / VM +# unspecified or == 0 --> will be set to the actual amount of physical RAM +# > 0 <= amount of physical RAM --> will be set to the specified number +# > amount of physical RAM --> will be set to the actual amount of physical RAM +default_maxmemory = @DEFMAXMEMSZ@ + # Shared file system type: # - virtio-fs (default) # - virtio-fs-nydus diff --git a/src/runtime/config/configuration-fc.toml.in b/src/runtime/config/configuration-fc.toml.in index b6892bd17c..8761d8a02e 100644 --- a/src/runtime/config/configuration-fc.toml.in +++ b/src/runtime/config/configuration-fc.toml.in @@ -91,6 +91,7 @@ default_bridges = @DEFBRIDGES@ # Default memory size in MiB for SB/VM. # If unspecified then it will be set @DEFMEMSZ@ MiB. default_memory = @DEFMEMSZ@ + # # Default memory slots per SB/VM. # If unspecified then it will be set @DEFMEMSLOTS@. @@ -104,6 +105,12 @@ default_memory = @DEFMEMSZ@ # Default 0 #memory_offset = 0 +# Default maximum memory in MiB per SB / VM +# unspecified or == 0 --> will be set to the actual amount of physical RAM +# > 0 <= amount of physical RAM --> will be set to the specified number +# > amount of physical RAM --> will be set to the actual amount of physical RAM +default_maxmemory = @DEFMAXMEMSZ@ + # Block storage driver to be used for the hypervisor in case the container # rootfs is backed by a block device. This is virtio-scsi, virtio-blk # or nvdimm. diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index 702b71aadd..115cd19ccd 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -134,6 +134,12 @@ default_memory = @DEFMEMSZ@ # This is will determine the times that memory will be hotadded to sandbox/VM. #memory_slots = @DEFMEMSLOTS@ +# Default maximum memory in MiB per SB / VM +# unspecified or == 0 --> will be set to the actual amount of physical RAM +# > 0 <= amount of physical RAM --> will be set to the specified number +# > amount of physical RAM --> will be set to the actual amount of physical RAM +default_maxmemory = @DEFMAXMEMSZ@ + # The size in MiB will be plused to max memory of hypervisor. # It is the memory address space for the NVDIMM devie. # If set block storage driver (block_device_driver) to "nvdimm", From 323271403e7b44e60ed81f16eeaecd6020999778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 27 Jun 2022 18:46:11 +0200 Subject: [PATCH 4/4] virtcontainers: Remove unused function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While working on the previous commits, some of the functions become non-used. Let's simply remove them. Signed-off-by: Fabiano Fidêncio --- src/runtime/virtcontainers/hypervisor.go | 35 ------------- src/runtime/virtcontainers/hypervisor_test.go | 50 ------------------- src/runtime/virtcontainers/qemu.go | 12 ----- 3 files changed, 97 deletions(-) diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index e39885dac8..17fc7a8757 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -11,7 +11,6 @@ import ( "fmt" "os" "runtime" - "strconv" "strings" "github.com/pkg/errors" @@ -50,7 +49,6 @@ const ( // MockHypervisor is a mock hypervisor for testing purposes MockHypervisor HypervisorType = "mock" - procMemInfo = "/proc/meminfo" procCPUInfo = "/proc/cpuinfo" defaultVCPUs = 1 @@ -799,39 +797,6 @@ func DeserializeParams(parameters []string) []Param { return params } -func GetHostMemorySizeKb(memInfoPath string) (uint64, error) { - f, err := os.Open(memInfoPath) - if err != nil { - return 0, err - } - defer f.Close() - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - // Expected format: ["MemTotal:", "1234", "kB"] - parts := strings.Fields(scanner.Text()) - - // Sanity checks: Skip malformed entries. - if len(parts) < 3 || parts[0] != "MemTotal:" || parts[2] != "kB" { - continue - } - - sizeKb, err := strconv.ParseUint(parts[1], 0, 64) - if err != nil { - continue - } - - return sizeKb, nil - } - - // Handle errors that may have occurred during the reading of the file. - if err := scanner.Err(); err != nil { - return 0, err - } - - return 0, fmt.Errorf("unable get MemTotal from %s", memInfoPath) -} - // CheckCmdline checks whether an option or parameter is present in the kernel command line. // Search is case-insensitive. // Takes path to file that contains the kernel command line, desired option, and permitted values diff --git a/src/runtime/virtcontainers/hypervisor_test.go b/src/runtime/virtcontainers/hypervisor_test.go index ec475e86eb..424518915c 100644 --- a/src/runtime/virtcontainers/hypervisor_test.go +++ b/src/runtime/virtcontainers/hypervisor_test.go @@ -8,7 +8,6 @@ package virtcontainers import ( "fmt" "os" - "path/filepath" "testing" ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" @@ -372,55 +371,6 @@ func TestAddKernelParamInvalid(t *testing.T) { assert.Error(err) } -func TestGetHostMemorySizeKb(t *testing.T) { - assert := assert.New(t) - type testData struct { - contents string - expectedResult int - expectError bool - } - - data := []testData{ - { - ` - MemTotal: 1 kB - MemFree: 2 kB - SwapTotal: 3 kB - SwapFree: 4 kB - `, - 1024, - false, - }, - { - ` - MemFree: 2 kB - SwapTotal: 3 kB - SwapFree: 4 kB - `, - 0, - true, - }, - } - - dir := t.TempDir() - - file := filepath.Join(dir, "meminfo") - _, err := GetHostMemorySizeKb(file) - assert.Error(err) - - for _, d := range data { - err = os.WriteFile(file, []byte(d.contents), os.FileMode(0640)) - assert.NoError(err) - defer os.Remove(file) - - hostMemKb, err := GetHostMemorySizeKb(file) - - assert.False((d.expectError && err == nil)) - assert.False((!d.expectError && err != nil)) - assert.NotEqual(hostMemKb, d.expectedResult) - } -} - func TestCheckCmdline(t *testing.T) { assert := assert.New(t) diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index f2c501ac3a..f0cac0a897 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -302,18 +302,6 @@ func (q *qemu) cpuTopology() govmmQemu.SMP { return q.arch.cpuTopology(q.config.NumVCPUs, q.config.DefaultMaxVCPUs) } -func (q *qemu) hostMemMB() (uint64, error) { - hostMemKb, err := GetHostMemorySizeKb(procMemInfo) - if err != nil { - return 0, fmt.Errorf("Unable to read memory info: %s", err) - } - if hostMemKb == 0 { - return 0, fmt.Errorf("Error host memory size 0") - } - - return hostMemKb / 1024, nil -} - func (q *qemu) memoryTopology() (govmmQemu.Memory, error) { hostMemMb := q.config.DefaultMaxMemorySize memMb := uint64(q.config.MemorySize)