diff --git a/src/runtime/config/configuration-qemu-coco-dev.toml.in b/src/runtime/config/configuration-qemu-coco-dev.toml.in index 05ef3b54bb..4d6d5c31f3 100644 --- a/src/runtime/config/configuration-qemu-coco-dev.toml.in +++ b/src/runtime/config/configuration-qemu-coco-dev.toml.in @@ -273,6 +273,16 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # Default false #enable_mem_prealloc = true +# Reclaim guest freed memory. +# Enabling this will result in the VM balloon device having f_reporting=on set. +# Then the hypervisor will use it to reclaim guest freed memory. +# This is useful for reducing the amount of memory used by a VM. +# Enabling this feature may sometimes reduce the speed of memory access in +# the VM. +# +# Default false +#reclaim_guest_freed_memory = true + # Enable huge pages for VM RAM, default false # Enabling this will result in the VM memory # being allocated using huge pages. diff --git a/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in b/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in index c5088ce01b..26d6468022 100644 --- a/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in +++ b/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in @@ -290,6 +290,16 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # Default false #enable_mem_prealloc = true +# Reclaim guest freed memory. +# Enabling this will result in the VM balloon device having f_reporting=on set. +# Then the hypervisor will use it to reclaim guest freed memory. +# This is useful for reducing the amount of memory used by a VM. +# Enabling this feature may sometimes reduce the speed of memory access in +# the VM. +# +# Default false +#reclaim_guest_freed_memory = true + # Enable huge pages for VM RAM, default false # Enabling this will result in the VM memory # being allocated using huge pages. diff --git a/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in b/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in index 123b2a2be7..f615633c1a 100644 --- a/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in +++ b/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in @@ -266,6 +266,16 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # Default false #enable_mem_prealloc = true +# Reclaim guest freed memory. +# Enabling this will result in the VM balloon device having f_reporting=on set. +# Then the hypervisor will use it to reclaim guest freed memory. +# This is useful for reducing the amount of memory used by a VM. +# Enabling this feature may sometimes reduce the speed of memory access in +# the VM. +# +# Default false +#reclaim_guest_freed_memory = true + # Enable huge pages for VM RAM, default false # Enabling this will result in the VM memory # being allocated using huge pages. diff --git a/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in b/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in index 8f92987f96..d5b0c037ec 100644 --- a/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in +++ b/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in @@ -271,6 +271,16 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # Default false #enable_mem_prealloc = true +# Reclaim guest freed memory. +# Enabling this will result in the VM balloon device having f_reporting=on set. +# Then the hypervisor will use it to reclaim guest freed memory. +# This is useful for reducing the amount of memory used by a VM. +# Enabling this feature may sometimes reduce the speed of memory access in +# the VM. +# +# Default false +#reclaim_guest_freed_memory = true + # Enable huge pages for VM RAM, default false # Enabling this will result in the VM memory # being allocated using huge pages. diff --git a/src/runtime/config/configuration-qemu-se.toml.in b/src/runtime/config/configuration-qemu-se.toml.in index fba6f4c34d..2af86d8c1f 100644 --- a/src/runtime/config/configuration-qemu-se.toml.in +++ b/src/runtime/config/configuration-qemu-se.toml.in @@ -257,6 +257,16 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # Default false #enable_mem_prealloc = true +# Reclaim guest freed memory. +# Enabling this will result in the VM balloon device having f_reporting=on set. +# Then the hypervisor will use it to reclaim guest freed memory. +# This is useful for reducing the amount of memory used by a VM. +# Enabling this feature may sometimes reduce the speed of memory access in +# the VM. +# +# Default false +#reclaim_guest_freed_memory = true + # Enable huge pages for VM RAM, default false # Enabling this will result in the VM memory # being allocated using huge pages. diff --git a/src/runtime/config/configuration-qemu-snp.toml.in b/src/runtime/config/configuration-qemu-snp.toml.in index e6d5dcd2bf..f51cf6c5cd 100644 --- a/src/runtime/config/configuration-qemu-snp.toml.in +++ b/src/runtime/config/configuration-qemu-snp.toml.in @@ -290,6 +290,16 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # Default false #enable_mem_prealloc = true +# Reclaim guest freed memory. +# Enabling this will result in the VM balloon device having f_reporting=on set. +# Then the hypervisor will use it to reclaim guest freed memory. +# This is useful for reducing the amount of memory used by a VM. +# Enabling this feature may sometimes reduce the speed of memory access in +# the VM. +# +# Default false +#reclaim_guest_freed_memory = true + # Enable huge pages for VM RAM, default false # Enabling this will result in the VM memory # being allocated using huge pages. diff --git a/src/runtime/config/configuration-qemu-tdx.toml.in b/src/runtime/config/configuration-qemu-tdx.toml.in index de8b34c436..d84ec3cf39 100644 --- a/src/runtime/config/configuration-qemu-tdx.toml.in +++ b/src/runtime/config/configuration-qemu-tdx.toml.in @@ -267,6 +267,16 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # Default false #enable_mem_prealloc = true +# Reclaim guest freed memory. +# Enabling this will result in the VM balloon device having f_reporting=on set. +# Then the hypervisor will use it to reclaim guest freed memory. +# This is useful for reducing the amount of memory used by a VM. +# Enabling this feature may sometimes reduce the speed of memory access in +# the VM. +# +# Default false +#reclaim_guest_freed_memory = true + # Enable huge pages for VM RAM, default false # Enabling this will result in the VM memory # being allocated using huge pages. diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index 4afb3291ed..40ed29aeb8 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -272,6 +272,16 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # Default false #enable_mem_prealloc = true +# Reclaim guest freed memory. +# Enabling this will result in the VM balloon device having f_reporting=on set. +# Then the hypervisor will use it to reclaim guest freed memory. +# This is useful for reducing the amount of memory used by a VM. +# Enabling this feature may sometimes reduce the speed of memory access in +# the VM. +# +# Default false +#reclaim_guest_freed_memory = true + # Enable huge pages for VM RAM, default false # Enabling this will result in the VM memory # being allocated using huge pages. diff --git a/src/runtime/pkg/device/config/config.go b/src/runtime/pkg/device/config/config.go index 42bcccff8a..f535b3ebca 100644 --- a/src/runtime/pkg/device/config/config.go +++ b/src/runtime/pkg/device/config/config.go @@ -439,6 +439,14 @@ type RNGDev struct { Filename string } +// BalloonDev represents a balloon device +type BalloonDev struct { + ID string + DeflateOnOOM bool + DisableModern bool + FreePageReporting bool +} + // VhostUserDeviceAttrs represents data shared by most vhost-user devices type VhostUserDeviceAttrs struct { DevID string diff --git a/src/runtime/pkg/govmm/qemu/qemu.go b/src/runtime/pkg/govmm/qemu/qemu.go index 249a12c856..718156705a 100644 --- a/src/runtime/pkg/govmm/qemu/qemu.go +++ b/src/runtime/pkg/govmm/qemu/qemu.go @@ -2409,9 +2409,10 @@ func (v RngDevice) deviceName(config *Config) string { // BalloonDevice represents a memory balloon device. // nolint: govet type BalloonDevice struct { - DeflateOnOOM bool - DisableModern bool - ID string + DeflateOnOOM bool + DisableModern bool + FreePageReporting bool + ID string // ROMFile specifies the ROM file being used for this device. ROMFile string @@ -2458,6 +2459,11 @@ func (b BalloonDevice) QemuParams(config *Config) []string { if s := b.Transport.disableModern(config, b.DisableModern); s != "" { deviceParams = append(deviceParams, s) } + if b.FreePageReporting { + deviceParams = append(deviceParams, "free-page-reporting=on") + } else { + deviceParams = append(deviceParams, "free-page-reporting=off") + } qemuParams = append(qemuParams, "-device") qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) diff --git a/src/runtime/pkg/govmm/qemu/qemu_arch_base_test.go b/src/runtime/pkg/govmm/qemu/qemu_arch_base_test.go index 10fd0bdffa..a14e0fb032 100644 --- a/src/runtime/pkg/govmm/qemu/qemu_arch_base_test.go +++ b/src/runtime/pkg/govmm/qemu/qemu_arch_base_test.go @@ -80,14 +80,19 @@ func TestAppendVirtioBalloon(t *testing.T) { var OnDisableModern = ",disable-modern=true" var OffDisableModern = ",disable-modern=false" - testAppend(balloonDevice, deviceString+OffDeflateOnOMM+OffDisableModern, t) + var OnFreePageReporting = ",free-page-reporting=on" + var OffFreePageReporting = ",free-page-reporting=off" + + testAppend(balloonDevice, deviceString+OffDeflateOnOMM+OffDisableModern+OffFreePageReporting, t) balloonDevice.DeflateOnOOM = true - testAppend(balloonDevice, deviceString+OnDeflateOnOMM+OffDisableModern, t) + testAppend(balloonDevice, deviceString+OnDeflateOnOMM+OffDisableModern+OffFreePageReporting, t) balloonDevice.DisableModern = true - testAppend(balloonDevice, deviceString+OnDeflateOnOMM+OnDisableModern, t) + testAppend(balloonDevice, deviceString+OnDeflateOnOMM+OnDisableModern+OffFreePageReporting, t) + balloonDevice.FreePageReporting = true + testAppend(balloonDevice, deviceString+OnDeflateOnOMM+OnDisableModern+OnFreePageReporting, t) } func TestAppendPCIBridgeDevice(t *testing.T) { diff --git a/src/runtime/pkg/govmm/qemu/qemu_s390x_test.go b/src/runtime/pkg/govmm/qemu/qemu_s390x_test.go index 311e286e94..5e36623bbf 100644 --- a/src/runtime/pkg/govmm/qemu/qemu_s390x_test.go +++ b/src/runtime/pkg/govmm/qemu/qemu_s390x_test.go @@ -35,10 +35,17 @@ func TestAppendVirtioBalloon(t *testing.T) { var OnDeflateOnOMM = ",deflate-on-oom=on" var OffDeflateOnOMM = ",deflate-on-oom=off" - testAppend(balloonDevice, deviceString+devnoOptios+OffDeflateOnOMM, t) + + var OnFreePageReporting = ",free-page-reporting=on" + var OffFreePageReporting = ",free-page-reporting=off" + + testAppend(balloonDevice, deviceString+devnoOptios+OffDeflateOnOMM+OffFreePageReporting, t) balloonDevice.DeflateOnOOM = true - testAppend(balloonDevice, deviceString+devnoOptios+OnDeflateOnOMM, t) + testAppend(balloonDevice, deviceString+devnoOptios+OnDeflateOnOMM+OffFreePageReporting, t) + + balloonDevice.FreePageReporting = true + testAppend(balloonDevice, deviceString+devnoOptios+OnDeflateOnOMM+OnFreePageReporting, t) } func TestAppendDeviceFSCCW(t *testing.T) { diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 85f6c72588..39d2739ce7 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -952,6 +952,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { VirtioFSQueueSize: h.VirtioFSQueueSize, VirtioFSExtraArgs: h.VirtioFSExtraArgs, MemPrealloc: h.MemPrealloc, + ReclaimGuestFreedMemory: h.ReclaimGuestFreedMemory, HugePages: h.HugePages, IOMMU: h.IOMMU, IOMMUPlatform: h.getIOMMUPlatform(), diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index e7d00f7ea3..282d1d1a95 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -139,6 +139,7 @@ const ( scsiControllerID = "scsi0" rngID = "rng0" fallbackFileBackedMemDir = "/dev/shm" + balloonID = "balloon0" qemuStopSandboxTimeoutSecs = 15 @@ -632,6 +633,9 @@ func (q *qemu) prepareInitdataMount(config *HypervisorConfig) error { } // CreateVM is the Hypervisor VM creation implementation for govmmQemu. +// This function is complex and there's not much to be done about it, unfortunately. +// +//nolint:gocyclo func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervisorConfig *HypervisorConfig) error { // Save the tracing context q.ctx = ctx @@ -801,6 +805,20 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi } } + if q.config.ReclaimGuestFreedMemory && !q.config.ConfidentialGuest { + balloonDev := config.BalloonDev{ + ID: balloonID, + DeflateOnOOM: true, + DisableModern: false, + FreePageReporting: true, + } + + qemuConfig.Devices, err = q.arch.appendBalloonDevice(ctx, qemuConfig.Devices, balloonDev) + if err != nil { + return err + } + } + if machine.Type == QemuQ35 || machine.Type == QemuVirt { if err := q.createPCIeTopology(&qemuConfig, hypervisorConfig, machine.Type, network); err != nil { q.Logger().WithError(err).Errorf("Cannot create PCIe topology") diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index 18c193628c..2b6ab7ce5b 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -116,6 +116,9 @@ type qemuArch interface { // appendRNGDevice appends a RNG device to devices appendRNGDevice(ctx context.Context, devices []govmmQemu.Device, rngDevice config.RNGDev) ([]govmmQemu.Device, error) + // appendBalloonDevice appends a Balloon device to devices + appendBalloonDevice(ctx context.Context, devices []govmmQemu.Device, BalloonDevice config.BalloonDev) ([]govmmQemu.Device, error) + // setEndpointDevicePath sets the appropriate PCI or CCW device path for an endpoint setEndpointDevicePath(endpoint Endpoint, bridgeAddr int, devAddr string) error @@ -738,6 +741,19 @@ func (q *qemuArchBase) appendRNGDevice(_ context.Context, devices []govmmQemu.De return devices, nil } +func (q *qemuArchBase) appendBalloonDevice(_ context.Context, devices []govmmQemu.Device, balloonDev config.BalloonDev) ([]govmmQemu.Device, error) { + devices = append(devices, + govmmQemu.BalloonDevice{ + ID: balloonDev.ID, + DeflateOnOOM: balloonDev.DeflateOnOOM, + DisableModern: balloonDev.DisableModern, + FreePageReporting: balloonDev.FreePageReporting, + }, + ) + + return devices, nil +} + func (q *qemuArchBase) setEndpointDevicePath(endpoint Endpoint, bridgeAddr int, devAddr string) error { bridgeSlot, err := types.PciSlotFromInt(bridgeAddr) if err != nil {