memoryDevice: reconstruct memoryDevice

If kata-runtime supports memory hotplug via probe interface, we need to reconstruct
memoryDevice to store relevant status, which are addr and probe. addr specifies the
physical address of the memory device, and probe determines it is hotplugged via
acpi-driven or probe interface.

Fixes: #1149

Signed-off-by: Penny Zheng <penny.zheng@arm.com>

virtcontainers/fc.go

@@ -683,8 +683,8 @@ func (fc *firecracker) hypervisorConfig() HypervisorConfig {
 	return fc.config
 }
 
-func (fc *firecracker) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) {
+func (fc *firecracker) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) {
-	return 0, nil
+	return 0, memoryDevice{}, nil
 }
 
 func (fc *firecracker) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) {

virtcontainers/hypervisor.go

@@ -91,6 +91,8 @@ const (
 type memoryDevice struct {
 	slot   int
 	sizeMB int
+	addr   uint64
+	probe  bool
 }
 
 // Set sets an hypervisor type based on the input string.
@@ -592,7 +594,7 @@ type hypervisor interface {
 	addDevice(devInfo interface{}, devType deviceType) error
 	hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error)
 	hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error)
-	resizeMemory(memMB uint32, memoryBlockSizeMB uint32) (uint32, error)
+	resizeMemory(memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error)
 	resizeVCPUs(vcpus uint32) (uint32, uint32, error)
 	getSandboxConsole(sandboxID string) (string, error)
 	disconnect()

virtcontainers/mock_hypervisor.go

@@ -84,8 +84,8 @@ func (m *mockHypervisor) getSandboxConsole(sandboxID string) (string, error) {
 	return "", nil
 }
 
-func (m *mockHypervisor) resizeMemory(memMB uint32, memorySectionSizeMB uint32) (uint32, error) {
+func (m *mockHypervisor) resizeMemory(memMB uint32, memorySectionSizeMB uint32, probe bool) (uint32, memoryDevice, error) {
-	return 0, nil
+	return 0, memoryDevice{}, nil
 }
 func (m *mockHypervisor) resizeVCPUs(cpus uint32) (uint32, uint32, error) {
 	return 0, 0, nil

virtcontainers/qemu.go

@@ -1240,7 +1240,19 @@ func (q *qemu) hotplugAddMemory(memDev *memoryDevice) (int, error) {
 		q.Logger().WithError(err).Error("hotplug memory")
 		return 0, err
 	}
-
+	// if guest kernel only supports memory hotplug via probe interface, we need to get address of hot-add memory device
+	if memDev.probe {
+		memoryDevices, err := q.qmpMonitorCh.qmp.ExecQueryMemoryDevices(q.qmpMonitorCh.ctx)
+		if err != nil {
+			return 0, fmt.Errorf("failed to query memory devices: %v", err)
+		}
+		if len(memoryDevices) != 0 {
+			q.Logger().WithField("addr", fmt.Sprintf("0x%x", memoryDevices[len(memoryDevices)-1].Data.Addr)).Debug("recently hot-add memory device")
+			memDev.addr = memoryDevices[len(memoryDevices)-1].Data.Addr
+		} else {
+			return 0, fmt.Errorf("failed to probe address of recently hot-add memory device, no device exists")
+		}
+	}
 	q.state.HotpluggedMemory += memDev.sizeMB
 	return memDev.sizeMB, q.store.Store(store.Hypervisor, q.state)
 }
@@ -1370,32 +1382,33 @@ func (q *qemu) disconnect() {
 // the memory to remove has to be at least the size of one slot.
 // To return memory back we are resizing the VM memory balloon.
 // A longer term solution is evaluate solutions like virtio-mem
-func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) {
+func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) {
 
 	currentMemory := q.config.MemorySize + uint32(q.state.HotpluggedMemory)
 	err := q.qmpSetup()
 	if err != nil {
-		return 0, err
+		return 0, memoryDevice{}, err
 	}
+	var addMemDevice memoryDevice
 	switch {
 	case currentMemory < reqMemMB:
 		//hotplug
 		addMemMB := reqMemMB - currentMemory
 		memHotplugMB, err := calcHotplugMemMiBSize(addMemMB, memoryBlockSizeMB)
 		if err != nil {
-			return currentMemory, err
+			return currentMemory, memoryDevice{}, err
 		}
 
-		addMemDevice := &memoryDevice{
+		addMemDevice.sizeMB = int(memHotplugMB)
-			sizeMB: int(memHotplugMB),
+		addMemDevice.probe = probe
-		}
+
-		data, err := q.hotplugAddDevice(addMemDevice, memoryDev)
+		data, err := q.hotplugAddDevice(&addMemDevice, memoryDev)
 		if err != nil {
-			return currentMemory, err
+			return currentMemory, addMemDevice, err
 		}
 		memoryAdded, ok := data.(int)
 		if !ok {
-			return currentMemory, fmt.Errorf("Could not get the memory added, got %+v", data)
+			return currentMemory, addMemDevice, fmt.Errorf("Could not get the memory added, got %+v", data)
 		}
 		currentMemory += uint32(memoryAdded)
 	case currentMemory > reqMemMB:
@@ -1403,31 +1416,31 @@ func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32,
 		addMemMB := currentMemory - reqMemMB
 		memHotunplugMB, err := calcHotplugMemMiBSize(addMemMB, memoryBlockSizeMB)
 		if err != nil {
-			return currentMemory, err
+			return currentMemory, memoryDevice{}, err
 		}
 
-		addMemDevice := &memoryDevice{
+		addMemDevice.sizeMB = int(memHotunplugMB)
-			sizeMB: int(memHotunplugMB),
+		addMemDevice.probe = probe
-		}
+
-		data, err := q.hotplugRemoveDevice(addMemDevice, memoryDev)
+		data, err := q.hotplugRemoveDevice(&addMemDevice, memoryDev)
 		if err != nil {
-			return currentMemory, err
+			return currentMemory, addMemDevice, err
 		}
 		memoryRemoved, ok := data.(int)
 		if !ok {
-			return currentMemory, fmt.Errorf("Could not get the memory removed, got %+v", data)
+			return currentMemory, addMemDevice, fmt.Errorf("Could not get the memory removed, got %+v", data)
 		}
 		//FIXME: This is to check memory hotplugRemoveDevice reported 0, as this is not supported.
 		// In the future if this is implemented this validation should be removed.
 		if memoryRemoved != 0 {
-			return currentMemory, fmt.Errorf("memory hot unplug is not supported, something went wrong")
+			return currentMemory, addMemDevice, fmt.Errorf("memory hot unplug is not supported, something went wrong")
 		}
 		currentMemory -= uint32(memoryRemoved)
 	}
 
 	// currentMemory is the current memory (updated) of the VM, return to caller to allow verify
 	// the current VM memory state.
-	return currentMemory, nil
+	return currentMemory, addMemDevice, nil
 }
 
 // genericAppendBridges appends to devices the given bridges

virtcontainers/qemu_test.go

@@ -393,9 +393,9 @@ func TestHotplugUnsupportedDeviceType(t *testing.T) {
 	}
 	q.store = vcStore
 
-	_, err = q.hotplugAddDevice(&memoryDevice{0, 128}, fsDev)
+	_, err = q.hotplugAddDevice(&memoryDevice{0, 128, uint64(0), false}, fsDev)
 	assert.Error(err)
-	_, err = q.hotplugRemoveDevice(&memoryDevice{0, 128}, fsDev)
+	_, err = q.hotplugRemoveDevice(&memoryDevice{0, 128, uint64(0), false}, fsDev)
 	assert.Error(err)
 }
 

virtcontainers/sandbox.go

@@ -1673,7 +1673,7 @@ func (s *Sandbox) updateResources() error {
 
 	// Update Memory
 	s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory")
-	newMemory, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB)
+	newMemory, _, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
 	if err != nil {
 		return err
 	}

virtcontainers/vm.go

@@ -373,7 +373,7 @@ func (v *VM) AddCPUs(num uint32) error {
 func (v *VM) AddMemory(numMB uint32) error {
 	if numMB > 0 {
 		v.logger().Infof("hot adding %d MB memory", numMB)
-		dev := &memoryDevice{1, int(numMB)}
+		dev := &memoryDevice{1, int(numMB), 0, false}
 		if _, err := v.hypervisor.hotplugAddDevice(dev, memoryDev); err != nil {
 			return err
 		}