runtime: Sandbox: Add addSwap and removeSwap

addSwap will create a swap file, hotplug it to hypervisor as a special
block device and let agent to setup it in the guest kernel.
removeSwap will remove the swap file.

Just QEMU support addSwap.

Fixes: #2201

Signed-off-by: Hui Zhu <teawater@antfin.com>

src/runtime/virtcontainers/acrn.go

@@ -517,6 +517,10 @@ func (a *Acrn) stopSandbox(ctx context.Context, waitOnly bool) (err error) {
 }
 
 func (a *Acrn) updateBlockDevice(drive *config.BlockDrive) error {
+	if drive.Swap {
+		return fmt.Errorf("Acrn doesn't support swap")
+	}
+
 	var err error
 	if drive.File == "" || drive.Index >= AcrnBlkDevPoolSz {
 		return fmt.Errorf("Empty filepath or invalid drive index, Dive ID:%s, Drive Index:%d",

src/runtime/virtcontainers/agent.go

@@ -12,6 +12,7 @@ import (
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
 	pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/net/context"
@@ -189,6 +190,9 @@ type agent interface {
 	// copyFile copies file from host to container's rootfs
 	copyFile(ctx context.Context, src, dst string) error
 
+	// Tell the agent to setup the swapfile in the guest
+	addSwap(ctx context.Context, PCIPath vcTypes.PciPath) error
+
 	// markDead tell agent that the guest is dead
 	markDead(ctx context.Context)
 

src/runtime/virtcontainers/clh.go

@@ -416,6 +416,10 @@ func clhPciInfoToPath(pciInfo chclient.PciDeviceInfo) (vcTypes.PciPath, error) {
 }
 
 func (clh *cloudHypervisor) hotplugAddBlockDevice(drive *config.BlockDrive) error {
+	if drive.Swap {
+		return fmt.Errorf("cloudHypervisor doesn't support swap")
+	}
+
 	if clh.config.BlockDeviceDriver != config.VirtioBlock {
 		return fmt.Errorf("incorrect hypervisor configuration on 'block_device_driver':"+
 			" using '%v' but only support '%v'", clh.config.BlockDeviceDriver, config.VirtioBlock)

src/runtime/virtcontainers/device/config/config.go

@@ -182,6 +182,9 @@ type BlockDrive struct {
 	// Pmem enables persistent memory. Use File as backing file
 	// for a nvdimm device in the guest
 	Pmem bool
+
+	// This block device is for swap
+	Swap bool
 }
 
 // VFIODeviceType indicates VFIO device type

src/runtime/virtcontainers/fc.go

@@ -1036,6 +1036,10 @@ func (fc *firecracker) addDevice(ctx context.Context, devInfo interface{}, devTy
 // hotplugBlockDevice supported in Firecracker VMM
 // hot add or remove a block device.
 func (fc *firecracker) hotplugBlockDevice(ctx context.Context, drive config.BlockDrive, op operation) (interface{}, error) {
+	if drive.Swap {
+		return nil, fmt.Errorf("firecracker doesn't support swap")
+	}
+
 	var path string
 	var err error
 	driveID := fcDriveIndexToID(drive.Index)

src/runtime/virtcontainers/kata_agent.go

@@ -143,6 +143,7 @@ const (
 	grpcStopTracingRequest       = "grpc.StopTracingRequest"
 	grpcGetOOMEventRequest       = "grpc.GetOOMEventRequest"
 	grpcGetMetricsRequest        = "grpc.GetMetricsRequest"
+	grpcAddSwapRequest           = "grpc.AddSwapRequest"
 )
 
 // newKataAgent returns an agent from an agent type.
@@ -2024,6 +2025,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) {
 	k.reqHandlers[grpcGetMetricsRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
 		return k.client.AgentServiceClient.GetMetrics(ctx, req.(*grpc.GetMetricsRequest))
 	}
+	k.reqHandlers[grpcAddSwapRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
+		return k.client.AgentServiceClient.AddSwap(ctx, req.(*grpc.AddSwapRequest))
+	}
 }
 
 func (k *kataAgent) getReqContext(ctx context.Context, reqName string) (newCtx context.Context, cancel context.CancelFunc) {
@@ -2184,6 +2188,14 @@ func (k *kataAgent) copyFile(ctx context.Context, src, dst string) error {
 	return nil
 }
 
+func (k *kataAgent) addSwap(ctx context.Context, PCIPath vcTypes.PciPath) error {
+	span, ctx := katatrace.Trace(ctx, k.Logger(), "addSwap", kataAgentTracingTags)
+	defer span.End()
+
+	_, err := k.sendReq(ctx, &grpc.AddSwapRequest{PCIPath: PCIPath.ToArray()})
+	return err
+}
+
 func (k *kataAgent) markDead(ctx context.Context) {
 	k.Logger().Infof("mark agent dead")
 	k.dead = true

src/runtime/virtcontainers/mock_agent.go

@@ -12,6 +12,7 @@ import (
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
 	pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/net/context"
@@ -215,6 +216,11 @@ func (n *mockAgent) copyFile(ctx context.Context, src, dst string) error {
 	return nil
 }
 
+// addSwap is the Noop agent setup swap. It does nothing.
+func (n *mockAgent) addSwap(ctx context.Context, PCIPath vcTypes.PciPath) error {
+	return nil
+}
+
 func (n *mockAgent) markDead(ctx context.Context) {
 }
 

src/runtime/virtcontainers/pkg/types/pcipath.go

@@ -75,6 +75,14 @@ func (p PciPath) IsNil() bool {
 	return p.slots == nil
 }
 
+func (p PciPath) ToArray() []uint32 {
+	var slots []uint32
+	for _, slot := range p.slots {
+		slots = append(slots, uint32(slot.slot))
+	}
+	return slots
+}
+
 func PciPathFromString(s string) (PciPath, error) {
 	if s == "" {
 		return PciPath{}, nil

src/runtime/virtcontainers/qemu.go

@@ -1232,7 +1232,9 @@ func (q *qemu) hotplugAddBlockDevice(ctx context.Context, drive *config.BlockDri
 		return nil
 	}
 
-	if q.config.BlockDeviceCacheSet {
+	if drive.Swap {
+		err = q.qmpMonitorCh.qmp.ExecuteBlockdevAddWithCache(q.qmpMonitorCh.ctx, drive.File, drive.ID, false, false, false)
+	} else if q.config.BlockDeviceCacheSet {
 		err = q.qmpMonitorCh.qmp.ExecuteBlockdevAddWithCache(q.qmpMonitorCh.ctx, drive.File, drive.ID, q.config.BlockDeviceCacheDirect, q.config.BlockDeviceCacheNoflush, drive.ReadOnly)
 	} else {
 		err = q.qmpMonitorCh.qmp.ExecuteBlockdevAdd(q.qmpMonitorCh.ctx, drive.File, drive.ID, drive.ReadOnly)
@@ -1248,25 +1250,8 @@ func (q *qemu) hotplugAddBlockDevice(ctx context.Context, drive *config.BlockDri
 	}()
 
 	switch {
-	case q.config.BlockDeviceDriver == config.VirtioBlockCCW:
+	case drive.Swap:
-		driver := "virtio-blk-ccw"
+		fallthrough
-
-		addr, bridge, err := q.arch.addDeviceToBridge(ctx, drive.ID, types.CCW)
-		if err != nil {
-			return err
-		}
-		var devNoHotplug string
-		devNoHotplug, err = bridge.AddressFormatCCW(addr)
-		if err != nil {
-			return err
-		}
-		drive.DevNo, err = bridge.AddressFormatCCWForVirtServer(addr)
-		if err != nil {
-			return err
-		}
-		if err = q.qmpMonitorCh.qmp.ExecuteDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, devNoHotplug, "", true, false); err != nil {
-			return err
-		}
 	case q.config.BlockDeviceDriver == config.VirtioBlock:
 		driver := "virtio-blk-pci"
 		addr, bridge, err := q.arch.addDeviceToBridge(ctx, drive.ID, types.PCI)
@@ -1296,6 +1281,25 @@ func (q *qemu) hotplugAddBlockDevice(ctx context.Context, drive *config.BlockDri
 		if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil {
 			return err
 		}
+	case q.config.BlockDeviceDriver == config.VirtioBlockCCW:
+		driver := "virtio-blk-ccw"
+
+		addr, bridge, err := q.arch.addDeviceToBridge(ctx, drive.ID, types.CCW)
+		if err != nil {
+			return err
+		}
+		var devNoHotplug string
+		devNoHotplug, err = bridge.AddressFormatCCW(addr)
+		if err != nil {
+			return err
+		}
+		drive.DevNo, err = bridge.AddressFormatCCWForVirtServer(addr)
+		if err != nil {
+			return err
+		}
+		if err = q.qmpMonitorCh.qmp.ExecuteDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, devNoHotplug, "", true, false); err != nil {
+			return err
+		}
 	case q.config.BlockDeviceDriver == config.VirtioSCSI:
 		driver := "scsi-hd"
 
@@ -1369,7 +1373,7 @@ func (q *qemu) hotplugBlockDevice(ctx context.Context, drive *config.BlockDrive,
 	if op == addDevice {
 		return q.hotplugAddBlockDevice(ctx, drive, op, devID)
 	}
-	if q.config.BlockDeviceDriver == config.VirtioBlock {
+	if !drive.Swap && q.config.BlockDeviceDriver == config.VirtioBlock {
 		if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil {
 			return err
 		}

src/runtime/virtcontainers/sandbox.go

@@ -14,6 +14,8 @@ import (
 	"math"
 	"net"
 	"os"
+	"os/exec"
+	"path/filepath"
 	"strings"
 	"sync"
 	"syscall"
@@ -996,6 +998,87 @@ func (cw *consoleWatcher) stop() {
 	}
 }
 
+func (s *Sandbox) addSwap(ctx context.Context, swapID string, size int64) (*config.BlockDrive, error) {
+	swapFile := filepath.Join(getSandboxPath(s.id), swapID)
+
+	swapFD, err := os.OpenFile(swapFile, os.O_CREATE, 0600)
+	if err != nil {
+		err = fmt.Errorf("creat swapfile %s fail %s", swapFile, err.Error())
+		s.Logger().WithError(err).Error("addSwap")
+		return nil, err
+	}
+	swapFD.Close()
+	defer func() {
+		if err != nil {
+			os.Remove(swapFile)
+		}
+	}()
+
+	// Check the size
+	pagesize := os.Getpagesize()
+	// mkswap refuses areas smaller than 10 pages.
+	size = int64(math.Max(float64(size), float64(pagesize*10)))
+	// Swapfile need a page to store the metadata
+	size += int64(pagesize)
+
+	err = os.Truncate(swapFile, size)
+	if err != nil {
+		err = fmt.Errorf("truncate swapfile %s fail %s", swapFile, err.Error())
+		s.Logger().WithError(err).Error("addSwap")
+		return nil, err
+	}
+
+	err = exec.CommandContext(ctx, "/sbin/mkswap", swapFile).Run()
+	if err != nil {
+		err = fmt.Errorf("mkswap swapfile %s fail %s", swapFile, err.Error())
+		s.Logger().WithError(err).Error("addSwap")
+		return nil, err
+	}
+
+	blockDevice := &config.BlockDrive{
+		File:   swapFile,
+		Format: "raw",
+		ID:     swapID,
+		Swap:   true,
+	}
+	_, err = s.hypervisor.hotplugAddDevice(ctx, blockDevice, blockDev)
+	if err != nil {
+		err = fmt.Errorf("add swapfile %s device to VM fail %s", swapFile, err.Error())
+		s.Logger().WithError(err).Error("addSwap")
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			_, e := s.hypervisor.hotplugRemoveDevice(ctx, blockDevice, blockDev)
+			if e != nil {
+				s.Logger().Errorf("remove swapfile %s to VM fail %s", swapFile, e.Error())
+			}
+		}
+	}()
+
+	err = s.agent.addSwap(ctx, blockDevice.PCIPath)
+	if err != nil {
+		err = fmt.Errorf("agent add swapfile %s PCIPath %+v to VM fail %s", swapFile, blockDevice.PCIPath, err.Error())
+		s.Logger().WithError(err).Error("addSwap")
+		return nil, err
+	}
+
+	s.Logger().Infof("add swapfile %s size %d PCIPath %+v to VM success", swapFile, size, blockDevice.PCIPath)
+
+	return blockDevice, nil
+}
+
+func (s *Sandbox) removeSwap(ctx context.Context, blockDevice *config.BlockDrive) error {
+	err := os.Remove(blockDevice.File)
+	if err != nil {
+		err = fmt.Errorf("remove swapfile %s fail %s", blockDevice.File, err.Error())
+		s.Logger().WithError(err).Error("removeSwap")
+	} else {
+		s.Logger().Infof("remove swapfile %s success", blockDevice.File)
+	}
+	return err
+}
+
 // startVM starts the VM.
 func (s *Sandbox) startVM(ctx context.Context) (err error) {
 	span, ctx := katatrace.Trace(ctx, s.Logger(), "startVM", s.tracingTags())
@@ -1074,6 +1157,14 @@ func (s *Sandbox) startVM(ctx context.Context) (err error) {
 
 	s.Logger().Info("Agent started in the sandbox")
 
+	defer func() {
+		if err != nil {
+			if e := s.agent.stopSandbox(ctx, s); e != nil {
+				s.Logger().WithError(e).WithField("sandboxid", s.id).Warning("Agent did not stop sandbox")
+			}
+		}
+	}()
+
 	return nil
 }
 
@@ -1846,6 +1937,7 @@ func (s *Sandbox) updateResources(ctx context.Context) error {
 	if err := s.agent.onlineCPUMem(ctx, 0, false); err != nil {
 		return err
 	}
+
 	return nil
 }