From 243d4b86894a5e4c4bb310eac4c6034722a0275f Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Mon, 5 Jul 2021 16:28:32 +0800 Subject: [PATCH] runtime: Sandbox: Add addSwap and removeSwap addSwap will create a swap file, hotplug it to hypervisor as a special block device and let agent to setup it in the guest kernel. removeSwap will remove the swap file. Just QEMU support addSwap. Fixes: #2201 Signed-off-by: Hui Zhu --- src/runtime/virtcontainers/acrn.go | 4 + src/runtime/virtcontainers/agent.go | 4 + src/runtime/virtcontainers/clh.go | 4 + .../virtcontainers/device/config/config.go | 3 + src/runtime/virtcontainers/fc.go | 4 + src/runtime/virtcontainers/kata_agent.go | 12 +++ src/runtime/virtcontainers/mock_agent.go | 6 ++ .../virtcontainers/pkg/types/pcipath.go | 8 ++ src/runtime/virtcontainers/qemu.go | 46 +++++----- src/runtime/virtcontainers/sandbox.go | 92 +++++++++++++++++++ 10 files changed, 162 insertions(+), 21 deletions(-) diff --git a/src/runtime/virtcontainers/acrn.go b/src/runtime/virtcontainers/acrn.go index dd2d59954b..07eb92c8cf 100644 --- a/src/runtime/virtcontainers/acrn.go +++ b/src/runtime/virtcontainers/acrn.go @@ -517,6 +517,10 @@ func (a *Acrn) stopSandbox(ctx context.Context, waitOnly bool) (err error) { } func (a *Acrn) updateBlockDevice(drive *config.BlockDrive) error { + if drive.Swap { + return fmt.Errorf("Acrn doesn't support swap") + } + var err error if drive.File == "" || drive.Index >= AcrnBlkDevPoolSz { return fmt.Errorf("Empty filepath or invalid drive index, Dive ID:%s, Drive Index:%d", diff --git a/src/runtime/virtcontainers/agent.go b/src/runtime/virtcontainers/agent.go index 51fdf67dce..bf709b1ad7 100644 --- a/src/runtime/virtcontainers/agent.go +++ b/src/runtime/virtcontainers/agent.go @@ -12,6 +12,7 @@ import ( persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" + vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" specs "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/net/context" @@ -189,6 +190,9 @@ type agent interface { // copyFile copies file from host to container's rootfs copyFile(ctx context.Context, src, dst string) error + // Tell the agent to setup the swapfile in the guest + addSwap(ctx context.Context, PCIPath vcTypes.PciPath) error + // markDead tell agent that the guest is dead markDead(ctx context.Context) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 053d60b750..5cf2a5de93 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -416,6 +416,10 @@ func clhPciInfoToPath(pciInfo chclient.PciDeviceInfo) (vcTypes.PciPath, error) { } func (clh *cloudHypervisor) hotplugAddBlockDevice(drive *config.BlockDrive) error { + if drive.Swap { + return fmt.Errorf("cloudHypervisor doesn't support swap") + } + if clh.config.BlockDeviceDriver != config.VirtioBlock { return fmt.Errorf("incorrect hypervisor configuration on 'block_device_driver':"+ " using '%v' but only support '%v'", clh.config.BlockDeviceDriver, config.VirtioBlock) diff --git a/src/runtime/virtcontainers/device/config/config.go b/src/runtime/virtcontainers/device/config/config.go index 5c866623de..8ad8c6266c 100644 --- a/src/runtime/virtcontainers/device/config/config.go +++ b/src/runtime/virtcontainers/device/config/config.go @@ -182,6 +182,9 @@ type BlockDrive struct { // Pmem enables persistent memory. Use File as backing file // for a nvdimm device in the guest Pmem bool + + // This block device is for swap + Swap bool } // VFIODeviceType indicates VFIO device type diff --git a/src/runtime/virtcontainers/fc.go b/src/runtime/virtcontainers/fc.go index f7775117d5..9b4ffa7829 100644 --- a/src/runtime/virtcontainers/fc.go +++ b/src/runtime/virtcontainers/fc.go @@ -1036,6 +1036,10 @@ func (fc *firecracker) addDevice(ctx context.Context, devInfo interface{}, devTy // hotplugBlockDevice supported in Firecracker VMM // hot add or remove a block device. func (fc *firecracker) hotplugBlockDevice(ctx context.Context, drive config.BlockDrive, op operation) (interface{}, error) { + if drive.Swap { + return nil, fmt.Errorf("firecracker doesn't support swap") + } + var path string var err error driveID := fcDriveIndexToID(drive.Index) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 2edb295f7c..a8b62127ae 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -143,6 +143,7 @@ const ( grpcStopTracingRequest = "grpc.StopTracingRequest" grpcGetOOMEventRequest = "grpc.GetOOMEventRequest" grpcGetMetricsRequest = "grpc.GetMetricsRequest" + grpcAddSwapRequest = "grpc.AddSwapRequest" ) // newKataAgent returns an agent from an agent type. @@ -2024,6 +2025,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { k.reqHandlers[grpcGetMetricsRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { return k.client.AgentServiceClient.GetMetrics(ctx, req.(*grpc.GetMetricsRequest)) } + k.reqHandlers[grpcAddSwapRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { + return k.client.AgentServiceClient.AddSwap(ctx, req.(*grpc.AddSwapRequest)) + } } func (k *kataAgent) getReqContext(ctx context.Context, reqName string) (newCtx context.Context, cancel context.CancelFunc) { @@ -2184,6 +2188,14 @@ func (k *kataAgent) copyFile(ctx context.Context, src, dst string) error { return nil } +func (k *kataAgent) addSwap(ctx context.Context, PCIPath vcTypes.PciPath) error { + span, ctx := katatrace.Trace(ctx, k.Logger(), "addSwap", kataAgentTracingTags) + defer span.End() + + _, err := k.sendReq(ctx, &grpc.AddSwapRequest{PCIPath: PCIPath.ToArray()}) + return err +} + func (k *kataAgent) markDead(ctx context.Context) { k.Logger().Infof("mark agent dead") k.dead = true diff --git a/src/runtime/virtcontainers/mock_agent.go b/src/runtime/virtcontainers/mock_agent.go index 17bd03d540..5be0b4a2f8 100644 --- a/src/runtime/virtcontainers/mock_agent.go +++ b/src/runtime/virtcontainers/mock_agent.go @@ -12,6 +12,7 @@ import ( persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" + vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" specs "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/net/context" @@ -215,6 +216,11 @@ func (n *mockAgent) copyFile(ctx context.Context, src, dst string) error { return nil } +// addSwap is the Noop agent setup swap. It does nothing. +func (n *mockAgent) addSwap(ctx context.Context, PCIPath vcTypes.PciPath) error { + return nil +} + func (n *mockAgent) markDead(ctx context.Context) { } diff --git a/src/runtime/virtcontainers/pkg/types/pcipath.go b/src/runtime/virtcontainers/pkg/types/pcipath.go index 786ab67a3e..5b7b3db91a 100644 --- a/src/runtime/virtcontainers/pkg/types/pcipath.go +++ b/src/runtime/virtcontainers/pkg/types/pcipath.go @@ -75,6 +75,14 @@ func (p PciPath) IsNil() bool { return p.slots == nil } +func (p PciPath) ToArray() []uint32 { + var slots []uint32 + for _, slot := range p.slots { + slots = append(slots, uint32(slot.slot)) + } + return slots +} + func PciPathFromString(s string) (PciPath, error) { if s == "" { return PciPath{}, nil diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index f340e36e4c..8bdc0439fd 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -1232,7 +1232,9 @@ func (q *qemu) hotplugAddBlockDevice(ctx context.Context, drive *config.BlockDri return nil } - if q.config.BlockDeviceCacheSet { + if drive.Swap { + err = q.qmpMonitorCh.qmp.ExecuteBlockdevAddWithCache(q.qmpMonitorCh.ctx, drive.File, drive.ID, false, false, false) + } else if q.config.BlockDeviceCacheSet { err = q.qmpMonitorCh.qmp.ExecuteBlockdevAddWithCache(q.qmpMonitorCh.ctx, drive.File, drive.ID, q.config.BlockDeviceCacheDirect, q.config.BlockDeviceCacheNoflush, drive.ReadOnly) } else { err = q.qmpMonitorCh.qmp.ExecuteBlockdevAdd(q.qmpMonitorCh.ctx, drive.File, drive.ID, drive.ReadOnly) @@ -1248,25 +1250,8 @@ func (q *qemu) hotplugAddBlockDevice(ctx context.Context, drive *config.BlockDri }() switch { - case q.config.BlockDeviceDriver == config.VirtioBlockCCW: - driver := "virtio-blk-ccw" - - addr, bridge, err := q.arch.addDeviceToBridge(ctx, drive.ID, types.CCW) - if err != nil { - return err - } - var devNoHotplug string - devNoHotplug, err = bridge.AddressFormatCCW(addr) - if err != nil { - return err - } - drive.DevNo, err = bridge.AddressFormatCCWForVirtServer(addr) - if err != nil { - return err - } - if err = q.qmpMonitorCh.qmp.ExecuteDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, devNoHotplug, "", true, false); err != nil { - return err - } + case drive.Swap: + fallthrough case q.config.BlockDeviceDriver == config.VirtioBlock: driver := "virtio-blk-pci" addr, bridge, err := q.arch.addDeviceToBridge(ctx, drive.ID, types.PCI) @@ -1296,6 +1281,25 @@ func (q *qemu) hotplugAddBlockDevice(ctx context.Context, drive *config.BlockDri if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil { return err } + case q.config.BlockDeviceDriver == config.VirtioBlockCCW: + driver := "virtio-blk-ccw" + + addr, bridge, err := q.arch.addDeviceToBridge(ctx, drive.ID, types.CCW) + if err != nil { + return err + } + var devNoHotplug string + devNoHotplug, err = bridge.AddressFormatCCW(addr) + if err != nil { + return err + } + drive.DevNo, err = bridge.AddressFormatCCWForVirtServer(addr) + if err != nil { + return err + } + if err = q.qmpMonitorCh.qmp.ExecuteDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, devNoHotplug, "", true, false); err != nil { + return err + } case q.config.BlockDeviceDriver == config.VirtioSCSI: driver := "scsi-hd" @@ -1369,7 +1373,7 @@ func (q *qemu) hotplugBlockDevice(ctx context.Context, drive *config.BlockDrive, if op == addDevice { return q.hotplugAddBlockDevice(ctx, drive, op, devID) } - if q.config.BlockDeviceDriver == config.VirtioBlock { + if !drive.Swap && q.config.BlockDeviceDriver == config.VirtioBlock { if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil { return err } diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 2a41a1b4b9..c5ab8d5647 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -14,6 +14,8 @@ import ( "math" "net" "os" + "os/exec" + "path/filepath" "strings" "sync" "syscall" @@ -996,6 +998,87 @@ func (cw *consoleWatcher) stop() { } } +func (s *Sandbox) addSwap(ctx context.Context, swapID string, size int64) (*config.BlockDrive, error) { + swapFile := filepath.Join(getSandboxPath(s.id), swapID) + + swapFD, err := os.OpenFile(swapFile, os.O_CREATE, 0600) + if err != nil { + err = fmt.Errorf("creat swapfile %s fail %s", swapFile, err.Error()) + s.Logger().WithError(err).Error("addSwap") + return nil, err + } + swapFD.Close() + defer func() { + if err != nil { + os.Remove(swapFile) + } + }() + + // Check the size + pagesize := os.Getpagesize() + // mkswap refuses areas smaller than 10 pages. + size = int64(math.Max(float64(size), float64(pagesize*10))) + // Swapfile need a page to store the metadata + size += int64(pagesize) + + err = os.Truncate(swapFile, size) + if err != nil { + err = fmt.Errorf("truncate swapfile %s fail %s", swapFile, err.Error()) + s.Logger().WithError(err).Error("addSwap") + return nil, err + } + + err = exec.CommandContext(ctx, "/sbin/mkswap", swapFile).Run() + if err != nil { + err = fmt.Errorf("mkswap swapfile %s fail %s", swapFile, err.Error()) + s.Logger().WithError(err).Error("addSwap") + return nil, err + } + + blockDevice := &config.BlockDrive{ + File: swapFile, + Format: "raw", + ID: swapID, + Swap: true, + } + _, err = s.hypervisor.hotplugAddDevice(ctx, blockDevice, blockDev) + if err != nil { + err = fmt.Errorf("add swapfile %s device to VM fail %s", swapFile, err.Error()) + s.Logger().WithError(err).Error("addSwap") + return nil, err + } + defer func() { + if err != nil { + _, e := s.hypervisor.hotplugRemoveDevice(ctx, blockDevice, blockDev) + if e != nil { + s.Logger().Errorf("remove swapfile %s to VM fail %s", swapFile, e.Error()) + } + } + }() + + err = s.agent.addSwap(ctx, blockDevice.PCIPath) + if err != nil { + err = fmt.Errorf("agent add swapfile %s PCIPath %+v to VM fail %s", swapFile, blockDevice.PCIPath, err.Error()) + s.Logger().WithError(err).Error("addSwap") + return nil, err + } + + s.Logger().Infof("add swapfile %s size %d PCIPath %+v to VM success", swapFile, size, blockDevice.PCIPath) + + return blockDevice, nil +} + +func (s *Sandbox) removeSwap(ctx context.Context, blockDevice *config.BlockDrive) error { + err := os.Remove(blockDevice.File) + if err != nil { + err = fmt.Errorf("remove swapfile %s fail %s", blockDevice.File, err.Error()) + s.Logger().WithError(err).Error("removeSwap") + } else { + s.Logger().Infof("remove swapfile %s success", blockDevice.File) + } + return err +} + // startVM starts the VM. func (s *Sandbox) startVM(ctx context.Context) (err error) { span, ctx := katatrace.Trace(ctx, s.Logger(), "startVM", s.tracingTags()) @@ -1074,6 +1157,14 @@ func (s *Sandbox) startVM(ctx context.Context) (err error) { s.Logger().Info("Agent started in the sandbox") + defer func() { + if err != nil { + if e := s.agent.stopSandbox(ctx, s); e != nil { + s.Logger().WithError(e).WithField("sandboxid", s.id).Warning("Agent did not stop sandbox") + } + } + }() + return nil } @@ -1846,6 +1937,7 @@ func (s *Sandbox) updateResources(ctx context.Context) error { if err := s.agent.onlineCPUMem(ctx, 0, false); err != nil { return err } + return nil }