github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-04-29 12:14:48 +00:00
Code Issues Projects Releases Wiki Activity

runtime: Sandbox: Add addSwap and removeSwap

Browse Source 
addSwap will create a swap file, hotplug it to hypervisor as a special
block device and let agent to setup it in the guest kernel.
removeSwap will remove the swap file.

Just QEMU support addSwap.

Fixes: #2201

Signed-off-by: Hui Zhu <teawater@antfin.com>
This commit is contained in:
Hui Zhu 2021-07-05 16:28:32 +08:00
parent e1b91986d7
commit 243d4b8689
10 changed files with 162 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/runtime/virtcontainers/acrn.go
View File

@ -517,6 +517,10 @@ func (a *Acrn) stopSandbox(ctx context.Context, waitOnly bool) (err error) {
}
func (a *Acrn) updateBlockDevice(drive *config.BlockDrive) error {
if drive.Swap {
return fmt.Errorf("Acrn doesn't support swap")
}
var err error
if drive.File == "" || drive.Index >= AcrnBlkDevPoolSz {
return fmt.Errorf("Empty filepath or invalid drive index, Dive ID:%s, Drive Index:%d",

4
src/runtime/virtcontainers/agent.go
View File

@ -12,6 +12,7 @@ import (
persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
specs "github.com/opencontainers/runtime-spec/specs-go"
"golang.org/x/net/context"
@ -189,6 +190,9 @@ type agent interface {
// copyFile copies file from host to container's rootfs
copyFile(ctx context.Context, src, dst string) error
// Tell the agent to setup the swapfile in the guest
addSwap(ctx context.Context, PCIPath vcTypes.PciPath) error
// markDead tell agent that the guest is dead
markDead(ctx context.Context)

4
src/runtime/virtcontainers/clh.go
View File

@ -416,6 +416,10 @@ func clhPciInfoToPath(pciInfo chclient.PciDeviceInfo) (vcTypes.PciPath, error) {
}
func (clh *cloudHypervisor) hotplugAddBlockDevice(drive *config.BlockDrive) error {
if drive.Swap {
return fmt.Errorf("cloudHypervisor doesn't support swap")
}
if clh.config.BlockDeviceDriver != config.VirtioBlock {
return fmt.Errorf("incorrect hypervisor configuration on 'block_device_driver':"+
" using '%v' but only support '%v'", clh.config.BlockDeviceDriver, config.VirtioBlock)

3
src/runtime/virtcontainers/device/config/config.go
View File

@ -182,6 +182,9 @@ type BlockDrive struct {
// Pmem enables persistent memory. Use File as backing file
// for a nvdimm device in the guest
Pmem bool
// This block device is for swap
Swap bool
}
// VFIODeviceType indicates VFIO device type

4
src/runtime/virtcontainers/fc.go
View File

@ -1036,6 +1036,10 @@ func (fc *firecracker) addDevice(ctx context.Context, devInfo interface{}, devTy
// hotplugBlockDevice supported in Firecracker VMM
// hot add or remove a block device.
func (fc *firecracker) hotplugBlockDevice(ctx context.Context, drive config.BlockDrive, op operation) (interface{}, error) {
if drive.Swap {
return nil, fmt.Errorf("firecracker doesn't support swap")
}
var path string
var err error
driveID := fcDriveIndexToID(drive.Index)

12
src/runtime/virtcontainers/kata_agent.go
View File

@ -143,6 +143,7 @@ const (
grpcStopTracingRequest = "grpc.StopTracingRequest"
grpcGetOOMEventRequest = "grpc.GetOOMEventRequest"
grpcGetMetricsRequest = "grpc.GetMetricsRequest"
grpcAddSwapRequest = "grpc.AddSwapRequest"
)
// newKataAgent returns an agent from an agent type.
@ -2024,6 +2025,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) {
k.reqHandlers[grpcGetMetricsRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
return k.client.AgentServiceClient.GetMetrics(ctx, req.(*grpc.GetMetricsRequest))
}
k.reqHandlers[grpcAddSwapRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
return k.client.AgentServiceClient.AddSwap(ctx, req.(*grpc.AddSwapRequest))
}
}
func (k *kataAgent) getReqContext(ctx context.Context, reqName string) (newCtx context.Context, cancel context.CancelFunc) {
@ -2184,6 +2188,14 @@ func (k *kataAgent) copyFile(ctx context.Context, src, dst string) error {
return nil
}
func (k *kataAgent) addSwap(ctx context.Context, PCIPath vcTypes.PciPath) error {
span, ctx := katatrace.Trace(ctx, k.Logger(), "addSwap", kataAgentTracingTags)
defer span.End()
_, err := k.sendReq(ctx, &grpc.AddSwapRequest{PCIPath: PCIPath.ToArray()})
return err
}
func (k *kataAgent) markDead(ctx context.Context) {
k.Logger().Infof("mark agent dead")
k.dead = true

6
src/runtime/virtcontainers/mock_agent.go
View File

@ -12,6 +12,7 @@ import (
persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
specs "github.com/opencontainers/runtime-spec/specs-go"
"golang.org/x/net/context"
@ -215,6 +216,11 @@ func (n *mockAgent) copyFile(ctx context.Context, src, dst string) error {
return nil
}
// addSwap is the Noop agent setup swap. It does nothing.
func (n *mockAgent) addSwap(ctx context.Context, PCIPath vcTypes.PciPath) error {
return nil
}
func (n *mockAgent) markDead(ctx context.Context) {
}

8
src/runtime/virtcontainers/pkg/types/pcipath.go
View File

@ -75,6 +75,14 @@ func (p PciPath) IsNil() bool {
return p.slots == nil
}
func (p PciPath) ToArray() []uint32 {
var slots []uint32
for _, slot := range p.slots {
slots = append(slots, uint32(slot.slot))
}
return slots
}
func PciPathFromString(s string) (PciPath, error) {
if s == "" {
return PciPath{}, nil

46
src/runtime/virtcontainers/qemu.go
View File

@ -1232,7 +1232,9 @@ func (q *qemu) hotplugAddBlockDevice(ctx context.Context, drive *config.BlockDri
return nil
}
if q.config.BlockDeviceCacheSet {
if drive.Swap {
err = q.qmpMonitorCh.qmp.ExecuteBlockdevAddWithCache(q.qmpMonitorCh.ctx, drive.File, drive.ID, false, false, false)
} else if q.config.BlockDeviceCacheSet {
err = q.qmpMonitorCh.qmp.ExecuteBlockdevAddWithCache(q.qmpMonitorCh.ctx, drive.File, drive.ID, q.config.BlockDeviceCacheDirect, q.config.BlockDeviceCacheNoflush, drive.ReadOnly)
} else {
err = q.qmpMonitorCh.qmp.ExecuteBlockdevAdd(q.qmpMonitorCh.ctx, drive.File, drive.ID, drive.ReadOnly)
@ -1248,25 +1250,8 @@ func (q *qemu) hotplugAddBlockDevice(ctx context.Context, drive *config.BlockDri
}()
switch {
case q.config.BlockDeviceDriver == config.VirtioBlockCCW:
driver := "virtio-blk-ccw"
addr, bridge, err := q.arch.addDeviceToBridge(ctx, drive.ID, types.CCW)
if err != nil {
return err
}
var devNoHotplug string
devNoHotplug, err = bridge.AddressFormatCCW(addr)
if err != nil {
return err
}
drive.DevNo, err = bridge.AddressFormatCCWForVirtServer(addr)
if err != nil {
return err
}
if err = q.qmpMonitorCh.qmp.ExecuteDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, devNoHotplug, "", true, false); err != nil {
return err
}
case drive.Swap:
fallthrough
case q.config.BlockDeviceDriver == config.VirtioBlock:
driver := "virtio-blk-pci"
addr, bridge, err := q.arch.addDeviceToBridge(ctx, drive.ID, types.PCI)
@ -1296,6 +1281,25 @@ func (q *qemu) hotplugAddBlockDevice(ctx context.Context, drive *config.BlockDri
if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil {
return err
}
case q.config.BlockDeviceDriver == config.VirtioBlockCCW:
driver := "virtio-blk-ccw"
addr, bridge, err := q.arch.addDeviceToBridge(ctx, drive.ID, types.CCW)
if err != nil {
return err
}
var devNoHotplug string
devNoHotplug, err = bridge.AddressFormatCCW(addr)
if err != nil {
return err
}
drive.DevNo, err = bridge.AddressFormatCCWForVirtServer(addr)
if err != nil {
return err
}
if err = q.qmpMonitorCh.qmp.ExecuteDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, devNoHotplug, "", true, false); err != nil {
return err
}
case q.config.BlockDeviceDriver == config.VirtioSCSI:
driver := "scsi-hd"
@ -1369,7 +1373,7 @@ func (q *qemu) hotplugBlockDevice(ctx context.Context, drive *config.BlockDrive,
if op == addDevice {
return q.hotplugAddBlockDevice(ctx, drive, op, devID)
}
if q.config.BlockDeviceDriver == config.VirtioBlock {
if !drive.Swap && q.config.BlockDeviceDriver == config.VirtioBlock {
if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil {
return err
}

92
src/runtime/virtcontainers/sandbox.go
View File

@ -14,6 +14,8 @@ import (
"math"
"net"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"syscall"
@ -996,6 +998,87 @@ func (cw *consoleWatcher) stop() {
}
}
func (s *Sandbox) addSwap(ctx context.Context, swapID string, size int64) (*config.BlockDrive, error) {
swapFile := filepath.Join(getSandboxPath(s.id), swapID)
swapFD, err := os.OpenFile(swapFile, os.O_CREATE, 0600)
if err != nil {
err = fmt.Errorf("creat swapfile %s fail %s", swapFile, err.Error())
s.Logger().WithError(err).Error("addSwap")
return nil, err
}
swapFD.Close()
defer func() {
if err != nil {
os.Remove(swapFile)
}
}()
// Check the size
pagesize := os.Getpagesize()
// mkswap refuses areas smaller than 10 pages.
size = int64(math.Max(float64(size), float64(pagesize*10)))
// Swapfile need a page to store the metadata
size += int64(pagesize)
err = os.Truncate(swapFile, size)
if err != nil {
err = fmt.Errorf("truncate swapfile %s fail %s", swapFile, err.Error())
s.Logger().WithError(err).Error("addSwap")
return nil, err
}
err = exec.CommandContext(ctx, "/sbin/mkswap", swapFile).Run()
if err != nil {
err = fmt.Errorf("mkswap swapfile %s fail %s", swapFile, err.Error())
s.Logger().WithError(err).Error("addSwap")
return nil, err
}
blockDevice := &config.BlockDrive{
File: swapFile,
Format: "raw",
ID: swapID,
Swap: true,
}
_, err = s.hypervisor.hotplugAddDevice(ctx, blockDevice, blockDev)
if err != nil {
err = fmt.Errorf("add swapfile %s device to VM fail %s", swapFile, err.Error())
s.Logger().WithError(err).Error("addSwap")
return nil, err
}
defer func() {
if err != nil {
_, e := s.hypervisor.hotplugRemoveDevice(ctx, blockDevice, blockDev)
if e != nil {
s.Logger().Errorf("remove swapfile %s to VM fail %s", swapFile, e.Error())
}
}
}()
err = s.agent.addSwap(ctx, blockDevice.PCIPath)
if err != nil {
err = fmt.Errorf("agent add swapfile %s PCIPath %+v to VM fail %s", swapFile, blockDevice.PCIPath, err.Error())
s.Logger().WithError(err).Error("addSwap")
return nil, err
}
s.Logger().Infof("add swapfile %s size %d PCIPath %+v to VM success", swapFile, size, blockDevice.PCIPath)
return blockDevice, nil
}
func (s *Sandbox) removeSwap(ctx context.Context, blockDevice *config.BlockDrive) error {
err := os.Remove(blockDevice.File)
if err != nil {
err = fmt.Errorf("remove swapfile %s fail %s", blockDevice.File, err.Error())
s.Logger().WithError(err).Error("removeSwap")
} else {
s.Logger().Infof("remove swapfile %s success", blockDevice.File)
}
return err
}
// startVM starts the VM.
func (s *Sandbox) startVM(ctx context.Context) (err error) {
span, ctx := katatrace.Trace(ctx, s.Logger(), "startVM", s.tracingTags())
@ -1074,6 +1157,14 @@ func (s *Sandbox) startVM(ctx context.Context) (err error) {
s.Logger().Info("Agent started in the sandbox")
defer func() {
if err != nil {
if e := s.agent.stopSandbox(ctx, s); e != nil {
s.Logger().WithError(e).WithField("sandboxid", s.id).Warning("Agent did not stop sandbox")
}
}
}()
return nil
}
@ -1846,6 +1937,7 @@ func (s *Sandbox) updateResources(ctx context.Context) error {
if err := s.agent.onlineCPUMem(ctx, 0, false); err != nil {
return err
}
return nil
}