persist: manage "hypervisor.json" with new store

Fixes #803

Merge "hypervisor.json" into "persist.json", so the new store can take
care of hypervisor data now.

Signed-off-by: Wei Zhang <weizhang555.zw@gmail.com>

virtcontainers/acrn.go

@@ -17,6 +17,7 @@ import (
 	"time"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
@@ -230,7 +231,18 @@ func (a *acrn) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto
 	a.store = vcStore
 	a.config = *hypervisorConfig
 	a.arch = newAcrnArch(a.config)
-	if err = a.store.Load(store.Hypervisor, &a.state); err != nil {
+
+	var create bool
+
+	if a.store != nil { //use old store
+		if err = a.store.Load(store.Hypervisor, &a.info); err != nil {
+			create = true
+		}
+	} else if a.info.PID == 0 { // new store
+		create = true
+	}
+
+	if create {
 		// acrn currently supports only known UUIDs for security
 		// reasons (FuSa). When launching VM, only these pre-defined
 		// UUID should be used else VM launch will fail. acrn team is
@@ -246,15 +258,11 @@ func (a *acrn) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto
 			return err
 		}
 
-		if err = a.store.Store(store.Hypervisor, a.state); err != nil {
+		if err = a.storeInfo(); err != nil {
 			return err
 		}
 	}
 
-	if err = a.store.Load(store.Hypervisor, &a.info); err != nil {
-		a.Logger().WithField("function", "setup").WithError(err).Info("No info could be fetched")
-	}
-
 	return nil
 }
 
@@ -619,3 +627,24 @@ func (a *acrn) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig,
 func (a *acrn) toGrpc() ([]byte, error) {
 	return nil, errors.New("acrn is not supported by VM cache")
 }
+
+func (a *acrn) storeInfo() error {
+	if a.store != nil {
+		if err := a.store.Store(store.Hypervisor, a.info); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (a *acrn) save() (s persistapi.HypervisorState) {
+	s.Pid = a.pid()
+	s.Type = string(AcrnHypervisor)
+	s.UUID = a.state.UUID
+	return
+}
+
+func (a *acrn) load(s persistapi.HypervisorState) {
+	a.info.PID = s.Pid
+	a.state.UUID = s.UUID
+}

virtcontainers/fc.go

@@ -22,6 +22,7 @@ import (
 
 	httptransport "github.com/go-openapi/runtime/client"
 	"github.com/go-openapi/strfmt"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client"
 	models "github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client/models"
 	ops "github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client/operations"
@@ -234,8 +235,10 @@ func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS N
 
 	// No need to return an error from there since there might be nothing
 	// to fetch if this is the first time the hypervisor is created.
-	if err := fc.store.Load(store.Hypervisor, &fc.info); err != nil {
-		fc.Logger().WithField("function", "init").WithError(err).Info("No info could be fetched")
+	if fc.store != nil {
+		if err := fc.store.Load(store.Hypervisor, &fc.info); err != nil {
+			fc.Logger().WithField("function", "init").WithError(err).Info("No info could be fetched")
+		}
 	}
 
 	return nil
@@ -388,7 +391,10 @@ func (fc *firecracker) fcInit(timeout int) error {
 	fc.state.set(apiReady)
 
 	// Store VMM information
-	return fc.store.Store(store.Hypervisor, fc.info)
+	if fc.store != nil {
+		return fc.store.Store(store.Hypervisor, fc.info)
+	}
+	return nil
 }
 
 func (fc *firecracker) fcEnd() (err error) {
@@ -988,3 +994,13 @@ func (fc *firecracker) fromGrpc(ctx context.Context, hypervisorConfig *Hyperviso
 func (fc *firecracker) toGrpc() ([]byte, error) {
 	return nil, errors.New("firecracker is not supported by VM cache")
 }
+
+func (fc *firecracker) save() (s persistapi.HypervisorState) {
+	s.Pid = fc.pid()
+	s.Type = string(FirecrackerHypervisor)
+	return
+}
+
+func (fc *firecracker) load(s persistapi.HypervisorState) {
+	fc.info.PID = s.Pid
+}

virtcontainers/hypervisor.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 )
@@ -670,4 +671,7 @@ type hypervisor interface {
 	pid() int
 	fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, store *store.VCStore, j []byte) error
 	toGrpc() ([]byte, error)
+
+	save() persistapi.HypervisorState
+	load(persistapi.HypervisorState)
 }

virtcontainers/mock_hypervisor.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"os"
 
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 )
@@ -114,3 +115,9 @@ func (m *mockHypervisor) fromGrpc(ctx context.Context, hypervisorConfig *Hypervi
 func (m *mockHypervisor) toGrpc() ([]byte, error) {
 	return nil, errors.New("firecracker is not supported by VM cache")
 }
+
+func (m *mockHypervisor) save() (s persistapi.HypervisorState) {
+	return
+}
+
+func (m *mockHypervisor) load(s persistapi.HypervisorState) {}

virtcontainers/persist.go

@@ -57,7 +57,9 @@ func (s *Sandbox) dumpState(ss *persistapi.SandboxState, cs map[string]persistap
 	}
 }
 
-func (s *Sandbox) dumpHypervisor(ss *persistapi.SandboxState, cs map[string]persistapi.ContainerState) {
+func (s *Sandbox) dumpHypervisor(ss *persistapi.SandboxState) {
+	ss.HypervisorState = s.hypervisor.save()
+	// BlockIndex will be moved from sandbox state to hypervisor state later
 	ss.HypervisorState.BlockIndex = s.state.BlockIndex
 }
 
@@ -160,7 +162,7 @@ func (s *Sandbox) Save() error {
 
 	s.dumpVersion(&ss)
 	s.dumpState(&ss, cs)
-	s.dumpHypervisor(&ss, cs)
+	s.dumpHypervisor(&ss)
 	s.dumpDevices(&ss, cs)
 	s.dumpProcess(cs)
 	s.dumpMounts(cs)
@@ -190,6 +192,10 @@ func (c *Container) loadContState(cs persistapi.ContainerState) {
 	}
 }
 
+func (s *Sandbox) loadHypervisor(hs persistapi.HypervisorState) {
+	s.hypervisor.load(hs)
+}
+
 func (s *Sandbox) loadDevices(devStates []persistapi.DeviceState) {
 	s.devManager.LoadDevices(devStates)
 }
@@ -237,6 +243,7 @@ func (s *Sandbox) Restore() error {
 	}
 
 	s.loadState(ss)
+	s.loadHypervisor(ss.HypervisorState)
 	s.loadDevices(ss.Devices)
 	return nil
 }

virtcontainers/persist/api/hypervisor.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2016 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package persistapi
+
+// Bridge is a bridge where devices can be hot plugged
+type Bridge struct {
+	// DeviceAddr contains information about devices plugged and its address in the bridge
+	DeviceAddr map[uint32]string
+
+	// Type is the type of the bridge (pci, pcie, etc)
+	Type string
+
+	//ID is used to identify the bridge in the hypervisor
+	ID string
+
+	// Addr is the PCI/e slot of the bridge
+	Addr int
+}
+
+// CPUDevice represents a CPU device which was hot-added in a running VM
+type CPUDevice struct {
+	// ID is used to identify this CPU in the hypervisor options.
+	ID string
+}
+
+type HypervisorState struct {
+	Pid int
+	// Type of hypervisor, E.g. qemu/firecracker/acrn.
+	Type       string
+	BlockIndex int
+	UUID       string
+
+	// Belows are qemu specific
+	// Refs: virtcontainers/qemu.go:QemuState
+	Bridges []Bridge
+	// HotpluggedCPUs is the list of CPUs that were hot-added
+	HotpluggedVCPUs      []CPUDevice
+	HotpluggedMemory     int
+	HotplugVFIOOnRootBus bool
+}

virtcontainers/persist/api/sandbox.go

@@ -8,44 +8,6 @@ package persistapi
 
 // ============= sandbox level resources =============
 
-// SetFunc is function hook used for setting sandbox/container state
-// It can be registered to dynamically set state files when dump
-type SetFunc (func(*SandboxState, map[string]ContainerState) error)
-
-// Bridge is a bridge where devices can be hot plugged
-type Bridge struct {
-	// DeviceAddr contains information about devices plugged and its address in the bridge
-	DeviceAddr map[uint32]string
-
-	// Type is the type of the bridge (pci, pcie, etc)
-	Type string
-
-	//ID is used to identify the bridge in the hypervisor
-	ID string
-
-	// Addr is the PCI/e slot of the bridge
-	Addr int
-}
-
-// CPUDevice represents a CPU device which was hot-added in a running VM
-type CPUDevice struct {
-	// ID is used to identify this CPU in the hypervisor options.
-	ID string
-}
-
-// HypervisorState saves state of hypervisor
-// Refs: virtcontainers/qemu.go:QemuState
-type HypervisorState struct {
-	Pid     int
-	Bridges []Bridge
-	// HotpluggedCPUs is the list of CPUs that were hot-added
-	HotpluggedVCPUs      []CPUDevice
-	HotpluggedMemory     int
-	UUID                 string
-	HotplugVFIOOnRootBus bool
-	BlockIndex           int
-}
-
 // ProxyState save proxy state data
 type ProxyState struct {
 	// Pid of proxy process

virtcontainers/qemu.go

@@ -29,9 +29,11 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
+
 	"golang.org/x/sys/unix"
 )
 
@@ -252,7 +254,17 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto
 		q.nvdimmCount = 0
 	}
 
-	if err = q.store.Load(store.Hypervisor, &q.state); err != nil {
+	var create bool
+	if q.store != nil { //use old store
+		if err := q.store.Load(store.Hypervisor, &q.state); err != nil {
+			// hypervisor doesn't exist, create new one
+			create = true
+		}
+	} else if q.state.UUID == "" { // new store
+		create = true
+	}
+
+	if create {
 		q.Logger().Debug("Creating bridges")
 		q.state.Bridges = q.arch.bridges(q.config.DefaultBridges)
 
@@ -267,7 +279,7 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto
 			return err
 		}
 
-		if err = q.store.Store(store.Hypervisor, q.state); err != nil {
+		if err = q.storeState(); err != nil {
 			return err
 		}
 	}
@@ -1222,7 +1234,7 @@ func (q *qemu) hotplugAddDevice(devInfo interface{}, devType deviceType) (interf
 		return data, err
 	}
 
-	return data, q.store.Store(store.Hypervisor, q.state)
+	return data, q.storeState()
 }
 
 func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) {
@@ -1234,7 +1246,7 @@ func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (int
 		return data, err
 	}
 
-	return data, q.store.Store(store.Hypervisor, q.state)
+	return data, q.storeState()
 }
 
 func (q *qemu) hotplugCPUs(vcpus uint32, op operation) (uint32, error) {
@@ -1314,12 +1326,12 @@ func (q *qemu) hotplugAddCPUs(amount uint32) (uint32, error) {
 		hotpluggedVCPUs++
 		if hotpluggedVCPUs == amount {
 			// All vCPUs were hotplugged
-			return amount, q.store.Store(store.Hypervisor, q.state)
+			return amount, q.storeState()
 		}
 	}
 
 	// All vCPUs were NOT hotplugged
-	if err := q.store.Store(store.Hypervisor, q.state); err != nil {
+	if err := q.storeState(); err != nil {
 		q.Logger().Errorf("failed to save hypervisor state after hotplug %d vCPUs: %v", hotpluggedVCPUs, err)
 	}
 
@@ -1339,7 +1351,7 @@ func (q *qemu) hotplugRemoveCPUs(amount uint32) (uint32, error) {
 		// get the last vCPUs and try to remove it
 		cpu := q.state.HotpluggedVCPUs[len(q.state.HotpluggedVCPUs)-1]
 		if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, cpu.ID); err != nil {
-			_ = q.store.Store(store.Hypervisor, q.state)
+			q.storeState()
 			return i, fmt.Errorf("failed to hotunplug CPUs, only %d CPUs were hotunplugged: %v", i, err)
 		}
 
@@ -1347,7 +1359,7 @@ func (q *qemu) hotplugRemoveCPUs(amount uint32) (uint32, error) {
 		q.state.HotpluggedVCPUs = q.state.HotpluggedVCPUs[:len(q.state.HotpluggedVCPUs)-1]
 	}
 
-	return amount, q.store.Store(store.Hypervisor, q.state)
+	return amount, q.storeState()
 }
 
 func (q *qemu) hotplugMemory(memDev *memoryDevice, op operation) (int, error) {
@@ -1453,7 +1465,7 @@ func (q *qemu) hotplugAddMemory(memDev *memoryDevice) (int, error) {
 		}
 	}
 	q.state.HotpluggedMemory += memDev.sizeMB
-	return memDev.sizeMB, q.store.Store(store.Hypervisor, q.state)
+	return memDev.sizeMB, q.storeState()
 }
 
 func (q *qemu) pauseSandbox() error {
@@ -1907,3 +1919,57 @@ func (q *qemu) toGrpc() ([]byte, error) {
 
 	return json.Marshal(&qp)
 }
+
+func (q *qemu) storeState() error {
+	if q.store != nil {
+		if err := q.store.Store(store.Hypervisor, q.state); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (q *qemu) save() (s persistapi.HypervisorState) {
+	s.Pid = q.pid()
+	s.Type = string(QemuHypervisor)
+	s.UUID = q.state.UUID
+	s.HotpluggedMemory = q.state.HotpluggedMemory
+	s.HotplugVFIOOnRootBus = q.state.HotplugVFIOOnRootBus
+
+	for _, bridge := range q.state.Bridges {
+		s.Bridges = append(s.Bridges, persistapi.Bridge{
+			DeviceAddr: bridge.Address,
+			Type:       string(bridge.Type),
+			ID:         bridge.ID,
+			Addr:       bridge.Addr,
+		})
+	}
+
+	for _, cpu := range q.state.HotpluggedVCPUs {
+		s.HotpluggedVCPUs = append(s.HotpluggedVCPUs, persistapi.CPUDevice{
+			ID: cpu.ID,
+		})
+	}
+	return
+}
+
+func (q *qemu) load(s persistapi.HypervisorState) {
+	q.state.UUID = s.UUID
+	q.state.HotpluggedMemory = s.HotpluggedMemory
+	q.state.HotplugVFIOOnRootBus = s.HotplugVFIOOnRootBus
+
+	for _, bridge := range s.Bridges {
+		q.state.Bridges = append(q.state.Bridges, types.PCIBridge{
+			Address: bridge.DeviceAddr,
+			Type:    types.PCIType(bridge.Type),
+			ID:      bridge.ID,
+			Addr:    bridge.Addr,
+		})
+	}
+
+	for _, cpu := range s.HotpluggedVCPUs {
+		q.state.HotpluggedVCPUs = append(q.state.HotpluggedVCPUs, CPUDevice{
+			ID: cpu.ID,
+		})
+	}
+}

virtcontainers/sandbox.go

@@ -474,12 +474,10 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
 	}
 
 	if s.supportNewStore() {
-		s.devManager = deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, nil)
-
-		if err := s.Restore(); err == nil && s.state.State != "" {
+		// Restored successfully from newstore before.
+		if s.state.State != "" {
 			return s, nil
 		}
-
 	} else {
 		devices, err := s.store.LoadDevices()
 		if err != nil {
@@ -574,8 +572,20 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 		}
 	}()
 
-	if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.store); err != nil {
-		return nil, err
+	if s.supportNewStore() {
+		s.devManager = deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, nil)
+
+		// Ignore the error. Restore can fail for a new sandbox
+		s.Restore()
+
+		// new store doesn't require hypervisor to be stored immediately
+		if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, nil); err != nil {
+			return nil, err
+		}
+	} else {
+		if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.store); err != nil {
+			return nil, err
+		}
 	}
 
 	agentConfig, err := newAgentConfig(sandboxConfig.AgentType, sandboxConfig.AgentConfig)
@@ -913,6 +923,12 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, erro
 		return nil, err
 	}
 
+	if s.supportNewStore() {
+		if err := s.Save(); err != nil {
+			return nil, err
+		}
+	}
+
 	// Add network for vm
 	inf.PciAddr = endpoint.PciAddr()
 	return s.agent.updateInterface(inf)
@@ -930,6 +946,13 @@ func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, e
 			if err := s.store.Store(store.Network, s.networkNS); err != nil {
 				return inf, err
 			}
+
+			if s.supportNewStore() {
+				if err := s.Save(); err != nil {
+					return inf, err
+				}
+			}
+
 			break
 		}
 	}