Merge pull request #1889 from WeiZhang555/persist-data

persist: manage "hypervisor.json" with new store

virtcontainers/acrn.go

@@ -17,6 +17,7 @@ import (
 	"time"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
@@ -230,7 +231,18 @@ func (a *acrn) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto
 	a.store = vcStore
 	a.config = *hypervisorConfig
 	a.arch = newAcrnArch(a.config)
-	if err = a.store.Load(store.Hypervisor, &a.state); err != nil {
+
+	var create bool
+
+	if a.store != nil { //use old store
+		if err = a.store.Load(store.Hypervisor, &a.info); err != nil {
+			create = true
+		}
+	} else if a.info.PID == 0 { // new store
+		create = true
+	}
+
+	if create {
 		// acrn currently supports only known UUIDs for security
 		// reasons (FuSa). When launching VM, only these pre-defined
 		// UUID should be used else VM launch will fail. acrn team is
@@ -246,15 +258,11 @@ func (a *acrn) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto
 			return err
 		}
 
-		if err = a.store.Store(store.Hypervisor, a.state); err != nil {
+		if err = a.storeInfo(); err != nil {
 			return err
 		}
 	}
 
-	if err = a.store.Load(store.Hypervisor, &a.info); err != nil {
-		a.Logger().WithField("function", "setup").WithError(err).Info("No info could be fetched")
-	}
-
 	return nil
 }
 
@@ -619,3 +627,24 @@ func (a *acrn) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig,
 func (a *acrn) toGrpc() ([]byte, error) {
 	return nil, errors.New("acrn is not supported by VM cache")
 }
+
+func (a *acrn) storeInfo() error {
+	if a.store != nil {
+		if err := a.store.Store(store.Hypervisor, a.info); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (a *acrn) save() (s persistapi.HypervisorState) {
+	s.Pid = a.pid()
+	s.Type = string(AcrnHypervisor)
+	s.UUID = a.state.UUID
+	return
+}
+
+func (a *acrn) load(s persistapi.HypervisorState) {
+	a.info.PID = s.Pid
+	a.state.UUID = s.UUID
+}

virtcontainers/agent.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/kata-containers/agent/protocols/grpc"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 	"github.com/mitchellh/mapstructure"
@@ -255,4 +256,10 @@ type agent interface {
 
 	// cleanup removes all on disk information generated by the agent
 	cleanup(s *Sandbox)
+
+	// return data for saving
+	save() persistapi.AgentState
+
+	// load data from disk
+	load(persistapi.AgentState)
 }

virtcontainers/bridgedmacvlan_endpoint.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 
 	"github.com/containernetworking/plugins/pkg/ns"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
 // BridgedMacvlanEndpoint represents a macvlan endpoint that is bridged to the VM
@@ -115,3 +116,23 @@ func (endpoint *BridgedMacvlanEndpoint) HotAttach(h hypervisor) error {
 func (endpoint *BridgedMacvlanEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
 	return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot detach")
 }
+
+func (endpoint *BridgedMacvlanEndpoint) save() persistapi.NetworkEndpoint {
+	netpair := saveNetIfPair(&endpoint.NetPair)
+
+	return persistapi.NetworkEndpoint{
+		Type: string(endpoint.Type()),
+		BridgedMacvlan: &persistapi.BridgedMacvlanEndpoint{
+			NetPair: *netpair,
+		},
+	}
+}
+
+func (endpoint *BridgedMacvlanEndpoint) load(s persistapi.NetworkEndpoint) {
+	endpoint.EndpointType = BridgedMacvlanEndpointType
+
+	if s.BridgedMacvlan != nil {
+		netpair := loadNetIfPair(&s.BridgedMacvlan.NetPair)
+		endpoint.NetPair = *netpair
+	}
+}

virtcontainers/endpoint.go

@@ -7,6 +7,8 @@ package virtcontainers
 
 import (
 	"fmt"
+
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
 // Endpoint represents a physical or virtual network interface.
@@ -24,6 +26,9 @@ type Endpoint interface {
 	Detach(netNsCreated bool, netNsPath string) error
 	HotAttach(h hypervisor) error
 	HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error
+
+	save() persistapi.NetworkEndpoint
+	load(persistapi.NetworkEndpoint)
 }
 
 // EndpointType identifies the type of the network endpoint.
@@ -102,3 +107,79 @@ func (endpointType *EndpointType) String() string {
 		return ""
 	}
 }
+
+func saveTapIf(tapif *TapInterface) *persistapi.TapInterface {
+	if tapif == nil {
+		return nil
+	}
+
+	return &persistapi.TapInterface{
+		ID:   tapif.ID,
+		Name: tapif.Name,
+		TAPIface: persistapi.NetworkInterface{
+			Name:     tapif.TAPIface.Name,
+			HardAddr: tapif.TAPIface.HardAddr,
+			Addrs:    tapif.TAPIface.Addrs,
+		},
+	}
+}
+
+func loadTapIf(tapif *persistapi.TapInterface) *TapInterface {
+	if tapif == nil {
+		return nil
+	}
+
+	return &TapInterface{
+		ID:   tapif.ID,
+		Name: tapif.Name,
+		TAPIface: NetworkInterface{
+			Name:     tapif.TAPIface.Name,
+			HardAddr: tapif.TAPIface.HardAddr,
+			Addrs:    tapif.TAPIface.Addrs,
+		},
+	}
+}
+
+func saveNetIfPair(pair *NetworkInterfacePair) *persistapi.NetworkInterfacePair {
+	if pair == nil {
+		return nil
+	}
+
+	epVirtIf := pair.VirtIface
+
+	tapif := saveTapIf(&pair.TapInterface)
+
+	virtif := persistapi.NetworkInterface{
+		Name:     epVirtIf.Name,
+		HardAddr: epVirtIf.HardAddr,
+		Addrs:    epVirtIf.Addrs,
+	}
+
+	return &persistapi.NetworkInterfacePair{
+		TapInterface:         *tapif,
+		VirtIface:            virtif,
+		NetInterworkingModel: int(pair.NetInterworkingModel),
+	}
+}
+
+func loadNetIfPair(pair *persistapi.NetworkInterfacePair) *NetworkInterfacePair {
+	if pair == nil {
+		return nil
+	}
+
+	savedVirtIf := pair.VirtIface
+
+	tapif := loadTapIf(&pair.TapInterface)
+
+	virtif := NetworkInterface{
+		Name:     savedVirtIf.Name,
+		HardAddr: savedVirtIf.HardAddr,
+		Addrs:    savedVirtIf.Addrs,
+	}
+
+	return &NetworkInterfacePair{
+		TapInterface:         *tapif,
+		VirtIface:            virtif,
+		NetInterworkingModel: NetInterworkingModel(pair.NetInterworkingModel),
+	}
+}

virtcontainers/endpoint_test.go

@@ -6,6 +6,10 @@
 package virtcontainers
 
 import (
+	"io/ioutil"
+	"net"
+	"os"
+	"reflect"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -79,3 +83,41 @@ func TestIncorrectEndpointTypeString(t *testing.T) {
 	var endpointType EndpointType
 	testEndpointTypeString(t, &endpointType, "")
 }
+
+func TestSaveLoadIfPair(t *testing.T) {
+	macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04}
+
+	tmpfile, err := ioutil.TempFile("", "vc-save-load-net-")
+	assert.Nil(t, err)
+	defer os.Remove(tmpfile.Name())
+
+	netPair := &NetworkInterfacePair{
+		TapInterface: TapInterface{
+			ID:   "uniqueTestID-4",
+			Name: "br4_kata",
+			TAPIface: NetworkInterface{
+				Name:     "tap4_kata",
+				HardAddr: macAddr.String(),
+			},
+			VMFds:    []*os.File{tmpfile}, // won't be saved to disk
+			VhostFds: []*os.File{tmpfile}, // won't be saved to disk
+		},
+		VirtIface: NetworkInterface{
+			Name:     "eth4",
+			HardAddr: macAddr.String(),
+		},
+		NetInterworkingModel: DefaultNetInterworkingModel,
+	}
+
+	// Save to disk then load it back.
+	savedIfPair := saveNetIfPair(netPair)
+	loadedIfPair := loadNetIfPair(savedIfPair)
+
+	// Since VMFds and VhostFds are't saved, netPair and loadedIfPair are not equal.
+	assert.False(t, reflect.DeepEqual(netPair, loadedIfPair))
+
+	netPair.TapInterface.VMFds = nil
+	netPair.TapInterface.VhostFds = nil
+	// They are equal now.
+	assert.True(t, reflect.DeepEqual(netPair, loadedIfPair))
+}

virtcontainers/fc.go

@@ -22,6 +22,7 @@ import (
 
 	httptransport "github.com/go-openapi/runtime/client"
 	"github.com/go-openapi/strfmt"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client"
 	models "github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client/models"
 	ops "github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client/operations"
@@ -234,8 +235,10 @@ func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS N
 
 	// No need to return an error from there since there might be nothing
 	// to fetch if this is the first time the hypervisor is created.
-	if err := fc.store.Load(store.Hypervisor, &fc.info); err != nil {
+	if fc.store != nil {
-		fc.Logger().WithField("function", "init").WithError(err).Info("No info could be fetched")
+		if err := fc.store.Load(store.Hypervisor, &fc.info); err != nil {
+			fc.Logger().WithField("function", "init").WithError(err).Info("No info could be fetched")
+		}
 	}
 
 	return nil
@@ -321,11 +324,13 @@ func (fc *firecracker) fcInit(timeout int) error {
 
 	// Fetch sandbox network to be able to access it from the sandbox structure.
 	var networkNS NetworkNamespace
-	if err := fc.store.Load(store.Network, &networkNS); err == nil {
+	if fc.store != nil {
-		if networkNS.NetNsPath == "" {
+		if err := fc.store.Load(store.Network, &networkNS); err == nil {
-			fc.Logger().WithField("NETWORK NAMESPACE NULL", networkNS).Warn()
+			if networkNS.NetNsPath == "" {
+				fc.Logger().WithField("NETWORK NAMESPACE NULL", networkNS).Warn()
+			}
+			fc.netNSPath = networkNS.NetNsPath
 		}
-		fc.netNSPath = networkNS.NetNsPath
 	}
 
 	err := os.MkdirAll(fc.jailerRoot, store.DirMode)
@@ -388,7 +393,10 @@ func (fc *firecracker) fcInit(timeout int) error {
 	fc.state.set(apiReady)
 
 	// Store VMM information
-	return fc.store.Store(store.Hypervisor, fc.info)
+	if fc.store != nil {
+		return fc.store.Store(store.Hypervisor, fc.info)
+	}
+	return nil
 }
 
 func (fc *firecracker) fcEnd() (err error) {
@@ -988,3 +996,13 @@ func (fc *firecracker) fromGrpc(ctx context.Context, hypervisorConfig *Hyperviso
 func (fc *firecracker) toGrpc() ([]byte, error) {
 	return nil, errors.New("firecracker is not supported by VM cache")
 }
+
+func (fc *firecracker) save() (s persistapi.HypervisorState) {
+	s.Pid = fc.pid()
+	s.Type = string(FirecrackerHypervisor)
+	return
+}
+
+func (fc *firecracker) load(s persistapi.HypervisorState) {
+	fc.info.PID = s.Pid
+}

virtcontainers/hypervisor.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 )
@@ -670,4 +671,7 @@ type hypervisor interface {
 	pid() int
 	fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, store *store.VCStore, j []byte) error
 	toGrpc() ([]byte, error)
+
+	save() persistapi.HypervisorState
+	load(persistapi.HypervisorState)
 }

virtcontainers/ipvlan_endpoint.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 
 	"github.com/containernetworking/plugins/pkg/ns"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
 // IPVlanEndpoint represents a ipvlan endpoint that is bridged to the VM
@@ -118,3 +119,23 @@ func (endpoint *IPVlanEndpoint) HotAttach(h hypervisor) error {
 func (endpoint *IPVlanEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
 	return fmt.Errorf("IPVlanEndpoint does not support Hot detach")
 }
+
+func (endpoint *IPVlanEndpoint) save() persistapi.NetworkEndpoint {
+	netpair := saveNetIfPair(&endpoint.NetPair)
+
+	return persistapi.NetworkEndpoint{
+		Type: string(endpoint.Type()),
+		IPVlan: &persistapi.IPVlanEndpoint{
+			NetPair: *netpair,
+		},
+	}
+}
+
+func (endpoint *IPVlanEndpoint) load(s persistapi.NetworkEndpoint) {
+	endpoint.EndpointType = IPVlanEndpointType
+
+	if s.IPVlan != nil {
+		netpair := loadNetIfPair(&s.IPVlan.NetPair)
+		endpoint.NetPair = *netpair
+	}
+}

virtcontainers/kata_agent.go

@@ -24,6 +24,7 @@ import (
 	kataclient "github.com/kata-containers/agent/protocols/client"
 	"github.com/kata-containers/agent/protocols/grpc"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
 	ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
 	vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
@@ -286,8 +287,10 @@ func (k *kataAgent) init(ctx context.Context, sandbox *Sandbox, config interface
 	k.proxyBuiltIn = isProxyBuiltIn(sandbox.config.ProxyType)
 
 	// Fetch agent runtime info.
-	if err := sandbox.store.Load(store.Agent, &k.state); err != nil {
+	if !sandbox.supportNewStore() {
-		k.Logger().Debug("Could not retrieve anything from storage")
+		if err := sandbox.store.Load(store.Agent, &k.state); err != nil {
+			k.Logger().Debug("Could not retrieve anything from storage")
+		}
 	}
 
 	return disableVMShutdown, nil
@@ -686,7 +689,7 @@ func (k *kataAgent) setProxy(sandbox *Sandbox, proxy proxy, pid int, url string)
 	k.proxy = proxy
 	k.state.ProxyPid = pid
 	k.state.URL = url
-	if sandbox != nil {
+	if sandbox != nil && !sandbox.supportNewStore() {
 		if err := sandbox.store.Store(store.Agent, k.state); err != nil {
 			return err
 		}
@@ -841,9 +844,11 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
 	// clean up agent state
 	k.state.ProxyPid = -1
 	k.state.URL = ""
-	if err := sandbox.store.Store(store.Agent, k.state); err != nil {
+	if !sandbox.supportNewStore() {
-		// ignore error
+		if err := sandbox.store.Store(store.Agent, k.state); err != nil {
-		k.Logger().WithError(err).WithField("sandbox", sandbox.id).Error("failed to clean up agent state")
+			// ignore error
+			k.Logger().WithError(err).WithField("sandbox", sandbox.id).Error("failed to clean up agent state")
+		}
 	}
 
 	return nil
@@ -2074,3 +2079,15 @@ func (k *kataAgent) cleanup(s *Sandbox) {
 		k.Logger().WithError(err).Errorf("failed to cleanup vm share path %s", path)
 	}
 }
+
+func (k *kataAgent) save() persistapi.AgentState {
+	return persistapi.AgentState{
+		ProxyPid: k.state.ProxyPid,
+		URL:      k.state.URL,
+	}
+}
+
+func (k *kataAgent) load(s persistapi.AgentState) {
+	k.state.ProxyPid = s.ProxyPid
+	k.state.URL = s.URL
+}

virtcontainers/macvtap_endpoint.go

@@ -8,6 +8,8 @@ package virtcontainers
 import (
 	"fmt"
 	"os"
+
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
 // MacvtapEndpoint represents a macvtap endpoint
@@ -102,3 +104,20 @@ func (endpoint *MacvtapEndpoint) SetPciAddr(pciAddr string) {
 func (endpoint *MacvtapEndpoint) NetworkPair() *NetworkInterfacePair {
 	return nil
 }
+
+func (endpoint *MacvtapEndpoint) save() persistapi.NetworkEndpoint {
+	return persistapi.NetworkEndpoint{
+		Type: string(endpoint.Type()),
+
+		Macvtap: &persistapi.MacvtapEndpoint{
+			PCIAddr: endpoint.PCIAddr,
+		},
+	}
+}
+func (endpoint *MacvtapEndpoint) load(s persistapi.NetworkEndpoint) {
+	endpoint.EndpointType = MacvtapEndpointType
+
+	if s.Macvtap != nil {
+		endpoint.PCIAddr = s.Macvtap.PCIAddr
+	}
+}

virtcontainers/mock_hypervisor.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"os"
 
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 )
@@ -114,3 +115,9 @@ func (m *mockHypervisor) fromGrpc(ctx context.Context, hypervisorConfig *Hypervi
 func (m *mockHypervisor) toGrpc() ([]byte, error) {
 	return nil, errors.New("firecracker is not supported by VM cache")
 }
+
+func (m *mockHypervisor) save() (s persistapi.HypervisorState) {
+	return
+}
+
+func (m *mockHypervisor) load(s persistapi.HypervisorState) {}

virtcontainers/noop_agent.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/kata-containers/agent/protocols/grpc"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -233,3 +234,11 @@ func (n *noopAgent) markDead() {
 
 func (n *noopAgent) cleanup(s *Sandbox) {
 }
+
+// save is the Noop agent state saver. It does nothing.
+func (n *noopAgent) save() (s persistapi.AgentState) {
+	return
+}
+
+// load is the Noop agent state loader. It does nothing.
+func (n *noopAgent) load(s persistapi.AgentState) {}

virtcontainers/persist.go

@@ -57,7 +57,9 @@ func (s *Sandbox) dumpState(ss *persistapi.SandboxState, cs map[string]persistap
 	}
 }
 
-func (s *Sandbox) dumpHypervisor(ss *persistapi.SandboxState, cs map[string]persistapi.ContainerState) {
+func (s *Sandbox) dumpHypervisor(ss *persistapi.SandboxState) {
+	ss.HypervisorState = s.hypervisor.save()
+	// BlockIndex will be moved from sandbox state to hypervisor state later
 	ss.HypervisorState.BlockIndex = s.state.BlockIndex
 }
 
@@ -152,6 +154,23 @@ func (s *Sandbox) dumpMounts(cs map[string]persistapi.ContainerState) {
 	}
 }
 
+func (s *Sandbox) dumpAgent(ss *persistapi.SandboxState) {
+	if s.agent != nil {
+		ss.AgentState = s.agent.save()
+	}
+}
+
+func (s *Sandbox) dumpNetwork(ss *persistapi.SandboxState) {
+	ss.Network = persistapi.NetworkInfo{
+		NetNsPath:    s.networkNS.NetNsPath,
+		NetmonPID:    s.networkNS.NetmonPID,
+		NetNsCreated: s.networkNS.NetNsCreated,
+	}
+	for _, e := range s.networkNS.Endpoints {
+		ss.Network.Endpoints = append(ss.Network.Endpoints, e.save())
+	}
+}
+
 func (s *Sandbox) Save() error {
 	var (
 		ss = persistapi.SandboxState{}
@@ -160,10 +179,12 @@ func (s *Sandbox) Save() error {
 
 	s.dumpVersion(&ss)
 	s.dumpState(&ss, cs)
-	s.dumpHypervisor(&ss, cs)
+	s.dumpHypervisor(&ss)
 	s.dumpDevices(&ss, cs)
 	s.dumpProcess(cs)
 	s.dumpMounts(cs)
+	s.dumpAgent(&ss)
+	s.dumpNetwork(&ss)
 
 	if err := s.newStore.ToDisk(ss, cs); err != nil {
 		return err
@@ -190,6 +211,16 @@ func (c *Container) loadContState(cs persistapi.ContainerState) {
 	}
 }
 
+func (s *Sandbox) loadHypervisor(hs persistapi.HypervisorState) {
+	s.hypervisor.load(hs)
+}
+
+func (s *Sandbox) loadAgent(as persistapi.AgentState) {
+	if s.agent != nil {
+		s.agent.load(as)
+	}
+}
+
 func (s *Sandbox) loadDevices(devStates []persistapi.DeviceState) {
 	s.devManager.LoadDevices(devStates)
 }
@@ -229,6 +260,39 @@ func (c *Container) loadContProcess(cs persistapi.ContainerState) {
 	}
 }
 
+func (s *Sandbox) loadNetwork(netInfo persistapi.NetworkInfo) {
+	s.networkNS = NetworkNamespace{
+		NetNsPath:    netInfo.NetNsPath,
+		NetmonPID:    netInfo.NetmonPID,
+		NetNsCreated: netInfo.NetNsCreated,
+	}
+
+	for _, e := range netInfo.Endpoints {
+		var ep Endpoint
+		switch EndpointType(e.Type) {
+		case PhysicalEndpointType:
+			ep = &PhysicalEndpoint{}
+		case VethEndpointType:
+			ep = &VethEndpoint{}
+		case VhostUserEndpointType:
+			ep = &VhostUserEndpoint{}
+		case BridgedMacvlanEndpointType:
+			ep = &BridgedMacvlanEndpoint{}
+		case MacvtapEndpointType:
+			ep = &MacvtapEndpoint{}
+		case TapEndpointType:
+			ep = &TapEndpoint{}
+		case IPVlanEndpointType:
+			ep = &IPVlanEndpoint{}
+		default:
+			s.Logger().WithField("endpoint-type", e.Type).Error("unknown endpoint type")
+			continue
+		}
+		ep.load(e)
+		s.networkNS.Endpoints = append(s.networkNS.Endpoints, ep)
+	}
+}
+
 // Restore will restore sandbox data from persist file on disk
 func (s *Sandbox) Restore() error {
 	ss, _, err := s.newStore.FromDisk(s.id)
@@ -237,7 +301,10 @@ func (s *Sandbox) Restore() error {
 	}
 
 	s.loadState(ss)
+	s.loadHypervisor(ss.HypervisorState)
 	s.loadDevices(ss.Devices)
+	s.loadAgent(ss.AgentState)
+	s.loadNetwork(ss.Network)
 	return nil
 }
 

virtcontainers/persist/api/hypervisor.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2019 Huawei Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package persistapi
+
+// Bridge is a bridge where devices can be hot plugged
+type Bridge struct {
+	// DeviceAddr contains information about devices plugged and its address in the bridge
+	DeviceAddr map[uint32]string
+
+	// Type is the type of the bridge (pci, pcie, etc)
+	Type string
+
+	//ID is used to identify the bridge in the hypervisor
+	ID string
+
+	// Addr is the PCI/e slot of the bridge
+	Addr int
+}
+
+// CPUDevice represents a CPU device which was hot-added in a running VM
+type CPUDevice struct {
+	// ID is used to identify this CPU in the hypervisor options.
+	ID string
+}
+
+type HypervisorState struct {
+	Pid int
+	// Type of hypervisor, E.g. qemu/firecracker/acrn.
+	Type       string
+	BlockIndex int
+	UUID       string
+
+	// Belows are qemu specific
+	// Refs: virtcontainers/qemu.go:QemuState
+	Bridges []Bridge
+	// HotpluggedCPUs is the list of CPUs that were hot-added
+	HotpluggedVCPUs      []CPUDevice
+	HotpluggedMemory     int
+	HotplugVFIOOnRootBus bool
+}

virtcontainers/persist/api/network.go

@@ -6,27 +6,86 @@
 
 package persistapi
 
+import (
+	"github.com/vishvananda/netlink"
+)
+
 // ============= sandbox level resources =============
 
+type NetworkInterface struct {
+	Name     string
+	HardAddr string
+	Addrs    []netlink.Addr
+}
+
+// TapInterface defines a tap interface
+type TapInterface struct {
+	ID       string
+	Name     string
+	TAPIface NetworkInterface
+	// remove VMFds and VhostFds
+}
+
+// NetworkInterfacePair defines a pair between VM and virtual network interfaces.
+type NetworkInterfacePair struct {
+	TapInterface
+	VirtIface            NetworkInterface
+	NetInterworkingModel int
+}
+
+type PhysicalEndpoint struct {
+	BDF            string
+	Driver         string
+	VendorDeviceID string
+}
+
+type MacvtapEndpoint struct {
+	// This is for showing information.
+	// Remove this field won't impact anything.
+	PCIAddr string
+}
+
+type TapEndpoint struct {
+	TapInterface TapInterface
+}
+
+type BridgedMacvlanEndpoint struct {
+	NetPair NetworkInterfacePair
+}
+
+type VethEndpoint struct {
+	NetPair NetworkInterfacePair
+}
+
+type IPVlanEndpoint struct {
+	NetPair NetworkInterfacePair
+}
+
+type VhostUserEndpoint struct {
+	// This is for showing information.
+	// Remove these fields won't impact anything.
+	IfaceName string
+	PCIAddr   string
+}
+
 // NetworkEndpoint contains network interface information
 type NetworkEndpoint struct {
 	Type string
 
-	// ID used to pass the netdev option to qemu
+	// One and only one of these below are not nil according to Type.
-	ID string
+	Physical       *PhysicalEndpoint       `json:",omitempty"`
-
+	Veth           *VethEndpoint           `json:",omitempty"`
-	// Name of the interface
+	VhostUser      *VhostUserEndpoint      `json:",omitempty"`
-	Name string
+	BridgedMacvlan *BridgedMacvlanEndpoint `json:",omitempty"`
-
+	Macvtap        *MacvtapEndpoint        `json:",omitempty"`
-	// Index of interface
+	Tap            *TapEndpoint            `json:",omitempty"`
-	Index int
+	IPVlan         *IPVlanEndpoint         `json:",omitempty"`
 }
 
 // NetworkInfo contains network information of sandbox
 type NetworkInfo struct {
-	NetNsPath         string
+	NetNsPath    string
-	NetmonPID         int
+	NetmonPID    int
-	NetNsCreated      bool
+	NetNsCreated bool
-	InterworkingModel string
+	Endpoints    []NetworkEndpoint
-	Endpoints         []NetworkEndpoint
 }

virtcontainers/persist/api/sandbox.go

@@ -8,50 +8,12 @@ package persistapi
 
 // ============= sandbox level resources =============
 
-// SetFunc is function hook used for setting sandbox/container state
+// AgentState save agent state data
-// It can be registered to dynamically set state files when dump
+type AgentState struct {
-type SetFunc (func(*SandboxState, map[string]ContainerState) error)
-
-// Bridge is a bridge where devices can be hot plugged
-type Bridge struct {
-	// DeviceAddr contains information about devices plugged and its address in the bridge
-	DeviceAddr map[uint32]string
-
-	// Type is the type of the bridge (pci, pcie, etc)
-	Type string
-
-	//ID is used to identify the bridge in the hypervisor
-	ID string
-
-	// Addr is the PCI/e slot of the bridge
-	Addr int
-}
-
-// CPUDevice represents a CPU device which was hot-added in a running VM
-type CPUDevice struct {
-	// ID is used to identify this CPU in the hypervisor options.
-	ID string
-}
-
-// HypervisorState saves state of hypervisor
-// Refs: virtcontainers/qemu.go:QemuState
-type HypervisorState struct {
-	Pid     int
-	Bridges []Bridge
-	// HotpluggedCPUs is the list of CPUs that were hot-added
-	HotpluggedVCPUs      []CPUDevice
-	HotpluggedMemory     int
-	UUID                 string
-	HotplugVFIOOnRootBus bool
-	BlockIndex           int
-}
-
-// ProxyState save proxy state data
-type ProxyState struct {
 	// Pid of proxy process
-	Pid int
+	ProxyPid int
 
-	// URL to connect to proxy
+	// URL to connect to agent
 	URL string
 }
 
@@ -84,8 +46,8 @@ type SandboxState struct {
 	// HypervisorState saves hypervisor specific data
 	HypervisorState HypervisorState
 
-	// ProxyState saves state data of proxy process
+	// AgentState saves state data of agent
-	ProxyState ProxyState
+	AgentState AgentState
 
 	// Network saves network configuration of sandbox
 	Network NetworkInfo

virtcontainers/persist_test.go

@@ -83,7 +83,7 @@ func TestSandboxRestore(t *testing.T) {
 	assert.NoError(err)
 	assert.NotNil(sandbox.newStore)
 
-	// if we don't call ToDisk, we can get nothing from disk
+	// if we don't call Save(), we can get nothing from disk
 	err = sandbox.Restore()
 	assert.NotNil(t, err)
 	assert.True(os.IsNotExist(err))

virtcontainers/physical_endpoint.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/device/drivers"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/safchain/ethtool"
 )
 
@@ -200,3 +201,25 @@ func bindNICToVFIO(endpoint *PhysicalEndpoint) error {
 func bindNICToHost(endpoint *PhysicalEndpoint) error {
 	return drivers.BindDevicetoHost(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID)
 }
+
+func (endpoint *PhysicalEndpoint) save() persistapi.NetworkEndpoint {
+	return persistapi.NetworkEndpoint{
+		Type: string(endpoint.Type()),
+
+		Physical: &persistapi.PhysicalEndpoint{
+			BDF:            endpoint.BDF,
+			Driver:         endpoint.Driver,
+			VendorDeviceID: endpoint.VendorDeviceID,
+		},
+	}
+}
+
+func (endpoint *PhysicalEndpoint) load(s persistapi.NetworkEndpoint) {
+	endpoint.EndpointType = PhysicalEndpointType
+
+	if s.Physical != nil {
+		endpoint.BDF = s.Physical.BDF
+		endpoint.Driver = s.Physical.Driver
+		endpoint.VendorDeviceID = s.Physical.VendorDeviceID
+	}
+}

virtcontainers/qemu.go

@@ -29,9 +29,11 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
+
 	"golang.org/x/sys/unix"
 )
 
@@ -252,7 +254,17 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto
 		q.nvdimmCount = 0
 	}
 
-	if err = q.store.Load(store.Hypervisor, &q.state); err != nil {
+	var create bool
+	if q.store != nil { //use old store
+		if err := q.store.Load(store.Hypervisor, &q.state); err != nil {
+			// hypervisor doesn't exist, create new one
+			create = true
+		}
+	} else if q.state.UUID == "" { // new store
+		create = true
+	}
+
+	if create {
 		q.Logger().Debug("Creating bridges")
 		q.state.Bridges = q.arch.bridges(q.config.DefaultBridges)
 
@@ -267,7 +279,7 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto
 			return err
 		}
 
-		if err = q.store.Store(store.Hypervisor, q.state); err != nil {
+		if err = q.storeState(); err != nil {
 			return err
 		}
 	}
@@ -1222,7 +1234,7 @@ func (q *qemu) hotplugAddDevice(devInfo interface{}, devType deviceType) (interf
 		return data, err
 	}
 
-	return data, q.store.Store(store.Hypervisor, q.state)
+	return data, q.storeState()
 }
 
 func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) {
@@ -1234,7 +1246,7 @@ func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (int
 		return data, err
 	}
 
-	return data, q.store.Store(store.Hypervisor, q.state)
+	return data, q.storeState()
 }
 
 func (q *qemu) hotplugCPUs(vcpus uint32, op operation) (uint32, error) {
@@ -1314,12 +1326,12 @@ func (q *qemu) hotplugAddCPUs(amount uint32) (uint32, error) {
 		hotpluggedVCPUs++
 		if hotpluggedVCPUs == amount {
 			// All vCPUs were hotplugged
-			return amount, q.store.Store(store.Hypervisor, q.state)
+			return amount, q.storeState()
 		}
 	}
 
 	// All vCPUs were NOT hotplugged
-	if err := q.store.Store(store.Hypervisor, q.state); err != nil {
+	if err := q.storeState(); err != nil {
 		q.Logger().Errorf("failed to save hypervisor state after hotplug %d vCPUs: %v", hotpluggedVCPUs, err)
 	}
 
@@ -1339,7 +1351,7 @@ func (q *qemu) hotplugRemoveCPUs(amount uint32) (uint32, error) {
 		// get the last vCPUs and try to remove it
 		cpu := q.state.HotpluggedVCPUs[len(q.state.HotpluggedVCPUs)-1]
 		if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, cpu.ID); err != nil {
-			_ = q.store.Store(store.Hypervisor, q.state)
+			q.storeState()
 			return i, fmt.Errorf("failed to hotunplug CPUs, only %d CPUs were hotunplugged: %v", i, err)
 		}
 
@@ -1347,7 +1359,7 @@ func (q *qemu) hotplugRemoveCPUs(amount uint32) (uint32, error) {
 		q.state.HotpluggedVCPUs = q.state.HotpluggedVCPUs[:len(q.state.HotpluggedVCPUs)-1]
 	}
 
-	return amount, q.store.Store(store.Hypervisor, q.state)
+	return amount, q.storeState()
 }
 
 func (q *qemu) hotplugMemory(memDev *memoryDevice, op operation) (int, error) {
@@ -1453,7 +1465,7 @@ func (q *qemu) hotplugAddMemory(memDev *memoryDevice) (int, error) {
 		}
 	}
 	q.state.HotpluggedMemory += memDev.sizeMB
-	return memDev.sizeMB, q.store.Store(store.Hypervisor, q.state)
+	return memDev.sizeMB, q.storeState()
 }
 
 func (q *qemu) pauseSandbox() error {
@@ -1907,3 +1919,57 @@ func (q *qemu) toGrpc() ([]byte, error) {
 
 	return json.Marshal(&qp)
 }
+
+func (q *qemu) storeState() error {
+	if q.store != nil {
+		if err := q.store.Store(store.Hypervisor, q.state); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (q *qemu) save() (s persistapi.HypervisorState) {
+	s.Pid = q.pid()
+	s.Type = string(QemuHypervisor)
+	s.UUID = q.state.UUID
+	s.HotpluggedMemory = q.state.HotpluggedMemory
+	s.HotplugVFIOOnRootBus = q.state.HotplugVFIOOnRootBus
+
+	for _, bridge := range q.state.Bridges {
+		s.Bridges = append(s.Bridges, persistapi.Bridge{
+			DeviceAddr: bridge.Address,
+			Type:       string(bridge.Type),
+			ID:         bridge.ID,
+			Addr:       bridge.Addr,
+		})
+	}
+
+	for _, cpu := range q.state.HotpluggedVCPUs {
+		s.HotpluggedVCPUs = append(s.HotpluggedVCPUs, persistapi.CPUDevice{
+			ID: cpu.ID,
+		})
+	}
+	return
+}
+
+func (q *qemu) load(s persistapi.HypervisorState) {
+	q.state.UUID = s.UUID
+	q.state.HotpluggedMemory = s.HotpluggedMemory
+	q.state.HotplugVFIOOnRootBus = s.HotplugVFIOOnRootBus
+
+	for _, bridge := range s.Bridges {
+		q.state.Bridges = append(q.state.Bridges, types.PCIBridge{
+			Address: bridge.DeviceAddr,
+			Type:    types.PCIType(bridge.Type),
+			ID:      bridge.ID,
+			Addr:    bridge.Addr,
+		})
+	}
+
+	for _, cpu := range s.HotpluggedVCPUs {
+		q.state.HotpluggedVCPUs = append(q.state.HotpluggedVCPUs, CPUDevice{
+			ID: cpu.ID,
+		})
+	}
+}

virtcontainers/sandbox.go

@@ -467,20 +467,18 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
 		s.Logger().WithField("features", s.config.Experimental).Infof("Enable experimental features")
 	}
 
-	// Fetch sandbox network to be able to access it from the sandbox structure.
-	var networkNS NetworkNamespace
-	if err := s.store.Load(store.Network, &networkNS); err == nil {
-		s.networkNS = networkNS
-	}
-
 	if s.supportNewStore() {
-		s.devManager = deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, nil)
+		// Restored successfully from newstore before.
-
+		if s.state.State != "" {
-		if err := s.Restore(); err == nil && s.state.State != "" {
 			return s, nil
 		}
-
 	} else {
+		// Fetch sandbox network to be able to access it from the sandbox structure.
+		var networkNS NetworkNamespace
+		if err := s.store.Load(store.Network, &networkNS); err == nil {
+			s.networkNS = networkNS
+		}
+
 		devices, err := s.store.LoadDevices()
 		if err != nil {
 			s.Logger().WithError(err).WithField("sandboxid", s.id).Warning("load sandbox devices failed")
@@ -565,17 +563,24 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 		if err != nil {
 			s.Logger().WithError(err).WithField("sandboxid", s.id).Error("Create new sandbox failed")
 			globalSandboxList.removeSandbox(s.id)
-		}
-	}()
-
-	defer func() {
-		if err != nil {
 			s.store.Delete()
 		}
 	}()
 
-	if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.store); err != nil {
+	if s.supportNewStore() {
-		return nil, err
+		s.devManager = deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, nil)
+
+		// Ignore the error. Restore can fail for a new sandbox
+		s.Restore()
+
+		// new store doesn't require hypervisor to be stored immediately
+		if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, nil); err != nil {
+			return nil, err
+		}
+	} else {
+		if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.store); err != nil {
+			return nil, err
+		}
 	}
 
 	agentConfig, err := newAgentConfig(sandboxConfig.AgentType, sandboxConfig.AgentConfig)
@@ -836,7 +841,10 @@ func (s *Sandbox) createNetwork() error {
 	}
 
 	// Store the network
-	return s.store.Store(store.Network, s.networkNS)
+	if !s.supportNewStore() {
+		return s.store.Store(store.Network, s.networkNS)
+	}
+	return nil
 }
 
 func (s *Sandbox) postCreatedNetwork() error {
@@ -909,8 +917,14 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, erro
 
 	// Update the sandbox storage
 	s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint)
-	if err := s.store.Store(store.Network, s.networkNS); err != nil {
+	if s.supportNewStore() {
-		return nil, err
+		if err := s.Save(); err != nil {
+			return nil, err
+		}
+	} else {
+		if err := s.store.Store(store.Network, s.networkNS); err != nil {
+			return nil, err
+		}
 	}
 
 	// Add network for vm
@@ -927,9 +941,17 @@ func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, e
 				return inf, err
 			}
 			s.networkNS.Endpoints = append(s.networkNS.Endpoints[:i], s.networkNS.Endpoints[i+1:]...)
-			if err := s.store.Store(store.Network, s.networkNS); err != nil {
+
-				return inf, err
+			if s.supportNewStore() {
+				if err := s.Save(); err != nil {
+					return inf, err
+				}
+			} else {
+				if err := s.store.Store(store.Network, s.networkNS); err != nil {
+					return inf, err
+				}
 			}
+
 			break
 		}
 	}
@@ -1001,8 +1023,11 @@ func (s *Sandbox) startVM() (err error) {
 				return err
 			}
 		}
-		if err := s.store.Store(store.Network, s.networkNS); err != nil {
+
-			return err
+		if !s.supportNewStore() {
+			if err := s.store.Store(store.Network, s.networkNS); err != nil {
+				return err
+			}
 		}
 	}
 

virtcontainers/tap_endpoint.go

@@ -11,6 +11,7 @@ import (
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/vishvananda/netlink"
 
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
 )
 
@@ -187,3 +188,22 @@ func unTapNetwork(name string) error {
 	}
 	return nil
 }
+
+func (endpoint *TapEndpoint) save() persistapi.NetworkEndpoint {
+	tapif := saveTapIf(&endpoint.TapInterface)
+
+	return persistapi.NetworkEndpoint{
+		Type: string(endpoint.Type()),
+		Tap: &persistapi.TapEndpoint{
+			TapInterface: *tapif,
+		},
+	}
+}
+func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) {
+	endpoint.EndpointType = TapEndpointType
+
+	if s.Tap != nil {
+		tapif := loadTapIf(&s.Tap.TapInterface)
+		endpoint.TapInterface = *tapif
+	}
+}

virtcontainers/veth_endpoint.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 
 	"github.com/containernetworking/plugins/pkg/ns"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
 // VethEndpoint gathers a network pair and its properties.
@@ -141,3 +142,23 @@ func (endpoint *VethEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPa
 	}
 	return nil
 }
+
+func (endpoint *VethEndpoint) save() persistapi.NetworkEndpoint {
+	netpair := saveNetIfPair(&endpoint.NetPair)
+
+	return persistapi.NetworkEndpoint{
+		Type: string(endpoint.Type()),
+		Veth: &persistapi.VethEndpoint{
+			NetPair: *netpair,
+		},
+	}
+}
+
+func (endpoint *VethEndpoint) load(s persistapi.NetworkEndpoint) {
+	endpoint.EndpointType = VethEndpointType
+
+	if s.Veth != nil {
+		netpair := loadNetIfPair(&s.Veth.NetPair)
+		endpoint.NetPair = *netpair
+	}
+}

virtcontainers/vhostuser_endpoint.go

@@ -11,6 +11,7 @@ import (
 	"os"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -149,3 +150,22 @@ func vhostUserSocketPath(info interface{}) (string, error) {
 	}
 
 }
+
+func (endpoint *VhostUserEndpoint) save() persistapi.NetworkEndpoint {
+	return persistapi.NetworkEndpoint{
+		Type: string(endpoint.Type()),
+		VhostUser: &persistapi.VhostUserEndpoint{
+			IfaceName: endpoint.IfaceName,
+			PCIAddr:   endpoint.PCIAddr,
+		},
+	}
+}
+
+func (endpoint *VhostUserEndpoint) load(s persistapi.NetworkEndpoint) {
+	endpoint.EndpointType = VhostUserEndpointType
+
+	if s.VhostUser != nil {
+		endpoint.IfaceName = s.VhostUser.IfaceName
+		endpoint.PCIAddr = s.VhostUser.PCIAddr
+	}
+}