compatibility: keep oldstore for compatibility

Keep old store restore functions for keeping backward compatibility, if
old store files are found from disk, restore them with old store first.

Signed-off-by: Wei Zhang <weizhang555.zw@gmail.com>

virtcontainers/container.go

@@ -29,6 +29,7 @@ import (
 	"github.com/kata-containers/runtime/pkg/rootless"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/device/manager"
+	"github.com/kata-containers/runtime/virtcontainers/store"
 )
 
 // https://github.com/torvalds/linux/blob/master/include/uapi/linux/major.h
@@ -335,6 +336,8 @@ type Container struct {
 	systemMountsInfo SystemMountsInfo
 
 	ctx context.Context
+
+	store *store.VCStore
 }
 
 // ID returns the container identifier string.
@@ -426,9 +429,17 @@ func (c *Container) setContainerState(state types.StateString) error {
 	// update in-memory state
 	c.state.State = state
 
-	// flush data to storage
-	if err := c.sandbox.Save(); err != nil {
-		return err
+	if useOldStore(c.sandbox.ctx) {
+		// experimental runtime use "persist.json" which doesn't need "state.json" anymore
+		// update on-disk state
+		if err := c.store.Store(store.State, c.state); err != nil {
+			return err
+		}
+	} else {
+		// flush data to storage
+		if err := c.sandbox.Save(); err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -678,31 +689,76 @@ func newContainer(sandbox *Sandbox, contConfig *ContainerConfig) (*Container, er
 		ctx:           sandbox.ctx,
 	}
 
-	// experimental runtime use "persist.json" instead of legacy "state.json" as storage
-	err := c.Restore()
-	if err == nil {
-		//container restored
-		return c, nil
-	}
+	if useOldStore(sandbox.ctx) {
+		ctrStore, err := store.NewVCContainerStore(sandbox.ctx, c.sandboxID, c.id)
+		if err != nil {
+			return nil, err
+		}
+		c.store = ctrStore
+		state, err := c.store.LoadContainerState()
+		if err == nil {
+			c.state = state
+		}
 
-	// Unexpected error
-	if !os.IsNotExist(err) && err != errContainerPersistNotExist {
-		return nil, err
+		var process Process
+		if err := c.store.Load(store.Process, &process); err == nil {
+			c.process = process
+		}
+	} else {
+		// experimental runtime use "persist.json" instead of legacy "state.json" as storage
+		err := c.Restore()
+		if err == nil {
+			//container restored
+			return c, nil
+		}
+
+		// Unexpected error
+		if !os.IsNotExist(err) && err != errContainerPersistNotExist {
+			return nil, err
+		}
 	}
 
 	// Go to next step for first created container
-	if err = c.createMounts(); err != nil {
+	if err := c.createMounts(); err != nil {
 		return nil, err
 	}
 
-	if err = c.createDevices(contConfig); err != nil {
+	if err := c.createDevices(contConfig); err != nil {
 		return nil, err
 	}
 
 	return c, nil
 }
 
+func (c *Container) loadMounts() ([]Mount, error) {
+	var mounts []Mount
+	if err := c.store.Load(store.Mounts, &mounts); err != nil {
+		return []Mount{}, err
+	}
+
+	return mounts, nil
+}
+
+func (c *Container) loadDevices() ([]ContainerDevice, error) {
+	var devices []ContainerDevice
+
+	if err := c.store.Load(store.DeviceIDs, &devices); err != nil {
+		return []ContainerDevice{}, err
+	}
+
+	return devices, nil
+}
+
 func (c *Container) createMounts() error {
+	if useOldStore(c.sandbox.ctx) {
+		mounts, err := c.loadMounts()
+		if err == nil {
+			// restore mounts from disk
+			c.mounts = mounts
+			return nil
+		}
+	}
+
 	// Create block devices for newly created container
 	if err := c.createBlockDevices(); err != nil {
 		return err
@@ -712,6 +768,18 @@ func (c *Container) createMounts() error {
 }
 
 func (c *Container) createDevices(contConfig *ContainerConfig) error {
+	// If sandbox supports "newstore", only newly created container can reach this function,
+	// so we don't call restore when `supportNewStore` is true
+	if useOldStore(c.sandbox.ctx) {
+		// Devices will be found in storage after create stage has completed.
+		// We load devices from storage at all other stages.
+		storedDevices, err := c.loadDevices()
+		if err == nil {
+			c.devices = storedDevices
+			return nil
+		}
+	}
+
 	// If devices were not found in storage, create Device implementations
 	// from the configuration. This should happen at create.
 	var storedDevices []ContainerDevice

virtcontainers/kata_agent.go

@@ -31,6 +31,7 @@ import (
 	ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
 	vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
+	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	opentracing "github.com/opentracing/opentracing-go"
@@ -317,6 +318,12 @@ func (k *kataAgent) init(ctx context.Context, sandbox *Sandbox, config interface
 
 	k.proxyBuiltIn = isProxyBuiltIn(sandbox.config.ProxyType)
 
+	// Fetch agent runtime info.
+	if useOldStore(sandbox.ctx) {
+		if err := sandbox.store.Load(store.Agent, &k.state); err != nil {
+			k.Logger().Debug("Could not retrieve anything from storage")
+		}
+	}
 	return disableVMShutdown, nil
 }
 

virtcontainers/persist.go

@@ -6,12 +6,14 @@
 package virtcontainers
 
 import (
+	"context"
 	"errors"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	exp "github.com/kata-containers/runtime/virtcontainers/experimental"
 	"github.com/kata-containers/runtime/virtcontainers/persist"
 	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 	"github.com/mitchellh/mapstructure"
 )
@@ -559,3 +561,25 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
 	}
 	return sconfig, nil
 }
+
+var oldstoreKey = struct{}{}
+
+func loadSandboxConfigFromOldStore(ctx context.Context, sid string) (*SandboxConfig, context.Context, error) {
+	var config SandboxConfig
+	// We're bootstrapping
+	vcStore, err := store.NewVCSandboxStore(ctx, sid)
+	if err != nil {
+		return nil, ctx, err
+	}
+
+	if err := vcStore.Load(store.Configuration, &config); err != nil {
+		return nil, ctx, err
+	}
+
+	return &config, context.WithValue(ctx, oldstoreKey, true), nil
+}
+
+func useOldStore(ctx context.Context) bool {
+	v := ctx.Value(oldstoreKey)
+	return v != nil
+}

virtcontainers/sandbox.go

@@ -37,6 +37,7 @@ import (
 	"github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/compatoci"
 	vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
+	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
@@ -177,7 +178,9 @@ type Sandbox struct {
 	factory    Factory
 	hypervisor hypervisor
 	agent      agent
-	newStore   persistapi.PersistDriver
+	store      *store.VCStore
+	// store is used to replace VCStore step by step
+	newStore persistapi.PersistDriver
 
 	network Network
 	monitor *monitor
@@ -544,16 +547,47 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 		}
 	}()
 
-	s.devManager = deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, nil)
+	if useOldStore(ctx) {
+		vcStore, err := store.NewVCSandboxStore(ctx, s.id)
+		if err != nil {
+			return nil, err
+		}
 
-	// Ignore the error. Restore can fail for a new sandbox
-	if err := s.Restore(); err != nil {
-		s.Logger().WithError(err).Debug("restore sandbox failed")
-	}
+		s.store = vcStore
 
-	// new store doesn't require hypervisor to be stored immediately
-	if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.stateful); err != nil {
-		return nil, err
+		// Fetch sandbox network to be able to access it from the sandbox structure.
+		var networkNS NetworkNamespace
+		if err = s.store.Load(store.Network, &networkNS); err == nil {
+			s.networkNS = networkNS
+		}
+
+		devices, err := s.store.LoadDevices()
+		if err != nil {
+			s.Logger().WithError(err).WithField("sandboxid", s.id).Warning("load sandbox devices failed")
+		}
+		s.devManager = deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, devices)
+
+		// Load sandbox state. The hypervisor.createSandbox call, may need to access statei.
+		state, err := s.store.LoadState()
+		if err == nil {
+			s.state = state
+		}
+
+		if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.stateful); err != nil {
+			return nil, err
+		}
+	} else {
+		s.devManager = deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, nil)
+
+		// Ignore the error. Restore can fail for a new sandbox
+		if err := s.Restore(); err != nil {
+			s.Logger().WithError(err).Debug("restore sandbox failed")
+		}
+
+		// new store doesn't require hypervisor to be stored immediately
+		if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.stateful); err != nil {
+			return nil, err
+		}
 	}
 
 	agentConfig, err := newAgentConfig(sandboxConfig.AgentType, sandboxConfig.AgentConfig)
@@ -612,12 +646,22 @@ func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err
 
 	var config SandboxConfig
 
-	c, err := loadSandboxConfig(sandboxID)
+	// Try to load sandbox config from old store at first.
+	c, ctx, err := loadSandboxConfigFromOldStore(ctx, sandboxID)
 	if err != nil {
-		return nil, err
+		virtLog.Warningf("failed to get sandbox config from old store: %v", err)
+		// If we failed to load sandbox config from old store, try again with new store.
+		c, err = loadSandboxConfig(sandboxID)
+		if err != nil {
+			virtLog.Warningf("failed to get sandbox config from new store: %v", err)
+			return nil, err
+		}
 	}
 	config = *c
 
+	if useOldStore(ctx) {
+		virtLog.Infof("Warning: old store has been deprecated.")
+	}
 	// fetchSandbox is not suppose to create new sandbox VM.
 	sandbox, err = createSandbox(ctx, config, nil)
 	if err != nil {
@@ -1478,6 +1522,10 @@ func (s *Sandbox) setSandboxState(state types.StateString) error {
 
 	// update in-memory state
 	s.state.State = state
+
+	if useOldStore(s.ctx) {
+		return s.store.Store(store.State, s.state)
+	}
 	return nil
 }
 

virtcontainers/virtcontainers_test.go

@@ -16,6 +16,7 @@ import (
 	"testing"
 
 	"github.com/kata-containers/runtime/virtcontainers/persist/fs"
+	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 	"github.com/sirupsen/logrus"
 )
@@ -60,10 +61,14 @@ func cleanUp() {
 	os.RemoveAll(testDir)
 	os.MkdirAll(testDir, DirMode)
 
+	store.DeleteAll()
+	store.VCStorePrefix = ""
+
 	setup()
 }
 
 func setup() {
+	store.VCStorePrefix = testDir
 	os.Mkdir(filepath.Join(testDir, testBundle), DirMode)
 
 	for _, filename := range []string{testQemuKernelPath, testQemuInitrdPath, testQemuImagePath, testQemuPath} {