Merge pull request #2179 from WeiZhang555/persist-storage

Support "Configuration" in "newstore" feature

cli/main.go

@@ -21,6 +21,7 @@ import (
 	"github.com/kata-containers/runtime/pkg/rootless"
 	"github.com/kata-containers/runtime/pkg/signals"
 	vc "github.com/kata-containers/runtime/virtcontainers"
+	exp "github.com/kata-containers/runtime/virtcontainers/experimental"
 	vf "github.com/kata-containers/runtime/virtcontainers/factory"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -346,6 +347,11 @@ func beforeSubcommands(c *cli.Context) error {
 		"arguments": `"` + args + `"`,
 	}
 
+	err = addExpFeatures(c, runtimeConfig)
+	if err != nil {
+		return err
+	}
+
 	kataLog.WithFields(fields).Info()
 
 	// make the data accessible to the sub-commands.
@@ -401,6 +407,24 @@ func setupTracing(context *cli.Context, rootSpanName string) error {
 	return nil
 }
 
+// add supported experimental features in context
+func addExpFeatures(clictx *cli.Context, runtimeConfig oci.RuntimeConfig) error {
+	ctx, err := cliContextToContext(clictx)
+	if err != nil {
+		return err
+	}
+
+	var exps []string
+	for _, e := range runtimeConfig.Experimental {
+		exps = append(exps, e.Name)
+	}
+
+	ctx = exp.ContextWithExp(ctx, exps)
+	// Add tracer to metadata and update the context
+	clictx.App.Metadata["context"] = ctx
+	return nil
+}
+
 func afterSubcommands(c *cli.Context) error {
 	ctx, err := cliContextToContext(c)
 	if err != nil {

virtcontainers/api.go

@@ -13,6 +13,7 @@ import (
 
 	deviceApi "github.com/kata-containers/runtime/virtcontainers/device/api"
 	deviceConfig "github.com/kata-containers/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/runtime/virtcontainers/persist/fs"
 	vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
@@ -307,7 +308,14 @@ func ListSandbox(ctx context.Context) ([]SandboxStatus, error) {
 	span, ctx := trace(ctx, "ListSandbox")
 	defer span.Finish()
 
-	dir, err := os.Open(store.ConfigStoragePath())
+	var sbsdir string
+	if supportNewStore(ctx) {
+		sbsdir = fs.RunStoragePath()
+	} else {
+		sbsdir = store.RunStoragePath()
+	}
+
+	dir, err := os.Open(sbsdir)
 	if err != nil {
 		if os.IsNotExist(err) {
 			// No sandbox directory is not an error

virtcontainers/container.go

@@ -417,6 +417,7 @@ func (c *Container) storeContainer() error {
 		if err := c.sandbox.Save(); err != nil {
 			return err
 		}
+		return nil
 	}
 	return c.store.Store(store.Configuration, *(c.config))
 }

virtcontainers/experimental/experimental.go

@@ -6,6 +6,7 @@
 package experimental
 
 import (
+	"context"
 	"fmt"
 	"regexp"
 )
@@ -22,8 +23,11 @@ type Feature struct {
 	ExpRelease string
 }
 
+type contextKey struct{}
+
 var (
 	supportedFeatures = make(map[string]Feature)
+	expContextKey     = contextKey{}
 )
 
 // Register register a new experimental feature
@@ -61,3 +65,16 @@ func validateFeature(feature Feature) error {
 
 	return nil
 }
+
+func ContextWithExp(ctx context.Context, names []string) context.Context {
+	return context.WithValue(ctx, expContextKey, names)
+}
+
+func ExpFromContext(ctx context.Context) []string {
+	value := ctx.Value(expContextKey)
+	if value == nil {
+		return nil
+	}
+	names := value.([]string)
+	return names
+}

virtcontainers/persist.go

@@ -13,6 +13,7 @@ import (
 	"github.com/kata-containers/runtime/virtcontainers/persist"
 	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/types"
+	"github.com/mitchellh/mapstructure"
 )
 
 var (
@@ -171,6 +172,119 @@ func (s *Sandbox) dumpNetwork(ss *persistapi.SandboxState) {
 	}
 }
 
+func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
+	sconfig := s.config
+	ss.Config = persistapi.SandboxConfig{
+		HypervisorType: string(sconfig.HypervisorType),
+		AgentType:      string(sconfig.AgentType),
+		ProxyType:      string(sconfig.ProxyType),
+		ProxyConfig: persistapi.ProxyConfig{
+			Path:  sconfig.ProxyConfig.Path,
+			Debug: sconfig.ProxyConfig.Debug,
+		},
+		ShimType: string(sconfig.ShimType),
+		NetworkConfig: persistapi.NetworkConfig{
+			NetNSPath:         sconfig.NetworkConfig.NetNSPath,
+			NetNsCreated:      sconfig.NetworkConfig.NetNsCreated,
+			DisableNewNetNs:   sconfig.NetworkConfig.DisableNewNetNs,
+			InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel),
+		},
+
+		ShmSize:             sconfig.ShmSize,
+		SharePidNs:          sconfig.SharePidNs,
+		Stateful:            sconfig.Stateful,
+		SystemdCgroup:       sconfig.SystemdCgroup,
+		SandboxCgroupOnly:   sconfig.SandboxCgroupOnly,
+		DisableGuestSeccomp: sconfig.DisableGuestSeccomp,
+	}
+
+	for _, e := range sconfig.Experimental {
+		ss.Config.Experimental = append(ss.Config.Experimental, e.Name)
+	}
+
+	ss.Config.HypervisorConfig = persistapi.HypervisorConfig{
+		NumVCPUs:                sconfig.HypervisorConfig.NumVCPUs,
+		DefaultMaxVCPUs:         sconfig.HypervisorConfig.DefaultMaxVCPUs,
+		MemorySize:              sconfig.HypervisorConfig.MemorySize,
+		DefaultBridges:          sconfig.HypervisorConfig.DefaultBridges,
+		Msize9p:                 sconfig.HypervisorConfig.Msize9p,
+		MemSlots:                sconfig.HypervisorConfig.MemSlots,
+		MemOffset:               sconfig.HypervisorConfig.MemOffset,
+		VirtioFSCacheSize:       sconfig.HypervisorConfig.VirtioFSCacheSize,
+		KernelPath:              sconfig.HypervisorConfig.KernelPath,
+		ImagePath:               sconfig.HypervisorConfig.ImagePath,
+		InitrdPath:              sconfig.HypervisorConfig.InitrdPath,
+		FirmwarePath:            sconfig.HypervisorConfig.FirmwarePath,
+		MachineAccelerators:     sconfig.HypervisorConfig.MachineAccelerators,
+		HypervisorPath:          sconfig.HypervisorConfig.HypervisorPath,
+		HypervisorCtlPath:       sconfig.HypervisorConfig.HypervisorCtlPath,
+		JailerPath:              sconfig.HypervisorConfig.JailerPath,
+		BlockDeviceDriver:       sconfig.HypervisorConfig.BlockDeviceDriver,
+		HypervisorMachineType:   sconfig.HypervisorConfig.HypervisorMachineType,
+		MemoryPath:              sconfig.HypervisorConfig.MemoryPath,
+		DevicesStatePath:        sconfig.HypervisorConfig.DevicesStatePath,
+		EntropySource:           sconfig.HypervisorConfig.EntropySource,
+		SharedFS:                sconfig.HypervisorConfig.SharedFS,
+		VirtioFSDaemon:          sconfig.HypervisorConfig.VirtioFSDaemon,
+		VirtioFSCache:           sconfig.HypervisorConfig.VirtioFSCache,
+		VirtioFSExtraArgs:       sconfig.HypervisorConfig.VirtioFSExtraArgs[:],
+		BlockDeviceCacheSet:     sconfig.HypervisorConfig.BlockDeviceCacheSet,
+		BlockDeviceCacheDirect:  sconfig.HypervisorConfig.BlockDeviceCacheDirect,
+		BlockDeviceCacheNoflush: sconfig.HypervisorConfig.BlockDeviceCacheNoflush,
+		DisableBlockDeviceUse:   sconfig.HypervisorConfig.DisableBlockDeviceUse,
+		EnableIOThreads:         sconfig.HypervisorConfig.EnableIOThreads,
+		Debug:                   sconfig.HypervisorConfig.Debug,
+		MemPrealloc:             sconfig.HypervisorConfig.MemPrealloc,
+		HugePages:               sconfig.HypervisorConfig.HugePages,
+		FileBackedMemRootDir:    sconfig.HypervisorConfig.FileBackedMemRootDir,
+		Realtime:                sconfig.HypervisorConfig.Realtime,
+		Mlock:                   sconfig.HypervisorConfig.Mlock,
+		DisableNestingChecks:    sconfig.HypervisorConfig.DisableNestingChecks,
+		UseVSock:                sconfig.HypervisorConfig.UseVSock,
+		HotplugVFIOOnRootBus:    sconfig.HypervisorConfig.HotplugVFIOOnRootBus,
+		BootToBeTemplate:        sconfig.HypervisorConfig.BootToBeTemplate,
+		BootFromTemplate:        sconfig.HypervisorConfig.BootFromTemplate,
+		DisableVhostNet:         sconfig.HypervisorConfig.DisableVhostNet,
+		GuestHookPath:           sconfig.HypervisorConfig.GuestHookPath,
+		VMid:                    sconfig.HypervisorConfig.VMid,
+	}
+
+	if sconfig.AgentType == "kata" {
+		var sagent KataAgentConfig
+		err := mapstructure.Decode(sconfig.AgentConfig, &sagent)
+		if err != nil {
+			s.Logger().WithError(err).Error("internal error: KataAgentConfig failed to decode")
+		} else {
+			ss.Config.KataAgentConfig = &persistapi.KataAgentConfig{
+				LongLiveConn: sagent.LongLiveConn,
+				UseVSock:     sagent.UseVSock,
+			}
+		}
+	}
+
+	if sconfig.ShimType == "kataShim" {
+		var shim ShimConfig
+		err := mapstructure.Decode(sconfig.ShimConfig, &shim)
+		if err != nil {
+			s.Logger().WithError(err).Error("internal error: ShimConfig failed to decode")
+		} else {
+			ss.Config.KataShimConfig = &persistapi.ShimConfig{
+				Path:  shim.Path,
+				Debug: shim.Debug,
+			}
+		}
+	}
+
+	for _, contConf := range sconfig.Containers {
+		ss.Config.ContainerConfigs = append(ss.Config.ContainerConfigs, persistapi.ContainerConfig{
+			ID:          contConf.ID,
+			Annotations: contConf.Annotations,
+			RootFs:      contConf.RootFs.Target,
+			Resources:   contConf.Resources,
+		})
+	}
+}
+
 func (s *Sandbox) Save() error {
 	var (
 		ss = persistapi.SandboxState{}
@@ -185,6 +299,7 @@ func (s *Sandbox) Save() error {
 	s.dumpMounts(cs)
 	s.dumpAgent(&ss)
 	s.dumpNetwork(&ss)
+	s.dumpConfig(&ss)
 
 	if err := s.newStore.ToDisk(ss, cs); err != nil {
 		return err
@@ -335,3 +450,119 @@ func (s *Sandbox) supportNewStore() bool {
 	}
 	return false
 }
+
+func loadSandboxConfig(id string) (*SandboxConfig, error) {
+	store, err := persist.GetDriver("fs")
+	if err != nil || store == nil {
+		return nil, errors.New("failed to get fs persist driver")
+	}
+
+	ss, _, err := store.FromDisk(id)
+	if err != nil {
+		return nil, err
+	}
+
+	savedConf := ss.Config
+	sconfig := &SandboxConfig{
+		ID:             id,
+		HypervisorType: HypervisorType(savedConf.HypervisorType),
+		AgentType:      AgentType(savedConf.AgentType),
+		ProxyType:      ProxyType(savedConf.ProxyType),
+		ProxyConfig: ProxyConfig{
+			Path:  savedConf.ProxyConfig.Path,
+			Debug: savedConf.ProxyConfig.Debug,
+		},
+		ShimType: ShimType(savedConf.ShimType),
+		NetworkConfig: NetworkConfig{
+			NetNSPath:         savedConf.NetworkConfig.NetNSPath,
+			NetNsCreated:      savedConf.NetworkConfig.NetNsCreated,
+			DisableNewNetNs:   savedConf.NetworkConfig.DisableNewNetNs,
+			InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel),
+		},
+
+		ShmSize:             savedConf.ShmSize,
+		SharePidNs:          savedConf.SharePidNs,
+		Stateful:            savedConf.Stateful,
+		SystemdCgroup:       savedConf.SystemdCgroup,
+		SandboxCgroupOnly:   savedConf.SandboxCgroupOnly,
+		DisableGuestSeccomp: savedConf.DisableGuestSeccomp,
+	}
+
+	for _, name := range savedConf.Experimental {
+		sconfig.Experimental = append(sconfig.Experimental, *exp.Get(name))
+	}
+
+	hconf := savedConf.HypervisorConfig
+	sconfig.HypervisorConfig = HypervisorConfig{
+		NumVCPUs:                hconf.NumVCPUs,
+		DefaultMaxVCPUs:         hconf.DefaultMaxVCPUs,
+		MemorySize:              hconf.MemorySize,
+		DefaultBridges:          hconf.DefaultBridges,
+		Msize9p:                 hconf.Msize9p,
+		MemSlots:                hconf.MemSlots,
+		MemOffset:               hconf.MemOffset,
+		VirtioFSCacheSize:       hconf.VirtioFSCacheSize,
+		KernelPath:              hconf.KernelPath,
+		ImagePath:               hconf.ImagePath,
+		InitrdPath:              hconf.InitrdPath,
+		FirmwarePath:            hconf.FirmwarePath,
+		MachineAccelerators:     hconf.MachineAccelerators,
+		HypervisorPath:          hconf.HypervisorPath,
+		HypervisorCtlPath:       hconf.HypervisorCtlPath,
+		JailerPath:              hconf.JailerPath,
+		BlockDeviceDriver:       hconf.BlockDeviceDriver,
+		HypervisorMachineType:   hconf.HypervisorMachineType,
+		MemoryPath:              hconf.MemoryPath,
+		DevicesStatePath:        hconf.DevicesStatePath,
+		EntropySource:           hconf.EntropySource,
+		SharedFS:                hconf.SharedFS,
+		VirtioFSDaemon:          hconf.VirtioFSDaemon,
+		VirtioFSCache:           hconf.VirtioFSCache,
+		VirtioFSExtraArgs:       hconf.VirtioFSExtraArgs[:],
+		BlockDeviceCacheSet:     hconf.BlockDeviceCacheSet,
+		BlockDeviceCacheDirect:  hconf.BlockDeviceCacheDirect,
+		BlockDeviceCacheNoflush: hconf.BlockDeviceCacheNoflush,
+		DisableBlockDeviceUse:   hconf.DisableBlockDeviceUse,
+		EnableIOThreads:         hconf.EnableIOThreads,
+		Debug:                   hconf.Debug,
+		MemPrealloc:             hconf.MemPrealloc,
+		HugePages:               hconf.HugePages,
+		FileBackedMemRootDir:    hconf.FileBackedMemRootDir,
+		Realtime:                hconf.Realtime,
+		Mlock:                   hconf.Mlock,
+		DisableNestingChecks:    hconf.DisableNestingChecks,
+		UseVSock:                hconf.UseVSock,
+		HotplugVFIOOnRootBus:    hconf.HotplugVFIOOnRootBus,
+		BootToBeTemplate:        hconf.BootToBeTemplate,
+		BootFromTemplate:        hconf.BootFromTemplate,
+		DisableVhostNet:         hconf.DisableVhostNet,
+		GuestHookPath:           hconf.GuestHookPath,
+		VMid:                    hconf.VMid,
+	}
+
+	if savedConf.AgentType == "kata" {
+		sconfig.AgentConfig = KataAgentConfig{
+			LongLiveConn: savedConf.KataAgentConfig.LongLiveConn,
+			UseVSock:     savedConf.KataAgentConfig.UseVSock,
+		}
+	}
+
+	if savedConf.ShimType == "kataShim" {
+		sconfig.ShimConfig = ShimConfig{
+			Path:  savedConf.KataShimConfig.Path,
+			Debug: savedConf.KataShimConfig.Debug,
+		}
+	}
+
+	for _, contConf := range savedConf.ContainerConfigs {
+		sconfig.Containers = append(sconfig.Containers, ContainerConfig{
+			ID:          contConf.ID,
+			Annotations: contConf.Annotations,
+			Resources:   contConf.Resources,
+			RootFs: RootFs{
+				Target: contConf.RootFs,
+			},
+		})
+	}
+	return sconfig, nil
+}

virtcontainers/persist/api/config.go

@@ -6,17 +6,9 @@
 
 package persistapi
 
-// Param is a key/value representation for hypervisor and kernel parameters.
+import (
-type Param struct {
+	specs "github.com/opencontainers/runtime-spec/specs-go"
-	Key   string
+)
-	Value string
-}
-
-// Asset saves hypervisor asset
-type Asset struct {
-	Path   string `json:"path"`
-	Custom bool   `json:"bool"`
-}
 
 // HypervisorConfig saves configurations of sandbox hypervisor
 type HypervisorConfig struct {
@@ -42,11 +34,8 @@ type HypervisorConfig struct {
 	// MemOffset specifies memory space for nvdimm device
 	MemOffset uint32
 
-	// KernelParams are additional guest kernel parameters.
+	// VirtioFSCacheSize is the DAX cache size in MiB
-	KernelParams []Param
+	VirtioFSCacheSize uint32
-
-	// HypervisorParams are additional hypervisor parameters.
-	HypervisorParams []Param
 
 	// KernelPath is the guest kernel host path.
 	KernelPath string
@@ -67,6 +56,12 @@ type HypervisorConfig struct {
 	// HypervisorPath is the hypervisor executable host path.
 	HypervisorPath string
 
+	// HypervisorCtlPath is the hypervisor ctl executable host path.
+	HypervisorCtlPath string
+
+	// JailerPath is the jailer executable host path.
+	JailerPath string
+
 	// BlockDeviceDriver specifies the driver to be used for block device
 	// either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver
 	BlockDeviceDriver string
@@ -87,11 +82,19 @@ type HypervisorConfig struct {
 	// entropy (/dev/random, /dev/urandom or real hardware RNG device)
 	EntropySource string
 
-	// customAssets is a map of assets.
+	// Shared file system type:
-	// Each value in that map takes precedence over the configured assets.
+	//   - virtio-9p (default)
-	// For example, if there is a value for the "kernel" key in this map,
+	//   - virtio-fs
-	// it will be used for the sandbox's kernel path instead of KernelPath.
+	SharedFS string
-	CustomAssets map[string]*Asset
+
+	// VirtioFSDaemon is the virtio-fs vhost-user daemon path
+	VirtioFSDaemon string
+
+	// VirtioFSCache cache mode for fs version cache or "none"
+	VirtioFSCache string
+
+	// VirtioFSExtraArgs passes options to virtiofsd daemon
+	VirtioFSExtraArgs []string
 
 	// BlockDeviceCacheSet specifies cache-related options will be set to block devices or not.
 	BlockDeviceCacheSet bool
@@ -154,6 +157,10 @@ type HypervisorConfig struct {
 
 	// GuestHookPath is the path within the VM that will be used for 'drop-in' hooks
 	GuestHookPath string
+
+	// VMid is the id of the VM that create the hypervisor if the VM is created by the factory.
+	// VMid is "" if the hypervisor is not created by the factory.
+	VMid string
 }
 
 // KataAgentConfig is a structure storing information needed
@@ -186,6 +193,18 @@ type ShimConfig struct {
 
 // NetworkConfig is the network configuration related to a network.
 type NetworkConfig struct {
+	NetNSPath         string
+	NetNsCreated      bool
+	DisableNewNetNs   bool
+	InterworkingModel int
+}
+
+type ContainerConfig struct {
+	ID          string
+	Annotations map[string]string
+	RootFs      string
+	// Resources for recoding update
+	Resources specs.LinuxResources
 }
 
 // SandboxConfig is a sandbox configuration.
@@ -197,15 +216,13 @@ type SandboxConfig struct {
 	// only one agent config can be non-nil according to agent type
 	AgentType       string
 	KataAgentConfig *KataAgentConfig `json:",omitempty"`
-	HyperstartConfig *HyperstartConfig `json:",omitempty"`
 
 	ProxyType   string
 	ProxyConfig ProxyConfig
 
 	ShimType       string
-	KataShimConfig ShimConfig
+	KataShimConfig *ShimConfig
 
-	NetworkModel  string
 	NetworkConfig NetworkConfig
 
 	ShmSize uint64
@@ -220,11 +237,18 @@ type SandboxConfig struct {
 	// SystemdCgroup enables systemd cgroup support
 	SystemdCgroup bool
 
+	// SandboxCgroupOnly enables cgroup only at podlevel in the host
+	SandboxCgroupOnly bool
+
+	DisableGuestSeccomp bool
+
 	// Experimental enables experimental features
-	Experimental bool
+	Experimental []string
 
 	// Information for fields not saved:
 	// * Annotation: this is kind of casual data, we don't need casual data in persist file,
 	// 				if you know this data needs to persist, please gives it
 	//				a specific field
+
+	ContainerConfigs []ContainerConfig
 }

virtcontainers/persist/fs/fs.go

@@ -14,6 +14,7 @@ import (
 	"path/filepath"
 	"syscall"
 
+	"github.com/kata-containers/runtime/pkg/rootless"
 	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/sirupsen/logrus"
 )
@@ -36,9 +37,15 @@ const storagePathSuffix = "vc"
 // sandboxPathSuffix is the suffix used for sandbox storage
 const sandboxPathSuffix = "sbs"
 
-// runStoragePath is the sandbox runtime directory.
+// RunStoragePath is the sandbox runtime directory.
 // It will contain one state.json and one lock file for each created sandbox.
-var runStoragePath = filepath.Join("/run", storagePathSuffix, sandboxPathSuffix)
+var RunStoragePath = func() string {
+	path := filepath.Join("/run", storagePathSuffix, sandboxPathSuffix)
+	if rootless.IsRootless() {
+		return filepath.Join(rootless.GetRootlessDir(), path)
+	}
+	return path
+}
 
 // FS storage driver implementation
 type FS struct {
@@ -76,7 +83,7 @@ func (fs *FS) sandboxDir() (string, error) {
 		return "", fmt.Errorf("sandbox container id required")
 	}
 
-	return filepath.Join(runStoragePath, id), nil
+	return filepath.Join(RunStoragePath(), id), nil
 }
 
 // ToDisk sandboxState and containerState to disk
@@ -254,8 +261,10 @@ func (fs *FS) unlock() error {
 	return nil
 }
 
-// TestSetRunStoragePath set runStoragePath to path
+// TestSetRunStoragePath set RunStoragePath to path
 // this function is only used for testing purpose
 func TestSetRunStoragePath(path string) {
-	runStoragePath = path
+	RunStoragePath = func() string {
+		return path
+	}
 }

virtcontainers/sandbox.go

@@ -242,8 +242,11 @@ func (s *Sandbox) SetAnnotations(annotations map[string]string) error {
 		s.config.Annotations[k] = v
 	}
 
+	if !s.supportNewStore() {
 		return s.store.Store(store.Configuration, *(s.config))
 	}
+	return nil
+}
 
 // GetAnnotations returns sandbox's annotations
 func (s *Sandbox) GetAnnotations() map[string]string {
@@ -617,6 +620,12 @@ func (s *Sandbox) storeSandbox() error {
 	span, _ := s.trace("storeSandbox")
 	defer span.Finish()
 
+	if s.supportNewStore() {
+		// flush data to storage
+		if err := s.Save(); err != nil {
+			return err
+		}
+	} else {
 		err := s.store.Store(store.Configuration, *(s.config))
 		if err != nil {
 			return err
@@ -628,14 +637,7 @@ func (s *Sandbox) storeSandbox() error {
 				return err
 			}
 		}
-
-	if s.supportNewStore() {
-		// flush data to storage
-		if err := s.Save(); err != nil {
-			return err
 	}
-	}
-
 	return nil
 }
 
@@ -673,6 +675,22 @@ func unlockSandbox(ctx context.Context, sandboxID, token string) error {
 	return store.Unlock(token)
 }
 
+func supportNewStore(ctx context.Context) bool {
+	if exp.Get("newstore") == nil {
+		return false
+	}
+
+	// check if client context enabled "newstore" feature
+	exps := exp.ExpFromContext(ctx)
+	for _, v := range exps {
+		if v == "newstore" {
+			return true
+		}
+	}
+
+	return false
+}
+
 // fetchSandbox fetches a sandbox config from a sandbox ID and returns a sandbox.
 func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err error) {
 	virtLog.Info("fetch sandbox")
@@ -685,16 +703,25 @@ func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err
 		return sandbox, err
 	}
 
+	var config SandboxConfig
+
+	if supportNewStore(ctx) {
+		c, err := loadSandboxConfig(sandboxID)
+		if err != nil {
+			return nil, err
+		}
+		config = *c
+	} else {
 		// We're bootstrapping
 		vcStore, err := store.NewVCSandboxStore(ctx, sandboxID)
 		if err != nil {
 			return nil, err
 		}
 
-	var config SandboxConfig
 		if err := vcStore.Load(store.Configuration, &config); err != nil {
 			return nil, err
 		}
+	}
 
 	// fetchSandbox is not suppose to create new sandbox VM.
 	sandbox, err = createSandbox(ctx, config, nil)