storage: set new storage driver as "experimental"

Set new persist storage driver "virtcontainers/persist/" as "experimental"
feature.
One day when this can fully work and we're ready to move to 2.0, we'll move
it from "experimental" feature to formal feature.
At that time, the "virtcontainers/filesystem_resource_storage.go" can be removed
completely.

Signed-off-by: Wei Zhang <zhangwei555@huawei.com>

virtcontainers/container.go

@@ -374,7 +374,15 @@ func (c *Container) SetPid(pid int) error {
 func (c *Container) setStateFstype(fstype string) error {
 	c.state.Fstype = fstype
 
-	return c.storeState()
+	if !c.sandbox.supportNewStore() {
+		// experimental runtime use "persist.json" which doesn't need "state.json" anymore
+		err := c.storeState()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 // GetAnnotations returns container's annotations
@@ -384,8 +392,10 @@ func (c *Container) GetAnnotations() map[string]string {
 
 // storeContainer stores a container config.
 func (c *Container) storeContainer() error {
-	if err := c.sandbox.newStore.ToDisk(); err != nil {
+	if c.sandbox.supportNewStore() {
-		return err
+		if err := c.sandbox.newStore.ToDisk(); err != nil {
+			return err
+		}
 	}
 	return c.store.Store(store.Configuration, *(c.config))
 }
@@ -436,16 +446,20 @@ func (c *Container) setContainerState(state types.StateString) error {
 	// update in-memory state
 	c.state.State = state
 
-	// update on-disk state
+	if c.sandbox.supportNewStore() {
-	err := c.store.Store(store.State, c.state)
+		// flush data to storage
-	if err != nil {
+		if err := c.sandbox.newStore.ToDisk(); err != nil {
-		return err
+			return err
+		}
+	} else {
+		// experimental runtime use "persist.json" which doesn't need "state.json" anymore
+		// update on-disk state
+		err := c.store.Store(store.State, c.state)
+		if err != nil {
+			return err
+		}
 	}
 
-	// flush data to storage
-	if err = c.sandbox.newStore.ToDisk(); err != nil {
-		return err
-	}
 	return nil
 }
 
@@ -687,10 +701,18 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err
 
 	c.store = ctrStore
 
-	/*state, err := c.store.LoadContainerState()
+	// experimental runtime use "persist.json" instead of legacy "state.json" as storage
-	if err == nil {
+	if c.sandbox.supportNewStore() {
-		c.state = state
+		if err := c.Restore(); err != nil &&
-	}*/
+			!os.IsNotExist(err) && err != errContainerPersistNotExist {
+			return nil, err
+		}
+	} else {
+		state, err := c.store.LoadContainerState()
+		if err == nil {
+			c.state = state
+		}
+	}
 
 	if err := c.Restore(); err != nil &&
 		!os.IsNotExist(err) && err != errContainerPersistNotExist {

virtcontainers/persist.go

@@ -9,6 +9,8 @@ import (
 	"errors"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
+	exp "github.com/kata-containers/runtime/virtcontainers/experimental"
+	"github.com/kata-containers/runtime/virtcontainers/persist"
 	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 )
@@ -158,3 +160,12 @@ func (c *Container) Restore() error {
 
 	return nil
 }
+
+func (s *Sandbox) supportNewStore() bool {
+	for _, f := range s.config.Experimental {
+		if f == persist.NewStoreFeature && exp.Get("newstore") != nil {
+			return true
+		}
+	}
+	return false
+}

virtcontainers/persist/manager.go

@@ -8,6 +8,7 @@ package persist
 import (
 	"fmt"
 
+	exp "github.com/kata-containers/runtime/virtcontainers/experimental"
 	"github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/persist/fs"
 )
@@ -15,14 +16,29 @@ import (
 type initFunc (func() (persistapi.PersistDriver, error))
 
 var (
+	// NewStoreFeature is an experimental feature
+	NewStoreFeature = exp.Feature{
+		Name:        "newstore",
+		Description: "This is a new storage driver which reorganized disk data structures, it has to be an experimental feature since it breaks backward compatibility.",
+		ExpRelease:  "2.0",
+	}
+	expErr           error
 	supportedDrivers = map[string]initFunc{
 
 		"fs": fs.Init,
 	}
 )
 
+func init() {
+	expErr = exp.Register(NewStoreFeature)
+}
+
 // GetDriver returns new PersistDriver according to driver name
 func GetDriver(name string) (persistapi.PersistDriver, error) {
+	if expErr != nil {
+		return nil, expErr
+	}
+
 	if f, ok := supportedDrivers[name]; ok {
 		return f()
 	}

virtcontainers/persist_test.go

@@ -0,0 +1,137 @@
+// Copyright (c) 2019 Huawei Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package virtcontainers
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	exp "github.com/kata-containers/runtime/virtcontainers/experimental"
+	"github.com/kata-containers/runtime/virtcontainers/persist"
+	"github.com/kata-containers/runtime/virtcontainers/types"
+)
+
+func testCreateExpSandbox() (*Sandbox, error) {
+	sconfig := SandboxConfig{
+		ID:               "test-exp",
+		HypervisorType:   MockHypervisor,
+		HypervisorConfig: newHypervisorConfig(nil, nil),
+		AgentType:        NoopAgentType,
+		NetworkConfig:    NetworkConfig{},
+		Volumes:          nil,
+		Containers:       nil,
+		Experimental:     []exp.Feature{persist.NewStoreFeature},
+	}
+
+	// support experimental
+	sandbox, err := createSandbox(context.Background(), sconfig, nil)
+	if err != nil {
+		return nil, fmt.Errorf("Could not create sandbox: %s", err)
+	}
+
+	if err := sandbox.agent.startSandbox(sandbox); err != nil {
+		return nil, err
+	}
+
+	return sandbox, nil
+}
+
+func TestSupportNewStore(t *testing.T) {
+	hConfig := newHypervisorConfig(nil, nil)
+	sandbox, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanUp()
+
+	// not support experimental
+	assert.False(t, sandbox.supportNewStore())
+
+	// support experimental
+	sandbox, err = testCreateExpSandbox()
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.True(t, sandbox.supportNewStore())
+}
+
+func TestSandboxRestore(t *testing.T) {
+	var err error
+	sconfig := SandboxConfig{
+		ID:           "test-exp",
+		Experimental: []exp.Feature{persist.NewStoreFeature},
+	}
+	container := make(map[string]*Container)
+	container["test-exp"] = &Container{}
+
+	sandbox := Sandbox{
+		id:         "test-exp",
+		containers: container,
+		hypervisor: &mockHypervisor{},
+		ctx:        context.Background(),
+		config:     &sconfig,
+	}
+
+	if sandbox.newStore, err = persist.GetDriver("fs"); err != nil || sandbox.newStore == nil {
+		t.Fatalf("failed to get fs persist driver")
+	}
+
+	// if we don't call ToDisk, we can get nothing from disk
+	err = sandbox.Restore()
+	assert.NotNil(t, err)
+	assert.True(t, os.IsNotExist(err))
+
+	// disk data are empty
+	err = sandbox.newStore.ToDisk()
+	assert.Nil(t, err)
+
+	err = sandbox.Restore()
+	assert.Nil(t, err)
+	assert.Equal(t, sandbox.state.State, types.StateString(""))
+	assert.Equal(t, sandbox.state.GuestMemoryBlockSizeMB, uint32(0))
+	assert.Equal(t, sandbox.state.BlockIndex, 0)
+	assert.Equal(t, sandbox.state.Pid, 0)
+
+	// register callback function
+	sandbox.stateSaveCallback()
+
+	sandbox.state.State = types.StateString("running")
+	sandbox.state.GuestMemoryBlockSizeMB = uint32(1024)
+	sandbox.state.BlockIndex = 2
+	sandbox.state.Pid = 10000
+	// flush data to disk
+	err = sandbox.newStore.ToDisk()
+	assert.Nil(t, err)
+
+	// empty the sandbox
+	sandbox.state = types.State{}
+
+	// restore data from disk
+	err = sandbox.Restore()
+	assert.Nil(t, err)
+	assert.Equal(t, sandbox.state.State, types.StateString("running"))
+	assert.Equal(t, sandbox.state.GuestMemoryBlockSizeMB, uint32(1024))
+	assert.Equal(t, sandbox.state.Pid, 10000)
+	// require hvStateSaveCallback to make it savable
+	assert.Equal(t, sandbox.state.BlockIndex, 0)
+
+	// after use hvStateSaveCallbck, BlockIndex can be saved now
+	sandbox.state.BlockIndex = 2
+	sandbox.hvStateSaveCallback()
+	err = sandbox.newStore.ToDisk()
+	assert.Nil(t, err)
+	err = sandbox.Restore()
+	assert.Nil(t, err)
+	assert.Equal(t, sandbox.state.State, types.StateString("running"))
+	assert.Equal(t, sandbox.state.GuestMemoryBlockSizeMB, uint32(1024))
+	assert.Equal(t, sandbox.state.Pid, 10000)
+	assert.Equal(t, sandbox.state.BlockIndex, 2)
+
+}

virtcontainers/sandbox.go

@@ -435,8 +435,10 @@ func (s *Sandbox) getAndStoreGuestDetails() error {
 		}
 		s.state.GuestMemoryHotplugProbe = guestDetailRes.SupportMemHotplugProbe
 
-		if err = s.store.Store(store.State, s.state); err != nil {
+		if !s.supportNewStore() {
-			return err
+			if err = s.store.Store(store.State, s.state); err != nil {
+				return err
+			}
 		}
 	}
 
@@ -477,29 +479,31 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
 	}
 	s.devManager = deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, devices)
 
-	// register persist hook for now, data will be written to disk by ToDisk()
+	if s.supportNewStore() {
-	s.stateSaveCallback()
+		// register persist hook for now, data will be written to disk by ToDisk()
-	s.hvStateSaveCallback()
+		s.stateSaveCallback()
-	s.devicesSaveCallback()
+		s.hvStateSaveCallback()
+		s.devicesSaveCallback()
 
-	if err := s.Restore(); err == nil && s.state.State != "" {
+		if err := s.Restore(); err == nil && s.state.State != "" {
-		return s, nil
+			return s, nil
+		}
+
+		// if sandbox doesn't exist, set persist version to current version
+		// otherwise do nothing
+		s.verSaveCallback()
+	} else {
+		// We first try to fetch the sandbox state from storage.
+		// If it exists, this means this is a re-creation, i.e.
+		// we don't need to talk to the guest's agent, but only
+		// want to create the sandbox and its containers in memory.
+		state, err := s.store.LoadState()
+		if err == nil && state.State != "" {
+			s.state = state
+			return s, nil
+		}
 	}
 
-	// We first try to fetch the sandbox state from storage.
-	// If it exists, this means this is a re-creation, i.e.
-	// we don't need to talk to the guest's agent, but only
-	// want to create the sandbox and its containers in memory.
-	/* state, err := s.store.LoadState()
-	if err == nil && state.State != "" {
-		s.state = state
-		return s, nil
-	}*/
-
-	// if sandbox doesn't exist, set persist version to current version
-	// otherwise do nothing
-	s.verSaveCallback()
-
 	// Below code path is called only during create, because of earlier check.
 	if err := s.agent.createSandbox(s); err != nil {
 		return nil, err
@@ -608,9 +612,11 @@ func (s *Sandbox) storeSandbox() error {
 		}
 	}
 
-	// flush data to storage
+	if s.supportNewStore() {
-	if err = s.newStore.ToDisk(); err != nil {
+		// flush data to storage
-		return err
+		if err := s.newStore.ToDisk(); err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -1037,7 +1043,9 @@ func (s *Sandbox) addContainer(c *Container) error {
 	ann := c.GetAnnotations()
 	if ann[annotations.ContainerTypeKey] == string(PodSandbox) {
 		s.state.CgroupPath = c.state.CgroupPath
-		return s.store.Store(store.State, s.state)
+		if !s.supportNewStore() {
+			return s.store.Store(store.State, s.state)
+		}
 	}
 
 	return nil
@@ -1512,7 +1520,10 @@ func (s *Sandbox) setSandboxState(state types.StateString) error {
 	s.state.State = state
 
 	// update on-disk state
-	return s.store.Store(store.State, s.state)
+	if !s.supportNewStore() {
+		return s.store.Store(store.State, s.state)
+	}
+	return nil
 }
 
 func (s *Sandbox) pauseSetStates() error {
@@ -1544,9 +1555,12 @@ func (s *Sandbox) getAndSetSandboxBlockIndex() (int, error) {
 	// Increment so that container gets incremented block index
 	s.state.BlockIndex++
 
-	// update on-disk state
+	if !s.supportNewStore() {
-	if err := s.store.Store(store.State, s.state); err != nil {
+		// experimental runtime use "persist.json" which doesn't need "state.json" anymore
-		return -1, err
+		// update on-disk state
+		if err := s.store.Store(store.State, s.state); err != nil {
+			return -1, err
+		}
 	}
 
 	return currentIndex, nil
@@ -1557,9 +1571,12 @@ func (s *Sandbox) getAndSetSandboxBlockIndex() (int, error) {
 func (s *Sandbox) decrementSandboxBlockIndex() error {
 	s.state.BlockIndex--
 
-	// update on-disk state
+	if !s.supportNewStore() {
-	if err := s.store.Store(store.State, s.state); err != nil {
+		// experimental runtime use "persist.json" which doesn't need "state.json" anymore
-		return err
+		// update on-disk state
+		if err := s.store.Store(store.State, s.state); err != nil {
+			return err
+		}
 	}
 
 	return nil