diff --git a/virtcontainers/container.go b/virtcontainers/container.go index 3241cc2115..9858a2f2ab 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -374,7 +374,15 @@ func (c *Container) SetPid(pid int) error { func (c *Container) setStateFstype(fstype string) error { c.state.Fstype = fstype - return c.storeState() + if !c.sandbox.supportNewStore() { + // experimental runtime use "persist.json" which doesn't need "state.json" anymore + err := c.storeState() + if err != nil { + return err + } + } + + return nil } // GetAnnotations returns container's annotations @@ -384,8 +392,10 @@ func (c *Container) GetAnnotations() map[string]string { // storeContainer stores a container config. func (c *Container) storeContainer() error { - if err := c.sandbox.newStore.ToDisk(); err != nil { - return err + if c.sandbox.supportNewStore() { + if err := c.sandbox.newStore.ToDisk(); err != nil { + return err + } } return c.store.Store(store.Configuration, *(c.config)) } @@ -436,16 +446,20 @@ func (c *Container) setContainerState(state types.StateString) error { // update in-memory state c.state.State = state - // update on-disk state - err := c.store.Store(store.State, c.state) - if err != nil { - return err + if c.sandbox.supportNewStore() { + // flush data to storage + if err := c.sandbox.newStore.ToDisk(); err != nil { + return err + } + } else { + // experimental runtime use "persist.json" which doesn't need "state.json" anymore + // update on-disk state + err := c.store.Store(store.State, c.state) + if err != nil { + return err + } } - // flush data to storage - if err = c.sandbox.newStore.ToDisk(); err != nil { - return err - } return nil } @@ -687,10 +701,18 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err c.store = ctrStore - /*state, err := c.store.LoadContainerState() - if err == nil { - c.state = state - }*/ + // experimental runtime use "persist.json" instead of legacy "state.json" as storage + if c.sandbox.supportNewStore() { + if err := c.Restore(); err != nil && + !os.IsNotExist(err) && err != errContainerPersistNotExist { + return nil, err + } + } else { + state, err := c.store.LoadContainerState() + if err == nil { + c.state = state + } + } if err := c.Restore(); err != nil && !os.IsNotExist(err) && err != errContainerPersistNotExist { diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go index 3fb0afef2d..7fdc6fa3c9 100644 --- a/virtcontainers/persist.go +++ b/virtcontainers/persist.go @@ -9,6 +9,8 @@ import ( "errors" "github.com/kata-containers/runtime/virtcontainers/device/api" + exp "github.com/kata-containers/runtime/virtcontainers/experimental" + "github.com/kata-containers/runtime/virtcontainers/persist" persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/types" ) @@ -158,3 +160,12 @@ func (c *Container) Restore() error { return nil } + +func (s *Sandbox) supportNewStore() bool { + for _, f := range s.config.Experimental { + if f == persist.NewStoreFeature && exp.Get("newstore") != nil { + return true + } + } + return false +} diff --git a/virtcontainers/persist/manager.go b/virtcontainers/persist/manager.go index aad31fdaf9..8bbf3f7e08 100644 --- a/virtcontainers/persist/manager.go +++ b/virtcontainers/persist/manager.go @@ -8,6 +8,7 @@ package persist import ( "fmt" + exp "github.com/kata-containers/runtime/virtcontainers/experimental" "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/persist/fs" ) @@ -15,14 +16,29 @@ import ( type initFunc (func() (persistapi.PersistDriver, error)) var ( + // NewStoreFeature is an experimental feature + NewStoreFeature = exp.Feature{ + Name: "newstore", + Description: "This is a new storage driver which reorganized disk data structures, it has to be an experimental feature since it breaks backward compatibility.", + ExpRelease: "2.0", + } + expErr error supportedDrivers = map[string]initFunc{ "fs": fs.Init, } ) +func init() { + expErr = exp.Register(NewStoreFeature) +} + // GetDriver returns new PersistDriver according to driver name func GetDriver(name string) (persistapi.PersistDriver, error) { + if expErr != nil { + return nil, expErr + } + if f, ok := supportedDrivers[name]; ok { return f() } diff --git a/virtcontainers/persist_test.go b/virtcontainers/persist_test.go new file mode 100644 index 0000000000..96354f5272 --- /dev/null +++ b/virtcontainers/persist_test.go @@ -0,0 +1,137 @@ +// Copyright (c) 2019 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + + exp "github.com/kata-containers/runtime/virtcontainers/experimental" + "github.com/kata-containers/runtime/virtcontainers/persist" + "github.com/kata-containers/runtime/virtcontainers/types" +) + +func testCreateExpSandbox() (*Sandbox, error) { + sconfig := SandboxConfig{ + ID: "test-exp", + HypervisorType: MockHypervisor, + HypervisorConfig: newHypervisorConfig(nil, nil), + AgentType: NoopAgentType, + NetworkConfig: NetworkConfig{}, + Volumes: nil, + Containers: nil, + Experimental: []exp.Feature{persist.NewStoreFeature}, + } + + // support experimental + sandbox, err := createSandbox(context.Background(), sconfig, nil) + if err != nil { + return nil, fmt.Errorf("Could not create sandbox: %s", err) + } + + if err := sandbox.agent.startSandbox(sandbox); err != nil { + return nil, err + } + + return sandbox, nil +} + +func TestSupportNewStore(t *testing.T) { + hConfig := newHypervisorConfig(nil, nil) + sandbox, err := testCreateSandbox(t, testSandboxID, MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, nil, nil) + if err != nil { + t.Fatal(err) + } + defer cleanUp() + + // not support experimental + assert.False(t, sandbox.supportNewStore()) + + // support experimental + sandbox, err = testCreateExpSandbox() + if err != nil { + t.Fatal(err) + } + assert.True(t, sandbox.supportNewStore()) +} + +func TestSandboxRestore(t *testing.T) { + var err error + sconfig := SandboxConfig{ + ID: "test-exp", + Experimental: []exp.Feature{persist.NewStoreFeature}, + } + container := make(map[string]*Container) + container["test-exp"] = &Container{} + + sandbox := Sandbox{ + id: "test-exp", + containers: container, + hypervisor: &mockHypervisor{}, + ctx: context.Background(), + config: &sconfig, + } + + if sandbox.newStore, err = persist.GetDriver("fs"); err != nil || sandbox.newStore == nil { + t.Fatalf("failed to get fs persist driver") + } + + // if we don't call ToDisk, we can get nothing from disk + err = sandbox.Restore() + assert.NotNil(t, err) + assert.True(t, os.IsNotExist(err)) + + // disk data are empty + err = sandbox.newStore.ToDisk() + assert.Nil(t, err) + + err = sandbox.Restore() + assert.Nil(t, err) + assert.Equal(t, sandbox.state.State, types.StateString("")) + assert.Equal(t, sandbox.state.GuestMemoryBlockSizeMB, uint32(0)) + assert.Equal(t, sandbox.state.BlockIndex, 0) + assert.Equal(t, sandbox.state.Pid, 0) + + // register callback function + sandbox.stateSaveCallback() + + sandbox.state.State = types.StateString("running") + sandbox.state.GuestMemoryBlockSizeMB = uint32(1024) + sandbox.state.BlockIndex = 2 + sandbox.state.Pid = 10000 + // flush data to disk + err = sandbox.newStore.ToDisk() + assert.Nil(t, err) + + // empty the sandbox + sandbox.state = types.State{} + + // restore data from disk + err = sandbox.Restore() + assert.Nil(t, err) + assert.Equal(t, sandbox.state.State, types.StateString("running")) + assert.Equal(t, sandbox.state.GuestMemoryBlockSizeMB, uint32(1024)) + assert.Equal(t, sandbox.state.Pid, 10000) + // require hvStateSaveCallback to make it savable + assert.Equal(t, sandbox.state.BlockIndex, 0) + + // after use hvStateSaveCallbck, BlockIndex can be saved now + sandbox.state.BlockIndex = 2 + sandbox.hvStateSaveCallback() + err = sandbox.newStore.ToDisk() + assert.Nil(t, err) + err = sandbox.Restore() + assert.Nil(t, err) + assert.Equal(t, sandbox.state.State, types.StateString("running")) + assert.Equal(t, sandbox.state.GuestMemoryBlockSizeMB, uint32(1024)) + assert.Equal(t, sandbox.state.Pid, 10000) + assert.Equal(t, sandbox.state.BlockIndex, 2) + +} diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index ca0fe5cac3..be161b6b4a 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -435,8 +435,10 @@ func (s *Sandbox) getAndStoreGuestDetails() error { } s.state.GuestMemoryHotplugProbe = guestDetailRes.SupportMemHotplugProbe - if err = s.store.Store(store.State, s.state); err != nil { - return err + if !s.supportNewStore() { + if err = s.store.Store(store.State, s.state); err != nil { + return err + } } } @@ -477,29 +479,31 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac } s.devManager = deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, devices) - // register persist hook for now, data will be written to disk by ToDisk() - s.stateSaveCallback() - s.hvStateSaveCallback() - s.devicesSaveCallback() + if s.supportNewStore() { + // register persist hook for now, data will be written to disk by ToDisk() + s.stateSaveCallback() + s.hvStateSaveCallback() + s.devicesSaveCallback() - if err := s.Restore(); err == nil && s.state.State != "" { - return s, nil + if err := s.Restore(); err == nil && s.state.State != "" { + return s, nil + } + + // if sandbox doesn't exist, set persist version to current version + // otherwise do nothing + s.verSaveCallback() + } else { + // We first try to fetch the sandbox state from storage. + // If it exists, this means this is a re-creation, i.e. + // we don't need to talk to the guest's agent, but only + // want to create the sandbox and its containers in memory. + state, err := s.store.LoadState() + if err == nil && state.State != "" { + s.state = state + return s, nil + } } - // We first try to fetch the sandbox state from storage. - // If it exists, this means this is a re-creation, i.e. - // we don't need to talk to the guest's agent, but only - // want to create the sandbox and its containers in memory. - /* state, err := s.store.LoadState() - if err == nil && state.State != "" { - s.state = state - return s, nil - }*/ - - // if sandbox doesn't exist, set persist version to current version - // otherwise do nothing - s.verSaveCallback() - // Below code path is called only during create, because of earlier check. if err := s.agent.createSandbox(s); err != nil { return nil, err @@ -608,9 +612,11 @@ func (s *Sandbox) storeSandbox() error { } } - // flush data to storage - if err = s.newStore.ToDisk(); err != nil { - return err + if s.supportNewStore() { + // flush data to storage + if err := s.newStore.ToDisk(); err != nil { + return err + } } return nil @@ -1037,7 +1043,9 @@ func (s *Sandbox) addContainer(c *Container) error { ann := c.GetAnnotations() if ann[annotations.ContainerTypeKey] == string(PodSandbox) { s.state.CgroupPath = c.state.CgroupPath - return s.store.Store(store.State, s.state) + if !s.supportNewStore() { + return s.store.Store(store.State, s.state) + } } return nil @@ -1512,7 +1520,10 @@ func (s *Sandbox) setSandboxState(state types.StateString) error { s.state.State = state // update on-disk state - return s.store.Store(store.State, s.state) + if !s.supportNewStore() { + return s.store.Store(store.State, s.state) + } + return nil } func (s *Sandbox) pauseSetStates() error { @@ -1544,9 +1555,12 @@ func (s *Sandbox) getAndSetSandboxBlockIndex() (int, error) { // Increment so that container gets incremented block index s.state.BlockIndex++ - // update on-disk state - if err := s.store.Store(store.State, s.state); err != nil { - return -1, err + if !s.supportNewStore() { + // experimental runtime use "persist.json" which doesn't need "state.json" anymore + // update on-disk state + if err := s.store.Store(store.State, s.state); err != nil { + return -1, err + } } return currentIndex, nil @@ -1557,9 +1571,12 @@ func (s *Sandbox) getAndSetSandboxBlockIndex() (int, error) { func (s *Sandbox) decrementSandboxBlockIndex() error { s.state.BlockIndex-- - // update on-disk state - if err := s.store.Store(store.State, s.state); err != nil { - return err + if !s.supportNewStore() { + // experimental runtime use "persist.json" which doesn't need "state.json" anymore + // update on-disk state + if err := s.store.Store(store.State, s.state); err != nil { + return err + } } return nil