diff --git a/virtcontainers/api.go b/virtcontainers/api.go index 8925f458f1..c3197c1c50 100644 --- a/virtcontainers/api.go +++ b/virtcontainers/api.go @@ -224,6 +224,10 @@ func StartSandbox(ctx context.Context, sandboxID string) (VCSandbox, error) { return nil, err } + if err = s.storeSandbox(); err != nil { + return nil, err + } + return s, nil } @@ -256,6 +260,10 @@ func StopSandbox(ctx context.Context, sandboxID string) (VCSandbox, error) { return nil, err } + if err = s.storeSandbox(); err != nil { + return nil, err + } + return s, nil } @@ -394,6 +402,10 @@ func CreateContainer(ctx context.Context, sandboxID string, containerConfig Cont return nil, nil, err } + if err = s.storeSandbox(); err != nil { + return nil, nil, err + } + return s, c, nil } diff --git a/virtcontainers/container.go b/virtcontainers/container.go index 2c7e8c0572..e18a9a1a39 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -374,6 +374,9 @@ func (c *Container) GetAnnotations() map[string]string { // storeContainer stores a container config. func (c *Container) storeContainer() error { + if err := c.sandbox.newStore.Dump(); err != nil { + return err + } return c.store.Store(store.Configuration, *(c.config)) } diff --git a/virtcontainers/container_test.go b/virtcontainers/container_test.go index 5e28e5e29a..e475e93056 100644 --- a/virtcontainers/container_test.go +++ b/virtcontainers/container_test.go @@ -20,6 +20,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" "github.com/kata-containers/runtime/virtcontainers/device/manager" + "github.com/kata-containers/runtime/virtcontainers/persist" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" @@ -247,6 +248,10 @@ func TestContainerAddDriveDir(t *testing.T) { assert.Nil(t, err) container.store = containerStore + if sandbox.newStore, err = persist.GetDriver("fs"); err != nil || sandbox.newStore == nil { + t.Fatalf("failed to get fs persist driver") + } + // create state file path := store.ContainerRuntimeRootPath(testSandboxID, container.ID()) stateFilePath := filepath.Join(path, store.StateFile) diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go new file mode 100644 index 0000000000..605f3ab1f4 --- /dev/null +++ b/virtcontainers/persist.go @@ -0,0 +1,96 @@ +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + //"fmt" + + //"github.com/sirupsen/logrus" + + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" + "github.com/kata-containers/runtime/virtcontainers/types" +) + +func (s *Sandbox) dumpState(ss *persistapi.SandboxState, cs map[string]persistapi.ContainerState) error { + ss.SandboxContainer = s.id + ss.GuestMemoryBlockSizeMB = s.state.GuestMemoryBlockSizeMB + ss.State = string(s.state.State) + + for id, cont := range s.containers { + cs[id] = persistapi.ContainerState{ + State: string(cont.state.State), + Rootfs: persistapi.RootfsState{ + BlockDeviceID: cont.state.BlockDeviceID, + FsType: cont.state.Fstype, + }, + } + } + return nil +} + +func (s *Sandbox) dumpHypervisor(ss *persistapi.SandboxState, cs map[string]persistapi.ContainerState) error { + ss.HypervisorState.BlockIndex = s.state.BlockIndex + return nil +} + +// PersistVersion set persist data version to current version in runtime +func (s *Sandbox) persistVersion() { + s.newStore.RegisterHook("version", func(ss *persistapi.SandboxState, cs map[string]persistapi.ContainerState) error { + ss.PersistVersion = persistapi.CurPersistVersion + return nil + }) +} + +// PersistState register hook to set sandbox and container state to persist +func (s *Sandbox) persistState() { + s.newStore.RegisterHook("state", s.dumpState) +} + +// PersistHvState register hook to save hypervisor state to persist data +func (s *Sandbox) persistHvState() { + s.newStore.RegisterHook("hypervisor", s.dumpHypervisor) +} + +// Restore will restore data from persist disk on disk +func (s *Sandbox) Restore() error { + if err := s.newStore.Restore(s.id); err != nil { + return err + } + + ss, _, err := s.newStore.GetStates() + if err != nil { + return err + } + + /* + // TODO: need more modifications, restoring containers + // will make sandbox.addContainer failing + if s.containers == nil { + s.containers = make(map[string]*Container) + } + + for id, cont := range cs { + s.containers[id] = &Container{ + state: State{ + State: stateString(cont.State), + BlockDeviceID: cont.Rootfs.BlockDeviceID, + Fstype: cont.Rootfs.FsType, + Pid: cont.ShimPid, + }, + } + } + + sbxCont, ok := s.containers[ss.SandboxContainer] + if !ok { + return fmt.Errorf("failed to get sandbox container state") + } + */ + s.state.GuestMemoryBlockSizeMB = ss.GuestMemoryBlockSizeMB + s.state.BlockIndex = ss.HypervisorState.BlockIndex + s.state.State = types.StateString(ss.State) + + return nil +} diff --git a/virtcontainers/persist/api/interface.go b/virtcontainers/persist/api/interface.go new file mode 100644 index 0000000000..0fb06e5be9 --- /dev/null +++ b/virtcontainers/persist/api/interface.go @@ -0,0 +1,16 @@ +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package persistapi + +// PersistDriver is interface describing operations to save/restore persist data +type PersistDriver interface { + // Dump persist data to + Dump() error + Restore(sid string) error + Destroy() error + GetStates() (*SandboxState, map[string]ContainerState, error) + RegisterHook(name string, f SetFunc) +} diff --git a/virtcontainers/persist/fs/fs.go b/virtcontainers/persist/fs/fs.go new file mode 100644 index 0000000000..a6a3ebd60f --- /dev/null +++ b/virtcontainers/persist/fs/fs.go @@ -0,0 +1,270 @@ +// Copyright (c) 2016 Intel Corporation +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package fs + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "syscall" + + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" +) + +// persistFile is the file name for JSON sandbox/container configuration +const persistFile = "persist.json" + +// dirMode is the permission bits used for creating a directory +const dirMode = os.FileMode(0750) + +// fileMode is the permission bits used for creating a file +const fileMode = os.FileMode(0640) + +// storagePathSuffix is the suffix used for all storage paths +// +// Note: this very brief path represents "virtcontainers". It is as +// terse as possible to minimise path length. +const storagePathSuffix = "vc" + +// sandboxPathSuffix is the suffix used for sandbox storage +const sandboxPathSuffix = "sbs" + +// runStoragePath is the sandbox runtime directory. +// It will contain one state.json and one lock file for each created sandbox. +var runStoragePath = filepath.Join("/run", storagePathSuffix, sandboxPathSuffix) + +// FS storage driver implementation +type FS struct { + sandboxState *persistapi.SandboxState + containerState map[string]persistapi.ContainerState + setFuncs map[string]persistapi.SetFunc + + lockFile *os.File +} + +// Name returns driver name +func Name() string { + return "fs" +} + +// Init FS persist driver and return abstract PersistDriver +func Init() (persistapi.PersistDriver, error) { + return &FS{ + sandboxState: &persistapi.SandboxState{}, + containerState: make(map[string]persistapi.ContainerState), + setFuncs: make(map[string]persistapi.SetFunc), + }, nil +} + +func (fs *FS) sandboxDir() (string, error) { + if fs.sandboxState.SandboxContainer == "" { + return "", fmt.Errorf("sandbox container id required") + } + + return filepath.Join(runStoragePath, fs.sandboxState.SandboxContainer), nil +} + +// Dump sandboxState and containerState to disk +func (fs *FS) Dump() (retErr error) { + // call registered hooks to set sandboxState and containerState + for _, fun := range fs.setFuncs { + fun(fs.sandboxState, fs.containerState) + } + + // if error happened, destroy all dirs + defer func() { + if retErr != nil { + // TODO: log error + fs.Destroy() + } + }() + + sandboxDir, err := fs.sandboxDir() + if err != nil { + return err + } + + if err := os.MkdirAll(sandboxDir, dirMode); err != nil { + return err + } + + if err := fs.lock(); err != nil { + return err + } + defer fs.unlock() + + // persist sandbox configuration data + sandboxFile := filepath.Join(sandboxDir, persistFile) + f, err := os.OpenFile(sandboxFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileMode) + if err != nil { + return err + } + defer f.Close() + + if err := json.NewEncoder(f).Encode(fs.sandboxState); err != nil { + return err + } + + // persist container configuration data + for cid, cstate := range fs.containerState { + cdir := filepath.Join(sandboxDir, cid) + if err := os.MkdirAll(cdir, dirMode); err != nil { + return err + } + + cfile := filepath.Join(cdir, persistFile) + cf, err := os.OpenFile(cfile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileMode) + if err != nil { + return err + } + + if err := json.NewEncoder(cf).Encode(cstate); err != nil { + return err + } + cf.Close() + } + + return nil +} + +// Restore state for sandbox with name sid +func (fs *FS) Restore(sid string) error { + if sid == "" { + return fmt.Errorf("restore requires sandbox id") + } + + fs.sandboxState.SandboxContainer = sid + sandboxDir, err := fs.sandboxDir() + if err != nil { + return err + } + + if err := os.MkdirAll(sandboxDir, dirMode); err != nil { + return err + } + + if err := fs.lock(); err != nil { + return err + } + defer fs.unlock() + + // get sandbox configuration from persist data + sandboxFile := filepath.Join(sandboxDir, persistFile) + f, err := os.OpenFile(sandboxFile, os.O_RDONLY, fileMode) + if err != nil { + return err + } + defer f.Close() + + if err := json.NewDecoder(f).Decode(fs.sandboxState); err != nil { + return err + } + + // walk sandbox dir and find container + d, err := os.OpenFile(sandboxDir, os.O_RDONLY, fileMode) + if err != nil { + return err + } + defer d.Close() + + files, err := d.Readdir(-1) + if err != nil { + return err + } + + for _, file := range files { + if !file.IsDir() { + continue + } + + cid := file.Name() + cfile := filepath.Join(sandboxDir, cid, persistFile) + cf, err := os.OpenFile(cfile, os.O_RDONLY, fileMode) + if err != nil { + // if persist.json doesn't exist, ignore and go to next + if os.IsNotExist(err) { + continue + } + return err + } + + var cstate persistapi.ContainerState + if err := json.NewDecoder(cf).Decode(&cstate); err != nil { + return err + } + cf.Close() + + fs.containerState[cid] = cstate + } + return nil +} + +// Destroy removes everything from disk +func (fs *FS) Destroy() error { + sandboxDir, err := fs.sandboxDir() + if err != nil { + return err + } + + if err := os.RemoveAll(sandboxDir); err != nil { + return err + } + return nil +} + +// GetStates returns SandboxState and ContainerState +func (fs *FS) GetStates() (*persistapi.SandboxState, map[string]persistapi.ContainerState, error) { + return fs.sandboxState, fs.containerState, nil +} + +// RegisterHook registers processing hooks for Dump +func (fs *FS) RegisterHook(name string, f persistapi.SetFunc) { + // only accept last registered hook with same name + fs.setFuncs[name] = f +} + +func (fs *FS) lock() error { + sandboxDir, err := fs.sandboxDir() + if err != nil { + return err + } + + f, err := os.Open(sandboxDir) + if err != nil { + return err + } + + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + f.Close() + return err + } + fs.lockFile = f + + return nil +} + +func (fs *FS) unlock() error { + if fs.lockFile == nil { + return nil + } + + lockFile := fs.lockFile + defer lockFile.Close() + fs.lockFile = nil + if err := syscall.Flock(int(lockFile.Fd()), syscall.LOCK_UN); err != nil { + return err + } + + return nil +} + +// TestSetRunStoragePath set runStoragePath to path +// this function is only used for testing purpose +func TestSetRunStoragePath(path string) { + runStoragePath = path +} diff --git a/virtcontainers/persist/manager.go b/virtcontainers/persist/manager.go new file mode 100644 index 0000000000..c52bfc0327 --- /dev/null +++ b/virtcontainers/persist/manager.go @@ -0,0 +1,31 @@ +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package persist + +import ( + "fmt" + + "github.com/kata-containers/runtime/virtcontainers/persist/api" + "github.com/kata-containers/runtime/virtcontainers/persist/fs" +) + +type initFunc (func() (persistapi.PersistDriver, error)) + +var ( + supportedDrivers = map[string]initFunc{ + "fs": fs.Init, + } + defaultDriver = "fs" +) + +// GetDriver returns new PersistDriver according to driver name +func GetDriver(name string) (persistapi.PersistDriver, error) { + if f, ok := supportedDrivers[name]; ok { + return f() + } + + return nil, fmt.Errorf("failed to get storage driver %q", name) +} diff --git a/virtcontainers/persist/plugin/README.md b/virtcontainers/persist/plugin/README.md new file mode 100644 index 0000000000..ec230d15b8 --- /dev/null +++ b/virtcontainers/persist/plugin/README.md @@ -0,0 +1,2 @@ +This package is a simple placeholder currently which will contain persist storage plugin support, +e.g. leveldb, sqlite and other possible storage implementations. diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 1faf9f4441..a507f9c318 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -27,6 +27,8 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/drivers" deviceManager "github.com/kata-containers/runtime/virtcontainers/device/manager" exp "github.com/kata-containers/runtime/virtcontainers/experimental" + "github.com/kata-containers/runtime/virtcontainers/persist" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/kata-containers/runtime/virtcontainers/store" @@ -159,8 +161,11 @@ type Sandbox struct { hypervisor hypervisor agent agent store *store.VCStore - network Network - monitor *monitor + // store is used to replace VCStore step by step + newStore persistapi.PersistDriver + + network Network + monitor *monitor config *SandboxConfig @@ -472,15 +477,25 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac } s.devManager = deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, devices) + // register persist hook for now, data will be written to disk by Dump() + s.persistState() + s.persistHvState() + + if err := s.Restore(); err == nil && s.state.State != "" { + return s, nil + } // We first try to fetch the sandbox state from storage. // If it exists, this means this is a re-creation, i.e. // we don't need to talk to the guest's agent, but only // want to create the sandbox and its containers in memory. - state, err := s.store.LoadState() + /* state, err := s.store.LoadState() if err == nil && state.State != "" { s.state = state return s, nil - } + }*/ + + // if sandbox doesn't exist, set persist version to current version + s.persistVersion() // Below code path is called only during create, because of earlier check. if err := s.agent.createSandbox(s); err != nil { @@ -536,6 +551,10 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor s.store = vcStore + if s.newStore, err = persist.GetDriver("fs"); err != nil || s.newStore == nil { + return nil, fmt.Errorf("failed to get fs persist driver") + } + if err = globalSandboxList.addSandbox(s); err != nil { return nil, err } @@ -586,6 +605,10 @@ func (s *Sandbox) storeSandbox() error { } } + if err = s.newStore.Dump(); err != nil { + return err + } + return nil } @@ -1078,6 +1101,10 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro return nil, err } + if err = s.storeSandbox(); err != nil { + return nil, err + } + return c, nil } @@ -1094,6 +1121,11 @@ func (s *Sandbox) StartContainer(containerID string) (VCContainer, error) { if err != nil { return nil, err } + + if err = s.storeSandbox(); err != nil { + return nil, err + } + //Fixme Container delete from sandbox, need to update resources return c, nil @@ -1112,6 +1144,9 @@ func (s *Sandbox) StopContainer(containerID string) (VCContainer, error) { return nil, err } + if err = s.storeSandbox(); err != nil { + return nil, err + } return c, nil } @@ -1124,7 +1159,14 @@ func (s *Sandbox) KillContainer(containerID string, signal syscall.Signal, all b } // Send a signal to the process. - return c.kill(signal, all) + if err := c.kill(signal, all); err != nil { + return err + } + + if err = s.storeSandbox(); err != nil { + return err + } + return nil } // DeleteContainer deletes a container from the sandbox @@ -1158,6 +1200,9 @@ func (s *Sandbox) DeleteContainer(containerID string) (VCContainer, error) { return nil, err } + if err = s.storeSandbox(); err != nil { + return nil, err + } return c, nil } @@ -1236,7 +1281,13 @@ func (s *Sandbox) UpdateContainer(containerID string, resources specs.LinuxResou return err } - return c.storeContainer() + if err := c.storeContainer(); err != nil { + return err + } + if err = s.storeSandbox(); err != nil { + return err + } + return nil } // StatsContainer return the stats of a running container @@ -1263,7 +1314,14 @@ func (s *Sandbox) PauseContainer(containerID string) error { } // Pause the container. - return c.pause() + if err := c.pause(); err != nil { + return err + } + + if err = s.storeSandbox(); err != nil { + return err + } + return nil } // ResumeContainer resumes a paused container. @@ -1275,7 +1333,14 @@ func (s *Sandbox) ResumeContainer(containerID string) error { } // Resume the container. - return c.resume() + if err := c.resume(); err != nil { + return err + } + + if err = s.storeSandbox(); err != nil { + return err + } + return nil } // createContainers registers all containers to the proxy, create the @@ -1306,6 +1371,9 @@ func (s *Sandbox) createContainers() error { if err := s.updateCgroups(); err != nil { return err } + if err := s.storeSandbox(); err != nil { + return err + } return nil } @@ -1327,6 +1395,10 @@ func (s *Sandbox) Start() error { } } + if err := s.storeSandbox(); err != nil { + return err + } + s.Logger().Info("Sandbox is started") return nil @@ -1362,7 +1434,15 @@ func (s *Sandbox) Stop() error { } // Remove the network. - return s.removeNetwork() + if err := s.removeNetwork(); err != nil { + return err + } + + if err := s.storeSandbox(); err != nil { + return err + } + + return nil } // Pause pauses the sandbox @@ -1379,7 +1459,15 @@ func (s *Sandbox) Pause() error { s.monitor.stop() } - return s.pauseSetStates() + if err := s.pauseSetStates(); err != nil { + return err + } + + if err := s.storeSandbox(); err != nil { + return err + } + + return nil } // Resume resumes the sandbox @@ -1388,7 +1476,15 @@ func (s *Sandbox) Resume() error { return err } - return s.resumeSetStates() + if err := s.resumeSetStates(); err != nil { + return err + } + + if err := s.storeSandbox(); err != nil { + return err + } + + return nil } // list lists all sandbox running on the host. diff --git a/virtcontainers/virtcontainers_test.go b/virtcontainers/virtcontainers_test.go index 4faf72f647..424a83ed6d 100644 --- a/virtcontainers/virtcontainers_test.go +++ b/virtcontainers/virtcontainers_test.go @@ -14,6 +14,7 @@ import ( "path/filepath" "testing" + "github.com/kata-containers/runtime/virtcontainers/persist/fs" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/sirupsen/logrus" ) @@ -104,6 +105,7 @@ func TestMain(m *testing.M) { // allow the tests to run without affecting the host system. store.ConfigStoragePath = filepath.Join(testDir, store.StoragePathSuffix, "config") store.RunStoragePath = filepath.Join(testDir, store.StoragePathSuffix, "run") + fs.TestSetRunStoragePath(filepath.Join(testDir, "vc", "sbs")) // set now that configStoragePath has been overridden. sandboxDirConfig = filepath.Join(store.ConfigStoragePath, testSandboxID)