From a6b33684699b575ec6a655401e5c847740dd4863 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 13 May 2019 15:26:45 +0800 Subject: [PATCH] persist: merge more files with `persist.json` Fixes #803 Merge more container storage files with `persist.json` including: * devices.json * mounts.json * process.json Signed-off-by: Wei Zhang --- virtcontainers/container.go | 115 +++++++++++++++--------- virtcontainers/kata_agent.go | 9 +- virtcontainers/persist.go | 115 ++++++++++++++++++++++-- virtcontainers/persist/api/container.go | 3 +- virtcontainers/persist/api/sandbox.go | 2 +- 5 files changed, 185 insertions(+), 59 deletions(-) diff --git a/virtcontainers/container.go b/virtcontainers/container.go index cb6e12d40b..4ae6754333 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -372,7 +372,10 @@ func (c *Container) GetPid() int { func (c *Container) SetPid(pid int) error { c.process.Pid = pid - return c.storeProcess() + if !c.sandbox.supportNewStore() { + return c.storeProcess() + } + return nil } func (c *Container) setStateFstype(fstype string) error { @@ -592,8 +595,10 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) ( sharedDirMounts = append(sharedDirMounts, sharedDirMount) } - if err := c.storeMounts(); err != nil { - return nil, nil, err + if !c.sandbox.supportNewStore() { + if err := c.storeMounts(); err != nil { + return nil, nil, err + } } return sharedDirMounts, ignoredMounts, nil @@ -716,20 +721,27 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err // experimental runtime use "persist.json" instead of legacy "state.json" as storage if c.sandbox.supportNewStore() { - if err := c.Restore(); err != nil && - !os.IsNotExist(err) && err != errContainerPersistNotExist { + err := c.Restore() + if err == nil { + //container restored + return c, nil + } + + // Unexpected error + if !os.IsNotExist(err) && err != errContainerPersistNotExist { return nil, err } + // Go to next step for first created container } else { state, err := c.store.LoadContainerState() if err == nil { c.state = state } - } - var process Process - if err := c.store.Load(store.Process, &process); err == nil { - c.process = process + var process Process + if err := c.store.Load(store.Process, &process); err == nil { + c.process = process + } } if err = c.createMounts(); err != nil { @@ -744,45 +756,56 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err } func (c *Container) createMounts() error { - mounts, err := c.loadMounts() - if err == nil { - // restore mounts from disk - c.mounts = mounts - } else { - // Create block devices for newly created container - if err := c.createBlockDevices(); err != nil { - return err + // If sandbox supports "newstore", only newly created container can reach this function, + // so we don't call restore when `supportNewStore` is true + if !c.sandbox.supportNewStore() { + mounts, err := c.loadMounts() + if err == nil { + // restore mounts from disk + c.mounts = mounts + return nil } } + // Create block devices for newly created container + if err := c.createBlockDevices(); err != nil { + return err + } + return nil } func (c *Container) createDevices(contConfig ContainerConfig) error { - // Devices will be found in storage after create stage has completed. - // We load devices from storage at all other stages. - storedDevices, err := c.loadDevices() - if err == nil { - c.devices = storedDevices - } else { - // If devices were not found in storage, create Device implementations - // from the configuration. This should happen at create. - for _, info := range contConfig.DeviceInfos { - dev, err := c.sandbox.devManager.NewDevice(info) - if err != nil { - return err - } - - storedDevices = append(storedDevices, ContainerDevice{ - ID: dev.DeviceID(), - ContainerPath: info.ContainerPath, - FileMode: info.FileMode, - UID: info.UID, - GID: info.GID, - }) + // If sandbox supports "newstore", only newly created container can reach this function, + // so we don't call restore when `supportNewStore` is true + if !c.sandbox.supportNewStore() { + // Devices will be found in storage after create stage has completed. + // We load devices from storage at all other stages. + storedDevices, err := c.loadDevices() + if err == nil { + c.devices = storedDevices + return nil } - c.devices = filterDevices(c, storedDevices) } + + // If devices were not found in storage, create Device implementations + // from the configuration. This should happen at create. + var storedDevices []ContainerDevice + for _, info := range contConfig.DeviceInfos { + dev, err := c.sandbox.devManager.NewDevice(info) + if err != nil { + return err + } + + storedDevices = append(storedDevices, ContainerDevice{ + ID: dev.DeviceID(), + ContainerPath: info.ContainerPath, + FileMode: info.FileMode, + UID: info.UID, + GID: info.GID, + }) + } + c.devices = filterDevices(c, storedDevices) return nil } @@ -838,8 +861,10 @@ func (c *Container) create() (err error) { // inside the VM c.getSystemMountInfo() - if err = c.storeDevices(); err != nil { - return + if !c.sandbox.supportNewStore() { + if err = c.storeDevices(); err != nil { + return + } } process, err := c.sandbox.agent.createContainer(c.sandbox, c) @@ -852,9 +877,11 @@ func (c *Container) create() (err error) { return } - // Store the container process returned by the agent. - if err = c.storeProcess(); err != nil { - return + if !c.sandbox.supportNewStore() { + // Store the container process returned by the agent. + if err = c.storeProcess(); err != nil { + return + } } if err = c.setContainerState(types.StateReady); err != nil { diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 956635dcc2..aae7e1d265 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -1311,9 +1311,12 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage { // Add the block device to the list of container devices, to make sure the // device is detached with detachDevices() for a container. c.devices = append(c.devices, ContainerDevice{ID: id, ContainerPath: m.Destination}) - if err := c.storeDevices(); err != nil { - k.Logger().WithField("device", id).WithError(err).Error("store device failed") - return nil + + if !c.sandbox.supportNewStore() { + if err := c.storeDevices(); err != nil { + k.Logger().WithField("device", id).WithError(err).Error("store device failed") + return nil + } } vol := &grpc.Storage{} diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go index 821240f884..eaa1ac489e 100644 --- a/virtcontainers/persist.go +++ b/virtcontainers/persist.go @@ -99,6 +99,59 @@ func (s *Sandbox) dumpDevices(ss *persistapi.SandboxState, cs map[string]persist } } +func (s *Sandbox) dumpProcess(cs map[string]persistapi.ContainerState) { + for id, cont := range s.containers { + state := persistapi.ContainerState{} + if v, ok := cs[id]; ok { + state = v + } + + state.Process = persistapi.Process{ + Token: cont.process.Token, + Pid: cont.process.Pid, + StartTime: cont.process.StartTime, + } + + cs[id] = state + } + + // delete removed containers + for id := range cs { + if _, ok := s.containers[id]; !ok { + delete(cs, id) + } + } +} + +func (s *Sandbox) dumpMounts(cs map[string]persistapi.ContainerState) { + for id, cont := range s.containers { + state := persistapi.ContainerState{} + if v, ok := cs[id]; ok { + state = v + } + + for _, m := range cont.mounts { + state.Mounts = append(state.Mounts, persistapi.Mount{ + Source: m.Source, + Destination: m.Destination, + Options: m.Options, + HostPath: m.HostPath, + ReadOnly: m.ReadOnly, + BlockDeviceID: m.BlockDeviceID, + }) + } + + cs[id] = state + } + + // delete removed containers + for id := range cs { + if _, ok := s.containers[id]; !ok { + delete(cs, id) + } + } +} + func (s *Sandbox) Save() error { var ( ss = persistapi.SandboxState{} @@ -109,6 +162,8 @@ func (s *Sandbox) Save() error { s.dumpState(&ss, cs) s.dumpHypervisor(&ss, cs) s.dumpDevices(&ss, cs) + s.dumpProcess(cs) + s.dumpMounts(cs) if err := s.newStore.ToDisk(ss, cs); err != nil { return err @@ -126,10 +181,54 @@ func (s *Sandbox) loadState(ss persistapi.SandboxState) { s.state.GuestMemoryHotplugProbe = ss.GuestMemoryHotplugProbe } +func (c *Container) loadContState(cs persistapi.ContainerState) { + c.state = types.ContainerState{ + State: types.StateString(cs.State), + BlockDeviceID: cs.Rootfs.BlockDeviceID, + Fstype: cs.Rootfs.FsType, + CgroupPath: cs.CgroupPath, + } +} + func (s *Sandbox) loadDevices(devStates []persistapi.DeviceState) { s.devManager.LoadDevices(devStates) } +func (c *Container) loadContDevices(cs persistapi.ContainerState) { + c.devices = nil + for _, dev := range cs.DeviceMaps { + c.devices = append(c.devices, ContainerDevice{ + ID: dev.ID, + ContainerPath: dev.ContainerPath, + FileMode: dev.FileMode, + UID: dev.UID, + GID: dev.GID, + }) + } +} + +func (c *Container) loadContMounts(cs persistapi.ContainerState) { + c.mounts = nil + for _, m := range cs.Mounts { + c.mounts = append(c.mounts, Mount{ + Source: m.Source, + Destination: m.Destination, + Options: m.Options, + HostPath: m.HostPath, + ReadOnly: m.ReadOnly, + BlockDeviceID: m.BlockDeviceID, + }) + } +} + +func (c *Container) loadContProcess(cs persistapi.ContainerState) { + c.process = Process{ + Token: cs.Process.Token, + Pid: cs.Process.Pid, + StartTime: cs.Process.StartTime, + } +} + // Restore will restore sandbox data from persist file on disk func (s *Sandbox) Restore() error { ss, _, err := s.newStore.FromDisk(s.id) @@ -144,22 +243,20 @@ func (s *Sandbox) Restore() error { // Restore will restore container data from persist file on disk func (c *Container) Restore() error { - _, cs, err := c.sandbox.newStore.FromDisk(c.sandbox.id) + _, css, err := c.sandbox.newStore.FromDisk(c.sandbox.id) if err != nil { return err } - if _, ok := cs[c.id]; !ok { + cs, ok := css[c.id] + if !ok { return errContainerPersistNotExist } - c.state = types.ContainerState{ - State: types.StateString(cs[c.id].State), - BlockDeviceID: cs[c.id].Rootfs.BlockDeviceID, - Fstype: cs[c.id].Rootfs.FsType, - CgroupPath: cs[c.id].CgroupPath, - } - + c.loadContState(cs) + c.loadContDevices(cs) + c.loadContProcess(cs) + c.loadContMounts(cs) return nil } diff --git a/virtcontainers/persist/api/container.go b/virtcontainers/persist/api/container.go index 6b98323476..0418b81412 100644 --- a/virtcontainers/persist/api/container.go +++ b/virtcontainers/persist/api/container.go @@ -103,8 +103,7 @@ type ContainerState struct { Mounts []Mount // Process on host representing container process - // FIXME: []Process or Process ? - Process []Process + Process Process // BundlePath saves container OCI config.json, which can be unmarshaled // and translated to "CompatOCISpec" diff --git a/virtcontainers/persist/api/sandbox.go b/virtcontainers/persist/api/sandbox.go index da17276106..b78a5a447d 100644 --- a/virtcontainers/persist/api/sandbox.go +++ b/virtcontainers/persist/api/sandbox.go @@ -14,7 +14,7 @@ type SetFunc (func(*SandboxState, map[string]ContainerState) error) // Bridge is a bridge where devices can be hot plugged type Bridge struct { - // Address contains information about devices plugged and its address in the bridge + // DeviceAddr contains information about devices plugged and its address in the bridge DeviceAddr map[uint32]string // Type is the type of the bridge (pci, pcie, etc)