Merge pull request #1668 from WeiZhang555/use-newstore

persist: merge more files with `persist.json`

virtcontainers/container.go

@@ -372,7 +372,10 @@ func (c *Container) GetPid() int {
 func (c *Container) SetPid(pid int) error {
 	c.process.Pid = pid
 
-	return c.storeProcess()
+	if !c.sandbox.supportNewStore() {
+		return c.storeProcess()
+	}
+	return nil
 }
 
 func (c *Container) setStateFstype(fstype string) error {
@@ -592,8 +595,10 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) (
 		sharedDirMounts = append(sharedDirMounts, sharedDirMount)
 	}
 
-	if err := c.storeMounts(); err != nil {
+	if !c.sandbox.supportNewStore() {
-		return nil, nil, err
+		if err := c.storeMounts(); err != nil {
+			return nil, nil, err
+		}
 	}
 
 	return sharedDirMounts, ignoredMounts, nil
@@ -716,20 +721,27 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err
 
 	// experimental runtime use "persist.json" instead of legacy "state.json" as storage
 	if c.sandbox.supportNewStore() {
-		if err := c.Restore(); err != nil &&
+		err := c.Restore()
-			!os.IsNotExist(err) && err != errContainerPersistNotExist {
+		if err == nil {
+			//container restored
+			return c, nil
+		}
+
+		// Unexpected error
+		if !os.IsNotExist(err) && err != errContainerPersistNotExist {
 			return nil, err
 		}
+		// Go to next step for first created container
 	} else {
 		state, err := c.store.LoadContainerState()
 		if err == nil {
 			c.state = state
 		}
-	}
 
-	var process Process
+		var process Process
-	if err := c.store.Load(store.Process, &process); err == nil {
+		if err := c.store.Load(store.Process, &process); err == nil {
-		c.process = process
+			c.process = process
+		}
 	}
 
 	if err = c.createMounts(); err != nil {
@@ -744,45 +756,56 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err
 }
 
 func (c *Container) createMounts() error {
-	mounts, err := c.loadMounts()
+	// If sandbox supports "newstore", only newly created container can reach this function,
-	if err == nil {
+	// so we don't call restore when `supportNewStore` is true
-		// restore mounts from disk
+	if !c.sandbox.supportNewStore() {
-		c.mounts = mounts
+		mounts, err := c.loadMounts()
-	} else {
+		if err == nil {
-		// Create block devices for newly created container
+			// restore mounts from disk
-		if err := c.createBlockDevices(); err != nil {
+			c.mounts = mounts
-			return err
+			return nil
 		}
 	}
 
+	// Create block devices for newly created container
+	if err := c.createBlockDevices(); err != nil {
+		return err
+	}
+
 	return nil
 }
 
 func (c *Container) createDevices(contConfig ContainerConfig) error {
-	// Devices will be found in storage after create stage has completed.
+	// If sandbox supports "newstore", only newly created container can reach this function,
-	// We load devices from storage at all other stages.
+	// so we don't call restore when `supportNewStore` is true
-	storedDevices, err := c.loadDevices()
+	if !c.sandbox.supportNewStore() {
-	if err == nil {
+		// Devices will be found in storage after create stage has completed.
-		c.devices = storedDevices
+		// We load devices from storage at all other stages.
-	} else {
+		storedDevices, err := c.loadDevices()
-		// If devices were not found in storage, create Device implementations
+		if err == nil {
-		// from the configuration. This should happen at create.
+			c.devices = storedDevices
-		for _, info := range contConfig.DeviceInfos {
+			return nil
-			dev, err := c.sandbox.devManager.NewDevice(info)
-			if err != nil {
-				return err
-			}
-
-			storedDevices = append(storedDevices, ContainerDevice{
-				ID:            dev.DeviceID(),
-				ContainerPath: info.ContainerPath,
-				FileMode:      info.FileMode,
-				UID:           info.UID,
-				GID:           info.GID,
-			})
 		}
-		c.devices = filterDevices(c, storedDevices)
 	}
+
+	// If devices were not found in storage, create Device implementations
+	// from the configuration. This should happen at create.
+	var storedDevices []ContainerDevice
+	for _, info := range contConfig.DeviceInfos {
+		dev, err := c.sandbox.devManager.NewDevice(info)
+		if err != nil {
+			return err
+		}
+
+		storedDevices = append(storedDevices, ContainerDevice{
+			ID:            dev.DeviceID(),
+			ContainerPath: info.ContainerPath,
+			FileMode:      info.FileMode,
+			UID:           info.UID,
+			GID:           info.GID,
+		})
+	}
+	c.devices = filterDevices(c, storedDevices)
 	return nil
 }
 
@@ -838,8 +861,10 @@ func (c *Container) create() (err error) {
 	// inside the VM
 	c.getSystemMountInfo()
 
-	if err = c.storeDevices(); err != nil {
+	if !c.sandbox.supportNewStore() {
-		return
+		if err = c.storeDevices(); err != nil {
+			return
+		}
 	}
 
 	process, err := c.sandbox.agent.createContainer(c.sandbox, c)
@@ -852,9 +877,11 @@ func (c *Container) create() (err error) {
 		return
 	}
 
-	// Store the container process returned by the agent.
+	if !c.sandbox.supportNewStore() {
-	if err = c.storeProcess(); err != nil {
+		// Store the container process returned by the agent.
-		return
+		if err = c.storeProcess(); err != nil {
+			return
+		}
 	}
 
 	if err = c.setContainerState(types.StateReady); err != nil {

virtcontainers/kata_agent.go

@@ -1311,9 +1311,12 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
 		// Add the block device to the list of container devices, to make sure the
 		// device is detached with detachDevices() for a container.
 		c.devices = append(c.devices, ContainerDevice{ID: id, ContainerPath: m.Destination})
-		if err := c.storeDevices(); err != nil {
+
-			k.Logger().WithField("device", id).WithError(err).Error("store device failed")
+		if !c.sandbox.supportNewStore() {
-			return nil
+			if err := c.storeDevices(); err != nil {
+				k.Logger().WithField("device", id).WithError(err).Error("store device failed")
+				return nil
+			}
 		}
 
 		vol := &grpc.Storage{}

virtcontainers/persist.go

@@ -99,6 +99,59 @@ func (s *Sandbox) dumpDevices(ss *persistapi.SandboxState, cs map[string]persist
 	}
 }
 
+func (s *Sandbox) dumpProcess(cs map[string]persistapi.ContainerState) {
+	for id, cont := range s.containers {
+		state := persistapi.ContainerState{}
+		if v, ok := cs[id]; ok {
+			state = v
+		}
+
+		state.Process = persistapi.Process{
+			Token:     cont.process.Token,
+			Pid:       cont.process.Pid,
+			StartTime: cont.process.StartTime,
+		}
+
+		cs[id] = state
+	}
+
+	// delete removed containers
+	for id := range cs {
+		if _, ok := s.containers[id]; !ok {
+			delete(cs, id)
+		}
+	}
+}
+
+func (s *Sandbox) dumpMounts(cs map[string]persistapi.ContainerState) {
+	for id, cont := range s.containers {
+		state := persistapi.ContainerState{}
+		if v, ok := cs[id]; ok {
+			state = v
+		}
+
+		for _, m := range cont.mounts {
+			state.Mounts = append(state.Mounts, persistapi.Mount{
+				Source:        m.Source,
+				Destination:   m.Destination,
+				Options:       m.Options,
+				HostPath:      m.HostPath,
+				ReadOnly:      m.ReadOnly,
+				BlockDeviceID: m.BlockDeviceID,
+			})
+		}
+
+		cs[id] = state
+	}
+
+	// delete removed containers
+	for id := range cs {
+		if _, ok := s.containers[id]; !ok {
+			delete(cs, id)
+		}
+	}
+}
+
 func (s *Sandbox) Save() error {
 	var (
 		ss = persistapi.SandboxState{}
@@ -109,6 +162,8 @@ func (s *Sandbox) Save() error {
 	s.dumpState(&ss, cs)
 	s.dumpHypervisor(&ss, cs)
 	s.dumpDevices(&ss, cs)
+	s.dumpProcess(cs)
+	s.dumpMounts(cs)
 
 	if err := s.newStore.ToDisk(ss, cs); err != nil {
 		return err
@@ -126,10 +181,54 @@ func (s *Sandbox) loadState(ss persistapi.SandboxState) {
 	s.state.GuestMemoryHotplugProbe = ss.GuestMemoryHotplugProbe
 }
 
+func (c *Container) loadContState(cs persistapi.ContainerState) {
+	c.state = types.ContainerState{
+		State:         types.StateString(cs.State),
+		BlockDeviceID: cs.Rootfs.BlockDeviceID,
+		Fstype:        cs.Rootfs.FsType,
+		CgroupPath:    cs.CgroupPath,
+	}
+}
+
 func (s *Sandbox) loadDevices(devStates []persistapi.DeviceState) {
 	s.devManager.LoadDevices(devStates)
 }
 
+func (c *Container) loadContDevices(cs persistapi.ContainerState) {
+	c.devices = nil
+	for _, dev := range cs.DeviceMaps {
+		c.devices = append(c.devices, ContainerDevice{
+			ID:            dev.ID,
+			ContainerPath: dev.ContainerPath,
+			FileMode:      dev.FileMode,
+			UID:           dev.UID,
+			GID:           dev.GID,
+		})
+	}
+}
+
+func (c *Container) loadContMounts(cs persistapi.ContainerState) {
+	c.mounts = nil
+	for _, m := range cs.Mounts {
+		c.mounts = append(c.mounts, Mount{
+			Source:        m.Source,
+			Destination:   m.Destination,
+			Options:       m.Options,
+			HostPath:      m.HostPath,
+			ReadOnly:      m.ReadOnly,
+			BlockDeviceID: m.BlockDeviceID,
+		})
+	}
+}
+
+func (c *Container) loadContProcess(cs persistapi.ContainerState) {
+	c.process = Process{
+		Token:     cs.Process.Token,
+		Pid:       cs.Process.Pid,
+		StartTime: cs.Process.StartTime,
+	}
+}
+
 // Restore will restore sandbox data from persist file on disk
 func (s *Sandbox) Restore() error {
 	ss, _, err := s.newStore.FromDisk(s.id)
@@ -144,22 +243,20 @@ func (s *Sandbox) Restore() error {
 
 // Restore will restore container data from persist file on disk
 func (c *Container) Restore() error {
-	_, cs, err := c.sandbox.newStore.FromDisk(c.sandbox.id)
+	_, css, err := c.sandbox.newStore.FromDisk(c.sandbox.id)
 	if err != nil {
 		return err
 	}
 
-	if _, ok := cs[c.id]; !ok {
+	cs, ok := css[c.id]
+	if !ok {
 		return errContainerPersistNotExist
 	}
 
-	c.state = types.ContainerState{
+	c.loadContState(cs)
-		State:         types.StateString(cs[c.id].State),
+	c.loadContDevices(cs)
-		BlockDeviceID: cs[c.id].Rootfs.BlockDeviceID,
+	c.loadContProcess(cs)
-		Fstype:        cs[c.id].Rootfs.FsType,
+	c.loadContMounts(cs)
-		CgroupPath:    cs[c.id].CgroupPath,
-	}
-
 	return nil
 }
 

virtcontainers/persist/api/container.go

@@ -103,8 +103,7 @@ type ContainerState struct {
 	Mounts []Mount
 
 	// Process on host representing container process
-	// FIXME: []Process or Process ?
+	Process Process
-	Process []Process
 
 	// BundlePath saves container OCI config.json, which can be unmarshaled
 	// and translated to "CompatOCISpec"

virtcontainers/persist/api/sandbox.go

@@ -14,7 +14,7 @@ type SetFunc (func(*SandboxState, map[string]ContainerState) error)
 
 // Bridge is a bridge where devices can be hot plugged
 type Bridge struct {
-	// Address contains information about devices plugged and its address in the bridge
+	// DeviceAddr contains information about devices plugged and its address in the bridge
 	DeviceAddr map[uint32]string
 
 	// Type is the type of the bridge (pci, pcie, etc)