persist: persist device data

Persist device information to relative file

Signed-off-by: Wei Zhang <zhangwei555@huawei.com>

virtcontainers/container.go

@@ -258,8 +258,18 @@ type SystemMountsInfo struct {
 type ContainerDevice struct {
 	// ID is device id referencing the device from sandbox's device manager
 	ID string
+
 	// ContainerPath is device path displayed in container
 	ContainerPath string
+
+	// FileMode permission bits for the device.
+	FileMode os.FileMode
+
+	// UID is user ID in the container namespace
+	UID uint32
+
+	// GID is group ID in the container namespace
+	GID uint32
 }
 
 // RootFs describes the container's rootfs.
@@ -711,6 +721,9 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err
 			storedDevices = append(storedDevices, ContainerDevice{
 				ID:            dev.DeviceID(),
 				ContainerPath: info.ContainerPath,
+				FileMode:      info.FileMode,
+				UID:           info.UID,
+				GID:           info.GID,
 			})
 		}
 		c.devices = filterDevices(sandbox, c, storedDevices)

virtcontainers/container_test.go

@@ -237,6 +237,10 @@ func TestContainerAddDriveDir(t *testing.T) {
 	assert.Nil(t, err)
 	sandbox.store = sandboxStore
 
+	if sandbox.newStore, err = persist.GetDriver("fs"); err != nil || sandbox.newStore == nil {
+		t.Fatalf("failed to get fs persist driver")
+	}
+
 	contID := "100"
 	container := Container{
 		sandbox: sandbox,
@@ -248,10 +252,6 @@ func TestContainerAddDriveDir(t *testing.T) {
 	assert.Nil(t, err)
 	container.store = containerStore
 
-	if sandbox.newStore, err = persist.GetDriver("fs"); err != nil || sandbox.newStore == nil {
-		t.Fatalf("failed to get fs persist driver")
-	}
-
 	// create state file
 	path := store.ContainerRuntimeRootPath(testSandboxID, container.ID())
 	stateFilePath := filepath.Join(path, store.StateFile)

virtcontainers/device/api/interface.go

@@ -10,6 +10,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
 var devLogger = logrus.WithField("subsystem", "device")
@@ -63,6 +64,9 @@ type Device interface {
 	Reference() uint
 	// Dereference removes one reference to device then returns final ref count
 	Dereference() uint
+
+	// Persist convert and return data in persist format
+	Dump() persistapi.DeviceState
 }
 
 // DeviceManager can be used to create a new device, this can be used as single

virtcontainers/device/drivers/block.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -146,5 +147,25 @@ func (device *BlockDevice) GetDeviceInfo() interface{} {
 	return device.BlockDrive
 }
 
+// Dump convert and return data in persist format
+func (device *BlockDevice) Dump() persistapi.DeviceState {
+	ds := device.GenericDevice.Dump()
+	ds.Type = string(device.DeviceType())
+
+	drive := device.BlockDrive
+	ds.BlockDrive = &persistapi.BlockDrive{
+		File:     drive.File,
+		Format:   drive.Format,
+		ID:       drive.ID,
+		Index:    drive.Index,
+		MmioAddr: drive.MmioAddr,
+		PCIAddr:  drive.PCIAddr,
+		SCSIAddr: drive.SCSIAddr,
+		NvdimmID: drive.NvdimmID,
+		VirtPath: drive.VirtPath,
+	}
+	return ds
+}
+
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes

virtcontainers/device/drivers/generic.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
 // GenericDevice refers to a device that is neither a VFIO device or block device.
@@ -115,3 +116,19 @@ func (device *GenericDevice) bumpAttachCount(attach bool) (skip bool, err error)
 		}
 	}
 }
+
+// Dump convert and return data in persist format
+func (device *GenericDevice) Dump() persistapi.DeviceState {
+	info := device.DeviceInfo
+	return persistapi.DeviceState{
+		ID:          device.ID,
+		Type:        string(device.DeviceType()),
+		RefCount:    device.RefCount,
+		AttachCount: device.AttachCount,
+
+		DevType:       info.DevType,
+		Major:         info.Major,
+		Minor:         info.Minor,
+		DriverOptions: info.DriverOptions,
+	}
+}

virtcontainers/device/drivers/vfio.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -139,6 +140,23 @@ func (device *VFIODevice) GetDeviceInfo() interface{} {
 	return device.VfioDevs
 }
 
+// Dump convert and return data in persist format
+func (device *VFIODevice) Dump() persistapi.DeviceState {
+	ds := device.GenericDevice.Dump()
+	ds.Type = string(device.DeviceType())
+
+	devs := device.VfioDevs
+	for _, dev := range devs {
+		ds.VFIODevs = append(ds.VFIODevs, &persistapi.VFIODev{
+			ID:       dev.ID,
+			Type:     string(dev.Type),
+			BDF:      dev.BDF,
+			SysfsDev: dev.SysfsDev,
+		})
+	}
+	return ds
+}
+
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes
 

virtcontainers/device/drivers/vhost_user_blk.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -71,5 +72,18 @@ func (device *VhostUserBlkDevice) GetDeviceInfo() interface{} {
 	return &device.VhostUserDeviceAttrs
 }
 
+// Dump convert and return data in persist format
+func (device *VhostUserBlkDevice) Dump() persistapi.DeviceState {
+	ds := device.GenericDevice.Dump()
+	ds.Type = string(device.DeviceType())
+	ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{
+		DevID:      device.DevID,
+		SocketPath: device.SocketPath,
+		Type:       string(device.Type),
+		MacAddress: device.MacAddress,
+	}
+	return ds
+}
+
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes

virtcontainers/device/drivers/vhost_user_net.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -72,5 +73,18 @@ func (device *VhostUserNetDevice) GetDeviceInfo() interface{} {
 	return &device.VhostUserDeviceAttrs
 }
 
+// Dump convert and return data in persist format
+func (device *VhostUserNetDevice) Dump() persistapi.DeviceState {
+	ds := device.GenericDevice.Dump()
+	ds.Type = string(device.DeviceType())
+	ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{
+		DevID:      device.DevID,
+		SocketPath: device.SocketPath,
+		Type:       string(device.Type),
+		MacAddress: device.MacAddress,
+	}
+	return ds
+}
+
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes

virtcontainers/device/drivers/vhost_user_scsi.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -72,5 +73,18 @@ func (device *VhostUserSCSIDevice) GetDeviceInfo() interface{} {
 	return &device.VhostUserDeviceAttrs
 }
 
+// Dump convert and return data in persist format
+func (device *VhostUserSCSIDevice) Dump() persistapi.DeviceState {
+	ds := device.GenericDevice.Dump()
+	ds.Type = string(device.DeviceType())
+	ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{
+		DevID:      device.DevID,
+		SocketPath: device.SocketPath,
+		Type:       string(device.Type),
+		MacAddress: device.MacAddress,
+	}
+	return ds
+}
+
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes

virtcontainers/mount_test.go

@@ -225,7 +225,7 @@ func TestGetDeviceForPathBindMount(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	defer syscall.Unmount(dest, 0)
+	defer syscall.Unmount(dest, syscall.MNT_DETACH)
 
 	destFile := filepath.Join(dest, "test")
 	_, err = os.Create(destFile)

virtcontainers/persist.go

@@ -10,6 +10,7 @@ import (
 
 	//"github.com/sirupsen/logrus"
 
+	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 )
@@ -20,14 +21,25 @@ func (s *Sandbox) dumpState(ss *persistapi.SandboxState, cs map[string]persistap
 	ss.State = string(s.state.State)
 
 	for id, cont := range s.containers {
-		cs[id] = persistapi.ContainerState{
-			State: string(cont.state.State),
-			Rootfs: persistapi.RootfsState{
-				BlockDeviceID: cont.state.BlockDeviceID,
-				FsType:        cont.state.Fstype,
-			},
+		state := persistapi.ContainerState{}
+		if v, ok := cs[id]; ok {
+			state = v
+		}
+		state.State = string(cont.state.State)
+		state.Rootfs = persistapi.RootfsState{
+			BlockDeviceID: cont.state.BlockDeviceID,
+			FsType:        cont.state.Fstype,
+		}
+		cs[id] = state
+	}
+
+	// delete removed containers
+	for id := range cs {
+		if _, ok := s.containers[id]; !ok {
+			delete(cs, id)
 		}
 	}
+
 	return nil
 }
 
@@ -36,6 +48,46 @@ func (s *Sandbox) dumpHypervisor(ss *persistapi.SandboxState, cs map[string]pers
 	return nil
 }
 
+func deviceToDeviceState(devices []api.Device) (dss []persistapi.DeviceState) {
+	for _, dev := range devices {
+		dss = append(dss, dev.Dump())
+	}
+	return
+}
+
+func (s *Sandbox) dumpDevices(ss *persistapi.SandboxState, cs map[string]persistapi.ContainerState) error {
+	ss.Devices = deviceToDeviceState(s.devManager.GetAllDevices())
+
+	for id, cont := range s.containers {
+		state := persistapi.ContainerState{}
+		if v, ok := cs[id]; ok {
+			state = v
+		}
+
+		state.DeviceMaps = nil
+		for _, dev := range cont.devices {
+			state.DeviceMaps = append(state.DeviceMaps, persistapi.DeviceMap{
+				ID:            dev.ID,
+				ContainerPath: dev.ContainerPath,
+				FileMode:      dev.FileMode,
+				UID:           dev.UID,
+				GID:           dev.GID,
+			})
+		}
+
+		cs[id] = state
+	}
+
+	// delete removed containers
+	for id := range cs {
+		if _, ok := s.containers[id]; !ok {
+			delete(cs, id)
+		}
+	}
+
+	return nil
+}
+
 // PersistVersion set persist data version to current version in runtime
 func (s *Sandbox) persistVersion() {
 	s.newStore.RegisterHook("version", func(ss *persistapi.SandboxState, cs map[string]persistapi.ContainerState) error {
@@ -54,6 +106,11 @@ func (s *Sandbox) persistHvState() {
 	s.newStore.RegisterHook("hypervisor", s.dumpHypervisor)
 }
 
+// PersistDevices register hook to save device informations
+func (s *Sandbox) persistDevices() {
+	s.newStore.RegisterHook("devices", s.dumpDevices)
+}
+
 // Restore will restore data from persist disk on disk
 func (s *Sandbox) Restore() error {
 	if err := s.newStore.Restore(s.id); err != nil {

virtcontainers/persist/api/device.go

@@ -23,6 +23,9 @@ type BlockDrive struct {
 	// Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index
 	Index int
 
+	// MmioAddr is used to identify the slot at which the drive is attached (order?).
+	MmioAddr string
+
 	// PCIAddr is the PCI address used to identify the slot at which the drive is attached.
 	PCIAddr string
 
@@ -30,6 +33,9 @@ type BlockDrive struct {
 	// SCSI address is in the format SCSI-Id:LUN
 	SCSIAddr string
 
+	// NvdimmID is the nvdimm id inside the VM
+	NvdimmID string
+
 	// VirtPath at which the device appears inside the VM, outside of the container mount namespace
 	VirtPath string
 }

virtcontainers/sandbox.go

@@ -480,10 +480,12 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
 	// register persist hook for now, data will be written to disk by Dump()
 	s.persistState()
 	s.persistHvState()
+	s.persistDevices()
 
 	if err := s.Restore(); err == nil && s.state.State != "" {
 		return s, nil
 	}
+
 	// We first try to fetch the sandbox state from storage.
 	// If it exists, this means this is a re-creation, i.e.
 	// we don't need to talk to the guest's agent, but only
@@ -495,6 +497,7 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
 	}*/
 
 	// if sandbox doesn't exist, set persist version to current version
+	// otherwise do nothing
 	s.persistVersion()
 
 	// Below code path is called only during create, because of earlier check.