diff --git a/virtcontainers/container.go b/virtcontainers/container.go index e18a9a1a39..7cbecb266e 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -258,8 +258,18 @@ type SystemMountsInfo struct { type ContainerDevice struct { // ID is device id referencing the device from sandbox's device manager ID string + // ContainerPath is device path displayed in container ContainerPath string + + // FileMode permission bits for the device. + FileMode os.FileMode + + // UID is user ID in the container namespace + UID uint32 + + // GID is group ID in the container namespace + GID uint32 } // RootFs describes the container's rootfs. @@ -711,6 +721,9 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err storedDevices = append(storedDevices, ContainerDevice{ ID: dev.DeviceID(), ContainerPath: info.ContainerPath, + FileMode: info.FileMode, + UID: info.UID, + GID: info.GID, }) } c.devices = filterDevices(sandbox, c, storedDevices) diff --git a/virtcontainers/container_test.go b/virtcontainers/container_test.go index e475e93056..333d2e4889 100644 --- a/virtcontainers/container_test.go +++ b/virtcontainers/container_test.go @@ -237,6 +237,10 @@ func TestContainerAddDriveDir(t *testing.T) { assert.Nil(t, err) sandbox.store = sandboxStore + if sandbox.newStore, err = persist.GetDriver("fs"); err != nil || sandbox.newStore == nil { + t.Fatalf("failed to get fs persist driver") + } + contID := "100" container := Container{ sandbox: sandbox, @@ -248,10 +252,6 @@ func TestContainerAddDriveDir(t *testing.T) { assert.Nil(t, err) container.store = containerStore - if sandbox.newStore, err = persist.GetDriver("fs"); err != nil || sandbox.newStore == nil { - t.Fatalf("failed to get fs persist driver") - } - // create state file path := store.ContainerRuntimeRootPath(testSandboxID, container.ID()) stateFilePath := filepath.Join(path, store.StateFile) diff --git a/virtcontainers/device/api/interface.go b/virtcontainers/device/api/interface.go index da15831aef..560a40d1b3 100644 --- a/virtcontainers/device/api/interface.go +++ b/virtcontainers/device/api/interface.go @@ -10,6 +10,7 @@ import ( "github.com/sirupsen/logrus" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" ) var devLogger = logrus.WithField("subsystem", "device") @@ -63,6 +64,9 @@ type Device interface { Reference() uint // Dereference removes one reference to device then returns final ref count Dereference() uint + + // Persist convert and return data in persist format + Dump() persistapi.DeviceState } // DeviceManager can be used to create a new device, this can be used as single diff --git a/virtcontainers/device/drivers/block.go b/virtcontainers/device/drivers/block.go index fdb18f4187..85ad8c63e4 100644 --- a/virtcontainers/device/drivers/block.go +++ b/virtcontainers/device/drivers/block.go @@ -11,6 +11,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -146,5 +147,25 @@ func (device *BlockDevice) GetDeviceInfo() interface{} { return device.BlockDrive } +// Dump convert and return data in persist format +func (device *BlockDevice) Dump() persistapi.DeviceState { + ds := device.GenericDevice.Dump() + ds.Type = string(device.DeviceType()) + + drive := device.BlockDrive + ds.BlockDrive = &persistapi.BlockDrive{ + File: drive.File, + Format: drive.Format, + ID: drive.ID, + Index: drive.Index, + MmioAddr: drive.MmioAddr, + PCIAddr: drive.PCIAddr, + SCSIAddr: drive.SCSIAddr, + NvdimmID: drive.NvdimmID, + VirtPath: drive.VirtPath, + } + return ds +} + // It should implement GetAttachCount() and DeviceID() as api.Device implementation // here it shares function from *GenericDevice so we don't need duplicate codes diff --git a/virtcontainers/device/drivers/generic.go b/virtcontainers/device/drivers/generic.go index ff60fc3fb4..e5b7e30ee3 100644 --- a/virtcontainers/device/drivers/generic.go +++ b/virtcontainers/device/drivers/generic.go @@ -11,6 +11,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" ) // GenericDevice refers to a device that is neither a VFIO device or block device. @@ -115,3 +116,19 @@ func (device *GenericDevice) bumpAttachCount(attach bool) (skip bool, err error) } } } + +// Dump convert and return data in persist format +func (device *GenericDevice) Dump() persistapi.DeviceState { + info := device.DeviceInfo + return persistapi.DeviceState{ + ID: device.ID, + Type: string(device.DeviceType()), + RefCount: device.RefCount, + AttachCount: device.AttachCount, + + DevType: info.DevType, + Major: info.Major, + Minor: info.Minor, + DriverOptions: info.DriverOptions, + } +} diff --git a/virtcontainers/device/drivers/vfio.go b/virtcontainers/device/drivers/vfio.go index f717692258..82573ccb65 100644 --- a/virtcontainers/device/drivers/vfio.go +++ b/virtcontainers/device/drivers/vfio.go @@ -17,6 +17,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -139,6 +140,23 @@ func (device *VFIODevice) GetDeviceInfo() interface{} { return device.VfioDevs } +// Dump convert and return data in persist format +func (device *VFIODevice) Dump() persistapi.DeviceState { + ds := device.GenericDevice.Dump() + ds.Type = string(device.DeviceType()) + + devs := device.VfioDevs + for _, dev := range devs { + ds.VFIODevs = append(ds.VFIODevs, &persistapi.VFIODev{ + ID: dev.ID, + Type: string(dev.Type), + BDF: dev.BDF, + SysfsDev: dev.SysfsDev, + }) + } + return ds +} + // It should implement GetAttachCount() and DeviceID() as api.Device implementation // here it shares function from *GenericDevice so we don't need duplicate codes diff --git a/virtcontainers/device/drivers/vhost_user_blk.go b/virtcontainers/device/drivers/vhost_user_blk.go index 78b2885dc1..588c46590b 100644 --- a/virtcontainers/device/drivers/vhost_user_blk.go +++ b/virtcontainers/device/drivers/vhost_user_blk.go @@ -11,6 +11,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -71,5 +72,18 @@ func (device *VhostUserBlkDevice) GetDeviceInfo() interface{} { return &device.VhostUserDeviceAttrs } +// Dump convert and return data in persist format +func (device *VhostUserBlkDevice) Dump() persistapi.DeviceState { + ds := device.GenericDevice.Dump() + ds.Type = string(device.DeviceType()) + ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{ + DevID: device.DevID, + SocketPath: device.SocketPath, + Type: string(device.Type), + MacAddress: device.MacAddress, + } + return ds +} + // It should implement GetAttachCount() and DeviceID() as api.Device implementation // here it shares function from *GenericDevice so we don't need duplicate codes diff --git a/virtcontainers/device/drivers/vhost_user_net.go b/virtcontainers/device/drivers/vhost_user_net.go index a276bb6fe5..8f5abb32e2 100644 --- a/virtcontainers/device/drivers/vhost_user_net.go +++ b/virtcontainers/device/drivers/vhost_user_net.go @@ -11,6 +11,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -72,5 +73,18 @@ func (device *VhostUserNetDevice) GetDeviceInfo() interface{} { return &device.VhostUserDeviceAttrs } +// Dump convert and return data in persist format +func (device *VhostUserNetDevice) Dump() persistapi.DeviceState { + ds := device.GenericDevice.Dump() + ds.Type = string(device.DeviceType()) + ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{ + DevID: device.DevID, + SocketPath: device.SocketPath, + Type: string(device.Type), + MacAddress: device.MacAddress, + } + return ds +} + // It should implement GetAttachCount() and DeviceID() as api.Device implementation // here it shares function from *GenericDevice so we don't need duplicate codes diff --git a/virtcontainers/device/drivers/vhost_user_scsi.go b/virtcontainers/device/drivers/vhost_user_scsi.go index e4e2a27afa..543949ceb4 100644 --- a/virtcontainers/device/drivers/vhost_user_scsi.go +++ b/virtcontainers/device/drivers/vhost_user_scsi.go @@ -11,6 +11,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -72,5 +73,18 @@ func (device *VhostUserSCSIDevice) GetDeviceInfo() interface{} { return &device.VhostUserDeviceAttrs } +// Dump convert and return data in persist format +func (device *VhostUserSCSIDevice) Dump() persistapi.DeviceState { + ds := device.GenericDevice.Dump() + ds.Type = string(device.DeviceType()) + ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{ + DevID: device.DevID, + SocketPath: device.SocketPath, + Type: string(device.Type), + MacAddress: device.MacAddress, + } + return ds +} + // It should implement GetAttachCount() and DeviceID() as api.Device implementation // here it shares function from *GenericDevice so we don't need duplicate codes diff --git a/virtcontainers/mount_test.go b/virtcontainers/mount_test.go index 2edc87e4a7..71d109483a 100644 --- a/virtcontainers/mount_test.go +++ b/virtcontainers/mount_test.go @@ -225,7 +225,7 @@ func TestGetDeviceForPathBindMount(t *testing.T) { t.Fatal(err) } - defer syscall.Unmount(dest, 0) + defer syscall.Unmount(dest, syscall.MNT_DETACH) destFile := filepath.Join(dest, "test") _, err = os.Create(destFile) diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go index 605f3ab1f4..a47ceea9b2 100644 --- a/virtcontainers/persist.go +++ b/virtcontainers/persist.go @@ -10,6 +10,7 @@ import ( //"github.com/sirupsen/logrus" + "github.com/kata-containers/runtime/virtcontainers/device/api" persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/types" ) @@ -20,14 +21,25 @@ func (s *Sandbox) dumpState(ss *persistapi.SandboxState, cs map[string]persistap ss.State = string(s.state.State) for id, cont := range s.containers { - cs[id] = persistapi.ContainerState{ - State: string(cont.state.State), - Rootfs: persistapi.RootfsState{ - BlockDeviceID: cont.state.BlockDeviceID, - FsType: cont.state.Fstype, - }, + state := persistapi.ContainerState{} + if v, ok := cs[id]; ok { + state = v + } + state.State = string(cont.state.State) + state.Rootfs = persistapi.RootfsState{ + BlockDeviceID: cont.state.BlockDeviceID, + FsType: cont.state.Fstype, + } + cs[id] = state + } + + // delete removed containers + for id := range cs { + if _, ok := s.containers[id]; !ok { + delete(cs, id) } } + return nil } @@ -36,6 +48,46 @@ func (s *Sandbox) dumpHypervisor(ss *persistapi.SandboxState, cs map[string]pers return nil } +func deviceToDeviceState(devices []api.Device) (dss []persistapi.DeviceState) { + for _, dev := range devices { + dss = append(dss, dev.Dump()) + } + return +} + +func (s *Sandbox) dumpDevices(ss *persistapi.SandboxState, cs map[string]persistapi.ContainerState) error { + ss.Devices = deviceToDeviceState(s.devManager.GetAllDevices()) + + for id, cont := range s.containers { + state := persistapi.ContainerState{} + if v, ok := cs[id]; ok { + state = v + } + + state.DeviceMaps = nil + for _, dev := range cont.devices { + state.DeviceMaps = append(state.DeviceMaps, persistapi.DeviceMap{ + ID: dev.ID, + ContainerPath: dev.ContainerPath, + FileMode: dev.FileMode, + UID: dev.UID, + GID: dev.GID, + }) + } + + cs[id] = state + } + + // delete removed containers + for id := range cs { + if _, ok := s.containers[id]; !ok { + delete(cs, id) + } + } + + return nil +} + // PersistVersion set persist data version to current version in runtime func (s *Sandbox) persistVersion() { s.newStore.RegisterHook("version", func(ss *persistapi.SandboxState, cs map[string]persistapi.ContainerState) error { @@ -54,6 +106,11 @@ func (s *Sandbox) persistHvState() { s.newStore.RegisterHook("hypervisor", s.dumpHypervisor) } +// PersistDevices register hook to save device informations +func (s *Sandbox) persistDevices() { + s.newStore.RegisterHook("devices", s.dumpDevices) +} + // Restore will restore data from persist disk on disk func (s *Sandbox) Restore() error { if err := s.newStore.Restore(s.id); err != nil { diff --git a/virtcontainers/persist/api/device.go b/virtcontainers/persist/api/device.go index 2b30d14cf7..6460fcbd7e 100644 --- a/virtcontainers/persist/api/device.go +++ b/virtcontainers/persist/api/device.go @@ -23,6 +23,9 @@ type BlockDrive struct { // Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index Index int + // MmioAddr is used to identify the slot at which the drive is attached (order?). + MmioAddr string + // PCIAddr is the PCI address used to identify the slot at which the drive is attached. PCIAddr string @@ -30,6 +33,9 @@ type BlockDrive struct { // SCSI address is in the format SCSI-Id:LUN SCSIAddr string + // NvdimmID is the nvdimm id inside the VM + NvdimmID string + // VirtPath at which the device appears inside the VM, outside of the container mount namespace VirtPath string } diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index a507f9c318..77540b416a 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -480,10 +480,12 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac // register persist hook for now, data will be written to disk by Dump() s.persistState() s.persistHvState() + s.persistDevices() if err := s.Restore(); err == nil && s.state.State != "" { return s, nil } + // We first try to fetch the sandbox state from storage. // If it exists, this means this is a re-creation, i.e. // we don't need to talk to the guest's agent, but only @@ -495,6 +497,7 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac }*/ // if sandbox doesn't exist, set persist version to current version + // otherwise do nothing s.persistVersion() // Below code path is called only during create, because of earlier check.