devices: use device manager to manage all devices

Fixes #50

Previously the devices are created with device manager and laterly
attached to hypervisor with "device.Attach()", this could work, but
there's no way to remember the reference count for every device, which
means if we plug one device to hypervisor twice, it's truly inserted
twice, but actually we only need to insert once but use it in many
places.

Use device manager as a consolidated entrypoint of device management can
give us a way to handle many "references" to single device, because it
can save all devices and remember it's use count.

Signed-off-by: Wei Zhang <zhangwei555@huawei.com>

virtcontainers/container.go

@@ -440,8 +440,7 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) (
 		if c.checkBlockDeviceSupport() && stat.Mode&unix.S_IFBLK == unix.S_IFBLK {
 			// TODO: remove dependency of package drivers
 			b := &drivers.BlockDevice{
-				DevType: config.DeviceBlock,
+				DeviceInfo: &config.DeviceInfo{
-				DeviceInfo: config.DeviceInfo{
 					HostPath:      m.Source,
 					ContainerPath: m.Destination,
 					DevType:       "b",
@@ -574,12 +573,13 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err
 	} else {
 		// If devices were not found in storage, create Device implementations
 		// from the configuration. This should happen at create.
-
+		for _, info := range contConfig.DeviceInfos {
-		devices, err := sandbox.devManager.NewDevices(contConfig.DeviceInfos)
+			dev, err := sandbox.devManager.NewDevice(info)
-		if err != nil {
+			if err != nil {
-			return &Container{}, err
+				return &Container{}, err
+			}
+			c.devices = append(c.devices, dev)
 		}
-		c.devices = devices
 	}
 	return c, nil
 }
@@ -1022,7 +1022,7 @@ func (c *Container) hotplugDrive() error {
 
 	// Add drive with id as container id
 	devID := utils.MakeNameID("drive", c.id, maxDevIDSize)
-	drive := drivers.Drive{
+	drive := config.BlockDrive{
 		File:   devicePath,
 		Format: "raw",
 		ID:     devID,
@@ -1059,7 +1059,7 @@ func (c *Container) removeDrive() (err error) {
 		c.Logger().Info("unplugging block device")
 
 		devID := utils.MakeNameID("drive", c.id, maxDevIDSize)
-		drive := &drivers.Drive{
+		drive := &config.BlockDrive{
 			ID: devID,
 		}
 

virtcontainers/device/api/interface.go

@@ -27,6 +27,7 @@ func DeviceLogger() *logrus.Entry {
 // DeviceReceiver is an interface used for accepting devices
 // a device should be attached/added/plugged to a DeviceReceiver
 type DeviceReceiver interface {
+	// these are for hotplug/hot-unplug devices to/from hypervisor
 	HotplugAddDevice(Device, config.DeviceType) error
 	HotplugRemoveDevice(Device, config.DeviceType) error
 
@@ -51,11 +52,29 @@ type VhostUserDevice interface {
 type Device interface {
 	Attach(DeviceReceiver) error
 	Detach(DeviceReceiver) error
+	// ID returns device identifier
+	DeviceID() string
+	// DeviceType indicates which kind of device it is
+	// e.g. block, vfio or vhost user
 	DeviceType() config.DeviceType
+	// GetDeviceInfo returns device information that the device is created based on
+	GetDeviceInfo() *config.DeviceInfo
+	// GetDeviceDrive returns device specific data used for hotplugging by hypervisor
+	// Caller could cast the return value to device specific struct
+	// e.g. Block device returns *config.BlockDrive and
+	// vfio device returns *config.VFIODrive
+	GetDeviceDrive() interface{}
+	// IsAttached checks if the device is attached
+	IsAttached() bool
 }
 
 // DeviceManager can be used to create a new device, this can be used as single
 // device management object.
 type DeviceManager interface {
-	NewDevices(devInfos []config.DeviceInfo) ([]Device, error)
+	NewDevice(config.DeviceInfo) (Device, error)
+	AttachDevice(string, DeviceReceiver) error
+	DetachDevice(string, DeviceReceiver) error
+	IsDeviceAttached(string) bool
+	GetDeviceByID(string) Device
+	GetAllDevices() []Device
 }

virtcontainers/device/config/config.go

@@ -49,10 +49,10 @@ var SysIOMMUPath = "/sys/kernel/iommu_groups"
 
 // DeviceInfo is an embedded type that contains device data common to all types of devices.
 type DeviceInfo struct {
-	// Device path on host
+	// Hostpath is device path on host
 	HostPath string
 
-	// Device path inside the container
+	// ContainerPath is device path inside container
 	ContainerPath string
 
 	// Type of device: c, b, u or p
@@ -87,6 +87,40 @@ type DeviceInfo struct {
 	DriverOptions map[string]string
 }
 
+// BlockDrive represents a block storage drive which may be used in case the storage
+// driver has an underlying block storage device.
+type BlockDrive struct {
+	// File is the path to the disk-image/device which will be used with this drive
+	File string
+
+	// Format of the drive
+	Format string
+
+	// ID is used to identify this drive in the hypervisor options.
+	ID string
+
+	// Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index
+	Index int
+
+	// PCIAddr is the PCI address used to identify the slot at which the drive is attached.
+	PCIAddr string
+
+	// SCSI Address of the block device, in case the device is attached using SCSI driver
+	// SCSI address is in the format SCSI-Id:LUN
+	SCSIAddr string
+
+	// VirtPath at which the device appears inside the VM, outside of the container mount namespace
+	VirtPath string
+}
+
+// VFIODrive represents a VFIO drive used for hotplugging
+type VFIODrive struct {
+	// ID is used to identify this drive in the hypervisor options.
+	ID string
+	// BDF (Bus:Device.Function) of the PCI address
+	BDF string
+}
+
 // VhostUserDeviceAttrs represents data shared by most vhost-user devices
 type VhostUserDeviceAttrs struct {
 	DevType    DeviceType

virtcontainers/device/drivers/block.go

@@ -7,7 +7,6 @@
 package drivers
 
 import (
-	"encoding/hex"
 	"path/filepath"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
@@ -17,48 +16,17 @@ import (
 
 const maxDevIDSize = 31
 
-// Drive represents a block storage drive which may be used in case the storage
-// driver has an underlying block storage device.
-type Drive struct {
-
-	// Path to the disk-image/device which will be used with this drive
-	File string
-
-	// Format of the drive
-	Format string
-
-	// ID is used to identify this drive in the hypervisor options.
-	ID string
-
-	// Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index
-	Index int
-
-	// PCIAddr is the PCI address used to identify the slot at which the drive is attached.
-	PCIAddr string
-}
-
 // BlockDevice refers to a block storage device implementation.
 type BlockDevice struct {
-	DevType    config.DeviceType
+	ID         string
-	DeviceInfo config.DeviceInfo
+	DeviceInfo *config.DeviceInfo
-
+	BlockDrive *config.BlockDrive
-	// SCSI Address of the block device, in case the device is attached using SCSI driver
-	// SCSI address is in the format SCSI-Id:LUN
-	SCSIAddr string
-
-	// Path at which the device appears inside the VM, outside of the container mount namespace
-	VirtPath string
-
-	// PCI Slot of the block device
-	PCIAddr string
-
-	BlockDrive *Drive
 }
 
 // NewBlockDevice creates a new block device based on DeviceInfo
-func NewBlockDevice(devInfo config.DeviceInfo) *BlockDevice {
+func NewBlockDevice(devInfo *config.DeviceInfo) *BlockDevice {
 	return &BlockDevice{
-		DevType:    config.DeviceBlock,
+		ID:         devInfo.ID,
 		DeviceInfo: devInfo,
 	}
 }
@@ -66,13 +34,6 @@ func NewBlockDevice(devInfo config.DeviceInfo) *BlockDevice {
 // Attach is standard interface of api.Device, it's used to add device to some
 // DeviceReceiver
 func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
-	randBytes, err := utils.GenerateRandomBytes(8)
-	if err != nil {
-		return err
-	}
-
-	device.DeviceInfo.ID = hex.EncodeToString(randBytes)
-
 	// Increment the block index for the sandbox. This is used to determine the name
 	// for the block device in the case where the block device is used as container
 	// rootfs and the predicted block device name needs to be provided to the agent.
@@ -88,21 +49,13 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
 		return err
 	}
 
-	drive := Drive{
+	drive := &config.BlockDrive{
 		File:   device.DeviceInfo.HostPath,
 		Format: "raw",
 		ID:     utils.MakeNameID("drive", device.DeviceInfo.ID, maxDevIDSize),
 		Index:  index,
 	}
 
-	deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device")
-	device.BlockDrive = &drive
-	if err = devReceiver.HotplugAddDevice(device, config.DeviceBlock); err != nil {
-		return err
-	}
-
-	device.DeviceInfo.Hotplugged = true
-
 	driveName, err := utils.GetVirtDriveName(index)
 	if err != nil {
 		return err
@@ -110,36 +63,61 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
 
 	customOptions := device.DeviceInfo.DriverOptions
 	if customOptions != nil && customOptions["block-driver"] == "virtio-blk" {
-		device.VirtPath = filepath.Join("/dev", driveName)
+		drive.VirtPath = filepath.Join("/dev", driveName)
-		device.PCIAddr = drive.PCIAddr
 	} else {
 		scsiAddr, err := utils.GetSCSIAddress(index)
 		if err != nil {
 			return err
 		}
 
-		device.SCSIAddr = scsiAddr
+		drive.SCSIAddr = scsiAddr
 	}
 
+	deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device")
+	device.BlockDrive = drive
+	if err = devReceiver.HotplugAddDevice(device, config.DeviceBlock); err != nil {
+		return err
+	}
+
+	device.DeviceInfo.Hotplugged = true
+
 	return nil
 }
 
 // Detach is standard interface of api.Device, it's used to remove device from some
 // DeviceReceiver
 func (device *BlockDevice) Detach(devReceiver api.DeviceReceiver) error {
-	if device.DeviceInfo.Hotplugged {
+	deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device")
-		deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device")
-
-		if err := devReceiver.HotplugRemoveDevice(device, config.DeviceBlock); err != nil {
-			deviceLogger().WithError(err).Error("Failed to unplug block device")
-			return err
-		}
 
+	if err := devReceiver.HotplugRemoveDevice(device, config.DeviceBlock); err != nil {
+		deviceLogger().WithError(err).Error("Failed to unplug block device")
+		return err
 	}
+	device.DeviceInfo.Hotplugged = false
 	return nil
 }
 
+// IsAttached checks if the device is attached
+func (device *BlockDevice) IsAttached() bool {
+	return device.DeviceInfo.Hotplugged
+}
+
 // DeviceType is standard interface of api.Device, it returns device type
 func (device *BlockDevice) DeviceType() config.DeviceType {
-	return device.DevType
+	return config.DeviceBlock
+}
+
+// DeviceID returns device ID
+func (device *BlockDevice) DeviceID() string {
+	return device.ID
+}
+
+// GetDeviceInfo returns device information that the device is created based on
+func (device *BlockDevice) GetDeviceInfo() *config.DeviceInfo {
+	return device.DeviceInfo
+}
+
+// GetDeviceDrive returns device information used for creating
+func (device *BlockDevice) GetDeviceDrive() interface{} {
+	return device.BlockDrive
 }

virtcontainers/device/drivers/generic.go

@@ -13,14 +13,14 @@ import (
 
 // GenericDevice refers to a device that is neither a VFIO device or block device.
 type GenericDevice struct {
-	DevType    config.DeviceType
+	ID         string
-	DeviceInfo config.DeviceInfo
+	DeviceInfo *config.DeviceInfo
 }
 
 // NewGenericDevice creates a new GenericDevice
-func NewGenericDevice(devInfo config.DeviceInfo) *GenericDevice {
+func NewGenericDevice(devInfo *config.DeviceInfo) *GenericDevice {
 	return &GenericDevice{
-		DevType:    config.DeviceGeneric,
+		ID:         devInfo.ID,
 		DeviceInfo: devInfo,
 	}
 }
@@ -35,7 +35,27 @@ func (device *GenericDevice) Detach(devReceiver api.DeviceReceiver) error {
 	return nil
 }
 
+// IsAttached checks if the device is attached
+func (device *GenericDevice) IsAttached() bool {
+	return device.DeviceInfo.Hotplugged
+}
+
+// DeviceID returns device ID
+func (device *GenericDevice) DeviceID() string {
+	return device.ID
+}
+
 // DeviceType is standard interface of api.Device, it returns device type
 func (device *GenericDevice) DeviceType() config.DeviceType {
-	return device.DevType
+	return config.DeviceGeneric
+}
+
+// GetDeviceInfo returns device information that the device is created based on
+func (device *GenericDevice) GetDeviceInfo() *config.DeviceInfo {
+	return device.DeviceInfo
+}
+
+// GetDeviceDrive returns device information used for creating
+func (device *GenericDevice) GetDeviceDrive() interface{} {
+	return device.DeviceInfo
 }

virtcontainers/device/drivers/vfio.go

@@ -7,7 +7,6 @@
 package drivers
 
 import (
-	"encoding/hex"
 	"fmt"
 	"io/ioutil"
 	"path/filepath"
@@ -31,15 +30,15 @@ const (
 // VFIODevice is a vfio device meant to be passed to the hypervisor
 // to be used by the Virtual Machine.
 type VFIODevice struct {
-	DevType    config.DeviceType
+	ID         string
-	DeviceInfo config.DeviceInfo
+	DeviceInfo *config.DeviceInfo
-	BDF        string
+	vfioDrives []*config.VFIODrive
 }
 
 // NewVFIODevice create a new VFIO device
-func NewVFIODevice(devInfo config.DeviceInfo) *VFIODevice {
+func NewVFIODevice(devInfo *config.DeviceInfo) *VFIODevice {
 	return &VFIODevice{
-		DevType:    config.DeviceVFIO,
+		ID:         devInfo.ID,
 		DeviceInfo: devInfo,
 	}
 }
@@ -57,44 +56,62 @@ func (device *VFIODevice) Attach(devReceiver api.DeviceReceiver) error {
 
 	// Pass all devices in iommu group
 	for _, deviceFile := range deviceFiles {
-
 		//Get bdf of device eg 0000:00:1c.0
 		deviceBDF, err := getBDF(deviceFile.Name())
 		if err != nil {
 			return err
 		}
-
+		vfio := &config.VFIODrive{
-		device.BDF = deviceBDF
+			ID:  utils.MakeNameID("vfio", device.DeviceInfo.ID, maxDevIDSize),
-
+			BDF: deviceBDF,
-		randBytes, err := utils.GenerateRandomBytes(8)
-		if err != nil {
-			return err
 		}
-		device.DeviceInfo.ID = hex.EncodeToString(randBytes)
+		device.vfioDrives = append(device.vfioDrives, vfio)
-
-		if err := devReceiver.HotplugAddDevice(device, config.DeviceVFIO); err != nil {
-			deviceLogger().WithError(err).Error("Failed to add device")
-			return err
-		}
-
-		deviceLogger().WithFields(logrus.Fields{
-			"device-group": device.DeviceInfo.HostPath,
-			"device-type":  "vfio-passthrough",
-		}).Info("Device group attached")
 	}
 
+	// hotplug a VFIO device is actually hotplugging a group of iommu devices
+	if err := devReceiver.HotplugAddDevice(device, config.DeviceVFIO); err != nil {
+		deviceLogger().WithError(err).Error("Failed to add device")
+		return err
+	}
+
+	deviceLogger().WithFields(logrus.Fields{
+		"device-group": device.DeviceInfo.HostPath,
+		"device-type":  "vfio-passthrough",
+	}).Info("Device group attached")
+	device.DeviceInfo.Hotplugged = true
 	return nil
 }
 
 // Detach is standard interface of api.Device, it's used to remove device from some
 // DeviceReceiver
 func (device *VFIODevice) Detach(devReceiver api.DeviceReceiver) error {
+	device.DeviceInfo.Hotplugged = false
 	return nil
 }
 
+// IsAttached checks if the device is attached
+func (device *VFIODevice) IsAttached() bool {
+	return device.DeviceInfo.Hotplugged
+}
+
 // DeviceType is standard interface of api.Device, it returns device type
 func (device *VFIODevice) DeviceType() config.DeviceType {
-	return device.DevType
+	return config.DeviceVFIO
+}
+
+// DeviceID returns device ID
+func (device *VFIODevice) DeviceID() string {
+	return device.ID
+}
+
+// GetDeviceInfo returns device information that the device is created based on
+func (device *VFIODevice) GetDeviceInfo() *config.DeviceInfo {
+	return device.DeviceInfo
+}
+
+// GetDeviceDrive returns device information used for creating
+func (device *VFIODevice) GetDeviceDrive() interface{} {
+	return device.vfioDrives
 }
 
 // getBDF returns the BDF of pci device

virtcontainers/device/manager/manager.go

@@ -7,11 +7,16 @@
 package manager
 
 import (
+	"encoding/hex"
+	"errors"
+	"sync"
+
 	"github.com/sirupsen/logrus"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/device/drivers"
+	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
 const (
@@ -21,59 +26,162 @@ const (
 	VirtioSCSI string = "virtio-scsi"
 )
 
+var (
+	// ErrIDExhausted represents that devices are too many
+	// and no more IDs can be generated
+	ErrIDExhausted = errors.New("IDs are exhausted")
+	// ErrDeviceNotExist represents device hasn't been created before
+	ErrDeviceNotExist = errors.New("device with specified ID hasn't been created")
+	// ErrDeviceAttached represents the device is already attached
+	ErrDeviceAttached = errors.New("device is already attached")
+	// ErrDeviceNotAttached represents the device isn't attached
+	ErrDeviceNotAttached = errors.New("device isn't attached")
+)
+
 type deviceManager struct {
 	blockDriver string
+
+	devices map[string]api.Device
+	sync.RWMutex
 }
 
 func deviceLogger() *logrus.Entry {
 	return api.DeviceLogger().WithField("subsystem", "device")
 }
 
+// NewDeviceManager creates a deviceManager object behaved as api.DeviceManager
+func NewDeviceManager(blockDriver string, devices []api.Device) api.DeviceManager {
+	dm := &deviceManager{
+		devices: make(map[string]api.Device),
+	}
+	if blockDriver == VirtioBlock {
+		dm.blockDriver = VirtioBlock
+	} else {
+		dm.blockDriver = VirtioSCSI
+	}
+
+	for _, dev := range devices {
+		dm.devices[dev.DeviceID()] = dev
+	}
+	return dm
+}
+
 // createDevice creates one device based on DeviceInfo
 func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (api.Device, error) {
 	path, err := config.GetHostPathFunc(devInfo)
 	if err != nil {
 		return nil, err
 	}
-
 	devInfo.HostPath = path
+
+	// device ID must be generated by manager instead of device itself
+	// in case of ID collision
+	if devInfo.ID, err = dm.newDeviceID(); err != nil {
+		return nil, err
+	}
 	if isVFIO(path) {
-		return drivers.NewVFIODevice(devInfo), nil
+		return drivers.NewVFIODevice(&devInfo), nil
 	} else if isBlock(devInfo) {
 		if devInfo.DriverOptions == nil {
 			devInfo.DriverOptions = make(map[string]string)
 		}
 		devInfo.DriverOptions["block-driver"] = dm.blockDriver
-		return drivers.NewBlockDevice(devInfo), nil
+		return drivers.NewBlockDevice(&devInfo), nil
 	} else {
 		deviceLogger().WithField("device", path).Info("Device has not been passed to the container")
-		return drivers.NewGenericDevice(devInfo), nil
+		return drivers.NewGenericDevice(&devInfo), nil
 	}
 }
 
-// NewDevices creates bundles of devices based on array of DeviceInfo
+// NewDevice creates bundles of devices based on array of DeviceInfo
-func (dm *deviceManager) NewDevices(devInfos []config.DeviceInfo) ([]api.Device, error) {
+func (dm *deviceManager) NewDevice(devInfo config.DeviceInfo) (api.Device, error) {
-	var devices []api.Device
+	dm.Lock()
+	defer dm.Unlock()
+	dev, err := dm.createDevice(devInfo)
+	if err == nil {
+		dm.devices[dev.DeviceID()] = dev
+	}
+	return dev, err
+}
 
-	for _, devInfo := range devInfos {
+func (dm *deviceManager) newDeviceID() (string, error) {
-		device, err := dm.createDevice(devInfo)
+	for i := 0; i < 5; i++ {
+		// generate an random ID
+		randBytes, err := utils.GenerateRandomBytes(8)
 		if err != nil {
-			return nil, err
+			return "", err
 		}
-		devices = append(devices, device)
+		id := hex.EncodeToString(randBytes)
-	}
 
-	return devices, nil
+		// check ID collision, choose another one if ID is in use
+		if _, ok := dm.devices[id]; !ok {
+			return id, nil
+		}
+	}
+	return "", ErrIDExhausted
 }
 
-// NewDeviceManager creates a deviceManager object behaved as api.DeviceManager
+func (dm *deviceManager) AttachDevice(id string, dr api.DeviceReceiver) error {
-func NewDeviceManager(blockDriver string) api.DeviceManager {
+	dm.Lock()
-	dm := &deviceManager{}
+	defer dm.Unlock()
-	if blockDriver == VirtioBlock {
+
-		dm.blockDriver = VirtioBlock
+	d, ok := dm.devices[id]
-	} else {
+	if !ok {
-		dm.blockDriver = VirtioSCSI
+		return ErrDeviceNotExist
 	}
 
-	return dm
+	if d.IsAttached() {
+		return ErrDeviceAttached
+	}
+
+	if err := d.Attach(dr); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (dm *deviceManager) DetachDevice(id string, dr api.DeviceReceiver) error {
+	dm.Lock()
+	defer dm.Unlock()
+
+	d, ok := dm.devices[id]
+	if !ok {
+		return ErrDeviceNotExist
+	}
+	if !d.IsAttached() {
+		return ErrDeviceNotAttached
+	}
+
+	if err := d.Detach(dr); err != nil {
+		return err
+	}
+	return nil
+}
+func (dm *deviceManager) GetDeviceByID(id string) api.Device {
+	dm.RLock()
+	defer dm.RUnlock()
+	if d, ok := dm.devices[id]; ok {
+		return d
+	}
+	return nil
+}
+
+func (dm *deviceManager) GetAllDevices() []api.Device {
+	dm.RLock()
+	defer dm.RUnlock()
+	devices := []api.Device{}
+	for _, v := range dm.devices {
+		devices = append(devices, v)
+	}
+	return devices
+}
+
+func (dm *deviceManager) IsDeviceAttached(id string) bool {
+	dm.RLock()
+	defer dm.RUnlock()
+	d, ok := dm.devices[id]
+	if !ok {
+		return false
+	}
+	return d.IsAttached()
 }

virtcontainers/hyperstart_agent.go

@@ -14,15 +14,16 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/sirupsen/logrus"
+	"github.com/vishvananda/netlink"
+
 	proxyClient "github.com/clearcontainers/proxy/client"
+	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/device/drivers"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/hyperstart"
 	ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
-
-	"github.com/sirupsen/logrus"
-	"github.com/vishvananda/netlink"
 )
 
 var defaultSockPathTemplates = []string{"%s/%s/hyper.sock", "%s/%s/tty.sock"}
@@ -231,6 +232,32 @@ func fsMapFromMounts(mounts []Mount) []*hyperstart.FsmapDescriptor {
 	return fsmap
 }
 
+func fsMapFromDevices(c *Container) ([]*hyperstart.FsmapDescriptor, error) {
+	var fsmap []*hyperstart.FsmapDescriptor
+	for _, dev := range c.devices {
+		device := c.sandbox.devManager.GetDeviceByID(dev.DeviceID())
+		if device == nil {
+			return nil, fmt.Errorf("can't find device: %#v", dev)
+		}
+		blockDev := device.(*drivers.BlockDevice)
+
+		d, ok := blockDev.GetDeviceDrive().(*config.BlockDrive)
+		if !ok || d == nil {
+			return nil, fmt.Errorf("can't retrieve block device information")
+		}
+
+		fsmapDesc := &hyperstart.FsmapDescriptor{
+			Source:       d.VirtPath,
+			Path:         blockDev.DeviceInfo.ContainerPath,
+			AbsolutePath: true,
+			DockerVolume: false,
+			SCSIAddr:     d.SCSIAddr,
+		}
+		fsmap = append(fsmap, fsmapDesc)
+	}
+	return fsmap, nil
+}
+
 // init is the agent initialization implementation for hyperstart.
 func (h *hyper) init(sandbox *Sandbox, config interface{}) (err error) {
 	switch c := config.(type) {
@@ -497,20 +524,11 @@ func (h *hyper) startOneContainer(sandbox *Sandbox, c *Container) error {
 	h.handleBlockVolumes(c)
 
 	// Append container mounts for block devices passed with --device.
-	for _, device := range c.devices {
+	fsmapDev, err := fsMapFromDevices(c)
-		d, ok := device.(*drivers.BlockDevice)
+	if err != nil {
-
+		return err
-		if ok {
-			fsmapDesc := &hyperstart.FsmapDescriptor{
-				Source:       d.VirtPath,
-				Path:         d.DeviceInfo.ContainerPath,
-				AbsolutePath: true,
-				DockerVolume: false,
-				SCSIAddr:     d.SCSIAddr,
-			}
-			fsmap = append(fsmap, fsmapDesc)
-		}
 	}
+	fsmap = append(fsmap, fsmapDev...)
 
 	// Assign fsmap for hyperstart to mount these at the correct location within the container
 	container.Fsmap = fsmap

virtcontainers/kata_agent.go

@@ -733,12 +733,12 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, devices []api.Devic
 			ContainerPath: d.DeviceInfo.ContainerPath,
 		}
 
-		if d.SCSIAddr == "" {
+		if d.BlockDrive.SCSIAddr == "" {
 			kataDevice.Type = kataBlkDevType
-			kataDevice.Id = d.PCIAddr
+			kataDevice.Id = d.BlockDrive.PCIAddr
 		} else {
 			kataDevice.Type = kataSCSIDevType
-			kataDevice.Id = d.SCSIAddr
+			kataDevice.Id = d.BlockDrive.SCSIAddr
 		}
 
 		deviceList = append(deviceList, kataDevice)
@@ -970,10 +970,10 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
 
 		if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
 			vol.Driver = kataBlkDevType
-			vol.Source = b.VirtPath
+			vol.Source = b.BlockDrive.VirtPath
 		} else {
 			vol.Driver = kataSCSIDevType
-			vol.Source = b.SCSIAddr
+			vol.Source = b.BlockDrive.SCSIAddr
 		}
 
 		vol.MountPoint = b.DeviceInfo.ContainerPath

virtcontainers/network.go

@@ -25,6 +25,7 @@ import (
 	"github.com/vishvananda/netns"
 	"golang.org/x/sys/unix"
 
+	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/device/drivers"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
@@ -343,7 +344,8 @@ func (endpoint *PhysicalEndpoint) Attach(h hypervisor) error {
 		return err
 	}
 
-	d := drivers.VFIODevice{
+	// TODO: use device manager as general device management entrance
+	d := config.VFIODrive{
 		BDF: endpoint.BDF,
 	}
 

virtcontainers/qemu.go

@@ -20,7 +20,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
-	deviceDrivers "github.com/kata-containers/runtime/virtcontainers/device/drivers"
+	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -656,7 +656,7 @@ func (q *qemu) removeDeviceFromBridge(ID string) error {
 	return err
 }
 
-func (q *qemu) hotplugBlockDevice(drive *deviceDrivers.Drive, op operation) error {
+func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error {
 	err := q.qmpSetup()
 	if err != nil {
 		return err
@@ -717,13 +717,13 @@ func (q *qemu) hotplugBlockDevice(drive *deviceDrivers.Drive, op operation) erro
 	return nil
 }
 
-func (q *qemu) hotplugVFIODevice(device deviceDrivers.VFIODevice, op operation) error {
+func (q *qemu) hotplugVFIODevice(device *config.VFIODrive, op operation) error {
 	err := q.qmpSetup()
 	if err != nil {
 		return err
 	}
 
-	devID := "vfio-" + device.DeviceInfo.ID
+	devID := device.ID
 
 	if op == addDevice {
 		addr, bridge, err := q.addDeviceToBridge(devID)
@@ -750,15 +750,13 @@ func (q *qemu) hotplugVFIODevice(device deviceDrivers.VFIODevice, op operation)
 func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operation) (interface{}, error) {
 	switch devType {
 	case blockDev:
-		// TODO: find a way to remove dependency of deviceDrivers lib @weizhang555
+		drive := devInfo.(*config.BlockDrive)
-		drive := devInfo.(*deviceDrivers.Drive)
 		return nil, q.hotplugBlockDevice(drive, op)
 	case cpuDev:
 		vcpus := devInfo.(uint32)
 		return q.hotplugCPUs(vcpus, op)
 	case vfioDev:
-		// TODO: find a way to remove dependency of deviceDrivers lib @weizhang555
+		device := devInfo.(*config.VFIODrive)
-		device := devInfo.(deviceDrivers.VFIODevice)
 		return nil, q.hotplugVFIODevice(device, op)
 	case memoryDev:
 		memdev := devInfo.(*memoryDevice)
@@ -958,10 +956,10 @@ func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error {
 		q.qemuConfig.Devices = q.arch.appendSocket(q.qemuConfig.Devices, v)
 	case Endpoint:
 		q.qemuConfig.Devices = q.arch.appendNetwork(q.qemuConfig.Devices, v)
-	case deviceDrivers.Drive:
+	case config.BlockDrive:
 		q.qemuConfig.Devices = q.arch.appendBlockDevice(q.qemuConfig.Devices, v)
 
-	case deviceDrivers.VFIODevice:
+	case config.VFIODrive:
 		q.qemuConfig.Devices = q.arch.appendVFIODevice(q.qemuConfig.Devices, v)
 	default:
 		break

virtcontainers/qemu_arch_base.go

@@ -14,6 +14,7 @@ import (
 	govmmQemu "github.com/intel/govmm/qemu"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
+	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/device/drivers"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
@@ -72,13 +73,13 @@ type qemuArch interface {
 	appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device
 
 	// appendBlockDevice appends a block drive to devices
-	appendBlockDevice(devices []govmmQemu.Device, drive drivers.Drive) []govmmQemu.Device
+	appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device
 
 	// appendVhostUserDevice appends a vhost user device to devices
 	appendVhostUserDevice(devices []govmmQemu.Device, vhostUserDevice api.VhostUserDevice) []govmmQemu.Device
 
 	// appendVFIODevice appends a VFIO device to devices
-	appendVFIODevice(devices []govmmQemu.Device, vfioDevice drivers.VFIODevice) []govmmQemu.Device
+	appendVFIODevice(devices []govmmQemu.Device, vfioDevice config.VFIODrive) []govmmQemu.Device
 
 	// handleImagePath handles the Hypervisor Config image path
 	handleImagePath(config HypervisorConfig)
@@ -286,7 +287,7 @@ func (q *qemuArchBase) appendImage(devices []govmmQemu.Device, path string) ([]g
 
 	id := utils.MakeNameID("image", hex.EncodeToString(randBytes), maxDevIDSize)
 
-	drive := drivers.Drive{
+	drive := config.BlockDrive{
 		File:   path,
 		Format: "raw",
 		ID:     id,
@@ -430,7 +431,7 @@ func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoi
 	return devices
 }
 
-func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive drivers.Drive) []govmmQemu.Device {
+func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device {
 	if drive.File == "" || drive.ID == "" || drive.Format == "" {
 		return devices
 	}
@@ -476,14 +477,14 @@ func (q *qemuArchBase) appendVhostUserDevice(devices []govmmQemu.Device, vhostUs
 	return devices
 }
 
-func (q *qemuArchBase) appendVFIODevice(devices []govmmQemu.Device, vfioDevice drivers.VFIODevice) []govmmQemu.Device {
+func (q *qemuArchBase) appendVFIODevice(devices []govmmQemu.Device, vfioDrive config.VFIODrive) []govmmQemu.Device {
-	if vfioDevice.BDF == "" {
+	if vfioDrive.BDF == "" {
 		return devices
 	}
 
 	devices = append(devices,
 		govmmQemu.VFIODevice{
-			BDF: vfioDevice.BDF,
+			BDF: vfioDrive.BDF,
 		},
 	)
 

virtcontainers/sandbox.go

@@ -758,7 +758,7 @@ func newSandbox(sandboxConfig SandboxConfig, factory Factory) (*Sandbox, error)
 		storage:         &filesystem{},
 		network:         network,
 		config:          &sandboxConfig,
-		devManager:      deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver),
+		devManager:      deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, nil),
 		volumes:         sandboxConfig.Volumes,
 		containers:      map[string]*Container{},
 		runPath:         filepath.Join(runStoragePath, sandboxConfig.ID),