From 2885eb05327c502828b454310e48f6d680f833b4 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 27 Jun 2018 20:15:43 +0800 Subject: [PATCH] devices: use device manager to manage all devices Fixes #50 Previously the devices are created with device manager and laterly attached to hypervisor with "device.Attach()", this could work, but there's no way to remember the reference count for every device, which means if we plug one device to hypervisor twice, it's truly inserted twice, but actually we only need to insert once but use it in many places. Use device manager as a consolidated entrypoint of device management can give us a way to handle many "references" to single device, because it can save all devices and remember it's use count. Signed-off-by: Wei Zhang --- virtcontainers/container.go | 18 +-- virtcontainers/device/api/interface.go | 21 +++- virtcontainers/device/config/config.go | 38 +++++- virtcontainers/device/drivers/block.go | 106 +++++++--------- virtcontainers/device/drivers/generic.go | 30 ++++- virtcontainers/device/drivers/vfio.go | 67 ++++++---- virtcontainers/device/manager/manager.go | 150 +++++++++++++++++++---- virtcontainers/hyperstart_agent.go | 50 +++++--- virtcontainers/kata_agent.go | 10 +- virtcontainers/network.go | 4 +- virtcontainers/qemu.go | 18 ++- virtcontainers/qemu_arch_base.go | 15 +-- virtcontainers/sandbox.go | 2 +- 13 files changed, 362 insertions(+), 167 deletions(-) diff --git a/virtcontainers/container.go b/virtcontainers/container.go index 941b7d3968..8b2e127afe 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -440,8 +440,7 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) ( if c.checkBlockDeviceSupport() && stat.Mode&unix.S_IFBLK == unix.S_IFBLK { // TODO: remove dependency of package drivers b := &drivers.BlockDevice{ - DevType: config.DeviceBlock, - DeviceInfo: config.DeviceInfo{ + DeviceInfo: &config.DeviceInfo{ HostPath: m.Source, ContainerPath: m.Destination, DevType: "b", @@ -574,12 +573,13 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err } else { // If devices were not found in storage, create Device implementations // from the configuration. This should happen at create. - - devices, err := sandbox.devManager.NewDevices(contConfig.DeviceInfos) - if err != nil { - return &Container{}, err + for _, info := range contConfig.DeviceInfos { + dev, err := sandbox.devManager.NewDevice(info) + if err != nil { + return &Container{}, err + } + c.devices = append(c.devices, dev) } - c.devices = devices } return c, nil } @@ -1022,7 +1022,7 @@ func (c *Container) hotplugDrive() error { // Add drive with id as container id devID := utils.MakeNameID("drive", c.id, maxDevIDSize) - drive := drivers.Drive{ + drive := config.BlockDrive{ File: devicePath, Format: "raw", ID: devID, @@ -1059,7 +1059,7 @@ func (c *Container) removeDrive() (err error) { c.Logger().Info("unplugging block device") devID := utils.MakeNameID("drive", c.id, maxDevIDSize) - drive := &drivers.Drive{ + drive := &config.BlockDrive{ ID: devID, } diff --git a/virtcontainers/device/api/interface.go b/virtcontainers/device/api/interface.go index d6a4072d33..7f75bd64e5 100644 --- a/virtcontainers/device/api/interface.go +++ b/virtcontainers/device/api/interface.go @@ -27,6 +27,7 @@ func DeviceLogger() *logrus.Entry { // DeviceReceiver is an interface used for accepting devices // a device should be attached/added/plugged to a DeviceReceiver type DeviceReceiver interface { + // these are for hotplug/hot-unplug devices to/from hypervisor HotplugAddDevice(Device, config.DeviceType) error HotplugRemoveDevice(Device, config.DeviceType) error @@ -51,11 +52,29 @@ type VhostUserDevice interface { type Device interface { Attach(DeviceReceiver) error Detach(DeviceReceiver) error + // ID returns device identifier + DeviceID() string + // DeviceType indicates which kind of device it is + // e.g. block, vfio or vhost user DeviceType() config.DeviceType + // GetDeviceInfo returns device information that the device is created based on + GetDeviceInfo() *config.DeviceInfo + // GetDeviceDrive returns device specific data used for hotplugging by hypervisor + // Caller could cast the return value to device specific struct + // e.g. Block device returns *config.BlockDrive and + // vfio device returns *config.VFIODrive + GetDeviceDrive() interface{} + // IsAttached checks if the device is attached + IsAttached() bool } // DeviceManager can be used to create a new device, this can be used as single // device management object. type DeviceManager interface { - NewDevices(devInfos []config.DeviceInfo) ([]Device, error) + NewDevice(config.DeviceInfo) (Device, error) + AttachDevice(string, DeviceReceiver) error + DetachDevice(string, DeviceReceiver) error + IsDeviceAttached(string) bool + GetDeviceByID(string) Device + GetAllDevices() []Device } diff --git a/virtcontainers/device/config/config.go b/virtcontainers/device/config/config.go index 1ab83ddc91..c099a5cfca 100644 --- a/virtcontainers/device/config/config.go +++ b/virtcontainers/device/config/config.go @@ -49,10 +49,10 @@ var SysIOMMUPath = "/sys/kernel/iommu_groups" // DeviceInfo is an embedded type that contains device data common to all types of devices. type DeviceInfo struct { - // Device path on host + // Hostpath is device path on host HostPath string - // Device path inside the container + // ContainerPath is device path inside container ContainerPath string // Type of device: c, b, u or p @@ -87,6 +87,40 @@ type DeviceInfo struct { DriverOptions map[string]string } +// BlockDrive represents a block storage drive which may be used in case the storage +// driver has an underlying block storage device. +type BlockDrive struct { + // File is the path to the disk-image/device which will be used with this drive + File string + + // Format of the drive + Format string + + // ID is used to identify this drive in the hypervisor options. + ID string + + // Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index + Index int + + // PCIAddr is the PCI address used to identify the slot at which the drive is attached. + PCIAddr string + + // SCSI Address of the block device, in case the device is attached using SCSI driver + // SCSI address is in the format SCSI-Id:LUN + SCSIAddr string + + // VirtPath at which the device appears inside the VM, outside of the container mount namespace + VirtPath string +} + +// VFIODrive represents a VFIO drive used for hotplugging +type VFIODrive struct { + // ID is used to identify this drive in the hypervisor options. + ID string + // BDF (Bus:Device.Function) of the PCI address + BDF string +} + // VhostUserDeviceAttrs represents data shared by most vhost-user devices type VhostUserDeviceAttrs struct { DevType DeviceType diff --git a/virtcontainers/device/drivers/block.go b/virtcontainers/device/drivers/block.go index 175dc8cea2..da41cf2ecb 100644 --- a/virtcontainers/device/drivers/block.go +++ b/virtcontainers/device/drivers/block.go @@ -7,7 +7,6 @@ package drivers import ( - "encoding/hex" "path/filepath" "github.com/kata-containers/runtime/virtcontainers/device/api" @@ -17,48 +16,17 @@ import ( const maxDevIDSize = 31 -// Drive represents a block storage drive which may be used in case the storage -// driver has an underlying block storage device. -type Drive struct { - - // Path to the disk-image/device which will be used with this drive - File string - - // Format of the drive - Format string - - // ID is used to identify this drive in the hypervisor options. - ID string - - // Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index - Index int - - // PCIAddr is the PCI address used to identify the slot at which the drive is attached. - PCIAddr string -} - // BlockDevice refers to a block storage device implementation. type BlockDevice struct { - DevType config.DeviceType - DeviceInfo config.DeviceInfo - - // SCSI Address of the block device, in case the device is attached using SCSI driver - // SCSI address is in the format SCSI-Id:LUN - SCSIAddr string - - // Path at which the device appears inside the VM, outside of the container mount namespace - VirtPath string - - // PCI Slot of the block device - PCIAddr string - - BlockDrive *Drive + ID string + DeviceInfo *config.DeviceInfo + BlockDrive *config.BlockDrive } // NewBlockDevice creates a new block device based on DeviceInfo -func NewBlockDevice(devInfo config.DeviceInfo) *BlockDevice { +func NewBlockDevice(devInfo *config.DeviceInfo) *BlockDevice { return &BlockDevice{ - DevType: config.DeviceBlock, + ID: devInfo.ID, DeviceInfo: devInfo, } } @@ -66,13 +34,6 @@ func NewBlockDevice(devInfo config.DeviceInfo) *BlockDevice { // Attach is standard interface of api.Device, it's used to add device to some // DeviceReceiver func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) { - randBytes, err := utils.GenerateRandomBytes(8) - if err != nil { - return err - } - - device.DeviceInfo.ID = hex.EncodeToString(randBytes) - // Increment the block index for the sandbox. This is used to determine the name // for the block device in the case where the block device is used as container // rootfs and the predicted block device name needs to be provided to the agent. @@ -88,21 +49,13 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) { return err } - drive := Drive{ + drive := &config.BlockDrive{ File: device.DeviceInfo.HostPath, Format: "raw", ID: utils.MakeNameID("drive", device.DeviceInfo.ID, maxDevIDSize), Index: index, } - deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device") - device.BlockDrive = &drive - if err = devReceiver.HotplugAddDevice(device, config.DeviceBlock); err != nil { - return err - } - - device.DeviceInfo.Hotplugged = true - driveName, err := utils.GetVirtDriveName(index) if err != nil { return err @@ -110,36 +63,61 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) { customOptions := device.DeviceInfo.DriverOptions if customOptions != nil && customOptions["block-driver"] == "virtio-blk" { - device.VirtPath = filepath.Join("/dev", driveName) - device.PCIAddr = drive.PCIAddr + drive.VirtPath = filepath.Join("/dev", driveName) } else { scsiAddr, err := utils.GetSCSIAddress(index) if err != nil { return err } - device.SCSIAddr = scsiAddr + drive.SCSIAddr = scsiAddr } + deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device") + device.BlockDrive = drive + if err = devReceiver.HotplugAddDevice(device, config.DeviceBlock); err != nil { + return err + } + + device.DeviceInfo.Hotplugged = true + return nil } // Detach is standard interface of api.Device, it's used to remove device from some // DeviceReceiver func (device *BlockDevice) Detach(devReceiver api.DeviceReceiver) error { - if device.DeviceInfo.Hotplugged { - deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device") - - if err := devReceiver.HotplugRemoveDevice(device, config.DeviceBlock); err != nil { - deviceLogger().WithError(err).Error("Failed to unplug block device") - return err - } + deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device") + if err := devReceiver.HotplugRemoveDevice(device, config.DeviceBlock); err != nil { + deviceLogger().WithError(err).Error("Failed to unplug block device") + return err } + device.DeviceInfo.Hotplugged = false return nil } +// IsAttached checks if the device is attached +func (device *BlockDevice) IsAttached() bool { + return device.DeviceInfo.Hotplugged +} + // DeviceType is standard interface of api.Device, it returns device type func (device *BlockDevice) DeviceType() config.DeviceType { - return device.DevType + return config.DeviceBlock +} + +// DeviceID returns device ID +func (device *BlockDevice) DeviceID() string { + return device.ID +} + +// GetDeviceInfo returns device information that the device is created based on +func (device *BlockDevice) GetDeviceInfo() *config.DeviceInfo { + return device.DeviceInfo +} + +// GetDeviceDrive returns device information used for creating +func (device *BlockDevice) GetDeviceDrive() interface{} { + return device.BlockDrive } diff --git a/virtcontainers/device/drivers/generic.go b/virtcontainers/device/drivers/generic.go index 9609d811f5..e339af2105 100644 --- a/virtcontainers/device/drivers/generic.go +++ b/virtcontainers/device/drivers/generic.go @@ -13,14 +13,14 @@ import ( // GenericDevice refers to a device that is neither a VFIO device or block device. type GenericDevice struct { - DevType config.DeviceType - DeviceInfo config.DeviceInfo + ID string + DeviceInfo *config.DeviceInfo } // NewGenericDevice creates a new GenericDevice -func NewGenericDevice(devInfo config.DeviceInfo) *GenericDevice { +func NewGenericDevice(devInfo *config.DeviceInfo) *GenericDevice { return &GenericDevice{ - DevType: config.DeviceGeneric, + ID: devInfo.ID, DeviceInfo: devInfo, } } @@ -35,7 +35,27 @@ func (device *GenericDevice) Detach(devReceiver api.DeviceReceiver) error { return nil } +// IsAttached checks if the device is attached +func (device *GenericDevice) IsAttached() bool { + return device.DeviceInfo.Hotplugged +} + +// DeviceID returns device ID +func (device *GenericDevice) DeviceID() string { + return device.ID +} + // DeviceType is standard interface of api.Device, it returns device type func (device *GenericDevice) DeviceType() config.DeviceType { - return device.DevType + return config.DeviceGeneric +} + +// GetDeviceInfo returns device information that the device is created based on +func (device *GenericDevice) GetDeviceInfo() *config.DeviceInfo { + return device.DeviceInfo +} + +// GetDeviceDrive returns device information used for creating +func (device *GenericDevice) GetDeviceDrive() interface{} { + return device.DeviceInfo } diff --git a/virtcontainers/device/drivers/vfio.go b/virtcontainers/device/drivers/vfio.go index 4243465a30..313411ab1c 100644 --- a/virtcontainers/device/drivers/vfio.go +++ b/virtcontainers/device/drivers/vfio.go @@ -7,7 +7,6 @@ package drivers import ( - "encoding/hex" "fmt" "io/ioutil" "path/filepath" @@ -31,15 +30,15 @@ const ( // VFIODevice is a vfio device meant to be passed to the hypervisor // to be used by the Virtual Machine. type VFIODevice struct { - DevType config.DeviceType - DeviceInfo config.DeviceInfo - BDF string + ID string + DeviceInfo *config.DeviceInfo + vfioDrives []*config.VFIODrive } // NewVFIODevice create a new VFIO device -func NewVFIODevice(devInfo config.DeviceInfo) *VFIODevice { +func NewVFIODevice(devInfo *config.DeviceInfo) *VFIODevice { return &VFIODevice{ - DevType: config.DeviceVFIO, + ID: devInfo.ID, DeviceInfo: devInfo, } } @@ -57,44 +56,62 @@ func (device *VFIODevice) Attach(devReceiver api.DeviceReceiver) error { // Pass all devices in iommu group for _, deviceFile := range deviceFiles { - //Get bdf of device eg 0000:00:1c.0 deviceBDF, err := getBDF(deviceFile.Name()) if err != nil { return err } - - device.BDF = deviceBDF - - randBytes, err := utils.GenerateRandomBytes(8) - if err != nil { - return err + vfio := &config.VFIODrive{ + ID: utils.MakeNameID("vfio", device.DeviceInfo.ID, maxDevIDSize), + BDF: deviceBDF, } - device.DeviceInfo.ID = hex.EncodeToString(randBytes) - - if err := devReceiver.HotplugAddDevice(device, config.DeviceVFIO); err != nil { - deviceLogger().WithError(err).Error("Failed to add device") - return err - } - - deviceLogger().WithFields(logrus.Fields{ - "device-group": device.DeviceInfo.HostPath, - "device-type": "vfio-passthrough", - }).Info("Device group attached") + device.vfioDrives = append(device.vfioDrives, vfio) } + // hotplug a VFIO device is actually hotplugging a group of iommu devices + if err := devReceiver.HotplugAddDevice(device, config.DeviceVFIO); err != nil { + deviceLogger().WithError(err).Error("Failed to add device") + return err + } + + deviceLogger().WithFields(logrus.Fields{ + "device-group": device.DeviceInfo.HostPath, + "device-type": "vfio-passthrough", + }).Info("Device group attached") + device.DeviceInfo.Hotplugged = true return nil } // Detach is standard interface of api.Device, it's used to remove device from some // DeviceReceiver func (device *VFIODevice) Detach(devReceiver api.DeviceReceiver) error { + device.DeviceInfo.Hotplugged = false return nil } +// IsAttached checks if the device is attached +func (device *VFIODevice) IsAttached() bool { + return device.DeviceInfo.Hotplugged +} + // DeviceType is standard interface of api.Device, it returns device type func (device *VFIODevice) DeviceType() config.DeviceType { - return device.DevType + return config.DeviceVFIO +} + +// DeviceID returns device ID +func (device *VFIODevice) DeviceID() string { + return device.ID +} + +// GetDeviceInfo returns device information that the device is created based on +func (device *VFIODevice) GetDeviceInfo() *config.DeviceInfo { + return device.DeviceInfo +} + +// GetDeviceDrive returns device information used for creating +func (device *VFIODevice) GetDeviceDrive() interface{} { + return device.vfioDrives } // getBDF returns the BDF of pci device diff --git a/virtcontainers/device/manager/manager.go b/virtcontainers/device/manager/manager.go index 0e30e6e5b2..e14a53d266 100644 --- a/virtcontainers/device/manager/manager.go +++ b/virtcontainers/device/manager/manager.go @@ -7,11 +7,16 @@ package manager import ( + "encoding/hex" + "errors" + "sync" + "github.com/sirupsen/logrus" "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" + "github.com/kata-containers/runtime/virtcontainers/utils" ) const ( @@ -21,59 +26,162 @@ const ( VirtioSCSI string = "virtio-scsi" ) +var ( + // ErrIDExhausted represents that devices are too many + // and no more IDs can be generated + ErrIDExhausted = errors.New("IDs are exhausted") + // ErrDeviceNotExist represents device hasn't been created before + ErrDeviceNotExist = errors.New("device with specified ID hasn't been created") + // ErrDeviceAttached represents the device is already attached + ErrDeviceAttached = errors.New("device is already attached") + // ErrDeviceNotAttached represents the device isn't attached + ErrDeviceNotAttached = errors.New("device isn't attached") +) + type deviceManager struct { blockDriver string + + devices map[string]api.Device + sync.RWMutex } func deviceLogger() *logrus.Entry { return api.DeviceLogger().WithField("subsystem", "device") } +// NewDeviceManager creates a deviceManager object behaved as api.DeviceManager +func NewDeviceManager(blockDriver string, devices []api.Device) api.DeviceManager { + dm := &deviceManager{ + devices: make(map[string]api.Device), + } + if blockDriver == VirtioBlock { + dm.blockDriver = VirtioBlock + } else { + dm.blockDriver = VirtioSCSI + } + + for _, dev := range devices { + dm.devices[dev.DeviceID()] = dev + } + return dm +} + // createDevice creates one device based on DeviceInfo func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (api.Device, error) { path, err := config.GetHostPathFunc(devInfo) if err != nil { return nil, err } - devInfo.HostPath = path + + // device ID must be generated by manager instead of device itself + // in case of ID collision + if devInfo.ID, err = dm.newDeviceID(); err != nil { + return nil, err + } if isVFIO(path) { - return drivers.NewVFIODevice(devInfo), nil + return drivers.NewVFIODevice(&devInfo), nil } else if isBlock(devInfo) { if devInfo.DriverOptions == nil { devInfo.DriverOptions = make(map[string]string) } devInfo.DriverOptions["block-driver"] = dm.blockDriver - return drivers.NewBlockDevice(devInfo), nil + return drivers.NewBlockDevice(&devInfo), nil } else { deviceLogger().WithField("device", path).Info("Device has not been passed to the container") - return drivers.NewGenericDevice(devInfo), nil + return drivers.NewGenericDevice(&devInfo), nil } } -// NewDevices creates bundles of devices based on array of DeviceInfo -func (dm *deviceManager) NewDevices(devInfos []config.DeviceInfo) ([]api.Device, error) { - var devices []api.Device +// NewDevice creates bundles of devices based on array of DeviceInfo +func (dm *deviceManager) NewDevice(devInfo config.DeviceInfo) (api.Device, error) { + dm.Lock() + defer dm.Unlock() + dev, err := dm.createDevice(devInfo) + if err == nil { + dm.devices[dev.DeviceID()] = dev + } + return dev, err +} - for _, devInfo := range devInfos { - device, err := dm.createDevice(devInfo) +func (dm *deviceManager) newDeviceID() (string, error) { + for i := 0; i < 5; i++ { + // generate an random ID + randBytes, err := utils.GenerateRandomBytes(8) if err != nil { - return nil, err + return "", err } - devices = append(devices, device) - } + id := hex.EncodeToString(randBytes) - return devices, nil + // check ID collision, choose another one if ID is in use + if _, ok := dm.devices[id]; !ok { + return id, nil + } + } + return "", ErrIDExhausted } -// NewDeviceManager creates a deviceManager object behaved as api.DeviceManager -func NewDeviceManager(blockDriver string) api.DeviceManager { - dm := &deviceManager{} - if blockDriver == VirtioBlock { - dm.blockDriver = VirtioBlock - } else { - dm.blockDriver = VirtioSCSI +func (dm *deviceManager) AttachDevice(id string, dr api.DeviceReceiver) error { + dm.Lock() + defer dm.Unlock() + + d, ok := dm.devices[id] + if !ok { + return ErrDeviceNotExist } - return dm + if d.IsAttached() { + return ErrDeviceAttached + } + + if err := d.Attach(dr); err != nil { + return err + } + return nil +} + +func (dm *deviceManager) DetachDevice(id string, dr api.DeviceReceiver) error { + dm.Lock() + defer dm.Unlock() + + d, ok := dm.devices[id] + if !ok { + return ErrDeviceNotExist + } + if !d.IsAttached() { + return ErrDeviceNotAttached + } + + if err := d.Detach(dr); err != nil { + return err + } + return nil +} +func (dm *deviceManager) GetDeviceByID(id string) api.Device { + dm.RLock() + defer dm.RUnlock() + if d, ok := dm.devices[id]; ok { + return d + } + return nil +} + +func (dm *deviceManager) GetAllDevices() []api.Device { + dm.RLock() + defer dm.RUnlock() + devices := []api.Device{} + for _, v := range dm.devices { + devices = append(devices, v) + } + return devices +} + +func (dm *deviceManager) IsDeviceAttached(id string) bool { + dm.RLock() + defer dm.RUnlock() + d, ok := dm.devices[id] + if !ok { + return false + } + return d.IsAttached() } diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go index cd2f9a358c..c60397c3a7 100644 --- a/virtcontainers/hyperstart_agent.go +++ b/virtcontainers/hyperstart_agent.go @@ -14,15 +14,16 @@ import ( "syscall" "time" + "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" + proxyClient "github.com/clearcontainers/proxy/client" + "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" "github.com/kata-containers/runtime/virtcontainers/pkg/hyperstart" ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" "github.com/kata-containers/runtime/virtcontainers/utils" specs "github.com/opencontainers/runtime-spec/specs-go" - - "github.com/sirupsen/logrus" - "github.com/vishvananda/netlink" ) var defaultSockPathTemplates = []string{"%s/%s/hyper.sock", "%s/%s/tty.sock"} @@ -231,6 +232,32 @@ func fsMapFromMounts(mounts []Mount) []*hyperstart.FsmapDescriptor { return fsmap } +func fsMapFromDevices(c *Container) ([]*hyperstart.FsmapDescriptor, error) { + var fsmap []*hyperstart.FsmapDescriptor + for _, dev := range c.devices { + device := c.sandbox.devManager.GetDeviceByID(dev.DeviceID()) + if device == nil { + return nil, fmt.Errorf("can't find device: %#v", dev) + } + blockDev := device.(*drivers.BlockDevice) + + d, ok := blockDev.GetDeviceDrive().(*config.BlockDrive) + if !ok || d == nil { + return nil, fmt.Errorf("can't retrieve block device information") + } + + fsmapDesc := &hyperstart.FsmapDescriptor{ + Source: d.VirtPath, + Path: blockDev.DeviceInfo.ContainerPath, + AbsolutePath: true, + DockerVolume: false, + SCSIAddr: d.SCSIAddr, + } + fsmap = append(fsmap, fsmapDesc) + } + return fsmap, nil +} + // init is the agent initialization implementation for hyperstart. func (h *hyper) init(sandbox *Sandbox, config interface{}) (err error) { switch c := config.(type) { @@ -497,20 +524,11 @@ func (h *hyper) startOneContainer(sandbox *Sandbox, c *Container) error { h.handleBlockVolumes(c) // Append container mounts for block devices passed with --device. - for _, device := range c.devices { - d, ok := device.(*drivers.BlockDevice) - - if ok { - fsmapDesc := &hyperstart.FsmapDescriptor{ - Source: d.VirtPath, - Path: d.DeviceInfo.ContainerPath, - AbsolutePath: true, - DockerVolume: false, - SCSIAddr: d.SCSIAddr, - } - fsmap = append(fsmap, fsmapDesc) - } + fsmapDev, err := fsMapFromDevices(c) + if err != nil { + return err } + fsmap = append(fsmap, fsmapDev...) // Assign fsmap for hyperstart to mount these at the correct location within the container container.Fsmap = fsmap diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 831862af22..e6ff9c6d00 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -733,12 +733,12 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, devices []api.Devic ContainerPath: d.DeviceInfo.ContainerPath, } - if d.SCSIAddr == "" { + if d.BlockDrive.SCSIAddr == "" { kataDevice.Type = kataBlkDevType - kataDevice.Id = d.PCIAddr + kataDevice.Id = d.BlockDrive.PCIAddr } else { kataDevice.Type = kataSCSIDevType - kataDevice.Id = d.SCSIAddr + kataDevice.Id = d.BlockDrive.SCSIAddr } deviceList = append(deviceList, kataDevice) @@ -970,10 +970,10 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage { if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock { vol.Driver = kataBlkDevType - vol.Source = b.VirtPath + vol.Source = b.BlockDrive.VirtPath } else { vol.Driver = kataSCSIDevType - vol.Source = b.SCSIAddr + vol.Source = b.BlockDrive.SCSIAddr } vol.MountPoint = b.DeviceInfo.ContainerPath diff --git a/virtcontainers/network.go b/virtcontainers/network.go index 15401af655..ef76157407 100644 --- a/virtcontainers/network.go +++ b/virtcontainers/network.go @@ -25,6 +25,7 @@ import ( "github.com/vishvananda/netns" "golang.org/x/sys/unix" + "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" "github.com/kata-containers/runtime/virtcontainers/utils" @@ -343,7 +344,8 @@ func (endpoint *PhysicalEndpoint) Attach(h hypervisor) error { return err } - d := drivers.VFIODevice{ + // TODO: use device manager as general device management entrance + d := config.VFIODrive{ BDF: endpoint.BDF, } diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 1d84fa88d7..8a932939c7 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -20,7 +20,7 @@ import ( "github.com/sirupsen/logrus" "github.com/kata-containers/runtime/virtcontainers/device/api" - deviceDrivers "github.com/kata-containers/runtime/virtcontainers/device/drivers" + "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -656,7 +656,7 @@ func (q *qemu) removeDeviceFromBridge(ID string) error { return err } -func (q *qemu) hotplugBlockDevice(drive *deviceDrivers.Drive, op operation) error { +func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error { err := q.qmpSetup() if err != nil { return err @@ -717,13 +717,13 @@ func (q *qemu) hotplugBlockDevice(drive *deviceDrivers.Drive, op operation) erro return nil } -func (q *qemu) hotplugVFIODevice(device deviceDrivers.VFIODevice, op operation) error { +func (q *qemu) hotplugVFIODevice(device *config.VFIODrive, op operation) error { err := q.qmpSetup() if err != nil { return err } - devID := "vfio-" + device.DeviceInfo.ID + devID := device.ID if op == addDevice { addr, bridge, err := q.addDeviceToBridge(devID) @@ -750,15 +750,13 @@ func (q *qemu) hotplugVFIODevice(device deviceDrivers.VFIODevice, op operation) func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operation) (interface{}, error) { switch devType { case blockDev: - // TODO: find a way to remove dependency of deviceDrivers lib @weizhang555 - drive := devInfo.(*deviceDrivers.Drive) + drive := devInfo.(*config.BlockDrive) return nil, q.hotplugBlockDevice(drive, op) case cpuDev: vcpus := devInfo.(uint32) return q.hotplugCPUs(vcpus, op) case vfioDev: - // TODO: find a way to remove dependency of deviceDrivers lib @weizhang555 - device := devInfo.(deviceDrivers.VFIODevice) + device := devInfo.(*config.VFIODrive) return nil, q.hotplugVFIODevice(device, op) case memoryDev: memdev := devInfo.(*memoryDevice) @@ -958,10 +956,10 @@ func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error { q.qemuConfig.Devices = q.arch.appendSocket(q.qemuConfig.Devices, v) case Endpoint: q.qemuConfig.Devices = q.arch.appendNetwork(q.qemuConfig.Devices, v) - case deviceDrivers.Drive: + case config.BlockDrive: q.qemuConfig.Devices = q.arch.appendBlockDevice(q.qemuConfig.Devices, v) - case deviceDrivers.VFIODevice: + case config.VFIODrive: q.qemuConfig.Devices = q.arch.appendVFIODevice(q.qemuConfig.Devices, v) default: break diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index e0188c4d3d..53356b9d4c 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -14,6 +14,7 @@ import ( govmmQemu "github.com/intel/govmm/qemu" "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -72,13 +73,13 @@ type qemuArch interface { appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device // appendBlockDevice appends a block drive to devices - appendBlockDevice(devices []govmmQemu.Device, drive drivers.Drive) []govmmQemu.Device + appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device // appendVhostUserDevice appends a vhost user device to devices appendVhostUserDevice(devices []govmmQemu.Device, vhostUserDevice api.VhostUserDevice) []govmmQemu.Device // appendVFIODevice appends a VFIO device to devices - appendVFIODevice(devices []govmmQemu.Device, vfioDevice drivers.VFIODevice) []govmmQemu.Device + appendVFIODevice(devices []govmmQemu.Device, vfioDevice config.VFIODrive) []govmmQemu.Device // handleImagePath handles the Hypervisor Config image path handleImagePath(config HypervisorConfig) @@ -286,7 +287,7 @@ func (q *qemuArchBase) appendImage(devices []govmmQemu.Device, path string) ([]g id := utils.MakeNameID("image", hex.EncodeToString(randBytes), maxDevIDSize) - drive := drivers.Drive{ + drive := config.BlockDrive{ File: path, Format: "raw", ID: id, @@ -430,7 +431,7 @@ func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoi return devices } -func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive drivers.Drive) []govmmQemu.Device { +func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device { if drive.File == "" || drive.ID == "" || drive.Format == "" { return devices } @@ -476,14 +477,14 @@ func (q *qemuArchBase) appendVhostUserDevice(devices []govmmQemu.Device, vhostUs return devices } -func (q *qemuArchBase) appendVFIODevice(devices []govmmQemu.Device, vfioDevice drivers.VFIODevice) []govmmQemu.Device { - if vfioDevice.BDF == "" { +func (q *qemuArchBase) appendVFIODevice(devices []govmmQemu.Device, vfioDrive config.VFIODrive) []govmmQemu.Device { + if vfioDrive.BDF == "" { return devices } devices = append(devices, govmmQemu.VFIODevice{ - BDF: vfioDevice.BDF, + BDF: vfioDrive.BDF, }, ) diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index b3339d252f..f6a63eb23b 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -758,7 +758,7 @@ func newSandbox(sandboxConfig SandboxConfig, factory Factory) (*Sandbox, error) storage: &filesystem{}, network: network, config: &sandboxConfig, - devManager: deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver), + devManager: deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, nil), volumes: sandboxConfig.Volumes, containers: map[string]*Container{}, runPath: filepath.Join(runStoragePath, sandboxConfig.ID),