github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-07-01 17:52:40 +00:00
Code Issues Projects Releases Wiki Activity

devices: use device manager to manage all devices

Browse Source 
Fixes #50

Previously the devices are created with device manager and laterly
attached to hypervisor with "device.Attach()", this could work, but
there's no way to remember the reference count for every device, which
means if we plug one device to hypervisor twice, it's truly inserted
twice, but actually we only need to insert once but use it in many
places.

Use device manager as a consolidated entrypoint of device management can
give us a way to handle many "references" to single device, because it
can save all devices and remember it's use count.

Signed-off-by: Wei Zhang <zhangwei555@huawei.com>
This commit is contained in:
Wei Zhang 2018-06-27 20:15:43 +08:00 committed by z00280905
parent 67b5841153
commit 2885eb0532
13 changed files with 362 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
virtcontainers/container.go
View File

@ -440,8 +440,7 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) (
if c.checkBlockDeviceSupport() && stat.Mode&unix.S_IFBLK == unix.S_IFBLK {
// TODO: remove dependency of package drivers
b := &drivers.BlockDevice{
DevType: config.DeviceBlock,
DeviceInfo: config.DeviceInfo{
DeviceInfo: &config.DeviceInfo{
HostPath: m.Source,
ContainerPath: m.Destination,
DevType: "b",
@ -574,12 +573,13 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err
} else {
// If devices were not found in storage, create Device implementations
// from the configuration. This should happen at create.
devices, err := sandbox.devManager.NewDevices(contConfig.DeviceInfos)
if err != nil {
return &Container{}, err
for _, info := range contConfig.DeviceInfos {
dev, err := sandbox.devManager.NewDevice(info)
if err != nil {
return &Container{}, err
}
c.devices = append(c.devices, dev)
}
c.devices = devices
}
return c, nil
}
@ -1022,7 +1022,7 @@ func (c *Container) hotplugDrive() error {
// Add drive with id as container id
devID := utils.MakeNameID("drive", c.id, maxDevIDSize)
drive := drivers.Drive{
drive := config.BlockDrive{
File: devicePath,
Format: "raw",
ID: devID,
@ -1059,7 +1059,7 @@ func (c *Container) removeDrive() (err error) {
c.Logger().Info("unplugging block device")
devID := utils.MakeNameID("drive", c.id, maxDevIDSize)
drive := &drivers.Drive{
drive := &config.BlockDrive{
ID: devID,
}

21
virtcontainers/device/api/interface.go
View File

@ -27,6 +27,7 @@ func DeviceLogger() *logrus.Entry {
// DeviceReceiver is an interface used for accepting devices
// a device should be attached/added/plugged to a DeviceReceiver
type DeviceReceiver interface {
// these are for hotplug/hot-unplug devices to/from hypervisor
HotplugAddDevice(Device, config.DeviceType) error
HotplugRemoveDevice(Device, config.DeviceType) error
@ -51,11 +52,29 @@ type VhostUserDevice interface {
type Device interface {
Attach(DeviceReceiver) error
Detach(DeviceReceiver) error
// ID returns device identifier
DeviceID() string
// DeviceType indicates which kind of device it is
// e.g. block, vfio or vhost user
DeviceType() config.DeviceType
// GetDeviceInfo returns device information that the device is created based on
GetDeviceInfo() *config.DeviceInfo
// GetDeviceDrive returns device specific data used for hotplugging by hypervisor
// Caller could cast the return value to device specific struct
// e.g. Block device returns *config.BlockDrive and
// vfio device returns *config.VFIODrive
GetDeviceDrive() interface{}
// IsAttached checks if the device is attached
IsAttached() bool
}
// DeviceManager can be used to create a new device, this can be used as single
// device management object.
type DeviceManager interface {
NewDevices(devInfos []config.DeviceInfo) ([]Device, error)
NewDevice(config.DeviceInfo) (Device, error)
AttachDevice(string, DeviceReceiver) error
DetachDevice(string, DeviceReceiver) error
IsDeviceAttached(string) bool
GetDeviceByID(string) Device
GetAllDevices() []Device
}

38
virtcontainers/device/config/config.go
View File

@ -49,10 +49,10 @@ var SysIOMMUPath = "/sys/kernel/iommu_groups"
// DeviceInfo is an embedded type that contains device data common to all types of devices.
type DeviceInfo struct {
// Device path on host
// Hostpath is device path on host
HostPath string
// Device path inside the container
// ContainerPath is device path inside container
ContainerPath string
// Type of device: c, b, u or p
@ -87,6 +87,40 @@ type DeviceInfo struct {
DriverOptions map[string]string
}
// BlockDrive represents a block storage drive which may be used in case the storage
// driver has an underlying block storage device.
type BlockDrive struct {
// File is the path to the disk-image/device which will be used with this drive
File string
// Format of the drive
Format string
// ID is used to identify this drive in the hypervisor options.
ID string
// Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index
Index int
// PCIAddr is the PCI address used to identify the slot at which the drive is attached.
PCIAddr string
// SCSI Address of the block device, in case the device is attached using SCSI driver
// SCSI address is in the format SCSI-Id:LUN
SCSIAddr string
// VirtPath at which the device appears inside the VM, outside of the container mount namespace
VirtPath string
}
// VFIODrive represents a VFIO drive used for hotplugging
type VFIODrive struct {
// ID is used to identify this drive in the hypervisor options.
ID string
// BDF (Bus:Device.Function) of the PCI address
BDF string
}
// VhostUserDeviceAttrs represents data shared by most vhost-user devices
type VhostUserDeviceAttrs struct {
DevType DeviceType

106
virtcontainers/device/drivers/block.go
View File

@ -7,7 +7,6 @@
package drivers
import (
"encoding/hex"
"path/filepath"
"github.com/kata-containers/runtime/virtcontainers/device/api"
@ -17,48 +16,17 @@ import (
const maxDevIDSize = 31
// Drive represents a block storage drive which may be used in case the storage
// driver has an underlying block storage device.
type Drive struct {
// Path to the disk-image/device which will be used with this drive
File string
// Format of the drive
Format string
// ID is used to identify this drive in the hypervisor options.
ID string
// Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index
Index int
// PCIAddr is the PCI address used to identify the slot at which the drive is attached.
PCIAddr string
}
// BlockDevice refers to a block storage device implementation.
type BlockDevice struct {
DevType config.DeviceType
DeviceInfo config.DeviceInfo
// SCSI Address of the block device, in case the device is attached using SCSI driver
// SCSI address is in the format SCSI-Id:LUN
SCSIAddr string
// Path at which the device appears inside the VM, outside of the container mount namespace
VirtPath string
// PCI Slot of the block device
PCIAddr string
BlockDrive *Drive
ID string
DeviceInfo *config.DeviceInfo
BlockDrive *config.BlockDrive
}
// NewBlockDevice creates a new block device based on DeviceInfo
func NewBlockDevice(devInfo config.DeviceInfo) *BlockDevice {
func NewBlockDevice(devInfo *config.DeviceInfo) *BlockDevice {
return &BlockDevice{
DevType: config.DeviceBlock,
ID: devInfo.ID,
DeviceInfo: devInfo,
}
}
@ -66,13 +34,6 @@ func NewBlockDevice(devInfo config.DeviceInfo) *BlockDevice {
// Attach is standard interface of api.Device, it's used to add device to some
// DeviceReceiver
func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
randBytes, err := utils.GenerateRandomBytes(8)
if err != nil {
return err
}
device.DeviceInfo.ID = hex.EncodeToString(randBytes)
// Increment the block index for the sandbox. This is used to determine the name
// for the block device in the case where the block device is used as container
// rootfs and the predicted block device name needs to be provided to the agent.
@ -88,21 +49,13 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
return err
}
drive := Drive{
drive := &config.BlockDrive{
File: device.DeviceInfo.HostPath,
Format: "raw",
ID: utils.MakeNameID("drive", device.DeviceInfo.ID, maxDevIDSize),
Index: index,
}
deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device")
device.BlockDrive = &drive
if err = devReceiver.HotplugAddDevice(device, config.DeviceBlock); err != nil {
return err
}
device.DeviceInfo.Hotplugged = true
driveName, err := utils.GetVirtDriveName(index)
if err != nil {
return err
@ -110,36 +63,61 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
customOptions := device.DeviceInfo.DriverOptions
if customOptions != nil && customOptions["block-driver"] == "virtio-blk" {
device.VirtPath = filepath.Join("/dev", driveName)
device.PCIAddr = drive.PCIAddr
drive.VirtPath = filepath.Join("/dev", driveName)
} else {
scsiAddr, err := utils.GetSCSIAddress(index)
if err != nil {
return err
}
device.SCSIAddr = scsiAddr
drive.SCSIAddr = scsiAddr
}
deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device")
device.BlockDrive = drive
if err = devReceiver.HotplugAddDevice(device, config.DeviceBlock); err != nil {
return err
}
device.DeviceInfo.Hotplugged = true
return nil
}
// Detach is standard interface of api.Device, it's used to remove device from some
// DeviceReceiver
func (device *BlockDevice) Detach(devReceiver api.DeviceReceiver) error {
if device.DeviceInfo.Hotplugged {
deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device")
if err := devReceiver.HotplugRemoveDevice(device, config.DeviceBlock); err != nil {
deviceLogger().WithError(err).Error("Failed to unplug block device")
return err
}
deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device")
if err := devReceiver.HotplugRemoveDevice(device, config.DeviceBlock); err != nil {
deviceLogger().WithError(err).Error("Failed to unplug block device")
return err
}
device.DeviceInfo.Hotplugged = false
return nil
}
// IsAttached checks if the device is attached
func (device *BlockDevice) IsAttached() bool {
return device.DeviceInfo.Hotplugged
}
// DeviceType is standard interface of api.Device, it returns device type
func (device *BlockDevice) DeviceType() config.DeviceType {
return device.DevType
return config.DeviceBlock
}
// DeviceID returns device ID
func (device *BlockDevice) DeviceID() string {
return device.ID
}
// GetDeviceInfo returns device information that the device is created based on
func (device *BlockDevice) GetDeviceInfo() *config.DeviceInfo {
return device.DeviceInfo
}
// GetDeviceDrive returns device information used for creating
func (device *BlockDevice) GetDeviceDrive() interface{} {
return device.BlockDrive
}

30
virtcontainers/device/drivers/generic.go
View File

@ -13,14 +13,14 @@ import (
// GenericDevice refers to a device that is neither a VFIO device or block device.
type GenericDevice struct {
DevType config.DeviceType
DeviceInfo config.DeviceInfo
ID string
DeviceInfo *config.DeviceInfo
}
// NewGenericDevice creates a new GenericDevice
func NewGenericDevice(devInfo config.DeviceInfo) *GenericDevice {
func NewGenericDevice(devInfo *config.DeviceInfo) *GenericDevice {
return &GenericDevice{
DevType: config.DeviceGeneric,
ID: devInfo.ID,
DeviceInfo: devInfo,
}
}
@ -35,7 +35,27 @@ func (device *GenericDevice) Detach(devReceiver api.DeviceReceiver) error {
return nil
}
// IsAttached checks if the device is attached
func (device *GenericDevice) IsAttached() bool {
return device.DeviceInfo.Hotplugged
}
// DeviceID returns device ID
func (device *GenericDevice) DeviceID() string {
return device.ID
}
// DeviceType is standard interface of api.Device, it returns device type
func (device *GenericDevice) DeviceType() config.DeviceType {
return device.DevType
return config.DeviceGeneric
}
// GetDeviceInfo returns device information that the device is created based on
func (device *GenericDevice) GetDeviceInfo() *config.DeviceInfo {
return device.DeviceInfo
}
// GetDeviceDrive returns device information used for creating
func (device *GenericDevice) GetDeviceDrive() interface{} {
return device.DeviceInfo
}

67
virtcontainers/device/drivers/vfio.go
View File

@ -7,7 +7,6 @@
package drivers
import (
"encoding/hex"
"fmt"
"io/ioutil"
"path/filepath"
@ -31,15 +30,15 @@ const (
// VFIODevice is a vfio device meant to be passed to the hypervisor
// to be used by the Virtual Machine.
type VFIODevice struct {
DevType config.DeviceType
DeviceInfo config.DeviceInfo
BDF string
ID string
DeviceInfo *config.DeviceInfo
vfioDrives []*config.VFIODrive
}
// NewVFIODevice create a new VFIO device
func NewVFIODevice(devInfo config.DeviceInfo) *VFIODevice {
func NewVFIODevice(devInfo *config.DeviceInfo) *VFIODevice {
return &VFIODevice{
DevType: config.DeviceVFIO,
ID: devInfo.ID,
DeviceInfo: devInfo,
}
}
@ -57,44 +56,62 @@ func (device *VFIODevice) Attach(devReceiver api.DeviceReceiver) error {
// Pass all devices in iommu group
for _, deviceFile := range deviceFiles {
//Get bdf of device eg 0000:00:1c.0
deviceBDF, err := getBDF(deviceFile.Name())
if err != nil {
return err
}
device.BDF = deviceBDF
randBytes, err := utils.GenerateRandomBytes(8)
if err != nil {
return err
vfio := &config.VFIODrive{
ID: utils.MakeNameID("vfio", device.DeviceInfo.ID, maxDevIDSize),
BDF: deviceBDF,
}
device.DeviceInfo.ID = hex.EncodeToString(randBytes)
if err := devReceiver.HotplugAddDevice(device, config.DeviceVFIO); err != nil {
deviceLogger().WithError(err).Error("Failed to add device")
return err
}
deviceLogger().WithFields(logrus.Fields{
"device-group": device.DeviceInfo.HostPath,
"device-type": "vfio-passthrough",
}).Info("Device group attached")
device.vfioDrives = append(device.vfioDrives, vfio)
}
// hotplug a VFIO device is actually hotplugging a group of iommu devices
if err := devReceiver.HotplugAddDevice(device, config.DeviceVFIO); err != nil {
deviceLogger().WithError(err).Error("Failed to add device")
return err
}
deviceLogger().WithFields(logrus.Fields{
"device-group": device.DeviceInfo.HostPath,
"device-type": "vfio-passthrough",
}).Info("Device group attached")
device.DeviceInfo.Hotplugged = true
return nil
}
// Detach is standard interface of api.Device, it's used to remove device from some
// DeviceReceiver
func (device *VFIODevice) Detach(devReceiver api.DeviceReceiver) error {
device.DeviceInfo.Hotplugged = false
return nil
}
// IsAttached checks if the device is attached
func (device *VFIODevice) IsAttached() bool {
return device.DeviceInfo.Hotplugged
}
// DeviceType is standard interface of api.Device, it returns device type
func (device *VFIODevice) DeviceType() config.DeviceType {
return device.DevType
return config.DeviceVFIO
}
// DeviceID returns device ID
func (device *VFIODevice) DeviceID() string {
return device.ID
}
// GetDeviceInfo returns device information that the device is created based on
func (device *VFIODevice) GetDeviceInfo() *config.DeviceInfo {
return device.DeviceInfo
}
// GetDeviceDrive returns device information used for creating
func (device *VFIODevice) GetDeviceDrive() interface{} {
return device.vfioDrives
}
// getBDF returns the BDF of pci device

150
virtcontainers/device/manager/manager.go
View File

@ -7,11 +7,16 @@
package manager
import (
"encoding/hex"
"errors"
"sync"
"github.com/sirupsen/logrus"
"github.com/kata-containers/runtime/virtcontainers/device/api"
"github.com/kata-containers/runtime/virtcontainers/device/config"
"github.com/kata-containers/runtime/virtcontainers/device/drivers"
"github.com/kata-containers/runtime/virtcontainers/utils"
)
const (
@ -21,59 +26,162 @@ const (
VirtioSCSI string = "virtio-scsi"
)
var (
// ErrIDExhausted represents that devices are too many
// and no more IDs can be generated
ErrIDExhausted = errors.New("IDs are exhausted")
// ErrDeviceNotExist represents device hasn't been created before
ErrDeviceNotExist = errors.New("device with specified ID hasn't been created")
// ErrDeviceAttached represents the device is already attached
ErrDeviceAttached = errors.New("device is already attached")
// ErrDeviceNotAttached represents the device isn't attached
ErrDeviceNotAttached = errors.New("device isn't attached")
)
type deviceManager struct {
blockDriver string
devices map[string]api.Device
sync.RWMutex
}
func deviceLogger() *logrus.Entry {
return api.DeviceLogger().WithField("subsystem", "device")
}
// NewDeviceManager creates a deviceManager object behaved as api.DeviceManager
func NewDeviceManager(blockDriver string, devices []api.Device) api.DeviceManager {
dm := &deviceManager{
devices: make(map[string]api.Device),
}
if blockDriver == VirtioBlock {
dm.blockDriver = VirtioBlock
} else {
dm.blockDriver = VirtioSCSI
}
for _, dev := range devices {
dm.devices[dev.DeviceID()] = dev
}
return dm
}
// createDevice creates one device based on DeviceInfo
func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (api.Device, error) {
path, err := config.GetHostPathFunc(devInfo)
if err != nil {
return nil, err
}
devInfo.HostPath = path
// device ID must be generated by manager instead of device itself
// in case of ID collision
if devInfo.ID, err = dm.newDeviceID(); err != nil {
return nil, err
}
if isVFIO(path) {
return drivers.NewVFIODevice(devInfo), nil
return drivers.NewVFIODevice(&devInfo), nil
} else if isBlock(devInfo) {
if devInfo.DriverOptions == nil {
devInfo.DriverOptions = make(map[string]string)
}
devInfo.DriverOptions["block-driver"] = dm.blockDriver
return drivers.NewBlockDevice(devInfo), nil
return drivers.NewBlockDevice(&devInfo), nil
} else {
deviceLogger().WithField("device", path).Info("Device has not been passed to the container")
return drivers.NewGenericDevice(devInfo), nil
return drivers.NewGenericDevice(&devInfo), nil
}
}
// NewDevices creates bundles of devices based on array of DeviceInfo
func (dm *deviceManager) NewDevices(devInfos []config.DeviceInfo) ([]api.Device, error) {
var devices []api.Device
// NewDevice creates bundles of devices based on array of DeviceInfo
func (dm *deviceManager) NewDevice(devInfo config.DeviceInfo) (api.Device, error) {
dm.Lock()
defer dm.Unlock()
dev, err := dm.createDevice(devInfo)
if err == nil {
dm.devices[dev.DeviceID()] = dev
}
return dev, err
}
for _, devInfo := range devInfos {
device, err := dm.createDevice(devInfo)
func (dm *deviceManager) newDeviceID() (string, error) {
for i := 0; i < 5; i++ {
// generate an random ID
randBytes, err := utils.GenerateRandomBytes(8)
if err != nil {
return nil, err
return "", err
}
devices = append(devices, device)
}
id := hex.EncodeToString(randBytes)
return devices, nil
// check ID collision, choose another one if ID is in use
if _, ok := dm.devices[id]; !ok {
return id, nil
}
}
return "", ErrIDExhausted
}
// NewDeviceManager creates a deviceManager object behaved as api.DeviceManager
func NewDeviceManager(blockDriver string) api.DeviceManager {
dm := &deviceManager{}
if blockDriver == VirtioBlock {
dm.blockDriver = VirtioBlock
} else {
dm.blockDriver = VirtioSCSI
func (dm *deviceManager) AttachDevice(id string, dr api.DeviceReceiver) error {
dm.Lock()
defer dm.Unlock()
d, ok := dm.devices[id]
if !ok {
return ErrDeviceNotExist
}
return dm
if d.IsAttached() {
return ErrDeviceAttached
}
if err := d.Attach(dr); err != nil {
return err
}
return nil
}
func (dm *deviceManager) DetachDevice(id string, dr api.DeviceReceiver) error {
dm.Lock()
defer dm.Unlock()
d, ok := dm.devices[id]
if !ok {
return ErrDeviceNotExist
}
if !d.IsAttached() {
return ErrDeviceNotAttached
}
if err := d.Detach(dr); err != nil {
return err
}
return nil
}
func (dm *deviceManager) GetDeviceByID(id string) api.Device {
dm.RLock()
defer dm.RUnlock()
if d, ok := dm.devices[id]; ok {
return d
}
return nil
}
func (dm *deviceManager) GetAllDevices() []api.Device {
dm.RLock()
defer dm.RUnlock()
devices := []api.Device{}
for _, v := range dm.devices {
devices = append(devices, v)
}
return devices
}
func (dm *deviceManager) IsDeviceAttached(id string) bool {
dm.RLock()
defer dm.RUnlock()
d, ok := dm.devices[id]
if !ok {
return false
}
return d.IsAttached()
}

50
virtcontainers/hyperstart_agent.go
View File

@ -14,15 +14,16 @@ import (
"syscall"
"time"
"github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
proxyClient "github.com/clearcontainers/proxy/client"
"github.com/kata-containers/runtime/virtcontainers/device/config"
"github.com/kata-containers/runtime/virtcontainers/device/drivers"
"github.com/kata-containers/runtime/virtcontainers/pkg/hyperstart"
ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
"github.com/kata-containers/runtime/virtcontainers/utils"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
)
var defaultSockPathTemplates = []string{"%s/%s/hyper.sock", "%s/%s/tty.sock"}
@ -231,6 +232,32 @@ func fsMapFromMounts(mounts []Mount) []*hyperstart.FsmapDescriptor {
return fsmap
}
func fsMapFromDevices(c *Container) ([]*hyperstart.FsmapDescriptor, error) {
var fsmap []*hyperstart.FsmapDescriptor
for _, dev := range c.devices {
device := c.sandbox.devManager.GetDeviceByID(dev.DeviceID())
if device == nil {
return nil, fmt.Errorf("can't find device: %#v", dev)
}
blockDev := device.(*drivers.BlockDevice)
d, ok := blockDev.GetDeviceDrive().(*config.BlockDrive)
if !ok || d == nil {
return nil, fmt.Errorf("can't retrieve block device information")
}
fsmapDesc := &hyperstart.FsmapDescriptor{
Source: d.VirtPath,
Path: blockDev.DeviceInfo.ContainerPath,
AbsolutePath: true,
DockerVolume: false,
SCSIAddr: d.SCSIAddr,
}
fsmap = append(fsmap, fsmapDesc)
}
return fsmap, nil
}
// init is the agent initialization implementation for hyperstart.
func (h *hyper) init(sandbox *Sandbox, config interface{}) (err error) {
switch c := config.(type) {
@ -497,20 +524,11 @@ func (h *hyper) startOneContainer(sandbox *Sandbox, c *Container) error {
h.handleBlockVolumes(c)
// Append container mounts for block devices passed with --device.
for _, device := range c.devices {
d, ok := device.(*drivers.BlockDevice)
if ok {
fsmapDesc := &hyperstart.FsmapDescriptor{
Source: d.VirtPath,
Path: d.DeviceInfo.ContainerPath,
AbsolutePath: true,
DockerVolume: false,
SCSIAddr: d.SCSIAddr,
}
fsmap = append(fsmap, fsmapDesc)
}
fsmapDev, err := fsMapFromDevices(c)
if err != nil {
return err
}
fsmap = append(fsmap, fsmapDev...)
// Assign fsmap for hyperstart to mount these at the correct location within the container
container.Fsmap = fsmap

10
virtcontainers/kata_agent.go
View File

@ -733,12 +733,12 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, devices []api.Devic
ContainerPath: d.DeviceInfo.ContainerPath,
}
if d.SCSIAddr == "" {
if d.BlockDrive.SCSIAddr == "" {
kataDevice.Type = kataBlkDevType
kataDevice.Id = d.PCIAddr
kataDevice.Id = d.BlockDrive.PCIAddr
} else {
kataDevice.Type = kataSCSIDevType
kataDevice.Id = d.SCSIAddr
kataDevice.Id = d.BlockDrive.SCSIAddr
}
deviceList = append(deviceList, kataDevice)
@ -970,10 +970,10 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
vol.Driver = kataBlkDevType
vol.Source = b.VirtPath
vol.Source = b.BlockDrive.VirtPath
} else {
vol.Driver = kataSCSIDevType
vol.Source = b.SCSIAddr
vol.Source = b.BlockDrive.SCSIAddr
}
vol.MountPoint = b.DeviceInfo.ContainerPath

4
virtcontainers/network.go
View File

@ -25,6 +25,7 @@ import (
"github.com/vishvananda/netns"
"golang.org/x/sys/unix"
"github.com/kata-containers/runtime/virtcontainers/device/config"
"github.com/kata-containers/runtime/virtcontainers/device/drivers"
"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
"github.com/kata-containers/runtime/virtcontainers/utils"
@ -343,7 +344,8 @@ func (endpoint *PhysicalEndpoint) Attach(h hypervisor) error {
return err
}
d := drivers.VFIODevice{
// TODO: use device manager as general device management entrance
d := config.VFIODrive{
BDF: endpoint.BDF,
}

18
virtcontainers/qemu.go
View File

@ -20,7 +20,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/kata-containers/runtime/virtcontainers/device/api"
deviceDrivers "github.com/kata-containers/runtime/virtcontainers/device/drivers"
"github.com/kata-containers/runtime/virtcontainers/device/config"
"github.com/kata-containers/runtime/virtcontainers/utils"
)
@ -656,7 +656,7 @@ func (q *qemu) removeDeviceFromBridge(ID string) error {
return err
}
func (q *qemu) hotplugBlockDevice(drive *deviceDrivers.Drive, op operation) error {
func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error {
err := q.qmpSetup()
if err != nil {
return err
@ -717,13 +717,13 @@ func (q *qemu) hotplugBlockDevice(drive *deviceDrivers.Drive, op operation) erro
return nil
}
func (q *qemu) hotplugVFIODevice(device deviceDrivers.VFIODevice, op operation) error {
func (q *qemu) hotplugVFIODevice(device *config.VFIODrive, op operation) error {
err := q.qmpSetup()
if err != nil {
return err
}
devID := "vfio-" + device.DeviceInfo.ID
devID := device.ID
if op == addDevice {
addr, bridge, err := q.addDeviceToBridge(devID)
@ -750,15 +750,13 @@ func (q *qemu) hotplugVFIODevice(device deviceDrivers.VFIODevice, op operation)
func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operation) (interface{}, error) {
switch devType {
case blockDev:
// TODO: find a way to remove dependency of deviceDrivers lib @weizhang555
drive := devInfo.(*deviceDrivers.Drive)
drive := devInfo.(*config.BlockDrive)
return nil, q.hotplugBlockDevice(drive, op)
case cpuDev:
vcpus := devInfo.(uint32)
return q.hotplugCPUs(vcpus, op)
case vfioDev:
// TODO: find a way to remove dependency of deviceDrivers lib @weizhang555
device := devInfo.(deviceDrivers.VFIODevice)
device := devInfo.(*config.VFIODrive)
return nil, q.hotplugVFIODevice(device, op)
case memoryDev:
memdev := devInfo.(*memoryDevice)
@ -958,10 +956,10 @@ func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error {
q.qemuConfig.Devices = q.arch.appendSocket(q.qemuConfig.Devices, v)
case Endpoint:
q.qemuConfig.Devices = q.arch.appendNetwork(q.qemuConfig.Devices, v)
case deviceDrivers.Drive:
case config.BlockDrive:
q.qemuConfig.Devices = q.arch.appendBlockDevice(q.qemuConfig.Devices, v)
case deviceDrivers.VFIODevice:
case config.VFIODrive:
q.qemuConfig.Devices = q.arch.appendVFIODevice(q.qemuConfig.Devices, v)
default:
break

15
virtcontainers/qemu_arch_base.go
View File

@ -14,6 +14,7 @@ import (
govmmQemu "github.com/intel/govmm/qemu"
"github.com/kata-containers/runtime/virtcontainers/device/api"
"github.com/kata-containers/runtime/virtcontainers/device/config"
"github.com/kata-containers/runtime/virtcontainers/device/drivers"
"github.com/kata-containers/runtime/virtcontainers/utils"
)
@ -72,13 +73,13 @@ type qemuArch interface {
appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device
// appendBlockDevice appends a block drive to devices
appendBlockDevice(devices []govmmQemu.Device, drive drivers.Drive) []govmmQemu.Device
appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device
// appendVhostUserDevice appends a vhost user device to devices
appendVhostUserDevice(devices []govmmQemu.Device, vhostUserDevice api.VhostUserDevice) []govmmQemu.Device
// appendVFIODevice appends a VFIO device to devices
appendVFIODevice(devices []govmmQemu.Device, vfioDevice drivers.VFIODevice) []govmmQemu.Device
appendVFIODevice(devices []govmmQemu.Device, vfioDevice config.VFIODrive) []govmmQemu.Device
// handleImagePath handles the Hypervisor Config image path
handleImagePath(config HypervisorConfig)
@ -286,7 +287,7 @@ func (q *qemuArchBase) appendImage(devices []govmmQemu.Device, path string) ([]g
id := utils.MakeNameID("image", hex.EncodeToString(randBytes), maxDevIDSize)
drive := drivers.Drive{
drive := config.BlockDrive{
File: path,
Format: "raw",
ID: id,
@ -430,7 +431,7 @@ func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoi
return devices
}
func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive drivers.Drive) []govmmQemu.Device {
func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device {
if drive.File == "" || drive.ID == "" || drive.Format == "" {
return devices
}
@ -476,14 +477,14 @@ func (q *qemuArchBase) appendVhostUserDevice(devices []govmmQemu.Device, vhostUs
return devices
}
func (q *qemuArchBase) appendVFIODevice(devices []govmmQemu.Device, vfioDevice drivers.VFIODevice) []govmmQemu.Device {
if vfioDevice.BDF == "" {
func (q *qemuArchBase) appendVFIODevice(devices []govmmQemu.Device, vfioDrive config.VFIODrive) []govmmQemu.Device {
if vfioDrive.BDF == "" {
return devices
}
devices = append(devices,
govmmQemu.VFIODevice{
BDF: vfioDevice.BDF,
BDF: vfioDrive.BDF,
},
)

2
virtcontainers/sandbox.go
View File

@ -758,7 +758,7 @@ func newSandbox(sandboxConfig SandboxConfig, factory Factory) (*Sandbox, error)
storage: &filesystem{},
network: network,
config: &sandboxConfig,
devManager: deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver),
devManager: deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, nil),
volumes: sandboxConfig.Volumes,
containers: map[string]*Container{},
runPath: filepath.Join(runStoragePath, sandboxConfig.ID),