diff --git a/virtcontainers/container.go b/virtcontainers/container.go index 941b7d3968..8b2e127afe 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -440,8 +440,7 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) ( if c.checkBlockDeviceSupport() && stat.Mode&unix.S_IFBLK == unix.S_IFBLK { // TODO: remove dependency of package drivers b := &drivers.BlockDevice{ - DevType: config.DeviceBlock, - DeviceInfo: config.DeviceInfo{ + DeviceInfo: &config.DeviceInfo{ HostPath: m.Source, ContainerPath: m.Destination, DevType: "b", @@ -574,12 +573,13 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err } else { // If devices were not found in storage, create Device implementations // from the configuration. This should happen at create. - - devices, err := sandbox.devManager.NewDevices(contConfig.DeviceInfos) - if err != nil { - return &Container{}, err + for _, info := range contConfig.DeviceInfos { + dev, err := sandbox.devManager.NewDevice(info) + if err != nil { + return &Container{}, err + } + c.devices = append(c.devices, dev) } - c.devices = devices } return c, nil } @@ -1022,7 +1022,7 @@ func (c *Container) hotplugDrive() error { // Add drive with id as container id devID := utils.MakeNameID("drive", c.id, maxDevIDSize) - drive := drivers.Drive{ + drive := config.BlockDrive{ File: devicePath, Format: "raw", ID: devID, @@ -1059,7 +1059,7 @@ func (c *Container) removeDrive() (err error) { c.Logger().Info("unplugging block device") devID := utils.MakeNameID("drive", c.id, maxDevIDSize) - drive := &drivers.Drive{ + drive := &config.BlockDrive{ ID: devID, } diff --git a/virtcontainers/device/api/interface.go b/virtcontainers/device/api/interface.go index d6a4072d33..7f75bd64e5 100644 --- a/virtcontainers/device/api/interface.go +++ b/virtcontainers/device/api/interface.go @@ -27,6 +27,7 @@ func DeviceLogger() *logrus.Entry { // DeviceReceiver is an interface used for accepting devices // a device should be attached/added/plugged to a DeviceReceiver type DeviceReceiver interface { + // these are for hotplug/hot-unplug devices to/from hypervisor HotplugAddDevice(Device, config.DeviceType) error HotplugRemoveDevice(Device, config.DeviceType) error @@ -51,11 +52,29 @@ type VhostUserDevice interface { type Device interface { Attach(DeviceReceiver) error Detach(DeviceReceiver) error + // ID returns device identifier + DeviceID() string + // DeviceType indicates which kind of device it is + // e.g. block, vfio or vhost user DeviceType() config.DeviceType + // GetDeviceInfo returns device information that the device is created based on + GetDeviceInfo() *config.DeviceInfo + // GetDeviceDrive returns device specific data used for hotplugging by hypervisor + // Caller could cast the return value to device specific struct + // e.g. Block device returns *config.BlockDrive and + // vfio device returns *config.VFIODrive + GetDeviceDrive() interface{} + // IsAttached checks if the device is attached + IsAttached() bool } // DeviceManager can be used to create a new device, this can be used as single // device management object. type DeviceManager interface { - NewDevices(devInfos []config.DeviceInfo) ([]Device, error) + NewDevice(config.DeviceInfo) (Device, error) + AttachDevice(string, DeviceReceiver) error + DetachDevice(string, DeviceReceiver) error + IsDeviceAttached(string) bool + GetDeviceByID(string) Device + GetAllDevices() []Device } diff --git a/virtcontainers/device/config/config.go b/virtcontainers/device/config/config.go index 1ab83ddc91..c099a5cfca 100644 --- a/virtcontainers/device/config/config.go +++ b/virtcontainers/device/config/config.go @@ -49,10 +49,10 @@ var SysIOMMUPath = "/sys/kernel/iommu_groups" // DeviceInfo is an embedded type that contains device data common to all types of devices. type DeviceInfo struct { - // Device path on host + // Hostpath is device path on host HostPath string - // Device path inside the container + // ContainerPath is device path inside container ContainerPath string // Type of device: c, b, u or p @@ -87,6 +87,40 @@ type DeviceInfo struct { DriverOptions map[string]string } +// BlockDrive represents a block storage drive which may be used in case the storage +// driver has an underlying block storage device. +type BlockDrive struct { + // File is the path to the disk-image/device which will be used with this drive + File string + + // Format of the drive + Format string + + // ID is used to identify this drive in the hypervisor options. + ID string + + // Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index + Index int + + // PCIAddr is the PCI address used to identify the slot at which the drive is attached. + PCIAddr string + + // SCSI Address of the block device, in case the device is attached using SCSI driver + // SCSI address is in the format SCSI-Id:LUN + SCSIAddr string + + // VirtPath at which the device appears inside the VM, outside of the container mount namespace + VirtPath string +} + +// VFIODrive represents a VFIO drive used for hotplugging +type VFIODrive struct { + // ID is used to identify this drive in the hypervisor options. + ID string + // BDF (Bus:Device.Function) of the PCI address + BDF string +} + // VhostUserDeviceAttrs represents data shared by most vhost-user devices type VhostUserDeviceAttrs struct { DevType DeviceType diff --git a/virtcontainers/device/drivers/block.go b/virtcontainers/device/drivers/block.go index 175dc8cea2..da41cf2ecb 100644 --- a/virtcontainers/device/drivers/block.go +++ b/virtcontainers/device/drivers/block.go @@ -7,7 +7,6 @@ package drivers import ( - "encoding/hex" "path/filepath" "github.com/kata-containers/runtime/virtcontainers/device/api" @@ -17,48 +16,17 @@ import ( const maxDevIDSize = 31 -// Drive represents a block storage drive which may be used in case the storage -// driver has an underlying block storage device. -type Drive struct { - - // Path to the disk-image/device which will be used with this drive - File string - - // Format of the drive - Format string - - // ID is used to identify this drive in the hypervisor options. - ID string - - // Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index - Index int - - // PCIAddr is the PCI address used to identify the slot at which the drive is attached. - PCIAddr string -} - // BlockDevice refers to a block storage device implementation. type BlockDevice struct { - DevType config.DeviceType - DeviceInfo config.DeviceInfo - - // SCSI Address of the block device, in case the device is attached using SCSI driver - // SCSI address is in the format SCSI-Id:LUN - SCSIAddr string - - // Path at which the device appears inside the VM, outside of the container mount namespace - VirtPath string - - // PCI Slot of the block device - PCIAddr string - - BlockDrive *Drive + ID string + DeviceInfo *config.DeviceInfo + BlockDrive *config.BlockDrive } // NewBlockDevice creates a new block device based on DeviceInfo -func NewBlockDevice(devInfo config.DeviceInfo) *BlockDevice { +func NewBlockDevice(devInfo *config.DeviceInfo) *BlockDevice { return &BlockDevice{ - DevType: config.DeviceBlock, + ID: devInfo.ID, DeviceInfo: devInfo, } } @@ -66,13 +34,6 @@ func NewBlockDevice(devInfo config.DeviceInfo) *BlockDevice { // Attach is standard interface of api.Device, it's used to add device to some // DeviceReceiver func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) { - randBytes, err := utils.GenerateRandomBytes(8) - if err != nil { - return err - } - - device.DeviceInfo.ID = hex.EncodeToString(randBytes) - // Increment the block index for the sandbox. This is used to determine the name // for the block device in the case where the block device is used as container // rootfs and the predicted block device name needs to be provided to the agent. @@ -88,21 +49,13 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) { return err } - drive := Drive{ + drive := &config.BlockDrive{ File: device.DeviceInfo.HostPath, Format: "raw", ID: utils.MakeNameID("drive", device.DeviceInfo.ID, maxDevIDSize), Index: index, } - deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device") - device.BlockDrive = &drive - if err = devReceiver.HotplugAddDevice(device, config.DeviceBlock); err != nil { - return err - } - - device.DeviceInfo.Hotplugged = true - driveName, err := utils.GetVirtDriveName(index) if err != nil { return err @@ -110,36 +63,61 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) { customOptions := device.DeviceInfo.DriverOptions if customOptions != nil && customOptions["block-driver"] == "virtio-blk" { - device.VirtPath = filepath.Join("/dev", driveName) - device.PCIAddr = drive.PCIAddr + drive.VirtPath = filepath.Join("/dev", driveName) } else { scsiAddr, err := utils.GetSCSIAddress(index) if err != nil { return err } - device.SCSIAddr = scsiAddr + drive.SCSIAddr = scsiAddr } + deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device") + device.BlockDrive = drive + if err = devReceiver.HotplugAddDevice(device, config.DeviceBlock); err != nil { + return err + } + + device.DeviceInfo.Hotplugged = true + return nil } // Detach is standard interface of api.Device, it's used to remove device from some // DeviceReceiver func (device *BlockDevice) Detach(devReceiver api.DeviceReceiver) error { - if device.DeviceInfo.Hotplugged { - deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device") - - if err := devReceiver.HotplugRemoveDevice(device, config.DeviceBlock); err != nil { - deviceLogger().WithError(err).Error("Failed to unplug block device") - return err - } + deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device") + if err := devReceiver.HotplugRemoveDevice(device, config.DeviceBlock); err != nil { + deviceLogger().WithError(err).Error("Failed to unplug block device") + return err } + device.DeviceInfo.Hotplugged = false return nil } +// IsAttached checks if the device is attached +func (device *BlockDevice) IsAttached() bool { + return device.DeviceInfo.Hotplugged +} + // DeviceType is standard interface of api.Device, it returns device type func (device *BlockDevice) DeviceType() config.DeviceType { - return device.DevType + return config.DeviceBlock +} + +// DeviceID returns device ID +func (device *BlockDevice) DeviceID() string { + return device.ID +} + +// GetDeviceInfo returns device information that the device is created based on +func (device *BlockDevice) GetDeviceInfo() *config.DeviceInfo { + return device.DeviceInfo +} + +// GetDeviceDrive returns device information used for creating +func (device *BlockDevice) GetDeviceDrive() interface{} { + return device.BlockDrive } diff --git a/virtcontainers/device/drivers/generic.go b/virtcontainers/device/drivers/generic.go index 9609d811f5..e339af2105 100644 --- a/virtcontainers/device/drivers/generic.go +++ b/virtcontainers/device/drivers/generic.go @@ -13,14 +13,14 @@ import ( // GenericDevice refers to a device that is neither a VFIO device or block device. type GenericDevice struct { - DevType config.DeviceType - DeviceInfo config.DeviceInfo + ID string + DeviceInfo *config.DeviceInfo } // NewGenericDevice creates a new GenericDevice -func NewGenericDevice(devInfo config.DeviceInfo) *GenericDevice { +func NewGenericDevice(devInfo *config.DeviceInfo) *GenericDevice { return &GenericDevice{ - DevType: config.DeviceGeneric, + ID: devInfo.ID, DeviceInfo: devInfo, } } @@ -35,7 +35,27 @@ func (device *GenericDevice) Detach(devReceiver api.DeviceReceiver) error { return nil } +// IsAttached checks if the device is attached +func (device *GenericDevice) IsAttached() bool { + return device.DeviceInfo.Hotplugged +} + +// DeviceID returns device ID +func (device *GenericDevice) DeviceID() string { + return device.ID +} + // DeviceType is standard interface of api.Device, it returns device type func (device *GenericDevice) DeviceType() config.DeviceType { - return device.DevType + return config.DeviceGeneric +} + +// GetDeviceInfo returns device information that the device is created based on +func (device *GenericDevice) GetDeviceInfo() *config.DeviceInfo { + return device.DeviceInfo +} + +// GetDeviceDrive returns device information used for creating +func (device *GenericDevice) GetDeviceDrive() interface{} { + return device.DeviceInfo } diff --git a/virtcontainers/device/drivers/vfio.go b/virtcontainers/device/drivers/vfio.go index 4243465a30..313411ab1c 100644 --- a/virtcontainers/device/drivers/vfio.go +++ b/virtcontainers/device/drivers/vfio.go @@ -7,7 +7,6 @@ package drivers import ( - "encoding/hex" "fmt" "io/ioutil" "path/filepath" @@ -31,15 +30,15 @@ const ( // VFIODevice is a vfio device meant to be passed to the hypervisor // to be used by the Virtual Machine. type VFIODevice struct { - DevType config.DeviceType - DeviceInfo config.DeviceInfo - BDF string + ID string + DeviceInfo *config.DeviceInfo + vfioDrives []*config.VFIODrive } // NewVFIODevice create a new VFIO device -func NewVFIODevice(devInfo config.DeviceInfo) *VFIODevice { +func NewVFIODevice(devInfo *config.DeviceInfo) *VFIODevice { return &VFIODevice{ - DevType: config.DeviceVFIO, + ID: devInfo.ID, DeviceInfo: devInfo, } } @@ -57,44 +56,62 @@ func (device *VFIODevice) Attach(devReceiver api.DeviceReceiver) error { // Pass all devices in iommu group for _, deviceFile := range deviceFiles { - //Get bdf of device eg 0000:00:1c.0 deviceBDF, err := getBDF(deviceFile.Name()) if err != nil { return err } - - device.BDF = deviceBDF - - randBytes, err := utils.GenerateRandomBytes(8) - if err != nil { - return err + vfio := &config.VFIODrive{ + ID: utils.MakeNameID("vfio", device.DeviceInfo.ID, maxDevIDSize), + BDF: deviceBDF, } - device.DeviceInfo.ID = hex.EncodeToString(randBytes) - - if err := devReceiver.HotplugAddDevice(device, config.DeviceVFIO); err != nil { - deviceLogger().WithError(err).Error("Failed to add device") - return err - } - - deviceLogger().WithFields(logrus.Fields{ - "device-group": device.DeviceInfo.HostPath, - "device-type": "vfio-passthrough", - }).Info("Device group attached") + device.vfioDrives = append(device.vfioDrives, vfio) } + // hotplug a VFIO device is actually hotplugging a group of iommu devices + if err := devReceiver.HotplugAddDevice(device, config.DeviceVFIO); err != nil { + deviceLogger().WithError(err).Error("Failed to add device") + return err + } + + deviceLogger().WithFields(logrus.Fields{ + "device-group": device.DeviceInfo.HostPath, + "device-type": "vfio-passthrough", + }).Info("Device group attached") + device.DeviceInfo.Hotplugged = true return nil } // Detach is standard interface of api.Device, it's used to remove device from some // DeviceReceiver func (device *VFIODevice) Detach(devReceiver api.DeviceReceiver) error { + device.DeviceInfo.Hotplugged = false return nil } +// IsAttached checks if the device is attached +func (device *VFIODevice) IsAttached() bool { + return device.DeviceInfo.Hotplugged +} + // DeviceType is standard interface of api.Device, it returns device type func (device *VFIODevice) DeviceType() config.DeviceType { - return device.DevType + return config.DeviceVFIO +} + +// DeviceID returns device ID +func (device *VFIODevice) DeviceID() string { + return device.ID +} + +// GetDeviceInfo returns device information that the device is created based on +func (device *VFIODevice) GetDeviceInfo() *config.DeviceInfo { + return device.DeviceInfo +} + +// GetDeviceDrive returns device information used for creating +func (device *VFIODevice) GetDeviceDrive() interface{} { + return device.vfioDrives } // getBDF returns the BDF of pci device diff --git a/virtcontainers/device/manager/manager.go b/virtcontainers/device/manager/manager.go index 0e30e6e5b2..e14a53d266 100644 --- a/virtcontainers/device/manager/manager.go +++ b/virtcontainers/device/manager/manager.go @@ -7,11 +7,16 @@ package manager import ( + "encoding/hex" + "errors" + "sync" + "github.com/sirupsen/logrus" "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" + "github.com/kata-containers/runtime/virtcontainers/utils" ) const ( @@ -21,59 +26,162 @@ const ( VirtioSCSI string = "virtio-scsi" ) +var ( + // ErrIDExhausted represents that devices are too many + // and no more IDs can be generated + ErrIDExhausted = errors.New("IDs are exhausted") + // ErrDeviceNotExist represents device hasn't been created before + ErrDeviceNotExist = errors.New("device with specified ID hasn't been created") + // ErrDeviceAttached represents the device is already attached + ErrDeviceAttached = errors.New("device is already attached") + // ErrDeviceNotAttached represents the device isn't attached + ErrDeviceNotAttached = errors.New("device isn't attached") +) + type deviceManager struct { blockDriver string + + devices map[string]api.Device + sync.RWMutex } func deviceLogger() *logrus.Entry { return api.DeviceLogger().WithField("subsystem", "device") } +// NewDeviceManager creates a deviceManager object behaved as api.DeviceManager +func NewDeviceManager(blockDriver string, devices []api.Device) api.DeviceManager { + dm := &deviceManager{ + devices: make(map[string]api.Device), + } + if blockDriver == VirtioBlock { + dm.blockDriver = VirtioBlock + } else { + dm.blockDriver = VirtioSCSI + } + + for _, dev := range devices { + dm.devices[dev.DeviceID()] = dev + } + return dm +} + // createDevice creates one device based on DeviceInfo func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (api.Device, error) { path, err := config.GetHostPathFunc(devInfo) if err != nil { return nil, err } - devInfo.HostPath = path + + // device ID must be generated by manager instead of device itself + // in case of ID collision + if devInfo.ID, err = dm.newDeviceID(); err != nil { + return nil, err + } if isVFIO(path) { - return drivers.NewVFIODevice(devInfo), nil + return drivers.NewVFIODevice(&devInfo), nil } else if isBlock(devInfo) { if devInfo.DriverOptions == nil { devInfo.DriverOptions = make(map[string]string) } devInfo.DriverOptions["block-driver"] = dm.blockDriver - return drivers.NewBlockDevice(devInfo), nil + return drivers.NewBlockDevice(&devInfo), nil } else { deviceLogger().WithField("device", path).Info("Device has not been passed to the container") - return drivers.NewGenericDevice(devInfo), nil + return drivers.NewGenericDevice(&devInfo), nil } } -// NewDevices creates bundles of devices based on array of DeviceInfo -func (dm *deviceManager) NewDevices(devInfos []config.DeviceInfo) ([]api.Device, error) { - var devices []api.Device +// NewDevice creates bundles of devices based on array of DeviceInfo +func (dm *deviceManager) NewDevice(devInfo config.DeviceInfo) (api.Device, error) { + dm.Lock() + defer dm.Unlock() + dev, err := dm.createDevice(devInfo) + if err == nil { + dm.devices[dev.DeviceID()] = dev + } + return dev, err +} - for _, devInfo := range devInfos { - device, err := dm.createDevice(devInfo) +func (dm *deviceManager) newDeviceID() (string, error) { + for i := 0; i < 5; i++ { + // generate an random ID + randBytes, err := utils.GenerateRandomBytes(8) if err != nil { - return nil, err + return "", err } - devices = append(devices, device) - } + id := hex.EncodeToString(randBytes) - return devices, nil + // check ID collision, choose another one if ID is in use + if _, ok := dm.devices[id]; !ok { + return id, nil + } + } + return "", ErrIDExhausted } -// NewDeviceManager creates a deviceManager object behaved as api.DeviceManager -func NewDeviceManager(blockDriver string) api.DeviceManager { - dm := &deviceManager{} - if blockDriver == VirtioBlock { - dm.blockDriver = VirtioBlock - } else { - dm.blockDriver = VirtioSCSI +func (dm *deviceManager) AttachDevice(id string, dr api.DeviceReceiver) error { + dm.Lock() + defer dm.Unlock() + + d, ok := dm.devices[id] + if !ok { + return ErrDeviceNotExist } - return dm + if d.IsAttached() { + return ErrDeviceAttached + } + + if err := d.Attach(dr); err != nil { + return err + } + return nil +} + +func (dm *deviceManager) DetachDevice(id string, dr api.DeviceReceiver) error { + dm.Lock() + defer dm.Unlock() + + d, ok := dm.devices[id] + if !ok { + return ErrDeviceNotExist + } + if !d.IsAttached() { + return ErrDeviceNotAttached + } + + if err := d.Detach(dr); err != nil { + return err + } + return nil +} +func (dm *deviceManager) GetDeviceByID(id string) api.Device { + dm.RLock() + defer dm.RUnlock() + if d, ok := dm.devices[id]; ok { + return d + } + return nil +} + +func (dm *deviceManager) GetAllDevices() []api.Device { + dm.RLock() + defer dm.RUnlock() + devices := []api.Device{} + for _, v := range dm.devices { + devices = append(devices, v) + } + return devices +} + +func (dm *deviceManager) IsDeviceAttached(id string) bool { + dm.RLock() + defer dm.RUnlock() + d, ok := dm.devices[id] + if !ok { + return false + } + return d.IsAttached() } diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go index cd2f9a358c..c60397c3a7 100644 --- a/virtcontainers/hyperstart_agent.go +++ b/virtcontainers/hyperstart_agent.go @@ -14,15 +14,16 @@ import ( "syscall" "time" + "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" + proxyClient "github.com/clearcontainers/proxy/client" + "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" "github.com/kata-containers/runtime/virtcontainers/pkg/hyperstart" ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" "github.com/kata-containers/runtime/virtcontainers/utils" specs "github.com/opencontainers/runtime-spec/specs-go" - - "github.com/sirupsen/logrus" - "github.com/vishvananda/netlink" ) var defaultSockPathTemplates = []string{"%s/%s/hyper.sock", "%s/%s/tty.sock"} @@ -231,6 +232,32 @@ func fsMapFromMounts(mounts []Mount) []*hyperstart.FsmapDescriptor { return fsmap } +func fsMapFromDevices(c *Container) ([]*hyperstart.FsmapDescriptor, error) { + var fsmap []*hyperstart.FsmapDescriptor + for _, dev := range c.devices { + device := c.sandbox.devManager.GetDeviceByID(dev.DeviceID()) + if device == nil { + return nil, fmt.Errorf("can't find device: %#v", dev) + } + blockDev := device.(*drivers.BlockDevice) + + d, ok := blockDev.GetDeviceDrive().(*config.BlockDrive) + if !ok || d == nil { + return nil, fmt.Errorf("can't retrieve block device information") + } + + fsmapDesc := &hyperstart.FsmapDescriptor{ + Source: d.VirtPath, + Path: blockDev.DeviceInfo.ContainerPath, + AbsolutePath: true, + DockerVolume: false, + SCSIAddr: d.SCSIAddr, + } + fsmap = append(fsmap, fsmapDesc) + } + return fsmap, nil +} + // init is the agent initialization implementation for hyperstart. func (h *hyper) init(sandbox *Sandbox, config interface{}) (err error) { switch c := config.(type) { @@ -497,20 +524,11 @@ func (h *hyper) startOneContainer(sandbox *Sandbox, c *Container) error { h.handleBlockVolumes(c) // Append container mounts for block devices passed with --device. - for _, device := range c.devices { - d, ok := device.(*drivers.BlockDevice) - - if ok { - fsmapDesc := &hyperstart.FsmapDescriptor{ - Source: d.VirtPath, - Path: d.DeviceInfo.ContainerPath, - AbsolutePath: true, - DockerVolume: false, - SCSIAddr: d.SCSIAddr, - } - fsmap = append(fsmap, fsmapDesc) - } + fsmapDev, err := fsMapFromDevices(c) + if err != nil { + return err } + fsmap = append(fsmap, fsmapDev...) // Assign fsmap for hyperstart to mount these at the correct location within the container container.Fsmap = fsmap diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 831862af22..e6ff9c6d00 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -733,12 +733,12 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, devices []api.Devic ContainerPath: d.DeviceInfo.ContainerPath, } - if d.SCSIAddr == "" { + if d.BlockDrive.SCSIAddr == "" { kataDevice.Type = kataBlkDevType - kataDevice.Id = d.PCIAddr + kataDevice.Id = d.BlockDrive.PCIAddr } else { kataDevice.Type = kataSCSIDevType - kataDevice.Id = d.SCSIAddr + kataDevice.Id = d.BlockDrive.SCSIAddr } deviceList = append(deviceList, kataDevice) @@ -970,10 +970,10 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage { if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock { vol.Driver = kataBlkDevType - vol.Source = b.VirtPath + vol.Source = b.BlockDrive.VirtPath } else { vol.Driver = kataSCSIDevType - vol.Source = b.SCSIAddr + vol.Source = b.BlockDrive.SCSIAddr } vol.MountPoint = b.DeviceInfo.ContainerPath diff --git a/virtcontainers/network.go b/virtcontainers/network.go index 15401af655..ef76157407 100644 --- a/virtcontainers/network.go +++ b/virtcontainers/network.go @@ -25,6 +25,7 @@ import ( "github.com/vishvananda/netns" "golang.org/x/sys/unix" + "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" "github.com/kata-containers/runtime/virtcontainers/utils" @@ -343,7 +344,8 @@ func (endpoint *PhysicalEndpoint) Attach(h hypervisor) error { return err } - d := drivers.VFIODevice{ + // TODO: use device manager as general device management entrance + d := config.VFIODrive{ BDF: endpoint.BDF, } diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 1d84fa88d7..8a932939c7 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -20,7 +20,7 @@ import ( "github.com/sirupsen/logrus" "github.com/kata-containers/runtime/virtcontainers/device/api" - deviceDrivers "github.com/kata-containers/runtime/virtcontainers/device/drivers" + "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -656,7 +656,7 @@ func (q *qemu) removeDeviceFromBridge(ID string) error { return err } -func (q *qemu) hotplugBlockDevice(drive *deviceDrivers.Drive, op operation) error { +func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error { err := q.qmpSetup() if err != nil { return err @@ -717,13 +717,13 @@ func (q *qemu) hotplugBlockDevice(drive *deviceDrivers.Drive, op operation) erro return nil } -func (q *qemu) hotplugVFIODevice(device deviceDrivers.VFIODevice, op operation) error { +func (q *qemu) hotplugVFIODevice(device *config.VFIODrive, op operation) error { err := q.qmpSetup() if err != nil { return err } - devID := "vfio-" + device.DeviceInfo.ID + devID := device.ID if op == addDevice { addr, bridge, err := q.addDeviceToBridge(devID) @@ -750,15 +750,13 @@ func (q *qemu) hotplugVFIODevice(device deviceDrivers.VFIODevice, op operation) func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operation) (interface{}, error) { switch devType { case blockDev: - // TODO: find a way to remove dependency of deviceDrivers lib @weizhang555 - drive := devInfo.(*deviceDrivers.Drive) + drive := devInfo.(*config.BlockDrive) return nil, q.hotplugBlockDevice(drive, op) case cpuDev: vcpus := devInfo.(uint32) return q.hotplugCPUs(vcpus, op) case vfioDev: - // TODO: find a way to remove dependency of deviceDrivers lib @weizhang555 - device := devInfo.(deviceDrivers.VFIODevice) + device := devInfo.(*config.VFIODrive) return nil, q.hotplugVFIODevice(device, op) case memoryDev: memdev := devInfo.(*memoryDevice) @@ -958,10 +956,10 @@ func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error { q.qemuConfig.Devices = q.arch.appendSocket(q.qemuConfig.Devices, v) case Endpoint: q.qemuConfig.Devices = q.arch.appendNetwork(q.qemuConfig.Devices, v) - case deviceDrivers.Drive: + case config.BlockDrive: q.qemuConfig.Devices = q.arch.appendBlockDevice(q.qemuConfig.Devices, v) - case deviceDrivers.VFIODevice: + case config.VFIODrive: q.qemuConfig.Devices = q.arch.appendVFIODevice(q.qemuConfig.Devices, v) default: break diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index e0188c4d3d..53356b9d4c 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -14,6 +14,7 @@ import ( govmmQemu "github.com/intel/govmm/qemu" "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -72,13 +73,13 @@ type qemuArch interface { appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device // appendBlockDevice appends a block drive to devices - appendBlockDevice(devices []govmmQemu.Device, drive drivers.Drive) []govmmQemu.Device + appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device // appendVhostUserDevice appends a vhost user device to devices appendVhostUserDevice(devices []govmmQemu.Device, vhostUserDevice api.VhostUserDevice) []govmmQemu.Device // appendVFIODevice appends a VFIO device to devices - appendVFIODevice(devices []govmmQemu.Device, vfioDevice drivers.VFIODevice) []govmmQemu.Device + appendVFIODevice(devices []govmmQemu.Device, vfioDevice config.VFIODrive) []govmmQemu.Device // handleImagePath handles the Hypervisor Config image path handleImagePath(config HypervisorConfig) @@ -286,7 +287,7 @@ func (q *qemuArchBase) appendImage(devices []govmmQemu.Device, path string) ([]g id := utils.MakeNameID("image", hex.EncodeToString(randBytes), maxDevIDSize) - drive := drivers.Drive{ + drive := config.BlockDrive{ File: path, Format: "raw", ID: id, @@ -430,7 +431,7 @@ func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoi return devices } -func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive drivers.Drive) []govmmQemu.Device { +func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device { if drive.File == "" || drive.ID == "" || drive.Format == "" { return devices } @@ -476,14 +477,14 @@ func (q *qemuArchBase) appendVhostUserDevice(devices []govmmQemu.Device, vhostUs return devices } -func (q *qemuArchBase) appendVFIODevice(devices []govmmQemu.Device, vfioDevice drivers.VFIODevice) []govmmQemu.Device { - if vfioDevice.BDF == "" { +func (q *qemuArchBase) appendVFIODevice(devices []govmmQemu.Device, vfioDrive config.VFIODrive) []govmmQemu.Device { + if vfioDrive.BDF == "" { return devices } devices = append(devices, govmmQemu.VFIODevice{ - BDF: vfioDevice.BDF, + BDF: vfioDrive.BDF, }, ) diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index b3339d252f..f6a63eb23b 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -758,7 +758,7 @@ func newSandbox(sandboxConfig SandboxConfig, factory Factory) (*Sandbox, error) storage: &filesystem{}, network: network, config: &sandboxConfig, - devManager: deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver), + devManager: deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, nil), volumes: sandboxConfig.Volumes, containers: map[string]*Container{}, runPath: filepath.Join(runStoragePath, sandboxConfig.ID),