github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-06-30 09:13:29 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1965 from jschintag/virtio-blk-ccw

Browse Source 
s390x: add virtio-blk-ccw support
This commit is contained in:
James O. D. Hunt 2019-09-05 08:46:08 +01:00 committed by GitHub
commit 94c47dcecd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 621 additions and 319 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/katautils/config.go
View File

@ -340,7 +340,7 @@ func (h hypervisor) defaultBridges() uint32 {
}
func (h hypervisor) blockDeviceDriver() (string, error) {
supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm}
supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm, config.VirtioBlockCCW}
if h.BlockDeviceDriver == "" {
return defaultBlockDeviceDriver, nil

6
virtcontainers/device/config/config.go
View File

@ -48,6 +48,9 @@ const (
// VirtioBlock means use virtio-blk for hotplugging drives
VirtioBlock = "virtio-blk"
// VirtioBlockCCW means use virtio-blk for hotplugging drives
VirtioBlockCCW = "virtio-blk-ccw"
// VirtioSCSI means use virtio-scsi for hotplugging drives
VirtioSCSI = "virtio-scsi"
@ -138,6 +141,9 @@ type BlockDrive struct {
// VirtPath at which the device appears inside the VM, outside of the container mount namespace
VirtPath string
// DevNo identifies the css bus id for virtio-blk-ccw
DevNo string
}
// VFIODeviceType indicates VFIO device type

4
virtcontainers/device/drivers/block.go
View File

@ -84,6 +84,8 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
switch customOptions["block-driver"] {
case "virtio-blk":
globalIdx = index
case "virtio-blk-ccw":
globalIdx = index
case "virtio-mmio":
//With firecracker the rootfs for the VM itself
//sits at /dev/vda and consumes the first index.
@ -164,6 +166,7 @@ func (device *BlockDevice) Save() persistapi.DeviceState {
SCSIAddr: drive.SCSIAddr,
NvdimmID: drive.NvdimmID,
VirtPath: drive.VirtPath,
DevNo: drive.DevNo,
}
}
return ds
@ -188,6 +191,7 @@ func (device *BlockDevice) Load(ds persistapi.DeviceState) {
SCSIAddr: bd.SCSIAddr,
NvdimmID: bd.NvdimmID,
VirtPath: bd.VirtPath,
DevNo: bd.DevNo,
}
}

4
virtcontainers/device/manager/manager.go
View File

@ -25,6 +25,8 @@ const (
VirtioMmio string = "virtio-mmio"
// VirtioBlock indicates block driver is virtio-blk based
VirtioBlock string = "virtio-blk"
// VirtioBlockCCW indicates block driver is virtio-blk-ccw based
VirtioBlockCCW string = "virtio-blk-ccw"
// VirtioSCSI indicates block driver is virtio-scsi based
VirtioSCSI string = "virtio-scsi"
// Nvdimm indicates block driver is nvdimm based
@ -66,6 +68,8 @@ func NewDeviceManager(blockDriver string, devices []api.Device) api.DeviceManage
dm.blockDriver = VirtioBlock
} else if blockDriver == Nvdimm {
dm.blockDriver = Nvdimm
} else if blockDriver == VirtioBlockCCW {
dm.blockDriver = VirtioBlockCCW
} else {
dm.blockDriver = VirtioSCSI
}

44
virtcontainers/kata_agent.go
View File

@ -75,6 +75,7 @@ var (
kata9pDevType = "9p"
kataMmioBlkDevType = "mmioblk"
kataBlkDevType = "blk"
kataBlkCCWDevType = "blk-ccw"
kataSCSIDevType = "scsi"
kataNvdimmDevType = "nvdimm"
kataVirtioFSDevType = "virtio-fs"
@ -1106,6 +1107,9 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, c *Container) []*gr
kataDevice.Type = kataMmioBlkDevType
kataDevice.Id = d.VirtPath
kataDevice.VmPath = d.VirtPath
case config.VirtioBlockCCW:
kataDevice.Type = kataBlkCCWDevType
kataDevice.Id = d.DevNo
case config.VirtioBlock:
kataDevice.Type = kataBlkDevType
kataDevice.Id = d.PCIAddr
@ -1160,11 +1164,14 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat
k.Logger().Error("malformed block drive")
return nil, fmt.Errorf("malformed block drive")
}
if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio {
switch {
case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio:
rootfs.Driver = kataMmioBlkDevType
rootfs.Source = blockDrive.VirtPath
} else if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock {
case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlockCCW:
rootfs.Driver = kataBlkCCWDevType
rootfs.Source = blockDrive.DevNo
case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock:
rootfs.Driver = kataBlkDevType
if blockDrive.PCIAddr == "" {
rootfs.Source = blockDrive.VirtPath
@ -1172,10 +1179,14 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat
rootfs.Source = blockDrive.PCIAddr
}
} else {
case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioSCSI:
rootfs.Driver = kataSCSIDevType
rootfs.Source = blockDrive.SCSIAddr
default:
return nil, fmt.Errorf("Unknown block device driver: %s", sandbox.config.HypervisorConfig.BlockDeviceDriver)
}
rootfs.MountPoint = rootPathParent
rootfs.Fstype = c.state.Fstype
@ -1281,7 +1292,10 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
// Note this call modifies the list of container devices to make sure
// all hotplugged devices are unplugged, so this needs be done
// after devices passed with --device are handled.
volumeStorages := k.handleBlockVolumes(c)
volumeStorages, err := k.handleBlockVolumes(c)
if err != nil {
return nil, err
}
if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil {
return nil, err
}
@ -1400,7 +1414,7 @@ func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, r
// handleBlockVolumes handles volumes that are block devices files
// by passing the block devices as Storage to the agent.
func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
var volumeStorages []*grpc.Storage
@ -1418,7 +1432,7 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
if !c.sandbox.supportNewStore() {
if err := c.storeDevices(); err != nil {
k.Logger().WithField("device", id).WithError(err).Error("store device failed")
return nil
return nil, err
}
}
@ -1427,22 +1441,28 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
device := c.sandbox.devManager.GetDeviceByID(id)
if device == nil {
k.Logger().WithField("device", id).Error("failed to find device by id")
return nil
return nil, fmt.Errorf("Failed to find device by id (id=%s)", id)
}
blockDrive, ok := device.GetDeviceInfo().(*config.BlockDrive)
if !ok || blockDrive == nil {
k.Logger().Error("malformed block drive")
continue
}
if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock {
switch {
case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlockCCW:
vol.Driver = kataBlkCCWDevType
vol.Source = blockDrive.DevNo
case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock:
vol.Driver = kataBlkDevType
vol.Source = blockDrive.PCIAddr
} else if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio {
case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio:
vol.Driver = kataMmioBlkDevType
vol.Source = blockDrive.VirtPath
} else {
case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioSCSI:
vol.Driver = kataSCSIDevType
vol.Source = blockDrive.SCSIAddr
default:
return nil, fmt.Errorf("Unknown block device driver: %s", c.sandbox.config.HypervisorConfig.BlockDeviceDriver)
}
vol.MountPoint = m.Destination
@ -1452,7 +1472,7 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
volumeStorages = append(volumeStorages, vol)
}
return volumeStorages
return volumeStorages, nil
}
// handlePidNamespace checks if Pid namespace for a container needs to be shared with its sandbox

3
virtcontainers/persist/api/device.go
View File

@ -38,6 +38,9 @@ type BlockDrive struct {
// VirtPath at which the device appears inside the VM, outside of the container mount namespace
VirtPath string
// DevNo
DevNo string
}
// VFIODev represents a VFIO drive used for hotplugging

120
virtcontainers/qemu.go
View File

@ -61,7 +61,7 @@ type CPUDevice struct {
// QemuState keeps Qemu's state
type QemuState struct {
Bridges []types.PCIBridge
Bridges []types.Bridge
// HotpluggedCPUs is the list of CPUs that were hot-added
HotpluggedVCPUs []CPUDevice
HotpluggedMemory int
@ -266,9 +266,11 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto
create = true
}
q.arch.setBridges(q.state.Bridges)
if create {
q.Logger().Debug("Creating bridges")
q.state.Bridges = q.arch.bridges(q.config.DefaultBridges)
q.arch.bridges(q.config.DefaultBridges)
q.Logger().Debug("Creating UUID")
q.state.UUID = uuid.Generate().String()
@ -403,7 +405,7 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
// Add bridges before any other devices. This way we make sure that
// bridge gets the first available PCI address i.e bridgePCIStartAddr
devices = q.arch.appendBridges(devices, q.state.Bridges)
devices = q.arch.appendBridges(devices)
devices = q.arch.appendConsole(devices, console)
@ -984,38 +986,6 @@ func (q *qemu) qmpShutdown() {
}
}
func (q *qemu) addDeviceToBridge(ID string) (string, types.PCIBridge, error) {
var err error
var addr uint32
if len(q.state.Bridges) == 0 {
return "", types.PCIBridge{}, errors.New("failed to get available address from bridges")
}
// looking for an empty address in the bridges
for _, b := range q.state.Bridges {
addr, err = b.AddDevice(ID)
if err == nil {
return fmt.Sprintf("%02x", addr), b, nil
}
}
return "", types.PCIBridge{}, fmt.Errorf("no more bridge slots available")
}
func (q *qemu) removeDeviceFromBridge(ID string) error {
var err error
for _, b := range q.state.Bridges {
err = b.RemoveDevice(ID)
if err == nil {
// device was removed correctly
return nil
}
}
return err
}
func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, devID string) (err error) {
if q.config.BlockDeviceDriver == config.Nvdimm {
var blocksize int64
@ -1050,16 +1020,36 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
}
}()
if q.config.BlockDeviceDriver == config.VirtioBlock {
switch {
case q.config.BlockDeviceDriver == config.VirtioBlockCCW:
driver := "virtio-blk-ccw"
addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.CCW)
if err != nil {
return err
}
var devNoHotplug string
devNoHotplug, err = bridge.AddressFormatCCW(addr)
if err != nil {
return err
}
drive.DevNo, err = bridge.AddressFormatCCWForVirtServer(addr)
if err != nil {
return err
}
if err = q.qmpMonitorCh.qmp.ExecuteDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, devNoHotplug, "", true, false); err != nil {
return err
}
case q.config.BlockDeviceDriver == config.VirtioBlock:
driver := "virtio-blk-pci"
addr, bridge, err := q.addDeviceToBridge(drive.ID)
addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.PCI)
if err != nil {
return err
}
defer func() {
if err != nil {
q.removeDeviceFromBridge(drive.ID)
q.arch.removeDeviceFromBridge(drive.ID)
}
}()
@ -1069,7 +1059,7 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil {
return err
}
} else {
case q.config.BlockDeviceDriver == config.VirtioSCSI:
driver := "scsi-hd"
// Bus exposed by the SCSI Controller
@ -1084,6 +1074,8 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
if err = q.qmpMonitorCh.qmp.ExecuteSCSIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, bus, romFile, scsiID, lun, true, defaultDisableModern); err != nil {
return err
}
default:
return fmt.Errorf("Block device %s not recognized", q.config.BlockDeviceDriver)
}
return nil
@ -1101,7 +1093,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
err = q.hotplugAddBlockDevice(drive, op, devID)
} else {
if q.config.BlockDeviceDriver == config.VirtioBlock {
if err := q.removeDeviceFromBridge(drive.ID); err != nil {
if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil {
return err
}
}
@ -1141,14 +1133,14 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
}
}
addr, bridge, err := q.addDeviceToBridge(devID)
addr, bridge, err := q.arch.addDeviceToBridge(devID, types.PCI)
if err != nil {
return err
}
defer func() {
if err != nil {
q.removeDeviceFromBridge(devID)
q.arch.removeDeviceFromBridge(devID)
}
}()
@ -1162,7 +1154,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
}
} else {
if !q.state.HotplugVFIOOnRootBus {
if err := q.removeDeviceFromBridge(devID); err != nil {
if err := q.arch.removeDeviceFromBridge(devID); err != nil {
return err
}
}
@ -1228,14 +1220,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
}
}()
addr, bridge, err := q.addDeviceToBridge(tap.ID)
addr, bridge, err := q.arch.addDeviceToBridge(tap.ID, types.PCI)
if err != nil {
return err
}
defer func() {
if err != nil {
q.removeDeviceFromBridge(tap.ID)
q.arch.removeDeviceFromBridge(tap.ID)
}
}()
@ -1248,12 +1240,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
return err
}
if machine.Type == QemuCCWVirtio {
return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), bridge.ID, int(q.config.NumVCPUs))
devNoHotplug := fmt.Sprintf("fe.%x.%x", bridge.Addr, addr)
return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs))
}
return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, romFile, int(q.config.NumVCPUs), defaultDisableModern)
}
if err := q.removeDeviceFromBridge(tap.ID); err != nil {
if err := q.arch.removeDeviceFromBridge(tap.ID); err != nil {
return err
}
@ -1750,7 +1744,7 @@ func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe boo
// genericAppendBridges appends to devices the given bridges
// nolint: unused, deadcode
func genericAppendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge, machineType string) []govmmQemu.Device {
func genericAppendBridges(devices []govmmQemu.Device, bridges []types.Bridge, machineType string) []govmmQemu.Device {
bus := defaultPCBridgeBus
switch machineType {
case QemuQ35, QemuVirt:
@ -1762,6 +1756,9 @@ func genericAppendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge,
if b.Type == types.PCIE {
t = govmmQemu.PCIEBridge
}
if b.Type == types.CCW {
continue
}
bridges[idx].Addr = bridgePCIStartAddr + idx
@ -1781,10 +1778,9 @@ func genericAppendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge,
return devices
}
// nolint: unused, deadcode
func genericBridges(number uint32, machineType string) []types.PCIBridge {
var bridges []types.PCIBridge
var bt types.PCIType
func genericBridges(number uint32, machineType string) []types.Bridge {
var bridges []types.Bridge
var bt types.Type
switch machineType {
case QemuQ35:
@ -1798,17 +1794,13 @@ func genericBridges(number uint32, machineType string) []types.PCIBridge {
case QemuPseries:
bt = types.PCI
case QemuCCWVirtio:
bt = types.PCI
bt = types.CCW
default:
return nil
}
for i := uint32(0); i < number; i++ {
bridges = append(bridges, types.PCIBridge{
Type: bt,
ID: fmt.Sprintf("%s-bridge-%d", bt, i),
Address: make(map[uint32]string),
})
bridges = append(bridges, types.NewBridge(bt, fmt.Sprintf("%s-bridge-%d", bt, i), make(map[uint32]string), 0))
}
return bridges
@ -1990,6 +1982,7 @@ func (q *qemu) toGrpc() ([]byte, error) {
func (q *qemu) storeState() error {
if q.store != nil {
q.state.Bridges = q.arch.getBridges()
if err := q.store.Store(store.Hypervisor, q.state); err != nil {
return err
}
@ -2008,9 +2001,9 @@ func (q *qemu) save() (s persistapi.HypervisorState) {
s.HotpluggedMemory = q.state.HotpluggedMemory
s.HotplugVFIOOnRootBus = q.state.HotplugVFIOOnRootBus
for _, bridge := range q.state.Bridges {
for _, bridge := range q.arch.getBridges() {
s.Bridges = append(s.Bridges, persistapi.Bridge{
DeviceAddr: bridge.Address,
DeviceAddr: bridge.Devices,
Type: string(bridge.Type),
ID: bridge.ID,
Addr: bridge.Addr,
@ -2032,12 +2025,7 @@ func (q *qemu) load(s persistapi.HypervisorState) {
q.state.VirtiofsdPid = s.VirtiofsdPid
for _, bridge := range s.Bridges {
q.state.Bridges = append(q.state.Bridges, types.PCIBridge{
Address: bridge.DeviceAddr,
Type: types.PCIType(bridge.Type),
ID: bridge.ID,
Addr: bridge.Addr,
})
q.state.Bridges = append(q.state.Bridges, types.NewBridge(types.Type(bridge.Type), bridge.ID, bridge.DeviceAddr, bridge.Addr))
}
for _, cpu := range s.HotpluggedVCPUs {

8
virtcontainers/qemu_amd64.go
View File

@ -122,8 +122,8 @@ func (q *qemuAmd64) capabilities() types.Capabilities {
return caps
}
func (q *qemuAmd64) bridges(number uint32) []types.PCIBridge {
return genericBridges(number, q.machineType)
func (q *qemuAmd64) bridges(number uint32) {
q.Bridges = genericBridges(number, q.machineType)
}
func (q *qemuAmd64) cpuModel() string {
@ -173,6 +173,6 @@ func (q *qemuAmd64) appendImage(devices []govmmQemu.Device, path string) ([]govm
}
// appendBridges appends to devices the given bridges
func (q *qemuAmd64) appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device {
return genericAppendBridges(devices, bridges, q.machineType)
func (q *qemuAmd64) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device {
return genericAppendBridges(devices, q.Bridges, q.machineType)
}

23
virtcontainers/qemu_amd64_test.go
View File

@ -40,29 +40,32 @@ func TestQemuAmd64Bridges(t *testing.T) {
amd64 := newTestQemu(QemuPC)
len := 5
bridges := amd64.bridges(uint32(len))
amd64.bridges(uint32(len))
bridges := amd64.getBridges()
assert.Len(bridges, len)
for i, b := range bridges {
id := fmt.Sprintf("%s-bridge-%d", types.PCI, i)
assert.Equal(types.PCI, b.Type)
assert.Equal(id, b.ID)
assert.NotNil(b.Address)
assert.NotNil(b.Devices)
}
amd64 = newTestQemu(QemuQ35)
bridges = amd64.bridges(uint32(len))
amd64.bridges(uint32(len))
bridges = amd64.getBridges()
assert.Len(bridges, len)
for i, b := range bridges {
id := fmt.Sprintf("%s-bridge-%d", types.PCI, i)
assert.Equal(types.PCI, b.Type)
assert.Equal(id, b.ID)
assert.NotNil(b.Address)
assert.NotNil(b.Devices)
}
amd64 = newTestQemu(QemuQ35 + QemuPC)
bridges = amd64.bridges(uint32(len))
amd64.bridges(uint32(len))
bridges = amd64.getBridges()
assert.Nil(bridges)
}
@ -143,10 +146,11 @@ func TestQemuAmd64AppendBridges(t *testing.T) {
// check PC
amd64 := newTestQemu(QemuPC)
bridges := amd64.bridges(1)
amd64.bridges(1)
bridges := amd64.getBridges()
assert.Len(bridges, 1)
devices = amd64.appendBridges(devices, bridges)
devices = amd64.appendBridges(devices)
assert.Len(devices, 1)
expectedOut := []govmmQemu.Device{
@ -165,11 +169,12 @@ func TestQemuAmd64AppendBridges(t *testing.T) {
// Check Q35
amd64 = newTestQemu(QemuQ35)
bridges = amd64.bridges(1)
amd64.bridges(1)
bridges = amd64.getBridges()
assert.Len(bridges, 1)
devices = []govmmQemu.Device{}
devices = amd64.appendBridges(devices, bridges)
devices = amd64.appendBridges(devices)
assert.Len(devices, 1)
expectedOut = []govmmQemu.Device{

263
virtcontainers/qemu_arch_base.go
View File

@ -8,6 +8,7 @@ package virtcontainers
import (
"context"
"encoding/hex"
"errors"
"fmt"
"os"
"strconv"
@ -48,8 +49,8 @@ type qemuArch interface {
//capabilities returns the capabilities supported by QEMU
capabilities() types.Capabilities
// bridges returns the number bridges for the machine type
bridges(number uint32) []types.PCIBridge
// bridges sets the number bridges for the machine type
bridges(number uint32)
// cpuTopology returns the CPU topology for the given amount of vcpus
cpuTopology(vcpus, maxvcpus uint32) govmmQemu.SMP
@ -70,7 +71,7 @@ type qemuArch interface {
appendSCSIController(devices []govmmQemu.Device, enableIOThreads bool) ([]govmmQemu.Device, *govmmQemu.IOThread)
// appendBridges appends bridges to devices
appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device
appendBridges(devices []govmmQemu.Device) []govmmQemu.Device
// append9PVolume appends a 9P volume to devices
append9PVolume(devices []govmmQemu.Device, volume types.Volume) []govmmQemu.Device
@ -96,6 +97,21 @@ type qemuArch interface {
// appendRNGDevice appends a RNG device to devices
appendRNGDevice(devices []govmmQemu.Device, rngDevice config.RNGDev) []govmmQemu.Device
// addDeviceToBridge adds devices to the bus
addDeviceToBridge(ID string, t types.Type) (string, types.Bridge, error)
// removeDeviceFromBridge removes devices to the bus
removeDeviceFromBridge(ID string) error
// getBridges grants access to Bridges
getBridges() []types.Bridge
// setBridges grants access to Bridges
setBridges(bridges []types.Bridge)
// addBridge adds a new Bridge to the list of Bridges
addBridge(types.Bridge)
// handleImagePath handles the Hypervisor Config image path
handleImagePath(config HypervisorConfig)
@ -117,6 +133,7 @@ type qemuArchBase struct {
kernelParamsNonDebug []Param
kernelParamsDebug []Param
kernelParams []Param
Bridges []types.Bridge
}
const (
@ -242,18 +259,10 @@ func (q *qemuArchBase) capabilities() types.Capabilities {
return caps
}
func (q *qemuArchBase) bridges(number uint32) []types.PCIBridge {
var bridges []types.PCIBridge
func (q *qemuArchBase) bridges(number uint32) {
for i := uint32(0); i < number; i++ {
bridges = append(bridges, types.PCIBridge{
Type: types.PCI,
ID: fmt.Sprintf("%s-bridge-%d", types.PCI, i),
Address: make(map[uint32]string),
})
q.Bridges = append(q.Bridges, types.NewBridge(types.PCI, fmt.Sprintf("%s-bridge-%d", types.PCI, i), make(map[uint32]string), 0))
}
return bridges
}
func (q *qemuArchBase) cpuTopology(vcpus, maxvcpus uint32) govmmQemu.SMP {
@ -306,14 +315,14 @@ func (q *qemuArchBase) appendConsole(devices []govmmQemu.Device, path string) []
return devices
}
func (q *qemuArchBase) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) {
func genericImage(path string) (config.BlockDrive, error) {
if _, err := os.Stat(path); os.IsNotExist(err) {
return nil, err
return config.BlockDrive{}, err
}
randBytes, err := utils.GenerateRandomBytes(8)
if err != nil {
return nil, err
return config.BlockDrive{}, err
}
id := utils.MakeNameID("image", hex.EncodeToString(randBytes), maxDevIDSize)
@ -324,13 +333,21 @@ func (q *qemuArchBase) appendImage(devices []govmmQemu.Device, path string) ([]g
ID: id,
}
return drive, nil
}
func (q *qemuArchBase) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) {
drive, err := genericImage(path)
if err != nil {
return nil, err
}
return q.appendBlockDevice(devices, drive), nil
}
func (q *qemuArchBase) appendSCSIController(devices []govmmQemu.Device, enableIOThreads bool) ([]govmmQemu.Device, *govmmQemu.IOThread) {
func genericSCSIController(enableIOThreads, nestedRun bool) (govmmQemu.SCSIController, *govmmQemu.IOThread) {
scsiController := govmmQemu.SCSIController{
ID: scsiControllerID,
DisableModern: q.nestedRun,
DisableModern: nestedRun,
}
var t *govmmQemu.IOThread
@ -345,20 +362,27 @@ func (q *qemuArchBase) appendSCSIController(devices []govmmQemu.Device, enableIO
scsiController.IOThread = t.ID
}
devices = append(devices, scsiController)
return scsiController, t
}
func (q *qemuArchBase) appendSCSIController(devices []govmmQemu.Device, enableIOThreads bool) ([]govmmQemu.Device, *govmmQemu.IOThread) {
d, t := genericSCSIController(enableIOThreads, q.nestedRun)
devices = append(devices, d)
return devices, t
}
// appendBridges appends to devices the given bridges
func (q *qemuArchBase) appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device {
for idx, b := range bridges {
func (q *qemuArchBase) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device {
for idx, b := range q.Bridges {
if b.Type == types.CCW {
continue
}
t := govmmQemu.PCIBridge
if b.Type == types.PCIE {
t = govmmQemu.PCIEBridge
}
bridges[idx].Addr = bridgePCIStartAddr + idx
q.Bridges[idx].Addr = bridgePCIStartAddr + idx
devices = append(devices,
govmmQemu.BridgeDevice{
@ -368,7 +392,7 @@ func (q *qemuArchBase) appendBridges(devices []govmmQemu.Device, bridges []types
// Each bridge is required to be assigned a unique chassis id > 0
Chassis: idx + 1,
SHPC: true,
Addr: strconv.FormatInt(int64(bridges[idx].Addr), 10),
Addr: strconv.FormatInt(int64(q.Bridges[idx].Addr), 10),
},
)
}
@ -376,28 +400,30 @@ func (q *qemuArchBase) appendBridges(devices []govmmQemu.Device, bridges []types
return devices
}
func (q *qemuArchBase) append9PVolume(devices []govmmQemu.Device, volume types.Volume) []govmmQemu.Device {
if volume.MountTag == "" || volume.HostPath == "" {
return devices
}
func generic9PVolume(volume types.Volume, nestedRun bool) govmmQemu.FSDevice {
devID := fmt.Sprintf("extra-9p-%s", volume.MountTag)
if len(devID) > maxDevIDSize {
devID = devID[:maxDevIDSize]
}
devices = append(devices,
govmmQemu.FSDevice{
Driver: govmmQemu.Virtio9P,
FSDriver: govmmQemu.Local,
ID: devID,
Path: volume.HostPath,
MountTag: volume.MountTag,
SecurityModel: govmmQemu.None,
DisableModern: q.nestedRun,
},
)
return govmmQemu.FSDevice{
Driver: govmmQemu.Virtio9P,
FSDriver: govmmQemu.Local,
ID: devID,
Path: volume.HostPath,
MountTag: volume.MountTag,
SecurityModel: govmmQemu.None,
DisableModern: nestedRun,
}
}
func (q *qemuArchBase) append9PVolume(devices []govmmQemu.Device, volume types.Volume) []govmmQemu.Device {
if volume.MountTag == "" || volume.HostPath == "" {
return devices
}
d := generic9PVolume(volume, q.nestedRun)
devices = append(devices, d)
return devices
}
@ -452,70 +478,83 @@ func networkModelToQemuType(model NetInterworkingModel) govmmQemu.NetDeviceType
}
}
func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device {
func genericNetwork(endpoint Endpoint, vhost, nestedRun bool, index int) (govmmQemu.NetDevice, error) {
var d govmmQemu.NetDevice
switch ep := endpoint.(type) {
case *VethEndpoint, *BridgedMacvlanEndpoint, *IPVlanEndpoint:
netPair := ep.NetworkPair()
devices = append(devices,
govmmQemu.NetDevice{
Type: networkModelToQemuType(netPair.NetInterworkingModel),
Driver: govmmQemu.VirtioNet,
ID: fmt.Sprintf("network-%d", q.networkIndex),
IFName: netPair.TAPIface.Name,
MACAddress: netPair.TAPIface.HardAddr,
DownScript: "no",
Script: "no",
VHost: q.vhost,
DisableModern: q.nestedRun,
FDs: netPair.VMFds,
VhostFDs: netPair.VhostFds,
},
)
q.networkIndex++
d = govmmQemu.NetDevice{
Type: networkModelToQemuType(netPair.NetInterworkingModel),
Driver: govmmQemu.VirtioNet,
ID: fmt.Sprintf("network-%d", index),
IFName: netPair.TAPIface.Name,
MACAddress: netPair.TAPIface.HardAddr,
DownScript: "no",
Script: "no",
VHost: vhost,
DisableModern: nestedRun,
FDs: netPair.VMFds,
VhostFDs: netPair.VhostFds,
}
case *MacvtapEndpoint:
devices = append(devices,
govmmQemu.NetDevice{
Type: govmmQemu.MACVTAP,
Driver: govmmQemu.VirtioNet,
ID: fmt.Sprintf("network-%d", q.networkIndex),
IFName: ep.Name(),
MACAddress: ep.HardwareAddr(),
DownScript: "no",
Script: "no",
VHost: q.vhost,
DisableModern: q.nestedRun,
FDs: ep.VMFds,
VhostFDs: ep.VhostFds,
},
)
q.networkIndex++
d = govmmQemu.NetDevice{
Type: govmmQemu.MACVTAP,
Driver: govmmQemu.VirtioNet,
ID: fmt.Sprintf("network-%d", index),
IFName: ep.Name(),
MACAddress: ep.HardwareAddr(),
DownScript: "no",
Script: "no",
VHost: vhost,
DisableModern: nestedRun,
FDs: ep.VMFds,
VhostFDs: ep.VhostFds,
}
default:
return govmmQemu.NetDevice{}, fmt.Errorf("Unknown type for endpoint")
}
return d, nil
}
func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device {
d, err := genericNetwork(endpoint, q.vhost, q.nestedRun, q.networkIndex)
if err != nil {
virtLog.WithField("subsystem", "qemuArch").WithError(err).Error("Failed to append network")
return devices
}
q.networkIndex++
devices = append(devices, d)
return devices
}
func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device {
func genericBlockDevice(drive config.BlockDrive, nestedRun bool) (govmmQemu.BlockDevice, error) {
if drive.File == "" || drive.ID == "" || drive.Format == "" {
return devices
return govmmQemu.BlockDevice{}, fmt.Errorf("Empty File, ID or Format for drive %v", drive)
}
if len(drive.ID) > maxDevIDSize {
drive.ID = drive.ID[:maxDevIDSize]
}
devices = append(devices,
govmmQemu.BlockDevice{
Driver: govmmQemu.VirtioBlock,
ID: drive.ID,
File: drive.File,
AIO: govmmQemu.Threads,
Format: govmmQemu.BlockDeviceFormat(drive.Format),
Interface: "none",
DisableModern: q.nestedRun,
},
)
return govmmQemu.BlockDevice{
Driver: govmmQemu.VirtioBlock,
ID: drive.ID,
File: drive.File,
AIO: govmmQemu.Threads,
Format: govmmQemu.BlockDeviceFormat(drive.Format),
Interface: "none",
DisableModern: nestedRun,
}, nil
}
func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device {
d, err := genericBlockDevice(drive, q.nestedRun)
if err != nil {
virtLog.WithField("subsystem", "qemuArch").WithError(err).Error("Failed to append block device")
return devices
}
devices = append(devices, d)
return devices
}
@ -592,3 +631,55 @@ func (q *qemuArchBase) setIgnoreSharedMemoryMigrationCaps(ctx context.Context, q
})
return err
}
func (q *qemuArchBase) addDeviceToBridge(ID string, t types.Type) (string, types.Bridge, error) {
var err error
var addr uint32
if len(q.Bridges) == 0 {
return "", types.Bridge{}, errors.New("failed to get available address from bridges")
}
// looking for an empty address in the bridges
for _, b := range q.Bridges {
if t != b.Type {
continue
}
addr, err = b.AddDevice(ID)
if err == nil {
switch t {
case types.CCW:
return fmt.Sprintf("%04x", addr), b, nil
case types.PCI, types.PCIE:
return fmt.Sprintf("%02x", addr), b, nil
}
}
}
return "", types.Bridge{}, fmt.Errorf("no more bridge slots available")
}
func (q *qemuArchBase) removeDeviceFromBridge(ID string) error {
var err error
for _, b := range q.Bridges {
err = b.RemoveDevice(ID)
if err == nil {
// device was removed correctly
return nil
}
}
return err
}
func (q *qemuArchBase) getBridges() []types.Bridge {
return q.Bridges
}
func (q *qemuArchBase) setBridges(bridges []types.Bridge) {
q.Bridges = bridges
}
func (q *qemuArchBase) addBridge(b types.Bridge) {
q.Bridges = append(q.Bridges, b)
}

40
virtcontainers/qemu_arch_base_test.go
View File

@ -18,6 +18,7 @@ import (
"github.com/kata-containers/runtime/virtcontainers/device/config"
"github.com/kata-containers/runtime/virtcontainers/store"
"github.com/kata-containers/runtime/virtcontainers/types"
"github.com/pkg/errors"
)
const (
@ -147,14 +148,44 @@ func TestQemuArchBaseBridges(t *testing.T) {
qemuArchBase := newQemuArchBase()
len := 5
bridges := qemuArchBase.bridges(uint32(len))
qemuArchBase.bridges(uint32(len))
bridges := qemuArchBase.getBridges()
assert.Len(bridges, len)
for i, b := range bridges {
id := fmt.Sprintf("%s-bridge-%d", types.PCI, i)
assert.Equal(types.PCI, b.Type)
assert.Equal(id, b.ID)
assert.NotNil(b.Address)
assert.NotNil(b.Devices)
}
}
func TestQemuAddDeviceToBridge(t *testing.T) {
assert := assert.New(t)
// addDeviceToBridge successfully
q := newQemuArchBase()
q.machineType = QemuPC
q.bridges(1)
for i := uint32(1); i <= types.PCIBridgeMaxCapacity; i++ {
_, _, err := q.addDeviceToBridge(fmt.Sprintf("qemu-bridge-%d", i), types.PCI)
assert.Nil(err)
}
// fail to add device to bridge cause no more available bridge slot
_, _, err := q.addDeviceToBridge("qemu-bridge-31", types.PCI)
exceptErr := errors.New("no more bridge slots available")
assert.Equal(exceptErr.Error(), err.Error())
// addDeviceToBridge fails cause q.Bridges == 0
q = newQemuArchBase()
q.machineType = QemuPCLite
q.bridges(0)
_, _, err = q.addDeviceToBridge("qemu-bridge", types.PCI)
if assert.Error(err) {
exceptErr = errors.New("failed to get available address from bridges")
assert.Equal(exceptErr.Error(), err.Error())
}
}
@ -283,10 +314,11 @@ func TestQemuArchBaseAppendBridges(t *testing.T) {
assert := assert.New(t)
qemuArchBase := newQemuArchBase()
bridges := qemuArchBase.bridges(1)
qemuArchBase.bridges(1)
bridges := qemuArchBase.getBridges()
assert.Len(bridges, 1)
devices = qemuArchBase.appendBridges(devices, bridges)
devices = qemuArchBase.appendBridges(devices)
assert.Len(devices, 1)
expectedOut := []govmmQemu.Device{

9
virtcontainers/qemu_arm64.go
View File

@ -14,7 +14,6 @@ import (
"time"
govmmQemu "github.com/intel/govmm/qemu"
"github.com/kata-containers/runtime/virtcontainers/types"
"github.com/sirupsen/logrus"
)
@ -162,13 +161,13 @@ func newQemuArch(config HypervisorConfig) qemuArch {
return q
}
func (q *qemuArm64) bridges(number uint32) []types.PCIBridge {
return genericBridges(number, q.machineType)
func (q *qemuArm64) bridges(number uint32) {
q.Bridges = genericBridges(number, q.machineType)
}
// appendBridges appends to devices the given bridges
func (q *qemuArm64) appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device {
return genericAppendBridges(devices, bridges, q.machineType)
func (q *qemuArm64) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device {
return genericAppendBridges(devices, q.Bridges, q.machineType)
}
func (q *qemuArm64) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) {

5
virtcontainers/qemu_arm64_test.go
View File

@ -105,11 +105,12 @@ func TestQemuArm64AppendBridges(t *testing.T) {
arm64 := newTestQemu(QemuVirt)
bridges := arm64.bridges(1)
arm64.bridges(1)
bridges := arm64.getBridges()
assert.Len(bridges, 1)
devices = []govmmQemu.Device{}
devices = arm64.appendBridges(devices, bridges)
devices = arm64.appendBridges(devices)
assert.Len(devices, 1)
expectedOut := []govmmQemu.Device{

8
virtcontainers/qemu_ppc64le.go
View File

@ -105,8 +105,8 @@ func (q *qemuPPC64le) capabilities() types.Capabilities {
return caps
}
func (q *qemuPPC64le) bridges(number uint32) []types.PCIBridge {
return genericBridges(number, q.machineType)
func (q *qemuPPC64le) bridges(number uint32) {
q.Bridges = genericBridges(number, q.machineType)
}
func (q *qemuPPC64le) cpuModel() string {
@ -152,6 +152,6 @@ func (q *qemuPPC64le) appendImage(devices []govmmQemu.Device, path string) ([]go
}
// appendBridges appends to devices the given bridges
func (q *qemuPPC64le) appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device {
return genericAppendBridges(devices, bridges, q.machineType)
func (q *qemuPPC64le) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device {
return genericAppendBridges(devices, q.Bridges, q.machineType)
}

147
virtcontainers/qemu_s390x.go
View File

@ -40,6 +40,8 @@ var kernelParams = []Param{
var kernelRootParams = commonVirtioblkKernelRootParams
var ccwbridge = types.NewBridge(types.CCW, "", make(map[uint32]string, types.CCWBridgeMaxCapacity), 0)
var supportedQemuMachines = []govmmQemu.Machine{
{
Type: QemuCCWVirtio,
@ -72,6 +74,8 @@ func newQemuArch(config HypervisorConfig) qemuArch {
kernelParams: kernelParams,
},
}
// Set first bridge type to CCW
q.Bridges = append(q.Bridges, ccwbridge)
if config.ImagePath != "" {
q.kernelParams = append(q.kernelParams, kernelRootParams...)
@ -82,22 +86,32 @@ func newQemuArch(config HypervisorConfig) qemuArch {
return q
}
func (q *qemuS390x) bridges(number uint32) []types.PCIBridge {
return genericBridges(number, q.machineType)
}
// appendBridges appends to devices the given bridges
func (q *qemuS390x) appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device {
return genericAppendBridges(devices, bridges, q.machineType)
func (q *qemuS390x) bridges(number uint32) {
q.Bridges = genericBridges(number, q.machineType)
}
// appendConsole appends a console to devices.
// The function has been overwriten to correctly set the driver to the CCW device
func (q *qemuS390x) appendConsole(devices []govmmQemu.Device, path string) []govmmQemu.Device {
id := "serial0"
addr, b, err := q.addDeviceToBridge(id, types.CCW)
if err != nil {
virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append console")
return devices
}
var devno string
devno, err = b.AddressFormatCCW(addr)
if err != nil {
virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append console")
return devices
}
serial := govmmQemu.SerialDevice{
Driver: virtioSerialCCW,
ID: "serial0",
ID: id,
DisableModern: q.nestedRun,
DevNo: devno,
}
devices = append(devices, serial)
@ -115,6 +129,36 @@ func (q *qemuS390x) appendConsole(devices []govmmQemu.Device, path string) []gov
return devices
}
func (q *qemuS390x) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) {
drive, err := genericImage(path)
if err != nil {
virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append image")
return nil, err
}
return q.appendBlockDevice(devices, drive), nil
}
func (q *qemuS390x) appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device {
d, err := genericBlockDevice(drive, false)
if err != nil {
virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append blk-dev")
return devices
}
addr, b, err := q.addDeviceToBridge(drive.ID, types.CCW)
if err != nil {
virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append blk-dev")
return devices
}
d.DevNo, err = b.AddressFormatCCW(addr)
if err != nil {
virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append blk-dev")
return devices
}
devices = append(devices, d)
return devices
}
// appendVhostUserDevice throws an error if vhost devices are tried to be used.
// See issue https://github.com/kata-containers/runtime/issues/659
func (q *qemuS390x) appendVhostUserDevice(devices []govmmQemu.Device, attr config.VhostUserDeviceAttrs) ([]govmmQemu.Device, error) {
@ -126,3 +170,90 @@ func (q *qemuS390x) appendVhostUserDevice(devices []govmmQemu.Device, attr confi
func (q *qemuS390x) supportGuestMemoryHotplug() bool {
return false
}
func (q *qemuS390x) appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device {
d, err := genericNetwork(endpoint, false, false, q.networkIndex)
if err != nil {
virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append network")
return devices
}
q.networkIndex++
addr, b, err := q.addDeviceToBridge(d.ID, types.CCW)
if err != nil {
virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append network")
return devices
}
d.DevNo, err = b.AddressFormatCCW(addr)
if err != nil {
virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append network")
return devices
}
devices = append(devices, d)
return devices
}
func (q *qemuS390x) appendRNGDevice(devices []govmmQemu.Device, rngDev config.RNGDev) []govmmQemu.Device {
addr, b, err := q.addDeviceToBridge(rngDev.ID, types.CCW)
if err != nil {
virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append RNG-Device")
return devices
}
var devno string
devno, err = b.AddressFormatCCW(addr)
if err != nil {
virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append RNG-Device")
return devices
}
devices = append(devices,
govmmQemu.RngDevice{
ID: rngDev.ID,
Filename: rngDev.Filename,
DevNo: devno,
},
)
return devices
}
func (q *qemuS390x) append9PVolume(devices []govmmQemu.Device, volume types.Volume) []govmmQemu.Device {
if volume.MountTag == "" || volume.HostPath == "" {
return devices
}
d := generic9PVolume(volume, false)
addr, b, err := q.addDeviceToBridge(d.ID, types.CCW)
if err != nil {
virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append 9p-Volume")
return devices
}
d.DevNo, err = b.AddressFormatCCW(addr)
if err != nil {
virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append 9p-Volume")
return devices
}
devices = append(devices, d)
return devices
}
// appendBridges appends to devices the given bridges
func (q *qemuS390x) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device {
return genericAppendBridges(devices, q.Bridges, q.machineType)
}
func (q *qemuS390x) appendSCSIController(devices []govmmQemu.Device, enableIOThreads bool) ([]govmmQemu.Device, *govmmQemu.IOThread) {
d, t := genericSCSIController(enableIOThreads, q.nestedRun)
addr, b, err := q.addDeviceToBridge(d.ID, types.CCW)
if err != nil {
virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append scsi-controller")
return devices, nil
}
d.DevNo, err = b.AddressFormatCCW(addr)
if err != nil {
virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append scsi-controller")
return devices, nil
}
devices = append(devices, d)
return devices, t
}

38
virtcontainers/qemu_test.go
View File

@ -421,44 +421,6 @@ func TestQemuGrpc(t *testing.T) {
assert.True(q.id == q2.id)
}
func TestQemuAddDeviceToBridge(t *testing.T) {
assert := assert.New(t)
config := newQemuConfig()
config.DefaultBridges = defaultBridges
// addDeviceToBridge successfully
config.HypervisorMachineType = QemuPC
q := &qemu{
config: config,
arch: newQemuArch(config),
}
q.state.Bridges = q.arch.bridges(q.config.DefaultBridges)
// get pciBridgeMaxCapacity value from virtcontainers/types/pci.go
const pciBridgeMaxCapacity = 30
for i := uint32(1); i <= pciBridgeMaxCapacity; i++ {
_, _, err := q.addDeviceToBridge(fmt.Sprintf("qemu-bridge-%d", i))
assert.Nil(err)
}
// fail to add device to bridge cause no more available bridge slot
_, _, err := q.addDeviceToBridge("qemu-bridge-31")
exceptErr := errors.New("no more bridge slots available")
assert.Equal(exceptErr.Error(), err.Error())
// addDeviceToBridge fails cause q.state.Bridges == 0
config.HypervisorMachineType = QemuPCLite
q = &qemu{
config: config,
arch: newQemuArch(config),
}
q.state.Bridges = q.arch.bridges(q.config.DefaultBridges)
_, _, err = q.addDeviceToBridge("qemu-bridge")
exceptErr = errors.New("failed to get available address from bridges")
assert.Equal(exceptErr.Error(), err.Error())
}
func TestQemuFileBackedMem(t *testing.T) {
assert := assert.New(t)

115
virtcontainers/types/bridges.go Normal file
View File

@ -0,0 +1,115 @@
// Copyright (c) 2017 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
//
package types
import "fmt"
// Type represents a type of bus and bridge.
type Type string
const PCIBridgeMaxCapacity = 30
const (
// PCI represents a PCI bus and bridge
PCI Type = "pci"
// PCIE represents a PCIe bus and bridge
PCIE Type = "pcie"
)
const CCWBridgeMaxCapacity = 0xffff
const (
CCW Type = "ccw"
)
type Bridge struct {
// Devices contains information about devices plugged and its address in the bridge
Devices map[uint32]string
// ID is used to identify the bridge in the hypervisor
ID string
// Addr is the slot of the bridge
Addr int
// Type is the type of the bridge (pci, pcie, etc)
Type Type
// MaxCapacity is the max capacity of the bridge
MaxCapacity uint32
}
func NewBridge(bt Type, id string, devices map[uint32]string, addr int) Bridge {
var maxCapacity uint32
switch bt {
case PCI:
fallthrough
case PCIE:
maxCapacity = PCIBridgeMaxCapacity
case CCW:
maxCapacity = CCWBridgeMaxCapacity
default:
maxCapacity = 0
}
return Bridge{
Devices: devices,
ID: id,
Addr: addr,
Type: bt,
MaxCapacity: maxCapacity,
}
}
func (b *Bridge) AddDevice(ID string) (uint32, error) {
var addr uint32
// looking for the first available address
for i := uint32(1); i <= b.MaxCapacity; i++ {
if _, ok := b.Devices[i]; !ok {
addr = i
break
}
}
if addr == 0 {
return 0, fmt.Errorf("Unable to hot plug device on bridge: there are no empty slots")
}
// save address and device
b.Devices[addr] = ID
return addr, nil
}
func (b *Bridge) RemoveDevice(ID string) error {
// check if the device was hot plugged in the bridge
for addr, devID := range b.Devices {
if devID == ID {
// free address to re-use the same slot with other devices
delete(b.Devices, addr)
return nil
}
}
return fmt.Errorf("Unable to hot unplug device %s: not present on bridge", ID)
}
// AddressFormatCCW returns the address format for the device number. The channel subsystem-ID 0xfe is reserved to the virtual channel and the address format is in the form fe.n.dddd, where n is subchannel set ID and ddd the device number. More details at https://www.ibm.com/support/knowledgecenter/en/linuxonibm/com.ibm.linux.z.ldva/ldva_t_configuringSCSIdevices.html
func (b *Bridge) AddressFormatCCW(addr string) (string, error) {
if b.Type != CCW {
return "", fmt.Errorf("Expected bridge type %T, got %T (%+v)", CCW, b.Type, b)
}
return fmt.Sprintf("fe.%x.%s", b.Addr, addr), nil
}
// AddressFormatCCWForVirtServer returns the address format for the virtual server. The address format is in the form of 0.n.dddd
func (b *Bridge) AddressFormatCCWForVirtServer(addr string) (string, error) {
if b.Type != CCW {
return "", fmt.Errorf("Wrong bridge type")
}
return fmt.Sprintf("0.%x.%s", b.Addr, addr), nil
}

29
virtcontainers/types/pci_test.go → virtcontainers/types/bridges_test.go
View File

@ -12,15 +12,12 @@ import (
"github.com/stretchr/testify/assert"
)
func TestAddRemoveDevice(t *testing.T) {
func testAddRemoveDevice(t *testing.T, b *Bridge) {
assert := assert.New(t)
// create a bridge
bridges := []*PCIBridge{{make(map[uint32]string), PCI, "rgb123", 5}}
// add device
devID := "abc123"
b := bridges[0]
addr, err := b.AddDevice(devID)
assert.NoError(err)
if addr < 1 {
@ -35,9 +32,9 @@ func TestAddRemoveDevice(t *testing.T) {
assert.NoError(err)
// add device when the bridge is full
bridges[0].Address = make(map[uint32]string)
for i := uint32(1); i <= pciBridgeMaxCapacity; i++ {
bridges[0].Address[i] = fmt.Sprintf("%d", i)
b.Devices = make(map[uint32]string)
for i := uint32(1); i <= b.MaxCapacity; i++ {
b.Devices[i] = fmt.Sprintf("%d", i)
}
addr, err = b.AddDevice(devID)
assert.Error(err)
@ -45,3 +42,19 @@ func TestAddRemoveDevice(t *testing.T) {
assert.Fail("address should be 0")
}
}
func TestAddRemoveDevicePCI(t *testing.T) {
// create a pci bridge
bridges := []*Bridge{{make(map[uint32]string), "rgb123", 5, PCI, PCIBridgeMaxCapacity}}
testAddRemoveDevice(t, bridges[0])
}
func TestAddRemoveDeviceCCW(t *testing.T) {
// create a CCW bridge
bridges := []*Bridge{{make(map[uint32]string), "rgb123", 5, CCW, CCWBridgeMaxCapacity}}
testAddRemoveDevice(t, bridges[0])
}

72
virtcontainers/types/pci.go
View File

@ -1,72 +0,0 @@
// Copyright (c) 2017 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
//
package types
import "fmt"
// PCIType represents a type of PCI bus and bridge.
type PCIType string
const (
// PCI represents a PCI bus and bridge
PCI PCIType = "pci"
// PCIE represents a PCIe bus and bridge
PCIE PCIType = "pcie"
)
const pciBridgeMaxCapacity = 30
// PCIBridge is a PCI or PCIe bridge where devices can be hot plugged
type PCIBridge struct {
// Address contains information about devices plugged and its address in the bridge
Address map[uint32]string
// Type is the PCI type of the bridge (pci, pcie, etc)
Type PCIType
// ID is used to identify the bridge in the hypervisor
ID string
// Addr is the PCI/e slot of the bridge
Addr int
}
// AddDevice on success adds the device ID to the PCI bridge and returns
// the address where the device was added.
func (b *PCIBridge) AddDevice(ID string) (uint32, error) {
var addr uint32
// looking for the first available address
for i := uint32(1); i <= pciBridgeMaxCapacity; i++ {
if _, ok := b.Address[i]; !ok {
addr = i
break
}
}
if addr == 0 {
return 0, fmt.Errorf("Unable to hot plug device on bridge: there are not empty slots")
}
// save address and device
b.Address[addr] = ID
return addr, nil
}
// RemoveDevice removes the device ID from the PCI bridge.
func (b *PCIBridge) RemoveDevice(ID string) error {
// check if the device was hot plugged in the bridge
for addr, devID := range b.Address {
if devID == ID {
// free address to re-use the same slot with other devices
delete(b.Address, addr)
return nil
}
}
return fmt.Errorf("Unable to hot unplug device %s: not present on bridge", ID)
}