diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index a920b57952..9c3432d2b5 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -340,7 +340,7 @@ func (h hypervisor) defaultBridges() uint32 { } func (h hypervisor) blockDeviceDriver() (string, error) { - supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm} + supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm, config.VirtioBlockCCW} if h.BlockDeviceDriver == "" { return defaultBlockDeviceDriver, nil diff --git a/virtcontainers/device/config/config.go b/virtcontainers/device/config/config.go index 93fedc1872..4eda53b997 100644 --- a/virtcontainers/device/config/config.go +++ b/virtcontainers/device/config/config.go @@ -48,6 +48,9 @@ const ( // VirtioBlock means use virtio-blk for hotplugging drives VirtioBlock = "virtio-blk" + // VirtioBlockCCW means use virtio-blk for hotplugging drives + VirtioBlockCCW = "virtio-blk-ccw" + // VirtioSCSI means use virtio-scsi for hotplugging drives VirtioSCSI = "virtio-scsi" @@ -138,6 +141,9 @@ type BlockDrive struct { // VirtPath at which the device appears inside the VM, outside of the container mount namespace VirtPath string + + // DevNo identifies the css bus id for virtio-blk-ccw + DevNo string } // VFIODeviceType indicates VFIO device type diff --git a/virtcontainers/device/drivers/block.go b/virtcontainers/device/drivers/block.go index 1e23bc75c5..0be1d6afe0 100644 --- a/virtcontainers/device/drivers/block.go +++ b/virtcontainers/device/drivers/block.go @@ -84,6 +84,8 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) { switch customOptions["block-driver"] { case "virtio-blk": globalIdx = index + case "virtio-blk-ccw": + globalIdx = index case "virtio-mmio": //With firecracker the rootfs for the VM itself //sits at /dev/vda and consumes the first index. @@ -164,6 +166,7 @@ func (device *BlockDevice) Save() persistapi.DeviceState { SCSIAddr: drive.SCSIAddr, NvdimmID: drive.NvdimmID, VirtPath: drive.VirtPath, + DevNo: drive.DevNo, } } return ds @@ -188,6 +191,7 @@ func (device *BlockDevice) Load(ds persistapi.DeviceState) { SCSIAddr: bd.SCSIAddr, NvdimmID: bd.NvdimmID, VirtPath: bd.VirtPath, + DevNo: bd.DevNo, } } diff --git a/virtcontainers/device/manager/manager.go b/virtcontainers/device/manager/manager.go index 6647b1c944..78a2c281bd 100644 --- a/virtcontainers/device/manager/manager.go +++ b/virtcontainers/device/manager/manager.go @@ -25,6 +25,8 @@ const ( VirtioMmio string = "virtio-mmio" // VirtioBlock indicates block driver is virtio-blk based VirtioBlock string = "virtio-blk" + // VirtioBlockCCW indicates block driver is virtio-blk-ccw based + VirtioBlockCCW string = "virtio-blk-ccw" // VirtioSCSI indicates block driver is virtio-scsi based VirtioSCSI string = "virtio-scsi" // Nvdimm indicates block driver is nvdimm based @@ -66,6 +68,8 @@ func NewDeviceManager(blockDriver string, devices []api.Device) api.DeviceManage dm.blockDriver = VirtioBlock } else if blockDriver == Nvdimm { dm.blockDriver = Nvdimm + } else if blockDriver == VirtioBlockCCW { + dm.blockDriver = VirtioBlockCCW } else { dm.blockDriver = VirtioSCSI } diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index c933971106..098583ac00 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -75,6 +75,7 @@ var ( kata9pDevType = "9p" kataMmioBlkDevType = "mmioblk" kataBlkDevType = "blk" + kataBlkCCWDevType = "blk-ccw" kataSCSIDevType = "scsi" kataNvdimmDevType = "nvdimm" kataVirtioFSDevType = "virtio-fs" @@ -1106,6 +1107,9 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, c *Container) []*gr kataDevice.Type = kataMmioBlkDevType kataDevice.Id = d.VirtPath kataDevice.VmPath = d.VirtPath + case config.VirtioBlockCCW: + kataDevice.Type = kataBlkCCWDevType + kataDevice.Id = d.DevNo case config.VirtioBlock: kataDevice.Type = kataBlkDevType kataDevice.Id = d.PCIAddr @@ -1160,11 +1164,14 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat k.Logger().Error("malformed block drive") return nil, fmt.Errorf("malformed block drive") } - - if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio { + switch { + case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio: rootfs.Driver = kataMmioBlkDevType rootfs.Source = blockDrive.VirtPath - } else if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock { + case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlockCCW: + rootfs.Driver = kataBlkCCWDevType + rootfs.Source = blockDrive.DevNo + case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock: rootfs.Driver = kataBlkDevType if blockDrive.PCIAddr == "" { rootfs.Source = blockDrive.VirtPath @@ -1172,10 +1179,14 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat rootfs.Source = blockDrive.PCIAddr } - } else { + case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioSCSI: + rootfs.Driver = kataSCSIDevType rootfs.Source = blockDrive.SCSIAddr + default: + return nil, fmt.Errorf("Unknown block device driver: %s", sandbox.config.HypervisorConfig.BlockDeviceDriver) } + rootfs.MountPoint = rootPathParent rootfs.Fstype = c.state.Fstype @@ -1281,7 +1292,10 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, // Note this call modifies the list of container devices to make sure // all hotplugged devices are unplugged, so this needs be done // after devices passed with --device are handled. - volumeStorages := k.handleBlockVolumes(c) + volumeStorages, err := k.handleBlockVolumes(c) + if err != nil { + return nil, err + } if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil { return nil, err } @@ -1400,7 +1414,7 @@ func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, r // handleBlockVolumes handles volumes that are block devices files // by passing the block devices as Storage to the agent. -func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage { +func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { var volumeStorages []*grpc.Storage @@ -1418,7 +1432,7 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage { if !c.sandbox.supportNewStore() { if err := c.storeDevices(); err != nil { k.Logger().WithField("device", id).WithError(err).Error("store device failed") - return nil + return nil, err } } @@ -1427,22 +1441,28 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage { device := c.sandbox.devManager.GetDeviceByID(id) if device == nil { k.Logger().WithField("device", id).Error("failed to find device by id") - return nil + return nil, fmt.Errorf("Failed to find device by id (id=%s)", id) } blockDrive, ok := device.GetDeviceInfo().(*config.BlockDrive) if !ok || blockDrive == nil { k.Logger().Error("malformed block drive") continue } - if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock { + switch { + case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlockCCW: + vol.Driver = kataBlkCCWDevType + vol.Source = blockDrive.DevNo + case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock: vol.Driver = kataBlkDevType vol.Source = blockDrive.PCIAddr - } else if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio { + case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio: vol.Driver = kataMmioBlkDevType vol.Source = blockDrive.VirtPath - } else { + case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioSCSI: vol.Driver = kataSCSIDevType vol.Source = blockDrive.SCSIAddr + default: + return nil, fmt.Errorf("Unknown block device driver: %s", c.sandbox.config.HypervisorConfig.BlockDeviceDriver) } vol.MountPoint = m.Destination @@ -1452,7 +1472,7 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage { volumeStorages = append(volumeStorages, vol) } - return volumeStorages + return volumeStorages, nil } // handlePidNamespace checks if Pid namespace for a container needs to be shared with its sandbox diff --git a/virtcontainers/persist/api/device.go b/virtcontainers/persist/api/device.go index b1b3c1a6f9..e82276ccf9 100644 --- a/virtcontainers/persist/api/device.go +++ b/virtcontainers/persist/api/device.go @@ -38,6 +38,9 @@ type BlockDrive struct { // VirtPath at which the device appears inside the VM, outside of the container mount namespace VirtPath string + + // DevNo + DevNo string } // VFIODev represents a VFIO drive used for hotplugging diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 8eb69034ba..2a6143caad 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -61,7 +61,7 @@ type CPUDevice struct { // QemuState keeps Qemu's state type QemuState struct { - Bridges []types.PCIBridge + Bridges []types.Bridge // HotpluggedCPUs is the list of CPUs that were hot-added HotpluggedVCPUs []CPUDevice HotpluggedMemory int @@ -266,9 +266,11 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto create = true } + q.arch.setBridges(q.state.Bridges) + if create { q.Logger().Debug("Creating bridges") - q.state.Bridges = q.arch.bridges(q.config.DefaultBridges) + q.arch.bridges(q.config.DefaultBridges) q.Logger().Debug("Creating UUID") q.state.UUID = uuid.Generate().String() @@ -403,7 +405,7 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I // Add bridges before any other devices. This way we make sure that // bridge gets the first available PCI address i.e bridgePCIStartAddr - devices = q.arch.appendBridges(devices, q.state.Bridges) + devices = q.arch.appendBridges(devices) devices = q.arch.appendConsole(devices, console) @@ -984,38 +986,6 @@ func (q *qemu) qmpShutdown() { } } -func (q *qemu) addDeviceToBridge(ID string) (string, types.PCIBridge, error) { - var err error - var addr uint32 - - if len(q.state.Bridges) == 0 { - return "", types.PCIBridge{}, errors.New("failed to get available address from bridges") - } - - // looking for an empty address in the bridges - for _, b := range q.state.Bridges { - addr, err = b.AddDevice(ID) - if err == nil { - return fmt.Sprintf("%02x", addr), b, nil - } - } - - return "", types.PCIBridge{}, fmt.Errorf("no more bridge slots available") -} - -func (q *qemu) removeDeviceFromBridge(ID string) error { - var err error - for _, b := range q.state.Bridges { - err = b.RemoveDevice(ID) - if err == nil { - // device was removed correctly - return nil - } - } - - return err -} - func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, devID string) (err error) { if q.config.BlockDeviceDriver == config.Nvdimm { var blocksize int64 @@ -1050,16 +1020,36 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev } }() - if q.config.BlockDeviceDriver == config.VirtioBlock { + switch { + case q.config.BlockDeviceDriver == config.VirtioBlockCCW: + driver := "virtio-blk-ccw" + + addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.CCW) + if err != nil { + return err + } + var devNoHotplug string + devNoHotplug, err = bridge.AddressFormatCCW(addr) + if err != nil { + return err + } + drive.DevNo, err = bridge.AddressFormatCCWForVirtServer(addr) + if err != nil { + return err + } + if err = q.qmpMonitorCh.qmp.ExecuteDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, devNoHotplug, "", true, false); err != nil { + return err + } + case q.config.BlockDeviceDriver == config.VirtioBlock: driver := "virtio-blk-pci" - addr, bridge, err := q.addDeviceToBridge(drive.ID) + addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.PCI) if err != nil { return err } defer func() { if err != nil { - q.removeDeviceFromBridge(drive.ID) + q.arch.removeDeviceFromBridge(drive.ID) } }() @@ -1069,7 +1059,7 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil { return err } - } else { + case q.config.BlockDeviceDriver == config.VirtioSCSI: driver := "scsi-hd" // Bus exposed by the SCSI Controller @@ -1084,6 +1074,8 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev if err = q.qmpMonitorCh.qmp.ExecuteSCSIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, bus, romFile, scsiID, lun, true, defaultDisableModern); err != nil { return err } + default: + return fmt.Errorf("Block device %s not recognized", q.config.BlockDeviceDriver) } return nil @@ -1101,7 +1093,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error err = q.hotplugAddBlockDevice(drive, op, devID) } else { if q.config.BlockDeviceDriver == config.VirtioBlock { - if err := q.removeDeviceFromBridge(drive.ID); err != nil { + if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil { return err } } @@ -1141,14 +1133,14 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro } } - addr, bridge, err := q.addDeviceToBridge(devID) + addr, bridge, err := q.arch.addDeviceToBridge(devID, types.PCI) if err != nil { return err } defer func() { if err != nil { - q.removeDeviceFromBridge(devID) + q.arch.removeDeviceFromBridge(devID) } }() @@ -1162,7 +1154,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro } } else { if !q.state.HotplugVFIOOnRootBus { - if err := q.removeDeviceFromBridge(devID); err != nil { + if err := q.arch.removeDeviceFromBridge(devID); err != nil { return err } } @@ -1228,14 +1220,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { } }() - addr, bridge, err := q.addDeviceToBridge(tap.ID) + addr, bridge, err := q.arch.addDeviceToBridge(tap.ID, types.PCI) if err != nil { return err } defer func() { if err != nil { - q.removeDeviceFromBridge(tap.ID) + q.arch.removeDeviceFromBridge(tap.ID) } }() @@ -1248,12 +1240,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { return err } if machine.Type == QemuCCWVirtio { - return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), bridge.ID, int(q.config.NumVCPUs)) + devNoHotplug := fmt.Sprintf("fe.%x.%x", bridge.Addr, addr) + return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs)) } return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, romFile, int(q.config.NumVCPUs), defaultDisableModern) + } - if err := q.removeDeviceFromBridge(tap.ID); err != nil { + if err := q.arch.removeDeviceFromBridge(tap.ID); err != nil { return err } @@ -1750,7 +1744,7 @@ func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe boo // genericAppendBridges appends to devices the given bridges // nolint: unused, deadcode -func genericAppendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge, machineType string) []govmmQemu.Device { +func genericAppendBridges(devices []govmmQemu.Device, bridges []types.Bridge, machineType string) []govmmQemu.Device { bus := defaultPCBridgeBus switch machineType { case QemuQ35, QemuVirt: @@ -1762,6 +1756,9 @@ func genericAppendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge, if b.Type == types.PCIE { t = govmmQemu.PCIEBridge } + if b.Type == types.CCW { + continue + } bridges[idx].Addr = bridgePCIStartAddr + idx @@ -1781,10 +1778,9 @@ func genericAppendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge, return devices } -// nolint: unused, deadcode -func genericBridges(number uint32, machineType string) []types.PCIBridge { - var bridges []types.PCIBridge - var bt types.PCIType +func genericBridges(number uint32, machineType string) []types.Bridge { + var bridges []types.Bridge + var bt types.Type switch machineType { case QemuQ35: @@ -1798,17 +1794,13 @@ func genericBridges(number uint32, machineType string) []types.PCIBridge { case QemuPseries: bt = types.PCI case QemuCCWVirtio: - bt = types.PCI + bt = types.CCW default: return nil } for i := uint32(0); i < number; i++ { - bridges = append(bridges, types.PCIBridge{ - Type: bt, - ID: fmt.Sprintf("%s-bridge-%d", bt, i), - Address: make(map[uint32]string), - }) + bridges = append(bridges, types.NewBridge(bt, fmt.Sprintf("%s-bridge-%d", bt, i), make(map[uint32]string), 0)) } return bridges @@ -1990,6 +1982,7 @@ func (q *qemu) toGrpc() ([]byte, error) { func (q *qemu) storeState() error { if q.store != nil { + q.state.Bridges = q.arch.getBridges() if err := q.store.Store(store.Hypervisor, q.state); err != nil { return err } @@ -2008,9 +2001,9 @@ func (q *qemu) save() (s persistapi.HypervisorState) { s.HotpluggedMemory = q.state.HotpluggedMemory s.HotplugVFIOOnRootBus = q.state.HotplugVFIOOnRootBus - for _, bridge := range q.state.Bridges { + for _, bridge := range q.arch.getBridges() { s.Bridges = append(s.Bridges, persistapi.Bridge{ - DeviceAddr: bridge.Address, + DeviceAddr: bridge.Devices, Type: string(bridge.Type), ID: bridge.ID, Addr: bridge.Addr, @@ -2032,12 +2025,7 @@ func (q *qemu) load(s persistapi.HypervisorState) { q.state.VirtiofsdPid = s.VirtiofsdPid for _, bridge := range s.Bridges { - q.state.Bridges = append(q.state.Bridges, types.PCIBridge{ - Address: bridge.DeviceAddr, - Type: types.PCIType(bridge.Type), - ID: bridge.ID, - Addr: bridge.Addr, - }) + q.state.Bridges = append(q.state.Bridges, types.NewBridge(types.Type(bridge.Type), bridge.ID, bridge.DeviceAddr, bridge.Addr)) } for _, cpu := range s.HotpluggedVCPUs { diff --git a/virtcontainers/qemu_amd64.go b/virtcontainers/qemu_amd64.go index 280b94ad65..7014083f31 100644 --- a/virtcontainers/qemu_amd64.go +++ b/virtcontainers/qemu_amd64.go @@ -122,8 +122,8 @@ func (q *qemuAmd64) capabilities() types.Capabilities { return caps } -func (q *qemuAmd64) bridges(number uint32) []types.PCIBridge { - return genericBridges(number, q.machineType) +func (q *qemuAmd64) bridges(number uint32) { + q.Bridges = genericBridges(number, q.machineType) } func (q *qemuAmd64) cpuModel() string { @@ -173,6 +173,6 @@ func (q *qemuAmd64) appendImage(devices []govmmQemu.Device, path string) ([]govm } // appendBridges appends to devices the given bridges -func (q *qemuAmd64) appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device { - return genericAppendBridges(devices, bridges, q.machineType) +func (q *qemuAmd64) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device { + return genericAppendBridges(devices, q.Bridges, q.machineType) } diff --git a/virtcontainers/qemu_amd64_test.go b/virtcontainers/qemu_amd64_test.go index 5fb91da548..176b97edb5 100644 --- a/virtcontainers/qemu_amd64_test.go +++ b/virtcontainers/qemu_amd64_test.go @@ -40,29 +40,32 @@ func TestQemuAmd64Bridges(t *testing.T) { amd64 := newTestQemu(QemuPC) len := 5 - bridges := amd64.bridges(uint32(len)) + amd64.bridges(uint32(len)) + bridges := amd64.getBridges() assert.Len(bridges, len) for i, b := range bridges { id := fmt.Sprintf("%s-bridge-%d", types.PCI, i) assert.Equal(types.PCI, b.Type) assert.Equal(id, b.ID) - assert.NotNil(b.Address) + assert.NotNil(b.Devices) } amd64 = newTestQemu(QemuQ35) - bridges = amd64.bridges(uint32(len)) + amd64.bridges(uint32(len)) + bridges = amd64.getBridges() assert.Len(bridges, len) for i, b := range bridges { id := fmt.Sprintf("%s-bridge-%d", types.PCI, i) assert.Equal(types.PCI, b.Type) assert.Equal(id, b.ID) - assert.NotNil(b.Address) + assert.NotNil(b.Devices) } amd64 = newTestQemu(QemuQ35 + QemuPC) - bridges = amd64.bridges(uint32(len)) + amd64.bridges(uint32(len)) + bridges = amd64.getBridges() assert.Nil(bridges) } @@ -143,10 +146,11 @@ func TestQemuAmd64AppendBridges(t *testing.T) { // check PC amd64 := newTestQemu(QemuPC) - bridges := amd64.bridges(1) + amd64.bridges(1) + bridges := amd64.getBridges() assert.Len(bridges, 1) - devices = amd64.appendBridges(devices, bridges) + devices = amd64.appendBridges(devices) assert.Len(devices, 1) expectedOut := []govmmQemu.Device{ @@ -165,11 +169,12 @@ func TestQemuAmd64AppendBridges(t *testing.T) { // Check Q35 amd64 = newTestQemu(QemuQ35) - bridges = amd64.bridges(1) + amd64.bridges(1) + bridges = amd64.getBridges() assert.Len(bridges, 1) devices = []govmmQemu.Device{} - devices = amd64.appendBridges(devices, bridges) + devices = amd64.appendBridges(devices) assert.Len(devices, 1) expectedOut = []govmmQemu.Device{ diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index d8abcf5066..20d73505c3 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -8,6 +8,7 @@ package virtcontainers import ( "context" "encoding/hex" + "errors" "fmt" "os" "strconv" @@ -48,8 +49,8 @@ type qemuArch interface { //capabilities returns the capabilities supported by QEMU capabilities() types.Capabilities - // bridges returns the number bridges for the machine type - bridges(number uint32) []types.PCIBridge + // bridges sets the number bridges for the machine type + bridges(number uint32) // cpuTopology returns the CPU topology for the given amount of vcpus cpuTopology(vcpus, maxvcpus uint32) govmmQemu.SMP @@ -70,7 +71,7 @@ type qemuArch interface { appendSCSIController(devices []govmmQemu.Device, enableIOThreads bool) ([]govmmQemu.Device, *govmmQemu.IOThread) // appendBridges appends bridges to devices - appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device + appendBridges(devices []govmmQemu.Device) []govmmQemu.Device // append9PVolume appends a 9P volume to devices append9PVolume(devices []govmmQemu.Device, volume types.Volume) []govmmQemu.Device @@ -96,6 +97,21 @@ type qemuArch interface { // appendRNGDevice appends a RNG device to devices appendRNGDevice(devices []govmmQemu.Device, rngDevice config.RNGDev) []govmmQemu.Device + // addDeviceToBridge adds devices to the bus + addDeviceToBridge(ID string, t types.Type) (string, types.Bridge, error) + + // removeDeviceFromBridge removes devices to the bus + removeDeviceFromBridge(ID string) error + + // getBridges grants access to Bridges + getBridges() []types.Bridge + + // setBridges grants access to Bridges + setBridges(bridges []types.Bridge) + + // addBridge adds a new Bridge to the list of Bridges + addBridge(types.Bridge) + // handleImagePath handles the Hypervisor Config image path handleImagePath(config HypervisorConfig) @@ -117,6 +133,7 @@ type qemuArchBase struct { kernelParamsNonDebug []Param kernelParamsDebug []Param kernelParams []Param + Bridges []types.Bridge } const ( @@ -242,18 +259,10 @@ func (q *qemuArchBase) capabilities() types.Capabilities { return caps } -func (q *qemuArchBase) bridges(number uint32) []types.PCIBridge { - var bridges []types.PCIBridge - +func (q *qemuArchBase) bridges(number uint32) { for i := uint32(0); i < number; i++ { - bridges = append(bridges, types.PCIBridge{ - Type: types.PCI, - ID: fmt.Sprintf("%s-bridge-%d", types.PCI, i), - Address: make(map[uint32]string), - }) + q.Bridges = append(q.Bridges, types.NewBridge(types.PCI, fmt.Sprintf("%s-bridge-%d", types.PCI, i), make(map[uint32]string), 0)) } - - return bridges } func (q *qemuArchBase) cpuTopology(vcpus, maxvcpus uint32) govmmQemu.SMP { @@ -306,14 +315,14 @@ func (q *qemuArchBase) appendConsole(devices []govmmQemu.Device, path string) [] return devices } -func (q *qemuArchBase) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) { +func genericImage(path string) (config.BlockDrive, error) { if _, err := os.Stat(path); os.IsNotExist(err) { - return nil, err + return config.BlockDrive{}, err } randBytes, err := utils.GenerateRandomBytes(8) if err != nil { - return nil, err + return config.BlockDrive{}, err } id := utils.MakeNameID("image", hex.EncodeToString(randBytes), maxDevIDSize) @@ -324,13 +333,21 @@ func (q *qemuArchBase) appendImage(devices []govmmQemu.Device, path string) ([]g ID: id, } + return drive, nil +} + +func (q *qemuArchBase) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) { + drive, err := genericImage(path) + if err != nil { + return nil, err + } return q.appendBlockDevice(devices, drive), nil } -func (q *qemuArchBase) appendSCSIController(devices []govmmQemu.Device, enableIOThreads bool) ([]govmmQemu.Device, *govmmQemu.IOThread) { +func genericSCSIController(enableIOThreads, nestedRun bool) (govmmQemu.SCSIController, *govmmQemu.IOThread) { scsiController := govmmQemu.SCSIController{ ID: scsiControllerID, - DisableModern: q.nestedRun, + DisableModern: nestedRun, } var t *govmmQemu.IOThread @@ -345,20 +362,27 @@ func (q *qemuArchBase) appendSCSIController(devices []govmmQemu.Device, enableIO scsiController.IOThread = t.ID } - devices = append(devices, scsiController) + return scsiController, t +} +func (q *qemuArchBase) appendSCSIController(devices []govmmQemu.Device, enableIOThreads bool) ([]govmmQemu.Device, *govmmQemu.IOThread) { + d, t := genericSCSIController(enableIOThreads, q.nestedRun) + devices = append(devices, d) return devices, t } // appendBridges appends to devices the given bridges -func (q *qemuArchBase) appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device { - for idx, b := range bridges { +func (q *qemuArchBase) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device { + for idx, b := range q.Bridges { + if b.Type == types.CCW { + continue + } t := govmmQemu.PCIBridge if b.Type == types.PCIE { t = govmmQemu.PCIEBridge } - bridges[idx].Addr = bridgePCIStartAddr + idx + q.Bridges[idx].Addr = bridgePCIStartAddr + idx devices = append(devices, govmmQemu.BridgeDevice{ @@ -368,7 +392,7 @@ func (q *qemuArchBase) appendBridges(devices []govmmQemu.Device, bridges []types // Each bridge is required to be assigned a unique chassis id > 0 Chassis: idx + 1, SHPC: true, - Addr: strconv.FormatInt(int64(bridges[idx].Addr), 10), + Addr: strconv.FormatInt(int64(q.Bridges[idx].Addr), 10), }, ) } @@ -376,28 +400,30 @@ func (q *qemuArchBase) appendBridges(devices []govmmQemu.Device, bridges []types return devices } -func (q *qemuArchBase) append9PVolume(devices []govmmQemu.Device, volume types.Volume) []govmmQemu.Device { - if volume.MountTag == "" || volume.HostPath == "" { - return devices - } - +func generic9PVolume(volume types.Volume, nestedRun bool) govmmQemu.FSDevice { devID := fmt.Sprintf("extra-9p-%s", volume.MountTag) if len(devID) > maxDevIDSize { devID = devID[:maxDevIDSize] } - devices = append(devices, - govmmQemu.FSDevice{ - Driver: govmmQemu.Virtio9P, - FSDriver: govmmQemu.Local, - ID: devID, - Path: volume.HostPath, - MountTag: volume.MountTag, - SecurityModel: govmmQemu.None, - DisableModern: q.nestedRun, - }, - ) + return govmmQemu.FSDevice{ + Driver: govmmQemu.Virtio9P, + FSDriver: govmmQemu.Local, + ID: devID, + Path: volume.HostPath, + MountTag: volume.MountTag, + SecurityModel: govmmQemu.None, + DisableModern: nestedRun, + } +} +func (q *qemuArchBase) append9PVolume(devices []govmmQemu.Device, volume types.Volume) []govmmQemu.Device { + if volume.MountTag == "" || volume.HostPath == "" { + return devices + } + + d := generic9PVolume(volume, q.nestedRun) + devices = append(devices, d) return devices } @@ -452,70 +478,83 @@ func networkModelToQemuType(model NetInterworkingModel) govmmQemu.NetDeviceType } } -func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device { +func genericNetwork(endpoint Endpoint, vhost, nestedRun bool, index int) (govmmQemu.NetDevice, error) { + var d govmmQemu.NetDevice switch ep := endpoint.(type) { case *VethEndpoint, *BridgedMacvlanEndpoint, *IPVlanEndpoint: netPair := ep.NetworkPair() - devices = append(devices, - govmmQemu.NetDevice{ - Type: networkModelToQemuType(netPair.NetInterworkingModel), - Driver: govmmQemu.VirtioNet, - ID: fmt.Sprintf("network-%d", q.networkIndex), - IFName: netPair.TAPIface.Name, - MACAddress: netPair.TAPIface.HardAddr, - DownScript: "no", - Script: "no", - VHost: q.vhost, - DisableModern: q.nestedRun, - FDs: netPair.VMFds, - VhostFDs: netPair.VhostFds, - }, - ) - q.networkIndex++ + d = govmmQemu.NetDevice{ + Type: networkModelToQemuType(netPair.NetInterworkingModel), + Driver: govmmQemu.VirtioNet, + ID: fmt.Sprintf("network-%d", index), + IFName: netPair.TAPIface.Name, + MACAddress: netPair.TAPIface.HardAddr, + DownScript: "no", + Script: "no", + VHost: vhost, + DisableModern: nestedRun, + FDs: netPair.VMFds, + VhostFDs: netPair.VhostFds, + } case *MacvtapEndpoint: - devices = append(devices, - govmmQemu.NetDevice{ - Type: govmmQemu.MACVTAP, - Driver: govmmQemu.VirtioNet, - ID: fmt.Sprintf("network-%d", q.networkIndex), - IFName: ep.Name(), - MACAddress: ep.HardwareAddr(), - DownScript: "no", - Script: "no", - VHost: q.vhost, - DisableModern: q.nestedRun, - FDs: ep.VMFds, - VhostFDs: ep.VhostFds, - }, - ) - q.networkIndex++ - + d = govmmQemu.NetDevice{ + Type: govmmQemu.MACVTAP, + Driver: govmmQemu.VirtioNet, + ID: fmt.Sprintf("network-%d", index), + IFName: ep.Name(), + MACAddress: ep.HardwareAddr(), + DownScript: "no", + Script: "no", + VHost: vhost, + DisableModern: nestedRun, + FDs: ep.VMFds, + VhostFDs: ep.VhostFds, + } + default: + return govmmQemu.NetDevice{}, fmt.Errorf("Unknown type for endpoint") } + return d, nil +} + +func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device { + d, err := genericNetwork(endpoint, q.vhost, q.nestedRun, q.networkIndex) + if err != nil { + virtLog.WithField("subsystem", "qemuArch").WithError(err).Error("Failed to append network") + return devices + } + q.networkIndex++ + devices = append(devices, d) return devices } -func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device { +func genericBlockDevice(drive config.BlockDrive, nestedRun bool) (govmmQemu.BlockDevice, error) { if drive.File == "" || drive.ID == "" || drive.Format == "" { - return devices + return govmmQemu.BlockDevice{}, fmt.Errorf("Empty File, ID or Format for drive %v", drive) } if len(drive.ID) > maxDevIDSize { drive.ID = drive.ID[:maxDevIDSize] } - devices = append(devices, - govmmQemu.BlockDevice{ - Driver: govmmQemu.VirtioBlock, - ID: drive.ID, - File: drive.File, - AIO: govmmQemu.Threads, - Format: govmmQemu.BlockDeviceFormat(drive.Format), - Interface: "none", - DisableModern: q.nestedRun, - }, - ) + return govmmQemu.BlockDevice{ + Driver: govmmQemu.VirtioBlock, + ID: drive.ID, + File: drive.File, + AIO: govmmQemu.Threads, + Format: govmmQemu.BlockDeviceFormat(drive.Format), + Interface: "none", + DisableModern: nestedRun, + }, nil +} +func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device { + d, err := genericBlockDevice(drive, q.nestedRun) + if err != nil { + virtLog.WithField("subsystem", "qemuArch").WithError(err).Error("Failed to append block device") + return devices + } + devices = append(devices, d) return devices } @@ -592,3 +631,55 @@ func (q *qemuArchBase) setIgnoreSharedMemoryMigrationCaps(ctx context.Context, q }) return err } + +func (q *qemuArchBase) addDeviceToBridge(ID string, t types.Type) (string, types.Bridge, error) { + var err error + var addr uint32 + + if len(q.Bridges) == 0 { + return "", types.Bridge{}, errors.New("failed to get available address from bridges") + } + + // looking for an empty address in the bridges + for _, b := range q.Bridges { + if t != b.Type { + continue + } + addr, err = b.AddDevice(ID) + if err == nil { + switch t { + case types.CCW: + return fmt.Sprintf("%04x", addr), b, nil + case types.PCI, types.PCIE: + return fmt.Sprintf("%02x", addr), b, nil + } + } + } + + return "", types.Bridge{}, fmt.Errorf("no more bridge slots available") +} + +func (q *qemuArchBase) removeDeviceFromBridge(ID string) error { + var err error + for _, b := range q.Bridges { + err = b.RemoveDevice(ID) + if err == nil { + // device was removed correctly + return nil + } + } + + return err +} + +func (q *qemuArchBase) getBridges() []types.Bridge { + return q.Bridges +} + +func (q *qemuArchBase) setBridges(bridges []types.Bridge) { + q.Bridges = bridges +} + +func (q *qemuArchBase) addBridge(b types.Bridge) { + q.Bridges = append(q.Bridges, b) +} diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go index 07eb7cf614..ce853f90cc 100644 --- a/virtcontainers/qemu_arch_base_test.go +++ b/virtcontainers/qemu_arch_base_test.go @@ -18,6 +18,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/pkg/errors" ) const ( @@ -147,14 +148,44 @@ func TestQemuArchBaseBridges(t *testing.T) { qemuArchBase := newQemuArchBase() len := 5 - bridges := qemuArchBase.bridges(uint32(len)) + qemuArchBase.bridges(uint32(len)) + bridges := qemuArchBase.getBridges() assert.Len(bridges, len) for i, b := range bridges { id := fmt.Sprintf("%s-bridge-%d", types.PCI, i) assert.Equal(types.PCI, b.Type) assert.Equal(id, b.ID) - assert.NotNil(b.Address) + assert.NotNil(b.Devices) + } +} + +func TestQemuAddDeviceToBridge(t *testing.T) { + assert := assert.New(t) + + // addDeviceToBridge successfully + q := newQemuArchBase() + q.machineType = QemuPC + + q.bridges(1) + for i := uint32(1); i <= types.PCIBridgeMaxCapacity; i++ { + _, _, err := q.addDeviceToBridge(fmt.Sprintf("qemu-bridge-%d", i), types.PCI) + assert.Nil(err) + } + + // fail to add device to bridge cause no more available bridge slot + _, _, err := q.addDeviceToBridge("qemu-bridge-31", types.PCI) + exceptErr := errors.New("no more bridge slots available") + assert.Equal(exceptErr.Error(), err.Error()) + + // addDeviceToBridge fails cause q.Bridges == 0 + q = newQemuArchBase() + q.machineType = QemuPCLite + q.bridges(0) + _, _, err = q.addDeviceToBridge("qemu-bridge", types.PCI) + if assert.Error(err) { + exceptErr = errors.New("failed to get available address from bridges") + assert.Equal(exceptErr.Error(), err.Error()) } } @@ -283,10 +314,11 @@ func TestQemuArchBaseAppendBridges(t *testing.T) { assert := assert.New(t) qemuArchBase := newQemuArchBase() - bridges := qemuArchBase.bridges(1) + qemuArchBase.bridges(1) + bridges := qemuArchBase.getBridges() assert.Len(bridges, 1) - devices = qemuArchBase.appendBridges(devices, bridges) + devices = qemuArchBase.appendBridges(devices) assert.Len(devices, 1) expectedOut := []govmmQemu.Device{ diff --git a/virtcontainers/qemu_arm64.go b/virtcontainers/qemu_arm64.go index 6cf3191079..9e8549bbbe 100644 --- a/virtcontainers/qemu_arm64.go +++ b/virtcontainers/qemu_arm64.go @@ -14,7 +14,6 @@ import ( "time" govmmQemu "github.com/intel/govmm/qemu" - "github.com/kata-containers/runtime/virtcontainers/types" "github.com/sirupsen/logrus" ) @@ -162,13 +161,13 @@ func newQemuArch(config HypervisorConfig) qemuArch { return q } -func (q *qemuArm64) bridges(number uint32) []types.PCIBridge { - return genericBridges(number, q.machineType) +func (q *qemuArm64) bridges(number uint32) { + q.Bridges = genericBridges(number, q.machineType) } // appendBridges appends to devices the given bridges -func (q *qemuArm64) appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device { - return genericAppendBridges(devices, bridges, q.machineType) +func (q *qemuArm64) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device { + return genericAppendBridges(devices, q.Bridges, q.machineType) } func (q *qemuArm64) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) { diff --git a/virtcontainers/qemu_arm64_test.go b/virtcontainers/qemu_arm64_test.go index 46d75641ec..ff20d79419 100644 --- a/virtcontainers/qemu_arm64_test.go +++ b/virtcontainers/qemu_arm64_test.go @@ -105,11 +105,12 @@ func TestQemuArm64AppendBridges(t *testing.T) { arm64 := newTestQemu(QemuVirt) - bridges := arm64.bridges(1) + arm64.bridges(1) + bridges := arm64.getBridges() assert.Len(bridges, 1) devices = []govmmQemu.Device{} - devices = arm64.appendBridges(devices, bridges) + devices = arm64.appendBridges(devices) assert.Len(devices, 1) expectedOut := []govmmQemu.Device{ diff --git a/virtcontainers/qemu_ppc64le.go b/virtcontainers/qemu_ppc64le.go index afad877815..e86d17d885 100644 --- a/virtcontainers/qemu_ppc64le.go +++ b/virtcontainers/qemu_ppc64le.go @@ -105,8 +105,8 @@ func (q *qemuPPC64le) capabilities() types.Capabilities { return caps } -func (q *qemuPPC64le) bridges(number uint32) []types.PCIBridge { - return genericBridges(number, q.machineType) +func (q *qemuPPC64le) bridges(number uint32) { + q.Bridges = genericBridges(number, q.machineType) } func (q *qemuPPC64le) cpuModel() string { @@ -152,6 +152,6 @@ func (q *qemuPPC64le) appendImage(devices []govmmQemu.Device, path string) ([]go } // appendBridges appends to devices the given bridges -func (q *qemuPPC64le) appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device { - return genericAppendBridges(devices, bridges, q.machineType) +func (q *qemuPPC64le) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device { + return genericAppendBridges(devices, q.Bridges, q.machineType) } diff --git a/virtcontainers/qemu_s390x.go b/virtcontainers/qemu_s390x.go index c1bf90f393..25350a46ff 100644 --- a/virtcontainers/qemu_s390x.go +++ b/virtcontainers/qemu_s390x.go @@ -40,6 +40,8 @@ var kernelParams = []Param{ var kernelRootParams = commonVirtioblkKernelRootParams +var ccwbridge = types.NewBridge(types.CCW, "", make(map[uint32]string, types.CCWBridgeMaxCapacity), 0) + var supportedQemuMachines = []govmmQemu.Machine{ { Type: QemuCCWVirtio, @@ -72,6 +74,8 @@ func newQemuArch(config HypervisorConfig) qemuArch { kernelParams: kernelParams, }, } + // Set first bridge type to CCW + q.Bridges = append(q.Bridges, ccwbridge) if config.ImagePath != "" { q.kernelParams = append(q.kernelParams, kernelRootParams...) @@ -82,22 +86,32 @@ func newQemuArch(config HypervisorConfig) qemuArch { return q } -func (q *qemuS390x) bridges(number uint32) []types.PCIBridge { - return genericBridges(number, q.machineType) -} - -// appendBridges appends to devices the given bridges -func (q *qemuS390x) appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device { - return genericAppendBridges(devices, bridges, q.machineType) +func (q *qemuS390x) bridges(number uint32) { + q.Bridges = genericBridges(number, q.machineType) } // appendConsole appends a console to devices. // The function has been overwriten to correctly set the driver to the CCW device func (q *qemuS390x) appendConsole(devices []govmmQemu.Device, path string) []govmmQemu.Device { + id := "serial0" + addr, b, err := q.addDeviceToBridge(id, types.CCW) + if err != nil { + virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append console") + return devices + } + + var devno string + devno, err = b.AddressFormatCCW(addr) + if err != nil { + virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append console") + return devices + } + serial := govmmQemu.SerialDevice{ Driver: virtioSerialCCW, - ID: "serial0", + ID: id, DisableModern: q.nestedRun, + DevNo: devno, } devices = append(devices, serial) @@ -115,6 +129,36 @@ func (q *qemuS390x) appendConsole(devices []govmmQemu.Device, path string) []gov return devices } +func (q *qemuS390x) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) { + drive, err := genericImage(path) + if err != nil { + virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append image") + return nil, err + } + + return q.appendBlockDevice(devices, drive), nil +} + +func (q *qemuS390x) appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device { + d, err := genericBlockDevice(drive, false) + if err != nil { + virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append blk-dev") + return devices + } + addr, b, err := q.addDeviceToBridge(drive.ID, types.CCW) + if err != nil { + virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append blk-dev") + return devices + } + d.DevNo, err = b.AddressFormatCCW(addr) + if err != nil { + virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append blk-dev") + return devices + } + devices = append(devices, d) + return devices +} + // appendVhostUserDevice throws an error if vhost devices are tried to be used. // See issue https://github.com/kata-containers/runtime/issues/659 func (q *qemuS390x) appendVhostUserDevice(devices []govmmQemu.Device, attr config.VhostUserDeviceAttrs) ([]govmmQemu.Device, error) { @@ -126,3 +170,90 @@ func (q *qemuS390x) appendVhostUserDevice(devices []govmmQemu.Device, attr confi func (q *qemuS390x) supportGuestMemoryHotplug() bool { return false } + +func (q *qemuS390x) appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device { + d, err := genericNetwork(endpoint, false, false, q.networkIndex) + if err != nil { + virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append network") + return devices + } + q.networkIndex++ + addr, b, err := q.addDeviceToBridge(d.ID, types.CCW) + if err != nil { + virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append network") + return devices + } + d.DevNo, err = b.AddressFormatCCW(addr) + if err != nil { + virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append network") + return devices + } + + devices = append(devices, d) + return devices +} + +func (q *qemuS390x) appendRNGDevice(devices []govmmQemu.Device, rngDev config.RNGDev) []govmmQemu.Device { + addr, b, err := q.addDeviceToBridge(rngDev.ID, types.CCW) + if err != nil { + virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append RNG-Device") + return devices + } + var devno string + devno, err = b.AddressFormatCCW(addr) + if err != nil { + virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append RNG-Device") + return devices + } + + devices = append(devices, + govmmQemu.RngDevice{ + ID: rngDev.ID, + Filename: rngDev.Filename, + DevNo: devno, + }, + ) + + return devices +} + +func (q *qemuS390x) append9PVolume(devices []govmmQemu.Device, volume types.Volume) []govmmQemu.Device { + if volume.MountTag == "" || volume.HostPath == "" { + return devices + } + d := generic9PVolume(volume, false) + addr, b, err := q.addDeviceToBridge(d.ID, types.CCW) + if err != nil { + virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append 9p-Volume") + return devices + } + d.DevNo, err = b.AddressFormatCCW(addr) + if err != nil { + virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append 9p-Volume") + return devices + } + devices = append(devices, d) + return devices +} + +// appendBridges appends to devices the given bridges +func (q *qemuS390x) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device { + return genericAppendBridges(devices, q.Bridges, q.machineType) +} + +func (q *qemuS390x) appendSCSIController(devices []govmmQemu.Device, enableIOThreads bool) ([]govmmQemu.Device, *govmmQemu.IOThread) { + d, t := genericSCSIController(enableIOThreads, q.nestedRun) + addr, b, err := q.addDeviceToBridge(d.ID, types.CCW) + if err != nil { + virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append scsi-controller") + return devices, nil + } + d.DevNo, err = b.AddressFormatCCW(addr) + if err != nil { + virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append scsi-controller") + return devices, nil + } + + devices = append(devices, d) + return devices, t +} diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index e95f5d044a..62799b9b2f 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -421,44 +421,6 @@ func TestQemuGrpc(t *testing.T) { assert.True(q.id == q2.id) } -func TestQemuAddDeviceToBridge(t *testing.T) { - assert := assert.New(t) - - config := newQemuConfig() - config.DefaultBridges = defaultBridges - - // addDeviceToBridge successfully - config.HypervisorMachineType = QemuPC - q := &qemu{ - config: config, - arch: newQemuArch(config), - } - - q.state.Bridges = q.arch.bridges(q.config.DefaultBridges) - // get pciBridgeMaxCapacity value from virtcontainers/types/pci.go - const pciBridgeMaxCapacity = 30 - for i := uint32(1); i <= pciBridgeMaxCapacity; i++ { - _, _, err := q.addDeviceToBridge(fmt.Sprintf("qemu-bridge-%d", i)) - assert.Nil(err) - } - - // fail to add device to bridge cause no more available bridge slot - _, _, err := q.addDeviceToBridge("qemu-bridge-31") - exceptErr := errors.New("no more bridge slots available") - assert.Equal(exceptErr.Error(), err.Error()) - - // addDeviceToBridge fails cause q.state.Bridges == 0 - config.HypervisorMachineType = QemuPCLite - q = &qemu{ - config: config, - arch: newQemuArch(config), - } - q.state.Bridges = q.arch.bridges(q.config.DefaultBridges) - _, _, err = q.addDeviceToBridge("qemu-bridge") - exceptErr = errors.New("failed to get available address from bridges") - assert.Equal(exceptErr.Error(), err.Error()) -} - func TestQemuFileBackedMem(t *testing.T) { assert := assert.New(t) diff --git a/virtcontainers/types/bridges.go b/virtcontainers/types/bridges.go new file mode 100644 index 0000000000..cb15a88f0e --- /dev/null +++ b/virtcontainers/types/bridges.go @@ -0,0 +1,115 @@ +// Copyright (c) 2017 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package types + +import "fmt" + +// Type represents a type of bus and bridge. +type Type string + +const PCIBridgeMaxCapacity = 30 + +const ( + // PCI represents a PCI bus and bridge + PCI Type = "pci" + + // PCIE represents a PCIe bus and bridge + PCIE Type = "pcie" +) + +const CCWBridgeMaxCapacity = 0xffff + +const ( + CCW Type = "ccw" +) + +type Bridge struct { + // Devices contains information about devices plugged and its address in the bridge + Devices map[uint32]string + + // ID is used to identify the bridge in the hypervisor + ID string + + // Addr is the slot of the bridge + Addr int + + // Type is the type of the bridge (pci, pcie, etc) + Type Type + + // MaxCapacity is the max capacity of the bridge + MaxCapacity uint32 +} + +func NewBridge(bt Type, id string, devices map[uint32]string, addr int) Bridge { + var maxCapacity uint32 + switch bt { + case PCI: + fallthrough + case PCIE: + maxCapacity = PCIBridgeMaxCapacity + case CCW: + maxCapacity = CCWBridgeMaxCapacity + default: + maxCapacity = 0 + } + return Bridge{ + Devices: devices, + ID: id, + Addr: addr, + Type: bt, + MaxCapacity: maxCapacity, + } +} + +func (b *Bridge) AddDevice(ID string) (uint32, error) { + var addr uint32 + + // looking for the first available address + for i := uint32(1); i <= b.MaxCapacity; i++ { + if _, ok := b.Devices[i]; !ok { + addr = i + break + } + } + + if addr == 0 { + return 0, fmt.Errorf("Unable to hot plug device on bridge: there are no empty slots") + } + + // save address and device + b.Devices[addr] = ID + return addr, nil +} + +func (b *Bridge) RemoveDevice(ID string) error { + // check if the device was hot plugged in the bridge + for addr, devID := range b.Devices { + if devID == ID { + // free address to re-use the same slot with other devices + delete(b.Devices, addr) + return nil + } + } + + return fmt.Errorf("Unable to hot unplug device %s: not present on bridge", ID) +} + +// AddressFormatCCW returns the address format for the device number. The channel subsystem-ID 0xfe is reserved to the virtual channel and the address format is in the form fe.n.dddd, where n is subchannel set ID and ddd the device number. More details at https://www.ibm.com/support/knowledgecenter/en/linuxonibm/com.ibm.linux.z.ldva/ldva_t_configuringSCSIdevices.html +func (b *Bridge) AddressFormatCCW(addr string) (string, error) { + if b.Type != CCW { + return "", fmt.Errorf("Expected bridge type %T, got %T (%+v)", CCW, b.Type, b) + } + + return fmt.Sprintf("fe.%x.%s", b.Addr, addr), nil +} + +// AddressFormatCCWForVirtServer returns the address format for the virtual server. The address format is in the form of 0.n.dddd +func (b *Bridge) AddressFormatCCWForVirtServer(addr string) (string, error) { + if b.Type != CCW { + return "", fmt.Errorf("Wrong bridge type") + } + return fmt.Sprintf("0.%x.%s", b.Addr, addr), nil +} diff --git a/virtcontainers/types/pci_test.go b/virtcontainers/types/bridges_test.go similarity index 51% rename from virtcontainers/types/pci_test.go rename to virtcontainers/types/bridges_test.go index 3a90cfed20..d8102eaa1d 100644 --- a/virtcontainers/types/pci_test.go +++ b/virtcontainers/types/bridges_test.go @@ -12,15 +12,12 @@ import ( "github.com/stretchr/testify/assert" ) -func TestAddRemoveDevice(t *testing.T) { +func testAddRemoveDevice(t *testing.T, b *Bridge) { assert := assert.New(t) - // create a bridge - bridges := []*PCIBridge{{make(map[uint32]string), PCI, "rgb123", 5}} - // add device devID := "abc123" - b := bridges[0] + addr, err := b.AddDevice(devID) assert.NoError(err) if addr < 1 { @@ -35,9 +32,9 @@ func TestAddRemoveDevice(t *testing.T) { assert.NoError(err) // add device when the bridge is full - bridges[0].Address = make(map[uint32]string) - for i := uint32(1); i <= pciBridgeMaxCapacity; i++ { - bridges[0].Address[i] = fmt.Sprintf("%d", i) + b.Devices = make(map[uint32]string) + for i := uint32(1); i <= b.MaxCapacity; i++ { + b.Devices[i] = fmt.Sprintf("%d", i) } addr, err = b.AddDevice(devID) assert.Error(err) @@ -45,3 +42,19 @@ func TestAddRemoveDevice(t *testing.T) { assert.Fail("address should be 0") } } + +func TestAddRemoveDevicePCI(t *testing.T) { + + // create a pci bridge + bridges := []*Bridge{{make(map[uint32]string), "rgb123", 5, PCI, PCIBridgeMaxCapacity}} + + testAddRemoveDevice(t, bridges[0]) +} + +func TestAddRemoveDeviceCCW(t *testing.T) { + + // create a CCW bridge + bridges := []*Bridge{{make(map[uint32]string), "rgb123", 5, CCW, CCWBridgeMaxCapacity}} + + testAddRemoveDevice(t, bridges[0]) +} diff --git a/virtcontainers/types/pci.go b/virtcontainers/types/pci.go deleted file mode 100644 index 78942600d7..0000000000 --- a/virtcontainers/types/pci.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2017 Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -package types - -import "fmt" - -// PCIType represents a type of PCI bus and bridge. -type PCIType string - -const ( - // PCI represents a PCI bus and bridge - PCI PCIType = "pci" - - // PCIE represents a PCIe bus and bridge - PCIE PCIType = "pcie" -) - -const pciBridgeMaxCapacity = 30 - -// PCIBridge is a PCI or PCIe bridge where devices can be hot plugged -type PCIBridge struct { - // Address contains information about devices plugged and its address in the bridge - Address map[uint32]string - - // Type is the PCI type of the bridge (pci, pcie, etc) - Type PCIType - - // ID is used to identify the bridge in the hypervisor - ID string - - // Addr is the PCI/e slot of the bridge - Addr int -} - -// AddDevice on success adds the device ID to the PCI bridge and returns -// the address where the device was added. -func (b *PCIBridge) AddDevice(ID string) (uint32, error) { - var addr uint32 - - // looking for the first available address - for i := uint32(1); i <= pciBridgeMaxCapacity; i++ { - if _, ok := b.Address[i]; !ok { - addr = i - break - } - } - - if addr == 0 { - return 0, fmt.Errorf("Unable to hot plug device on bridge: there are not empty slots") - } - - // save address and device - b.Address[addr] = ID - return addr, nil -} - -// RemoveDevice removes the device ID from the PCI bridge. -func (b *PCIBridge) RemoveDevice(ID string) error { - // check if the device was hot plugged in the bridge - for addr, devID := range b.Address { - if devID == ID { - // free address to re-use the same slot with other devices - delete(b.Address, addr) - return nil - } - } - - return fmt.Errorf("Unable to hot unplug device %s: not present on bridge", ID) -}