Merge pull request #1965 from jschintag/virtio-blk-ccw

s390x: add virtio-blk-ccw support

pkg/katautils/config.go

@@ -340,7 +340,7 @@ func (h hypervisor) defaultBridges() uint32 {
 }
 
 func (h hypervisor) blockDeviceDriver() (string, error) {
-	supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm}
+	supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm, config.VirtioBlockCCW}
 
 	if h.BlockDeviceDriver == "" {
 		return defaultBlockDeviceDriver, nil

virtcontainers/device/config/config.go

@@ -48,6 +48,9 @@ const (
 	// VirtioBlock means use virtio-blk for hotplugging drives
 	VirtioBlock = "virtio-blk"
 
+	// VirtioBlockCCW means use virtio-blk for hotplugging drives
+	VirtioBlockCCW = "virtio-blk-ccw"
+
 	// VirtioSCSI means use virtio-scsi for hotplugging drives
 	VirtioSCSI = "virtio-scsi"
 
@@ -138,6 +141,9 @@ type BlockDrive struct {
 
 	// VirtPath at which the device appears inside the VM, outside of the container mount namespace
 	VirtPath string
+
+	// DevNo identifies the css bus id for virtio-blk-ccw
+	DevNo string
 }
 
 // VFIODeviceType indicates VFIO device type

virtcontainers/device/drivers/block.go

@@ -84,6 +84,8 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
 		switch customOptions["block-driver"] {
 		case "virtio-blk":
 			globalIdx = index
+		case "virtio-blk-ccw":
+			globalIdx = index
 		case "virtio-mmio":
 			//With firecracker the rootfs for the VM itself
 			//sits at /dev/vda and consumes the first index.
@@ -164,6 +166,7 @@ func (device *BlockDevice) Save() persistapi.DeviceState {
 			SCSIAddr: drive.SCSIAddr,
 			NvdimmID: drive.NvdimmID,
 			VirtPath: drive.VirtPath,
+			DevNo:    drive.DevNo,
 		}
 	}
 	return ds
@@ -188,6 +191,7 @@ func (device *BlockDevice) Load(ds persistapi.DeviceState) {
 		SCSIAddr: bd.SCSIAddr,
 		NvdimmID: bd.NvdimmID,
 		VirtPath: bd.VirtPath,
+		DevNo:    bd.DevNo,
 	}
 }
 

virtcontainers/device/manager/manager.go

@@ -25,6 +25,8 @@ const (
 	VirtioMmio string = "virtio-mmio"
 	// VirtioBlock indicates block driver is virtio-blk based
 	VirtioBlock string = "virtio-blk"
+	// VirtioBlockCCW indicates block driver is virtio-blk-ccw based
+	VirtioBlockCCW string = "virtio-blk-ccw"
 	// VirtioSCSI indicates block driver is virtio-scsi based
 	VirtioSCSI string = "virtio-scsi"
 	// Nvdimm indicates block driver is nvdimm based
@@ -66,6 +68,8 @@ func NewDeviceManager(blockDriver string, devices []api.Device) api.DeviceManage
 		dm.blockDriver = VirtioBlock
 	} else if blockDriver == Nvdimm {
 		dm.blockDriver = Nvdimm
+	} else if blockDriver == VirtioBlockCCW {
+		dm.blockDriver = VirtioBlockCCW
 	} else {
 		dm.blockDriver = VirtioSCSI
 	}

virtcontainers/kata_agent.go

@@ -75,6 +75,7 @@ var (
 	kata9pDevType               = "9p"
 	kataMmioBlkDevType          = "mmioblk"
 	kataBlkDevType              = "blk"
+	kataBlkCCWDevType           = "blk-ccw"
 	kataSCSIDevType             = "scsi"
 	kataNvdimmDevType           = "nvdimm"
 	kataVirtioFSDevType         = "virtio-fs"
@@ -1106,6 +1107,9 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, c *Container) []*gr
 			kataDevice.Type = kataMmioBlkDevType
 			kataDevice.Id = d.VirtPath
 			kataDevice.VmPath = d.VirtPath
+		case config.VirtioBlockCCW:
+			kataDevice.Type = kataBlkCCWDevType
+			kataDevice.Id = d.DevNo
 		case config.VirtioBlock:
 			kataDevice.Type = kataBlkDevType
 			kataDevice.Id = d.PCIAddr
@@ -1160,11 +1164,14 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat
 			k.Logger().Error("malformed block drive")
 			return nil, fmt.Errorf("malformed block drive")
 		}
-
+		switch {
-		if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio {
+		case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio:
 			rootfs.Driver = kataMmioBlkDevType
 			rootfs.Source = blockDrive.VirtPath
-		} else if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock {
+		case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlockCCW:
+			rootfs.Driver = kataBlkCCWDevType
+			rootfs.Source = blockDrive.DevNo
+		case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock:
 			rootfs.Driver = kataBlkDevType
 			if blockDrive.PCIAddr == "" {
 				rootfs.Source = blockDrive.VirtPath
@@ -1172,10 +1179,14 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat
 				rootfs.Source = blockDrive.PCIAddr
 			}
 
-		} else {
+		case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioSCSI:
+
 			rootfs.Driver = kataSCSIDevType
 			rootfs.Source = blockDrive.SCSIAddr
+		default:
+			return nil, fmt.Errorf("Unknown block device driver: %s", sandbox.config.HypervisorConfig.BlockDeviceDriver)
 		}
+
 		rootfs.MountPoint = rootPathParent
 		rootfs.Fstype = c.state.Fstype
 
@@ -1281,7 +1292,10 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
 	// Note this call modifies the list of container devices to make sure
 	// all hotplugged devices are unplugged, so this needs be done
 	// after devices passed with --device are handled.
-	volumeStorages := k.handleBlockVolumes(c)
+	volumeStorages, err := k.handleBlockVolumes(c)
+	if err != nil {
+		return nil, err
+	}
 	if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil {
 		return nil, err
 	}
@@ -1400,7 +1414,7 @@ func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, r
 
 // handleBlockVolumes handles volumes that are block devices files
 // by passing the block devices as Storage to the agent.
-func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
+func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
 
 	var volumeStorages []*grpc.Storage
 
@@ -1418,7 +1432,7 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
 		if !c.sandbox.supportNewStore() {
 			if err := c.storeDevices(); err != nil {
 				k.Logger().WithField("device", id).WithError(err).Error("store device failed")
-				return nil
+				return nil, err
 			}
 		}
 
@@ -1427,22 +1441,28 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
 		device := c.sandbox.devManager.GetDeviceByID(id)
 		if device == nil {
 			k.Logger().WithField("device", id).Error("failed to find device by id")
-			return nil
+			return nil, fmt.Errorf("Failed to find device by id (id=%s)", id)
 		}
 		blockDrive, ok := device.GetDeviceInfo().(*config.BlockDrive)
 		if !ok || blockDrive == nil {
 			k.Logger().Error("malformed block drive")
 			continue
 		}
-		if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock {
+		switch {
+		case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlockCCW:
+			vol.Driver = kataBlkCCWDevType
+			vol.Source = blockDrive.DevNo
+		case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock:
 			vol.Driver = kataBlkDevType
 			vol.Source = blockDrive.PCIAddr
-		} else if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio {
+		case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio:
 			vol.Driver = kataMmioBlkDevType
 			vol.Source = blockDrive.VirtPath
-		} else {
+		case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioSCSI:
 			vol.Driver = kataSCSIDevType
 			vol.Source = blockDrive.SCSIAddr
+		default:
+			return nil, fmt.Errorf("Unknown block device driver: %s", c.sandbox.config.HypervisorConfig.BlockDeviceDriver)
 		}
 
 		vol.MountPoint = m.Destination
@@ -1452,7 +1472,7 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
 		volumeStorages = append(volumeStorages, vol)
 	}
 
-	return volumeStorages
+	return volumeStorages, nil
 }
 
 // handlePidNamespace checks if Pid namespace for a container needs to be shared with its sandbox

virtcontainers/persist/api/device.go

@@ -38,6 +38,9 @@ type BlockDrive struct {
 
 	// VirtPath at which the device appears inside the VM, outside of the container mount namespace
 	VirtPath string
+
+	// DevNo
+	DevNo string
 }
 
 // VFIODev represents a VFIO drive used for hotplugging

virtcontainers/qemu.go

@@ -61,7 +61,7 @@ type CPUDevice struct {
 
 // QemuState keeps Qemu's state
 type QemuState struct {
-	Bridges []types.PCIBridge
+	Bridges []types.Bridge
 	// HotpluggedCPUs is the list of CPUs that were hot-added
 	HotpluggedVCPUs      []CPUDevice
 	HotpluggedMemory     int
@@ -266,9 +266,11 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto
 		create = true
 	}
 
+	q.arch.setBridges(q.state.Bridges)
+
 	if create {
 		q.Logger().Debug("Creating bridges")
-		q.state.Bridges = q.arch.bridges(q.config.DefaultBridges)
+		q.arch.bridges(q.config.DefaultBridges)
 
 		q.Logger().Debug("Creating UUID")
 		q.state.UUID = uuid.Generate().String()
@@ -403,7 +405,7 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
 
 	// Add bridges before any other devices. This way we make sure that
 	// bridge gets the first available PCI address i.e bridgePCIStartAddr
-	devices = q.arch.appendBridges(devices, q.state.Bridges)
+	devices = q.arch.appendBridges(devices)
 
 	devices = q.arch.appendConsole(devices, console)
 
@@ -984,38 +986,6 @@ func (q *qemu) qmpShutdown() {
 	}
 }
 
-func (q *qemu) addDeviceToBridge(ID string) (string, types.PCIBridge, error) {
-	var err error
-	var addr uint32
-
-	if len(q.state.Bridges) == 0 {
-		return "", types.PCIBridge{}, errors.New("failed to get available address from bridges")
-	}
-
-	// looking for an empty address in the bridges
-	for _, b := range q.state.Bridges {
-		addr, err = b.AddDevice(ID)
-		if err == nil {
-			return fmt.Sprintf("%02x", addr), b, nil
-		}
-	}
-
-	return "", types.PCIBridge{}, fmt.Errorf("no more bridge slots available")
-}
-
-func (q *qemu) removeDeviceFromBridge(ID string) error {
-	var err error
-	for _, b := range q.state.Bridges {
-		err = b.RemoveDevice(ID)
-		if err == nil {
-			// device was removed correctly
-			return nil
-		}
-	}
-
-	return err
-}
-
 func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, devID string) (err error) {
 	if q.config.BlockDeviceDriver == config.Nvdimm {
 		var blocksize int64
@@ -1050,16 +1020,36 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
 		}
 	}()
 
-	if q.config.BlockDeviceDriver == config.VirtioBlock {
+	switch {
+	case q.config.BlockDeviceDriver == config.VirtioBlockCCW:
+		driver := "virtio-blk-ccw"
+
+		addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.CCW)
+		if err != nil {
+			return err
+		}
+		var devNoHotplug string
+		devNoHotplug, err = bridge.AddressFormatCCW(addr)
+		if err != nil {
+			return err
+		}
+		drive.DevNo, err = bridge.AddressFormatCCWForVirtServer(addr)
+		if err != nil {
+			return err
+		}
+		if err = q.qmpMonitorCh.qmp.ExecuteDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, devNoHotplug, "", true, false); err != nil {
+			return err
+		}
+	case q.config.BlockDeviceDriver == config.VirtioBlock:
 		driver := "virtio-blk-pci"
-		addr, bridge, err := q.addDeviceToBridge(drive.ID)
+		addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.PCI)
 		if err != nil {
 			return err
 		}
 
 		defer func() {
 			if err != nil {
-				q.removeDeviceFromBridge(drive.ID)
+				q.arch.removeDeviceFromBridge(drive.ID)
 			}
 		}()
 
@@ -1069,7 +1059,7 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
 		if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil {
 			return err
 		}
-	} else {
+	case q.config.BlockDeviceDriver == config.VirtioSCSI:
 		driver := "scsi-hd"
 
 		// Bus exposed by the SCSI Controller
@@ -1084,6 +1074,8 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
 		if err = q.qmpMonitorCh.qmp.ExecuteSCSIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, bus, romFile, scsiID, lun, true, defaultDisableModern); err != nil {
 			return err
 		}
+	default:
+		return fmt.Errorf("Block device %s not recognized", q.config.BlockDeviceDriver)
 	}
 
 	return nil
@@ -1101,7 +1093,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
 		err = q.hotplugAddBlockDevice(drive, op, devID)
 	} else {
 		if q.config.BlockDeviceDriver == config.VirtioBlock {
-			if err := q.removeDeviceFromBridge(drive.ID); err != nil {
+			if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil {
 				return err
 			}
 		}
@@ -1141,14 +1133,14 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
 			}
 		}
 
-		addr, bridge, err := q.addDeviceToBridge(devID)
+		addr, bridge, err := q.arch.addDeviceToBridge(devID, types.PCI)
 		if err != nil {
 			return err
 		}
 
 		defer func() {
 			if err != nil {
-				q.removeDeviceFromBridge(devID)
+				q.arch.removeDeviceFromBridge(devID)
 			}
 		}()
 
@@ -1162,7 +1154,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
 		}
 	} else {
 		if !q.state.HotplugVFIOOnRootBus {
-			if err := q.removeDeviceFromBridge(devID); err != nil {
+			if err := q.arch.removeDeviceFromBridge(devID); err != nil {
 				return err
 			}
 		}
@@ -1228,14 +1220,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
 			}
 		}()
 
-		addr, bridge, err := q.addDeviceToBridge(tap.ID)
+		addr, bridge, err := q.arch.addDeviceToBridge(tap.ID, types.PCI)
 		if err != nil {
 			return err
 		}
 
 		defer func() {
 			if err != nil {
-				q.removeDeviceFromBridge(tap.ID)
+				q.arch.removeDeviceFromBridge(tap.ID)
 			}
 		}()
 
@@ -1248,12 +1240,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
 			return err
 		}
 		if machine.Type == QemuCCWVirtio {
-			return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), bridge.ID, int(q.config.NumVCPUs))
+			devNoHotplug := fmt.Sprintf("fe.%x.%x", bridge.Addr, addr)
+			return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs))
 		}
 		return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, romFile, int(q.config.NumVCPUs), defaultDisableModern)
+
 	}
 
-	if err := q.removeDeviceFromBridge(tap.ID); err != nil {
+	if err := q.arch.removeDeviceFromBridge(tap.ID); err != nil {
 		return err
 	}
 
@@ -1750,7 +1744,7 @@ func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe boo
 
 // genericAppendBridges appends to devices the given bridges
 // nolint: unused, deadcode
-func genericAppendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge, machineType string) []govmmQemu.Device {
+func genericAppendBridges(devices []govmmQemu.Device, bridges []types.Bridge, machineType string) []govmmQemu.Device {
 	bus := defaultPCBridgeBus
 	switch machineType {
 	case QemuQ35, QemuVirt:
@@ -1762,6 +1756,9 @@ func genericAppendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge,
 		if b.Type == types.PCIE {
 			t = govmmQemu.PCIEBridge
 		}
+		if b.Type == types.CCW {
+			continue
+		}
 
 		bridges[idx].Addr = bridgePCIStartAddr + idx
 
@@ -1781,10 +1778,9 @@ func genericAppendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge,
 	return devices
 }
 
-// nolint: unused, deadcode
+func genericBridges(number uint32, machineType string) []types.Bridge {
-func genericBridges(number uint32, machineType string) []types.PCIBridge {
+	var bridges []types.Bridge
-	var bridges []types.PCIBridge
+	var bt types.Type
-	var bt types.PCIType
 
 	switch machineType {
 	case QemuQ35:
@@ -1798,17 +1794,13 @@ func genericBridges(number uint32, machineType string) []types.PCIBridge {
 	case QemuPseries:
 		bt = types.PCI
 	case QemuCCWVirtio:
-		bt = types.PCI
+		bt = types.CCW
 	default:
 		return nil
 	}
 
 	for i := uint32(0); i < number; i++ {
-		bridges = append(bridges, types.PCIBridge{
+		bridges = append(bridges, types.NewBridge(bt, fmt.Sprintf("%s-bridge-%d", bt, i), make(map[uint32]string), 0))
-			Type:    bt,
-			ID:      fmt.Sprintf("%s-bridge-%d", bt, i),
-			Address: make(map[uint32]string),
-		})
 	}
 
 	return bridges
@@ -1990,6 +1982,7 @@ func (q *qemu) toGrpc() ([]byte, error) {
 
 func (q *qemu) storeState() error {
 	if q.store != nil {
+		q.state.Bridges = q.arch.getBridges()
 		if err := q.store.Store(store.Hypervisor, q.state); err != nil {
 			return err
 		}
@@ -2008,9 +2001,9 @@ func (q *qemu) save() (s persistapi.HypervisorState) {
 	s.HotpluggedMemory = q.state.HotpluggedMemory
 	s.HotplugVFIOOnRootBus = q.state.HotplugVFIOOnRootBus
 
-	for _, bridge := range q.state.Bridges {
+	for _, bridge := range q.arch.getBridges() {
 		s.Bridges = append(s.Bridges, persistapi.Bridge{
-			DeviceAddr: bridge.Address,
+			DeviceAddr: bridge.Devices,
 			Type:       string(bridge.Type),
 			ID:         bridge.ID,
 			Addr:       bridge.Addr,
@@ -2032,12 +2025,7 @@ func (q *qemu) load(s persistapi.HypervisorState) {
 	q.state.VirtiofsdPid = s.VirtiofsdPid
 
 	for _, bridge := range s.Bridges {
-		q.state.Bridges = append(q.state.Bridges, types.PCIBridge{
+		q.state.Bridges = append(q.state.Bridges, types.NewBridge(types.Type(bridge.Type), bridge.ID, bridge.DeviceAddr, bridge.Addr))
-			Address: bridge.DeviceAddr,
-			Type:    types.PCIType(bridge.Type),
-			ID:      bridge.ID,
-			Addr:    bridge.Addr,
-		})
 	}
 
 	for _, cpu := range s.HotpluggedVCPUs {

virtcontainers/qemu_amd64.go

@@ -122,8 +122,8 @@ func (q *qemuAmd64) capabilities() types.Capabilities {
 	return caps
 }
 
-func (q *qemuAmd64) bridges(number uint32) []types.PCIBridge {
+func (q *qemuAmd64) bridges(number uint32) {
-	return genericBridges(number, q.machineType)
+	q.Bridges = genericBridges(number, q.machineType)
 }
 
 func (q *qemuAmd64) cpuModel() string {
@@ -173,6 +173,6 @@ func (q *qemuAmd64) appendImage(devices []govmmQemu.Device, path string) ([]govm
 }
 
 // appendBridges appends to devices the given bridges
-func (q *qemuAmd64) appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device {
+func (q *qemuAmd64) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device {
-	return genericAppendBridges(devices, bridges, q.machineType)
+	return genericAppendBridges(devices, q.Bridges, q.machineType)
 }

virtcontainers/qemu_amd64_test.go

@@ -40,29 +40,32 @@ func TestQemuAmd64Bridges(t *testing.T) {
 	amd64 := newTestQemu(QemuPC)
 	len := 5
 
-	bridges := amd64.bridges(uint32(len))
+	amd64.bridges(uint32(len))
+	bridges := amd64.getBridges()
 	assert.Len(bridges, len)
 
 	for i, b := range bridges {
 		id := fmt.Sprintf("%s-bridge-%d", types.PCI, i)
 		assert.Equal(types.PCI, b.Type)
 		assert.Equal(id, b.ID)
-		assert.NotNil(b.Address)
+		assert.NotNil(b.Devices)
 	}
 
 	amd64 = newTestQemu(QemuQ35)
-	bridges = amd64.bridges(uint32(len))
+	amd64.bridges(uint32(len))
+	bridges = amd64.getBridges()
 	assert.Len(bridges, len)
 
 	for i, b := range bridges {
 		id := fmt.Sprintf("%s-bridge-%d", types.PCI, i)
 		assert.Equal(types.PCI, b.Type)
 		assert.Equal(id, b.ID)
-		assert.NotNil(b.Address)
+		assert.NotNil(b.Devices)
 	}
 
 	amd64 = newTestQemu(QemuQ35 + QemuPC)
-	bridges = amd64.bridges(uint32(len))
+	amd64.bridges(uint32(len))
+	bridges = amd64.getBridges()
 	assert.Nil(bridges)
 }
 
@@ -143,10 +146,11 @@ func TestQemuAmd64AppendBridges(t *testing.T) {
 	// check PC
 	amd64 := newTestQemu(QemuPC)
 
-	bridges := amd64.bridges(1)
+	amd64.bridges(1)
+	bridges := amd64.getBridges()
 	assert.Len(bridges, 1)
 
-	devices = amd64.appendBridges(devices, bridges)
+	devices = amd64.appendBridges(devices)
 	assert.Len(devices, 1)
 
 	expectedOut := []govmmQemu.Device{
@@ -165,11 +169,12 @@ func TestQemuAmd64AppendBridges(t *testing.T) {
 	// Check Q35
 	amd64 = newTestQemu(QemuQ35)
 
-	bridges = amd64.bridges(1)
+	amd64.bridges(1)
+	bridges = amd64.getBridges()
 	assert.Len(bridges, 1)
 
 	devices = []govmmQemu.Device{}
-	devices = amd64.appendBridges(devices, bridges)
+	devices = amd64.appendBridges(devices)
 	assert.Len(devices, 1)
 
 	expectedOut = []govmmQemu.Device{

virtcontainers/qemu_arch_base.go

@@ -8,6 +8,7 @@ package virtcontainers
 import (
 	"context"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"os"
 	"strconv"
@@ -48,8 +49,8 @@ type qemuArch interface {
 	//capabilities returns the capabilities supported by QEMU
 	capabilities() types.Capabilities
 
-	// bridges returns the number bridges for the machine type
+	// bridges sets the number bridges for the machine type
-	bridges(number uint32) []types.PCIBridge
+	bridges(number uint32)
 
 	// cpuTopology returns the CPU topology for the given amount of vcpus
 	cpuTopology(vcpus, maxvcpus uint32) govmmQemu.SMP
@@ -70,7 +71,7 @@ type qemuArch interface {
 	appendSCSIController(devices []govmmQemu.Device, enableIOThreads bool) ([]govmmQemu.Device, *govmmQemu.IOThread)
 
 	// appendBridges appends bridges to devices
-	appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device
+	appendBridges(devices []govmmQemu.Device) []govmmQemu.Device
 
 	// append9PVolume appends a 9P volume to devices
 	append9PVolume(devices []govmmQemu.Device, volume types.Volume) []govmmQemu.Device
@@ -96,6 +97,21 @@ type qemuArch interface {
 	// appendRNGDevice appends a RNG device to devices
 	appendRNGDevice(devices []govmmQemu.Device, rngDevice config.RNGDev) []govmmQemu.Device
 
+	// addDeviceToBridge adds devices to the bus
+	addDeviceToBridge(ID string, t types.Type) (string, types.Bridge, error)
+
+	// removeDeviceFromBridge removes devices to the bus
+	removeDeviceFromBridge(ID string) error
+
+	// getBridges grants access to Bridges
+	getBridges() []types.Bridge
+
+	// setBridges grants access to Bridges
+	setBridges(bridges []types.Bridge)
+
+	// addBridge adds a new Bridge to the list of Bridges
+	addBridge(types.Bridge)
+
 	// handleImagePath handles the Hypervisor Config image path
 	handleImagePath(config HypervisorConfig)
 
@@ -117,6 +133,7 @@ type qemuArchBase struct {
 	kernelParamsNonDebug  []Param
 	kernelParamsDebug     []Param
 	kernelParams          []Param
+	Bridges               []types.Bridge
 }
 
 const (
@@ -242,18 +259,10 @@ func (q *qemuArchBase) capabilities() types.Capabilities {
 	return caps
 }
 
-func (q *qemuArchBase) bridges(number uint32) []types.PCIBridge {
+func (q *qemuArchBase) bridges(number uint32) {
-	var bridges []types.PCIBridge
-
 	for i := uint32(0); i < number; i++ {
-		bridges = append(bridges, types.PCIBridge{
+		q.Bridges = append(q.Bridges, types.NewBridge(types.PCI, fmt.Sprintf("%s-bridge-%d", types.PCI, i), make(map[uint32]string), 0))
-			Type:    types.PCI,
-			ID:      fmt.Sprintf("%s-bridge-%d", types.PCI, i),
-			Address: make(map[uint32]string),
-		})
 	}
-
-	return bridges
 }
 
 func (q *qemuArchBase) cpuTopology(vcpus, maxvcpus uint32) govmmQemu.SMP {
@@ -306,14 +315,14 @@ func (q *qemuArchBase) appendConsole(devices []govmmQemu.Device, path string) []
 	return devices
 }
 
-func (q *qemuArchBase) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) {
+func genericImage(path string) (config.BlockDrive, error) {
 	if _, err := os.Stat(path); os.IsNotExist(err) {
-		return nil, err
+		return config.BlockDrive{}, err
 	}
 
 	randBytes, err := utils.GenerateRandomBytes(8)
 	if err != nil {
-		return nil, err
+		return config.BlockDrive{}, err
 	}
 
 	id := utils.MakeNameID("image", hex.EncodeToString(randBytes), maxDevIDSize)
@@ -324,13 +333,21 @@ func (q *qemuArchBase) appendImage(devices []govmmQemu.Device, path string) ([]g
 		ID:     id,
 	}
 
+	return drive, nil
+}
+
+func (q *qemuArchBase) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) {
+	drive, err := genericImage(path)
+	if err != nil {
+		return nil, err
+	}
 	return q.appendBlockDevice(devices, drive), nil
 }
 
-func (q *qemuArchBase) appendSCSIController(devices []govmmQemu.Device, enableIOThreads bool) ([]govmmQemu.Device, *govmmQemu.IOThread) {
+func genericSCSIController(enableIOThreads, nestedRun bool) (govmmQemu.SCSIController, *govmmQemu.IOThread) {
 	scsiController := govmmQemu.SCSIController{
 		ID:            scsiControllerID,
-		DisableModern: q.nestedRun,
+		DisableModern: nestedRun,
 	}
 
 	var t *govmmQemu.IOThread
@@ -345,20 +362,27 @@ func (q *qemuArchBase) appendSCSIController(devices []govmmQemu.Device, enableIO
 		scsiController.IOThread = t.ID
 	}
 
-	devices = append(devices, scsiController)
+	return scsiController, t
+}
 
+func (q *qemuArchBase) appendSCSIController(devices []govmmQemu.Device, enableIOThreads bool) ([]govmmQemu.Device, *govmmQemu.IOThread) {
+	d, t := genericSCSIController(enableIOThreads, q.nestedRun)
+	devices = append(devices, d)
 	return devices, t
 }
 
 // appendBridges appends to devices the given bridges
-func (q *qemuArchBase) appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device {
+func (q *qemuArchBase) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device {
-	for idx, b := range bridges {
+	for idx, b := range q.Bridges {
+		if b.Type == types.CCW {
+			continue
+		}
 		t := govmmQemu.PCIBridge
 		if b.Type == types.PCIE {
 			t = govmmQemu.PCIEBridge
 		}
 
-		bridges[idx].Addr = bridgePCIStartAddr + idx
+		q.Bridges[idx].Addr = bridgePCIStartAddr + idx
 
 		devices = append(devices,
 			govmmQemu.BridgeDevice{
@@ -368,7 +392,7 @@ func (q *qemuArchBase) appendBridges(devices []govmmQemu.Device, bridges []types
 				// Each bridge is required to be assigned a unique chassis id > 0
 				Chassis: idx + 1,
 				SHPC:    true,
-				Addr:    strconv.FormatInt(int64(bridges[idx].Addr), 10),
+				Addr:    strconv.FormatInt(int64(q.Bridges[idx].Addr), 10),
 			},
 		)
 	}
@@ -376,28 +400,30 @@ func (q *qemuArchBase) appendBridges(devices []govmmQemu.Device, bridges []types
 	return devices
 }
 
-func (q *qemuArchBase) append9PVolume(devices []govmmQemu.Device, volume types.Volume) []govmmQemu.Device {
+func generic9PVolume(volume types.Volume, nestedRun bool) govmmQemu.FSDevice {
-	if volume.MountTag == "" || volume.HostPath == "" {
-		return devices
-	}
-
 	devID := fmt.Sprintf("extra-9p-%s", volume.MountTag)
 	if len(devID) > maxDevIDSize {
 		devID = devID[:maxDevIDSize]
 	}
 
-	devices = append(devices,
+	return govmmQemu.FSDevice{
-		govmmQemu.FSDevice{
+		Driver:        govmmQemu.Virtio9P,
-			Driver:        govmmQemu.Virtio9P,
+		FSDriver:      govmmQemu.Local,
-			FSDriver:      govmmQemu.Local,
+		ID:            devID,
-			ID:            devID,
+		Path:          volume.HostPath,
-			Path:          volume.HostPath,
+		MountTag:      volume.MountTag,
-			MountTag:      volume.MountTag,
+		SecurityModel: govmmQemu.None,
-			SecurityModel: govmmQemu.None,
+		DisableModern: nestedRun,
-			DisableModern: q.nestedRun,
+	}
-		},
+}
-	)
 
+func (q *qemuArchBase) append9PVolume(devices []govmmQemu.Device, volume types.Volume) []govmmQemu.Device {
+	if volume.MountTag == "" || volume.HostPath == "" {
+		return devices
+	}
+
+	d := generic9PVolume(volume, q.nestedRun)
+	devices = append(devices, d)
 	return devices
 }
 
@@ -452,70 +478,83 @@ func networkModelToQemuType(model NetInterworkingModel) govmmQemu.NetDeviceType
 	}
 }
 
-func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device {
+func genericNetwork(endpoint Endpoint, vhost, nestedRun bool, index int) (govmmQemu.NetDevice, error) {
+	var d govmmQemu.NetDevice
 	switch ep := endpoint.(type) {
 	case *VethEndpoint, *BridgedMacvlanEndpoint, *IPVlanEndpoint:
 		netPair := ep.NetworkPair()
-		devices = append(devices,
+		d = govmmQemu.NetDevice{
-			govmmQemu.NetDevice{
+			Type:          networkModelToQemuType(netPair.NetInterworkingModel),
-				Type:          networkModelToQemuType(netPair.NetInterworkingModel),
+			Driver:        govmmQemu.VirtioNet,
-				Driver:        govmmQemu.VirtioNet,
+			ID:            fmt.Sprintf("network-%d", index),
-				ID:            fmt.Sprintf("network-%d", q.networkIndex),
+			IFName:        netPair.TAPIface.Name,
-				IFName:        netPair.TAPIface.Name,
+			MACAddress:    netPair.TAPIface.HardAddr,
-				MACAddress:    netPair.TAPIface.HardAddr,
+			DownScript:    "no",
-				DownScript:    "no",
+			Script:        "no",
-				Script:        "no",
+			VHost:         vhost,
-				VHost:         q.vhost,
+			DisableModern: nestedRun,
-				DisableModern: q.nestedRun,
+			FDs:           netPair.VMFds,
-				FDs:           netPair.VMFds,
+			VhostFDs:      netPair.VhostFds,
-				VhostFDs:      netPair.VhostFds,
+		}
-			},
-		)
-		q.networkIndex++
 	case *MacvtapEndpoint:
-		devices = append(devices,
+		d = govmmQemu.NetDevice{
-			govmmQemu.NetDevice{
+			Type:          govmmQemu.MACVTAP,
-				Type:          govmmQemu.MACVTAP,
+			Driver:        govmmQemu.VirtioNet,
-				Driver:        govmmQemu.VirtioNet,
+			ID:            fmt.Sprintf("network-%d", index),
-				ID:            fmt.Sprintf("network-%d", q.networkIndex),
+			IFName:        ep.Name(),
-				IFName:        ep.Name(),
+			MACAddress:    ep.HardwareAddr(),
-				MACAddress:    ep.HardwareAddr(),
+			DownScript:    "no",
-				DownScript:    "no",
+			Script:        "no",
-				Script:        "no",
+			VHost:         vhost,
-				VHost:         q.vhost,
+			DisableModern: nestedRun,
-				DisableModern: q.nestedRun,
+			FDs:           ep.VMFds,
-				FDs:           ep.VMFds,
+			VhostFDs:      ep.VhostFds,
-				VhostFDs:      ep.VhostFds,
+		}
-			},
+	default:
-		)
+		return govmmQemu.NetDevice{}, fmt.Errorf("Unknown type for endpoint")
-		q.networkIndex++
-
 	}
 
+	return d, nil
+}
+
+func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device {
+	d, err := genericNetwork(endpoint, q.vhost, q.nestedRun, q.networkIndex)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemuArch").WithError(err).Error("Failed to append network")
+		return devices
+	}
+	q.networkIndex++
+	devices = append(devices, d)
 	return devices
 }
 
-func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device {
+func genericBlockDevice(drive config.BlockDrive, nestedRun bool) (govmmQemu.BlockDevice, error) {
 	if drive.File == "" || drive.ID == "" || drive.Format == "" {
-		return devices
+		return govmmQemu.BlockDevice{}, fmt.Errorf("Empty File, ID or Format for drive %v", drive)
 	}
 
 	if len(drive.ID) > maxDevIDSize {
 		drive.ID = drive.ID[:maxDevIDSize]
 	}
 
-	devices = append(devices,
+	return govmmQemu.BlockDevice{
-		govmmQemu.BlockDevice{
+		Driver:        govmmQemu.VirtioBlock,
-			Driver:        govmmQemu.VirtioBlock,
+		ID:            drive.ID,
-			ID:            drive.ID,
+		File:          drive.File,
-			File:          drive.File,
+		AIO:           govmmQemu.Threads,
-			AIO:           govmmQemu.Threads,
+		Format:        govmmQemu.BlockDeviceFormat(drive.Format),
-			Format:        govmmQemu.BlockDeviceFormat(drive.Format),
+		Interface:     "none",
-			Interface:     "none",
+		DisableModern: nestedRun,
-			DisableModern: q.nestedRun,
+	}, nil
-		},
+}
-	)
 
+func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device {
+	d, err := genericBlockDevice(drive, q.nestedRun)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemuArch").WithError(err).Error("Failed to append block device")
+		return devices
+	}
+	devices = append(devices, d)
 	return devices
 }
 
@@ -592,3 +631,55 @@ func (q *qemuArchBase) setIgnoreSharedMemoryMigrationCaps(ctx context.Context, q
 	})
 	return err
 }
+
+func (q *qemuArchBase) addDeviceToBridge(ID string, t types.Type) (string, types.Bridge, error) {
+	var err error
+	var addr uint32
+
+	if len(q.Bridges) == 0 {
+		return "", types.Bridge{}, errors.New("failed to get available address from bridges")
+	}
+
+	// looking for an empty address in the bridges
+	for _, b := range q.Bridges {
+		if t != b.Type {
+			continue
+		}
+		addr, err = b.AddDevice(ID)
+		if err == nil {
+			switch t {
+			case types.CCW:
+				return fmt.Sprintf("%04x", addr), b, nil
+			case types.PCI, types.PCIE:
+				return fmt.Sprintf("%02x", addr), b, nil
+			}
+		}
+	}
+
+	return "", types.Bridge{}, fmt.Errorf("no more bridge slots available")
+}
+
+func (q *qemuArchBase) removeDeviceFromBridge(ID string) error {
+	var err error
+	for _, b := range q.Bridges {
+		err = b.RemoveDevice(ID)
+		if err == nil {
+			// device was removed correctly
+			return nil
+		}
+	}
+
+	return err
+}
+
+func (q *qemuArchBase) getBridges() []types.Bridge {
+	return q.Bridges
+}
+
+func (q *qemuArchBase) setBridges(bridges []types.Bridge) {
+	q.Bridges = bridges
+}
+
+func (q *qemuArchBase) addBridge(b types.Bridge) {
+	q.Bridges = append(q.Bridges, b)
+}

virtcontainers/qemu_arch_base_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -147,14 +148,44 @@ func TestQemuArchBaseBridges(t *testing.T) {
 	qemuArchBase := newQemuArchBase()
 	len := 5
 
-	bridges := qemuArchBase.bridges(uint32(len))
+	qemuArchBase.bridges(uint32(len))
+	bridges := qemuArchBase.getBridges()
 	assert.Len(bridges, len)
 
 	for i, b := range bridges {
 		id := fmt.Sprintf("%s-bridge-%d", types.PCI, i)
 		assert.Equal(types.PCI, b.Type)
 		assert.Equal(id, b.ID)
-		assert.NotNil(b.Address)
+		assert.NotNil(b.Devices)
+	}
+}
+
+func TestQemuAddDeviceToBridge(t *testing.T) {
+	assert := assert.New(t)
+
+	// addDeviceToBridge successfully
+	q := newQemuArchBase()
+	q.machineType = QemuPC
+
+	q.bridges(1)
+	for i := uint32(1); i <= types.PCIBridgeMaxCapacity; i++ {
+		_, _, err := q.addDeviceToBridge(fmt.Sprintf("qemu-bridge-%d", i), types.PCI)
+		assert.Nil(err)
+	}
+
+	// fail to add device to bridge cause no more available bridge slot
+	_, _, err := q.addDeviceToBridge("qemu-bridge-31", types.PCI)
+	exceptErr := errors.New("no more bridge slots available")
+	assert.Equal(exceptErr.Error(), err.Error())
+
+	// addDeviceToBridge fails cause q.Bridges == 0
+	q = newQemuArchBase()
+	q.machineType = QemuPCLite
+	q.bridges(0)
+	_, _, err = q.addDeviceToBridge("qemu-bridge", types.PCI)
+	if assert.Error(err) {
+		exceptErr = errors.New("failed to get available address from bridges")
+		assert.Equal(exceptErr.Error(), err.Error())
 	}
 }
 
@@ -283,10 +314,11 @@ func TestQemuArchBaseAppendBridges(t *testing.T) {
 	assert := assert.New(t)
 	qemuArchBase := newQemuArchBase()
 
-	bridges := qemuArchBase.bridges(1)
+	qemuArchBase.bridges(1)
+	bridges := qemuArchBase.getBridges()
 	assert.Len(bridges, 1)
 
-	devices = qemuArchBase.appendBridges(devices, bridges)
+	devices = qemuArchBase.appendBridges(devices)
 	assert.Len(devices, 1)
 
 	expectedOut := []govmmQemu.Device{

virtcontainers/qemu_arm64.go

@@ -14,7 +14,6 @@ import (
 	"time"
 
 	govmmQemu "github.com/intel/govmm/qemu"
-	"github.com/kata-containers/runtime/virtcontainers/types"
 	"github.com/sirupsen/logrus"
 )
 
@@ -162,13 +161,13 @@ func newQemuArch(config HypervisorConfig) qemuArch {
 	return q
 }
 
-func (q *qemuArm64) bridges(number uint32) []types.PCIBridge {
+func (q *qemuArm64) bridges(number uint32) {
-	return genericBridges(number, q.machineType)
+	q.Bridges = genericBridges(number, q.machineType)
 }
 
 // appendBridges appends to devices the given bridges
-func (q *qemuArm64) appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device {
+func (q *qemuArm64) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device {
-	return genericAppendBridges(devices, bridges, q.machineType)
+	return genericAppendBridges(devices, q.Bridges, q.machineType)
 }
 
 func (q *qemuArm64) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) {

virtcontainers/qemu_arm64_test.go

@@ -105,11 +105,12 @@ func TestQemuArm64AppendBridges(t *testing.T) {
 
 	arm64 := newTestQemu(QemuVirt)
 
-	bridges := arm64.bridges(1)
+	arm64.bridges(1)
+	bridges := arm64.getBridges()
 	assert.Len(bridges, 1)
 
 	devices = []govmmQemu.Device{}
-	devices = arm64.appendBridges(devices, bridges)
+	devices = arm64.appendBridges(devices)
 	assert.Len(devices, 1)
 
 	expectedOut := []govmmQemu.Device{

virtcontainers/qemu_ppc64le.go

@@ -105,8 +105,8 @@ func (q *qemuPPC64le) capabilities() types.Capabilities {
 	return caps
 }
 
-func (q *qemuPPC64le) bridges(number uint32) []types.PCIBridge {
+func (q *qemuPPC64le) bridges(number uint32) {
-	return genericBridges(number, q.machineType)
+	q.Bridges = genericBridges(number, q.machineType)
 }
 
 func (q *qemuPPC64le) cpuModel() string {
@@ -152,6 +152,6 @@ func (q *qemuPPC64le) appendImage(devices []govmmQemu.Device, path string) ([]go
 }
 
 // appendBridges appends to devices the given bridges
-func (q *qemuPPC64le) appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device {
+func (q *qemuPPC64le) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device {
-	return genericAppendBridges(devices, bridges, q.machineType)
+	return genericAppendBridges(devices, q.Bridges, q.machineType)
 }

virtcontainers/qemu_s390x.go

@@ -40,6 +40,8 @@ var kernelParams = []Param{
 
 var kernelRootParams = commonVirtioblkKernelRootParams
 
+var ccwbridge = types.NewBridge(types.CCW, "", make(map[uint32]string, types.CCWBridgeMaxCapacity), 0)
+
 var supportedQemuMachines = []govmmQemu.Machine{
 	{
 		Type:    QemuCCWVirtio,
@@ -72,6 +74,8 @@ func newQemuArch(config HypervisorConfig) qemuArch {
 			kernelParams:          kernelParams,
 		},
 	}
+	// Set first bridge type to CCW
+	q.Bridges = append(q.Bridges, ccwbridge)
 
 	if config.ImagePath != "" {
 		q.kernelParams = append(q.kernelParams, kernelRootParams...)
@@ -82,22 +86,32 @@ func newQemuArch(config HypervisorConfig) qemuArch {
 	return q
 }
 
-func (q *qemuS390x) bridges(number uint32) []types.PCIBridge {
+func (q *qemuS390x) bridges(number uint32) {
-	return genericBridges(number, q.machineType)
+	q.Bridges = genericBridges(number, q.machineType)
-}
-
-// appendBridges appends to devices the given bridges
-func (q *qemuS390x) appendBridges(devices []govmmQemu.Device, bridges []types.PCIBridge) []govmmQemu.Device {
-	return genericAppendBridges(devices, bridges, q.machineType)
 }
 
 // appendConsole appends a console to devices.
 // The function has been overwriten to correctly set the driver to the CCW device
 func (q *qemuS390x) appendConsole(devices []govmmQemu.Device, path string) []govmmQemu.Device {
+	id := "serial0"
+	addr, b, err := q.addDeviceToBridge(id, types.CCW)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append console")
+		return devices
+	}
+
+	var devno string
+	devno, err = b.AddressFormatCCW(addr)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append console")
+		return devices
+	}
+
 	serial := govmmQemu.SerialDevice{
 		Driver:        virtioSerialCCW,
-		ID:            "serial0",
+		ID:            id,
 		DisableModern: q.nestedRun,
+		DevNo:         devno,
 	}
 
 	devices = append(devices, serial)
@@ -115,6 +129,36 @@ func (q *qemuS390x) appendConsole(devices []govmmQemu.Device, path string) []gov
 	return devices
 }
 
+func (q *qemuS390x) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) {
+	drive, err := genericImage(path)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append image")
+		return nil, err
+	}
+
+	return q.appendBlockDevice(devices, drive), nil
+}
+
+func (q *qemuS390x) appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device {
+	d, err := genericBlockDevice(drive, false)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append blk-dev")
+		return devices
+	}
+	addr, b, err := q.addDeviceToBridge(drive.ID, types.CCW)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append blk-dev")
+		return devices
+	}
+	d.DevNo, err = b.AddressFormatCCW(addr)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append blk-dev")
+		return devices
+	}
+	devices = append(devices, d)
+	return devices
+}
+
 // appendVhostUserDevice throws an error if vhost devices are tried to be used.
 // See issue https://github.com/kata-containers/runtime/issues/659
 func (q *qemuS390x) appendVhostUserDevice(devices []govmmQemu.Device, attr config.VhostUserDeviceAttrs) ([]govmmQemu.Device, error) {
@@ -126,3 +170,90 @@ func (q *qemuS390x) appendVhostUserDevice(devices []govmmQemu.Device, attr confi
 func (q *qemuS390x) supportGuestMemoryHotplug() bool {
 	return false
 }
+
+func (q *qemuS390x) appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device {
+	d, err := genericNetwork(endpoint, false, false, q.networkIndex)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append network")
+		return devices
+	}
+	q.networkIndex++
+	addr, b, err := q.addDeviceToBridge(d.ID, types.CCW)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append network")
+		return devices
+	}
+	d.DevNo, err = b.AddressFormatCCW(addr)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append network")
+		return devices
+	}
+
+	devices = append(devices, d)
+	return devices
+}
+
+func (q *qemuS390x) appendRNGDevice(devices []govmmQemu.Device, rngDev config.RNGDev) []govmmQemu.Device {
+	addr, b, err := q.addDeviceToBridge(rngDev.ID, types.CCW)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append RNG-Device")
+		return devices
+	}
+	var devno string
+	devno, err = b.AddressFormatCCW(addr)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append RNG-Device")
+		return devices
+	}
+
+	devices = append(devices,
+		govmmQemu.RngDevice{
+			ID:       rngDev.ID,
+			Filename: rngDev.Filename,
+			DevNo:    devno,
+		},
+	)
+
+	return devices
+}
+
+func (q *qemuS390x) append9PVolume(devices []govmmQemu.Device, volume types.Volume) []govmmQemu.Device {
+	if volume.MountTag == "" || volume.HostPath == "" {
+		return devices
+	}
+	d := generic9PVolume(volume, false)
+	addr, b, err := q.addDeviceToBridge(d.ID, types.CCW)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append 9p-Volume")
+		return devices
+	}
+	d.DevNo, err = b.AddressFormatCCW(addr)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append 9p-Volume")
+		return devices
+	}
+	devices = append(devices, d)
+	return devices
+}
+
+// appendBridges appends to devices the given bridges
+func (q *qemuS390x) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device {
+	return genericAppendBridges(devices, q.Bridges, q.machineType)
+}
+
+func (q *qemuS390x) appendSCSIController(devices []govmmQemu.Device, enableIOThreads bool) ([]govmmQemu.Device, *govmmQemu.IOThread) {
+	d, t := genericSCSIController(enableIOThreads, q.nestedRun)
+	addr, b, err := q.addDeviceToBridge(d.ID, types.CCW)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append scsi-controller")
+		return devices, nil
+	}
+	d.DevNo, err = b.AddressFormatCCW(addr)
+	if err != nil {
+		virtLog.WithField("subsystem", "qemus390x").WithError(err).Error("Failed to append scsi-controller")
+		return devices, nil
+	}
+
+	devices = append(devices, d)
+	return devices, t
+}

virtcontainers/qemu_test.go

@@ -421,44 +421,6 @@ func TestQemuGrpc(t *testing.T) {
 	assert.True(q.id == q2.id)
 }
 
-func TestQemuAddDeviceToBridge(t *testing.T) {
-	assert := assert.New(t)
-
-	config := newQemuConfig()
-	config.DefaultBridges = defaultBridges
-
-	// addDeviceToBridge successfully
-	config.HypervisorMachineType = QemuPC
-	q := &qemu{
-		config: config,
-		arch:   newQemuArch(config),
-	}
-
-	q.state.Bridges = q.arch.bridges(q.config.DefaultBridges)
-	// get pciBridgeMaxCapacity value from virtcontainers/types/pci.go
-	const pciBridgeMaxCapacity = 30
-	for i := uint32(1); i <= pciBridgeMaxCapacity; i++ {
-		_, _, err := q.addDeviceToBridge(fmt.Sprintf("qemu-bridge-%d", i))
-		assert.Nil(err)
-	}
-
-	// fail to add device to bridge cause no more available bridge slot
-	_, _, err := q.addDeviceToBridge("qemu-bridge-31")
-	exceptErr := errors.New("no more bridge slots available")
-	assert.Equal(exceptErr.Error(), err.Error())
-
-	// addDeviceToBridge fails cause q.state.Bridges == 0
-	config.HypervisorMachineType = QemuPCLite
-	q = &qemu{
-		config: config,
-		arch:   newQemuArch(config),
-	}
-	q.state.Bridges = q.arch.bridges(q.config.DefaultBridges)
-	_, _, err = q.addDeviceToBridge("qemu-bridge")
-	exceptErr = errors.New("failed to get available address from bridges")
-	assert.Equal(exceptErr.Error(), err.Error())
-}
-
 func TestQemuFileBackedMem(t *testing.T) {
 	assert := assert.New(t)
 

virtcontainers/types/bridges.go

@@ -0,0 +1,115 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package types
+
+import "fmt"
+
+// Type represents a type of bus and bridge.
+type Type string
+
+const PCIBridgeMaxCapacity = 30
+
+const (
+	// PCI represents a PCI bus and bridge
+	PCI Type = "pci"
+
+	// PCIE represents a PCIe bus and bridge
+	PCIE Type = "pcie"
+)
+
+const CCWBridgeMaxCapacity = 0xffff
+
+const (
+	CCW Type = "ccw"
+)
+
+type Bridge struct {
+	// Devices contains information about devices plugged and its address in the bridge
+	Devices map[uint32]string
+
+	// ID is used to identify the bridge in the hypervisor
+	ID string
+
+	// Addr is the slot of the bridge
+	Addr int
+
+	// Type is the type of the bridge (pci, pcie, etc)
+	Type Type
+
+	// MaxCapacity is the max capacity of the bridge
+	MaxCapacity uint32
+}
+
+func NewBridge(bt Type, id string, devices map[uint32]string, addr int) Bridge {
+	var maxCapacity uint32
+	switch bt {
+	case PCI:
+		fallthrough
+	case PCIE:
+		maxCapacity = PCIBridgeMaxCapacity
+	case CCW:
+		maxCapacity = CCWBridgeMaxCapacity
+	default:
+		maxCapacity = 0
+	}
+	return Bridge{
+		Devices:     devices,
+		ID:          id,
+		Addr:        addr,
+		Type:        bt,
+		MaxCapacity: maxCapacity,
+	}
+}
+
+func (b *Bridge) AddDevice(ID string) (uint32, error) {
+	var addr uint32
+
+	// looking for the first available address
+	for i := uint32(1); i <= b.MaxCapacity; i++ {
+		if _, ok := b.Devices[i]; !ok {
+			addr = i
+			break
+		}
+	}
+
+	if addr == 0 {
+		return 0, fmt.Errorf("Unable to hot plug device on bridge: there are no empty slots")
+	}
+
+	// save address and device
+	b.Devices[addr] = ID
+	return addr, nil
+}
+
+func (b *Bridge) RemoveDevice(ID string) error {
+	// check if the device was hot plugged in the bridge
+	for addr, devID := range b.Devices {
+		if devID == ID {
+			// free address to re-use the same slot with other devices
+			delete(b.Devices, addr)
+			return nil
+		}
+	}
+
+	return fmt.Errorf("Unable to hot unplug device %s: not present on bridge", ID)
+}
+
+// AddressFormatCCW returns the address format for the device number. The channel subsystem-ID 0xfe is reserved to the virtual channel and the address format is in the form fe.n.dddd, where n is subchannel set ID and ddd the device number. More details at https://www.ibm.com/support/knowledgecenter/en/linuxonibm/com.ibm.linux.z.ldva/ldva_t_configuringSCSIdevices.html
+func (b *Bridge) AddressFormatCCW(addr string) (string, error) {
+	if b.Type != CCW {
+		return "", fmt.Errorf("Expected bridge type %T, got %T (%+v)", CCW, b.Type, b)
+	}
+
+	return fmt.Sprintf("fe.%x.%s", b.Addr, addr), nil
+}
+
+// AddressFormatCCWForVirtServer returns the address format for the virtual server. The address format is in the form of 0.n.dddd
+func (b *Bridge) AddressFormatCCWForVirtServer(addr string) (string, error) {
+	if b.Type != CCW {
+		return "", fmt.Errorf("Wrong bridge type")
+	}
+	return fmt.Sprintf("0.%x.%s", b.Addr, addr), nil
+}

virtcontainers/types/bridges_test.go

@@ -12,15 +12,12 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestAddRemoveDevice(t *testing.T) {
+func testAddRemoveDevice(t *testing.T, b *Bridge) {
 	assert := assert.New(t)
 
-	// create a bridge
-	bridges := []*PCIBridge{{make(map[uint32]string), PCI, "rgb123", 5}}
-
 	// add device
 	devID := "abc123"
-	b := bridges[0]
+
 	addr, err := b.AddDevice(devID)
 	assert.NoError(err)
 	if addr < 1 {
@@ -35,9 +32,9 @@ func TestAddRemoveDevice(t *testing.T) {
 	assert.NoError(err)
 
 	// add device when the bridge is full
-	bridges[0].Address = make(map[uint32]string)
+	b.Devices = make(map[uint32]string)
-	for i := uint32(1); i <= pciBridgeMaxCapacity; i++ {
+	for i := uint32(1); i <= b.MaxCapacity; i++ {
-		bridges[0].Address[i] = fmt.Sprintf("%d", i)
+		b.Devices[i] = fmt.Sprintf("%d", i)
 	}
 	addr, err = b.AddDevice(devID)
 	assert.Error(err)
@@ -45,3 +42,19 @@ func TestAddRemoveDevice(t *testing.T) {
 		assert.Fail("address should be 0")
 	}
 }
+
+func TestAddRemoveDevicePCI(t *testing.T) {
+
+	// create a pci bridge
+	bridges := []*Bridge{{make(map[uint32]string), "rgb123", 5, PCI, PCIBridgeMaxCapacity}}
+
+	testAddRemoveDevice(t, bridges[0])
+}
+
+func TestAddRemoveDeviceCCW(t *testing.T) {
+
+	// create a CCW bridge
+	bridges := []*Bridge{{make(map[uint32]string), "rgb123", 5, CCW, CCWBridgeMaxCapacity}}
+
+	testAddRemoveDevice(t, bridges[0])
+}

virtcontainers/types/pci.go

@@ -1,72 +0,0 @@
-// Copyright (c) 2017 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-package types
-
-import "fmt"
-
-// PCIType represents a type of PCI bus and bridge.
-type PCIType string
-
-const (
-	// PCI represents a PCI bus and bridge
-	PCI PCIType = "pci"
-
-	// PCIE represents a PCIe bus and bridge
-	PCIE PCIType = "pcie"
-)
-
-const pciBridgeMaxCapacity = 30
-
-// PCIBridge is a PCI or PCIe bridge where devices can be hot plugged
-type PCIBridge struct {
-	// Address contains information about devices plugged and its address in the bridge
-	Address map[uint32]string
-
-	// Type is the PCI type of the bridge (pci, pcie, etc)
-	Type PCIType
-
-	// ID is used to identify the bridge in the hypervisor
-	ID string
-
-	// Addr is the PCI/e slot of the bridge
-	Addr int
-}
-
-// AddDevice on success adds the device ID to the PCI bridge and returns
-// the address where the device was added.
-func (b *PCIBridge) AddDevice(ID string) (uint32, error) {
-	var addr uint32
-
-	// looking for the first available address
-	for i := uint32(1); i <= pciBridgeMaxCapacity; i++ {
-		if _, ok := b.Address[i]; !ok {
-			addr = i
-			break
-		}
-	}
-
-	if addr == 0 {
-		return 0, fmt.Errorf("Unable to hot plug device on bridge: there are not empty slots")
-	}
-
-	// save address and device
-	b.Address[addr] = ID
-	return addr, nil
-}
-
-// RemoveDevice removes the device ID from the PCI bridge.
-func (b *PCIBridge) RemoveDevice(ID string) error {
-	// check if the device was hot plugged in the bridge
-	for addr, devID := range b.Address {
-		if devID == ID {
-			// free address to re-use the same slot with other devices
-			delete(b.Address, addr)
-			return nil
-		}
-	}
-
-	return fmt.Errorf("Unable to hot unplug device %s: not present on bridge", ID)
-}