diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index a920b57952..9c3432d2b5 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -340,7 +340,7 @@ func (h hypervisor) defaultBridges() uint32 { } func (h hypervisor) blockDeviceDriver() (string, error) { - supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm} + supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm, config.VirtioBlockCCW} if h.BlockDeviceDriver == "" { return defaultBlockDeviceDriver, nil diff --git a/virtcontainers/device/config/config.go b/virtcontainers/device/config/config.go index 93fedc1872..4eda53b997 100644 --- a/virtcontainers/device/config/config.go +++ b/virtcontainers/device/config/config.go @@ -48,6 +48,9 @@ const ( // VirtioBlock means use virtio-blk for hotplugging drives VirtioBlock = "virtio-blk" + // VirtioBlockCCW means use virtio-blk for hotplugging drives + VirtioBlockCCW = "virtio-blk-ccw" + // VirtioSCSI means use virtio-scsi for hotplugging drives VirtioSCSI = "virtio-scsi" @@ -138,6 +141,9 @@ type BlockDrive struct { // VirtPath at which the device appears inside the VM, outside of the container mount namespace VirtPath string + + // DevNo identifies the css bus id for virtio-blk-ccw + DevNo string } // VFIODeviceType indicates VFIO device type diff --git a/virtcontainers/device/drivers/block.go b/virtcontainers/device/drivers/block.go index 1e23bc75c5..0be1d6afe0 100644 --- a/virtcontainers/device/drivers/block.go +++ b/virtcontainers/device/drivers/block.go @@ -84,6 +84,8 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) { switch customOptions["block-driver"] { case "virtio-blk": globalIdx = index + case "virtio-blk-ccw": + globalIdx = index case "virtio-mmio": //With firecracker the rootfs for the VM itself //sits at /dev/vda and consumes the first index. @@ -164,6 +166,7 @@ func (device *BlockDevice) Save() persistapi.DeviceState { SCSIAddr: drive.SCSIAddr, NvdimmID: drive.NvdimmID, VirtPath: drive.VirtPath, + DevNo: drive.DevNo, } } return ds @@ -188,6 +191,7 @@ func (device *BlockDevice) Load(ds persistapi.DeviceState) { SCSIAddr: bd.SCSIAddr, NvdimmID: bd.NvdimmID, VirtPath: bd.VirtPath, + DevNo: bd.DevNo, } } diff --git a/virtcontainers/device/manager/manager.go b/virtcontainers/device/manager/manager.go index 6647b1c944..78a2c281bd 100644 --- a/virtcontainers/device/manager/manager.go +++ b/virtcontainers/device/manager/manager.go @@ -25,6 +25,8 @@ const ( VirtioMmio string = "virtio-mmio" // VirtioBlock indicates block driver is virtio-blk based VirtioBlock string = "virtio-blk" + // VirtioBlockCCW indicates block driver is virtio-blk-ccw based + VirtioBlockCCW string = "virtio-blk-ccw" // VirtioSCSI indicates block driver is virtio-scsi based VirtioSCSI string = "virtio-scsi" // Nvdimm indicates block driver is nvdimm based @@ -66,6 +68,8 @@ func NewDeviceManager(blockDriver string, devices []api.Device) api.DeviceManage dm.blockDriver = VirtioBlock } else if blockDriver == Nvdimm { dm.blockDriver = Nvdimm + } else if blockDriver == VirtioBlockCCW { + dm.blockDriver = VirtioBlockCCW } else { dm.blockDriver = VirtioSCSI } diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index c933971106..098583ac00 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -75,6 +75,7 @@ var ( kata9pDevType = "9p" kataMmioBlkDevType = "mmioblk" kataBlkDevType = "blk" + kataBlkCCWDevType = "blk-ccw" kataSCSIDevType = "scsi" kataNvdimmDevType = "nvdimm" kataVirtioFSDevType = "virtio-fs" @@ -1106,6 +1107,9 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, c *Container) []*gr kataDevice.Type = kataMmioBlkDevType kataDevice.Id = d.VirtPath kataDevice.VmPath = d.VirtPath + case config.VirtioBlockCCW: + kataDevice.Type = kataBlkCCWDevType + kataDevice.Id = d.DevNo case config.VirtioBlock: kataDevice.Type = kataBlkDevType kataDevice.Id = d.PCIAddr @@ -1160,11 +1164,14 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat k.Logger().Error("malformed block drive") return nil, fmt.Errorf("malformed block drive") } - - if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio { + switch { + case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio: rootfs.Driver = kataMmioBlkDevType rootfs.Source = blockDrive.VirtPath - } else if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock { + case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlockCCW: + rootfs.Driver = kataBlkCCWDevType + rootfs.Source = blockDrive.DevNo + case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock: rootfs.Driver = kataBlkDevType if blockDrive.PCIAddr == "" { rootfs.Source = blockDrive.VirtPath @@ -1172,10 +1179,14 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat rootfs.Source = blockDrive.PCIAddr } - } else { + case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioSCSI: + rootfs.Driver = kataSCSIDevType rootfs.Source = blockDrive.SCSIAddr + default: + return nil, fmt.Errorf("Unknown block device driver: %s", sandbox.config.HypervisorConfig.BlockDeviceDriver) } + rootfs.MountPoint = rootPathParent rootfs.Fstype = c.state.Fstype @@ -1281,7 +1292,10 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, // Note this call modifies the list of container devices to make sure // all hotplugged devices are unplugged, so this needs be done // after devices passed with --device are handled. - volumeStorages := k.handleBlockVolumes(c) + volumeStorages, err := k.handleBlockVolumes(c) + if err != nil { + return nil, err + } if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil { return nil, err } @@ -1400,7 +1414,7 @@ func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, r // handleBlockVolumes handles volumes that are block devices files // by passing the block devices as Storage to the agent. -func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage { +func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { var volumeStorages []*grpc.Storage @@ -1418,7 +1432,7 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage { if !c.sandbox.supportNewStore() { if err := c.storeDevices(); err != nil { k.Logger().WithField("device", id).WithError(err).Error("store device failed") - return nil + return nil, err } } @@ -1427,22 +1441,28 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage { device := c.sandbox.devManager.GetDeviceByID(id) if device == nil { k.Logger().WithField("device", id).Error("failed to find device by id") - return nil + return nil, fmt.Errorf("Failed to find device by id (id=%s)", id) } blockDrive, ok := device.GetDeviceInfo().(*config.BlockDrive) if !ok || blockDrive == nil { k.Logger().Error("malformed block drive") continue } - if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock { + switch { + case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlockCCW: + vol.Driver = kataBlkCCWDevType + vol.Source = blockDrive.DevNo + case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock: vol.Driver = kataBlkDevType vol.Source = blockDrive.PCIAddr - } else if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio { + case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio: vol.Driver = kataMmioBlkDevType vol.Source = blockDrive.VirtPath - } else { + case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioSCSI: vol.Driver = kataSCSIDevType vol.Source = blockDrive.SCSIAddr + default: + return nil, fmt.Errorf("Unknown block device driver: %s", c.sandbox.config.HypervisorConfig.BlockDeviceDriver) } vol.MountPoint = m.Destination @@ -1452,7 +1472,7 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage { volumeStorages = append(volumeStorages, vol) } - return volumeStorages + return volumeStorages, nil } // handlePidNamespace checks if Pid namespace for a container needs to be shared with its sandbox diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 1eac467d42..2a6143caad 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -986,38 +986,6 @@ func (q *qemu) qmpShutdown() { } } -func (q *qemu) addDeviceToBridge(ID string) (string, types.Bridge, error) { - var err error - var addr uint32 - - if len(q.state.Bridges) == 0 { - return "", types.Bridge{}, errors.New("failed to get available address from bridges") - } - - // looking for an empty address in the bridges - for _, b := range q.state.Bridges { - addr, err = b.AddDevice(ID) - if err == nil { - return fmt.Sprintf("%02x", addr), b, nil - } - } - - return "", types.Bridge{}, fmt.Errorf("no more bridge slots available") -} - -func (q *qemu) removeDeviceFromBridge(ID string) error { - var err error - for _, b := range q.state.Bridges { - err = b.RemoveDevice(ID) - if err == nil { - // device was removed correctly - return nil - } - } - - return err -} - func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, devID string) (err error) { if q.config.BlockDeviceDriver == config.Nvdimm { var blocksize int64 @@ -1052,16 +1020,36 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev } }() - if q.config.BlockDeviceDriver == config.VirtioBlock { + switch { + case q.config.BlockDeviceDriver == config.VirtioBlockCCW: + driver := "virtio-blk-ccw" + + addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.CCW) + if err != nil { + return err + } + var devNoHotplug string + devNoHotplug, err = bridge.AddressFormatCCW(addr) + if err != nil { + return err + } + drive.DevNo, err = bridge.AddressFormatCCWForVirtServer(addr) + if err != nil { + return err + } + if err = q.qmpMonitorCh.qmp.ExecuteDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, devNoHotplug, "", true, false); err != nil { + return err + } + case q.config.BlockDeviceDriver == config.VirtioBlock: driver := "virtio-blk-pci" - addr, bridge, err := q.addDeviceToBridge(drive.ID) + addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.PCI) if err != nil { return err } defer func() { if err != nil { - q.removeDeviceFromBridge(drive.ID) + q.arch.removeDeviceFromBridge(drive.ID) } }() @@ -1071,7 +1059,7 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil { return err } - } else { + case q.config.BlockDeviceDriver == config.VirtioSCSI: driver := "scsi-hd" // Bus exposed by the SCSI Controller @@ -1086,6 +1074,8 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev if err = q.qmpMonitorCh.qmp.ExecuteSCSIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, bus, romFile, scsiID, lun, true, defaultDisableModern); err != nil { return err } + default: + return fmt.Errorf("Block device %s not recognized", q.config.BlockDeviceDriver) } return nil @@ -1103,7 +1093,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error err = q.hotplugAddBlockDevice(drive, op, devID) } else { if q.config.BlockDeviceDriver == config.VirtioBlock { - if err := q.removeDeviceFromBridge(drive.ID); err != nil { + if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil { return err } } @@ -1143,14 +1133,14 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro } } - addr, bridge, err := q.addDeviceToBridge(devID) + addr, bridge, err := q.arch.addDeviceToBridge(devID, types.PCI) if err != nil { return err } defer func() { if err != nil { - q.removeDeviceFromBridge(devID) + q.arch.removeDeviceFromBridge(devID) } }() @@ -1164,7 +1154,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro } } else { if !q.state.HotplugVFIOOnRootBus { - if err := q.removeDeviceFromBridge(devID); err != nil { + if err := q.arch.removeDeviceFromBridge(devID); err != nil { return err } } @@ -1230,14 +1220,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { } }() - addr, bridge, err := q.addDeviceToBridge(tap.ID) + addr, bridge, err := q.arch.addDeviceToBridge(tap.ID, types.PCI) if err != nil { return err } defer func() { if err != nil { - q.removeDeviceFromBridge(tap.ID) + q.arch.removeDeviceFromBridge(tap.ID) } }() @@ -1250,12 +1240,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { return err } if machine.Type == QemuCCWVirtio { - return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), bridge.ID, int(q.config.NumVCPUs)) + devNoHotplug := fmt.Sprintf("fe.%x.%x", bridge.Addr, addr) + return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs)) } return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, romFile, int(q.config.NumVCPUs), defaultDisableModern) + } - if err := q.removeDeviceFromBridge(tap.ID); err != nil { + if err := q.arch.removeDeviceFromBridge(tap.ID); err != nil { return err }