runtime: Add support for VFIO-AP pass-through

Recognise when a device to be hot-plugged is an IBM Adjunct Processor
(AP) device and execute VFIO AP hot-plug accordingly. Includes unittest
for recognising and uses CCW for addDeviceToBridge in hotplugVFIODevice
if appropriate.

Fixes: #491

Signed-off-by: Jakob-Naucke <jakob.naucke@ibm.com>
Co-authored-by: Julio Montes <julio.montes@intel.com>
Reviewed-by: Alice Frosi <afrosi@redhat.com>

src/runtime/virtcontainers/qemu.go

@@ -1305,6 +1305,9 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
 			case config.VFIODeviceNormalType:
 				return q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, device.Bus, romFile)
 			case config.VFIODeviceMediatedType:
+				if utils.IsAPVFIOMediatedDevice(device.SysfsDev) {
+					return q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, device.SysfsDev)
+				}
 				return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, "", device.Bus, romFile)
 			default:
 				return fmt.Errorf("Incorrect VFIO device type found")
@@ -1326,6 +1329,9 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
 		case config.VFIODeviceNormalType:
 			return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID, romFile)
 		case config.VFIODeviceMediatedType:
+			if utils.IsAPVFIOMediatedDevice(device.SysfsDev) {
+				return q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, device.SysfsDev)
+			}
 			return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bridge.ID, romFile)
 		default:
 			return fmt.Errorf("Incorrect VFIO device type found")

src/runtime/virtcontainers/qemu_arch_base.go

@@ -712,30 +712,34 @@ func (q *qemuArchBase) setIgnoreSharedMemoryMigrationCaps(ctx context.Context, q
 }
 
 func (q *qemuArchBase) addDeviceToBridge(ID string, t types.Type) (string, types.Bridge, error) {
+	addr, b, err := genericAddDeviceToBridge(q.Bridges, ID, t)
+	if err != nil {
+		return "", b, err
+	}
+
+	return fmt.Sprintf("%02x", addr), b, nil
+}
+
+func genericAddDeviceToBridge(bridges []types.Bridge, ID string, t types.Type) (uint32, types.Bridge, error) {
 	var err error
 	var addr uint32
 
-	if len(q.Bridges) == 0 {
-		return "", types.Bridge{}, errors.New("failed to get available address from bridges")
+	if len(bridges) == 0 {
+		return 0, types.Bridge{}, errors.New("failed to get available address from bridges")
 	}
 
 	// looking for an empty address in the bridges
-	for _, b := range q.Bridges {
+	for _, b := range bridges {
 		if t != b.Type {
 			continue
 		}
 		addr, err = b.AddDevice(ID)
 		if err == nil {
-			switch t {
-			case types.CCW:
-				return fmt.Sprintf("%04x", addr), b, nil
-			case types.PCI, types.PCIE:
-				return fmt.Sprintf("%02x", addr), b, nil
-			}
+			return addr, b, nil
 		}
 	}
 
-	return "", types.Bridge{}, fmt.Errorf("no more bridge slots available")
+	return 0, types.Bridge{}, fmt.Errorf("no more bridge slots available")
 }
 
 func (q *qemuArchBase) removeDeviceFromBridge(ID string) error {

src/runtime/virtcontainers/qemu_s390x.go

@@ -269,3 +269,12 @@ func (q *qemuS390x) appendVSock(devices []govmmQemu.Device, vsock types.VSock) (
 func (q *qemuS390x) appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Device, error) {
 	return devices, fmt.Errorf("S390x does not support appending a vIOMMU")
 }
+
+func (q *qemuS390x) addDeviceToBridge(ID string, t types.Type) (string, types.Bridge, error) {
+	addr, b, err := genericAddDeviceToBridge(q.Bridges, ID, types.CCW)
+	if err != nil {
+		return "", b, err
+	}
+
+	return fmt.Sprintf("%04x", addr), b, nil
+}

src/runtime/virtcontainers/utils/utils_linux.go

@@ -92,7 +92,8 @@ func FindContextID() (*os.File, uint64, error) {
 const (
 	procMountsFile = "/proc/mounts"
 
-	fieldsPerLine = 6
+	fieldsPerLine  = 6
+	vfioAPSysfsDir = "vfio_ap"
 )
 
 const (
@@ -141,3 +142,15 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e
 		}
 	}
 }
+
+// IsAPVFIOMediatedDevice decides whether a device is a VFIO-AP device
+// by checking for the existence of "vfio_ap" in the path
+func IsAPVFIOMediatedDevice(sysfsdev string) bool {
+	split := strings.Split(sysfsdev, string(os.PathSeparator))
+	for _, el := range split {
+		if el == vfioAPSysfsDir {
+			return true
+		}
+	}
+	return false
+}

src/runtime/virtcontainers/utils/utils_linux_test.go

@@ -49,3 +49,19 @@ func TestGetDevicePathAndFsTypeSuccessful(t *testing.T) {
 	assert.Equal(path, "proc")
 	assert.Equal(fstype, "proc")
 }
+
+func TestIsAPVFIOMediatedDeviceFalse(t *testing.T) {
+	assert := assert.New(t)
+
+	// Should be false for a PCI device
+	isAPMdev := IsAPVFIOMediatedDevice("/sys/bus/pci/devices/0000:00:02.0/a297db4a-f4c2-11e6-90f6-d3b88d6c9525")
+	assert.False(isAPMdev)
+}
+
+func TestIsAPVFIOMediatedDeviceTrue(t *testing.T) {
+	assert := assert.New(t)
+
+	// Typical AP sysfsdev
+	isAPMdev := IsAPVFIOMediatedDevice("/sys/devices/vfio_ap/matrix/a297db4a-f4c2-11e6-90f6-d3b88d6c9525")
+	assert.True(isAPMdev)
+}