From 1236e22475f11e7519c5439c5e55dc06de0f0731 Mon Sep 17 00:00:00 2001 From: Jakob-Naucke Date: Wed, 26 Aug 2020 16:45:37 +0200 Subject: [PATCH] runtime: Add support for VFIO-AP pass-through Recognise when a device to be hot-plugged is an IBM Adjunct Processor (AP) device and execute VFIO AP hot-plug accordingly. Includes unittest for recognising and uses CCW for addDeviceToBridge in hotplugVFIODevice if appropriate. Fixes: #491 Signed-off-by: Jakob-Naucke Co-authored-by: Julio Montes Reviewed-by: Alice Frosi --- src/runtime/virtcontainers/qemu.go | 6 +++++ src/runtime/virtcontainers/qemu_arch_base.go | 24 +++++++++++-------- src/runtime/virtcontainers/qemu_s390x.go | 9 +++++++ .../virtcontainers/utils/utils_linux.go | 15 +++++++++++- .../virtcontainers/utils/utils_linux_test.go | 16 +++++++++++++ 5 files changed, 59 insertions(+), 11 deletions(-) diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index a9dac046a..0bddeb573 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -1305,6 +1305,9 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro case config.VFIODeviceNormalType: return q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, device.Bus, romFile) case config.VFIODeviceMediatedType: + if utils.IsAPVFIOMediatedDevice(device.SysfsDev) { + return q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, device.SysfsDev) + } return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, "", device.Bus, romFile) default: return fmt.Errorf("Incorrect VFIO device type found") @@ -1326,6 +1329,9 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro case config.VFIODeviceNormalType: return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID, romFile) case config.VFIODeviceMediatedType: + if utils.IsAPVFIOMediatedDevice(device.SysfsDev) { + return q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, device.SysfsDev) + } return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bridge.ID, romFile) default: return fmt.Errorf("Incorrect VFIO device type found") diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index 259907e87..363bc3fd9 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -712,30 +712,34 @@ func (q *qemuArchBase) setIgnoreSharedMemoryMigrationCaps(ctx context.Context, q } func (q *qemuArchBase) addDeviceToBridge(ID string, t types.Type) (string, types.Bridge, error) { + addr, b, err := genericAddDeviceToBridge(q.Bridges, ID, t) + if err != nil { + return "", b, err + } + + return fmt.Sprintf("%02x", addr), b, nil +} + +func genericAddDeviceToBridge(bridges []types.Bridge, ID string, t types.Type) (uint32, types.Bridge, error) { var err error var addr uint32 - if len(q.Bridges) == 0 { - return "", types.Bridge{}, errors.New("failed to get available address from bridges") + if len(bridges) == 0 { + return 0, types.Bridge{}, errors.New("failed to get available address from bridges") } // looking for an empty address in the bridges - for _, b := range q.Bridges { + for _, b := range bridges { if t != b.Type { continue } addr, err = b.AddDevice(ID) if err == nil { - switch t { - case types.CCW: - return fmt.Sprintf("%04x", addr), b, nil - case types.PCI, types.PCIE: - return fmt.Sprintf("%02x", addr), b, nil - } + return addr, b, nil } } - return "", types.Bridge{}, fmt.Errorf("no more bridge slots available") + return 0, types.Bridge{}, fmt.Errorf("no more bridge slots available") } func (q *qemuArchBase) removeDeviceFromBridge(ID string) error { diff --git a/src/runtime/virtcontainers/qemu_s390x.go b/src/runtime/virtcontainers/qemu_s390x.go index df890a7d9..132d7d781 100644 --- a/src/runtime/virtcontainers/qemu_s390x.go +++ b/src/runtime/virtcontainers/qemu_s390x.go @@ -269,3 +269,12 @@ func (q *qemuS390x) appendVSock(devices []govmmQemu.Device, vsock types.VSock) ( func (q *qemuS390x) appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Device, error) { return devices, fmt.Errorf("S390x does not support appending a vIOMMU") } + +func (q *qemuS390x) addDeviceToBridge(ID string, t types.Type) (string, types.Bridge, error) { + addr, b, err := genericAddDeviceToBridge(q.Bridges, ID, types.CCW) + if err != nil { + return "", b, err + } + + return fmt.Sprintf("%04x", addr), b, nil +} diff --git a/src/runtime/virtcontainers/utils/utils_linux.go b/src/runtime/virtcontainers/utils/utils_linux.go index ad870d63e..3c14e0cda 100644 --- a/src/runtime/virtcontainers/utils/utils_linux.go +++ b/src/runtime/virtcontainers/utils/utils_linux.go @@ -92,7 +92,8 @@ func FindContextID() (*os.File, uint64, error) { const ( procMountsFile = "/proc/mounts" - fieldsPerLine = 6 + fieldsPerLine = 6 + vfioAPSysfsDir = "vfio_ap" ) const ( @@ -141,3 +142,15 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e } } } + +// IsAPVFIOMediatedDevice decides whether a device is a VFIO-AP device +// by checking for the existence of "vfio_ap" in the path +func IsAPVFIOMediatedDevice(sysfsdev string) bool { + split := strings.Split(sysfsdev, string(os.PathSeparator)) + for _, el := range split { + if el == vfioAPSysfsDir { + return true + } + } + return false +} diff --git a/src/runtime/virtcontainers/utils/utils_linux_test.go b/src/runtime/virtcontainers/utils/utils_linux_test.go index 4554fa935..c7b2b8793 100644 --- a/src/runtime/virtcontainers/utils/utils_linux_test.go +++ b/src/runtime/virtcontainers/utils/utils_linux_test.go @@ -49,3 +49,19 @@ func TestGetDevicePathAndFsTypeSuccessful(t *testing.T) { assert.Equal(path, "proc") assert.Equal(fstype, "proc") } + +func TestIsAPVFIOMediatedDeviceFalse(t *testing.T) { + assert := assert.New(t) + + // Should be false for a PCI device + isAPMdev := IsAPVFIOMediatedDevice("/sys/bus/pci/devices/0000:00:02.0/a297db4a-f4c2-11e6-90f6-d3b88d6c9525") + assert.False(isAPMdev) +} + +func TestIsAPVFIOMediatedDeviceTrue(t *testing.T) { + assert := assert.New(t) + + // Typical AP sysfsdev + isAPMdev := IsAPVFIOMediatedDevice("/sys/devices/vfio_ap/matrix/a297db4a-f4c2-11e6-90f6-d3b88d6c9525") + assert.True(isAPMdev) +}