diff --git a/virtcontainers/device/config/config.go b/virtcontainers/device/config/config.go index 280a3e7d32..5f06d10c59 100644 --- a/virtcontainers/device/config/config.go +++ b/virtcontainers/device/config/config.go @@ -109,12 +109,33 @@ type BlockDrive struct { VirtPath string } +// VFIODeviceType indicates VFIO device type +type VFIODeviceType uint32 + +const ( + // VFIODeviceErrorType is the error type of VFIO device + VFIODeviceErrorType VFIODeviceType = iota + + // VFIODeviceNormalType is a normal VFIO device type + VFIODeviceNormalType + + // VFIODeviceMediatedType is a VFIO mediated device type + VFIODeviceMediatedType +) + // VFIODev represents a VFIO drive used for hotplugging type VFIODev struct { // ID is used to identify this drive in the hypervisor options. ID string + + // Type of VFIO device + Type VFIODeviceType + // BDF (Bus:Device.Function) of the PCI address BDF string + + // sysfsdev of VFIO mediated device + SysfsDev string } // RNGDev represents a random number generator device diff --git a/virtcontainers/device/drivers/vfio.go b/virtcontainers/device/drivers/vfio.go index 7b58eb781c..06568c07c4 100644 --- a/virtcontainers/device/drivers/vfio.go +++ b/virtcontainers/device/drivers/vfio.go @@ -67,13 +67,15 @@ func (device *VFIODevice) Attach(devReceiver api.DeviceReceiver) error { // Pass all devices in iommu group for i, deviceFile := range deviceFiles { //Get bdf of device eg 0000:00:1c.0 - deviceBDF, err := getBDF(deviceFile.Name()) + deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(deviceFile.Name(), iommuDevicesPath) if err != nil { return err } vfio := &config.VFIODev{ - ID: utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize), - BDF: deviceBDF, + ID: utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize), + Type: vfioDeviceType, + BDF: deviceBDF, + SysfsDev: deviceSysfsDev, } device.vfioDevs = append(device.vfioDevs, vfio) } @@ -130,17 +132,45 @@ func (device *VFIODevice) GetDeviceInfo() interface{} { // It should implement GetAttachCount() and DeviceID() as api.Device implementation // here it shares function from *GenericDevice so we don't need duplicate codes -// getBDF returns the BDF of pci device -// Expected input strng format is []:[][].[] eg. 0000:02:10.0 -func getBDF(deviceSysStr string) (string, error) { - tokens := strings.Split(deviceSysStr, ":") - - if len(tokens) != 3 { - return "", fmt.Errorf("Incorrect number of tokens found while parsing bdf for device : %s", deviceSysStr) +func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceSysfsDev string, vfioDeviceType config.VFIODeviceType, err error) { + tokens := strings.Split(deviceFileName, ":") + vfioDeviceType = config.VFIODeviceErrorType + if len(tokens) == 3 { + vfioDeviceType = config.VFIODeviceNormalType + } else { + tokens = strings.Split(deviceFileName, "-") + if len(tokens) == 5 { + vfioDeviceType = config.VFIODeviceMediatedType + } } - tokens = strings.SplitN(deviceSysStr, ":", 2) - return tokens[1], nil + switch vfioDeviceType { + case config.VFIODeviceNormalType: + // Get bdf of device eg. 0000:00:1c.0 + deviceBDF = getBDF(deviceFileName) + case config.VFIODeviceMediatedType: + // Get sysfsdev of device eg. /sys/devices/pci0000:00/0000:00:02.0/f79944e4-5a3d-11e8-99ce-479cbab002e4 + sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName) + deviceSysfsDev, err = getSysfsDev(sysfsDevStr) + default: + err = fmt.Errorf("Incorrect tokens found while parsing vfio details: %s", deviceFileName) + } + + return deviceBDF, deviceSysfsDev, vfioDeviceType, err +} + +// getBDF returns the BDF of pci device +// Expected input string format is []:[][].[] eg. 0000:02:10.0 +func getBDF(deviceSysStr string) string { + tokens := strings.SplitN(deviceSysStr, ":", 2) + return tokens[1] +} + +// getSysfsDev returns the sysfsdev of mediated device +// Expected input string format is absolute path to the sysfs dev node +// eg. /sys/kernel/iommu_groups/0/devices/f79944e4-5a3d-11e8-99ce-479cbab002e4 +func getSysfsDev(sysfsDevStr string) (string, error) { + return filepath.EvalSymlinks(sysfsDevStr) } // BindDevicetoVFIO binds the device to vfio driver after unbinding from host. diff --git a/virtcontainers/device/drivers/vfio_test.go b/virtcontainers/device/drivers/vfio_test.go index cf60098adf..59a441fc10 100644 --- a/virtcontainers/device/drivers/vfio_test.go +++ b/virtcontainers/device/drivers/vfio_test.go @@ -9,26 +9,38 @@ package drivers import ( "testing" + "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/stretchr/testify/assert" ) -func TestGetBDF(t *testing.T) { +func TestGetVFIODetails(t *testing.T) { type testData struct { deviceStr string - expectedBDF string + expectedStr string } data := []testData{ {"0000:02:10.0", "02:10.0"}, {"0000:0210.0", ""}, + {"f79944e4-5a3d-11e8-99ce-", ""}, + {"f79944e4-5a3d-11e8-99ce", ""}, {"test", ""}, {"", ""}, } for _, d := range data { - deviceBDF, err := getBDF(d.deviceStr) - assert.Equal(t, d.expectedBDF, deviceBDF) - if d.expectedBDF == "" { + deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(d.deviceStr, "") + + switch vfioDeviceType { + case config.VFIODeviceNormalType: + assert.Equal(t, d.expectedStr, deviceBDF) + case config.VFIODeviceMediatedType: + assert.Equal(t, d.expectedStr, deviceSysfsDev) + default: + assert.NotNil(t, err) + } + + if d.expectedStr == "" { assert.NotNil(t, err) } else { assert.Nil(t, err) diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 337cc68b8a..e2c236cccf 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -795,7 +795,14 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error { // for pc machine type instead of bridge. This is useful for devices that require // a large PCI BAR which is a currently a limitation with PCI bridges. if q.state.HotplugVFIOOnRootBus { - return q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF) + switch device.Type { + case config.VFIODeviceNormalType: + return q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF) + case config.VFIODeviceMediatedType: + return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, "", "") + default: + return fmt.Errorf("Incorrect VFIO device type found") + } } addr, bridge, err := q.addDeviceToBridge(devID) @@ -803,8 +810,13 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error { return err } - if err := q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID); err != nil { - return err + switch device.Type { + case config.VFIODeviceNormalType: + return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID) + case config.VFIODeviceMediatedType: + return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bridge.ID) + default: + return fmt.Errorf("Incorrect VFIO device type found") } } else { if !q.state.HotplugVFIOOnRootBus {