Merge pull request #533 from xindazhao/gpu-support

device: Add GPU device support

Gopkg.lock

@@ -127,7 +127,7 @@
   name = "github.com/intel/govmm"
   packages = ["qemu"]
   pruneopts = "NUT"
-  revision = "032705ba6aae05a9bf41e296cf89c8529cffb822"
+  revision = "9905ae92c5915c07abeb669eaa4d7f7408834b51"
 
 [[projects]]
   digest = "1:672470f31bc4e50f9ba09a1af7ab6035bf8b1452db64dfd79b1a22614bb30710"

Gopkg.toml

@@ -52,7 +52,7 @@
 
 [[constraint]]
   name = "github.com/intel/govmm"
-  revision = "032705ba6aae05a9bf41e296cf89c8529cffb822"
+  revision = "9905ae92c5915c07abeb669eaa4d7f7408834b51"
 
 [[constraint]]
   name = "github.com/kata-containers/agent"

vendor/github.com/intel/govmm/qemu/qmp.go

@@ -946,11 +946,13 @@ func (q *QMP) ExecutePCIVFIOMediatedDeviceAdd(ctx context.Context, devID, sysfsd
 		"id":       devID,
 		"driver":   "vfio-pci",
 		"sysfsdev": sysfsdev,
-		"addr":     addr,
 	}
 	if bus != "" {
 		args["bus"] = bus
 	}
+	if addr != "" {
+		args["addr"] = addr
+	}
 	return q.executeCommand(ctx, "device_add", args, nil)
 }
 

virtcontainers/device/config/config.go

@@ -109,12 +109,33 @@ type BlockDrive struct {
 	VirtPath string
 }
 
+// VFIODeviceType indicates VFIO device type
+type VFIODeviceType uint32
+
+const (
+	// VFIODeviceErrorType is the error type of VFIO device
+	VFIODeviceErrorType VFIODeviceType = iota
+
+	// VFIODeviceNormalType is a normal VFIO device type
+	VFIODeviceNormalType
+
+	// VFIODeviceMediatedType is a VFIO mediated device type
+	VFIODeviceMediatedType
+)
+
 // VFIODev represents a VFIO drive used for hotplugging
 type VFIODev struct {
 	// ID is used to identify this drive in the hypervisor options.
 	ID string
+
+	// Type of VFIO device
+	Type VFIODeviceType
+
 	// BDF (Bus:Device.Function) of the PCI address
 	BDF string
+
+	// sysfsdev of VFIO mediated device
+	SysfsDev string
 }
 
 // RNGDev represents a random number generator device

virtcontainers/device/drivers/vfio.go

@@ -67,13 +67,15 @@ func (device *VFIODevice) Attach(devReceiver api.DeviceReceiver) error {
 	// Pass all devices in iommu group
 	for i, deviceFile := range deviceFiles {
 		//Get bdf of device eg 0000:00:1c.0
-		deviceBDF, err := getBDF(deviceFile.Name())
+		deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(deviceFile.Name(), iommuDevicesPath)
 		if err != nil {
 			return err
 		}
 		vfio := &config.VFIODev{
 			ID:       utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize),
+			Type:     vfioDeviceType,
 			BDF:      deviceBDF,
+			SysfsDev: deviceSysfsDev,
 		}
 		device.vfioDevs = append(device.vfioDevs, vfio)
 	}
@@ -130,17 +132,45 @@ func (device *VFIODevice) GetDeviceInfo() interface{} {
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes
 
-// getBDF returns the BDF of pci device
-// Expected input strng format is [<domain>]:[<bus>][<slot>].[<func>] eg. 0000:02:10.0
-func getBDF(deviceSysStr string) (string, error) {
-	tokens := strings.Split(deviceSysStr, ":")
-
-	if len(tokens) != 3 {
-		return "", fmt.Errorf("Incorrect number of tokens found while parsing bdf for device : %s", deviceSysStr)
+func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceSysfsDev string, vfioDeviceType config.VFIODeviceType, err error) {
+	tokens := strings.Split(deviceFileName, ":")
+	vfioDeviceType = config.VFIODeviceErrorType
+	if len(tokens) == 3 {
+		vfioDeviceType = config.VFIODeviceNormalType
+	} else {
+		tokens = strings.Split(deviceFileName, "-")
+		if len(tokens) == 5 {
+			vfioDeviceType = config.VFIODeviceMediatedType
+		}
 	}
 
-	tokens = strings.SplitN(deviceSysStr, ":", 2)
-	return tokens[1], nil
+	switch vfioDeviceType {
+	case config.VFIODeviceNormalType:
+		// Get bdf of device eg. 0000:00:1c.0
+		deviceBDF = getBDF(deviceFileName)
+	case config.VFIODeviceMediatedType:
+		// Get sysfsdev of device eg. /sys/devices/pci0000:00/0000:00:02.0/f79944e4-5a3d-11e8-99ce-479cbab002e4
+		sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName)
+		deviceSysfsDev, err = getSysfsDev(sysfsDevStr)
+	default:
+		err = fmt.Errorf("Incorrect tokens found while parsing vfio details: %s", deviceFileName)
+	}
+
+	return deviceBDF, deviceSysfsDev, vfioDeviceType, err
+}
+
+// getBDF returns the BDF of pci device
+// Expected input string format is [<domain>]:[<bus>][<slot>].[<func>] eg. 0000:02:10.0
+func getBDF(deviceSysStr string) string {
+	tokens := strings.SplitN(deviceSysStr, ":", 2)
+	return tokens[1]
+}
+
+// getSysfsDev returns the sysfsdev of mediated device
+// Expected input string format is absolute path to the sysfs dev node
+// eg. /sys/kernel/iommu_groups/0/devices/f79944e4-5a3d-11e8-99ce-479cbab002e4
+func getSysfsDev(sysfsDevStr string) (string, error) {
+	return filepath.EvalSymlinks(sysfsDevStr)
 }
 
 // BindDevicetoVFIO binds the device to vfio driver after unbinding from host.

virtcontainers/device/drivers/vfio_test.go

@@ -9,26 +9,38 @@ package drivers
 import (
 	"testing"
 
+	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/stretchr/testify/assert"
 )
 
-func TestGetBDF(t *testing.T) {
+func TestGetVFIODetails(t *testing.T) {
 	type testData struct {
 		deviceStr   string
-		expectedBDF string
+		expectedStr string
 	}
 
 	data := []testData{
 		{"0000:02:10.0", "02:10.0"},
 		{"0000:0210.0", ""},
+		{"f79944e4-5a3d-11e8-99ce-", ""},
+		{"f79944e4-5a3d-11e8-99ce", ""},
 		{"test", ""},
 		{"", ""},
 	}
 
 	for _, d := range data {
-		deviceBDF, err := getBDF(d.deviceStr)
-		assert.Equal(t, d.expectedBDF, deviceBDF)
-		if d.expectedBDF == "" {
+		deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(d.deviceStr, "")
+
+		switch vfioDeviceType {
+		case config.VFIODeviceNormalType:
+			assert.Equal(t, d.expectedStr, deviceBDF)
+		case config.VFIODeviceMediatedType:
+			assert.Equal(t, d.expectedStr, deviceSysfsDev)
+		default:
+			assert.NotNil(t, err)
+		}
+
+		if d.expectedStr == "" {
 			assert.NotNil(t, err)
 		} else {
 			assert.Nil(t, err)

virtcontainers/qemu.go

@@ -795,7 +795,14 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error {
 		// for pc machine type instead of bridge. This is useful for devices that require
 		// a large PCI BAR which is a currently a limitation with PCI bridges.
 		if q.state.HotplugVFIOOnRootBus {
+			switch device.Type {
+			case config.VFIODeviceNormalType:
 				return q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF)
+			case config.VFIODeviceMediatedType:
+				return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, "", "")
+			default:
+				return fmt.Errorf("Incorrect VFIO device type found")
+			}
 		}
 
 		addr, bridge, err := q.addDeviceToBridge(devID)
@@ -803,8 +810,13 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error {
 			return err
 		}
 
-		if err := q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID); err != nil {
-			return err
+		switch device.Type {
+		case config.VFIODeviceNormalType:
+			return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID)
+		case config.VFIODeviceMediatedType:
+			return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bridge.ID)
+		default:
+			return fmt.Errorf("Incorrect VFIO device type found")
 		}
 	} else {
 		if !q.state.HotplugVFIOOnRootBus {