clh: Add vfio support

Add support to hotplug vfio devices.

Use hypervisor API to attach devices via hotplug.

Fixes: #2496

Signed-off-by: Jose Carlos Venegas Munoz <jose.carlos.venegas.munoz@intel.com>

virtcontainers/clh.go

@@ -52,6 +52,9 @@ const (
 	// Values based on:
 	clhTimeout    = 10
 	clhAPITimeout = 1
+	// Timeout for hot-plug - hotplug devices can take more time, than usual API calls
+	// Use longer time timeout for it.
+	clhHotPlugAPITimeout  = 5
 	clhStopSandboxTimeout = 3
 	clhSocket             = "clh.sock"
 	clhAPISocket          = "clh-api.sock"
@@ -76,6 +79,7 @@ type clhClient interface {
 	VmInfoGet(ctx context.Context) (chclient.VmInfo, *http.Response, error) //nolint:golint
 	BootVM(ctx context.Context) (*http.Response, error)
 	VmResizePut(ctx context.Context, vmResize chclient.VmResize) (*http.Response, error)
+	VmAddDevicePut(ctx context.Context, vmAddDevice chclient.VmAddDevice) (*http.Response, error)
 }
 
 type CloudHypervisorVersion struct {
@@ -372,9 +376,35 @@ func (clh *cloudHypervisor) getThreadIDs() (vcpuThreadIDs, error) {
 	return vcpuInfo, nil
 }
 
+func (clh *cloudHypervisor) hotPlugVFIODevice(device config.VFIODev) error {
+	cl := clh.client()
+	ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second)
+	defer cancel()
+
+	_, _, err := cl.VmmPingGet(ctx)
+	if err != nil {
+		return openAPIClientError(err)
+	}
+
+	_, err = cl.VmAddDevicePut(ctx, chclient.VmAddDevice{Path: device.SysfsDev})
+	if err != nil {
+		err = fmt.Errorf("Failed to hotplug device %+v %s", device, openAPIClientError(err))
+	}
+	return err
+}
+
 func (clh *cloudHypervisor) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) {
-	clh.Logger().WithField("function", "hotplugAddDevice").Warn("hotplug add device not supported")
+	span, _ := clh.trace("hotplugAddDevice")
-	return nil, nil
+	defer span.Finish()
+
+	switch devType {
+	case vfioDev:
+		device := devInfo.(*config.VFIODev)
+		return nil, clh.hotPlugVFIODevice(*device)
+	default:
+		return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType)
+	}
+
 }
 
 func (clh *cloudHypervisor) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) {

virtcontainers/clh_test.go

@@ -86,6 +86,11 @@ func (c *clhClientMock) VmResizePut(ctx context.Context, vmResize chclient.VmRes
 	return nil, nil
 }
 
+//nolint:golint
+func (c *clhClientMock) VmAddDevicePut(ctx context.Context, vmAddDevice chclient.VmAddDevice) (*http.Response, error) {
+	return nil, nil
+}
+
 func TestCloudHypervisorAddVSock(t *testing.T) {
 	assert := assert.New(t)
 	clh := cloudHypervisor{}

virtcontainers/device/drivers/vfio.go

@@ -194,6 +194,8 @@ func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceS
 	case config.VFIODeviceNormalType:
 		// Get bdf of device eg. 0000:00:1c.0
 		deviceBDF = getBDF(deviceFileName)
+		// Get sysfs path used by cloud-hypervisor
+		deviceSysfsDev = filepath.Join(config.SysBusPciDevicesPath, deviceFileName)
 	case config.VFIODeviceMediatedType:
 		// Get sysfsdev of device eg. /sys/devices/pci0000:00/0000:00:02.0/f79944e4-5a3d-11e8-99ce-479cbab002e4
 		sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName)