vfio: Added better handling of VFIO Control Devices

Depending on the vfio_mode we need to mount the
VFIO control device additionally into the container.

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>

src/runtime/pkg/device/drivers/utils.go

@@ -216,6 +216,7 @@ func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo) ([]*config.VFIODe
 				SysfsDev:  deviceSysfsDev,
 				Type:      config.VFIOAPDeviceMediatedType,
 				APDevices: devices,
+				Port:      device.Port,
 			}
 		default:
 			return nil, fmt.Errorf("Failed to append device: VFIO device type unrecognized")

src/runtime/pkg/device/manager/manager.go

@@ -120,7 +120,7 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device
 	if devInfo.ID, err = dm.newDeviceID(); err != nil {
 		return nil, err
 	}
-	if IsVFIO(devInfo.HostPath) {
+	if IsVFIODevice(devInfo.HostPath) {
 		return drivers.NewVFIODevice(&devInfo), nil
 	} else if IsVhostUserBlk(devInfo) {
 		if devInfo.DriverOptions == nil {
@@ -191,12 +191,12 @@ func (dm *deviceManager) AttachDevice(ctx context.Context, id string, dr api.Dev
 	dm.Lock()
 	defer dm.Unlock()
 
-	d, ok := dm.devices[id]
+	dev, ok := dm.devices[id]
 	if !ok {
 		return ErrDeviceNotExist
 	}
 
-	if err := d.Attach(ctx, dr); err != nil {
+	if err := dev.Attach(ctx, dr); err != nil {
 		return err
 	}
 	return nil

src/runtime/pkg/device/manager/utils.go

@@ -17,8 +17,15 @@ const (
 	vfioPath = "/dev/vfio/"
 )
 
+// IsVFIOControlDevice checks if the device provided is a vfio control device.
+// Depending no the vfio_mode we need to know if a device is a VFIO device
+// or the VFIO control device
+func IsVFIOControlDevice(path string) bool {
+	return path == filepath.Join(vfioPath, "vfio")
+}
+
 // IsVFIO checks if the device provided is a vfio group.
-func IsVFIO(hostPath string) bool {
+func IsVFIODevice(hostPath string) bool {
 	// Ignore /dev/vfio/vfio character device
 	if strings.HasPrefix(hostPath, filepath.Join(vfioPath, "vfio")) {
 		return false

src/runtime/pkg/device/manager/utils_test.go

@@ -31,7 +31,7 @@ func TestIsVFIO(t *testing.T) {
 	}
 
 	for _, d := range data {
-		isVFIO := IsVFIO(d.path)
+		isVFIO := IsVFIODevice(d.path)
 		assert.Equal(t, d.expected, isVFIO)
 	}
 }

src/runtime/virtcontainers/container.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager"
+	deviceManager "github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager"
 	volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
@@ -872,9 +873,16 @@ func (c *Container) create(ctx context.Context) (err error) {
 	// If cold-plug we've attached the devices already, do not try to
 	// attach them a second time.
 	coldPlugVFIO := (c.sandbox.config.HypervisorConfig.ColdPlugVFIO != config.NoPort)
+	modeVFIO := (c.sandbox.config.VfioMode == config.VFIOModeVFIO)
+
 	if coldPlugVFIO {
 		var cntDevices []ContainerDevice
 		for _, dev := range c.devices {
+			isVFIOControlDevice := deviceManager.IsVFIOControlDevice(dev.ContainerPath)
+			if isVFIOControlDevice && modeVFIO {
+				cntDevices = append(cntDevices, dev)
+			}
+
 			if strings.HasPrefix(dev.ContainerPath, vfioPath) {
 				c.Logger().WithFields(logrus.Fields{
 					"device": dev,

src/runtime/virtcontainers/sandbox.go

@@ -619,7 +619,9 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 	// Aggregate all the containner devices for hot-plug and use them to dedcue
 	// the correct amount of ports to reserve for the hypervisor.
 	hotPlugVFIO := (sandboxConfig.HypervisorConfig.HotPlugVFIO != config.NoPort)
-	stripVFIO := sandboxConfig.VfioMode == config.VFIOModeGuestKernel
+
+	modeGK := (sandboxConfig.VfioMode == config.VFIOModeGuestKernel)
+	modeVFIO := (sandboxConfig.VfioMode == config.VFIOModeVFIO)
 
 	var vfioDevices []config.DeviceInfo
 	// vhost-user-block device is a PCIe device in Virt, keep track of it
@@ -633,19 +635,26 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 				vhostUserBlkDevices = append(vhostUserBlkDevices, device)
 				continue
 			}
-			isVFIO := deviceManager.IsVFIO(device.ContainerPath)
+			isVFIODevice := deviceManager.IsVFIODevice(device.ContainerPath)
-			if hotPlugVFIO && isVFIO {
+			isVFIOControlDevice := deviceManager.IsVFIOControlDevice(device.ContainerPath)
+			// vfio_mode=vfio needs the VFIO control device add it to the list
+			// of devices to be added to the VM.
+			if modeVFIO && isVFIOControlDevice {
+				vfioDevices = append(vfioDevices, device)
+			}
+
+			if hotPlugVFIO && isVFIODevice {
 				vfioDevices = append(vfioDevices, device)
 				sandboxConfig.Containers[cnt].DeviceInfos[dev].Port = sandboxConfig.HypervisorConfig.HotPlugVFIO
 			}
-			if coldPlugVFIO && isVFIO {
+			if coldPlugVFIO && isVFIODevice {
 				device.ColdPlug = true
 				device.Port = sandboxConfig.HypervisorConfig.ColdPlugVFIO
 				vfioDevices = append(vfioDevices, device)
 				// We need to remove the devices marked for cold-plug
 				// otherwise at the container level the kata-agent
 				// will try to hot-plug them.
-				if stripVFIO {
+				if modeGK {
 					sandboxConfig.Containers[cnt].DeviceInfos[dev].ID = "remove-we-are-cold-plugging"
 				}
 			}
@@ -2053,26 +2062,26 @@ func (s *Sandbox) AddDevice(ctx context.Context, info config.DeviceInfo) (api.De
 	}
 
 	var err error
-	b, err := s.devManager.NewDevice(info)
+	add, err := s.devManager.NewDevice(info)
 	if err != nil {
 		return nil, err
 	}
 	defer func() {
 		if err != nil {
-			s.devManager.RemoveDevice(b.DeviceID())
+			s.devManager.RemoveDevice(add.DeviceID())
 		}
 	}()
 
-	if err = s.devManager.AttachDevice(ctx, b.DeviceID(), s); err != nil {
+	if err = s.devManager.AttachDevice(ctx, add.DeviceID(), s); err != nil {
 		return nil, err
 	}
 	defer func() {
 		if err != nil {
-			s.devManager.DetachDevice(ctx, b.DeviceID(), s)
+			s.devManager.DetachDevice(ctx, add.DeviceID(), s)
 		}
 	}()
 
-	return b, nil
+	return add, nil
 }
 
 // updateResources will: