qemu: Add command line for IOMMUFD

For each IOMMUFD device create an object and assign
it to the device, we need additional information that
is populated now correctly to decide if we run the old VFIO
or new VFIO backend.

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>

src/runtime/pkg/device/drivers/utils.go

@@ -233,7 +233,7 @@ func GetDeviceFromVFIODev(device config.DeviceInfo) ([]*config.VFIODev, error) {
 		ID:       id,
 		Type:     vfioDeviceType,
 		BDF:      deviceBDF,
-		SysfsDev: deviceSysfsDev,
+		SysfsDev: device.HostPath,
 		IsPCIe:   IsPCIeDevice(deviceBDF),
 		Class:    pciClass,
 		VendorID: vendorID,

src/runtime/pkg/device/drivers/vfio.go

@@ -71,6 +71,9 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece
 	// (1) Check if we have the new IOMMUFD or old container based VFIO
 	if strings.HasPrefix(device.DeviceInfo.HostPath, IommufdDevPath) {
 		device.VfioDevs, err = GetDeviceFromVFIODev(*device.DeviceInfo)
+		if err != nil {
+			return err
+		}
 	} else {
 		// Once we have
 		device.VfioDevs, err = GetAllVFIODevicesFromIOMMUGroup(*device.DeviceInfo)

src/runtime/pkg/govmm/qemu/qemu.go

@@ -25,6 +25,8 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
 )
 
 // Machine describes the machine type qemu will emulate.
@@ -1869,6 +1871,9 @@ func (b PCIeSwitchDownstreamPortDevice) Valid() bool {
 
 // VFIODevice represents a qemu vfio device meant for direct access by guest OS.
 type VFIODevice struct {
+	// ID index of the vfio device in devfs or sysfs used for IOMMUFD
+	ID string
+
 	// Bus-Device-Function of device
 	BDF string
 
@@ -1946,6 +1951,12 @@ func (vfioDev VFIODevice) QemuParams(config *Config) []string {
 		deviceParams = append(deviceParams, fmt.Sprintf("devno=%s", vfioDev.DevNo))
 	}
 
+	if strings.HasPrefix(vfioDev.SysfsDev, drivers.IommufdDevPath) {
+		qemuParams = append(qemuParams, "-object")
+		qemuParams = append(qemuParams, fmt.Sprintf("iommufd,id=iommufd%s", vfioDev.ID))
+		deviceParams = append(deviceParams, fmt.Sprintf("iommufd=iommufd%s", vfioDev.ID))
+	}
+
 	qemuParams = append(qemuParams, "-device")
 	qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
 

src/runtime/virtcontainers/qemu.go

@@ -792,7 +792,7 @@ func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig
 			return fmt.Errorf("Cannot get host path for device: %v err: %v", dev, err)
 		}
 
-		vfioDevices := []*config.VFIODev{}
+		var vfioDevices []*config.VFIODev
 		// This works for IOMMUFD enabled kernels > 6.x
 		// In the case of IOMMUFD the device.HostPath will look like
 		// /dev/vfio/devices/vfio0

src/runtime/virtcontainers/qemu_arch_base.go

@@ -708,10 +708,12 @@ func (q *qemuArchBase) appendVFIODevice(devices []govmmQemu.Device, vfioDev conf
 
 	devices = append(devices,
 		govmmQemu.VFIODevice{
+			ID:       vfioDev.ID,
 			BDF:      vfioDev.BDF,
 			VendorID: vfioDev.VendorID,
 			DeviceID: vfioDev.DeviceID,
 			Bus:      vfioDev.Bus,
+			SysfsDev: vfioDev.SysfsDev,
 		},
 	)