vfio: Rearrange the bus assignemnt

Refactor the bus assignment so that the call to GetAllVFIODevicesFromIOMMUGroup
can be used by any module without affecting the topology.

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>

src/runtime/pkg/device/config/config.go

@@ -370,6 +370,9 @@ type VFIODev struct {
 
 	// Rank identifies a device in a IOMMU group
 	Rank int
+
+	// Port is the PCIe port type to which the device is attached
+	Port PCIePort
 }
 
 // RNGDev represents a random number generator device

src/runtime/pkg/device/drivers/utils.go

@@ -194,24 +194,18 @@ func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo) ([]*config.VFIODe
 
 		switch vfioDeviceType {
 		case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType:
-			isPCIe := IsPCIeDevice(deviceBDF)
 			// Do not directly assign to `vfio` -- need to access field still
-			vfioPCI := config.VFIODev{
+			vfio = config.VFIODev{
 				ID:       id,
 				Type:     vfioDeviceType,
 				BDF:      deviceBDF,
 				SysfsDev: deviceSysfsDev,
-				IsPCIe:   isPCIe,
+				IsPCIe:   IsPCIeDevice(deviceBDF),
 				Class:    pciClass,
 				Rank:     -1,
-			}
+				Port:     device.Port,
-			if isPCIe {
-				vfioPCI.Rank = len(AllPCIeDevs)
-				AllPCIeDevs[deviceBDF] = true
-				vfioPCI.Bus = fmt.Sprintf("%s%d", config.PCIePortPrefixMapping[device.Port], vfioPCI.Rank)
 			}
 
-			vfio = vfioPCI
 		case config.VFIOAPDeviceMediatedType:
 			devices, err := GetAPVFIODevices(deviceSysfsDev)
 			if err != nil {

src/runtime/pkg/device/drivers/vfio.go

@@ -76,6 +76,13 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece
 	if err != nil {
 		return err
 	}
+	for _, vfio := range device.VfioDevs {
+		if vfio.IsPCIe {
+			vfio.Rank = len(AllPCIeDevs)
+			AllPCIeDevs[vfio.BDF] = true
+			vfio.Bus = fmt.Sprintf("%s%d", config.PCIePortPrefixMapping[vfio.Port], vfio.Rank)
+		}
+	}
 
 	coldPlug := device.DeviceInfo.ColdPlug
 	deviceLogger().WithField("cold-plug", coldPlug).Info("Attaching VFIO device")

src/runtime/pkg/govmm/qemu/qmp.go

@@ -656,7 +656,6 @@ func (q *QMP) executeCommand(ctx context.Context, name string, args map[string]i
 	filter *qmpEventFilter) error {
 
 	_, err := q.executeCommandWithResponse(ctx, name, args, nil, filter)
-
 	return err
 }
 
@@ -1192,7 +1191,6 @@ func (q *QMP) ExecutePCIVFIODeviceAdd(ctx context.Context, devID, bdf, addr, bus
 	if bus != "" {
 		args["bus"] = bus
 	}
-
 	return q.executeCommand(ctx, "device_add", args, nil)
 }
 

src/runtime/pkg/hypervisors/hypervisor_state.go

@@ -39,7 +39,6 @@ type HypervisorState struct {
 	// Belows are qemu specific
 	// Refs: virtcontainers/qemu.go:QemuState
 	Bridges []Bridge
-
 	// HotpluggedCPUs is the list of CPUs that were hot-added
 	HotpluggedVCPUs []CPUDevice
 

src/runtime/virtcontainers/qemu.go

@@ -704,14 +704,12 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
 			return err
 		}
 	}
-
 	if machine.Type == QemuQ35 {
 		if err := q.createPCIeTopology(&qemuConfig, hypervisorConfig); err != nil {
 			q.Logger().WithError(err).Errorf("Cannot create PCIe topology")
 			return err
 		}
 	}
-
 	q.qemuConfig = qemuConfig
 
 	q.virtiofsDaemon, err = q.createVirtiofsDaemon(hypervisorConfig.SharedPath)
@@ -787,12 +785,6 @@ func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig
 			if drivers.IsPCIeDevice(vfioDevice.BDF) {
 				numOfPluggablePorts = numOfPluggablePorts + 1
 			}
-			// Reset the Rank and AllPCIeDevsthe vfio module is going
-			// to assign the correct one in the case of hot-plug
-			if hypervisorConfig.HotPlugVFIO != config.NoPort {
-				vfioDevice.Rank = -1
-				drivers.AllPCIeDevs = map[string]bool{}
-			}
 		}
 	}
 

src/runtime/virtcontainers/sandbox.go

@@ -640,7 +640,6 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 				infos := sandboxConfig.Containers[cnt].DeviceInfos
 				infos = append(infos[:dev], infos[dev+1:]...)
 				sandboxConfig.Containers[cnt].DeviceInfos = infos
-
 			}
 		}
 	}
@@ -2017,9 +2016,7 @@ func (s *Sandbox) AppendDevice(ctx context.Context, device api.Device) error {
 		return s.hypervisor.AddDevice(ctx, device.GetDeviceInfo().(*config.VhostUserDeviceAttrs), VhostuserDev)
 	case config.DeviceVFIO:
 		vfioDevs := device.GetDeviceInfo().([]*config.VFIODev)
-		s.Logger().Info("### vfioDevs: ", vfioDevs)
 		for _, d := range vfioDevs {
-			s.Logger().Info("### vfioDev: ", d)
 			return s.hypervisor.AddDevice(ctx, *d, VfioDev)
 		}
 	default: