diff --git a/src/runtime/pkg/device/config/config.go b/src/runtime/pkg/device/config/config.go index 2431fa36f1..f6ac005571 100644 --- a/src/runtime/pkg/device/config/config.go +++ b/src/runtime/pkg/device/config/config.go @@ -370,6 +370,9 @@ type VFIODev struct { // Rank identifies a device in a IOMMU group Rank int + + // Port is the PCIe port type to which the device is attached + Port PCIePort } // RNGDev represents a random number generator device diff --git a/src/runtime/pkg/device/drivers/utils.go b/src/runtime/pkg/device/drivers/utils.go index ec1609fe93..ddb1a0ded3 100644 --- a/src/runtime/pkg/device/drivers/utils.go +++ b/src/runtime/pkg/device/drivers/utils.go @@ -194,24 +194,18 @@ func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo) ([]*config.VFIODe switch vfioDeviceType { case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType: - isPCIe := IsPCIeDevice(deviceBDF) // Do not directly assign to `vfio` -- need to access field still - vfioPCI := config.VFIODev{ + vfio = config.VFIODev{ ID: id, Type: vfioDeviceType, BDF: deviceBDF, SysfsDev: deviceSysfsDev, - IsPCIe: isPCIe, + IsPCIe: IsPCIeDevice(deviceBDF), Class: pciClass, Rank: -1, - } - if isPCIe { - vfioPCI.Rank = len(AllPCIeDevs) - AllPCIeDevs[deviceBDF] = true - vfioPCI.Bus = fmt.Sprintf("%s%d", config.PCIePortPrefixMapping[device.Port], vfioPCI.Rank) + Port: device.Port, } - vfio = vfioPCI case config.VFIOAPDeviceMediatedType: devices, err := GetAPVFIODevices(deviceSysfsDev) if err != nil { diff --git a/src/runtime/pkg/device/drivers/vfio.go b/src/runtime/pkg/device/drivers/vfio.go index ad6ec5cc3d..8f63b2fc6c 100644 --- a/src/runtime/pkg/device/drivers/vfio.go +++ b/src/runtime/pkg/device/drivers/vfio.go @@ -76,6 +76,13 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece if err != nil { return err } + for _, vfio := range device.VfioDevs { + if vfio.IsPCIe { + vfio.Rank = len(AllPCIeDevs) + AllPCIeDevs[vfio.BDF] = true + vfio.Bus = fmt.Sprintf("%s%d", config.PCIePortPrefixMapping[vfio.Port], vfio.Rank) + } + } coldPlug := device.DeviceInfo.ColdPlug deviceLogger().WithField("cold-plug", coldPlug).Info("Attaching VFIO device") diff --git a/src/runtime/pkg/govmm/qemu/qmp.go b/src/runtime/pkg/govmm/qemu/qmp.go index 5630ff9dec..6b020103da 100644 --- a/src/runtime/pkg/govmm/qemu/qmp.go +++ b/src/runtime/pkg/govmm/qemu/qmp.go @@ -656,7 +656,6 @@ func (q *QMP) executeCommand(ctx context.Context, name string, args map[string]i filter *qmpEventFilter) error { _, err := q.executeCommandWithResponse(ctx, name, args, nil, filter) - return err } @@ -1192,7 +1191,6 @@ func (q *QMP) ExecutePCIVFIODeviceAdd(ctx context.Context, devID, bdf, addr, bus if bus != "" { args["bus"] = bus } - return q.executeCommand(ctx, "device_add", args, nil) } diff --git a/src/runtime/pkg/hypervisors/hypervisor_state.go b/src/runtime/pkg/hypervisors/hypervisor_state.go index 20d6c7d657..db7e47c729 100644 --- a/src/runtime/pkg/hypervisors/hypervisor_state.go +++ b/src/runtime/pkg/hypervisors/hypervisor_state.go @@ -39,7 +39,6 @@ type HypervisorState struct { // Belows are qemu specific // Refs: virtcontainers/qemu.go:QemuState Bridges []Bridge - // HotpluggedCPUs is the list of CPUs that were hot-added HotpluggedVCPUs []CPUDevice diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index acd5fcc7e7..c1cde33c57 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -704,14 +704,12 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi return err } } - if machine.Type == QemuQ35 { if err := q.createPCIeTopology(&qemuConfig, hypervisorConfig); err != nil { q.Logger().WithError(err).Errorf("Cannot create PCIe topology") return err } } - q.qemuConfig = qemuConfig q.virtiofsDaemon, err = q.createVirtiofsDaemon(hypervisorConfig.SharedPath) @@ -787,12 +785,6 @@ func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig if drivers.IsPCIeDevice(vfioDevice.BDF) { numOfPluggablePorts = numOfPluggablePorts + 1 } - // Reset the Rank and AllPCIeDevsthe vfio module is going - // to assign the correct one in the case of hot-plug - if hypervisorConfig.HotPlugVFIO != config.NoPort { - vfioDevice.Rank = -1 - drivers.AllPCIeDevs = map[string]bool{} - } } } diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 9fe88b28ce..9c227ad56c 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -640,7 +640,6 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor infos := sandboxConfig.Containers[cnt].DeviceInfos infos = append(infos[:dev], infos[dev+1:]...) sandboxConfig.Containers[cnt].DeviceInfos = infos - } } } @@ -2017,9 +2016,7 @@ func (s *Sandbox) AppendDevice(ctx context.Context, device api.Device) error { return s.hypervisor.AddDevice(ctx, device.GetDeviceInfo().(*config.VhostUserDeviceAttrs), VhostuserDev) case config.DeviceVFIO: vfioDevs := device.GetDeviceInfo().([]*config.VFIODev) - s.Logger().Info("### vfioDevs: ", vfioDevs) for _, d := range vfioDevs { - s.Logger().Info("### vfioDev: ", d) return s.hypervisor.AddDevice(ctx, *d, VfioDev) } default: