From 72f2cb84e61f9b544d295beeb6f9b76f6019a411 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Thu, 15 Jun 2023 17:37:01 +0000 Subject: [PATCH] gpu: Reset cold or hot plug after overriding If we override the cold, hot plug with an annotation we need to reset the other plugging mechanism to NoPort otherwise both will be enabled. Signed-off-by: Zvonko Kaiser --- src/runtime/pkg/device/drivers/vfio.go | 2 -- src/runtime/pkg/oci/utils.go | 4 ++++ src/runtime/virtcontainers/qemu.go | 8 +++++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/runtime/pkg/device/drivers/vfio.go b/src/runtime/pkg/device/drivers/vfio.go index a9875f7e75..801b1a81f8 100644 --- a/src/runtime/pkg/device/drivers/vfio.go +++ b/src/runtime/pkg/device/drivers/vfio.go @@ -71,8 +71,6 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece } for _, vfio := range device.VfioDevs { if vfio.IsPCIe { - //vfio.Rank = len(AllPCIeDevs) - //AllPCIeDevs[vfio.BDF] = true busIndex := len(config.PCIeDevices[vfio.Port]) vfio.Bus = fmt.Sprintf("%s%d", config.PCIePortPrefixMapping[vfio.Port], busIndex) config.PCIeDevices[vfio.Port][vfio.BDF] = true diff --git a/src/runtime/pkg/oci/utils.go b/src/runtime/pkg/oci/utils.go index e53fc8e2ca..21e0271603 100644 --- a/src/runtime/pkg/oci/utils.go +++ b/src/runtime/pkg/oci/utils.go @@ -592,11 +592,15 @@ func addHypervisorHotColdPlugVfioOverrides(ocispec specs.Spec, sbConfig *vc.Sand if sbConfig.HypervisorConfig.HotPlugVFIO, err = addHypervisorPCIePortOverride(value); err != nil { return err } + // If hot-plug is specified disable cold-plug and vice versa + sbConfig.HypervisorConfig.ColdPlugVFIO = config.NoPort } if value, ok := ocispec.Annotations[vcAnnotations.ColdPlugVFIO]; ok { if sbConfig.HypervisorConfig.ColdPlugVFIO, err = addHypervisorPCIePortOverride(value); err != nil { return err } + // If cold-plug is specified disable hot-plug and vice versa + sbConfig.HypervisorConfig.HotPlugVFIO = config.NoPort } return nil } diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index ad94923ffb..80a05b3cf1 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -65,6 +65,11 @@ const romFile = "" // Default value is false. const defaultDisableModern = false +// A deeper PCIe topology than 5 is already not advisable just for the sake +// of having enough buffer we limit ourselves to 10 and exit if we reach +// the root bus +const maxPCIeTopoDepth = 10 + type qmpChannel struct { qmp *govmmQemu.QMP ctx context.Context @@ -1771,6 +1776,7 @@ func (q *qemu) qomGetPciPath(qemuID string) (types.PciPath, error) { } slots = append(slots, devSlot) + // This only works for Q35 and Virt r, _ := regexp.Compile(`^/machine/.*/pcie.0`) var parentPath = qemuID @@ -1778,7 +1784,7 @@ func (q *qemu) qomGetPciPath(qemuID string) (types.PciPath, error) { // than 5 is already not advisable just for the sake of having enough // buffer we limit ourselves to 10 and leave the loop early if we hit // the root bus. - for i := 1; i <= 10; i++ { + for i := 1; i <= maxPCIeTopoDepth; i++ { parenBusQOM, err := q.qmpMonitorCh.qmp.ExecQomGet(q.qmpMonitorCh.ctx, parentPath, "parent_bus") if err != nil { return types.PciPath{}, err