From f1aec98f9d0a5f996b96b17db42041caea0d227c Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Mon, 14 Aug 2023 09:16:07 +0000 Subject: [PATCH] qemu/virt: use pcie_root_port to do device hotplug for virt ACPI PCI device hotplug on qemu virt is not supported. The only way to hotplug pci device is pcie native way. Thus we need create pcie root port as default. Pcie root port number depends on following: 1. reserved one for network device as default; 2. virtio-mem dev; 3. add enough port for vhost user blk dev; Fixes: #7646 Signed-off-by: Jianyong Wu --- src/runtime/virtcontainers/qemu.go | 39 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 03b5fa676a..555be392c3 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -720,7 +720,7 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi } if machine.Type == QemuQ35 || machine.Type == QemuVirt { - if err := q.createPCIeTopology(&qemuConfig, hypervisorConfig, machine.Type); err != nil { + if err := q.createPCIeTopology(&qemuConfig, hypervisorConfig, machine.Type, network); err != nil { q.Logger().WithError(err).Errorf("Cannot create PCIe topology") return err } @@ -759,7 +759,7 @@ func (q *qemu) checkBpfEnabled() { // Max PCIe switch ports is 16 // There is only 64kB of IO memory each root,switch port will consume 4k hence // only 16 ports possible. -func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig *HypervisorConfig, machineType string) error { +func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig *HypervisorConfig, machineType string, network Network) error { // If no-port set just return no need to add PCIe Root Port or PCIe Switches if hypervisorConfig.HotPlugVFIO == config.NoPort && hypervisorConfig.ColdPlugVFIO == config.NoPort && machineType == QemuQ35 { @@ -787,8 +787,21 @@ func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig } // Get the number of hot(cold)-pluggable ports needed from the provided - // VFIO devices and VhostUserBlockDevices + // VFIO devices var numOfPluggablePorts uint32 = 0 + + // Fow now, pcie native hotplug is the only way for Arm to hotadd pci device. + if machineType == QemuVirt { + epNum, err := network.GetEndpointsNum() + if err != nil { + q.Logger().Warn("Fail to get network endpoints number") + } + virtPcieRootPortNum := len(hypervisorConfig.VhostUserBlkDevices) + epNum + if hypervisorConfig.VirtioMem { + virtPcieRootPortNum++ + } + numOfPluggablePorts += uint32(virtPcieRootPortNum) + } for _, dev := range hypervisorConfig.VFIODevices { var err error dev.HostPath, err = config.GetHostPath(dev, false, "") @@ -809,18 +822,11 @@ func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig vfioOnRootPort := (q.state.HotPlugVFIO == config.RootPort || q.state.ColdPlugVFIO == config.RootPort) vfioOnSwitchPort := (q.state.HotPlugVFIO == config.SwitchPort || q.state.ColdPlugVFIO == config.SwitchPort) - numOfVhostUserBlockDevices := len(hypervisorConfig.VhostUserBlkDevices) - // If number of PCIe root ports > 16 then bail out otherwise we may // use up all slots or IO memory on the root bus and vfio-XXX-pci devices // cannot be added which are crucial for Kata max slots on root bus is 32 // max slots on the complete pci(e) topology is 256 in QEMU if vfioOnRootPort { - // On Arm the vhost-user-block device is a PCIe device we need - // to account for it in the number of pluggable ports - if machineType == QemuVirt { - numOfPluggablePorts = numOfPluggablePorts + uint32(numOfVhostUserBlockDevices) - } if numOfPluggablePorts > maxPCIeRootPort { return fmt.Errorf("Number of PCIe Root Ports exceeed allowed max of %d", maxPCIeRootPort) } @@ -828,21 +834,16 @@ func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig return nil } if vfioOnSwitchPort { - // On Arm the vhost-user-block device is a PCIe device we need - // to account for it in the number of pluggable ports - if machineType == QemuVirt { - numOfPluggableRootPorts := uint32(numOfVhostUserBlockDevices) - if numOfPluggableRootPorts > maxPCIeRootPort { - return fmt.Errorf("Number of PCIe Root Ports exceeed allowed max of %d", maxPCIeRootPort) - } - qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, numOfPluggableRootPorts, memSize32bit, memSize64bit) - } if numOfPluggablePorts > maxPCIeSwitchPort { return fmt.Errorf("Number of PCIe Switch Ports exceeed allowed max of %d", maxPCIeSwitchPort) } qemuConfig.Devices = q.arch.appendPCIeSwitchPortDevice(qemuConfig.Devices, numOfPluggablePorts, memSize32bit, memSize64bit) return nil } + // If both Root Port and Switch Port are not enabled, check if QemuVirt need add pcie root port. + if machineType == QemuVirt { + qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, numOfPluggablePorts, memSize32bit, memSize64bit) + } return nil }