diff --git a/qemu/qemu.go b/qemu/qemu.go index f2b80ccd1..c9e0eb151 100644 --- a/qemu/qemu.go +++ b/qemu/qemu.go @@ -106,6 +106,9 @@ const ( // VirtioBlockCCW is the CCW block device driver VirtioBlockCCW DeviceDriver = "virtio-blk-ccw" + + // PCIeRootPort is a PCIe Root Port, the PCIe device should be hotplugged to this port. + PCIeRootPort DeviceDriver = "pcie-root-port" ) // disableModern returns the parameters with the disable-modern option. @@ -898,6 +901,102 @@ func (vhostuserDev VhostUserDevice) QemuParams(config *Config) []string { return qemuParams } +// PCIeRootPortDevice represents a memory balloon device. +type PCIeRootPortDevice struct { + ID string // format: rp{n}, n>=0 + + Bus string // default is pcie.0 + Chassis string // (slot, chassis) pair is mandatory and must be unique for each pcie-root-port, >=0, default is 0x00 + Slot string // >=0, default is 0x00 + + Multifunction bool // true => "on", false => "off", default is off + Addr string // >=0, default is 0x00 + + // The PCIE-PCI bridge can be hot-plugged only into pcie-root-port that has 'bus-reserve' property value to + // provide secondary bus for the hot-plugged bridge. + BusReserve string + Pref64Reserve string // reserve prefetched MMIO aperture, 64-bit + Pref32Reserve string // reserve prefetched MMIO aperture, 32-bit + MemReserve string // reserve non-prefetched MMIO aperture, 32-bit *only* + IOReserve string // IO reservation + + ROMFile string // ROMFile specifies the ROM file being used for this device. +} + +// QemuParams returns the qemu parameters built out of the PCIeRootPortDevice. +func (b PCIeRootPortDevice) QemuParams(_ *Config) []string { + var qemuParams []string + var deviceParams []string + driver := PCIeRootPort + + deviceParams = append(deviceParams, fmt.Sprintf("%s,id=%s", driver, b.ID)) + + if b.Bus == "" { + b.Bus = "pcie.0" + } + deviceParams = append(deviceParams, fmt.Sprintf("bus=%s", b.Bus)) + + if b.Chassis == "" { + b.Chassis = "0x00" + } + deviceParams = append(deviceParams, fmt.Sprintf("chassis=%s", b.Chassis)) + + if b.Slot == "" { + b.Slot = "0x00" + } + deviceParams = append(deviceParams, fmt.Sprintf("slot=%s", b.Slot)) + + multifunction := "off" + if b.Multifunction { + multifunction = "on" + if b.Addr == "" { + b.Addr = "0x00" + } + deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", b.Addr)) + } + deviceParams = append(deviceParams, fmt.Sprintf("multifunction=%v", multifunction)) + + if b.BusReserve != "" { + deviceParams = append(deviceParams, fmt.Sprintf("bus-reserve=%s", b.BusReserve)) + } + + if b.Pref64Reserve != "" { + deviceParams = append(deviceParams, fmt.Sprintf("pref64-reserve=%s", b.Pref64Reserve)) + } + + if b.Pref32Reserve != "" { + deviceParams = append(deviceParams, fmt.Sprintf("pref32-reserve=%s", b.Pref32Reserve)) + } + + if b.MemReserve != "" { + deviceParams = append(deviceParams, fmt.Sprintf("mem-reserve=%s", b.MemReserve)) + } + + if b.IOReserve != "" { + deviceParams = append(deviceParams, fmt.Sprintf("io-reserve=%s", b.IOReserve)) + } + + if isVirtioPCI[driver] && b.ROMFile != "" { + deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", b.ROMFile)) + } + + qemuParams = append(qemuParams, "-device") + qemuParams = append(qemuParams, strings.Join(deviceParams, ",")) + return qemuParams +} + +// Valid returns true if the PCIeRootPortDevice structure is valid and complete. +func (b PCIeRootPortDevice) Valid() bool { + // the "pref32-reserve" and "pref64-reserve" hints are mutually exclusive. + if b.Pref64Reserve != "" && b.Pref32Reserve != "" { + return false + } + if b.ID == "" { + return false + } + return true +} + // VFIODevice represents a qemu vfio device meant for direct access by guest OS. type VFIODevice struct { // Bus-Device-Function of device @@ -914,6 +1013,9 @@ type VFIODevice struct { // DeviceID specifies device id DeviceID string + + // Bus specifies device bus + Bus string } // Valid returns true if the VFIODevice structure is valid and complete. @@ -939,6 +1041,10 @@ func (vfioDev VFIODevice) QemuParams(config *Config) []string { deviceParams = append(deviceParams, fmt.Sprintf(",romfile=%s", vfioDev.ROMFile)) } + if vfioDev.Bus != "" { + deviceParams = append(deviceParams, fmt.Sprintf(",bus=%s", vfioDev.Bus)) + } + if isVirtioCCW[driver] { deviceParams = append(deviceParams, fmt.Sprintf(",devno=%s", vfioDev.DevNo)) } diff --git a/qemu/qemu_arch_base.go b/qemu/qemu_arch_base.go index d73c34f9a..3e2e03d00 100644 --- a/qemu/qemu_arch_base.go +++ b/qemu/qemu_arch_base.go @@ -59,6 +59,7 @@ var isVirtioPCI = map[DeviceDriver]bool{ VirtioScsi: true, PCIBridgeDriver: true, PCIePCIBridgeDriver: true, + PCIeRootPort: true, } // isVirtioCCW is a dummy map to return always false on no-s390x arch diff --git a/qemu/qemu_arch_base_test.go b/qemu/qemu_arch_base_test.go index 242eb8200..348f46538 100644 --- a/qemu/qemu_arch_base_test.go +++ b/qemu/qemu_arch_base_test.go @@ -28,6 +28,10 @@ var ( deviceVhostUserNetString = "-chardev socket,id=char1,path=/tmp/nonexistentsocket.socket -netdev type=vhost-user,id=net1,chardev=char1,vhostforce -device virtio-net-pci,netdev=net1,mac=00:11:22:33:44:55,romfile=efi-virtio.rom" deviceVSOCKString = "-device vhost-vsock-pci,disable-modern=true,id=vhost-vsock-pci0,guest-cid=4,romfile=efi-virtio.rom" deviceVFIOString = "-device vfio-pci,host=02:10.0,x-pci-vendor-id=0x1234,x-pci-device-id=0x5678,romfile=efi-virtio.rom" + devicePCIeRootPortSimpleString = "-device pcie-root-port,id=rp1,bus=pcie.0,chassis=0x00,slot=0x00,multifunction=off" + devicePCIeRootPortFullString = "-device pcie-root-port,id=rp2,bus=pcie.0,chassis=0x0,slot=0x1,addr=0x2,multifunction=on,bus-reserve=0x3,pref64-reserve=16G,mem-reserve=1G,io-reserve=512M,romfile=efi-virtio.rom" + deviceVFIOPCIeSimpleString = "-device vfio-pci,host=02:00.0,romfile=,bus=rp0" + deviceVFIOPCIeFullString = "-device vfio-pci,host=02:00.0,x-pci-vendor-id=0x10de,x-pci-device-id=0x15f8,romfile=efi-virtio.rom,bus=rp1" deviceSCSIControllerStr = "-device virtio-scsi-pci,id=foo,disable-modern=false,romfile=efi-virtio.rom" deviceSCSIControllerBusAddrStr = "-device virtio-scsi-pci,id=foo,bus=pci.0,addr=00:04.0,disable-modern=true,iothread=iothread1,romfile=efi-virtio.rom" deviceVhostUserSCSIString = "-chardev socket,id=char1,path=/tmp/nonexistentsocket.socket -device vhost-user-scsi-pci,id=scsi1,chardev=char1,romfile=efi-virtio.rom" @@ -95,3 +99,81 @@ func TestAppendVirtioBalloon(t *testing.T) { testAppend(balloonDevice, deviceString+OnDeflateOnOMM+OnDisableModern, t) } + +func TestAppendDevicePCIeRootPort(t *testing.T) { + var pcieRootPortID string + + // test empty ID + pcieRootPortDevice := PCIeRootPortDevice{} + if pcieRootPortDevice.Valid() { + t.Fatalf("failed to validdate empty ID") + } + + // test pref64_reserve and pre64_reserve + pcieRootPortID = "rp0" + pcieRootPortDevice = PCIeRootPortDevice{ + ID: pcieRootPortID, + Pref64Reserve: "16G", + Pref32Reserve: "256M", + } + if pcieRootPortDevice.Valid() { + t.Fatalf("failed to validate pref32-reserve and pref64-reserve for %v", pcieRootPortID) + } + + // default test + pcieRootPortID = "rp1" + pcieRootPortDevice = PCIeRootPortDevice{ + ID: pcieRootPortID, + } + if !pcieRootPortDevice.Valid() { + t.Fatalf("failed to validate for %v", pcieRootPortID) + } + testAppend(pcieRootPortDevice, devicePCIeRootPortSimpleString, t) + + // full test + pcieRootPortID = "rp2" + pcieRootPortDevice = PCIeRootPortDevice{ + ID: pcieRootPortID, + Multifunction: true, + Bus: "pcie.0", + Chassis: "0x0", + Slot: "0x1", + Addr: "0x2", + Pref64Reserve: "16G", + IOReserve: "512M", + MemReserve: "1G", + BusReserve: "0x3", + ROMFile: romfile, + } + if !pcieRootPortDevice.Valid() { + t.Fatalf("failed to validate for %v", pcieRootPortID) + } + testAppend(pcieRootPortDevice, devicePCIeRootPortFullString, t) +} + +func TestAppendDeviceVFIOPCIe(t *testing.T) { + // default test + pcieRootPortID := "rp0" + vfioDevice := VFIODevice{ + BDF: "02:00.0", + Bus: pcieRootPortID, + } + if isVirtioCCW[Vfio] { + vfioDevice.DevNo = DevNo + } + testAppend(vfioDevice, deviceVFIOPCIeSimpleString, t) + + // full test + pcieRootPortID = "rp1" + vfioDevice = VFIODevice{ + BDF: "02:00.0", + Bus: pcieRootPortID, + ROMFile: romfile, + VendorID: "0x10de", + DeviceID: "0x15f8", + } + if isVirtioCCW[Vfio] { + vfioDevice.DevNo = DevNo + } + testAppend(vfioDevice, deviceVFIOPCIeFullString, t) +} diff --git a/qemu/qmp.go b/qemu/qmp.go index 9d27dbeef..f874fb6af 100644 --- a/qemu/qmp.go +++ b/qemu/qmp.go @@ -1156,17 +1156,20 @@ func (q *QMP) ExecutePCIVhostUserDevAdd(ctx context.Context, driver, devID, char return q.executeCommand(ctx, "device_add", args, nil) } -// ExecuteVFIODeviceAdd adds a VFIO device to a QEMU instance -// using the device_add command. devID is the id of the device to add. -// Must be valid QMP identifier. bdf is the PCI bus-device-function -// of the pci device. -func (q *QMP) ExecuteVFIODeviceAdd(ctx context.Context, devID, bdf, romfile string) error { +// ExecuteVFIODeviceAdd adds a VFIO device to a QEMU instance using the device_add command. +// devID is the id of the device to add. Must be valid QMP identifier. +// bdf is the PCI bus-device-function of the pci device. +// bus is optional. When hot plugging a PCIe device, the bus can be the ID of the pcie-root-port. +func (q *QMP) ExecuteVFIODeviceAdd(ctx context.Context, devID, bdf, bus, romfile string) error { args := map[string]interface{}{ "id": devID, "driver": Vfio, "host": bdf, "romfile": romfile, } + if bus != "" { + args["bus"] = bus + } return q.executeCommand(ctx, "device_add", args, nil) } diff --git a/qemu/qmp_test.go b/qemu/qmp_test.go index 71f97bcad..dcd8e68ab 100644 --- a/qemu/qmp_test.go +++ b/qemu/qmp_test.go @@ -1079,6 +1079,27 @@ func TestQMPPCIVFIOMediatedDeviceAdd(t *testing.T) { <-disconnectedCh } +func TestQMPPCIVFIOPCIeDeviceAdd(t *testing.T) { + connectedCh := make(chan *QMPVersion) + disconnectedCh := make(chan struct{}) + buf := newQMPTestCommandBuffer(t) + buf.AddCommand("device_add", nil, "return", nil) + cfg := QMPConfig{Logger: qmpTestLogger{}} + q := startQMPLoop(buf, cfg, connectedCh, disconnectedCh) + checkVersion(t, connectedCh) + bdf := "04:00.0" + bus := "rp0" + addr := "0x1" + romfile := "" + devID := fmt.Sprintf("device_%s", volumeUUID) + err := q.ExecutePCIVFIODeviceAdd(context.Background(), devID, bdf, addr, bus, romfile) + if err != nil { + t.Fatalf("Unexpected error %v", err) + } + q.Shutdown() + <-disconnectedCh +} + // Checks that CPU are correctly added using device_add func TestQMPCPUDeviceAdd(t *testing.T) { drivers := []string{"host-x86_64-cpu", "host-s390x-cpu", "host-powerpc64-cpu"}