qemu: Add pcie-root-port device support.

qemu/qemu.go

@@ -106,6 +106,9 @@ const (
 
 	// VirtioBlockCCW is the CCW block device driver
 	VirtioBlockCCW DeviceDriver = "virtio-blk-ccw"
+
+	// PCIeRootPort is a PCIe Root Port, the PCIe device should be hotplugged to this port.
+	PCIeRootPort DeviceDriver = "pcie-root-port"
 )
 
 // disableModern returns the parameters with the disable-modern option.
@@ -898,6 +901,102 @@ func (vhostuserDev VhostUserDevice) QemuParams(config *Config) []string {
 	return qemuParams
 }
 
+// PCIeRootPortDevice represents a memory balloon device.
+type PCIeRootPortDevice struct {
+	ID string // format: rp{n}, n>=0
+
+	Bus     string // default is pcie.0
+	Chassis string // (slot, chassis) pair is mandatory and must be unique for each pcie-root-port, >=0, default is 0x00
+	Slot    string // >=0, default is 0x00
+
+	Multifunction bool   // true => "on", false => "off", default is off
+	Addr          string // >=0, default is 0x00
+
+	// The PCIE-PCI bridge can be hot-plugged only into pcie-root-port that has 'bus-reserve' property value to
+	// provide secondary bus for the hot-plugged bridge.
+	BusReserve    string
+	Pref64Reserve string // reserve prefetched MMIO aperture, 64-bit
+	Pref32Reserve string // reserve prefetched MMIO aperture, 32-bit
+	MemReserve    string // reserve non-prefetched MMIO aperture, 32-bit *only*
+	IOReserve     string // IO reservation
+
+	ROMFile string // ROMFile specifies the ROM file being used for this device.
+}
+
+// QemuParams returns the qemu parameters built out of the PCIeRootPortDevice.
+func (b PCIeRootPortDevice) QemuParams(_ *Config) []string {
+	var qemuParams []string
+	var deviceParams []string
+	driver := PCIeRootPort
+
+	deviceParams = append(deviceParams, fmt.Sprintf("%s,id=%s", driver, b.ID))
+
+	if b.Bus == "" {
+		b.Bus = "pcie.0"
+	}
+	deviceParams = append(deviceParams, fmt.Sprintf("bus=%s", b.Bus))
+
+	if b.Chassis == "" {
+		b.Chassis = "0x00"
+	}
+	deviceParams = append(deviceParams, fmt.Sprintf("chassis=%s", b.Chassis))
+
+	if b.Slot == "" {
+		b.Slot = "0x00"
+	}
+	deviceParams = append(deviceParams, fmt.Sprintf("slot=%s", b.Slot))
+
+	multifunction := "off"
+	if b.Multifunction {
+		multifunction = "on"
+		if b.Addr == "" {
+			b.Addr = "0x00"
+		}
+		deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", b.Addr))
+	}
+	deviceParams = append(deviceParams, fmt.Sprintf("multifunction=%v", multifunction))
+
+	if b.BusReserve != "" {
+		deviceParams = append(deviceParams, fmt.Sprintf("bus-reserve=%s", b.BusReserve))
+	}
+
+	if b.Pref64Reserve != "" {
+		deviceParams = append(deviceParams, fmt.Sprintf("pref64-reserve=%s", b.Pref64Reserve))
+	}
+
+	if b.Pref32Reserve != "" {
+		deviceParams = append(deviceParams, fmt.Sprintf("pref32-reserve=%s", b.Pref32Reserve))
+	}
+
+	if b.MemReserve != "" {
+		deviceParams = append(deviceParams, fmt.Sprintf("mem-reserve=%s", b.MemReserve))
+	}
+
+	if b.IOReserve != "" {
+		deviceParams = append(deviceParams, fmt.Sprintf("io-reserve=%s", b.IOReserve))
+	}
+
+	if isVirtioPCI[driver] && b.ROMFile != "" {
+		deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", b.ROMFile))
+	}
+
+	qemuParams = append(qemuParams, "-device")
+	qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
+	return qemuParams
+}
+
+// Valid returns true if the PCIeRootPortDevice structure is valid and complete.
+func (b PCIeRootPortDevice) Valid() bool {
+	// the "pref32-reserve" and "pref64-reserve" hints are mutually exclusive.
+	if b.Pref64Reserve != "" && b.Pref32Reserve != "" {
+		return false
+	}
+	if b.ID == "" {
+		return false
+	}
+	return true
+}
+
 // VFIODevice represents a qemu vfio device meant for direct access by guest OS.
 type VFIODevice struct {
 	// Bus-Device-Function of device
@@ -914,6 +1013,9 @@ type VFIODevice struct {
 
 	// DeviceID specifies device id
 	DeviceID string
+
+	// Bus specifies device bus
+	Bus string
 }
 
 // Valid returns true if the VFIODevice structure is valid and complete.
@@ -939,6 +1041,10 @@ func (vfioDev VFIODevice) QemuParams(config *Config) []string {
 		deviceParams = append(deviceParams, fmt.Sprintf(",romfile=%s", vfioDev.ROMFile))
 	}
 
+	if vfioDev.Bus != "" {
+		deviceParams = append(deviceParams, fmt.Sprintf(",bus=%s", vfioDev.Bus))
+	}
+
 	if isVirtioCCW[driver] {
 		deviceParams = append(deviceParams, fmt.Sprintf(",devno=%s", vfioDev.DevNo))
 	}

qemu/qemu_arch_base.go

@@ -59,6 +59,7 @@ var isVirtioPCI = map[DeviceDriver]bool{
 	VirtioScsi:          true,
 	PCIBridgeDriver:     true,
 	PCIePCIBridgeDriver: true,
+	PCIeRootPort:        true,
 }
 
 // isVirtioCCW is a dummy map to return always false on no-s390x arch

qemu/qemu_arch_base_test.go

@@ -28,6 +28,10 @@ var (
 	deviceVhostUserNetString       = "-chardev socket,id=char1,path=/tmp/nonexistentsocket.socket -netdev type=vhost-user,id=net1,chardev=char1,vhostforce -device virtio-net-pci,netdev=net1,mac=00:11:22:33:44:55,romfile=efi-virtio.rom"
 	deviceVSOCKString              = "-device vhost-vsock-pci,disable-modern=true,id=vhost-vsock-pci0,guest-cid=4,romfile=efi-virtio.rom"
 	deviceVFIOString               = "-device vfio-pci,host=02:10.0,x-pci-vendor-id=0x1234,x-pci-device-id=0x5678,romfile=efi-virtio.rom"
+	devicePCIeRootPortSimpleString = "-device pcie-root-port,id=rp1,bus=pcie.0,chassis=0x00,slot=0x00,multifunction=off"
+	devicePCIeRootPortFullString   = "-device pcie-root-port,id=rp2,bus=pcie.0,chassis=0x0,slot=0x1,addr=0x2,multifunction=on,bus-reserve=0x3,pref64-reserve=16G,mem-reserve=1G,io-reserve=512M,romfile=efi-virtio.rom"
+	deviceVFIOPCIeSimpleString     = "-device vfio-pci,host=02:00.0,romfile=,bus=rp0"
+	deviceVFIOPCIeFullString       = "-device vfio-pci,host=02:00.0,x-pci-vendor-id=0x10de,x-pci-device-id=0x15f8,romfile=efi-virtio.rom,bus=rp1"
 	deviceSCSIControllerStr        = "-device virtio-scsi-pci,id=foo,disable-modern=false,romfile=efi-virtio.rom"
 	deviceSCSIControllerBusAddrStr = "-device virtio-scsi-pci,id=foo,bus=pci.0,addr=00:04.0,disable-modern=true,iothread=iothread1,romfile=efi-virtio.rom"
 	deviceVhostUserSCSIString      = "-chardev socket,id=char1,path=/tmp/nonexistentsocket.socket -device vhost-user-scsi-pci,id=scsi1,chardev=char1,romfile=efi-virtio.rom"
@@ -95,3 +99,81 @@ func TestAppendVirtioBalloon(t *testing.T) {
 	testAppend(balloonDevice, deviceString+OnDeflateOnOMM+OnDisableModern, t)
 
 }
+
+func TestAppendDevicePCIeRootPort(t *testing.T) {
+	var pcieRootPortID string
+
+	// test empty ID
+	pcieRootPortDevice := PCIeRootPortDevice{}
+	if pcieRootPortDevice.Valid() {
+		t.Fatalf("failed to validdate empty ID")
+	}
+
+	// test pref64_reserve and pre64_reserve
+	pcieRootPortID = "rp0"
+	pcieRootPortDevice = PCIeRootPortDevice{
+		ID:            pcieRootPortID,
+		Pref64Reserve: "16G",
+		Pref32Reserve: "256M",
+	}
+	if pcieRootPortDevice.Valid() {
+		t.Fatalf("failed to validate pref32-reserve and pref64-reserve for %v", pcieRootPortID)
+	}
+
+	// default test
+	pcieRootPortID = "rp1"
+	pcieRootPortDevice = PCIeRootPortDevice{
+		ID: pcieRootPortID,
+	}
+	if !pcieRootPortDevice.Valid() {
+		t.Fatalf("failed to validate for %v", pcieRootPortID)
+	}
+	testAppend(pcieRootPortDevice, devicePCIeRootPortSimpleString, t)
+
+	// full test
+	pcieRootPortID = "rp2"
+	pcieRootPortDevice = PCIeRootPortDevice{
+		ID:            pcieRootPortID,
+		Multifunction: true,
+		Bus:           "pcie.0",
+		Chassis:       "0x0",
+		Slot:          "0x1",
+		Addr:          "0x2",
+		Pref64Reserve: "16G",
+		IOReserve:     "512M",
+		MemReserve:    "1G",
+		BusReserve:    "0x3",
+		ROMFile:       romfile,
+	}
+	if !pcieRootPortDevice.Valid() {
+		t.Fatalf("failed to validate for %v", pcieRootPortID)
+	}
+	testAppend(pcieRootPortDevice, devicePCIeRootPortFullString, t)
+}
+
+func TestAppendDeviceVFIOPCIe(t *testing.T) {
+	// default test
+	pcieRootPortID := "rp0"
+	vfioDevice := VFIODevice{
+		BDF: "02:00.0",
+		Bus: pcieRootPortID,
+	}
+	if isVirtioCCW[Vfio] {
+		vfioDevice.DevNo = DevNo
+	}
+	testAppend(vfioDevice, deviceVFIOPCIeSimpleString, t)
+
+	// full test
+	pcieRootPortID = "rp1"
+	vfioDevice = VFIODevice{
+		BDF:      "02:00.0",
+		Bus:      pcieRootPortID,
+		ROMFile:  romfile,
+		VendorID: "0x10de",
+		DeviceID: "0x15f8",
+	}
+	if isVirtioCCW[Vfio] {
+		vfioDevice.DevNo = DevNo
+	}
+	testAppend(vfioDevice, deviceVFIOPCIeFullString, t)
+}

qemu/qmp.go

@@ -1156,17 +1156,20 @@ func (q *QMP) ExecutePCIVhostUserDevAdd(ctx context.Context, driver, devID, char
 	return q.executeCommand(ctx, "device_add", args, nil)
 }
 
-// ExecuteVFIODeviceAdd adds a VFIO device to a QEMU instance
-// using the device_add command. devID is the id of the device to add.
-// Must be valid QMP identifier. bdf is the PCI bus-device-function
-// of the pci device.
-func (q *QMP) ExecuteVFIODeviceAdd(ctx context.Context, devID, bdf, romfile string) error {
+// ExecuteVFIODeviceAdd adds a VFIO device to a QEMU instance using the device_add command.
+// devID is the id of the device to add. Must be valid QMP identifier.
+// bdf is the PCI bus-device-function of the pci device.
+// bus is optional. When hot plugging a PCIe device, the bus can be the ID of the pcie-root-port.
+func (q *QMP) ExecuteVFIODeviceAdd(ctx context.Context, devID, bdf, bus, romfile string) error {
 	args := map[string]interface{}{
 		"id":      devID,
 		"driver":  Vfio,
 		"host":    bdf,
 		"romfile": romfile,
 	}
+	if bus != "" {
+		args["bus"] = bus
+	}
 	return q.executeCommand(ctx, "device_add", args, nil)
 }
 

qemu/qmp_test.go

@@ -1079,6 +1079,27 @@ func TestQMPPCIVFIOMediatedDeviceAdd(t *testing.T) {
 	<-disconnectedCh
 }
 
+func TestQMPPCIVFIOPCIeDeviceAdd(t *testing.T) {
+	connectedCh := make(chan *QMPVersion)
+	disconnectedCh := make(chan struct{})
+	buf := newQMPTestCommandBuffer(t)
+	buf.AddCommand("device_add", nil, "return", nil)
+	cfg := QMPConfig{Logger: qmpTestLogger{}}
+	q := startQMPLoop(buf, cfg, connectedCh, disconnectedCh)
+	checkVersion(t, connectedCh)
+	bdf := "04:00.0"
+	bus := "rp0"
+	addr := "0x1"
+	romfile := ""
+	devID := fmt.Sprintf("device_%s", volumeUUID)
+	err := q.ExecutePCIVFIODeviceAdd(context.Background(), devID, bdf, addr, bus, romfile)
+	if err != nil {
+		t.Fatalf("Unexpected error %v", err)
+	}
+	q.Shutdown()
+	<-disconnectedCh
+}
+
 // Checks that CPU are correctly added using device_add
 func TestQMPCPUDeviceAdd(t *testing.T) {
 	drivers := []string{"host-x86_64-cpu", "host-s390x-cpu", "host-powerpc64-cpu"}