HV: add vpci bridge operations support

add vpci bridge operations in hypervisor, to avoid SOS mis-operations
to affect other VM's PCI devices.

assumption: before hypervisor bootup, the physical pci-bridge shall be
configured correctly by BIOS or other bootloader; for ACS (Access
Control Service) capability, it is configured by BIOS to support the
devices under it to be isolated and allocated to different VMs.

to simplify the emulations of vpci bridge, set limitations as following:
  1. expose all configure space registers, but readonly
  2. BIST not support; by default is 0
  3. not support interrupt, including INTx and MSI.

TODO:
  1. configure tool can select whether a PCI bridge is emulated or pass
  through.

Open:
  1. SOS how to reset PCI device under the PCI bridge?

Tracked-On: #3381
Signed-off-by: Yin Fengwei <fengwei.yin@intel.com>
Signed-off-by: Minggui Cao <minggui.cao@intel.com>
Acked-by: Eddie Dong <eddie.dong@Intel.com>

hypervisor/Makefile

@@ -290,6 +290,7 @@ VP_DM_C_SRCS += dm/io_req.c
 VP_DM_C_SRCS += dm/vpci/vdev.c
 VP_DM_C_SRCS += dm/vpci/vpci.c
 VP_DM_C_SRCS += dm/vpci/vhostbridge.c
+VP_DM_C_SRCS += dm/vpci/vpci_bridge.c
 VP_DM_C_SRCS += dm/vpci/pci_pt.c
 VP_DM_C_SRCS += dm/vpci/vmsi.c
 VP_DM_C_SRCS += dm/vpci/vmsix.c

hypervisor/dm/vpci/vpci.c

@@ -494,7 +494,12 @@ static struct pci_vdev *vpci_init_vdev(struct acrn_vpci *vpci, struct acrn_vm_pc
 	if (dev_config->vdev_ops != NULL) {
 		vdev->vdev_ops = dev_config->vdev_ops;
 	} else {
-		vdev->vdev_ops = &pci_pt_dev_ops;
+		if (vdev->pdev->hdr_type == PCIM_HDRTYPE_BRIDGE) {
+			vdev->vdev_ops = &vpci_bridge_ops;
+		} else {
+			vdev->vdev_ops = &pci_pt_dev_ops;
+		}
+
 		ASSERT(dev_config->emu_type == PCI_DEV_TYPE_PTDEV,
 			"Only PCI_DEV_TYPE_PTDEV could not configure vdev_ops");
 		ASSERT(dev_config->pdev != NULL, "PCI PTDev is not present on platform!");

hypervisor/dm/vpci/vpci_bridge.c

@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2011 NetApp, Inc.
+ * Copyright (c) 2019 Intel Corporation
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+/*
+ * Emulate a PCI bridge: Intel Corporation Sunrise Point-LP (rev f1)
+ * Assumptions:
+ *   1. before hypervisor bootup, all PCI devices have been configured correctly
+ * by BIOS(boot loader). It's not expected service OS change the configure;
+ *   2. for ACS(Access Control Service) Capability in PCI bridge is enabled and configured
+ * by BIOS to support the devices under it isolated and allocated to different VMs.
+ *
+ * for this emulation of vpci bridge, limitations set as following:
+ *   1. all configure registers are readonly
+ *   2. BIST not support; by default is 0H
+ *   3. not support interrupt, including INTx and MSI.
+ *
+ * TODO:
+ *  1. configure tool can select whether a PCI bridge is emulated or pass through
+ *
+ * Open:
+ *   1. SOS how to reset PCI devices under the PCI bridge
+ */
+
+#include <vm.h>
+#include <errno.h>
+#include <logmsg.h>
+#include <pci.h>
+#include "vpci_priv.h"
+
+static void init_vpci_bridge(struct pci_vdev *vdev)
+{
+	uint32_t offset, val, capoff, msgctrl;
+
+	/* read PCI config space to virtual space */
+	for (offset = 0x00U; offset < 0x100U; offset += 4U) {
+		val = pci_pdev_read_cfg(vdev->pdev->bdf, offset, 4U);
+		pci_vdev_write_cfg_u32(vdev, offset, val);
+	}
+
+	/* emulated for type info */
+	pci_vdev_write_cfg_u16(vdev, PCIR_VENDOR, (uint16_t)0x8086U);
+	pci_vdev_write_cfg_u16(vdev, PCIR_DEVICE, (uint16_t)0x9d12U);
+
+	pci_vdev_write_cfg_u8(vdev, PCIR_REVID, (uint8_t)0xf1U);
+
+	pci_vdev_write_cfg_u8(vdev, PCIR_HDRTYPE, (uint8_t)(PCIM_HDRTYPE_BRIDGE | PCIM_MFDEV));
+	pci_vdev_write_cfg_u8(vdev, PCIR_CLASS, (uint8_t)PCIC_BRIDGE);
+	pci_vdev_write_cfg_u8(vdev, PCIR_SUBCLASS, (uint8_t)PCIS_BRIDGE_PCI);
+
+	/* for command regsiters, disable INTx */
+	val = pci_pdev_read_cfg(vdev->pdev->bdf, PCIR_COMMAND, 2U);
+	pci_vdev_write_cfg_u16(vdev, PCIR_COMMAND, (uint16_t)val | PCIM_CMD_INTxDIS);
+	pci_pdev_write_cfg(vdev->pdev->bdf, PCIR_COMMAND, 2U, (uint16_t)val | PCIM_CMD_INTxDIS);
+
+	/* disale MSI */
+	if (vdev->pdev->msi_capoff != 0x00UL) {
+		capoff = vdev->pdev->msi_capoff;
+		msgctrl = pci_vdev_read_cfg(vdev, capoff + PCIR_MSI_CTRL, 2U);
+
+		msgctrl &= ~PCIM_MSICTRL_MSI_ENABLE;
+		pci_pdev_write_cfg(vdev->pdev->bdf, capoff + PCIR_MSI_CTRL, 2U, msgctrl);
+		pci_vdev_write_cfg(vdev, capoff + PCIR_MSI_CTRL, 2U, msgctrl);
+	}
+}
+
+static void deinit_vpci_bridge(__unused struct pci_vdev *vdev)
+{
+}
+
+static int32_t vpci_bridge_read_cfg(const struct pci_vdev *vdev, uint32_t offset,
+	uint32_t bytes, uint32_t *val)
+{
+	if ((offset + bytes) <= 0x100U) {
+		*val = pci_vdev_read_cfg(vdev, offset, bytes);
+	} else {
+		/* just passthru read to physical device when read PCIE sapce > 0x100 */
+		*val = pci_pdev_read_cfg(vdev->pdev->bdf, offset, bytes);
+	}
+
+	return 0;
+}
+
+static int32_t vpci_bridge_write_cfg(__unused struct pci_vdev *vdev, __unused uint32_t offset,
+	__unused uint32_t bytes, __unused uint32_t val)
+{
+	return 0;
+}
+
+const struct pci_vdev_ops vpci_bridge_ops = {
+	.init_vdev         = init_vpci_bridge,
+	.deinit_vdev       = deinit_vpci_bridge,
+	.write_vdev_cfg    = vpci_bridge_write_cfg,
+	.read_vdev_cfg     = vpci_bridge_read_cfg,
+};

hypervisor/include/dm/vpci.h

@@ -129,6 +129,7 @@ struct acrn_vpci {
 };
 
 extern const struct pci_vdev_ops vhostbridge_ops;
+extern const struct pci_vdev_ops vpci_bridge_ops;
 void vpci_init(struct acrn_vm *vm);
 void vpci_cleanup(struct acrn_vm *vm);
 struct pci_vdev *pci_find_vdev(struct acrn_vpci *vpci, union pci_bdf vbdf);

hypervisor/include/hw/pci.h

@@ -122,6 +122,9 @@
 #define PCIC_BRIDGE           0x06U
 #define PCIS_BRIDGE_HOST      0x00U
 
+/* PCI device subclass */
+#define PCIS_BRIDGE_PCI       0x04U
+
 /* MSI-X definitions */
 #define PCIR_MSIX_CTRL        0x2U
 #define PCIR_MSIX_TABLE       0x4U