From e99ddf28c357840964c0e5dbaa1c776acb538c2f Mon Sep 17 00:00:00 2001 From: Li Fei1 Date: Fri, 13 Mar 2020 11:44:51 +0800 Subject: [PATCH] hv: vpci: handle the quirk part for pass through pci device cfg access in dm There're some PCI devices need special handler for vendor-specical feature or capability CFG access. The Intel GPU is one of them. In order to keep the ACRN-HV clean, we want to throw the qurik part of PCI CFG asccess to DM to handle. To achieve this, we implement per-device policy base on whether it needs quirk handler for a VM: each device could configure as "quirk pass through device" or not. For a "quirk pass through device", we will handle the general part in HV and the quirk part in DM. For a non "quirk pass through device", we will handle all the part in HV. Tracked-On: #4371 Signed-off-by: Li Fei1 --- devicemodel/hw/pci/passthrough.c | 44 +++++++++++++++++++-- devicemodel/include/public/vhm_ioctl_defs.h | 5 ++- hypervisor/dm/vpci/vpci.c | 41 +++++++++---------- hypervisor/dm/vpci/vpci_priv.h | 5 +++ hypervisor/include/dm/vpci.h | 2 + hypervisor/include/hw/pci.h | 9 ----- hypervisor/include/public/acrn_hv_defs.h | 5 ++- 7 files changed, 75 insertions(+), 36 deletions(-) diff --git a/devicemodel/hw/pci/passthrough.c b/devicemodel/hw/pci/passthrough.c index bea9d469f..1aa322162 100644 --- a/devicemodel/hw/pci/passthrough.c +++ b/devicemodel/hw/pci/passthrough.c @@ -123,6 +123,28 @@ read_config(struct pci_device *phys_dev, long reg, int width) return temp; } +static int +write_config(struct pci_device *phys_dev, long reg, int width, uint32_t data) +{ + int temp = -1; + + switch (width) { + case 1: + temp = pci_device_cfg_write_u8(phys_dev, data, reg); + break; + case 2: + temp = pci_device_cfg_write_u16(phys_dev, data, reg); + break; + case 4: + temp = pci_device_cfg_write_u32(phys_dev, data, reg); + break; + default: + warnx("%s: invalid reg width", __func__); + } + + return temp; +} + static int cfginit_cap(struct vmctx *ctx, struct passthru_dev *ptdev) { @@ -515,8 +537,10 @@ passthru_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts) /* initialize the EPT mapping for passthrough GPU opregion */ vm_map_ptdev_mmio(ctx, 0, 2, 0, GPU_OPREGION_GPA, GPU_OPREGION_SIZE, opregion_start_hpa); - pcidev.rsvd2[0] = GPU_GSM_GPA | (gsm_phys & ~PCIM_BDSM_GSM_MASK) ; - pcidev.rsvd2[1] = GPU_OPREGION_GPA | (opregion_phys & ~PCIM_ASLS_OPREGION_MASK); + pci_set_cfgdata32(dev, PCIR_BDSM, GPU_GSM_GPA | (gsm_phys & ~PCIM_BDSM_GSM_MASK)); + pci_set_cfgdata32(dev, PCIR_ASLS_CTL, GPU_OPREGION_GPA | (opregion_phys & ~PCIM_ASLS_OPREGION_MASK)); + + pcidev.type = QUIRK_PTDEV; } pcidev.virt_bdf = PCI_BDF(dev->bus, dev->slot, dev->func); @@ -628,13 +652,27 @@ static int passthru_cfgread(struct vmctx *ctx, int vcpu, struct pci_vdev *dev, int coff, int bytes, uint32_t *rv) { - return ~0; + struct passthru_dev *ptdev = dev->arg; + + if ((PCI_BDF(dev->bus, dev->slot, dev->func) == PCI_BDF_GPU) && + ((coff == PCIR_BDSM) || (coff == PCIR_ASLS_CTL))) + *rv = pci_get_cfgdata32(dev, coff); + else + *rv = read_config(ptdev->phys_dev, coff, bytes); + + return 0; } static int passthru_cfgwrite(struct vmctx *ctx, int vcpu, struct pci_vdev *dev, int coff, int bytes, uint32_t val) { + struct passthru_dev *ptdev = dev->arg; + + if (!((PCI_BDF(dev->bus, dev->slot, dev->func) == PCI_BDF_GPU) && + ((coff == PCIR_BDSM) || (coff == PCIR_ASLS_CTL)))) + write_config(ptdev->phys_dev, coff, bytes, val); + return 0; } diff --git a/devicemodel/include/public/vhm_ioctl_defs.h b/devicemodel/include/public/vhm_ioctl_defs.h index 6f99e1fb4..1e6571164 100644 --- a/devicemodel/include/public/vhm_ioctl_defs.h +++ b/devicemodel/include/public/vhm_ioctl_defs.h @@ -154,8 +154,9 @@ struct vm_memmap { * */ struct acrn_assign_pcidev { - /** reversed for externed compatibility */ - uint32_t rsvd1; +#define QUIRK_PTDEV (1 << 0) /* We will only handle general part in HV, others in DM */ + /** the type of the the pass-through PCI device */ + uint32_t type; /** virtual BDF# of the pass-through PCI device */ uint16_t virt_bdf; diff --git a/hypervisor/dm/vpci/vpci.c b/hypervisor/dm/vpci/vpci.c index ad47a1cd1..09f4a274c 100644 --- a/hypervisor/dm/vpci/vpci.c +++ b/hypervisor/dm/vpci/vpci.c @@ -82,11 +82,15 @@ static bool vpci_pio_cfgaddr_write(struct acrn_vcpu *vcpu, uint16_t addr, size_t cfg_addr->value = val & (~0x7f000003U); if (is_postlaunched_vm(vcpu->vm)) { + const struct pci_vdev *vdev; + vbdf.value = cfg_addr->bits.bdf; - /* For post-launched VM, ACRN will only handle PT device, all virtual PCI device + vdev = find_available_vdev(vpci, vbdf); + /* For post-launched VM, ACRN HV will only handle PT device, + * all virtual PCI device and QUIRK PT device * still need to deliver to ACRN DM to handle. */ - if (find_available_vdev(vpci, vbdf) == NULL) { + if ((vdev == NULL) || is_quirk_ptdev(vdev)) { ret = false; } } @@ -482,6 +486,8 @@ static void write_cfg_header(struct pci_vdev *vdev, static int32_t write_pt_dev_cfg(struct pci_vdev *vdev, uint32_t offset, uint32_t bytes, uint32_t val) { + int32_t ret = 0; + if (cfg_header_access(offset)) { write_cfg_header(vdev, offset, bytes, val); } else if (msicap_access(vdev, offset)) { @@ -491,20 +497,22 @@ static int32_t write_pt_dev_cfg(struct pci_vdev *vdev, uint32_t offset, } else if (sriovcap_access(vdev, offset)) { write_sriov_cap_reg(vdev, offset, bytes, val); } else { - /* For GVT-D, prevent stolen memory and opregion memory write */ - if (!(is_postlaunched_vm(vdev->vpci->vm) && is_gvtd(vdev->pdev->bdf) && - ((offset == PCIR_BDSM) || (offset == PCIR_ASLS_CTL)))) { + if (!is_quirk_ptdev(vdev)) { /* passthru to physical device */ pci_pdev_write_cfg(vdev->pdev->bdf, offset, bytes, val); + } else { + ret = -ENODEV; } } - return 0; + return ret; } static int32_t read_pt_dev_cfg(const struct pci_vdev *vdev, uint32_t offset, uint32_t bytes, uint32_t *val) { + int32_t ret = 0; + if (cfg_header_access(offset)) { read_cfg_header(vdev, offset, bytes, val); } else if (msicap_access(vdev, offset)) { @@ -514,17 +522,15 @@ static int32_t read_pt_dev_cfg(const struct pci_vdev *vdev, uint32_t offset, } else if (sriovcap_access(vdev, offset)) { read_sriov_cap_reg(vdev, offset, bytes, val); } else { - /* For GVT-D, just return GPA for stolen memory and opregion memory read. */ - if (is_postlaunched_vm(vdev->vpci->vm) && is_gvtd(vdev->pdev->bdf) && - ((offset == PCIR_BDSM) || (offset == PCIR_ASLS_CTL))) { - *val = pci_vdev_read_vcfg(vdev, offset, bytes); - } else { + if (!is_quirk_ptdev(vdev)) { /* passthru to physical device */ *val = pci_pdev_read_cfg(vdev->pdev->bdf, offset, bytes); + } else { + ret = -ENODEV; } } - return 0; + return ret; } static const struct pci_vdev_ops pci_pt_dev_ops = { @@ -546,7 +552,7 @@ static int32_t vpci_read_cfg(struct acrn_vpci *vpci, union pci_bdf bdf, spinlock_obtain(&vpci->lock); vdev = find_available_vdev(vpci, bdf); if (vdev != NULL) { - vdev->vdev_ops->read_vdev_cfg(vdev, offset, bytes, val); + ret = vdev->vdev_ops->read_vdev_cfg(vdev, offset, bytes, val); } else { if (is_postlaunched_vm(vpci->vm)) { ret = -ENODEV; @@ -568,7 +574,7 @@ static int32_t vpci_write_cfg(struct acrn_vpci *vpci, union pci_bdf bdf, spinlock_obtain(&vpci->lock); vdev = find_available_vdev(vpci, bdf); if (vdev != NULL) { - vdev->vdev_ops->write_vdev_cfg(vdev, offset, bytes, val); + ret = vdev->vdev_ops->write_vdev_cfg(vdev, offset, bytes, val); } else { if (!is_postlaunched_vm(vpci->vm)) { pr_acrnlog("%s %x:%x.%x not found! off: 0x%x, val: 0x%x\n", __func__, @@ -771,12 +777,7 @@ int32_t vpci_assign_pcidev(struct acrn_vm *tgt_vm, struct acrn_assign_pcidev *pc pci_vdev_write_vbar(vdev, idx, pcidev->bar[idx]); } - if (is_gvtd(bdf)) { - /* rsvd2[0U] for stolen memory GPA; rsvd2[1U] for opregion memory GPA */ - pci_vdev_write_vcfg(vdev, PCIR_BDSM, 4U, pcidev->rsvd2[0U]); - pci_vdev_write_vcfg(vdev, PCIR_ASLS_CTL, 4U, pcidev->rsvd2[1U]); - } - + vdev->flags |= pcidev->type; vdev->bdf.value = pcidev->virt_bdf; spinlock_release(&tgt_vm->vpci.lock); vdev_in_sos->new_owner = vdev; diff --git a/hypervisor/dm/vpci/vpci_priv.h b/hypervisor/dm/vpci/vpci_priv.h index 132dbb137..75ee2acb2 100644 --- a/hypervisor/dm/vpci/vpci_priv.h +++ b/hypervisor/dm/vpci/vpci_priv.h @@ -32,6 +32,11 @@ #include +static inline bool is_quirk_ptdev(const struct pci_vdev *vdev) +{ + return ((vdev->flags & QUIRK_PTDEV) != 0U); +} + static inline bool in_range(uint32_t value, uint32_t lower, uint32_t len) { return ((value >= lower) && (value < (lower + len))); diff --git a/hypervisor/include/dm/vpci.h b/hypervisor/include/dm/vpci.h index d8eb0e53f..6868af266 100644 --- a/hypervisor/include/dm/vpci.h +++ b/hypervisor/include/dm/vpci.h @@ -104,6 +104,8 @@ struct pci_vdev { union pci_cfgdata cfgdata; + uint32_t flags; + /* The bar info of the virtual PCI device. */ uint32_t nr_bars; /* 6 for normal device, 2 for bridge, 1 for cardbus */ struct pci_vbar vbars[PCI_BAR_COUNT]; diff --git a/hypervisor/include/hw/pci.h b/hypervisor/include/hw/pci.h index a200f6fa3..c594e77f8 100644 --- a/hypervisor/include/hw/pci.h +++ b/hypervisor/include/hw/pci.h @@ -186,10 +186,6 @@ #define HOST_BRIDGE_BDF 0U #define PCI_STD_NUM_BARS 6U -/* Graphics definitions */ -#define PCIR_BDSM 0x5CU /* BDSM graphics base data of stolen memory register */ -#define PCIR_ASLS_CTL 0xFCU /* Opregion start addr register */ - union pci_bdf { uint16_t value; struct { @@ -255,11 +251,6 @@ struct pci_cfg_ops { void (*pci_write_cfg)(union pci_bdf bdf, uint32_t offset, uint32_t bytes, uint32_t val); }; -static inline bool is_gvtd(union pci_bdf bdf) -{ - return (bdf.value == CONFIG_GPU_SBDF); -} - static inline uint32_t pci_bar_offset(uint32_t idx) { return PCIR_BARS + (idx << 2U); diff --git a/hypervisor/include/public/acrn_hv_defs.h b/hypervisor/include/public/acrn_hv_defs.h index 3ad054ff9..637ffd34b 100644 --- a/hypervisor/include/public/acrn_hv_defs.h +++ b/hypervisor/include/public/acrn_hv_defs.h @@ -272,8 +272,9 @@ struct hc_ptdev_irq { * the parameter for HC_ASSIGN_PCIDEV or HC_DEASSIGN_PCIDEV hypercall */ struct acrn_assign_pcidev { - /** reversed for externed compatibility */ - uint32_t rsvd1; +#define QUIRK_PTDEV (1U << 0) /* We will only handle general part in HV, others in DM */ + /** the type of the the pass-through PCI device */ + uint32_t type; /** virtual BDF# of the pass-through PCI device */ uint16_t virt_bdf;