From f27d47542a1872a56b5581a41adb8ec2d8dfd97e Mon Sep 17 00:00:00 2001 From: Junming Liu Date: Tue, 12 Nov 2019 20:53:15 +0000 Subject: [PATCH] dm:gvt:adjust pci bar region with reserved bar regions ACRN-DM has reserved gvt bar regions. This patch ensures other pci devices' bar regions won't overlap with reserved bar regions. v6 -> v7: * rename some struct and func v5 -> v6: * remove outdated comment * add comments for code reading * code cleaning about gvt bar0 and bar2 size v4 -> v5: * rename adjust_bar_region and adjust_bar_region_by_gvt_bars * change adjust_bar_region_by_gvt_bars interface for code cleaning v3 -> v4: * add static struct gvt_region instead of definition or pointer array. v2 -> v3: * repalce pci_emul_alloc_bar with gvt_reserve_resource when allocate gvt bars * use register_bar to detect if gvt bars confilts with pci devices v1 -> v2: * don't limit the gvt bar type is MEM32 when deal with pci bar * add is_two_region_overlap func to detect if two regions overlap * add region array to store gvt bar0 and bar2 regions Tracked-On: projectacrn#4005 Signed-off-by: Junming Liu Reviewed-by: Zhao Yakui Reviewed-by: Liu XinYun Reviewed-by: Shuo A Liu Acked-by: Yu Wang --- devicemodel/hw/pci/core.c | 84 +++++++++++++++++++++++++++++++++++---- devicemodel/hw/pci/gvt.c | 40 +++++-------------- 2 files changed, 86 insertions(+), 38 deletions(-) diff --git a/devicemodel/hw/pci/core.c b/devicemodel/hw/pci/core.c index 40512af41..1a98ae437 100644 --- a/devicemodel/hw/pci/core.c +++ b/devicemodel/hw/pci/core.c @@ -166,6 +166,36 @@ void destory_mmio_rsvd_rgns(struct pci_vdev *vdev){ reserved_bar_regions[i].vdev = NULL; } +static bool +is_mmio_rgns_overlap(uint64_t x1, uint64_t x2, uint64_t y1, uint64_t y2) +{ + if(x1 <= y2 && y1 <= x2) + return true; + return false; +} + +/* reserved_bar_regions has sorted mmio_rsvd_rgns. + * iterate all mmio_rsvd_rgn in reserved_bar_regions, + * if [base, base + size - 1] with any mmio_rsvd_rgn, + * adjust base addr to ensure [base, base + size - 1] + * won't overlap with reserved mmio_rsvd_rgn + */ +static void +adjust_bar_region(uint64_t *base, uint64_t size, int bar_type) +{ + int i; + + for(i = 0; i < REGION_NUMS; i++){ + if(!reserved_bar_regions[i].vdev || + reserved_bar_regions[i].bar_type != bar_type) + continue; + if(is_mmio_rgns_overlap(reserved_bar_regions[i].start, + reserved_bar_regions[i].end, *base, *base + size -1)){ + *base = roundup2(reserved_bar_regions[i].end + 1, size); + } + } +} + static inline void CFGWRITE(struct pci_vdev *dev, int coff, uint32_t val, int bytes) { @@ -525,10 +555,9 @@ pci_emul_mem_handler(struct vmctx *ctx, int vcpu, int dir, uint64_t addr, return 0; } - static int pci_emul_alloc_resource(uint64_t *baseptr, uint64_t limit, uint64_t size, - uint64_t *addr) + uint64_t *addr, int bar_type) { uint64_t base; @@ -539,6 +568,14 @@ pci_emul_alloc_resource(uint64_t *baseptr, uint64_t limit, uint64_t size, base = roundup2(*baseptr, size); + /* TODO:Currently, we only reserve gvt mmio regions, + * so ignore PCIBAR_IO when adjust_bar_region. + * If other devices also use reserved bar regions later, + * need remove pcibar_type != PCIBAR_IO condition + */ + if(bar_type != PCIBAR_IO) + adjust_bar_region(&base, size, bar_type); + if (base + size <= limit) { *addr = base; *baseptr = base + size; @@ -622,10 +659,10 @@ unregister_bar(struct pci_vdev *dev, int idx) modify_bar_registration(dev, idx, 0); } -static void +static int register_bar(struct pci_vdev *dev, int idx) { - modify_bar_registration(dev, idx, 1); + return modify_bar_registration(dev, idx, 1); } /* Are we decoding i/o port accesses for the emulated pci device? */ @@ -699,12 +736,28 @@ update_bar_address(struct vmctx *ctx, struct pci_vdev *dev, uint64_t addr, dev->dev_ops->vdev_update_bar_map(ctx, dev, idx, orig_addr); } +static struct mmio_rsvd_rgn * +get_mmio_rsvd_rgn_by_vdev_idx(struct pci_vdev *pdi, int idx) +{ + int i; + + for(i = 0; i < REGION_NUMS; i++){ + if(reserved_bar_regions[i].vdev && + reserved_bar_regions[i].idx == idx && + reserved_bar_regions[i].vdev == pdi) + return &reserved_bar_regions[i]; + } + + return NULL; +} + int pci_emul_alloc_pbar(struct pci_vdev *pdi, int idx, uint64_t hostbase, enum pcibar_type type, uint64_t size) { int error; uint64_t *baseptr, limit, addr, mask, lobits, bar; + struct mmio_rsvd_rgn *region; if ((size & (size - 1)) != 0) size = 1UL << flsl(size); /* round up to a power of 2 */ @@ -773,8 +826,12 @@ pci_emul_alloc_pbar(struct pci_vdev *pdi, int idx, uint64_t hostbase, return -1; } - if (baseptr != NULL) { - error = pci_emul_alloc_resource(baseptr, limit, size, &addr); + region = get_mmio_rsvd_rgn_by_vdev_idx(pdi, idx); + if(region) + addr = region->start; + + if (baseptr != NULL && !region) { + error = pci_emul_alloc_resource(baseptr, limit, size, &addr, type); if (error != 0) return error; } @@ -792,7 +849,20 @@ pci_emul_alloc_pbar(struct pci_vdev *pdi, int idx, uint64_t hostbase, pci_set_cfgdata32(pdi, PCIR_BAR(idx + 1), bar >> 32); } - register_bar(pdi, idx); + error = register_bar(pdi, idx); + + if(error != 0){ + /* FIXME: Currently, only gvt needs reserve regions. + * because gvt isn't firstly initialized, previous pci + * devices' bars may conflict with gvt bars. + * Use register_bar to detect this case, + * but this case rarely happen. + * If this case always happens, we need to + * change core.c code to ensure gvt firstly initialzed + */ + printf("%s failed to register_bar\n", pdi->name); + return error; + } return 0; } diff --git a/devicemodel/hw/pci/gvt.c b/devicemodel/hw/pci/gvt.c index 4fa08880e..166adff57 100644 --- a/devicemodel/hw/pci/gvt.c +++ b/devicemodel/hw/pci/gvt.c @@ -21,6 +21,8 @@ static int pci_gvt_debug; +static struct pci_vdev *gvt_dev; + #define DPRINTF(params) do { if (pci_gvt_debug) printf params; } while (0) #define WPRINTF(params) (printf params) @@ -89,8 +91,6 @@ gvt_init_config(struct pci_gvt *gvt) int ret; char name[PATH_MAX]; uint8_t cap_ptr = 0; - uint8_t aperture_size_reg; - uint16_t aperture_size = 256; char res_name[PATH_MAX]; char resource[512]; int res_fd; @@ -203,42 +203,15 @@ gvt_init_config(struct pci_gvt *gvt) /* processor graphics control register */ pci_set_cfgdata16(gvt->gvt_pi, 0x52, gvt->host_config[0x52]); - /* Alloc resource only and no need to register bar for gvt */ ret = pci_emul_alloc_bar(gvt->gvt_pi, 0, PCIBAR_MEM32, - 16 * 1024 * 1024); + bar0_end_addr - bar0_start_addr + 1); if (ret != 0) { pr_err("allocate gvt pci bar[0] failed\n"); return -1; } - /* same as host, but guest only use partition of it by ballon */ - aperture_size_reg = gvt->host_config[0x62]; - switch(aperture_size_reg & 0b1111){ - case 0b00000: - aperture_size = 128; - break; - case 0b00001: - aperture_size = 256; - break; - case 0b00011: - aperture_size = 512; - break; - case 0b00111: - aperture_size = 1024; - break; - case 0b01111: - aperture_size = 2048; - break; - case 0b11111: - aperture_size = 4096; - break; - default: - aperture_size = 256; - break; - } - ret = pci_emul_alloc_bar(gvt->gvt_pi, 2, PCIBAR_MEM32, - aperture_size * 1024 * 1024); + bar2_end_addr - bar2_start_addr + 1); if (ret != 0) { pr_err("allocate gvt pci bar[2] failed\n"); return -1; @@ -344,6 +317,8 @@ pci_gvt_init(struct vmctx *ctx, struct pci_vdev *pi, char *opts) gvt->gvt_pi = pi; guest_domid = ctx->vmid; + gvt_dev = pi; + ret = gvt_init_config(gvt); if (ret) @@ -354,6 +329,7 @@ pci_gvt_init(struct vmctx *ctx, struct pci_vdev *pi, char *opts) if(!ret) return ret; fail: + gvt_dev = NULL; ctx->gvt_enabled = false; perror("GVT: init failed\n"); free(gvt); @@ -372,8 +348,10 @@ pci_gvt_deinit(struct vmctx *ctx, struct pci_vdev *pi, char *opts) if (ret) WPRINTF(("GVT: %s: failed: errno=%d\n", __func__, ret)); + destory_mmio_rsvd_rgns(gvt_dev); free(gvt); pi->arg = NULL; + gvt_dev = NULL; } }