From 89908bf5106ed0e7253a13bf81a85df517ad8f3c Mon Sep 17 00:00:00 2001 From: Junming Liu Date: Tue, 12 Nov 2019 21:19:39 +0000 Subject: [PATCH] dm:gvt:update gvt bars before other pci devices write bar address The current design has the following problem: uos kernel may update gvt bars' regions, but ACRN-DM doesn't know this update. ACRN-DM only know out-of-date gvt bar regions, For simplicity, mark these bar regions as OOD bar regions. uos kernel may allocate OOD bar regions for other pci devices, which will result in ACRN-DM bar regions inconsistency with uos kernel. The new design is the following: When other pci device update bar regions (1) ACRN-DM updates gvt bars' regions provided by a system file. (2) ACRN-DM updates this pci device bar regions v5 -> v6: * add more comments v4 -> v5: * remove & for callback func assignment v3 -> v4: * compare gpu bar address to avoid unnecessary * unregistered/registered operation v2 -> v3: * call unregister_bar and register_bar when update gvt bars * update gvt reserved regions when update gvt bars Tracked-On: projectacrn#4005 Signed-off-by: Junming Liu Reviewed-by: Zhao Yakui Reviewed-by: Liu XinYun Reviewed-by: Shuo A Liu Acked-by: Yu Wang --- devicemodel/hw/pci/core.c | 57 +++++++++++++++++++---------- devicemodel/hw/pci/gvt.c | 65 ++++++++++++++++++++++++++++++++++ devicemodel/include/pci_core.h | 1 + devicemodel/include/vmmapi.h | 2 ++ 4 files changed, 106 insertions(+), 19 deletions(-) diff --git a/devicemodel/hw/pci/core.c b/devicemodel/hw/pci/core.c index 1a98ae437..bd7441297 100644 --- a/devicemodel/hw/pci/core.c +++ b/devicemodel/hw/pci/core.c @@ -710,6 +710,19 @@ update_bar_address(struct vmctx *ctx, struct pci_vdev *dev, uint64_t addr, if (decode) unregister_bar(dev, idx); + /* TODO:Currently, we only reserve gvt mmio regions, + * so ignore PCIBAR_IO when adjust_bar_region_with_reserved_bars. + * If other devices also use reserved bar regions later, + * need remove pcibar_type != PCIBAR_IO condition + */ + if(type != PCIBAR_IO && ctx->gvt_enabled) + /* uos kernel may update gvt bars' value, + * but ACRN-DM doesn't know this update. + * When other pci devices write bar address, + * ACRN-DM need update vgpu bars' info. + */ + ctx->update_gvt_bar(ctx); + switch (type) { case PCIBAR_IO: case PCIBAR_MEM32: @@ -867,30 +880,36 @@ pci_emul_alloc_pbar(struct pci_vdev *pdi, int idx, uint64_t hostbase, return 0; } +void +pci_emul_free_bar(struct pci_vdev *pdi, int idx) +{ + bool enabled; + + if ((pdi->bar[idx].type != PCIBAR_NONE) && + (pdi->bar[idx].type != PCIBAR_MEMHI64)){ + /* + * Check whether the bar is enabled or not, + * if it is disabled then it should have been + * unregistered in pci_emul_cmdsts_write. + */ + if (pdi->bar[idx].type == PCIBAR_IO) + enabled = porten(pdi); + else + enabled = memen(pdi); + + if (enabled) + unregister_bar(pdi, idx); + pdi->bar[idx].type = PCIBAR_NONE; + } +} + void pci_emul_free_bars(struct pci_vdev *pdi) { int i; - bool enabled; - for (i = 0; i < PCI_BARMAX; i++) { - if ((pdi->bar[i].type != PCIBAR_NONE) && - (pdi->bar[i].type != PCIBAR_MEMHI64)){ - /* - * Check whether the bar is enabled or not, - * if it is disabled then it should have been - * unregistered in pci_emul_cmdsts_write. - */ - if (pdi->bar[i].type == PCIBAR_IO) - enabled = porten(pdi); - else - enabled = memen(pdi); - - if (enabled) - unregister_bar(pdi, i); - pdi->bar[i].type = PCIBAR_NONE; - } - } + for (i = 0; i < PCI_BARMAX; i++) + pci_emul_free_bar(pdi, i); } #define CAP_START_OFFSET 0x40 diff --git a/devicemodel/hw/pci/gvt.c b/devicemodel/hw/pci/gvt.c index 166adff57..90449a30f 100644 --- a/devicemodel/hw/pci/gvt.c +++ b/devicemodel/hw/pci/gvt.c @@ -85,6 +85,70 @@ pci_gvt_read(struct vmctx *ctx, int vcpu, struct pci_vdev *pi, return 0; } +void +update_gvt_bar(struct vmctx *ctx) +{ + char bar_path[PATH_MAX]; + int bar_fd; + int ret; + char resource[76]; + char *next; + uint64_t bar0_start_addr, bar0_end_addr, bar2_start_addr, bar2_end_addr; + int i; + + /* "/sys/kernel/gvt/vmx/vgpu_bar_info" exposes vgpu bar regions. */ + snprintf(bar_path, sizeof(bar_path), + "/sys/kernel/gvt/vm%d/vgpu_bar_info", + ctx->vmid); + + if(access(bar_path, F_OK) == -1) + return; + + bar_fd = open(bar_path, O_RDONLY); + if(bar_fd == -1){ + perror("failed to open sys bar info\n"); + return; + } + + ret = pread(bar_fd, resource, 76, 0); + + close(bar_fd); + + if (ret < 76) { + perror("failed to read sys bar info\n"); + return; + } + + next = resource; + bar0_start_addr = strtoull(next, &next, 16); + bar0_end_addr = strtoull(next, &next, 16) + bar0_start_addr -1; + bar2_start_addr = strtoull(next, &next, 16); + bar2_end_addr = strtoull(next, &next, 16) + bar2_start_addr -1; + + for(i = 0; i < REGION_NUMS; i++){ + if(reserved_bar_regions[i].vdev && + reserved_bar_regions[i].vdev == gvt_dev){ + pci_emul_free_bar(gvt_dev, reserved_bar_regions[i].idx); + } + } + + destory_mmio_rsvd_rgns(gvt_dev); + + ret = create_mmio_rsvd_rgn(bar0_start_addr, + bar0_end_addr, 0, PCIBAR_MEM32, gvt_dev); + if(ret != 0) + return; + ret = create_mmio_rsvd_rgn(bar2_start_addr, + bar2_end_addr, 2, PCIBAR_MEM32, gvt_dev); + if(ret != 0) + return; + + pci_emul_alloc_bar(gvt_dev, 0, PCIBAR_MEM32, + bar0_end_addr - bar0_start_addr + 1); + pci_emul_alloc_bar(gvt_dev, 2, PCIBAR_MEM32, + bar2_end_addr - bar2_start_addr + 1); +} + static int gvt_init_config(struct pci_gvt *gvt) { @@ -142,6 +206,7 @@ gvt_init_config(struct pci_gvt *gvt) } ctx->gvt_enabled = true; + ctx->update_gvt_bar = &update_gvt_bar; /* In GVT-g design, it only use pci bar0 and bar2, * So we need reserve bar0 region and bar2 region only diff --git a/devicemodel/include/pci_core.h b/devicemodel/include/pci_core.h index c7b997779..69950dc01 100644 --- a/devicemodel/include/pci_core.h +++ b/devicemodel/include/pci_core.h @@ -277,6 +277,7 @@ int pci_emul_alloc_bar(struct pci_vdev *pdi, int idx, int pci_emul_alloc_pbar(struct pci_vdev *pdi, int idx, uint64_t hostbase, enum pcibar_type type, uint64_t size); +void pci_emul_free_bar(struct pci_vdev *pdi, int idx); void pci_emul_free_bars(struct pci_vdev *pdi); int pci_emul_add_capability(struct pci_vdev *dev, u_char *capdata, int caplen); diff --git a/devicemodel/include/vmmapi.h b/devicemodel/include/vmmapi.h index 8c7187cba..ab42358a1 100644 --- a/devicemodel/include/vmmapi.h +++ b/devicemodel/include/vmmapi.h @@ -70,6 +70,8 @@ struct vmctx { /* if gvt-g is enabled for current VM */ bool gvt_enabled; + + void (*update_gvt_bar)(struct vmctx *ctx); }; #define PROT_RW (PROT_READ | PROT_WRITE)