diff --git a/devicemodel/core/vmmapi.c b/devicemodel/core/vmmapi.c index 0a7b912ec..167654e66 100644 --- a/devicemodel/core/vmmapi.c +++ b/devicemodel/core/vmmapi.c @@ -128,6 +128,7 @@ vm_create(const char *name, uint64_t req_buf, int *vcpu_num) /* Pass uuid as parameter of create vm*/ uuid_copy(create_vm.uuid, vm_uuid); + ctx->gvt_enabled = false; ctx->fd = devfd; ctx->lowmem_limit = 2 * GB; ctx->highmem_gpa_base = PCI_EMUL_MEMLIMIT64; diff --git a/devicemodel/hw/pci/core.c b/devicemodel/hw/pci/core.c index e81a35a01..40512af41 100644 --- a/devicemodel/hw/pci/core.c +++ b/devicemodel/hw/pci/core.c @@ -91,6 +91,8 @@ static uint64_t pci_emul_membase64; extern bool skip_pci_mem64bar_workaround; +struct mmio_rsvd_rgn reserved_bar_regions[REGION_NUMS]; + #define PCI_EMUL_IOBASE 0x2000 #define PCI_EMUL_IOLIMIT 0x10000 @@ -106,6 +108,64 @@ static void pci_cfgrw(struct vmctx *ctx, int vcpu, int in, int bus, int slot, int func, int coff, int bytes, uint32_t *val); static void pci_emul_free_msixcap(struct pci_vdev *pdi); +int compare_mmio_rgns(const void *data1, const void *data2) +{ + struct mmio_rsvd_rgn *rng1, *rng2; + + rng1 = (struct mmio_rsvd_rgn*)data1; + rng2 = (struct mmio_rsvd_rgn*)data2; + + if(!rng1->vdev) + return 1; + if(!rng2->vdev) + return -1; + return (rng1->start - rng2->start); +} + +/* FIXME: the new registered region may overlap with exist mmio regions + * whatever they are registered by dm or reserved. + * Due to we only has gvt-g to use this feature, + * this case rarely happen. + */ +int create_mmio_rsvd_rgn(uint64_t start, + uint64_t end, int idx, int bar_type, struct pci_vdev *vdev) +{ + int i; + + if(bar_type == PCIBAR_IO){ + perror("fail to create PCIBAR_IO bar_type\n"); + return -1; + } + + for(i = 0; i < REGION_NUMS; i++){ + if(reserved_bar_regions[i].vdev == NULL){ + reserved_bar_regions[i].start = start; + reserved_bar_regions[i].end = end; + reserved_bar_regions[i].idx = idx; + reserved_bar_regions[i].bar_type = bar_type; + reserved_bar_regions[i].vdev = vdev; + + /* sort reserved_bar_regions array by "start" member, + * if this mmio_rsvd_rgn is not used, put it in the last. + */ + qsort((void*)reserved_bar_regions, REGION_NUMS, + sizeof(reserved_bar_regions[0]), compare_mmio_rgns); + return 0; + } + } + + perror("reserved_bar_regions is overflow\n"); + return -1; +} + +void destory_mmio_rsvd_rgns(struct pci_vdev *vdev){ + int i; + + for(i = 0; i < REGION_NUMS; i++) + if(reserved_bar_regions[i].vdev == vdev) + reserved_bar_regions[i].vdev = NULL; +} + static inline void CFGWRITE(struct pci_vdev *dev, int coff, uint32_t val, int bytes) { diff --git a/devicemodel/hw/pci/gvt.c b/devicemodel/hw/pci/gvt.c index ce61225ca..4fa08880e 100644 --- a/devicemodel/hw/pci/gvt.c +++ b/devicemodel/hw/pci/gvt.c @@ -91,7 +91,69 @@ gvt_init_config(struct pci_gvt *gvt) uint8_t cap_ptr = 0; uint8_t aperture_size_reg; uint16_t aperture_size = 256; + char res_name[PATH_MAX]; + char resource[512]; + int res_fd; + uint64_t bar0_start_addr; + uint64_t bar0_end_addr; + uint64_t bar2_start_addr; + uint64_t bar2_end_addr; + char *next; + struct vmctx *ctx; + /* get physical gpu bars info from + * "/sys/bus/PCI/devices/0000\:00\:02.0/resource" + */ + snprintf(res_name, sizeof(res_name), + "/sys/bus/pci/devices/%04x:%02x:%02x.%x/resource", + gvt->addr.domain, gvt->addr.bus, gvt->addr.slot, + gvt->addr.function); + res_fd = open(res_name, O_RDONLY); + if (res_fd == -1) { + perror("gvt:open host pci resource failed\n"); + return -1; + } + + ret = pread(res_fd, resource, 512, 0); + + close(res_fd); + + if (ret < 512) { + perror("failed to read host device resource space\n"); + return -1; + } + + next = resource; + bar0_start_addr = strtoull(next, &next, 16); + bar0_end_addr = strtoull(next, &next, 16); + + /* bar0 and bar2 have some distance, need pass the distance */ + next = next + 80; + bar2_start_addr = strtoull(next, &next, 16); + bar2_end_addr = strtoull(next, &next, 16); + + ctx = gvt->gvt_pi->vmctx; + if(bar0_start_addr < ctx->lowmem_limit + || bar2_start_addr < ctx->lowmem_limit + || bar0_end_addr > PCI_EMUL_ECFG_BASE + || bar2_end_addr > PCI_EMUL_ECFG_BASE){ + perror("gvt pci bases are out of range\n"); + return -1; + } + + ctx->gvt_enabled = true; + + /* In GVT-g design, it only use pci bar0 and bar2, + * So we need reserve bar0 region and bar2 region only + */ + ret = create_mmio_rsvd_rgn(bar0_start_addr, + bar0_end_addr, 0, PCIBAR_MEM32, gvt->gvt_pi); + if(ret != 0) + return -1; + ret = create_mmio_rsvd_rgn(bar2_start_addr, + bar2_end_addr, 2, PCIBAR_MEM32, gvt->gvt_pi); + if(ret != 0) + return -1; snprintf(name, sizeof(name), "/sys/bus/pci/devices/%04x:%02x:%02x.%x/config", gvt->addr.domain, gvt->addr.bus, gvt->addr.slot, @@ -292,6 +354,7 @@ pci_gvt_init(struct vmctx *ctx, struct pci_vdev *pi, char *opts) if(!ret) return ret; fail: + ctx->gvt_enabled = false; perror("GVT: init failed\n"); free(gvt); return -1; diff --git a/devicemodel/include/pci_core.h b/devicemodel/include/pci_core.h index ab066e175..c7b997779 100644 --- a/devicemodel/include/pci_core.h +++ b/devicemodel/include/pci_core.h @@ -45,6 +45,11 @@ #define PCI_EMUL_MEMBASE64 0x100000000UL /* 4GB */ #define PCI_EMUL_MEMLIMIT64 0x140000000UL /* 5GB */ +/* Currently,only gvt need reserved bar regions, + * so just hardcode REGION_NUMS=5 here + */ +#define REGION_NUMS 5 + struct vmctx; struct pci_vdev; struct memory_region; @@ -243,6 +248,20 @@ struct pciecap { } __attribute__((packed)); static_assert(sizeof(struct pciecap) == 60, "compile-time assertion failed"); +struct mmio_rsvd_rgn { + uint64_t start; + uint64_t end; + int idx; + int bar_type; + /* if vdev=NULL, it also indicates this mmio_rsvd_rgn is not used */ + struct pci_vdev *vdev; +}; + +extern struct mmio_rsvd_rgn reserved_bar_regions[REGION_NUMS]; +int create_mmio_rsvd_rgn(uint64_t start, + uint64_t end, int idx, int bar_type, struct pci_vdev *vdev); +void destory_mmio_rsvd_rgns(struct pci_vdev *vdev); + typedef void (*pci_lintr_cb)(int b, int s, int pin, int pirq_pin, int ioapic_irq, void *arg); diff --git a/devicemodel/include/vmmapi.h b/devicemodel/include/vmmapi.h index 1eeff5675..8c7187cba 100644 --- a/devicemodel/include/vmmapi.h +++ b/devicemodel/include/vmmapi.h @@ -67,6 +67,9 @@ struct vmctx { /* BSP state. guest loader needs to fill it */ struct acrn_set_vcpu_regs bsp_regs; + + /* if gvt-g is enabled for current VM */ + bool gvt_enabled; }; #define PROT_RW (PROT_READ | PROT_WRITE)