dm:gvt:reserve gvt bar regions in ACRN-DM

The current design has the following problem:
gvt uses some pci bar regions,
but ACRN-DM isn't aware of these regions.
So ACRN-DM may allocate these regions for other pci devices,
which will result in other pci devices bar regions
overlap with gvt bar regions.

The new design is the following:
(1) ACRN-DM reads gvt bar regions
which are provided by physical gpu;
(2) ACRN-DM reserves gvt bar regions

v6 -> v7:
	* use array to store reserved bar regions
	* rename some struct and func

v5 -> v6:
	* rename enable_gvt to gvt_enabled
	* add a interface to reserve bar regions
	* reserve gvt bar regions

Tracked-On: projectacrn#4005

Signed-off-by: Junming Liu <junming.liu@intel.com>
Reviewed-by: Zhao Yakui <yakui.zhao@intel.com>
Reviewed-by: Liu XinYun <xinyun.liu@intel.com>
Reviewed-by: Shuo A Liu <shuo.a.liu@intel.com>
Acked-by: Yu Wang <yu1.wang@intel.com>

devicemodel/core/vmmapi.c

@@ -128,6 +128,7 @@ vm_create(const char *name, uint64_t req_buf, int *vcpu_num)
 	/* Pass uuid as parameter of create vm*/
 	uuid_copy(create_vm.uuid, vm_uuid);
 
+	ctx->gvt_enabled = false;
 	ctx->fd = devfd;
 	ctx->lowmem_limit = 2 * GB;
 	ctx->highmem_gpa_base = PCI_EMUL_MEMLIMIT64;

devicemodel/hw/pci/core.c

@@ -91,6 +91,8 @@ static uint64_t pci_emul_membase64;
 
 extern bool skip_pci_mem64bar_workaround;
 
+struct mmio_rsvd_rgn reserved_bar_regions[REGION_NUMS];
+
 #define	PCI_EMUL_IOBASE		0x2000
 #define	PCI_EMUL_IOLIMIT	0x10000
 
@@ -106,6 +108,64 @@ static void pci_cfgrw(struct vmctx *ctx, int vcpu, int in, int bus, int slot,
 		      int func, int coff, int bytes, uint32_t *val);
 static void pci_emul_free_msixcap(struct pci_vdev *pdi);
 
+int compare_mmio_rgns(const void *data1, const void *data2)
+{
+    struct mmio_rsvd_rgn *rng1, *rng2;
+
+    rng1 = (struct mmio_rsvd_rgn*)data1;
+    rng2 = (struct mmio_rsvd_rgn*)data2;
+
+    if(!rng1->vdev)
+            return 1;
+    if(!rng2->vdev)
+            return -1;
+    return (rng1->start - rng2->start);
+}
+
+/* FIXME: the new registered region may overlap with exist mmio regions
+ * whatever they are registered by dm or reserved.
+ * Due to we only has gvt-g to use this feature,
+ * this case rarely happen.
+ */
+int create_mmio_rsvd_rgn(uint64_t start,
+	    uint64_t end, int idx, int bar_type, struct pci_vdev *vdev)
+{
+	int i;
+
+	if(bar_type == PCIBAR_IO){
+		perror("fail to create PCIBAR_IO bar_type\n");
+		return -1;
+	}
+
+	for(i = 0; i < REGION_NUMS; i++){
+		if(reserved_bar_regions[i].vdev == NULL){
+			reserved_bar_regions[i].start = start;
+			reserved_bar_regions[i].end = end;
+			reserved_bar_regions[i].idx = idx;
+			reserved_bar_regions[i].bar_type = bar_type;
+			reserved_bar_regions[i].vdev = vdev;
+
+			/* sort reserved_bar_regions array by "start" member,
+			 * if this mmio_rsvd_rgn is not used, put it in the last.
+			 */
+			qsort((void*)reserved_bar_regions, REGION_NUMS,
+					sizeof(reserved_bar_regions[0]),  compare_mmio_rgns);
+			return 0;
+		}
+	}
+
+	perror("reserved_bar_regions is overflow\n");
+	return -1;
+}
+
+void destory_mmio_rsvd_rgns(struct pci_vdev *vdev){
+    int i;
+
+    for(i = 0; i < REGION_NUMS; i++)
+		if(reserved_bar_regions[i].vdev == vdev)
+			reserved_bar_regions[i].vdev = NULL;
+}
+
 static inline void
 CFGWRITE(struct pci_vdev *dev, int coff, uint32_t val, int bytes)
 {

devicemodel/hw/pci/gvt.c

@@ -91,7 +91,69 @@ gvt_init_config(struct pci_gvt *gvt)
 	uint8_t cap_ptr = 0;
 	uint8_t aperture_size_reg;
 	uint16_t aperture_size = 256;
+	char res_name[PATH_MAX];
+	char resource[512];
+	int res_fd;
+	uint64_t bar0_start_addr;
+	uint64_t bar0_end_addr;
+	uint64_t bar2_start_addr;
+	uint64_t bar2_end_addr;
+	char *next;
+	struct vmctx *ctx;
 
+	/* get physical gpu bars info from
+	 * "/sys/bus/PCI/devices/0000\:00\:02.0/resource"
+	 */
+	snprintf(res_name, sizeof(res_name),
+		"/sys/bus/pci/devices/%04x:%02x:%02x.%x/resource",
+		gvt->addr.domain, gvt->addr.bus, gvt->addr.slot,
+		gvt->addr.function);
+	res_fd = open(res_name, O_RDONLY);
+	if (res_fd == -1) {
+		perror("gvt:open host pci resource failed\n");
+		return -1;
+	}
+
+	ret = pread(res_fd, resource, 512, 0);
+
+	close(res_fd);
+
+	if (ret < 512) {
+		perror("failed to read host device resource space\n");
+		return -1;
+	}
+
+	next = resource;
+	bar0_start_addr = strtoull(next, &next, 16);
+	bar0_end_addr = strtoull(next, &next, 16);
+
+	/* bar0 and bar2 have some distance, need pass the distance */
+	next = next + 80;
+	bar2_start_addr = strtoull(next, &next, 16);
+	bar2_end_addr = strtoull(next, &next, 16);
+
+	ctx = gvt->gvt_pi->vmctx;
+	if(bar0_start_addr < ctx->lowmem_limit
+		|| bar2_start_addr < ctx->lowmem_limit
+		|| bar0_end_addr > PCI_EMUL_ECFG_BASE
+		|| bar2_end_addr > PCI_EMUL_ECFG_BASE){
+		perror("gvt pci bases are out of range\n");
+		return -1;
+	}
+
+	ctx->gvt_enabled = true;
+
+	/* In GVT-g design, it only use pci bar0 and bar2,
+	 * So we need reserve bar0 region and bar2 region only
+	 */
+	ret = create_mmio_rsvd_rgn(bar0_start_addr,
+				bar0_end_addr, 0, PCIBAR_MEM32, gvt->gvt_pi);
+	if(ret != 0)
+		return -1;
+	ret = create_mmio_rsvd_rgn(bar2_start_addr,
+				bar2_end_addr, 2, PCIBAR_MEM32, gvt->gvt_pi);
+	if(ret != 0)
+		return -1;
 	snprintf(name, sizeof(name),
 		"/sys/bus/pci/devices/%04x:%02x:%02x.%x/config",
 		gvt->addr.domain, gvt->addr.bus, gvt->addr.slot,
@@ -292,6 +354,7 @@ pci_gvt_init(struct vmctx *ctx, struct pci_vdev *pi, char *opts)
 	if(!ret)
 		return ret;
 fail:
+	ctx->gvt_enabled = false;
 	perror("GVT: init failed\n");
 	free(gvt);
 	return -1;

devicemodel/include/pci_core.h

@@ -45,6 +45,11 @@
 #define	PCI_EMUL_MEMBASE64	0x100000000UL	/* 4GB */
 #define	PCI_EMUL_MEMLIMIT64	0x140000000UL	/* 5GB */
 
+/* Currently,only gvt need reserved bar regions,
+ * so just hardcode REGION_NUMS=5 here
+ */
+#define REGION_NUMS 5
+
 struct vmctx;
 struct pci_vdev;
 struct memory_region;
@@ -243,6 +248,20 @@ struct pciecap {
 } __attribute__((packed));
 static_assert(sizeof(struct pciecap) == 60, "compile-time assertion failed");
 
+struct mmio_rsvd_rgn {
+	uint64_t start;
+	uint64_t end;
+	int idx;
+	int bar_type;
+	/* if vdev=NULL, it also indicates this mmio_rsvd_rgn is not used */
+	struct pci_vdev *vdev;
+};
+
+extern struct mmio_rsvd_rgn reserved_bar_regions[REGION_NUMS];
+int create_mmio_rsvd_rgn(uint64_t start,
+                uint64_t end, int idx, int bar_type, struct pci_vdev *vdev);
+void destory_mmio_rsvd_rgns(struct pci_vdev *vdev);
+
 typedef void (*pci_lintr_cb)(int b, int s, int pin, int pirq_pin,
 			     int ioapic_irq, void *arg);
 

devicemodel/include/vmmapi.h

@@ -67,6 +67,9 @@ struct vmctx {
 
 	/* BSP state. guest loader needs to fill it */
 	struct acrn_set_vcpu_regs bsp_regs;
+
+	/* if gvt-g is enabled for current VM */
+	bool gvt_enabled;
 };
 
 #define	PROT_RW		(PROT_READ | PROT_WRITE)