acrn-config: remap PCI vbar address to high memory

1. If the device configure space is not fixed by ACPI and its pbar address
is above 4G or lower than 2G, then remap the vbar address to high memory.
2. Due to GPU might have huge MMIO space which would result in
HV_RAM_SIZE overflow, skip its vBAR remapping for now when the size high
than 1GB.

Tracked-On: #4458
Signed-off-by: Wei Liu <weix.w.liu@intel.com>
Acked-by: Victor Sun <victor.sun@intel.com>

misc/acrn-config/board_config/pci_devices_h.py

@@ -14,6 +14,32 @@ PCI_END_HEADER = r"""
 #endif /* PCI_DEVICES_H_ */"""
 
 
+MEM_ALIGN = 2 * board_cfg_lib.SIZE_M
+#TODO: Support 64Bit Bar for huge MMIO than HUGE_MMIO_LIMIT
+SUPPORT_HUGE_HI_MMIO = False
+HUGE_MMIO_LIMIT = board_cfg_lib.SIZE_2G / 2
+HI_MMIO_OFFSET = 0
+
+class Bar_Mem:
+    def __init__(self):
+        self.addr = 0
+        self.remapped = False
+
+
+class Bar_Attr:
+    def __init__(self):
+        self.name = 0
+        self.remappable = True
+
+
+class Pci_Dev_Bar_Desc:
+    def __init__(self):
+        self.pci_dev_dic = {}
+        self.pci_bar_dic = {}
+
+PCI_DEV_BAR_DESC = Pci_Dev_Bar_Desc()
+
+
 def get_value_after_str(line, key):
     """ Get the value after cstate string """
     idx = 0
@@ -26,80 +52,119 @@ def get_value_after_str(line, key):
     return line_in_list[idx + 1]
 
 
+def check_bar_remappable(line):
+    #TODO: check device BAR remappable per ACPI table
+
+    return True
+
+
+def get_size(line):
+
+    # get size string from format, Region n: Memory at x ... [size=NK]
+    size_str = line.split()[-1].strip(']').split('=')[1]
+    if 'M' in size_str:
+        size = int(size_str.strip('M')) * board_cfg_lib.SIZE_M
+    elif 'K' in size_str:
+        size = int(size_str.strip('K')) * board_cfg_lib.SIZE_K
+    else:
+        size = int(size_str)
+
+    return size
+
+
+def remap_bar_addr_to_high(bar_addr, line):
+    """Generate vbar address"""
+    global HI_MMIO_OFFSET
+    cur_addr = board_cfg_lib.round_up(bar_addr, MEM_ALIGN)
+    size = get_size(line)
+    HI_MMIO_OFFSET = board_cfg_lib.round_up(cur_addr + size, MEM_ALIGN)
+    return cur_addr
+
+
 def parser_pci():
     """ Parse PCI lines """
     cur_bdf = 0
     prev_bdf = 0
     tmp_bar_dic = {}
-    pci_dev_dic = {}
-    pci_bar_dic = {}
-    above_4G_mmio = False
     bar_addr = bar_num = '0'
+    cal_sub_pci_name = []
 
     pci_lines = board_cfg_lib.get_info(
         board_cfg_lib.BOARD_INFO_FILE, "<PCI_DEVICE>", "</PCI_DEVICE>")
+
     for line in pci_lines:
-        # get pci bar information into pci_bar_dic
+        tmp_bar_mem = Bar_Mem()
+        # get pci bar information into PCI_DEV_BAR_DESC
         if "Region" in line and "Memory at" in line:
             #ignore memory region from SR-IOV capabilities
             if "size=" not in line:
                  continue
+
+            bar_addr = int(get_value_after_str(line, "at"), 16)
             bar_num = line.split()[1].strip(':')
-            bar_addr = get_value_after_str(line, "at")
+            if bar_addr >= board_cfg_lib.SIZE_4G or bar_addr < board_cfg_lib.SIZE_2G:
-            if int(bar_addr, 16) > 0xffffffff:
+                if not tmp_bar_attr.remappable:
-                above_4G_mmio = True
+                    continue
-            tmp_bar_dic[int(bar_num)] = hex(int(bar_addr, 16))
+                #TODO: Support 64Bit Bar for huge MMIO than HUGE_MMIO_LIMIT
+                if not SUPPORT_HUGE_HI_MMIO and get_size(line) >=  HUGE_MMIO_LIMIT:
+                    tmp_bar_attr.remappable = False
+                    PCI_DEV_BAR_DESC.pci_dev_dic[pci_bdf] = tmp_bar_attr
+                    continue
+
+                bar_addr = remap_bar_addr_to_high(HI_MMIO_OFFSET, line)
+                tmp_bar_mem.remapped = True
+
+            tmp_bar_mem.addr = hex(bar_addr)
+            tmp_bar_dic[int(bar_num)] = tmp_bar_mem
         else:
+            tmp_bar_attr = Bar_Attr()
             prev_bdf = cur_bdf
             pci_bdf = line.split()[0]
-            pci_sub_name = " ".join(line.split(':')[1].split()[1:])
+            tmp_bar_attr.name = " ".join(line.split(':')[1].split()[1:])
 
             # remove '[*]' in pci subname
-            if '[' in pci_sub_name:
+            if '[' in tmp_bar_attr.name:
-                pci_sub_name = pci_sub_name.rsplit('[', 1)[0]
+                tmp_bar_attr.name = tmp_bar_attr.name.rsplit('[', 1)[0]
 
-            pci_dev_dic[pci_bdf] = pci_sub_name
+            cal_sub_pci_name.append(tmp_bar_attr.name)
+            tmp_bar_attr.remappable = check_bar_remappable(line)
+            PCI_DEV_BAR_DESC.pci_dev_dic[pci_bdf] = tmp_bar_attr
             cur_bdf = pci_bdf
 
             if not prev_bdf:
                 prev_bdf = cur_bdf
 
             if tmp_bar_dic and cur_bdf != prev_bdf:
-                pci_bar_dic[prev_bdf] = tmp_bar_dic
+                PCI_DEV_BAR_DESC.pci_bar_dic[prev_bdf] = tmp_bar_dic
 
             # clear the tmp_bar_dic before store the next dic
             tmp_bar_dic = {}
 
-    if above_4G_mmio:
-        board_cfg_lib.print_yel("Currently ACRN does not support BAR size above 4G, please double check below possible items in BIOS:\n\
-        1. GPU Aperture size is less than 1GB;\n\
-        2. the device MMIO mapping region is below 4GB", warn=True)
-
     # output all the pci device list to pci_device.h
-    sub_name_count = collections.Counter(pci_dev_dic.values())
+    sub_name_count = collections.Counter(cal_sub_pci_name)
 
     if tmp_bar_dic:
-        pci_bar_dic[cur_bdf] = tmp_bar_dic
+        PCI_DEV_BAR_DESC.pci_bar_dic[cur_bdf] = tmp_bar_dic
 
-    return (pci_dev_dic, pci_bar_dic, sub_name_count)
+    return sub_name_count
 
 
-def write_pbdf(i_cnt, bdf, subname, config):
+def write_pbdf(i_cnt, bdf, bar_attr, config):
     """
     Parser and generate pbdf
-    :param i_cnt: the number of pci devices have the same PCI subname
+    :param i_cnt: the number of pci devices have the same PCI sub class name
     :param bdf: it is a string what contains BDF
-    :param subname: it is a string belong to PIC subname
+    :param bar_attr: it is a class, contains PIC bar attribute
     :param config: it is a file pointer of pci information for writing to
     """
     # if there is only one host bridge, then will discard the index of suffix
-    if i_cnt == 0 and subname.upper() == "HOST BRIDGE":
+    if i_cnt == 0 and bar_attr.name.upper() == "HOST BRIDGE":
-        tmp_sub_name = "_".join(subname.split()).upper()
+        tmp_sub_name = "_".join(bar_attr.name.split()).upper()
     else:
-        if '-' in subname:
+        if '-' in bar_attr.name:
-            tmp_sub_name = board_cfg_lib.undline_name(subname) + "_" + str(i_cnt)
+            tmp_sub_name = board_cfg_lib.undline_name(bar_attr.name) + "_" + str(i_cnt)
         else:
-            tmp_sub_name = "_".join(subname.split()).upper() + "_" + str(i_cnt)
+            tmp_sub_name = "_".join(bar_attr.name.split()).upper() + "_" + str(i_cnt)
 
     bus = int(bdf.split(':')[0], 16)
     dev = int(bdf.split(':')[1].split('.')[0], 16)
@@ -108,34 +173,48 @@ def write_pbdf(i_cnt, bdf, subname, config):
     print("        .pbdf.bits = {{.b = 0x{:02X}U, .d = 0x{:02X}U, .f = 0x{:02X}U}}".format(
         bus, dev, fun), end="", file=config)
 
+    if not bar_attr.remappable:
+        align = ' ' * 48
+        print("\n{}/* TODO: add {} 64bit BAR support */".format(align, tmp_sub_name), file=config)
+        return
 
-def write_vbar(bdf, pci_bar_dic, config):
+
+def write_vbar(i_cnt, bdf, pci_bar_dic, bar_attr, config):
     """
     Parser and generate vbar
+    :param i_cnt: the number of pci devices have the same PCI sub class name
     :param bdf: it is a string what contains BDF
     :param pci_bar_dic: it is a dictionary of pci vbar for those BDF
+    :param bar_attr: it is a class, contains PIC bar attribute
     :param config: it is a file pointer of pci information for writing to
     """
     tail = 0
     align = ' ' * 48
+    ptdev_mmio_str = ''
+
+    tmp_sub_name = board_cfg_lib.undline_name(bar_attr.name) + "_" + str(i_cnt)
     if bdf in pci_bar_dic.keys():
         bar_list = list(pci_bar_dic[bdf].keys())
         bar_len = len(bar_list)
         bar_num = 0
         for bar_i in bar_list:
+            if not bar_attr.remappable:
+                return
+
             if tail == 0:
                 print(", \\", file=config)
                 tail += 1
             bar_num += 1
-            bar_val = pci_bar_dic[bdf][bar_i]
+            bar_val = pci_bar_dic[bdf][bar_i].addr
+            if pci_bar_dic[bdf][bar_i].remapped:
+                ptdev_mmio_str = 'PTDEV_HI_MMIO_START + '
 
             if bar_num == bar_len:
-                print("{}.vbar_base[{}] = {}UL".format(align, bar_i, bar_val), file=config)
+                print("{}.vbar_base[{}] = {}{}UL".format(align, bar_i, ptdev_mmio_str, bar_val), file=config)
             else:
-                print("{}.vbar_base[{}] = {}UL, \\".format(
+                print("{}.vbar_base[{}] = {}{}UL, \\".format(
-                    align, bar_i, bar_val), file=config)
+                    align, bar_i, ptdev_mmio_str, bar_val), file=config)
 
-        # print("", file=config)
     else:
         print("", file=config)
 
@@ -145,7 +224,6 @@ def generate_file(config):
     Get PCI device and generate pci_devices.h
     :param config: it is a file pointer of pci information for writing to
     """
-
     # write the license into pci
     print("{0}".format(board_cfg_lib.HEADER_LICENSE), file=config)
 
@@ -155,20 +233,22 @@ def generate_file(config):
     # write the header into pci
     print("{0}".format(PCI_HEADER), file=config)
 
-    (pci_dev_dic, pci_bar_dic, sub_name_count) = parser_pci()
+    sub_name_count = parser_pci()
 
+    print("#define %-32s" % "PTDEV_HI_MMIO_SIZE", "       {}UL".format(hex(HI_MMIO_OFFSET)), file=config)
 
     compared_bdf = []
     for cnt_sub_name in sub_name_count.keys():
         i_cnt = 0
-        for bdf, subname in pci_dev_dic.items():
+        for bdf, bar_attr in PCI_DEV_BAR_DESC.pci_dev_dic.items():
-            if cnt_sub_name == subname and bdf not in compared_bdf:
+            if cnt_sub_name == bar_attr.name and bdf not in compared_bdf:
                 compared_bdf.append(bdf)
             else:
                 continue
 
-            write_pbdf(i_cnt, bdf, subname, config)
+            print("",file=config)
-            write_vbar(bdf, pci_bar_dic, config)
+            write_pbdf(i_cnt, bdf, bar_attr, config)
+            write_vbar(i_cnt, bdf, PCI_DEV_BAR_DESC.pci_bar_dic, bar_attr, config)
 
             i_cnt += 1
 

misc/acrn-config/library/board_cfg_lib.py

@@ -39,6 +39,11 @@ KNOWN_HIDDEN_PDEVS_BOARD_DB = {
     'apl-up2':['00:0d:0'],
 }
 
+SIZE_K = common.SIZE_K
+SIZE_M = common.SIZE_M
+SIZE_2G = common.SIZE_2G
+SIZE_4G = common.SIZE_4G
+
 
 def prepare(check_git):
     """ check environment """
@@ -433,3 +438,8 @@ def undline_name(name):
     :return: name_str which contain'_'
     """
     return common.undline_name(name)
+
+
+def round_up(addr, mem_align):
+    """Keep memory align"""
+    return common.round_up(addr, mem_align)

misc/acrn-config/library/common.py

@@ -25,6 +25,11 @@ START_HPA_SIZE_LIST = ['0x20000000', '0x40000000', '0x80000000', 'CONFIG_SOS_RAM
 
 MULTI_ITEM = ["guest_flag", "pcpu_id", "input", "block", "network"]
 
+SIZE_K = 1024
+SIZE_M = SIZE_K * 1024
+SIZE_2G = 2 * SIZE_M * SIZE_K
+SIZE_4G = 2 * SIZE_2G
+
 
 class MultiItem():
 
@@ -681,3 +686,8 @@ def get_vuart_info_id(config_file, idx):
             vm_id += 1
 
     return tmp_tag
+
+
+def round_up(addr, mem_align):
+    """Keep memory align"""
+    return ((addr + (mem_align - 1)) & (~(mem_align - 1)))