From 91b06a35ee4cef8def490ee223e9d8a5be9396df Mon Sep 17 00:00:00 2001 From: Wei Liu Date: Fri, 6 Mar 2020 11:04:46 +0800 Subject: [PATCH] acrn-config: remap PCI vbar address to high memory 1. If the device configure space is not fixed by ACPI and its pbar address is above 4G or lower than 2G, then remap the vbar address to high memory. 2. Due to GPU might have huge MMIO space which would result in HV_RAM_SIZE overflow, skip its vBAR remapping for now when the size high than 1GB. Tracked-On: #4458 Signed-off-by: Wei Liu Acked-by: Victor Sun --- .../acrn-config/board_config/pci_devices_h.py | 162 +++++++++++++----- misc/acrn-config/library/board_cfg_lib.py | 10 ++ misc/acrn-config/library/common.py | 10 ++ 3 files changed, 141 insertions(+), 41 deletions(-) diff --git a/misc/acrn-config/board_config/pci_devices_h.py b/misc/acrn-config/board_config/pci_devices_h.py index 5e3bcbb31..e02f6ad3f 100644 --- a/misc/acrn-config/board_config/pci_devices_h.py +++ b/misc/acrn-config/board_config/pci_devices_h.py @@ -14,6 +14,32 @@ PCI_END_HEADER = r""" #endif /* PCI_DEVICES_H_ */""" +MEM_ALIGN = 2 * board_cfg_lib.SIZE_M +#TODO: Support 64Bit Bar for huge MMIO than HUGE_MMIO_LIMIT +SUPPORT_HUGE_HI_MMIO = False +HUGE_MMIO_LIMIT = board_cfg_lib.SIZE_2G / 2 +HI_MMIO_OFFSET = 0 + +class Bar_Mem: + def __init__(self): + self.addr = 0 + self.remapped = False + + +class Bar_Attr: + def __init__(self): + self.name = 0 + self.remappable = True + + +class Pci_Dev_Bar_Desc: + def __init__(self): + self.pci_dev_dic = {} + self.pci_bar_dic = {} + +PCI_DEV_BAR_DESC = Pci_Dev_Bar_Desc() + + def get_value_after_str(line, key): """ Get the value after cstate string """ idx = 0 @@ -26,80 +52,119 @@ def get_value_after_str(line, key): return line_in_list[idx + 1] +def check_bar_remappable(line): + #TODO: check device BAR remappable per ACPI table + + return True + + +def get_size(line): + + # get size string from format, Region n: Memory at x ... [size=NK] + size_str = line.split()[-1].strip(']').split('=')[1] + if 'M' in size_str: + size = int(size_str.strip('M')) * board_cfg_lib.SIZE_M + elif 'K' in size_str: + size = int(size_str.strip('K')) * board_cfg_lib.SIZE_K + else: + size = int(size_str) + + return size + + +def remap_bar_addr_to_high(bar_addr, line): + """Generate vbar address""" + global HI_MMIO_OFFSET + cur_addr = board_cfg_lib.round_up(bar_addr, MEM_ALIGN) + size = get_size(line) + HI_MMIO_OFFSET = board_cfg_lib.round_up(cur_addr + size, MEM_ALIGN) + return cur_addr + + def parser_pci(): """ Parse PCI lines """ cur_bdf = 0 prev_bdf = 0 tmp_bar_dic = {} - pci_dev_dic = {} - pci_bar_dic = {} - above_4G_mmio = False bar_addr = bar_num = '0' + cal_sub_pci_name = [] pci_lines = board_cfg_lib.get_info( board_cfg_lib.BOARD_INFO_FILE, "", "") + for line in pci_lines: - # get pci bar information into pci_bar_dic + tmp_bar_mem = Bar_Mem() + # get pci bar information into PCI_DEV_BAR_DESC if "Region" in line and "Memory at" in line: #ignore memory region from SR-IOV capabilities if "size=" not in line: continue + + bar_addr = int(get_value_after_str(line, "at"), 16) bar_num = line.split()[1].strip(':') - bar_addr = get_value_after_str(line, "at") - if int(bar_addr, 16) > 0xffffffff: - above_4G_mmio = True - tmp_bar_dic[int(bar_num)] = hex(int(bar_addr, 16)) + if bar_addr >= board_cfg_lib.SIZE_4G or bar_addr < board_cfg_lib.SIZE_2G: + if not tmp_bar_attr.remappable: + continue + #TODO: Support 64Bit Bar for huge MMIO than HUGE_MMIO_LIMIT + if not SUPPORT_HUGE_HI_MMIO and get_size(line) >= HUGE_MMIO_LIMIT: + tmp_bar_attr.remappable = False + PCI_DEV_BAR_DESC.pci_dev_dic[pci_bdf] = tmp_bar_attr + continue + + bar_addr = remap_bar_addr_to_high(HI_MMIO_OFFSET, line) + tmp_bar_mem.remapped = True + + tmp_bar_mem.addr = hex(bar_addr) + tmp_bar_dic[int(bar_num)] = tmp_bar_mem else: + tmp_bar_attr = Bar_Attr() prev_bdf = cur_bdf pci_bdf = line.split()[0] - pci_sub_name = " ".join(line.split(':')[1].split()[1:]) + tmp_bar_attr.name = " ".join(line.split(':')[1].split()[1:]) # remove '[*]' in pci subname - if '[' in pci_sub_name: - pci_sub_name = pci_sub_name.rsplit('[', 1)[0] + if '[' in tmp_bar_attr.name: + tmp_bar_attr.name = tmp_bar_attr.name.rsplit('[', 1)[0] - pci_dev_dic[pci_bdf] = pci_sub_name + cal_sub_pci_name.append(tmp_bar_attr.name) + tmp_bar_attr.remappable = check_bar_remappable(line) + PCI_DEV_BAR_DESC.pci_dev_dic[pci_bdf] = tmp_bar_attr cur_bdf = pci_bdf if not prev_bdf: prev_bdf = cur_bdf if tmp_bar_dic and cur_bdf != prev_bdf: - pci_bar_dic[prev_bdf] = tmp_bar_dic + PCI_DEV_BAR_DESC.pci_bar_dic[prev_bdf] = tmp_bar_dic # clear the tmp_bar_dic before store the next dic tmp_bar_dic = {} - if above_4G_mmio: - board_cfg_lib.print_yel("Currently ACRN does not support BAR size above 4G, please double check below possible items in BIOS:\n\ - 1. GPU Aperture size is less than 1GB;\n\ - 2. the device MMIO mapping region is below 4GB", warn=True) - # output all the pci device list to pci_device.h - sub_name_count = collections.Counter(pci_dev_dic.values()) + sub_name_count = collections.Counter(cal_sub_pci_name) if tmp_bar_dic: - pci_bar_dic[cur_bdf] = tmp_bar_dic + PCI_DEV_BAR_DESC.pci_bar_dic[cur_bdf] = tmp_bar_dic - return (pci_dev_dic, pci_bar_dic, sub_name_count) + return sub_name_count -def write_pbdf(i_cnt, bdf, subname, config): +def write_pbdf(i_cnt, bdf, bar_attr, config): """ Parser and generate pbdf - :param i_cnt: the number of pci devices have the same PCI subname + :param i_cnt: the number of pci devices have the same PCI sub class name :param bdf: it is a string what contains BDF - :param subname: it is a string belong to PIC subname + :param bar_attr: it is a class, contains PIC bar attribute :param config: it is a file pointer of pci information for writing to """ # if there is only one host bridge, then will discard the index of suffix - if i_cnt == 0 and subname.upper() == "HOST BRIDGE": - tmp_sub_name = "_".join(subname.split()).upper() + if i_cnt == 0 and bar_attr.name.upper() == "HOST BRIDGE": + tmp_sub_name = "_".join(bar_attr.name.split()).upper() else: - if '-' in subname: - tmp_sub_name = board_cfg_lib.undline_name(subname) + "_" + str(i_cnt) + if '-' in bar_attr.name: + tmp_sub_name = board_cfg_lib.undline_name(bar_attr.name) + "_" + str(i_cnt) else: - tmp_sub_name = "_".join(subname.split()).upper() + "_" + str(i_cnt) + tmp_sub_name = "_".join(bar_attr.name.split()).upper() + "_" + str(i_cnt) bus = int(bdf.split(':')[0], 16) dev = int(bdf.split(':')[1].split('.')[0], 16) @@ -108,34 +173,48 @@ def write_pbdf(i_cnt, bdf, subname, config): print(" .pbdf.bits = {{.b = 0x{:02X}U, .d = 0x{:02X}U, .f = 0x{:02X}U}}".format( bus, dev, fun), end="", file=config) + if not bar_attr.remappable: + align = ' ' * 48 + print("\n{}/* TODO: add {} 64bit BAR support */".format(align, tmp_sub_name), file=config) + return -def write_vbar(bdf, pci_bar_dic, config): + +def write_vbar(i_cnt, bdf, pci_bar_dic, bar_attr, config): """ Parser and generate vbar + :param i_cnt: the number of pci devices have the same PCI sub class name :param bdf: it is a string what contains BDF :param pci_bar_dic: it is a dictionary of pci vbar for those BDF + :param bar_attr: it is a class, contains PIC bar attribute :param config: it is a file pointer of pci information for writing to """ tail = 0 align = ' ' * 48 + ptdev_mmio_str = '' + + tmp_sub_name = board_cfg_lib.undline_name(bar_attr.name) + "_" + str(i_cnt) if bdf in pci_bar_dic.keys(): bar_list = list(pci_bar_dic[bdf].keys()) bar_len = len(bar_list) bar_num = 0 for bar_i in bar_list: + if not bar_attr.remappable: + return + if tail == 0: print(", \\", file=config) tail += 1 bar_num += 1 - bar_val = pci_bar_dic[bdf][bar_i] + bar_val = pci_bar_dic[bdf][bar_i].addr + if pci_bar_dic[bdf][bar_i].remapped: + ptdev_mmio_str = 'PTDEV_HI_MMIO_START + ' if bar_num == bar_len: - print("{}.vbar_base[{}] = {}UL".format(align, bar_i, bar_val), file=config) + print("{}.vbar_base[{}] = {}{}UL".format(align, bar_i, ptdev_mmio_str, bar_val), file=config) else: - print("{}.vbar_base[{}] = {}UL, \\".format( - align, bar_i, bar_val), file=config) + print("{}.vbar_base[{}] = {}{}UL, \\".format( + align, bar_i, ptdev_mmio_str, bar_val), file=config) - # print("", file=config) else: print("", file=config) @@ -145,7 +224,6 @@ def generate_file(config): Get PCI device and generate pci_devices.h :param config: it is a file pointer of pci information for writing to """ - # write the license into pci print("{0}".format(board_cfg_lib.HEADER_LICENSE), file=config) @@ -155,20 +233,22 @@ def generate_file(config): # write the header into pci print("{0}".format(PCI_HEADER), file=config) - (pci_dev_dic, pci_bar_dic, sub_name_count) = parser_pci() + sub_name_count = parser_pci() + print("#define %-32s" % "PTDEV_HI_MMIO_SIZE", " {}UL".format(hex(HI_MMIO_OFFSET)), file=config) compared_bdf = [] for cnt_sub_name in sub_name_count.keys(): i_cnt = 0 - for bdf, subname in pci_dev_dic.items(): - if cnt_sub_name == subname and bdf not in compared_bdf: + for bdf, bar_attr in PCI_DEV_BAR_DESC.pci_dev_dic.items(): + if cnt_sub_name == bar_attr.name and bdf not in compared_bdf: compared_bdf.append(bdf) else: continue - write_pbdf(i_cnt, bdf, subname, config) - write_vbar(bdf, pci_bar_dic, config) + print("",file=config) + write_pbdf(i_cnt, bdf, bar_attr, config) + write_vbar(i_cnt, bdf, PCI_DEV_BAR_DESC.pci_bar_dic, bar_attr, config) i_cnt += 1 diff --git a/misc/acrn-config/library/board_cfg_lib.py b/misc/acrn-config/library/board_cfg_lib.py index 80757e2a2..32966c797 100644 --- a/misc/acrn-config/library/board_cfg_lib.py +++ b/misc/acrn-config/library/board_cfg_lib.py @@ -39,6 +39,11 @@ KNOWN_HIDDEN_PDEVS_BOARD_DB = { 'apl-up2':['00:0d:0'], } +SIZE_K = common.SIZE_K +SIZE_M = common.SIZE_M +SIZE_2G = common.SIZE_2G +SIZE_4G = common.SIZE_4G + def prepare(check_git): """ check environment """ @@ -433,3 +438,8 @@ def undline_name(name): :return: name_str which contain'_' """ return common.undline_name(name) + + +def round_up(addr, mem_align): + """Keep memory align""" + return common.round_up(addr, mem_align) diff --git a/misc/acrn-config/library/common.py b/misc/acrn-config/library/common.py index 43894d819..01b06571c 100644 --- a/misc/acrn-config/library/common.py +++ b/misc/acrn-config/library/common.py @@ -25,6 +25,11 @@ START_HPA_SIZE_LIST = ['0x20000000', '0x40000000', '0x80000000', 'CONFIG_SOS_RAM MULTI_ITEM = ["guest_flag", "pcpu_id", "input", "block", "network"] +SIZE_K = 1024 +SIZE_M = SIZE_K * 1024 +SIZE_2G = 2 * SIZE_M * SIZE_K +SIZE_4G = 2 * SIZE_2G + class MultiItem(): @@ -681,3 +686,8 @@ def get_vuart_info_id(config_file, idx): vm_id += 1 return tmp_tag + + +def round_up(addr, mem_align): + """Keep memory align""" + return ((addr + (mem_align - 1)) & (~(mem_align - 1)))