From a2fd8c5a9d49aa76eabdb130eff1fc2ac5113bdc Mon Sep 17 00:00:00 2001 From: Li Fei1 Date: Mon, 31 Aug 2020 14:16:17 +0800 Subject: [PATCH] pci: mcfg: limit device bus numbers which could access by ECAM Per PCI Firmware Specification Revision 3.0, 4.1.2. MCFG Table Description: Memory Mapped Enhanced Configuration Space Base Address Allocation Structure assign the Start Bus Number and the End Bus Number which could decoded by the Host Bridge. We should not access the PCI device which bus number outside of the range of [Start Bus Number, End Bus Number). For ACRN, we should: 1. Don't detect PCI device which bus number outside the range of [Start Bus Number, End Bus Number) of MCFG ACPI Table. 2. Only trap the ECAM MMIO size: [MMCFG_BASE_ADDRESS, MMCFG_BASE_ADDRESS + (End Bus Number - Start Bus Number + 1) * 0x100000) for SOS. Tracked-On: #5233 Signed-off-by: Li Fei1 Acked-by: Eddie Dong --- hypervisor/acpi_parser/acpi_ext.c | 18 ++++++++++-------- hypervisor/arch/x86/cpu.c | 1 - hypervisor/arch/x86/guest/vm.c | 4 +++- hypervisor/dm/vpci/vhostbridge.c | 6 ++++-- hypervisor/dm/vpci/vpci.c | 19 +++++++++++++------ hypervisor/hw/pci.c | 26 +++++++++++++++++++------- hypervisor/include/dm/vacpi.h | 4 +++- hypervisor/include/dm/vpci.h | 2 +- hypervisor/include/hw/pci.h | 16 ++++++++++++++-- 9 files changed, 67 insertions(+), 29 deletions(-) diff --git a/hypervisor/acpi_parser/acpi_ext.c b/hypervisor/acpi_parser/acpi_ext.c index 35cedb4c8..d7db7ffcf 100644 --- a/hypervisor/acpi_parser/acpi_ext.c +++ b/hypervisor/acpi_parser/acpi_ext.c @@ -113,24 +113,24 @@ static void *get_facs_table(const uint8_t *facp_addr) } /* @pre mcfg_addr != NULL */ -static uint64_t parse_mmcfg_base(const uint8_t *mcfg_addr) +static struct acpi_mcfg_allocation *parse_mcfg_allocation_tables(const uint8_t *mcfg_addr) { - uint64_t base = 0UL; + struct acpi_mcfg_allocation *mcfg_table = NULL; uint32_t length = get_acpi_dt_dword(mcfg_addr, OFFSET_MCFG_LENGTH); if (length > OFFSET_MCFG_ENTRY1) { pr_fatal("Multiple PCI segment groups is not supported!"); } else { - base = get_acpi_dt_qword(mcfg_addr, OFFSET_MCFG_ENTRY0_BASE); + mcfg_table = (struct acpi_mcfg_allocation *)(mcfg_addr + OFFSET_MCFG_ENTRY0_BASE); } - return base; + return mcfg_table; } /* put all ACPI fix up code here */ int32_t acpi_fixup(void) { uint8_t *facp_addr = NULL, *facs_addr = NULL, *mcfg_addr = NULL; - uint64_t def_mmcfg_base = 0UL; + struct acpi_mcfg_allocation *mcfg_table = NULL; int32_t ret = 0; struct acpi_generic_address pm1a_cnt, pm1a_evt; struct pm_s_state_data *sx_data = get_host_sstate_data(); @@ -162,12 +162,14 @@ int32_t acpi_fixup(void) mcfg_addr = (uint8_t *)get_acpi_tbl(ACPI_SIG_MCFG); if (mcfg_addr != NULL) { - def_mmcfg_base = parse_mmcfg_base(mcfg_addr); - set_mmcfg_base(def_mmcfg_base); + mcfg_table = parse_mcfg_allocation_tables(mcfg_addr); + if (mcfg_table != NULL) { + set_mmcfg_region((struct pci_mmcfg_region*)mcfg_table); + } } if ((facp_addr == NULL) || (facs_addr == NULL) - || (mcfg_addr == NULL) || (def_mmcfg_base == 0UL)) { + || (mcfg_addr == NULL) || (mcfg_table == NULL)) { ret = -1; } diff --git a/hypervisor/arch/x86/cpu.c b/hypervisor/arch/x86/cpu.c index 021c6c118..fc9efef32 100644 --- a/hypervisor/arch/x86/cpu.c +++ b/hypervisor/arch/x86/cpu.c @@ -244,7 +244,6 @@ void init_pcpu_post(uint16_t pcpu_id) panic("failed to initialize iommu!"); } - hv_access_memory_region_update(get_mmcfg_base(), PCI_MMCONFIG_SIZE); #ifdef CONFIG_IVSHMEM_ENABLED init_ivshmem_shared_memory(); #endif diff --git a/hypervisor/arch/x86/guest/vm.c b/hypervisor/arch/x86/guest/vm.c index 78511a9ab..423d43353 100644 --- a/hypervisor/arch/x86/guest/vm.c +++ b/hypervisor/arch/x86/guest/vm.c @@ -292,6 +292,7 @@ static void prepare_sos_vm_memmap(struct acrn_vm *vm) uint32_t entries_count = vm->e820_entry_num; const struct e820_entry *p_e820 = vm->e820_entries; const struct mem_range *p_mem_range_info = get_mem_range_info(); + struct pci_mmcfg_region *pci_mmcfg; pr_dbg("sos_vm: bottom memory - 0x%lx, top memory - 0x%lx\n", p_mem_range_info->mem_bottom, p_mem_range_info->mem_top); @@ -354,7 +355,8 @@ static void prepare_sos_vm_memmap(struct acrn_vm *vm) ept_del_mr(vm, pml4_page, get_ap_trampoline_buf(), CONFIG_LOW_RAM_SIZE); /* unmap PCIe MMCONFIG region since it's owned by hypervisor */ - ept_del_mr(vm, (uint64_t *)vm->arch_vm.nworld_eptp, get_mmcfg_base(), PCI_MMCONFIG_SIZE); + pci_mmcfg = get_mmcfg_region(); + ept_del_mr(vm, (uint64_t *)vm->arch_vm.nworld_eptp, pci_mmcfg->address, get_pci_mmcfg_size(pci_mmcfg)); } /* Add EPT mapping of EPC reource for the VM */ diff --git a/hypervisor/dm/vpci/vhostbridge.c b/hypervisor/dm/vpci/vhostbridge.c index 7369a22d3..eed841e53 100644 --- a/hypervisor/dm/vpci/vhostbridge.c +++ b/hypervisor/dm/vpci/vhostbridge.c @@ -113,8 +113,10 @@ static void init_vhostbridge(struct pci_vdev *vdev) pci_vdev_write_vcfg(vdev, 0xf4U, 4U, 0x011c0f00U); if (is_prelaunched_vm(container_of(vdev->vpci, struct acrn_vm, vpci))) { - /* For pre-launched VMs, we only need to write an GPA that's reserved in guest ve820, and VIRT_PCI_MMCFG_BASE(0xE0000000) is fine. The trailing 1 is a ECAM enable-bit */ - pciexbar_low = VIRT_PCI_MMCFG_BASE | 0x1U; + /* For pre-launched VMs, we only need to write an GPA that's reserved in guest ve820, + * and UOS_VIRT_PCI_MMCFG_BASE(0xE0000000) is fine. The trailing 1 is a ECAM enable-bit + */ + pciexbar_low = UOS_VIRT_PCI_MMCFG_BASE | 0x1U; } else { /*Inject physical ECAM value to SOS vhostbridge since SOS may check PCIe-MMIO Base Address with it */ diff --git a/hypervisor/dm/vpci/vpci.c b/hypervisor/dm/vpci/vpci.c index ccda27353..03b8dbec9 100644 --- a/hypervisor/dm/vpci/vpci.c +++ b/hypervisor/dm/vpci/vpci.c @@ -174,7 +174,7 @@ static int32_t vpci_mmio_cfg_access(struct io_request *io_req, void *private_dat int32_t ret = 0; struct mmio_request *mmio = &io_req->reqs.mmio; struct acrn_vpci *vpci = (struct acrn_vpci *)private_data; - uint64_t pci_mmcofg_base = vpci->pci_mmcfg_base; + uint64_t pci_mmcofg_base = vpci->pci_mmcfg.address; uint64_t address = mmio->address; uint32_t reg_num = (uint32_t)(address & 0xfffUL); union pci_bdf bdf; @@ -216,7 +216,7 @@ void init_vpci(struct acrn_vm *vm) }; struct acrn_vm_config *vm_config; - uint64_t pci_mmcfg_base; + struct pci_mmcfg_region *pci_mmcfg; vm->iommu = create_iommu_domain(vm->vm_id, hva2hpa(vm->arch_vm.nworld_eptp), 48U); /* Build up vdev list for vm */ @@ -224,10 +224,17 @@ void init_vpci(struct acrn_vm *vm) vm_config = get_vm_config(vm->vm_id); /* virtual PCI MMCONFIG for SOS is same with the physical value */ - pci_mmcfg_base = (vm_config->load_order == SOS_VM) ? get_mmcfg_base() : VIRT_PCI_MMCFG_BASE; - vm->vpci.pci_mmcfg_base = pci_mmcfg_base; - register_mmio_emulation_handler(vm, vpci_mmio_cfg_access, - pci_mmcfg_base, pci_mmcfg_base + PCI_MMCONFIG_SIZE, &vm->vpci, false); + if (vm_config->load_order == SOS_VM) { + pci_mmcfg = get_mmcfg_region(); + vm->vpci.pci_mmcfg = *pci_mmcfg; + } else { + vm->vpci.pci_mmcfg.address = UOS_VIRT_PCI_MMCFG_BASE; + vm->vpci.pci_mmcfg.start_bus = UOS_VIRT_PCI_MMCFG_START_BUS; + vm->vpci.pci_mmcfg.end_bus = UOS_VIRT_PCI_MMCFG_END_BUS; + } + + register_mmio_emulation_handler(vm, vpci_mmio_cfg_access, vm->vpci.pci_mmcfg.address, + vm->vpci.pci_mmcfg.address + get_pci_mmcfg_size(&vm->vpci.pci_mmcfg), &vm->vpci, false); /* Intercept and handle I/O ports CF8h */ register_pio_emulation_handler(vm, PCI_CFGADDR_PIO_IDX, &pci_cfgaddr_range, diff --git a/hypervisor/hw/pci.c b/hypervisor/hw/pci.c index 30086f7b2..41df3a2c7 100644 --- a/hypervisor/hw/pci.c +++ b/hypervisor/hw/pci.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -52,18 +53,23 @@ static uint32_t num_pci_pdev; static struct pci_pdev pci_pdevs[CONFIG_MAX_PCI_DEV_NUM]; static struct hlist_head pdevs_hlist_heads[PDEV_HLIST_HASHSIZE]; -static uint64_t pci_mmcfg_base = DEFAULT_PCI_MMCFG_BASE; + +static struct pci_mmcfg_region phys_pci_mmcfg = { + .address = DEFAULT_PCI_MMCFG_BASE, + .start_bus = DEFAULT_PCI_MMCFG_START_BUS, + .end_bus = DEFAULT_PCI_MMCFG_END_BUS, +}; #ifdef CONFIG_ACPI_PARSE_ENABLED -void set_mmcfg_base(uint64_t mmcfg_base) +void set_mmcfg_region(struct pci_mmcfg_region *region) { - pci_mmcfg_base = mmcfg_base; + phys_pci_mmcfg = *region; } #endif -uint64_t get_mmcfg_base(void) +struct pci_mmcfg_region *get_mmcfg_region(void) { - return pci_mmcfg_base; + return &phys_pci_mmcfg; } #if defined(HV_DEBUG) @@ -143,11 +149,11 @@ static const struct pci_cfg_ops pci_pio_cfg_ops = { /* * @pre offset < 0x1000U - * @pre pci_mmcfg_base 4K-byte alignment + * @pre phys_pci_mmcfg.address 4K-byte alignment */ static inline uint32_t mmcfg_off_to_address(union pci_bdf bdf, uint32_t offset) { - return (uint32_t)pci_mmcfg_base + (((uint32_t)bdf.value << 12U) | offset); + return (uint32_t)phys_pci_mmcfg.address + (((uint32_t)bdf.value << 12U) | offset); } /* @@ -438,6 +444,10 @@ static void scan_pci_hierarchy(uint8_t bus, uint64_t buses_visited[BUSES_BITMAP_ &buses_visited[current_bus_index >> 6U]); pbdf.bits.b = current_bus_index; + if (pbdf.bits.b < phys_pci_mmcfg.start_bus || pbdf.bits.b > phys_pci_mmcfg.end_bus) { + continue; + } + for (dev = 0U; dev <= PCI_SLOTMAX; dev++) { pbdf.bits.d = dev; pbdf.bits.f = 0U; @@ -625,6 +635,8 @@ void init_pci_pdev_list(void) uint16_t bus; bool was_visited = false; + hv_access_memory_region_update(phys_pci_mmcfg.address, get_pci_mmcfg_size(&phys_pci_mmcfg)); + pci_parse_iommu_devscopes(&bdfs_from_drhds, &drhd_idx_pci_all); /* TODO: iterate over list of PCI Host Bridges found in ACPI namespace */ diff --git a/hypervisor/include/dm/vacpi.h b/hypervisor/include/dm/vacpi.h index 840f0894c..a5d57af8c 100644 --- a/hypervisor/include/dm/vacpi.h +++ b/hypervisor/include/dm/vacpi.h @@ -23,7 +23,9 @@ #define VIRT_XSDT_ADDR 0x7ff00080UL /* virtual PCI MMCFG address base for pre/post-launched VM. */ -#define VIRT_PCI_MMCFG_BASE 0xE0000000UL +#define UOS_VIRT_PCI_MMCFG_BASE 0xE0000000UL +#define UOS_VIRT_PCI_MMCFG_START_BUS 0x0U +#define UOS_VIRT_PCI_MMCFG_END_BUS 0xFFU void build_vrsdp(struct acrn_vm *vm); diff --git a/hypervisor/include/dm/vpci.h b/hypervisor/include/dm/vpci.h index 3feebdddf..7c856fce3 100644 --- a/hypervisor/include/dm/vpci.h +++ b/hypervisor/include/dm/vpci.h @@ -155,7 +155,7 @@ union pci_cfg_addr_reg { struct acrn_vpci { spinlock_t lock; union pci_cfg_addr_reg addr; - uint64_t pci_mmcfg_base; + struct pci_mmcfg_region pci_mmcfg; uint32_t pci_vdev_cnt; struct pci_vdev pci_vdevs[CONFIG_MAX_PCI_DEV_NUM]; struct hlist_head vdevs_hlist_heads [VDEV_LIST_HASHSIZE]; diff --git a/hypervisor/include/hw/pci.h b/hypervisor/include/hw/pci.h index a4bc48be3..d6cc5dae5 100644 --- a/hypervisor/include/hw/pci.h +++ b/hypervisor/include/hw/pci.h @@ -208,6 +208,13 @@ enum pci_bar_type { PCIBAR_MEM64HI, }; +struct pci_mmcfg_region { + uint64_t address; /* Base address, processor-relative */ + uint16_t pci_segment; /* PCI segment group number */ + uint8_t start_bus; /* Starting PCI Bus number */ + uint8_t end_bus; /* Final PCI Bus number */ +} __packed; + /* Basic MSIX capability info */ struct pci_msix_cap { uint32_t capoff; @@ -324,10 +331,15 @@ static inline bool bdf_is_equal(union pci_bdf a, union pci_bdf b) return (a.value == b.value); } +static inline uint64_t get_pci_mmcfg_size(struct pci_mmcfg_region *pci_mmcfg) +{ + return 0x100000UL * (pci_mmcfg->end_bus - pci_mmcfg->start_bus + 1U); +} + #ifdef CONFIG_ACPI_PARSE_ENABLED -void set_mmcfg_base(uint64_t mmcfg_base); +void set_mmcfg_region(struct pci_mmcfg_region *region); #endif -uint64_t get_mmcfg_base(void); +struct pci_mmcfg_region *get_mmcfg_region(void); struct pci_pdev *init_pdev(uint16_t pbdf, uint32_t drhd_index); uint32_t pci_pdev_read_cfg(union pci_bdf bdf, uint32_t offset, uint32_t bytes);