diff --git a/hypervisor/arch/x86/Kconfig b/hypervisor/arch/x86/Kconfig index 273422ab1..d1542c3e7 100644 --- a/hypervisor/arch/x86/Kconfig +++ b/hypervisor/arch/x86/Kconfig @@ -222,7 +222,7 @@ config HV_RAM_START config HV_RAM_SIZE hex "Size of the RAM region used by the hypervisor" - default 0x04000000 + default 0x06000000 help A 64-bit integer indicating the size of RAM used by the hypervisor. It is ensured at link time that the footprint of the hypervisor diff --git a/hypervisor/arch/x86/ept.c b/hypervisor/arch/x86/ept.c index 4952d95c3..80e812ccc 100644 --- a/hypervisor/arch/x86/ept.c +++ b/hypervisor/arch/x86/ept.c @@ -10,55 +10,18 @@ #define ACRN_DBG_EPT 6U -/** - * @pre pml4_addr != NULL - */ -void free_ept_mem(uint64_t *pml4_page) -{ - uint64_t *pdpt_page, *pd_page, *pt_page; - uint64_t *pml4e, *pdpte, *pde; - uint64_t pml4e_idx, pdpte_idx, pde_idx; - - for (pml4e_idx = 0U; pml4e_idx < PTRS_PER_PML4E; pml4e_idx++) { - pml4e = pml4_page + pml4e_idx; - if (pgentry_present(PTT_EPT, *pml4e) == 0UL) { - continue; - } - pdpt_page = pml4e_page_vaddr(*pml4e); - - for (pdpte_idx = 0U; pdpte_idx < PTRS_PER_PDPTE; pdpte_idx++) { - pdpte = pdpt_page + pdpte_idx; - if ((pgentry_present(PTT_EPT, *pdpte) == 0UL) || - pdpte_large(*pdpte) != 0UL) { - continue; - } - pd_page = pdpte_page_vaddr(*pdpte); - - for (pde_idx = 0U; pde_idx < PTRS_PER_PDE; pde_idx++) { - pde = pd_page + pde_idx; - if ((pgentry_present(PTT_EPT, *pde) == 0UL) || - pde_large(*pde) != 0UL) { - continue; - } - pt_page = pde_page_vaddr(*pde); - - /* Free page table entry table */ - free_paging_struct((void *)pt_page); - } - /* Free page directory entry table */ - free_paging_struct((void *)pd_page); - } - free_paging_struct((void *)pdpt_page); - } - free_paging_struct((void *)pml4_page); -} - void destroy_ept(struct vm *vm) { + /* Destroy secure world */ + if (vm->sworld_control.flag.active != 0UL) { + destroy_secure_world(vm, true); + } + if (vm->arch_vm.nworld_eptp != NULL) { - free_ept_mem((uint64_t *)vm->arch_vm.nworld_eptp); + (void)memset(vm->arch_vm.nworld_eptp, 0U, CPU_PAGE_SIZE); } } + /* using return value INVALID_HPA as error code */ uint64_t local_gpa2hpa(struct vm *vm, uint64_t gpa, uint32_t *size) { @@ -73,7 +36,7 @@ uint64_t local_gpa2hpa(struct vm *vm, uint64_t gpa, uint32_t *size) eptp = vm->arch_vm.nworld_eptp; } - pgentry = lookup_address((uint64_t *)eptp, gpa, &pg_size, PTT_EPT); + pgentry = lookup_address((uint64_t *)eptp, gpa, &pg_size, &vm->arch_vm.ept_mem_ops); if (pgentry != NULL) { hpa = ((*pgentry & (~(pg_size - 1UL))) | (gpa & (pg_size - 1UL))); @@ -222,9 +185,8 @@ void ept_mr_add(struct vm *vm, uint64_t *pml4_page, struct vcpu *vcpu; uint64_t prot = prot_orig; - dev_dbg(ACRN_DBG_EPT, "%s, vm[%d] hpa: 0x%016llx gpa: 0x%016llx ", - __func__, vm->vm_id, hpa, gpa); - dev_dbg(ACRN_DBG_EPT, "size: 0x%016llx prot: 0x%016x\n", size, prot); + dev_dbg(ACRN_DBG_EPT, "%s, vm[%d] hpa: 0x%016llx gpa: 0x%016llx size: 0x%016llx prot: 0x%016x\n", + __func__, vm->vm_id, hpa, gpa, size, prot); /* EPT & VT-d share the same page tables, set SNP bit * to force snooping of PCIe devices if the page @@ -234,7 +196,7 @@ void ept_mr_add(struct vm *vm, uint64_t *pml4_page, prot |= EPT_SNOOP_CTRL; } - mmu_add(pml4_page, hpa, gpa, size, prot, PTT_EPT); + mmu_add(pml4_page, hpa, gpa, size, prot, &vm->arch_vm.ept_mem_ops); foreach_vcpu(i, vm, vcpu) { vcpu_make_request(vcpu, ACRN_REQUEST_EPT_FLUSH); @@ -248,8 +210,9 @@ void ept_mr_modify(struct vm *vm, uint64_t *pml4_page, struct vcpu *vcpu; uint16_t i; - mmu_modify_or_del(pml4_page, gpa, size, - prot_set, prot_clr, PTT_EPT, MR_MODIFY); + dev_dbg(ACRN_DBG_EPT, "%s,vm[%d] gpa 0x%llx size 0x%llx\n", __func__, vm->vm_id, gpa, size); + + mmu_modify_or_del(pml4_page, gpa, size, prot_set, prot_clr, &vm->arch_vm.ept_mem_ops, MR_MODIFY); foreach_vcpu(i, vm, vcpu) { vcpu_make_request(vcpu, ACRN_REQUEST_EPT_FLUSH); @@ -258,17 +221,14 @@ void ept_mr_modify(struct vm *vm, uint64_t *pml4_page, /** * @pre [gpa,gpa+size) has been mapped into host physical memory region */ -void ept_mr_del(struct vm *vm, uint64_t *pml4_page, - uint64_t gpa, uint64_t size) +void ept_mr_del(struct vm *vm, uint64_t *pml4_page, uint64_t gpa, uint64_t size) { struct vcpu *vcpu; uint16_t i; - dev_dbg(ACRN_DBG_EPT, "%s,vm[%d] gpa 0x%llx size 0x%llx\n", - __func__, vm->vm_id, gpa, size); + dev_dbg(ACRN_DBG_EPT, "%s,vm[%d] gpa 0x%llx size 0x%llx\n", __func__, vm->vm_id, gpa, size); - mmu_modify_or_del(pml4_page, gpa, size, - 0UL, 0UL, PTT_EPT, MR_DEL); + mmu_modify_or_del(pml4_page, gpa, size, 0UL, 0UL, &vm->arch_vm.ept_mem_ops, MR_DEL); foreach_vcpu(i, vm, vcpu) { vcpu_make_request(vcpu, ACRN_REQUEST_EPT_FLUSH); diff --git a/hypervisor/arch/x86/guest/guest.c b/hypervisor/arch/x86/guest/guest.c index d54da65d9..6549c8a0f 100644 --- a/hypervisor/arch/x86/guest/guest.c +++ b/hypervisor/arch/x86/guest/guest.c @@ -619,6 +619,10 @@ int prepare_vm0_memmap_and_e820(struct vm *vm) "vm0: bottom memory - 0x%llx, top memory - 0x%llx\n", e820_mem.mem_bottom, e820_mem.mem_top); + if (e820_mem.mem_top > EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE)) { + panic("Please configure VM0_ADDRESS_SPACE correctly!\n"); + } + /* create real ept map for all ranges with UC */ ept_mr_add(vm, pml4_page, e820_mem.mem_bottom, e820_mem.mem_bottom, diff --git a/hypervisor/arch/x86/guest/vm.c b/hypervisor/arch/x86/guest/vm.c index fe37e26c2..31abfb7dc 100644 --- a/hypervisor/arch/x86/guest/vm.c +++ b/hypervisor/arch/x86/guest/vm.c @@ -87,12 +87,8 @@ int create_vm(struct vm_description *vm_desc, struct vm **rtn_vm) /* gpa_lowtop are used for system start up */ vm->hw.gpa_lowtop = 0UL; - vm->arch_vm.nworld_eptp = alloc_paging_struct(); - if (vm->arch_vm.nworld_eptp == NULL) { - pr_fatal("%s, alloc memory for EPTP failed\n", __func__); - status = -ENOMEM; - goto err; - } + init_ept_mem_ops(vm); + vm->arch_vm.nworld_eptp = vm->arch_vm.ept_mem_ops.get_pml4_page(vm->arch_vm.ept_mem_ops.info, 0UL); sanitize_pte((uint64_t *)vm->arch_vm.nworld_eptp); /* Only for SOS: Configure VM software information */ @@ -180,7 +176,7 @@ err: vioapic_cleanup(vm_ioapic(vm)); if (vm->arch_vm.nworld_eptp != NULL) { - free(vm->arch_vm.nworld_eptp); + (void)memset(vm->arch_vm.nworld_eptp, 0U, CPU_PAGE_SIZE); } return status; @@ -212,10 +208,6 @@ int shutdown_vm(struct vm *vm) /* cleanup vioapic */ vioapic_cleanup(vm_ioapic(vm)); - /* Destroy secure world */ - if (vm->sworld_control.flag.active != 0UL) { - destroy_secure_world(vm, true); - } /* Free EPT allocated resources assigned to VM */ destroy_ept(vm); diff --git a/hypervisor/arch/x86/mmu.c b/hypervisor/arch/x86/mmu.c index 5f8822a58..0d7ac2f90 100644 --- a/hypervisor/arch/x86/mmu.c +++ b/hypervisor/arch/x86/mmu.c @@ -238,39 +238,57 @@ void init_paging(void) struct e820_entry *entry; uint64_t hv_hpa; uint32_t i; + uint64_t low32_max_ram = 0UL; + uint64_t high64_max_ram; uint64_t attr_uc = (PAGE_TABLE | PAGE_CACHE_UC); pr_dbg("HV MMU Initialization"); /* Allocate memory for Hypervisor PML4 table */ - mmu_pml4_addr = alloc_paging_struct(); + mmu_pml4_addr = ppt_mem_ops.get_pml4_page(ppt_mem_ops.info, 0UL); init_e820(); obtain_e820_mem_info(); + /* align to 2MB */ + high64_max_ram = (e820_mem.mem_top + PDE_SIZE - 1UL) & PDE_MASK; + + if (high64_max_ram > (CONFIG_PLATFORM_RAM_SIZE + PLATFORM_LO_MMIO_SIZE) || + high64_max_ram < (1UL << 32U)) { + panic("Please configure HV_ADDRESS_SPACE correctly!\n"); + } + /* Map all memory regions to UC attribute */ - mmu_add((uint64_t *)mmu_pml4_addr, e820_mem.mem_bottom, - e820_mem.mem_bottom, e820_mem.mem_top - e820_mem.mem_bottom, - attr_uc, PTT_PRIMARY); + mmu_add((uint64_t *)mmu_pml4_addr, e820_mem.mem_bottom, e820_mem.mem_bottom, + high64_max_ram - e820_mem.mem_bottom, attr_uc, &ppt_mem_ops); /* Modify WB attribute for E820_TYPE_RAM */ for (i = 0U; i < e820_entries; i++) { entry = &e820[i]; if (entry->type == E820_TYPE_RAM) { - mmu_modify_or_del((uint64_t *)mmu_pml4_addr, - entry->baseaddr, entry->length, - PAGE_CACHE_WB, PAGE_CACHE_MASK, - PTT_PRIMARY, MR_MODIFY); + if (entry->baseaddr < (1UL << 32U)) { + uint64_t end = entry->baseaddr + entry->length; + if (end < (1UL << 32U) && (end > low32_max_ram)) { + low32_max_ram = end; + } + } } } + mmu_modify_or_del((uint64_t *)mmu_pml4_addr, 0UL, (low32_max_ram + PDE_SIZE - 1UL) & PDE_MASK, + PAGE_CACHE_WB, PAGE_CACHE_MASK, &ppt_mem_ops, MR_MODIFY); + + mmu_modify_or_del((uint64_t *)mmu_pml4_addr, (1UL << 32U), high64_max_ram - (1UL << 32U), + PAGE_CACHE_WB, PAGE_CACHE_MASK, &ppt_mem_ops, MR_MODIFY); + /* set the paging-structure entries' U/S flag * to supervisor-mode for hypervisor owned memroy. */ hv_hpa = get_hv_image_base(); - mmu_modify_or_del((uint64_t *)mmu_pml4_addr, hv_hpa, CONFIG_HV_RAM_SIZE, + mmu_modify_or_del((uint64_t *)mmu_pml4_addr, hv_hpa & PDE_MASK, + CONFIG_HV_RAM_SIZE + ((hv_hpa & (PDE_SIZE - 1UL)) != 0UL) ? PDE_SIZE : 0UL, PAGE_CACHE_WB, PAGE_CACHE_MASK | PAGE_USER, - PTT_PRIMARY, MR_MODIFY); + &ppt_mem_ops, MR_MODIFY); /* Enable paging */ enable_paging(hva2hpa(mmu_pml4_addr)); diff --git a/hypervisor/arch/x86/page.c b/hypervisor/arch/x86/page.c index 6d8cb594c..7c8a3c4c0 100644 --- a/hypervisor/arch/x86/page.c +++ b/hypervisor/arch/x86/page.c @@ -5,11 +5,6 @@ */ #include -#define PML4_PAGE_NUM(size) 1UL -#define PDPT_PAGE_NUM(size) (((size) + PML4E_SIZE - 1UL) >> PML4E_SHIFT) -#define PD_PAGE_NUM(size) (((size) + PDPTE_SIZE - 1UL) >> PDPTE_SHIFT) -#define PT_PAGE_NUM(size) (((size) + PDE_SIZE - 1UL) >> PDE_SHIFT) - #define DEFINE_PGTABLE_PAGE(prefix, lvl, LVL, size) \ static struct page prefix ## lvl ## _pages[LVL ## _PAGE_NUM(size)] @@ -66,8 +61,6 @@ const struct memory_ops ppt_mem_ops = { .get_pd_page = ppt_get_pd_page, }; -/* The size of the guest physical address space, covered by the EPT page table of a VM */ -#define EPT_ADDRESS_SPACE(size) ((size != 0UL) ? (size + PLATFORM_LO_MMIO_SIZE) : 0UL) DEFINE_PGTABLE_PAGE(vm0_, pml4, PML4, EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE)); DEFINE_PGTABLE_PAGE(vm0_, pdpt, PDPT, EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE)); DEFINE_PGTABLE_PAGE(vm0_, pd, PD, EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE)); @@ -79,13 +72,6 @@ static struct page uos_nworld_pdpt_pages[CONFIG_MAX_VM_NUM - 1U][PDPT_PAGE_NUM(E static struct page uos_nworld_pd_pages[CONFIG_MAX_VM_NUM - 1U][PD_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))]; static struct page uos_nworld_pt_pages[CONFIG_MAX_VM_NUM - 1U][PT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))]; -#define TRUSTY_PML4_PAGE_NUM(size) (1UL) -#define TRUSTY_PDPT_PAGE_NUM(size) (1UL) -#define TRUSTY_PD_PAGE_NUM(size) (PD_PAGE_NUM(size)) -#define TRUSTY_PT_PAGE_NUM(size) (PT_PAGE_NUM(size)) -#define TRUSTY_PGTABLE_PAGE_NUM(size) \ -(TRUSTY_PML4_PAGE_NUM(size) + TRUSTY_PDPT_PAGE_NUM(size) + TRUSTY_PD_PAGE_NUM(size) + TRUSTY_PT_PAGE_NUM(size)) - static struct page uos_sworld_pgtable_pages[CONFIG_MAX_VM_NUM - 1U][TRUSTY_PGTABLE_PAGE_NUM(TRUSTY_RAM_SIZE)]; /* ept: extended page table*/ diff --git a/hypervisor/arch/x86/pagetable.c b/hypervisor/arch/x86/pagetable.c index 85dc164e6..b0fb7b281 100644 --- a/hypervisor/arch/x86/pagetable.c +++ b/hypervisor/arch/x86/pagetable.c @@ -10,11 +10,9 @@ /* * Split a large page table into next level page table. */ -static int split_large_page(uint64_t *pte, - enum _page_table_level level, - enum _page_table_type ptt) +static void split_large_page(uint64_t *pte, enum _page_table_level level, + uint64_t vaddr, const struct memory_ops *mem_ops) { - int ret = -EINVAL; uint64_t *pbase; uint64_t ref_paddr, paddr, paddrinc; uint64_t i, ref_prot; @@ -24,23 +22,20 @@ static int split_large_page(uint64_t *pte, ref_paddr = (*pte) & PDPTE_PFN_MASK; paddrinc = PDE_SIZE; ref_prot = (*pte) & ~PDPTE_PFN_MASK; + pbase = (uint64_t *)mem_ops->get_pd_page(mem_ops->info, vaddr); break; case IA32E_PD: ref_paddr = (*pte) & PDE_PFN_MASK; paddrinc = PTE_SIZE; ref_prot = (*pte) & ~PDE_PFN_MASK; ref_prot &= ~PAGE_PSE; + pbase = (uint64_t *)mem_ops->get_pt_page(mem_ops->info, vaddr); break; default: - return ret; + panic("invalid paging table level: %d", level); } - dev_dbg(ACRN_DBG_MMU, "%s, paddr: 0x%llx\n", __func__, ref_paddr); - - pbase = (uint64_t *)alloc_paging_struct(); - if (pbase == NULL) { - return -ENOMEM; - } + dev_dbg(ACRN_DBG_MMU, "%s, paddr: 0x%llx, pbase: 0x%llx\n", __func__, ref_paddr, pbase); paddr = ref_paddr; for (i = 0UL; i < PTRS_PER_PTE; i++) { @@ -48,12 +43,10 @@ static int split_large_page(uint64_t *pte, paddr += paddrinc; } - ref_prot = (ptt == PTT_PRIMARY) ? PAGE_TABLE : EPT_RWX; + ref_prot = mem_ops->get_default_access_right(); set_pgentry(pte, hva2hpa((void *)pbase) | ref_prot); /* TODO: flush the TLB */ - - return 0; } static inline void local_modify_or_del_pte(uint64_t *pte, @@ -72,19 +65,11 @@ static inline void local_modify_or_del_pte(uint64_t *pte, /* * pgentry may means pml4e/pdpte/pde */ -static inline int construct_pgentry(enum _page_table_type ptt, uint64_t *pde) +static inline void construct_pgentry(uint64_t *pde, void *pd_page, uint64_t prot) { - uint64_t prot; - void *pd_page = alloc_paging_struct(); - if (pd_page == NULL) { - return -ENOMEM; - } - sanitize_pte((uint64_t *)pd_page); - prot = (ptt == PTT_PRIMARY) ? PAGE_TABLE: EPT_RWX; set_pgentry(pde, hva2hpa(pd_page) | prot); - return 0; } /* @@ -94,21 +79,18 @@ static inline int construct_pgentry(enum _page_table_type ptt, uint64_t *pde) * type: MR_DEL * delete [vaddr_start, vaddr_end) MT PT mapping */ -static void modify_or_del_pte(const uint64_t *pde, - uint64_t vaddr_start, uint64_t vaddr_end, - uint64_t prot_set, uint64_t prot_clr, - enum _page_table_type ptt, uint32_t type) +static void modify_or_del_pte(const uint64_t *pde, uint64_t vaddr_start, uint64_t vaddr_end, + uint64_t prot_set, uint64_t prot_clr, const struct memory_ops *mem_ops, uint32_t type) { uint64_t *pt_page = pde_page_vaddr(*pde); uint64_t vaddr = vaddr_start; uint64_t index = pte_index(vaddr); - dev_dbg(ACRN_DBG_MMU, "%s, vaddr: [0x%llx - 0x%llx]\n", - __func__, vaddr, vaddr_end); + dev_dbg(ACRN_DBG_MMU, "%s, vaddr: [0x%llx - 0x%llx]\n", __func__, vaddr, vaddr_end); for (; index < PTRS_PER_PTE; index++) { uint64_t *pte = pt_page + index; - if (pgentry_present(ptt, *pte) == 0UL) { + if (mem_ops->pgentry_present(*pte) == 0UL) { panic("invalid op, pte not present"); } @@ -127,34 +109,26 @@ static void modify_or_del_pte(const uint64_t *pde, * type: MR_DEL * delete [vaddr_start, vaddr_end) MT PT mapping */ -static void modify_or_del_pde(const uint64_t *pdpte, - uint64_t vaddr_start, uint64_t vaddr_end, - uint64_t prot_set, uint64_t prot_clr, - enum _page_table_type ptt, uint32_t type) +static void modify_or_del_pde(const uint64_t *pdpte, uint64_t vaddr_start, uint64_t vaddr_end, + uint64_t prot_set, uint64_t prot_clr, const struct memory_ops *mem_ops, uint32_t type) { uint64_t *pd_page = pdpte_page_vaddr(*pdpte); uint64_t vaddr = vaddr_start; uint64_t index = pde_index(vaddr); - dev_dbg(ACRN_DBG_MMU, "%s, vaddr: [0x%llx - 0x%llx]\n", - __func__, vaddr, vaddr_end); + dev_dbg(ACRN_DBG_MMU, "%s, vaddr: [0x%llx - 0x%llx]\n", __func__, vaddr, vaddr_end); for (; index < PTRS_PER_PDE; index++) { uint64_t *pde = pd_page + index; uint64_t vaddr_next = (vaddr & PDE_MASK) + PDE_SIZE; - if (pgentry_present(ptt, *pde) == 0UL) { + if (mem_ops->pgentry_present(*pde) == 0UL) { panic("invalid op, pde not present"); } if (pde_large(*pde) != 0UL) { - if (vaddr_next > vaddr_end || - !mem_aligned_check(vaddr, PDE_SIZE)) { - int ret = split_large_page(pde, IA32E_PD, ptt); - if (ret != 0) { - panic("split large PDE failed"); - } + if (vaddr_next > vaddr_end || !mem_aligned_check(vaddr, PDE_SIZE)) { + split_large_page(pde, IA32E_PD, vaddr, mem_ops); } else { - local_modify_or_del_pte(pde, - prot_set, prot_clr, type); + local_modify_or_del_pte(pde, prot_set, prot_clr, type); if (vaddr_next < vaddr_end) { vaddr = vaddr_next; continue; @@ -162,8 +136,7 @@ static void modify_or_del_pde(const uint64_t *pdpte, break; /* done */ } } - modify_or_del_pte(pde, vaddr, vaddr_end, - prot_set, prot_clr, ptt, type); + modify_or_del_pte(pde, vaddr, vaddr_end, prot_set, prot_clr, mem_ops, type); if (vaddr_next >= vaddr_end) { break; /* done */ } @@ -178,34 +151,27 @@ static void modify_or_del_pde(const uint64_t *pdpte, * type: MR_DEL * delete [vaddr_start, vaddr_end) MT PT mapping */ -static void modify_or_del_pdpte(const uint64_t *pml4e, - uint64_t vaddr_start, uint64_t vaddr_end, - uint64_t prot_set, uint64_t prot_clr, - enum _page_table_type ptt, uint32_t type) +static void modify_or_del_pdpte(const uint64_t *pml4e, uint64_t vaddr_start, uint64_t vaddr_end, + uint64_t prot_set, uint64_t prot_clr, const struct memory_ops *mem_ops, uint32_t type) { uint64_t *pdpt_page = pml4e_page_vaddr(*pml4e); uint64_t vaddr = vaddr_start; uint64_t index = pdpte_index(vaddr); - dev_dbg(ACRN_DBG_MMU, "%s, vaddr: [0x%llx - 0x%llx]\n", - __func__, vaddr, vaddr_end); + dev_dbg(ACRN_DBG_MMU, "%s, vaddr: [0x%llx - 0x%llx]\n", __func__, vaddr, vaddr_end); for (; index < PTRS_PER_PDPTE; index++) { uint64_t *pdpte = pdpt_page + index; uint64_t vaddr_next = (vaddr & PDPTE_MASK) + PDPTE_SIZE; - if (pgentry_present(ptt, *pdpte) == 0UL) { + if (mem_ops->pgentry_present(*pdpte) == 0UL) { panic("invalid op, pdpte not present"); } if (pdpte_large(*pdpte) != 0UL) { if (vaddr_next > vaddr_end || !mem_aligned_check(vaddr, PDPTE_SIZE)) { - int ret = split_large_page(pdpte, IA32E_PDPT, ptt); - if (ret != 0) { - panic("split large PDPTE failed"); - } + split_large_page(pdpte, IA32E_PDPT, vaddr, mem_ops); } else { - local_modify_or_del_pte(pdpte, - prot_set, prot_clr, type); + local_modify_or_del_pte(pdpte, prot_set, prot_clr, type); if (vaddr_next < vaddr_end) { vaddr = vaddr_next; continue; @@ -213,8 +179,7 @@ static void modify_or_del_pdpte(const uint64_t *pml4e, break; /* done */ } } - modify_or_del_pde(pdpte, vaddr, vaddr_end, - prot_set, prot_clr, ptt, type); + modify_or_del_pde(pdpte, vaddr, vaddr_end, prot_set, prot_clr, mem_ops, type); if (vaddr_next >= vaddr_end) { break; /* done */ } @@ -235,10 +200,8 @@ static void modify_or_del_pdpte(const uint64_t *pml4e, * type: MR_DEL * delete [vaddr_base, vaddr_base + size ) memory region page table mapping. */ -void mmu_modify_or_del(uint64_t *pml4_page, - uint64_t vaddr_base, uint64_t size, - uint64_t prot_set, uint64_t prot_clr, - enum _page_table_type ptt, uint32_t type) +void mmu_modify_or_del(uint64_t *pml4_page, uint64_t vaddr_base, uint64_t size, + uint64_t prot_set, uint64_t prot_clr, const struct memory_ops *mem_ops, uint32_t type) { uint64_t vaddr = round_page_up(vaddr_base); uint64_t vaddr_next, vaddr_end; @@ -251,11 +214,10 @@ void mmu_modify_or_del(uint64_t *pml4_page, while (vaddr < vaddr_end) { vaddr_next = (vaddr & PML4E_MASK) + PML4E_SIZE; pml4e = pml4e_offset(pml4_page, vaddr); - if (pgentry_present(ptt, *pml4e) == 0UL) { + if (mem_ops->pgentry_present(*pml4e) == 0UL) { panic("invalid op, pml4e not present"); } - modify_or_del_pdpte(pml4e, vaddr, vaddr_end, - prot_set, prot_clr, ptt, type); + modify_or_del_pdpte(pml4e, vaddr, vaddr_end, prot_set, prot_clr, mem_ops, type); vaddr = vaddr_next; } } @@ -264,9 +226,8 @@ void mmu_modify_or_del(uint64_t *pml4_page, * In PT level, * add [vaddr_start, vaddr_end) to [paddr_base, ...) MT PT mapping */ -static void add_pte(const uint64_t *pde, uint64_t paddr_start, - uint64_t vaddr_start, uint64_t vaddr_end, - uint64_t prot, enum _page_table_type ptt) +static void add_pte(const uint64_t *pde, uint64_t paddr_start, uint64_t vaddr_start, uint64_t vaddr_end, + uint64_t prot, const struct memory_ops *mem_ops) { uint64_t *pt_page = pde_page_vaddr(*pde); uint64_t vaddr = vaddr_start; @@ -278,7 +239,7 @@ static void add_pte(const uint64_t *pde, uint64_t paddr_start, for (; index < PTRS_PER_PTE; index++) { uint64_t *pte = pt_page + index; - if (pgentry_present(ptt, *pte) != 0UL) { + if (mem_ops->pgentry_present(*pte) != 0UL) { panic("invalid op, pte present"); } @@ -296,9 +257,8 @@ static void add_pte(const uint64_t *pde, uint64_t paddr_start, * In PD level, * add [vaddr_start, vaddr_end) to [paddr_base, ...) MT PT mapping */ -static void add_pde(const uint64_t *pdpte, uint64_t paddr_start, - uint64_t vaddr_start, uint64_t vaddr_end, - uint64_t prot, enum _page_table_type ptt) +static void add_pde(const uint64_t *pdpte, uint64_t paddr_start, uint64_t vaddr_start, uint64_t vaddr_end, + uint64_t prot, const struct memory_ops *mem_ops) { uint64_t *pd_page = pdpte_page_vaddr(*pdpte); uint64_t vaddr = vaddr_start; @@ -311,7 +271,7 @@ static void add_pde(const uint64_t *pdpte, uint64_t paddr_start, uint64_t *pde = pd_page + index; uint64_t vaddr_next = (vaddr & PDE_MASK) + PDE_SIZE; - if (pgentry_present(ptt, *pde) == 0UL) { + if (mem_ops->pgentry_present(*pde) == 0UL) { if (mem_aligned_check(paddr, PDE_SIZE) && mem_aligned_check(vaddr, PDE_SIZE) && (vaddr_next <= vaddr_end)) { @@ -323,13 +283,11 @@ static void add_pde(const uint64_t *pdpte, uint64_t paddr_start, } break; /* done */ } else { - int ret = construct_pgentry(ptt, pde); - if (ret != 0) { - panic("construct pde page table fail"); - } + void *pt_page = mem_ops->get_pt_page(mem_ops->info, vaddr); + construct_pgentry(pde, pt_page, mem_ops->get_default_access_right()); } } - add_pte(pde, paddr, vaddr, vaddr_end, prot, ptt); + add_pte(pde, paddr, vaddr, vaddr_end, prot, mem_ops); if (vaddr_next >= vaddr_end) { break; /* done */ } @@ -342,22 +300,20 @@ static void add_pde(const uint64_t *pdpte, uint64_t paddr_start, * In PDPT level, * add [vaddr_start, vaddr_end) to [paddr_base, ...) MT PT mapping */ -static void add_pdpte(const uint64_t *pml4e, uint64_t paddr_start, - uint64_t vaddr_start, uint64_t vaddr_end, - uint64_t prot, enum _page_table_type ptt) +static void add_pdpte(const uint64_t *pml4e, uint64_t paddr_start, uint64_t vaddr_start, uint64_t vaddr_end, + uint64_t prot, const struct memory_ops *mem_ops) { uint64_t *pdpt_page = pml4e_page_vaddr(*pml4e); uint64_t vaddr = vaddr_start; uint64_t paddr = paddr_start; uint64_t index = pdpte_index(vaddr); - dev_dbg(ACRN_DBG_MMU, "%s, paddr: 0x%llx, vaddr: [0x%llx - 0x%llx]\n", - __func__, paddr, vaddr, vaddr_end); + dev_dbg(ACRN_DBG_MMU, "%s, paddr: 0x%llx, vaddr: [0x%llx - 0x%llx]\n", __func__, paddr, vaddr, vaddr_end); for (; index < PTRS_PER_PDPTE; index++) { uint64_t *pdpte = pdpt_page + index; uint64_t vaddr_next = (vaddr & PDPTE_MASK) + PDPTE_SIZE; - if (pgentry_present(ptt, *pdpte) == 0UL) { + if (mem_ops->pgentry_present(*pdpte) == 0UL) { if (mem_aligned_check(paddr, PDPTE_SIZE) && mem_aligned_check(vaddr, PDPTE_SIZE) && (vaddr_next <= vaddr_end)) { @@ -369,13 +325,11 @@ static void add_pdpte(const uint64_t *pml4e, uint64_t paddr_start, } break; /* done */ } else { - int ret = construct_pgentry(ptt, pdpte); - if (ret != 0) { - panic("construct pdpte page table fail"); - } + void *pd_page = mem_ops->get_pd_page(mem_ops->info, vaddr); + construct_pgentry(pdpte, pd_page, mem_ops->get_default_access_right()); } } - add_pde(pdpte, paddr, vaddr, vaddr_end, prot, ptt); + add_pde(pdpte, paddr, vaddr, vaddr_end, prot, mem_ops); if (vaddr_next >= vaddr_end) { break; /* done */ } @@ -389,16 +343,14 @@ static void add_pdpte(const uint64_t *pml4e, uint64_t paddr_start, * add [vaddr_base, vaddr_base + size ) memory region page table mapping. * @pre: the prot should set before call this function. */ -void mmu_add(uint64_t *pml4_page, uint64_t paddr_base, - uint64_t vaddr_base, uint64_t size, - uint64_t prot, enum _page_table_type ptt) +void mmu_add(uint64_t *pml4_page, uint64_t paddr_base, uint64_t vaddr_base, uint64_t size, uint64_t prot, + const struct memory_ops *mem_ops) { uint64_t vaddr, vaddr_next, vaddr_end; uint64_t paddr; uint64_t *pml4e; - dev_dbg(ACRN_DBG_MMU, "%s, paddr 0x%llx, vaddr 0x%llx, size 0x%llx\n", - __func__, paddr_base, vaddr_base, size); + dev_dbg(ACRN_DBG_MMU, "%s, paddr 0x%llx, vaddr 0x%llx, size 0x%llx\n", __func__, paddr_base, vaddr_base, size); /* align address to page size*/ vaddr = round_page_up(vaddr_base); @@ -408,13 +360,11 @@ void mmu_add(uint64_t *pml4_page, uint64_t paddr_base, while (vaddr < vaddr_end) { vaddr_next = (vaddr & PML4E_MASK) + PML4E_SIZE; pml4e = pml4e_offset(pml4_page, vaddr); - if (pgentry_present(ptt, *pml4e) == 0UL) { - int ret = construct_pgentry(ptt, pml4e); - if (ret != 0) { - panic("construct pml4e page table fail"); - } + if (mem_ops->pgentry_present(*pml4e) == 0UL) { + void *pdpt_page = mem_ops->get_pdpt_page(mem_ops->info, vaddr); + construct_pgentry(pml4e, pdpt_page, mem_ops->get_default_access_right()); } - add_pdpte(pml4e, paddr, vaddr, vaddr_end, prot, ptt); + add_pdpte(pml4e, paddr, vaddr, vaddr_end, prot, mem_ops); paddr += (vaddr_next - vaddr); vaddr = vaddr_next; @@ -424,18 +374,17 @@ void mmu_add(uint64_t *pml4_page, uint64_t paddr_base, /** * @pre (pml4_page != NULL) && (pg_size != NULL) */ -uint64_t *lookup_address(uint64_t *pml4_page, - uint64_t addr, uint64_t *pg_size, enum _page_table_type ptt) +uint64_t *lookup_address(uint64_t *pml4_page, uint64_t addr, uint64_t *pg_size, const struct memory_ops *mem_ops) { uint64_t *pml4e, *pdpte, *pde, *pte; pml4e = pml4e_offset(pml4_page, addr); - if (pgentry_present(ptt, *pml4e) == 0UL) { + if (mem_ops->pgentry_present(*pml4e) == 0UL) { return NULL; } pdpte = pdpte_offset(pml4e, addr); - if (pgentry_present(ptt, *pdpte) == 0UL) { + if (mem_ops->pgentry_present(*pdpte) == 0UL) { return NULL; } else if (pdpte_large(*pdpte) != 0UL) { *pg_size = PDPTE_SIZE; @@ -443,7 +392,7 @@ uint64_t *lookup_address(uint64_t *pml4_page, } pde = pde_offset(pdpte, addr); - if (pgentry_present(ptt, *pde) == 0UL) { + if (mem_ops->pgentry_present(*pde) == 0UL) { return NULL; } else if (pde_large(*pde) != 0UL) { *pg_size = PDE_SIZE; @@ -451,7 +400,7 @@ uint64_t *lookup_address(uint64_t *pml4_page, } pte = pte_offset(pde, addr); - if (pgentry_present(ptt, *pte) == 0UL) { + if (mem_ops->pgentry_present(*pte) == 0UL) { return NULL; } else { *pg_size = PTE_SIZE; diff --git a/hypervisor/arch/x86/trusty.c b/hypervisor/arch/x86/trusty.c index 41c3a5678..aa5e42f00 100644 --- a/hypervisor/arch/x86/trusty.c +++ b/hypervisor/arch/x86/trusty.c @@ -97,14 +97,14 @@ static void create_secure_world_ept(struct vm *vm, uint64_t gpa_orig, * Normal World.PD/PT are shared in both Secure world's EPT * and Normal World's EPT */ - pml4_base = alloc_paging_struct(); + pml4_base = vm->arch_vm.ept_mem_ops.get_pml4_page(vm->arch_vm.ept_mem_ops.info, TRUSTY_EPT_REBASE_GPA); vm->arch_vm.sworld_eptp = pml4_base; sanitize_pte((uint64_t *)vm->arch_vm.sworld_eptp); /* The trusty memory is remapped to guest physical address * of gpa_rebased to gpa_rebased + size */ - sub_table_addr = alloc_paging_struct(); + sub_table_addr = vm->arch_vm.ept_mem_ops.get_pdpt_page(vm->arch_vm.ept_mem_ops.info, TRUSTY_EPT_REBASE_GPA); sworld_pml4e = hva2hpa(sub_table_addr) | table_present; set_pgentry((uint64_t *)pml4_base, sworld_pml4e); @@ -148,8 +148,6 @@ static void create_secure_world_ept(struct vm *vm, uint64_t gpa_orig, void destroy_secure_world(struct vm *vm, bool need_clr_mem) { - uint64_t j; - void *pdpt_addr; struct vm *vm0 = get_vm_from_vmid(0U); uint64_t hpa = vm->sworld_control.sworld_memory.base_hpa; uint64_t gpa_sos = vm->sworld_control.sworld_memory.base_gpa_in_sos; @@ -174,14 +172,8 @@ void destroy_secure_world(struct vm *vm, bool need_clr_mem) ept_mr_add(vm, vm->arch_vm.nworld_eptp, hpa, gpa_uos, size, EPT_RWX | EPT_WB); - /* Free trusty ept page-structures */ - pdpt_addr = - (void *)pml4e_page_vaddr(*(uint64_t *)vm->arch_vm.sworld_eptp); - /* identical PDPTEs except trusty memory */ - for (j = 0UL; j < NON_TRUSTY_PDPT_ENTRIES; j++) { - sanitize_pte_entry((uint64_t *)pdpt_addr + j); - } - free_ept_mem((uint64_t *)vm->arch_vm.sworld_eptp); + /* sanitize trusty ept page-structures */ + sanitize_pte((uint64_t *)vm->arch_vm.sworld_eptp); vm->arch_vm.sworld_eptp = NULL; } diff --git a/hypervisor/common/hypercall.c b/hypervisor/common/hypercall.c index 54c4d4ab8..46fd4345e 100644 --- a/hypervisor/common/hypercall.c +++ b/hypervisor/common/hypercall.c @@ -356,7 +356,7 @@ int32_t hcall_notify_ioreq_finish(uint16_t vmid, uint16_t vcpu_id) static int32_t local_set_vm_memory_region(struct vm *vm, struct vm *target_vm, const struct vm_memory_region *region) { - uint64_t hpa, base_paddr; + uint64_t hpa, base_paddr, gpa_end; uint64_t prot; uint64_t *pml4_page; @@ -366,6 +366,14 @@ static int32_t local_set_vm_memory_region(struct vm *vm, return -EINVAL; } + gpa_end = region->gpa + region->size; + if ((gpa_end > vm->arch_vm.ept_mem_ops.info->ept.top_address_space) && + (region->gpa < TRUSTY_EPT_REBASE_GPA)) { + pr_err("%s, invalid gpa: 0x%llx, size: 0x%llx, top_address_space: 0x%llx", __func__, + region->gpa, region->size, vm->arch_vm.ept_mem_ops.info->ept.top_address_space); + return -EINVAL; + } + dev_dbg(ACRN_DBG_HYCALL, "[vm%d] type=%d gpa=0x%x vm0_gpa=0x%x size=0x%x", target_vm->vm_id, region->type, region->gpa, diff --git a/hypervisor/dm/vpci/pci_pt.c b/hypervisor/dm/vpci/pci_pt.c index c1e312069..fb0bdd304 100644 --- a/hypervisor/dm/vpci/pci_pt.c +++ b/hypervisor/dm/vpci/pci_pt.c @@ -68,7 +68,7 @@ static int vdev_pt_init(struct pci_vdev *vdev) /* Create an iommu domain for target VM if not created */ if (vm->iommu == NULL) { if (vm->arch_vm.nworld_eptp == 0UL) { - vm->arch_vm.nworld_eptp = alloc_paging_struct(); + vm->arch_vm.nworld_eptp = vm->arch_vm.ept_mem_ops.get_pml4_page(vm->arch_vm.ept_mem_ops.info, 0UL); sanitize_pte((uint64_t *)vm->arch_vm.nworld_eptp); } vm->iommu = create_iommu_domain(vm->vm_id, diff --git a/hypervisor/include/arch/x86/mmu.h b/hypervisor/include/arch/x86/mmu.h index 589dff433..f5b19364c 100644 --- a/hypervisor/include/arch/x86/mmu.h +++ b/hypervisor/include/arch/x86/mmu.h @@ -35,6 +35,7 @@ #ifndef ASSEMBLER #include +#include /* Define cache line size (in bytes) */ #define CACHE_LINE_SIZE 64U @@ -53,11 +54,6 @@ static inline uint64_t round_page_down(uint64_t addr) return (addr & CPU_PAGE_MASK); } -enum _page_table_type { - PTT_PRIMARY = 0, /* Mapping for hypervisor */ - PTT_EPT = 1, -}; - /* Represent the 4 levels of translation tables in IA-32e paging mode */ enum _page_table_level { IA32E_PML4 = 0, @@ -83,13 +79,10 @@ void free_paging_struct(void *ptr); void enable_paging(uint64_t pml4_base_addr); void enable_smep(void); void init_paging(void); -void mmu_add(uint64_t *pml4_page, uint64_t paddr_base, - uint64_t vaddr_base, uint64_t size, - uint64_t prot, enum _page_table_type ptt); -void mmu_modify_or_del(uint64_t *pml4_page, - uint64_t vaddr_base, uint64_t size, - uint64_t prot_set, uint64_t prot_clr, - enum _page_table_type ptt, uint32_t type); +void mmu_add(uint64_t *pml4_page, uint64_t paddr_base, uint64_t vaddr_base, + uint64_t size, uint64_t prot, const struct memory_ops *mem_ops); +void mmu_modify_or_del(uint64_t *pml4_page, uint64_t vaddr_base, uint64_t size, + uint64_t prot_set, uint64_t prot_clr, const struct memory_ops *mem_ops, uint32_t type); int check_vmx_mmu_cap(void); uint16_t allocate_vpid(void); void flush_vpid_single(uint16_t vpid); @@ -100,7 +93,7 @@ bool check_continuous_hpa(struct vm *vm, uint64_t gpa_arg, uint64_t size_arg); *@pre (pml4_page != NULL) && (pg_size != NULL) */ uint64_t *lookup_address(uint64_t *pml4_page, uint64_t addr, - uint64_t *pg_size, enum _page_table_type ptt); + uint64_t *pg_size, const struct memory_ops *mem_ops); #pragma pack(1) @@ -166,7 +159,6 @@ void ept_mr_modify(struct vm *vm, uint64_t *pml4_page, uint64_t gpa, */ void ept_mr_del(struct vm *vm, uint64_t *pml4_page, uint64_t gpa, uint64_t size); -void free_ept_mem(uint64_t *pml4_page); int ept_violation_vmexit_handler(struct vcpu *vcpu); int ept_misconfig_vmexit_handler(__unused struct vcpu *vcpu); diff --git a/hypervisor/include/arch/x86/page.h b/hypervisor/include/arch/x86/page.h index 838440523..922960a8b 100644 --- a/hypervisor/include/arch/x86/page.h +++ b/hypervisor/include/arch/x86/page.h @@ -13,6 +13,21 @@ /* size of the low MMIO address space: 2GB */ #define PLATFORM_LO_MMIO_SIZE 0x80000000UL +#define PML4_PAGE_NUM(size) 1UL +#define PDPT_PAGE_NUM(size) (((size) + PML4E_SIZE - 1UL) >> PML4E_SHIFT) +#define PD_PAGE_NUM(size) (((size) + PDPTE_SIZE - 1UL) >> PDPTE_SHIFT) +#define PT_PAGE_NUM(size) (((size) + PDE_SIZE - 1UL) >> PDE_SHIFT) + +/* The size of the guest physical address space, covered by the EPT page table of a VM */ +#define EPT_ADDRESS_SPACE(size) ((size != 0UL) ? (size + PLATFORM_LO_MMIO_SIZE) : 0UL) + +#define TRUSTY_PML4_PAGE_NUM(size) (1UL) +#define TRUSTY_PDPT_PAGE_NUM(size) (1UL) +#define TRUSTY_PD_PAGE_NUM(size) (PD_PAGE_NUM(size)) +#define TRUSTY_PT_PAGE_NUM(size) (PT_PAGE_NUM(size)) +#define TRUSTY_PGTABLE_PAGE_NUM(size) \ +(TRUSTY_PML4_PAGE_NUM(size) + TRUSTY_PDPT_PAGE_NUM(size) + TRUSTY_PD_PAGE_NUM(size) + TRUSTY_PT_PAGE_NUM(size)) + struct page { uint8_t contents[PAGE_SIZE]; } __aligned(PAGE_SIZE); diff --git a/hypervisor/include/arch/x86/pgtable.h b/hypervisor/include/arch/x86/pgtable.h index 950f79467..32de62191 100644 --- a/hypervisor/include/arch/x86/pgtable.h +++ b/hypervisor/include/arch/x86/pgtable.h @@ -162,9 +162,4 @@ static inline uint64_t pdpte_large(uint64_t pdpte) return pdpte & PAGE_PSE; } -static inline uint64_t pgentry_present(enum _page_table_type ptt, uint64_t pte) -{ - return (ptt == PTT_PRIMARY) ? (pte & PAGE_PRESENT) : (pte & EPT_RWX); -} - #endif /* PGTABLE_H */