hv: mmu: replace dynamic memory allocation in memory

Replace dynamic memory allocation in memory management with static memory allocation.
Since the static memory allocation can guarantee the allocation never failed, so
split_large_page and construct_pgentry don't need to return a errno any more.
Besides, the destroy_ept don't need to free page any more. Instead, it would memset
the eptp to 0 to make sure we can't walk this paging table again.

Tracked-On: #861
Signed-off-by: Li, Fei1 <fei1.li@intel.com>

hypervisor/arch/x86/Kconfig

@@ -222,7 +222,7 @@ config HV_RAM_START
 
 config HV_RAM_SIZE
 	hex "Size of the RAM region used by the hypervisor"
-	default 0x04000000
+	default 0x06000000
 	help
 	  A 64-bit integer indicating the size of RAM used by the hypervisor.
 	  It is ensured at link time that the footprint of the hypervisor

hypervisor/arch/x86/ept.c

@@ -10,55 +10,18 @@
 
 #define ACRN_DBG_EPT	6U
 
-/**
- * @pre pml4_addr != NULL
- */
-void free_ept_mem(uint64_t *pml4_page)
-{
-	uint64_t *pdpt_page, *pd_page, *pt_page;
-	uint64_t *pml4e, *pdpte, *pde;
-	uint64_t pml4e_idx, pdpte_idx, pde_idx;
-
-	for (pml4e_idx = 0U; pml4e_idx < PTRS_PER_PML4E; pml4e_idx++) {
-		pml4e = pml4_page + pml4e_idx;
-		if (pgentry_present(PTT_EPT, *pml4e) == 0UL) {
-			continue;
-		}
-		pdpt_page = pml4e_page_vaddr(*pml4e);
-
-		for (pdpte_idx = 0U; pdpte_idx < PTRS_PER_PDPTE; pdpte_idx++) {
-			pdpte = pdpt_page + pdpte_idx;
-			if ((pgentry_present(PTT_EPT, *pdpte) == 0UL) ||
-					pdpte_large(*pdpte) != 0UL) {
-				continue;
-			}
-			pd_page = pdpte_page_vaddr(*pdpte);
-
-			for (pde_idx = 0U; pde_idx < PTRS_PER_PDE; pde_idx++) {
-				pde = pd_page + pde_idx;
-				if ((pgentry_present(PTT_EPT, *pde) == 0UL) ||
-						pde_large(*pde) != 0UL) {
-					continue;
-				}
-				pt_page = pde_page_vaddr(*pde);
-
-				/* Free page table entry table */
-				free_paging_struct((void *)pt_page);
-			}
-			/* Free page directory entry table */
-			free_paging_struct((void *)pd_page);
-		}
-		free_paging_struct((void *)pdpt_page);
-	}
-	free_paging_struct((void *)pml4_page);
-}
-
 void destroy_ept(struct vm *vm)
 {
+	/* Destroy secure world */
+	if (vm->sworld_control.flag.active != 0UL) {
+		destroy_secure_world(vm, true);
+	}
+
 	if (vm->arch_vm.nworld_eptp != NULL) {
-		free_ept_mem((uint64_t *)vm->arch_vm.nworld_eptp);
+		(void)memset(vm->arch_vm.nworld_eptp, 0U, CPU_PAGE_SIZE);
 	}
 }
+
 /* using return value INVALID_HPA as error code */
 uint64_t local_gpa2hpa(struct vm *vm, uint64_t gpa, uint32_t *size)
 {
@@ -73,7 +36,7 @@ uint64_t local_gpa2hpa(struct vm *vm, uint64_t gpa, uint32_t *size)
 		eptp = vm->arch_vm.nworld_eptp;
 	}
 
-	pgentry = lookup_address((uint64_t *)eptp, gpa, &pg_size, PTT_EPT);
+	pgentry = lookup_address((uint64_t *)eptp, gpa, &pg_size, &vm->arch_vm.ept_mem_ops);
 	if (pgentry != NULL) {
 		hpa = ((*pgentry & (~(pg_size - 1UL)))
 				| (gpa & (pg_size - 1UL)));
@@ -222,9 +185,8 @@ void ept_mr_add(struct vm *vm, uint64_t *pml4_page,
 	struct vcpu *vcpu;
 	uint64_t prot = prot_orig;
 
-	dev_dbg(ACRN_DBG_EPT, "%s, vm[%d] hpa: 0x%016llx gpa: 0x%016llx ",
+	dev_dbg(ACRN_DBG_EPT, "%s, vm[%d] hpa: 0x%016llx gpa: 0x%016llx size: 0x%016llx prot: 0x%016x\n",
-			__func__, vm->vm_id, hpa, gpa);
+			__func__, vm->vm_id, hpa, gpa, size, prot);
-	dev_dbg(ACRN_DBG_EPT, "size: 0x%016llx prot: 0x%016x\n", size, prot);
 
 	/* EPT & VT-d share the same page tables, set SNP bit
 	 * to force snooping of PCIe devices if the page
@@ -234,7 +196,7 @@ void ept_mr_add(struct vm *vm, uint64_t *pml4_page,
 		prot |= EPT_SNOOP_CTRL;
 	}
 
-	mmu_add(pml4_page, hpa, gpa, size, prot, PTT_EPT);
+	mmu_add(pml4_page, hpa, gpa, size, prot, &vm->arch_vm.ept_mem_ops);
 
 	foreach_vcpu(i, vm, vcpu) {
 		vcpu_make_request(vcpu, ACRN_REQUEST_EPT_FLUSH);
@@ -248,8 +210,9 @@ void ept_mr_modify(struct vm *vm, uint64_t *pml4_page,
 	struct vcpu *vcpu;
 	uint16_t i;
 
-	mmu_modify_or_del(pml4_page, gpa, size,
+	dev_dbg(ACRN_DBG_EPT, "%s,vm[%d] gpa 0x%llx size 0x%llx\n", __func__, vm->vm_id, gpa, size);
-			prot_set, prot_clr, PTT_EPT, MR_MODIFY);
+
+	mmu_modify_or_del(pml4_page, gpa, size, prot_set, prot_clr, &vm->arch_vm.ept_mem_ops, MR_MODIFY);
 
 	foreach_vcpu(i, vm, vcpu) {
 		vcpu_make_request(vcpu, ACRN_REQUEST_EPT_FLUSH);
@@ -258,17 +221,14 @@ void ept_mr_modify(struct vm *vm, uint64_t *pml4_page,
 /**
  * @pre [gpa,gpa+size) has been mapped into host physical memory region
  */
-void ept_mr_del(struct vm *vm, uint64_t *pml4_page,
+void ept_mr_del(struct vm *vm, uint64_t *pml4_page, uint64_t gpa, uint64_t size)
-		uint64_t gpa, uint64_t size)
 {
 	struct vcpu *vcpu;
 	uint16_t i;
 
-	dev_dbg(ACRN_DBG_EPT, "%s,vm[%d] gpa 0x%llx size 0x%llx\n",
+	dev_dbg(ACRN_DBG_EPT, "%s,vm[%d] gpa 0x%llx size 0x%llx\n", __func__, vm->vm_id, gpa, size);
-			__func__, vm->vm_id, gpa, size);
 
-	mmu_modify_or_del(pml4_page, gpa, size,
+	mmu_modify_or_del(pml4_page, gpa, size, 0UL, 0UL, &vm->arch_vm.ept_mem_ops, MR_DEL);
-			0UL, 0UL, PTT_EPT, MR_DEL);
 
 	foreach_vcpu(i, vm, vcpu) {
 		vcpu_make_request(vcpu, ACRN_REQUEST_EPT_FLUSH);

hypervisor/arch/x86/guest/guest.c

@@ -619,6 +619,10 @@ int prepare_vm0_memmap_and_e820(struct vm *vm)
 		"vm0: bottom memory - 0x%llx, top memory - 0x%llx\n",
 		e820_mem.mem_bottom, e820_mem.mem_top);
 
+	if (e820_mem.mem_top > EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE)) {
+		panic("Please configure VM0_ADDRESS_SPACE correctly!\n");
+	}
+
 	/* create real ept map for all ranges with UC */
 	ept_mr_add(vm, pml4_page,
 			e820_mem.mem_bottom, e820_mem.mem_bottom,

hypervisor/arch/x86/guest/vm.c

@@ -87,12 +87,8 @@ int create_vm(struct vm_description *vm_desc, struct vm **rtn_vm)
 	/* gpa_lowtop are used for system start up */
 	vm->hw.gpa_lowtop = 0UL;
 
-	vm->arch_vm.nworld_eptp = alloc_paging_struct();
+	init_ept_mem_ops(vm);
-	if (vm->arch_vm.nworld_eptp == NULL) {
+	vm->arch_vm.nworld_eptp = vm->arch_vm.ept_mem_ops.get_pml4_page(vm->arch_vm.ept_mem_ops.info, 0UL);
-		pr_fatal("%s, alloc memory for EPTP failed\n", __func__);
-		status = -ENOMEM;
-		goto err;
-	}
 	sanitize_pte((uint64_t *)vm->arch_vm.nworld_eptp);
 
 	/* Only for SOS: Configure VM software information */
@@ -180,7 +176,7 @@ err:
 	vioapic_cleanup(vm_ioapic(vm));
 
 	if (vm->arch_vm.nworld_eptp != NULL) {
-		free(vm->arch_vm.nworld_eptp);
+		(void)memset(vm->arch_vm.nworld_eptp, 0U, CPU_PAGE_SIZE);
 	}
 
 	return status;
@@ -212,10 +208,6 @@ int shutdown_vm(struct vm *vm)
 	/* cleanup vioapic */
 	vioapic_cleanup(vm_ioapic(vm));
 
-	/* Destroy secure world */
-	if (vm->sworld_control.flag.active != 0UL) {
-		destroy_secure_world(vm, true);
-	}
 	/* Free EPT allocated resources assigned to VM */
 	destroy_ept(vm);
 

hypervisor/arch/x86/mmu.c

@@ -238,39 +238,57 @@ void init_paging(void)
 	struct e820_entry *entry;
 	uint64_t hv_hpa;
 	uint32_t i;
+	uint64_t low32_max_ram = 0UL;
+	uint64_t high64_max_ram;
 	uint64_t attr_uc = (PAGE_TABLE | PAGE_CACHE_UC);
 
 	pr_dbg("HV MMU Initialization");
 
 	/* Allocate memory for Hypervisor PML4 table */
-	mmu_pml4_addr = alloc_paging_struct();
+	mmu_pml4_addr = ppt_mem_ops.get_pml4_page(ppt_mem_ops.info, 0UL);
 
 	init_e820();
 	obtain_e820_mem_info();
 
+	/* align to 2MB */
+	high64_max_ram = (e820_mem.mem_top + PDE_SIZE - 1UL) & PDE_MASK;
+
+	if (high64_max_ram > (CONFIG_PLATFORM_RAM_SIZE + PLATFORM_LO_MMIO_SIZE) ||
+			high64_max_ram < (1UL << 32U)) {
+		panic("Please configure HV_ADDRESS_SPACE correctly!\n");
+	}
+
 	/* Map all memory regions to UC attribute */
-	mmu_add((uint64_t *)mmu_pml4_addr, e820_mem.mem_bottom,
+	mmu_add((uint64_t *)mmu_pml4_addr, e820_mem.mem_bottom, e820_mem.mem_bottom,
-		e820_mem.mem_bottom, e820_mem.mem_top - e820_mem.mem_bottom,
+		high64_max_ram - e820_mem.mem_bottom, attr_uc, &ppt_mem_ops);
-		attr_uc, PTT_PRIMARY);
 
 	/* Modify WB attribute for E820_TYPE_RAM */
 	for (i = 0U; i < e820_entries; i++) {
 		entry = &e820[i];
 		if (entry->type == E820_TYPE_RAM) {
-			mmu_modify_or_del((uint64_t *)mmu_pml4_addr,
+			if (entry->baseaddr < (1UL << 32U)) {
-					entry->baseaddr, entry->length,
+				uint64_t end = entry->baseaddr + entry->length;
-					PAGE_CACHE_WB, PAGE_CACHE_MASK,
+				if (end < (1UL << 32U) && (end > low32_max_ram)) {
-					PTT_PRIMARY, MR_MODIFY);
+					low32_max_ram = end;
+				}
+			}
 		}
 	}
 
+	mmu_modify_or_del((uint64_t *)mmu_pml4_addr, 0UL, (low32_max_ram + PDE_SIZE - 1UL) & PDE_MASK,
+			PAGE_CACHE_WB, PAGE_CACHE_MASK, &ppt_mem_ops, MR_MODIFY);
+
+	mmu_modify_or_del((uint64_t *)mmu_pml4_addr, (1UL << 32U), high64_max_ram - (1UL << 32U),
+			PAGE_CACHE_WB, PAGE_CACHE_MASK, &ppt_mem_ops, MR_MODIFY);
+
 	/* set the paging-structure entries' U/S flag
 	 * to supervisor-mode for hypervisor owned memroy.
 	 */
 	hv_hpa = get_hv_image_base();
-	mmu_modify_or_del((uint64_t *)mmu_pml4_addr, hv_hpa, CONFIG_HV_RAM_SIZE,
+	mmu_modify_or_del((uint64_t *)mmu_pml4_addr, hv_hpa & PDE_MASK,
+		CONFIG_HV_RAM_SIZE + ((hv_hpa & (PDE_SIZE - 1UL)) != 0UL) ? PDE_SIZE : 0UL,
 			PAGE_CACHE_WB, PAGE_CACHE_MASK | PAGE_USER,
-			PTT_PRIMARY, MR_MODIFY);
+			&ppt_mem_ops, MR_MODIFY);
 
 	/* Enable paging */
 	enable_paging(hva2hpa(mmu_pml4_addr));

hypervisor/arch/x86/page.c

@@ -5,11 +5,6 @@
  */
 #include <hypervisor.h>
 
-#define PML4_PAGE_NUM(size)	1UL
-#define PDPT_PAGE_NUM(size)	(((size) + PML4E_SIZE - 1UL) >> PML4E_SHIFT)
-#define PD_PAGE_NUM(size)	(((size) + PDPTE_SIZE - 1UL) >> PDPTE_SHIFT)
-#define PT_PAGE_NUM(size)	(((size) + PDE_SIZE - 1UL) >> PDE_SHIFT)
-
 #define DEFINE_PGTABLE_PAGE(prefix, lvl, LVL, size)	\
 		static struct page prefix ## lvl ## _pages[LVL ## _PAGE_NUM(size)]
 
@@ -66,8 +61,6 @@ const struct memory_ops ppt_mem_ops = {
 	.get_pd_page = ppt_get_pd_page,
 };
 
-/* The size of the guest physical address space, covered by the EPT page table of a VM */
-#define EPT_ADDRESS_SPACE(size)	((size != 0UL) ? (size + PLATFORM_LO_MMIO_SIZE) : 0UL)
 DEFINE_PGTABLE_PAGE(vm0_, pml4, PML4, EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE));
 DEFINE_PGTABLE_PAGE(vm0_, pdpt, PDPT, EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE));
 DEFINE_PGTABLE_PAGE(vm0_, pd, PD, EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE));
@@ -79,13 +72,6 @@ static struct page uos_nworld_pdpt_pages[CONFIG_MAX_VM_NUM - 1U][PDPT_PAGE_NUM(E
 static struct page uos_nworld_pd_pages[CONFIG_MAX_VM_NUM - 1U][PD_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
 static struct page uos_nworld_pt_pages[CONFIG_MAX_VM_NUM - 1U][PT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
 
-#define TRUSTY_PML4_PAGE_NUM(size)	(1UL)
-#define TRUSTY_PDPT_PAGE_NUM(size)	(1UL)
-#define TRUSTY_PD_PAGE_NUM(size)	(PD_PAGE_NUM(size))
-#define TRUSTY_PT_PAGE_NUM(size)	(PT_PAGE_NUM(size))
-#define TRUSTY_PGTABLE_PAGE_NUM(size)	\
-(TRUSTY_PML4_PAGE_NUM(size) + TRUSTY_PDPT_PAGE_NUM(size) + TRUSTY_PD_PAGE_NUM(size) + TRUSTY_PT_PAGE_NUM(size))
-
 static struct page uos_sworld_pgtable_pages[CONFIG_MAX_VM_NUM - 1U][TRUSTY_PGTABLE_PAGE_NUM(TRUSTY_RAM_SIZE)];
 
 /* ept: extended page table*/

hypervisor/arch/x86/pagetable.c

@@ -10,11 +10,9 @@
 /*
  * Split a large page table into next level page table.
  */
-static int split_large_page(uint64_t *pte,
+static void split_large_page(uint64_t *pte, enum _page_table_level level,
-			enum _page_table_level level,
+		uint64_t vaddr, const struct memory_ops *mem_ops)
-			enum _page_table_type ptt)
 {
-	int ret = -EINVAL;
 	uint64_t *pbase;
 	uint64_t ref_paddr, paddr, paddrinc;
 	uint64_t i, ref_prot;
@@ -24,23 +22,20 @@ static int split_large_page(uint64_t *pte,
 		ref_paddr = (*pte) & PDPTE_PFN_MASK;
 		paddrinc = PDE_SIZE;
 		ref_prot = (*pte) & ~PDPTE_PFN_MASK;
+		pbase = (uint64_t *)mem_ops->get_pd_page(mem_ops->info, vaddr);
 		break;
 	case IA32E_PD:
 		ref_paddr = (*pte) & PDE_PFN_MASK;
 		paddrinc = PTE_SIZE;
 		ref_prot = (*pte) & ~PDE_PFN_MASK;
 		ref_prot &= ~PAGE_PSE;
+		pbase = (uint64_t *)mem_ops->get_pt_page(mem_ops->info, vaddr);
 		break;
 	default:
-		return ret;
+		panic("invalid paging table level: %d", level);
 	}
 
-	dev_dbg(ACRN_DBG_MMU, "%s, paddr: 0x%llx\n", __func__, ref_paddr);
+	dev_dbg(ACRN_DBG_MMU, "%s, paddr: 0x%llx, pbase: 0x%llx\n", __func__, ref_paddr, pbase);
-
-	pbase = (uint64_t *)alloc_paging_struct();
-	if (pbase == NULL) {
-		return -ENOMEM;
-	}
 
 	paddr = ref_paddr;
 	for (i = 0UL; i < PTRS_PER_PTE; i++) {
@@ -48,12 +43,10 @@ static int split_large_page(uint64_t *pte,
 		paddr += paddrinc;
 	}
 
-	ref_prot = (ptt == PTT_PRIMARY) ? PAGE_TABLE : EPT_RWX;
+	ref_prot = mem_ops->get_default_access_right();
 	set_pgentry(pte, hva2hpa((void *)pbase) | ref_prot);
 
 	/* TODO: flush the TLB */
-
-	return 0;
 }
 
 static inline void local_modify_or_del_pte(uint64_t *pte,
@@ -72,19 +65,11 @@ static inline void local_modify_or_del_pte(uint64_t *pte,
 /*
  * pgentry may means pml4e/pdpte/pde
  */
-static inline int construct_pgentry(enum _page_table_type ptt, uint64_t *pde)
+static inline void construct_pgentry(uint64_t *pde, void *pd_page, uint64_t prot)
 {
-	uint64_t prot;
-	void *pd_page = alloc_paging_struct();
-	if (pd_page == NULL) {
-		return -ENOMEM;
-	}
-
 	sanitize_pte((uint64_t *)pd_page);
 
-	prot = (ptt == PTT_PRIMARY) ? PAGE_TABLE: EPT_RWX;
 	set_pgentry(pde, hva2hpa(pd_page) | prot);
-	return 0;
 }
 
 /*
@@ -94,21 +79,18 @@ static inline int construct_pgentry(enum _page_table_type ptt, uint64_t *pde)
  * type: MR_DEL
  * delete [vaddr_start, vaddr_end) MT PT mapping
  */
-static void modify_or_del_pte(const uint64_t *pde,
+static void modify_or_del_pte(const uint64_t *pde, uint64_t vaddr_start, uint64_t vaddr_end,
-		uint64_t vaddr_start, uint64_t vaddr_end,
+		uint64_t prot_set, uint64_t prot_clr, const struct memory_ops *mem_ops, uint32_t type)
-		uint64_t prot_set, uint64_t prot_clr,
-		enum _page_table_type ptt, uint32_t type)
 {
 	uint64_t *pt_page = pde_page_vaddr(*pde);
 	uint64_t vaddr = vaddr_start;
 	uint64_t index = pte_index(vaddr);
 
-	dev_dbg(ACRN_DBG_MMU, "%s, vaddr: [0x%llx - 0x%llx]\n",
+	dev_dbg(ACRN_DBG_MMU, "%s, vaddr: [0x%llx - 0x%llx]\n", __func__, vaddr, vaddr_end);
-		__func__, vaddr, vaddr_end);
 	for (; index < PTRS_PER_PTE; index++) {
 		uint64_t *pte = pt_page + index;
 
-		if (pgentry_present(ptt, *pte) == 0UL) {
+		if (mem_ops->pgentry_present(*pte) == 0UL) {
 			panic("invalid op, pte not present");
 		}
 
@@ -127,34 +109,26 @@ static void modify_or_del_pte(const uint64_t *pde,
  * type: MR_DEL
  * delete [vaddr_start, vaddr_end) MT PT mapping
  */
-static void modify_or_del_pde(const uint64_t *pdpte,
+static void modify_or_del_pde(const uint64_t *pdpte, uint64_t vaddr_start, uint64_t vaddr_end,
-		uint64_t vaddr_start, uint64_t vaddr_end,
+		uint64_t prot_set, uint64_t prot_clr, const struct memory_ops *mem_ops, uint32_t type)
-		uint64_t prot_set, uint64_t prot_clr,
-		enum _page_table_type ptt, uint32_t type)
 {
 	uint64_t *pd_page = pdpte_page_vaddr(*pdpte);
 	uint64_t vaddr = vaddr_start;
 	uint64_t index = pde_index(vaddr);
 
-	dev_dbg(ACRN_DBG_MMU, "%s, vaddr: [0x%llx - 0x%llx]\n",
+	dev_dbg(ACRN_DBG_MMU, "%s, vaddr: [0x%llx - 0x%llx]\n", __func__, vaddr, vaddr_end);
-		__func__, vaddr, vaddr_end);
 	for (; index < PTRS_PER_PDE; index++) {
 		uint64_t *pde = pd_page + index;
 		uint64_t vaddr_next = (vaddr & PDE_MASK) + PDE_SIZE;
 
-		if (pgentry_present(ptt, *pde) == 0UL) {
+		if (mem_ops->pgentry_present(*pde) == 0UL) {
 			panic("invalid op, pde not present");
 		}
 		if (pde_large(*pde) != 0UL) {
-			if (vaddr_next > vaddr_end ||
+			if (vaddr_next > vaddr_end || !mem_aligned_check(vaddr, PDE_SIZE)) {
-					!mem_aligned_check(vaddr, PDE_SIZE)) {
+				split_large_page(pde, IA32E_PD, vaddr, mem_ops);
-				int ret = split_large_page(pde, IA32E_PD, ptt);
-				if (ret != 0) {
-					panic("split large PDE failed");
-				}
 			} else {
-				local_modify_or_del_pte(pde,
+				local_modify_or_del_pte(pde, prot_set, prot_clr, type);
-					prot_set, prot_clr, type);
 				if (vaddr_next < vaddr_end) {
 					vaddr = vaddr_next;
 					continue;
@@ -162,8 +136,7 @@ static void modify_or_del_pde(const uint64_t *pdpte,
 				break;	/* done */
 			}
 		}
-		modify_or_del_pte(pde, vaddr, vaddr_end,
+		modify_or_del_pte(pde, vaddr, vaddr_end, prot_set, prot_clr, mem_ops, type);
-				prot_set, prot_clr, ptt, type);
 		if (vaddr_next >= vaddr_end) {
 			break;	/* done */
 		}
@@ -178,34 +151,27 @@ static void modify_or_del_pde(const uint64_t *pdpte,
  * type: MR_DEL
  * delete [vaddr_start, vaddr_end) MT PT mapping
  */
-static void modify_or_del_pdpte(const uint64_t *pml4e,
+static void modify_or_del_pdpte(const uint64_t *pml4e, uint64_t vaddr_start, uint64_t vaddr_end,
-		uint64_t vaddr_start, uint64_t vaddr_end,
+		uint64_t prot_set, uint64_t prot_clr, const struct memory_ops *mem_ops, uint32_t type)
-		uint64_t prot_set, uint64_t prot_clr,
-		enum _page_table_type ptt, uint32_t type)
 {
 	uint64_t *pdpt_page = pml4e_page_vaddr(*pml4e);
 	uint64_t vaddr = vaddr_start;
 	uint64_t index = pdpte_index(vaddr);
 
-	dev_dbg(ACRN_DBG_MMU, "%s, vaddr: [0x%llx - 0x%llx]\n",
+	dev_dbg(ACRN_DBG_MMU, "%s, vaddr: [0x%llx - 0x%llx]\n", __func__, vaddr, vaddr_end);
-		__func__, vaddr, vaddr_end);
 	for (; index < PTRS_PER_PDPTE; index++) {
 		uint64_t *pdpte = pdpt_page + index;
 		uint64_t vaddr_next = (vaddr & PDPTE_MASK) + PDPTE_SIZE;
 
-		if (pgentry_present(ptt, *pdpte) == 0UL) {
+		if (mem_ops->pgentry_present(*pdpte) == 0UL) {
 			panic("invalid op, pdpte not present");
 		}
 		if (pdpte_large(*pdpte) != 0UL) {
 			if (vaddr_next > vaddr_end ||
 					!mem_aligned_check(vaddr, PDPTE_SIZE)) {
-				int ret = split_large_page(pdpte, IA32E_PDPT, ptt);
+				split_large_page(pdpte, IA32E_PDPT, vaddr, mem_ops);
-				if (ret != 0) {
-					panic("split large PDPTE failed");
-				}
 			} else {
-				local_modify_or_del_pte(pdpte,
+				local_modify_or_del_pte(pdpte, prot_set, prot_clr, type);
-					prot_set, prot_clr, type);
 				if (vaddr_next < vaddr_end) {
 					vaddr = vaddr_next;
 					continue;
@@ -213,8 +179,7 @@ static void modify_or_del_pdpte(const uint64_t *pml4e,
 				break;	/* done */
 			}
 		}
-		modify_or_del_pde(pdpte, vaddr, vaddr_end,
+		modify_or_del_pde(pdpte, vaddr, vaddr_end, prot_set, prot_clr, mem_ops, type);
-				prot_set, prot_clr, ptt, type);
 		if (vaddr_next >= vaddr_end) {
 			break;	/* done */
 		}
@@ -235,10 +200,8 @@ static void modify_or_del_pdpte(const uint64_t *pml4e,
  * type: MR_DEL
  * delete [vaddr_base, vaddr_base + size ) memory region page table mapping.
  */
-void mmu_modify_or_del(uint64_t *pml4_page,
+void mmu_modify_or_del(uint64_t *pml4_page, uint64_t vaddr_base, uint64_t size,
-		uint64_t vaddr_base, uint64_t size,
+		uint64_t prot_set, uint64_t prot_clr, const struct memory_ops *mem_ops, uint32_t type)
-		uint64_t prot_set, uint64_t prot_clr,
-		enum _page_table_type ptt, uint32_t type)
 {
 	uint64_t vaddr = round_page_up(vaddr_base);
 	uint64_t vaddr_next, vaddr_end;
@@ -251,11 +214,10 @@ void mmu_modify_or_del(uint64_t *pml4_page,
 	while (vaddr < vaddr_end) {
 		vaddr_next = (vaddr & PML4E_MASK) + PML4E_SIZE;
 		pml4e = pml4e_offset(pml4_page, vaddr);
-		if (pgentry_present(ptt, *pml4e) == 0UL) {
+		if (mem_ops->pgentry_present(*pml4e) == 0UL) {
 			panic("invalid op, pml4e not present");
 		}
-		modify_or_del_pdpte(pml4e, vaddr, vaddr_end,
+		modify_or_del_pdpte(pml4e, vaddr, vaddr_end, prot_set, prot_clr, mem_ops, type);
-					prot_set, prot_clr, ptt, type);
 		vaddr = vaddr_next;
 	}
 }
@@ -264,9 +226,8 @@ void mmu_modify_or_del(uint64_t *pml4_page,
  * In PT level,
  * add [vaddr_start, vaddr_end) to [paddr_base, ...) MT PT mapping
  */
-static void add_pte(const uint64_t *pde, uint64_t paddr_start,
+static void add_pte(const uint64_t *pde, uint64_t paddr_start, uint64_t vaddr_start, uint64_t vaddr_end,
-		uint64_t vaddr_start, uint64_t vaddr_end,
+		uint64_t prot, const struct memory_ops *mem_ops)
-		uint64_t prot, enum _page_table_type ptt)
 {
 	uint64_t *pt_page = pde_page_vaddr(*pde);
 	uint64_t vaddr = vaddr_start;
@@ -278,7 +239,7 @@ static void add_pte(const uint64_t *pde, uint64_t paddr_start,
 	for (; index < PTRS_PER_PTE; index++) {
 		uint64_t *pte = pt_page + index;
 
-		if (pgentry_present(ptt, *pte) != 0UL) {
+		if (mem_ops->pgentry_present(*pte) != 0UL) {
 			panic("invalid op, pte present");
 		}
 
@@ -296,9 +257,8 @@ static void add_pte(const uint64_t *pde, uint64_t paddr_start,
  * In PD level,
  * add [vaddr_start, vaddr_end) to [paddr_base, ...) MT PT mapping
  */
-static void add_pde(const uint64_t *pdpte, uint64_t paddr_start,
+static void add_pde(const uint64_t *pdpte, uint64_t paddr_start, uint64_t vaddr_start, uint64_t vaddr_end,
-		uint64_t vaddr_start, uint64_t vaddr_end,
+		uint64_t prot, const struct memory_ops *mem_ops)
-		uint64_t prot, enum _page_table_type ptt)
 {
 	uint64_t *pd_page = pdpte_page_vaddr(*pdpte);
 	uint64_t vaddr = vaddr_start;
@@ -311,7 +271,7 @@ static void add_pde(const uint64_t *pdpte, uint64_t paddr_start,
 		uint64_t *pde = pd_page + index;
 		uint64_t vaddr_next = (vaddr & PDE_MASK) + PDE_SIZE;
 
-		if (pgentry_present(ptt, *pde) == 0UL) {
+		if (mem_ops->pgentry_present(*pde) == 0UL) {
 			if (mem_aligned_check(paddr, PDE_SIZE) &&
 				mem_aligned_check(vaddr, PDE_SIZE) &&
 				(vaddr_next <= vaddr_end)) {
@@ -323,13 +283,11 @@ static void add_pde(const uint64_t *pdpte, uint64_t paddr_start,
 				}
 				break;	/* done */
 			} else {
-				int ret = construct_pgentry(ptt, pde);
+				void *pt_page = mem_ops->get_pt_page(mem_ops->info, vaddr);
-				if (ret != 0) {
+				construct_pgentry(pde, pt_page, mem_ops->get_default_access_right());
-					panic("construct pde page table fail");
-				}
 			}
 		}
-		add_pte(pde, paddr, vaddr, vaddr_end, prot, ptt);
+		add_pte(pde, paddr, vaddr, vaddr_end, prot, mem_ops);
 		if (vaddr_next >= vaddr_end) {
 			break;	/* done */
 		}
@@ -342,22 +300,20 @@ static void add_pde(const uint64_t *pdpte, uint64_t paddr_start,
  * In PDPT level,
  * add [vaddr_start, vaddr_end) to [paddr_base, ...) MT PT mapping
  */
-static void add_pdpte(const uint64_t *pml4e, uint64_t paddr_start,
+static void add_pdpte(const uint64_t *pml4e, uint64_t paddr_start, uint64_t vaddr_start, uint64_t vaddr_end,
-		uint64_t vaddr_start, uint64_t vaddr_end,
+		uint64_t prot, const struct memory_ops *mem_ops)
-		uint64_t prot, enum _page_table_type ptt)
 {
 	uint64_t *pdpt_page = pml4e_page_vaddr(*pml4e);
 	uint64_t vaddr = vaddr_start;
 	uint64_t paddr = paddr_start;
 	uint64_t index = pdpte_index(vaddr);
 
-	dev_dbg(ACRN_DBG_MMU, "%s, paddr: 0x%llx, vaddr: [0x%llx - 0x%llx]\n",
+	dev_dbg(ACRN_DBG_MMU, "%s, paddr: 0x%llx, vaddr: [0x%llx - 0x%llx]\n", __func__, paddr, vaddr, vaddr_end);
-		__func__, paddr, vaddr, vaddr_end);
 	for (; index < PTRS_PER_PDPTE; index++) {
 		uint64_t *pdpte = pdpt_page + index;
 		uint64_t vaddr_next = (vaddr & PDPTE_MASK) + PDPTE_SIZE;
 
-		if (pgentry_present(ptt, *pdpte) == 0UL) {
+		if (mem_ops->pgentry_present(*pdpte) == 0UL) {
 			if (mem_aligned_check(paddr, PDPTE_SIZE) &&
 				mem_aligned_check(vaddr, PDPTE_SIZE) &&
 				(vaddr_next <= vaddr_end)) {
@@ -369,13 +325,11 @@ static void add_pdpte(const uint64_t *pml4e, uint64_t paddr_start,
 				}
 				break;	/* done */
 			} else {
-				int ret = construct_pgentry(ptt, pdpte);
+				void *pd_page = mem_ops->get_pd_page(mem_ops->info, vaddr);
-				if (ret != 0) {
+				construct_pgentry(pdpte, pd_page, mem_ops->get_default_access_right());
-					panic("construct pdpte page table fail");
-				}
 			}
 		}
-		add_pde(pdpte, paddr, vaddr, vaddr_end, prot, ptt);
+		add_pde(pdpte, paddr, vaddr, vaddr_end, prot, mem_ops);
 		if (vaddr_next >= vaddr_end) {
 			break;	/* done */
 		}
@@ -389,16 +343,14 @@ static void add_pdpte(const uint64_t *pml4e, uint64_t paddr_start,
  * add [vaddr_base, vaddr_base + size ) memory region page table mapping.
  * @pre: the prot should set before call this function.
  */
-void mmu_add(uint64_t *pml4_page, uint64_t paddr_base,
+void mmu_add(uint64_t *pml4_page, uint64_t paddr_base, uint64_t vaddr_base, uint64_t size, uint64_t prot,
-		uint64_t vaddr_base, uint64_t size,
+		const struct memory_ops *mem_ops)
-		uint64_t prot, enum _page_table_type ptt)
 {
 	uint64_t vaddr, vaddr_next, vaddr_end;
 	uint64_t paddr;
 	uint64_t *pml4e;
 
-	dev_dbg(ACRN_DBG_MMU, "%s, paddr 0x%llx, vaddr 0x%llx, size 0x%llx\n",
+	dev_dbg(ACRN_DBG_MMU, "%s, paddr 0x%llx, vaddr 0x%llx, size 0x%llx\n", __func__, paddr_base, vaddr_base, size);
-		__func__, paddr_base, vaddr_base, size);
 
 	/* align address to page size*/
 	vaddr = round_page_up(vaddr_base);
@@ -408,13 +360,11 @@ void mmu_add(uint64_t *pml4_page, uint64_t paddr_base,
 	while (vaddr < vaddr_end) {
 		vaddr_next = (vaddr & PML4E_MASK) + PML4E_SIZE;
 		pml4e = pml4e_offset(pml4_page, vaddr);
-		if (pgentry_present(ptt, *pml4e) == 0UL) {
+		if (mem_ops->pgentry_present(*pml4e) == 0UL) {
-			int ret = construct_pgentry(ptt, pml4e);
+			void *pdpt_page = mem_ops->get_pdpt_page(mem_ops->info, vaddr);
-			if (ret != 0) {
+			construct_pgentry(pml4e, pdpt_page, mem_ops->get_default_access_right());
-				panic("construct pml4e page table fail");
-			}
 		}
-		add_pdpte(pml4e, paddr, vaddr, vaddr_end, prot, ptt);
+		add_pdpte(pml4e, paddr, vaddr, vaddr_end, prot, mem_ops);
 
 		paddr += (vaddr_next - vaddr);
 		vaddr = vaddr_next;
@@ -424,18 +374,17 @@ void mmu_add(uint64_t *pml4_page, uint64_t paddr_base,
 /**
  * @pre (pml4_page != NULL) && (pg_size != NULL)
  */
-uint64_t *lookup_address(uint64_t *pml4_page,
+uint64_t *lookup_address(uint64_t *pml4_page, uint64_t addr, uint64_t *pg_size, const struct memory_ops *mem_ops)
-		uint64_t addr, uint64_t *pg_size, enum _page_table_type ptt)
 {
 	uint64_t *pml4e, *pdpte, *pde, *pte;
 
 	pml4e = pml4e_offset(pml4_page, addr);
-	if (pgentry_present(ptt, *pml4e) == 0UL) {
+	if (mem_ops->pgentry_present(*pml4e) == 0UL) {
 		return NULL;
 	}
 
 	pdpte = pdpte_offset(pml4e, addr);
-	if (pgentry_present(ptt, *pdpte) == 0UL) {
+	if (mem_ops->pgentry_present(*pdpte) == 0UL) {
 		return NULL;
 	} else if (pdpte_large(*pdpte) != 0UL) {
 		*pg_size = PDPTE_SIZE;
@@ -443,7 +392,7 @@ uint64_t *lookup_address(uint64_t *pml4_page,
 	}
 
 	pde = pde_offset(pdpte, addr);
-	if (pgentry_present(ptt, *pde) == 0UL) {
+	if (mem_ops->pgentry_present(*pde) == 0UL) {
 		return NULL;
 	} else if (pde_large(*pde) != 0UL) {
 		*pg_size = PDE_SIZE;
@@ -451,7 +400,7 @@ uint64_t *lookup_address(uint64_t *pml4_page,
 	}
 
 	pte = pte_offset(pde, addr);
-	if (pgentry_present(ptt, *pte) == 0UL) {
+	if (mem_ops->pgentry_present(*pte) == 0UL) {
 		return NULL;
 	} else {
 		*pg_size = PTE_SIZE;

hypervisor/arch/x86/trusty.c

@@ -97,14 +97,14 @@ static void create_secure_world_ept(struct vm *vm, uint64_t gpa_orig,
 	 * Normal World.PD/PT are shared in both Secure world's EPT
 	 * and Normal World's EPT
 	 */
-	pml4_base = alloc_paging_struct();
+	pml4_base = vm->arch_vm.ept_mem_ops.get_pml4_page(vm->arch_vm.ept_mem_ops.info, TRUSTY_EPT_REBASE_GPA);
 	vm->arch_vm.sworld_eptp = pml4_base;
 	sanitize_pte((uint64_t *)vm->arch_vm.sworld_eptp);
 
 	/* The trusty memory is remapped to guest physical address
 	 * of gpa_rebased to gpa_rebased + size
 	 */
-	sub_table_addr = alloc_paging_struct();
+	sub_table_addr = vm->arch_vm.ept_mem_ops.get_pdpt_page(vm->arch_vm.ept_mem_ops.info, TRUSTY_EPT_REBASE_GPA);
 	sworld_pml4e = hva2hpa(sub_table_addr) | table_present;
 	set_pgentry((uint64_t *)pml4_base, sworld_pml4e);
 
@@ -148,8 +148,6 @@ static void create_secure_world_ept(struct vm *vm, uint64_t gpa_orig,
 
 void  destroy_secure_world(struct vm *vm, bool need_clr_mem)
 {
-	uint64_t j;
-	void *pdpt_addr;
 	struct vm *vm0 = get_vm_from_vmid(0U);
 	uint64_t hpa = vm->sworld_control.sworld_memory.base_hpa;
 	uint64_t gpa_sos = vm->sworld_control.sworld_memory.base_gpa_in_sos;
@@ -174,14 +172,8 @@ void  destroy_secure_world(struct vm *vm, bool need_clr_mem)
 	ept_mr_add(vm, vm->arch_vm.nworld_eptp,
 			hpa, gpa_uos, size, EPT_RWX | EPT_WB);
 
-	/* Free trusty ept page-structures */
+	/* sanitize trusty ept page-structures */
-	pdpt_addr =
+	sanitize_pte((uint64_t *)vm->arch_vm.sworld_eptp);
-		(void *)pml4e_page_vaddr(*(uint64_t *)vm->arch_vm.sworld_eptp);
-	/* identical PDPTEs except trusty memory */
-	for (j = 0UL; j < NON_TRUSTY_PDPT_ENTRIES; j++) {
-		sanitize_pte_entry((uint64_t *)pdpt_addr + j);
-	}
-	free_ept_mem((uint64_t *)vm->arch_vm.sworld_eptp);
 	vm->arch_vm.sworld_eptp = NULL;
 }
 

hypervisor/common/hypercall.c

@@ -356,7 +356,7 @@ int32_t hcall_notify_ioreq_finish(uint16_t vmid, uint16_t vcpu_id)
 static int32_t local_set_vm_memory_region(struct vm *vm,
 	struct vm *target_vm, const struct vm_memory_region *region)
 {
-	uint64_t hpa, base_paddr;
+	uint64_t hpa, base_paddr, gpa_end;
 	uint64_t prot;
 	uint64_t *pml4_page;
 
@@ -366,6 +366,14 @@ static int32_t local_set_vm_memory_region(struct vm *vm,
 		return -EINVAL;
 	}
 
+	gpa_end = region->gpa + region->size;
+	if ((gpa_end > vm->arch_vm.ept_mem_ops.info->ept.top_address_space) &&
+		(region->gpa < TRUSTY_EPT_REBASE_GPA)) {
+			pr_err("%s, invalid gpa: 0x%llx, size: 0x%llx, top_address_space: 0x%llx", __func__,
+				region->gpa, region->size, vm->arch_vm.ept_mem_ops.info->ept.top_address_space);
+			return -EINVAL;
+	}
+
 	dev_dbg(ACRN_DBG_HYCALL,
 		"[vm%d] type=%d gpa=0x%x vm0_gpa=0x%x size=0x%x",
 		target_vm->vm_id, region->type, region->gpa,

hypervisor/dm/vpci/pci_pt.c

@@ -68,7 +68,7 @@ static int vdev_pt_init(struct pci_vdev *vdev)
 	/* Create an iommu domain for target VM if not created */
 	if (vm->iommu == NULL) {
 		if (vm->arch_vm.nworld_eptp == 0UL) {
-			vm->arch_vm.nworld_eptp = alloc_paging_struct();
+			vm->arch_vm.nworld_eptp = vm->arch_vm.ept_mem_ops.get_pml4_page(vm->arch_vm.ept_mem_ops.info, 0UL);
 			sanitize_pte((uint64_t *)vm->arch_vm.nworld_eptp);
 		}
 		vm->iommu = create_iommu_domain(vm->vm_id,

hypervisor/include/arch/x86/mmu.h

@@ -35,6 +35,7 @@
 #ifndef ASSEMBLER
 
 #include <cpu.h>
+#include <page.h>
 
 /* Define cache line size (in bytes) */
 #define CACHE_LINE_SIZE		64U
@@ -53,11 +54,6 @@ static inline uint64_t round_page_down(uint64_t addr)
 	return (addr & CPU_PAGE_MASK);
 }
 
-enum _page_table_type {
-	PTT_PRIMARY = 0,	/* Mapping for hypervisor */
-	PTT_EPT = 1,
-};
-
 /* Represent the 4 levels of translation tables in IA-32e paging mode */
 enum _page_table_level {
 	IA32E_PML4 = 0,
@@ -83,13 +79,10 @@ void free_paging_struct(void *ptr);
 void enable_paging(uint64_t pml4_base_addr);
 void enable_smep(void);
 void init_paging(void);
-void mmu_add(uint64_t *pml4_page, uint64_t paddr_base,
+void mmu_add(uint64_t *pml4_page, uint64_t paddr_base, uint64_t vaddr_base,
-		uint64_t vaddr_base, uint64_t size,
+		uint64_t size, uint64_t prot, const struct memory_ops *mem_ops);
-		uint64_t prot, enum _page_table_type ptt);
+void mmu_modify_or_del(uint64_t *pml4_page, uint64_t vaddr_base, uint64_t size,
-void mmu_modify_or_del(uint64_t *pml4_page,
+		uint64_t prot_set, uint64_t prot_clr, const struct memory_ops *mem_ops, uint32_t type);
-		uint64_t vaddr_base, uint64_t size,
-		uint64_t prot_set, uint64_t prot_clr,
-		enum _page_table_type ptt, uint32_t type);
 int check_vmx_mmu_cap(void);
 uint16_t allocate_vpid(void);
 void flush_vpid_single(uint16_t vpid);
@@ -100,7 +93,7 @@ bool check_continuous_hpa(struct vm *vm, uint64_t gpa_arg, uint64_t size_arg);
  *@pre (pml4_page != NULL) && (pg_size != NULL)
  */
 uint64_t *lookup_address(uint64_t *pml4_page, uint64_t addr,
-		uint64_t *pg_size, enum _page_table_type ptt);
+		uint64_t *pg_size, const struct memory_ops *mem_ops);
 
 #pragma pack(1)
 
@@ -166,7 +159,6 @@ void ept_mr_modify(struct vm *vm, uint64_t *pml4_page, uint64_t gpa,
  */
 void ept_mr_del(struct vm *vm, uint64_t *pml4_page, uint64_t gpa,
 		uint64_t size);
-void free_ept_mem(uint64_t *pml4_page);
 int ept_violation_vmexit_handler(struct vcpu *vcpu);
 int ept_misconfig_vmexit_handler(__unused struct vcpu *vcpu);
 

hypervisor/include/arch/x86/page.h

@@ -13,6 +13,21 @@
 /* size of the low MMIO address space: 2GB */
 #define PLATFORM_LO_MMIO_SIZE	0x80000000UL
 
+#define PML4_PAGE_NUM(size)	1UL
+#define PDPT_PAGE_NUM(size)	(((size) + PML4E_SIZE - 1UL) >> PML4E_SHIFT)
+#define PD_PAGE_NUM(size)	(((size) + PDPTE_SIZE - 1UL) >> PDPTE_SHIFT)
+#define PT_PAGE_NUM(size)	(((size) + PDE_SIZE - 1UL) >> PDE_SHIFT)
+
+/* The size of the guest physical address space, covered by the EPT page table of a VM */
+#define EPT_ADDRESS_SPACE(size)	((size != 0UL) ? (size + PLATFORM_LO_MMIO_SIZE) : 0UL)
+
+#define TRUSTY_PML4_PAGE_NUM(size)	(1UL)
+#define TRUSTY_PDPT_PAGE_NUM(size)	(1UL)
+#define TRUSTY_PD_PAGE_NUM(size)	(PD_PAGE_NUM(size))
+#define TRUSTY_PT_PAGE_NUM(size)	(PT_PAGE_NUM(size))
+#define TRUSTY_PGTABLE_PAGE_NUM(size)	\
+(TRUSTY_PML4_PAGE_NUM(size) + TRUSTY_PDPT_PAGE_NUM(size) + TRUSTY_PD_PAGE_NUM(size) + TRUSTY_PT_PAGE_NUM(size))
+
 struct page {
 	uint8_t contents[PAGE_SIZE];
 } __aligned(PAGE_SIZE);

hypervisor/include/arch/x86/pgtable.h

@@ -162,9 +162,4 @@ static inline uint64_t pdpte_large(uint64_t pdpte)
 	return pdpte & PAGE_PSE;
 }
 
-static inline uint64_t pgentry_present(enum _page_table_type ptt, uint64_t pte)
-{
-	return (ptt == PTT_PRIMARY) ? (pte & PAGE_PRESENT) : (pte & EPT_RWX);
-}
-
 #endif /* PGTABLE_H */