From 2d55b49ea1ce8774951e05e5ff894174a1ed9519 Mon Sep 17 00:00:00 2001 From: Binbin Wu Date: Mon, 1 Jul 2019 13:39:20 +0800 Subject: [PATCH] hv: ept: flush cache for modified ept entries EPT tables are shared by MMU and IOMMU. Some IOMMUs don't support page-walk coherency, the cpu cache of EPT entires should be flushed to memory after modifications, so that the modifications are visible to the IOMMUs. This patch adds a new interface to flush the cache of modified EPT entires. There are different implementations for EPT/PPT entries: - For PPT, there is no need to flush the cpu cache after update. - For EPT, need to call iommu_flush_cache to make the modifications visible to IOMMUs. Tracked-On: #4120 Signed-off-by: Binbin Wu Reviewed-by: Anthony Xu --- hypervisor/arch/x86/guest/trusty.c | 8 +++---- hypervisor/arch/x86/guest/vm.c | 2 +- hypervisor/arch/x86/mmu.c | 10 ++++---- hypervisor/arch/x86/page.c | 13 +++++++++- hypervisor/arch/x86/pagetable.c | 34 +++++++++++++-------------- hypervisor/include/arch/x86/page.h | 1 + hypervisor/include/arch/x86/pgtable.h | 3 ++- 7 files changed, 42 insertions(+), 29 deletions(-) diff --git a/hypervisor/arch/x86/guest/trusty.c b/hypervisor/arch/x86/guest/trusty.c index 9fe38f84a..3aa3630dd 100644 --- a/hypervisor/arch/x86/guest/trusty.c +++ b/hypervisor/arch/x86/guest/trusty.c @@ -79,7 +79,7 @@ static void create_secure_world_ept(struct acrn_vm *vm, uint64_t gpa_orig, pml4_base = vm->arch_vm.ept_mem_ops.info->ept.sworld_pgtable_base; (void)memset(pml4_base, 0U, PAGE_SIZE); vm->arch_vm.sworld_eptp = pml4_base; - sanitize_pte((uint64_t *)vm->arch_vm.sworld_eptp); + sanitize_pte((uint64_t *)vm->arch_vm.sworld_eptp, &vm->arch_vm.ept_mem_ops); /* The trusty memory is remapped to guest physical address * of gpa_rebased to gpa_rebased + size @@ -88,7 +88,7 @@ static void create_secure_world_ept(struct acrn_vm *vm, uint64_t gpa_orig, TRUSTY_PML4_PAGE_NUM(TRUSTY_EPT_REBASE_GPA); (void)memset(sub_table_addr, 0U, PAGE_SIZE); sworld_pml4e = hva2hpa(sub_table_addr) | table_present; - set_pgentry((uint64_t *)pml4_base, sworld_pml4e); + set_pgentry((uint64_t *)pml4_base, sworld_pml4e, &vm->arch_vm.ept_mem_ops); nworld_pml4e = get_pgentry((uint64_t *)vm->arch_vm.nworld_eptp); @@ -102,7 +102,7 @@ static void create_secure_world_ept(struct acrn_vm *vm, uint64_t gpa_orig, pdpte = get_pgentry(src_pdpte_p); if ((pdpte & table_present) != 0UL) { pdpte &= ~EPT_EXE; - set_pgentry(dest_pdpte_p, pdpte); + set_pgentry(dest_pdpte_p, pdpte, &vm->arch_vm.ept_mem_ops); } src_pdpte_p++; dest_pdpte_p++; @@ -133,7 +133,7 @@ void destroy_secure_world(struct acrn_vm *vm, bool need_clr_mem) ept_mr_del(vm, vm->arch_vm.sworld_eptp, gpa_uos, size); /* sanitize trusty ept page-structures */ - sanitize_pte((uint64_t *)vm->arch_vm.sworld_eptp); + sanitize_pte((uint64_t *)vm->arch_vm.sworld_eptp, &vm->arch_vm.ept_mem_ops); vm->arch_vm.sworld_eptp = NULL; /* Restore memory to guest normal world */ diff --git a/hypervisor/arch/x86/guest/vm.c b/hypervisor/arch/x86/guest/vm.c index 05b126d42..f91c32dcd 100644 --- a/hypervisor/arch/x86/guest/vm.c +++ b/hypervisor/arch/x86/guest/vm.c @@ -333,7 +333,7 @@ int32_t create_vm(uint16_t vm_id, struct acrn_vm_config *vm_config, struct acrn_ init_ept_mem_ops(vm); vm->arch_vm.nworld_eptp = vm->arch_vm.ept_mem_ops.get_pml4_page(vm->arch_vm.ept_mem_ops.info); - sanitize_pte((uint64_t *)vm->arch_vm.nworld_eptp); + sanitize_pte((uint64_t *)vm->arch_vm.nworld_eptp, &vm->arch_vm.ept_mem_ops); /* Register default handlers for PIO & MMIO if it is SOS VM or Pre-launched VM */ if ((vm_config->type == SOS_VM) || (vm_config->type == PRE_LAUNCHED_VM)) { diff --git a/hypervisor/arch/x86/mmu.c b/hypervisor/arch/x86/mmu.c index 9e3e4701e..618c58f03 100644 --- a/hypervisor/arch/x86/mmu.c +++ b/hypervisor/arch/x86/mmu.c @@ -175,16 +175,16 @@ static inline uint64_t get_sanitized_page(void) return hva2hpa(sanitized_page); } -void sanitize_pte_entry(uint64_t *ptep) +void sanitize_pte_entry(uint64_t *ptep, const struct memory_ops *mem_ops) { - set_pgentry(ptep, get_sanitized_page()); + set_pgentry(ptep, get_sanitized_page(), mem_ops); } -void sanitize_pte(uint64_t *pt_page) +void sanitize_pte(uint64_t *pt_page, const struct memory_ops *mem_ops) { uint64_t i; for (i = 0UL; i < PTRS_PER_PTE; i++) { - sanitize_pte_entry(pt_page + i); + sanitize_pte_entry(pt_page + i, mem_ops); } } @@ -327,5 +327,5 @@ void init_paging(void) enable_paging(); /* set ptep in sanitized_page point to itself */ - sanitize_pte((uint64_t *)sanitized_page); + sanitize_pte((uint64_t *)sanitized_page, &ppt_mem_ops); } diff --git a/hypervisor/arch/x86/page.c b/hypervisor/arch/x86/page.c index 069065c76..a91cad289 100644 --- a/hypervisor/arch/x86/page.c +++ b/hypervisor/arch/x86/page.c @@ -10,6 +10,7 @@ #include #include #include +#include static struct page ppt_pml4_pages[PML4_PAGE_NUM(CONFIG_PLATFORM_RAM_SIZE + PLATFORM_LO_MMIO_SIZE)]; static struct page ppt_pdpt_pages[PDPT_PAGE_NUM(CONFIG_PLATFORM_RAM_SIZE + PLATFORM_LO_MMIO_SIZE)]; @@ -29,6 +30,10 @@ static inline uint64_t ppt_get_default_access_right(void) return (PAGE_PRESENT | PAGE_RW | PAGE_USER); } +static inline void ppt_clflush_pagewalk(const void* etry __attribute__((unused))) +{ +} + static inline uint64_t ppt_pgentry_present(uint64_t pte) { return pte & PAGE_PRESENT; @@ -62,6 +67,7 @@ const struct memory_ops ppt_mem_ops = { .get_pml4_page = ppt_get_pml4_page, .get_pdpt_page = ppt_get_pdpt_page, .get_pd_page = ppt_get_pd_page, + .clflush_pagewalk = ppt_clflush_pagewalk, }; static struct page sos_vm_pml4_pages[PML4_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE))]; @@ -107,6 +113,11 @@ static inline uint64_t ept_pgentry_present(uint64_t pte) return pte & EPT_RWX; } +static inline void ept_clflush_pagewalk(const void* etry) +{ + iommu_flush_cache(etry, sizeof(uint64_t)); +} + static inline struct page *ept_get_pml4_page(const union pgtable_pages_info *info) { struct page *pml4_page = info->ept.nworld_pml4_base; @@ -175,5 +186,5 @@ void init_ept_mem_ops(struct acrn_vm *vm) vm->arch_vm.ept_mem_ops.get_pdpt_page = ept_get_pdpt_page; vm->arch_vm.ept_mem_ops.get_pd_page = ept_get_pd_page; vm->arch_vm.ept_mem_ops.get_pt_page = ept_get_pt_page; - + vm->arch_vm.ept_mem_ops.clflush_pagewalk = ept_clflush_pagewalk; } diff --git a/hypervisor/arch/x86/pagetable.c b/hypervisor/arch/x86/pagetable.c index 7586ca2d3..74f85ee70 100644 --- a/hypervisor/arch/x86/pagetable.c +++ b/hypervisor/arch/x86/pagetable.c @@ -45,37 +45,37 @@ static void split_large_page(uint64_t *pte, enum _page_table_level level, paddr = ref_paddr; for (i = 0UL; i < PTRS_PER_PTE; i++) { - set_pgentry(pbase + i, paddr | ref_prot); + set_pgentry(pbase + i, paddr | ref_prot, mem_ops); paddr += paddrinc; } ref_prot = mem_ops->get_default_access_right(); - set_pgentry(pte, hva2hpa((void *)pbase) | ref_prot); + set_pgentry(pte, hva2hpa((void *)pbase) | ref_prot, mem_ops); /* TODO: flush the TLB */ } static inline void local_modify_or_del_pte(uint64_t *pte, - uint64_t prot_set, uint64_t prot_clr, uint32_t type) + uint64_t prot_set, uint64_t prot_clr, uint32_t type, const struct memory_ops *mem_ops) { if (type == MR_MODIFY) { uint64_t new_pte = *pte; new_pte &= ~prot_clr; new_pte |= prot_set; - set_pgentry(pte, new_pte); + set_pgentry(pte, new_pte, mem_ops); } else { - sanitize_pte_entry(pte); + sanitize_pte_entry(pte, mem_ops); } } /* * pgentry may means pml4e/pdpte/pde */ -static inline void construct_pgentry(uint64_t *pde, void *pd_page, uint64_t prot) +static inline void construct_pgentry(uint64_t *pde, void *pd_page, uint64_t prot, const struct memory_ops *mem_ops) { - sanitize_pte((uint64_t *)pd_page); + sanitize_pte((uint64_t *)pd_page, mem_ops); - set_pgentry(pde, hva2hpa(pd_page) | prot); + set_pgentry(pde, hva2hpa(pd_page) | prot, mem_ops); } /* @@ -106,7 +106,7 @@ static void modify_or_del_pte(const uint64_t *pde, uint64_t vaddr_start, uint64_ pr_warn("%s, vaddr: 0x%lx pte is not present.\n", __func__, vaddr); } } else { - local_modify_or_del_pte(pte, prot_set, prot_clr, type); + local_modify_or_del_pte(pte, prot_set, prot_clr, type, mem_ops); } vaddr += PTE_SIZE; @@ -142,7 +142,7 @@ static void modify_or_del_pde(const uint64_t *pdpte, uint64_t vaddr_start, uint6 if ((vaddr_next > vaddr_end) || (!mem_aligned_check(vaddr, PDE_SIZE))) { split_large_page(pde, IA32E_PD, vaddr, mem_ops); } else { - local_modify_or_del_pte(pde, prot_set, prot_clr, type); + local_modify_or_del_pte(pde, prot_set, prot_clr, type, mem_ops); if (vaddr_next < vaddr_end) { vaddr = vaddr_next; continue; @@ -187,7 +187,7 @@ static void modify_or_del_pdpte(const uint64_t *pml4e, uint64_t vaddr_start, uin (!mem_aligned_check(vaddr, PDPTE_SIZE))) { split_large_page(pdpte, IA32E_PDPT, vaddr, mem_ops); } else { - local_modify_or_del_pte(pdpte, prot_set, prot_clr, type); + local_modify_or_del_pte(pdpte, prot_set, prot_clr, type, mem_ops); if (vaddr_next < vaddr_end) { vaddr = vaddr_next; continue; @@ -261,7 +261,7 @@ static void add_pte(const uint64_t *pde, uint64_t paddr_start, uint64_t vaddr_st if (mem_ops->pgentry_present(*pte) != 0UL) { ASSERT(false, "invalid op, pte present"); } else { - set_pgentry(pte, paddr | prot); + set_pgentry(pte, paddr | prot, mem_ops); paddr += PTE_SIZE; vaddr += PTE_SIZE; @@ -294,7 +294,7 @@ static void add_pde(const uint64_t *pdpte, uint64_t paddr_start, uint64_t vaddr_ if (mem_aligned_check(paddr, PDE_SIZE) && mem_aligned_check(vaddr, PDE_SIZE) && (vaddr_next <= vaddr_end)) { - set_pgentry(pde, paddr | (prot | PAGE_PSE)); + set_pgentry(pde, paddr | (prot | PAGE_PSE), mem_ops); if (vaddr_next < vaddr_end) { paddr += (vaddr_next - vaddr); vaddr = vaddr_next; @@ -303,7 +303,7 @@ static void add_pde(const uint64_t *pdpte, uint64_t paddr_start, uint64_t vaddr_ break; /* done */ } else { void *pt_page = mem_ops->get_pt_page(mem_ops->info, vaddr); - construct_pgentry(pde, pt_page, mem_ops->get_default_access_right()); + construct_pgentry(pde, pt_page, mem_ops->get_default_access_right(), mem_ops); } } add_pte(pde, paddr, vaddr, vaddr_end, prot, mem_ops); @@ -336,7 +336,7 @@ static void add_pdpte(const uint64_t *pml4e, uint64_t paddr_start, uint64_t vadd if (mem_aligned_check(paddr, PDPTE_SIZE) && mem_aligned_check(vaddr, PDPTE_SIZE) && (vaddr_next <= vaddr_end)) { - set_pgentry(pdpte, paddr | (prot | PAGE_PSE)); + set_pgentry(pdpte, paddr | (prot | PAGE_PSE), mem_ops); if (vaddr_next < vaddr_end) { paddr += (vaddr_next - vaddr); vaddr = vaddr_next; @@ -345,7 +345,7 @@ static void add_pdpte(const uint64_t *pml4e, uint64_t paddr_start, uint64_t vadd break; /* done */ } else { void *pd_page = mem_ops->get_pd_page(mem_ops->info, vaddr); - construct_pgentry(pdpte, pd_page, mem_ops->get_default_access_right()); + construct_pgentry(pdpte, pd_page, mem_ops->get_default_access_right(), mem_ops); } } add_pde(pdpte, paddr, vaddr, vaddr_end, prot, mem_ops); @@ -381,7 +381,7 @@ void mmu_add(uint64_t *pml4_page, uint64_t paddr_base, uint64_t vaddr_base, uint pml4e = pml4e_offset(pml4_page, vaddr); if (mem_ops->pgentry_present(*pml4e) == 0UL) { void *pdpt_page = mem_ops->get_pdpt_page(mem_ops->info, vaddr); - construct_pgentry(pml4e, pdpt_page, mem_ops->get_default_access_right()); + construct_pgentry(pml4e, pdpt_page, mem_ops->get_default_access_right(), mem_ops); } add_pdpte(pml4e, paddr, vaddr, vaddr_end, prot, mem_ops); diff --git a/hypervisor/include/arch/x86/page.h b/hypervisor/include/arch/x86/page.h index 6e4c0a7bf..f8eb67bae 100644 --- a/hypervisor/include/arch/x86/page.h +++ b/hypervisor/include/arch/x86/page.h @@ -71,6 +71,7 @@ struct memory_ops { struct page *(*get_pd_page)(const union pgtable_pages_info *info, uint64_t gpa); struct page *(*get_pt_page)(const union pgtable_pages_info *info, uint64_t gpa); void *(*get_sworld_memory_base)(const union pgtable_pages_info *info); + void (*clflush_pagewalk)(const void *p); }; extern const struct memory_ops ppt_mem_ops; diff --git a/hypervisor/include/arch/x86/pgtable.h b/hypervisor/include/arch/x86/pgtable.h index d16e524ae..8332c4cec 100644 --- a/hypervisor/include/arch/x86/pgtable.h +++ b/hypervisor/include/arch/x86/pgtable.h @@ -168,9 +168,10 @@ static inline uint64_t get_pgentry(const uint64_t *pte) /* * pgentry may means pml4e/pdpte/pde/pte */ -static inline void set_pgentry(uint64_t *ptep, uint64_t pte) +static inline void set_pgentry(uint64_t *ptep, uint64_t pte, const struct memory_ops *mem_ops) { *ptep = pte; + mem_ops->clflush_pagewalk(ptep); } static inline uint64_t pde_large(uint64_t pde)