hv: Reserve space for VMs' EPT 4k pages after boot

As ACRN prepares to support servers with large amounts of memory
current logic to allocate space for 4K pages of EPT at compile time
will increase the size of .bss section of ACRN binary.

Bootloaders could run into a situation where they cannot
find enough contiguous space to load ACRN binary under 4GB,
which is typically heavily fragmented with E820 types Reserved,
ACPI data, 32-bit PCI hole etc.

This patch does the following
1) Works only for "direct" mode of vboot
2) reserves space for 4K pages of EPT, after boot by parsing
platform E820 table, for all types of VMs.

Size comparison:

w/o patch
Size of DRAM            Size of .bss
48 GB                   0xe1bbc98 (~226 MB)
128 GB                  0x222abc98 (~548 MB)

w/ patch
Size of DRAM            Size of .bss
48 GB                   0x1991c98 (~26 MB)
128 GB                  0x1a81c98 (~28 MB)

Tracked-On: #4563
Signed-off-by: Sainath Grandhi <sainath.grandhi@intel.com>
Acked-by: Eddie Dong <eddie.dong@intel.com>

hypervisor/Makefile

@@ -337,6 +337,9 @@ SYS_INIT_C_OBJS := $(patsubst %.c,$(HV_OBJDIR)/%.o,$(SYS_INIT_C_SRCS))
 ifneq ($(CONFIG_RELEASE),y)
 CFLAGS += -DHV_DEBUG -DPROFILING_ON -fno-omit-frame-pointer
 endif
+ifneq ($(FIRMWARE),uefi)
+CFLAGS += -DCONFIG_LAST_LEVEL_EPT_AT_BOOT
+endif
 PRE_BUILD_SRCS += pre_build/static_checks.c
 PRE_BUILD_OBJS := $(patsubst %.c,$(HV_OBJDIR)/%.o,$(PRE_BUILD_SRCS))
 

hypervisor/arch/x86/cpu.c

@@ -243,6 +243,12 @@ void init_pcpu_post(uint16_t pcpu_id)
 			panic("failed to initialize sgx!");
 		}
 
+		/*
+		 * Reserve memory from platform E820 for EPT 4K pages for all VMs
+		 */
+#ifdef CONFIG_LAST_LEVEL_EPT_AT_BOOT
+		reserve_buffer_for_ept_pages();
+#endif
 		/* Start all secondary cores */
 		startup_paddr = prepare_trampoline();
 		if (!start_pcpus(AP_MASK)) {

hypervisor/arch/x86/e820.c

@@ -11,6 +11,7 @@
 #include <mmu.h>
 #include <boot.h>
 #include <logmsg.h>
+#include <ept.h>
 
 /*
  * e820.c contains the related e820 operations; like HV to get memory info for its MMU setup;
@@ -51,63 +52,75 @@ static void obtain_mem_range_info(void)
 	}
 }
 
-/* get some RAM below 1MB in e820 entries, hide it from sos_vm, return its start address */
+/*
-uint64_t e820_alloc_low_memory(uint32_t size_arg)
+ * @brief reserve some RAM, hide it from sos_vm, return its start address
+ * @param size_arg Amount of memory to be found and marked reserved
+ * @param max_addr Maximum address below which memory is to be identified
+ *
+ * @pre hv_e820_entries_nr > 0U
+ * @pre (size_arg & 0xFFFU) == 0U
+ * @return base address of the memory region
+ */
+uint64_t e820_alloc_memory(uint32_t size_arg, uint64_t max_addr)
 {
-	uint32_t i;
+	int32_t i;
-	uint32_t size = size_arg;
+	uint64_t size = size_arg;
-	uint64_t ret = ACRN_INVALID_HPA;
+	uint64_t ret = INVALID_HPA;
 	struct e820_entry *entry, *new_entry;
 
-	/* We want memory in page boundary and integral multiple of pages */
+	for (i = (int32_t)hv_e820_entries_nr - 1; i >= 0; i--) {
-	size = (((size + PAGE_SIZE) - 1U) >> PAGE_SHIFT) << PAGE_SHIFT;
-
-	for (i = 0U; i < hv_e820_entries_nr; i++) {
 		entry = &hv_e820[i];
 		uint64_t start, end, length;
 
 		start = round_page_up(entry->baseaddr);
 		end = round_page_down(entry->baseaddr + entry->length);
-		length = end - start;
+		length = (end > start) ? (end - start) : 0UL;
-		length = (end > start) ? (end - start) : 0;
 
-		/* Search for available low memory */
+		if ((entry->type == E820_TYPE_RAM) && (length >= size) && ((start + size) <= max_addr)) {
-		if ((entry->type != E820_TYPE_RAM) || (length < size) || ((start + size) > MEM_1M)) {
+
-			continue;
+
+			/* found exact size of e820 entry */
+			if (length == size) {
+				entry->type = E820_TYPE_RESERVED;
+				hv_mem_range.total_mem_size -= size;
+				ret = start;
+			} else {
+
+				/*
+				 * found entry with available memory larger than requested (length > size)
+				 * Reserve memory if
+				 * 1) hv_e820_entries_nr < E820_MAX_ENTRIES
+				 * 2) if end of this "entry" is <= max_addr
+				 *    use memory from end of this e820 "entry".
+				 */
+
+				if ((hv_e820_entries_nr < E820_MAX_ENTRIES) && (end <= max_addr)) {
+
+					new_entry = &hv_e820[hv_e820_entries_nr];
+					new_entry->type = E820_TYPE_RESERVED;
+					new_entry->baseaddr = end - size;
+					new_entry->length = (entry->baseaddr + entry->length) - new_entry->baseaddr;
+					/* Shrink the existing entry and total available memory */
+					entry->length -= new_entry->length;
+					hv_mem_range.total_mem_size -= new_entry->length;
+					hv_e820_entries_nr++;
+
+				        ret = new_entry->baseaddr;
+				}
+			}
+
+			if (ret != INVALID_HPA) {
+				break;
+			}
 		}
-
-		/* found exact size of e820 entry */
-		if (length == size) {
-			entry->type = E820_TYPE_RESERVED;
-			hv_mem_range.total_mem_size -= size;
-			ret = start;
-			break;
-		}
-
-		/*
-		 * found entry with available memory larger than requested
-		 * allocate memory from the end of this entry at page boundary
-		 */
-		new_entry = &hv_e820[hv_e820_entries_nr];
-		new_entry->type = E820_TYPE_RESERVED;
-		new_entry->baseaddr = end - size;
-		new_entry->length = (entry->baseaddr + entry->length) - new_entry->baseaddr;
-
-		/* Shrink the existing entry and total available memory */
-		entry->length -= new_entry->length;
-		hv_mem_range.total_mem_size -= new_entry->length;
-		hv_e820_entries_nr++;
-
-	        ret = new_entry->baseaddr;
-		break;
 	}
 
-	if (ret == ACRN_INVALID_HPA) {
+	if (ret == INVALID_HPA) {
-		pr_fatal("Can't allocate memory under 1M from E820\n");
+		panic("Requested memory from E820 cannot be reserved!!");
 	}
+
 	return ret;
 }
-
 /* HV read multiboot header to get e820 entries info and calc total RAM info */
 void init_e820(void)
 {

hypervisor/arch/x86/guest/vm.c

@@ -320,7 +320,7 @@ static void prepare_sos_vm_memmap(struct acrn_vm *vm)
 
 	/* unmap AP trampoline code for security
 	 * 'allocate_pages()' in depri boot mode or
-	 * 'e820_alloc_low_memory()' in direct boot
+	 * 'e820_alloc_memory()' in direct boot
 	 * mode will ensure the base address of tramploline
 	 * code be page-aligned.
 	 */

hypervisor/arch/x86/page.c

@@ -12,6 +12,7 @@
 #include <vtd.h>
 #include <security.h>
 #include <vm.h>
+#include <vm_configurations.h>
 
 static struct page ppt_pml4_pages[PML4_PAGE_NUM(CONFIG_PLATFORM_RAM_SIZE + PLATFORM_LO_MMIO_SIZE)];
 static struct page ppt_pdpt_pages[PDPT_PAGE_NUM(CONFIG_PLATFORM_RAM_SIZE + PLATFORM_LO_MMIO_SIZE)];
@@ -80,19 +81,15 @@ const struct memory_ops ppt_mem_ops = {
 static struct page sos_vm_pml4_pages[SOS_VM_NUM][PML4_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE))];
 static struct page sos_vm_pdpt_pages[SOS_VM_NUM][PDPT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE))];
 static struct page sos_vm_pd_pages[SOS_VM_NUM][PD_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE))];
-static struct page sos_vm_pt_pages[SOS_VM_NUM][PT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE))];
-
 /* pre_uos_nworld_pml4_pages */
 static struct page pre_uos_nworld_pml4_pages[PRE_VM_NUM][PML4_PAGE_NUM(PRE_VM_EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
 static struct page pre_uos_nworld_pdpt_pages[PRE_VM_NUM][PDPT_PAGE_NUM(PRE_VM_EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
 static struct page pre_uos_nworld_pd_pages[PRE_VM_NUM][PD_PAGE_NUM(PRE_VM_EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
-static struct page pre_uos_nworld_pt_pages[PRE_VM_NUM][PT_PAGE_NUM(PRE_VM_EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
 
 /* post_uos_nworld_pml4_pages */
 static struct page post_uos_nworld_pml4_pages[MAX_POST_VM_NUM][PML4_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
 static struct page post_uos_nworld_pdpt_pages[MAX_POST_VM_NUM][PDPT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
 static struct page post_uos_nworld_pd_pages[MAX_POST_VM_NUM][PD_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
-static struct page post_uos_nworld_pt_pages[MAX_POST_VM_NUM][PT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
 
 static struct page post_uos_sworld_pgtable_pages[MAX_POST_VM_NUM][TRUSTY_PGTABLE_PAGE_NUM(TRUSTY_RAM_SIZE)];
 /* pre-assumption: TRUSTY_RAM_SIZE is 2M aligned */
@@ -101,6 +98,39 @@ static struct page post_uos_sworld_memory[MAX_POST_VM_NUM][TRUSTY_RAM_SIZE >> PA
 /* ept: extended page table*/
 static union pgtable_pages_info ept_pages_info[CONFIG_MAX_VM_NUM];
 
+
+#ifdef CONFIG_LAST_LEVEL_EPT_AT_BOOT
+/* Array with address space size for each type of load order of VM */
+static const uint64_t vm_address_space_size[MAX_LOAD_ORDER] = {
+	PRE_VM_EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE), /* for Pre-Launched VM */
+	EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE), /* for SOS VM */
+	EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE), /* for Post-Launched VM */
+};
+
+/*
+ * @brief Reserve space for EPT 4K pages from platform E820 table
+ */
+void reserve_buffer_for_ept_pages(void)
+{
+	uint64_t pt_base;
+	uint16_t vm_id;
+	uint32_t offset = 0U;
+	struct acrn_vm_config *vm_config;
+
+	pt_base = e820_alloc_memory(TOTAL_EPT_4K_PAGES_SIZE, ~0UL);
+	hv_access_memory_region_update(pt_base, TOTAL_EPT_4K_PAGES_SIZE);
+	for (vm_id = 0U; vm_id < CONFIG_MAX_VM_NUM; vm_id++) {
+		vm_config = get_vm_config(vm_id);
+		ept_pages_info[vm_id].ept.nworld_pt_base = (struct page *)(void *)(pt_base + offset);
+		offset += PT_PAGE_NUM(vm_address_space_size[vm_config->load_order])*MEM_4K;
+	}
+}
+#else
+static struct page sos_vm_pt_pages[SOS_VM_NUM][PT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE))];
+static struct page pre_uos_nworld_pt_pages[PRE_VM_NUM][PT_PAGE_NUM(PRE_VM_EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
+static struct page post_uos_nworld_pt_pages[MAX_POST_VM_NUM][PT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
+#endif
+
 void *get_reserve_sworld_memory_base(void)
 {
 	return post_uos_sworld_memory;
@@ -190,13 +220,17 @@ void init_ept_mem_ops(struct memory_ops *mem_ops, uint16_t vm_id)
 		ept_pages_info[vm_id].ept.nworld_pml4_base = sos_vm_pml4_pages[0U];
 		ept_pages_info[vm_id].ept.nworld_pdpt_base = sos_vm_pdpt_pages[0U];
 		ept_pages_info[vm_id].ept.nworld_pd_base = sos_vm_pd_pages[0U];
+#ifndef CONFIG_LAST_LEVEL_EPT_AT_BOOT
 		ept_pages_info[vm_id].ept.nworld_pt_base = sos_vm_pt_pages[0U];
+#endif
 	} else if (is_prelaunched_vm(vm)) {
-		ept_pages_info[vm_id].ept.top_address_space = EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE);
+		ept_pages_info[vm_id].ept.top_address_space = PRE_VM_EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE);
 		ept_pages_info[vm_id].ept.nworld_pml4_base = pre_uos_nworld_pml4_pages[vm_id];
 		ept_pages_info[vm_id].ept.nworld_pdpt_base = pre_uos_nworld_pdpt_pages[vm_id];
 		ept_pages_info[vm_id].ept.nworld_pd_base = pre_uos_nworld_pd_pages[vm_id];
+#ifndef CONFIG_LAST_LEVEL_EPT_AT_BOOT
 		ept_pages_info[vm_id].ept.nworld_pt_base = pre_uos_nworld_pt_pages[vm_id];
+#endif
 	} else {
 		uint16_t sos_vm_id = (get_sos_vm())->vm_id;
 		uint16_t page_idx = vmid_2_rel_vmid(sos_vm_id, vm_id) - 1U;
@@ -205,7 +239,9 @@ void init_ept_mem_ops(struct memory_ops *mem_ops, uint16_t vm_id)
 		ept_pages_info[vm_id].ept.nworld_pml4_base = post_uos_nworld_pml4_pages[page_idx];
 		ept_pages_info[vm_id].ept.nworld_pdpt_base = post_uos_nworld_pdpt_pages[page_idx];
 		ept_pages_info[vm_id].ept.nworld_pd_base = post_uos_nworld_pd_pages[page_idx];
+#ifndef CONFIG_LAST_LEVEL_EPT_AT_BOOT
 		ept_pages_info[vm_id].ept.nworld_pt_base = post_uos_nworld_pt_pages[page_idx];
+#endif
 		ept_pages_info[vm_id].ept.sworld_pgtable_base = post_uos_sworld_pgtable_pages[page_idx];
 		ept_pages_info[vm_id].ept.sworld_memory_base = post_uos_sworld_memory[page_idx];
 		mem_ops->get_sworld_memory_base = ept_get_sworld_memory_base;

hypervisor/boot/guest/direct_boot.c

@@ -11,13 +11,14 @@
 #include <cpu.h>
 #include <boot.h>
 #include <direct_boot.h>
+#include <mmu.h>
 
 /* AP trampoline code buffer base address. */
 static uint64_t ap_trampoline_buf;
 
 static void init_direct_boot(void)
 {
-	ap_trampoline_buf = e820_alloc_low_memory(CONFIG_LOW_RAM_SIZE);
+	ap_trampoline_buf = e820_alloc_memory(CONFIG_LOW_RAM_SIZE, MEM_1M);
 }
 
 /* @post: return != 0UL */

hypervisor/include/arch/x86/e820.h

@@ -37,9 +37,7 @@ struct mem_range {
 /* HV read multiboot header to get e820 entries info and calc total RAM info */
 void init_e820(void);
 
-/* get some RAM below 1MB in e820 entries, hide it from sos_vm, return its start address */
+uint64_t e820_alloc_memory(uint32_t size_arg, uint64_t max_addr);
-uint64_t e820_alloc_low_memory(uint32_t size_arg);
-
 /* get total number of the e820 entries */
 uint32_t get_e820_entries_count(void);
 

hypervisor/include/arch/x86/page.h

@@ -40,6 +40,10 @@
 
 #define PRE_VM_EPT_ADDRESS_SPACE(size)	(PTDEV_HI_MMIO_START + PTDEV_HI_MMIO_SIZE)
 
+#define TOTAL_EPT_4K_PAGES_SIZE		(PRE_VM_NUM*(PT_PAGE_NUM(PRE_VM_EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))*MEM_4K)) + \
+						(SOS_VM_NUM*(PT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE))*MEM_4K)) + \
+						(MAX_POST_VM_NUM*(PT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))*MEM_4K))
+
 #define TRUSTY_PML4_PAGE_NUM(size)	(1UL)
 #define TRUSTY_PDPT_PAGE_NUM(size)	(1UL)
 #define TRUSTY_PD_PAGE_NUM(size)	(PD_PAGE_NUM(size))
@@ -90,4 +94,7 @@ extern const struct memory_ops ppt_mem_ops;
 void init_ept_mem_ops(struct memory_ops *mem_ops, uint16_t vm_id);
 void *get_reserve_sworld_memory_base(void);
 
+#ifdef CONFIG_LAST_LEVEL_EPT_AT_BOOT
+void reserve_buffer_for_ept_pages(void);
+#endif
 #endif /* PAGE_H */

hypervisor/include/arch/x86/vm_config.h

@@ -38,9 +38,10 @@
  * POST_LAUNCHED_VM is launched by ACRN devicemodel, with/without LAPIC_PT depends on usecases.
  */
 enum acrn_vm_load_order {
-	PRE_LAUNCHED_VM = 1,
+	PRE_LAUNCHED_VM = 0,
 	SOS_VM,
-	POST_LAUNCHED_VM	/* Launched by Devicemodel in SOS_VM */
+	POST_LAUNCHED_VM,	/* Launched by Devicemodel in SOS_VM */
+	MAX_LOAD_ORDER
 };
 
 /* ACRN guest severity */