From b69d24b1d49c537202482dbc58b9d1e5bfb797fc Mon Sep 17 00:00:00 2001 From: Minggui Cao Date: Fri, 7 Dec 2018 14:41:02 +0800 Subject: [PATCH] HV: separate e820 related code as e820.c/h 1. e820 is modulized as separated files. 2. move boot_regs into multiboot.h as it's related with multiboot info header Tracked-On: #1842 Signed-off-by: Minggui Cao Acked-by: Anthony Xu --- hypervisor/Makefile | 1 + hypervisor/arch/x86/cpu.c | 1 + hypervisor/arch/x86/e820.c | 229 ++++++++++++++++++ hypervisor/arch/x86/guest/guest.c | 202 +-------------- hypervisor/arch/x86/guest/vm.c | 1 + hypervisor/arch/x86/mmu.c | 1 + hypervisor/arch/x86/trampoline.c | 1 + hypervisor/common/vm_load.c | 32 +-- hypervisor/include/arch/x86/e820.h | 35 +++ hypervisor/include/arch/x86/guest/guest.h | 29 --- hypervisor/include/arch/x86/multiboot.h | 3 + hypervisor/partition/apl-mrb/vm_description.c | 1 + hypervisor/partition/cb2_dnv/vm_description.c | 1 + 13 files changed, 276 insertions(+), 261 deletions(-) create mode 100644 hypervisor/arch/x86/e820.c create mode 100644 hypervisor/include/arch/x86/e820.h diff --git a/hypervisor/Makefile b/hypervisor/Makefile index 6b672aa33..0c8a43868 100644 --- a/hypervisor/Makefile +++ b/hypervisor/Makefile @@ -136,6 +136,7 @@ C_SRCS += arch/x86/lapic.c C_SRCS += arch/x86/cpu.c C_SRCS += arch/x86/cpuid.c C_SRCS += arch/x86/mmu.c +C_SRCS += arch/x86/e820.c C_SRCS += arch/x86/pagetable.c C_SRCS += arch/x86/page.c C_SRCS += arch/x86/notify.c diff --git a/hypervisor/arch/x86/cpu.c b/hypervisor/arch/x86/cpu.c index d3d564fcd..ff25d8b34 100644 --- a/hypervisor/arch/x86/cpu.c +++ b/hypervisor/arch/x86/cpu.c @@ -9,6 +9,7 @@ #include #include #include +#include spinlock_t trampoline_spinlock = { .head = 0U, diff --git a/hypervisor/arch/x86/e820.c b/hypervisor/arch/x86/e820.c new file mode 100644 index 000000000..4ab9630c6 --- /dev/null +++ b/hypervisor/arch/x86/e820.c @@ -0,0 +1,229 @@ +/* + * Copyright (C) 2018 Intel Corporation. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include + +/* for VM0 e820 */ +uint32_t e820_entries; +struct e820_entry e820[E820_MAX_ENTRIES]; +struct e820_mem_params e820_mem; + +#define ACRN_DBG_E820 6U + +void obtain_e820_mem_info(void) +{ + uint32_t i; + struct e820_entry *entry; + + e820_mem.mem_bottom = UINT64_MAX; + e820_mem.mem_top = 0x0UL; + e820_mem.total_mem_size = 0UL; + e820_mem.max_ram_blk_base = 0UL; + e820_mem.max_ram_blk_size = 0UL; + + for (i = 0U; i < e820_entries; i++) { + entry = &e820[i]; + if (e820_mem.mem_bottom > entry->baseaddr) { + e820_mem.mem_bottom = entry->baseaddr; + } + + if ((entry->baseaddr + entry->length) > e820_mem.mem_top) { + e820_mem.mem_top = entry->baseaddr + entry->length; + } + + if (entry->type == E820_TYPE_RAM) { + e820_mem.total_mem_size += entry->length; + if (entry->baseaddr == UOS_DEFAULT_START_ADDR) { + e820_mem.max_ram_blk_base = entry->baseaddr; + e820_mem.max_ram_blk_size = entry->length; + } + } + } +} + +void rebuild_vm0_e820(void) +{ + uint32_t i; + uint64_t entry_start; + uint64_t entry_end; + uint64_t hv_start_pa = get_hv_image_base(); + uint64_t hv_end_pa = hv_start_pa + CONFIG_HV_RAM_SIZE; + struct e820_entry *entry, new_entry = {0}; + + /* hypervisor mem need be filter out from e820 table + * it's hv itself + other hv reserved mem like vgt etc + */ + for (i = 0U; i < e820_entries; i++) { + entry = &e820[i]; + entry_start = entry->baseaddr; + entry_end = entry->baseaddr + entry->length; + + /* No need handle in these cases*/ + if ((entry->type != E820_TYPE_RAM) || (entry_end <= hv_start_pa) || (entry_start >= hv_end_pa)) { + continue; + } + + /* filter out hv mem and adjust length of this entry*/ + if ((entry_start < hv_start_pa) && (entry_end <= hv_end_pa)) { + entry->length = hv_start_pa - entry_start; + continue; + } + + /* filter out hv mem and need to create a new entry*/ + if ((entry_start < hv_start_pa) && (entry_end > hv_end_pa)) { + entry->length = hv_start_pa - entry_start; + new_entry.baseaddr = hv_end_pa; + new_entry.length = entry_end - hv_end_pa; + new_entry.type = E820_TYPE_RAM; + continue; + } + + /* This entry is within the range of hv mem + * change to E820_TYPE_RESERVED + */ + if ((entry_start >= hv_start_pa) && (entry_end <= hv_end_pa)) { + entry->type = E820_TYPE_RESERVED; + continue; + } + + if ((entry_start >= hv_start_pa) && (entry_start < hv_end_pa) && (entry_end > hv_end_pa)) { + entry->baseaddr = hv_end_pa; + entry->length = entry_end - hv_end_pa; + continue; + } + } + + if (new_entry.length > 0UL) { + e820_entries++; + ASSERT(e820_entries <= E820_MAX_ENTRIES, "e820 entry overflow"); + entry = &e820[e820_entries - 1]; + entry->baseaddr = new_entry.baseaddr; + entry->length = new_entry.length; + entry->type = new_entry.type; + } + + e820_mem.total_mem_size -= CONFIG_HV_RAM_SIZE; +} + +uint64_t e820_alloc_low_memory(uint32_t size_arg) +{ + uint32_t i; + uint32_t size = size_arg; + struct e820_entry *entry, *new_entry; + + /* We want memory in page boundary and integral multiple of pages */ + size = (((size + PAGE_SIZE) - 1U) >> PAGE_SHIFT) << PAGE_SHIFT; + + for (i = 0U; i < e820_entries; i++) { + entry = &e820[i]; + uint64_t start, end, length; + + start = round_page_up(entry->baseaddr); + end = round_page_down(entry->baseaddr + entry->length); + length = end - start; + length = (end > start) ? (end - start) : 0; + + /* Search for available low memory */ + if ((entry->type != E820_TYPE_RAM) || (length < size) || ((start + size) > MEM_1M)) { + continue; + } + + /* found exact size of e820 entry */ + if (length == size) { + entry->type = E820_TYPE_RESERVED; + e820_mem.total_mem_size -= size; + return start; + } + + /* + * found entry with available memory larger than requested + * allocate memory from the end of this entry at page boundary + */ + new_entry = &e820[e820_entries]; + new_entry->type = E820_TYPE_RESERVED; + new_entry->baseaddr = end - size; + new_entry->length = (entry->baseaddr + entry->length) - new_entry->baseaddr; + + /* Shrink the existing entry and total available memory */ + entry->length -= new_entry->length; + e820_mem.total_mem_size -= new_entry->length; + e820_entries++; + + return new_entry->baseaddr; + } + + pr_fatal("Can't allocate memory under 1M from E820\n"); + return ACRN_INVALID_HPA; +} + +void init_e820(void) +{ + uint32_t i; + + if (boot_regs[0] == MULTIBOOT_INFO_MAGIC) { + struct multiboot_info *mbi = (struct multiboot_info *)(hpa2hva((uint64_t)boot_regs[1])); + + pr_info("Multiboot info detected\n"); + if ((mbi->mi_flags & MULTIBOOT_INFO_HAS_MMAP) != 0U) { + struct multiboot_mmap *mmap = (struct multiboot_mmap *)hpa2hva((uint64_t)mbi->mi_mmap_addr); + + e820_entries = mbi->mi_mmap_length / sizeof(struct multiboot_mmap); + if (e820_entries > E820_MAX_ENTRIES) { + pr_err("Too many E820 entries %d\n", e820_entries); + e820_entries = E820_MAX_ENTRIES; + } + + dev_dbg(ACRN_DBG_E820, "mmap length 0x%x addr 0x%x entries %d\n", + mbi->mi_mmap_length, mbi->mi_mmap_addr, e820_entries); + + for (i = 0U; i < e820_entries; i++) { + e820[i].baseaddr = mmap[i].baseaddr; + e820[i].length = mmap[i].length; + e820[i].type = mmap[i].type; + + dev_dbg(ACRN_DBG_E820, "mmap table: %d type: 0x%x\n", i, mmap[i].type); + dev_dbg(ACRN_DBG_E820, "Base: 0x%016llx length: 0x%016llx", + mmap[i].baseaddr, mmap[i].length); + } + } + + } else { + panic("no multiboot info found"); + } +} + +#ifdef CONFIG_PARTITION_MODE +uint32_t create_e820_table(struct e820_entry *param_e820) +{ + uint32_t i; + + for (i = 0U; i < NUM_E820_ENTRIES; i++) { + param_e820[i].baseaddr = e820_default_entries[i].baseaddr; + param_e820[i].length = e820_default_entries[i].length; + param_e820[i].type = e820_default_entries[i].type; + } + + return NUM_E820_ENTRIES; +} +#else +uint32_t create_e820_table(struct e820_entry *param_e820) +{ + uint32_t i; + + ASSERT(e820_entries > 0U, "e820 should be inited"); + + for (i = 0U; i < e820_entries; i++) { + param_e820[i].baseaddr = e820[i].baseaddr; + param_e820[i].length = e820[i].length; + param_e820[i].type = e820[i].type; + } + + return e820_entries; +} +#endif diff --git a/hypervisor/arch/x86/guest/guest.c b/hypervisor/arch/x86/guest/guest.c index 830dfd5aa..946018bca 100644 --- a/hypervisor/arch/x86/guest/guest.c +++ b/hypervisor/arch/x86/guest/guest.c @@ -8,14 +8,10 @@ #include #include #include +#include #define ACRN_DBG_GUEST 6U -/* for VM0 e820 */ -uint32_t e820_entries; -struct e820_entry e820[E820_MAX_ENTRIES]; -struct e820_mem_params e820_mem; - struct page_walk_info { uint64_t top_entry; /* Top level paging structure entry */ uint32_t level; @@ -455,148 +451,6 @@ int copy_to_gva(struct acrn_vcpu *vcpu, void *h_ptr, uint64_t gva, return copy_gva(vcpu, h_ptr, gva, size, err_code, fault_addr, 0); } -void init_e820(void) -{ - uint32_t i; - - if (boot_regs[0] == MULTIBOOT_INFO_MAGIC) { - struct multiboot_info *mbi = (struct multiboot_info *) - (hpa2hva((uint64_t)boot_regs[1])); - - pr_info("Multiboot info detected\n"); - if ((mbi->mi_flags & MULTIBOOT_INFO_HAS_MMAP) != 0U) { - struct multiboot_mmap *mmap = - (struct multiboot_mmap *) - hpa2hva((uint64_t)mbi->mi_mmap_addr); - e820_entries = mbi->mi_mmap_length / sizeof(struct multiboot_mmap); - if (e820_entries > E820_MAX_ENTRIES) { - pr_err("Too many E820 entries %d\n", - e820_entries); - e820_entries = E820_MAX_ENTRIES; - } - dev_dbg(ACRN_DBG_GUEST, - "mmap length 0x%x addr 0x%x entries %d\n", - mbi->mi_mmap_length, mbi->mi_mmap_addr, - e820_entries); - for (i = 0U; i < e820_entries; i++) { - e820[i].baseaddr = mmap[i].baseaddr; - e820[i].length = mmap[i].length; - e820[i].type = mmap[i].type; - - dev_dbg(ACRN_DBG_GUEST, - "mmap table: %d type: 0x%x\n", - i, mmap[i].type); - dev_dbg(ACRN_DBG_GUEST, - "Base: 0x%016llx length: 0x%016llx", - mmap[i].baseaddr, mmap[i].length); - } - } - } else { - ASSERT(false, "no multiboot info found"); - } -} - - -void obtain_e820_mem_info(void) -{ - uint32_t i; - struct e820_entry *entry; - - e820_mem.mem_bottom = UINT64_MAX; - e820_mem.mem_top = 0x0UL; - e820_mem.total_mem_size = 0UL; - e820_mem.max_ram_blk_base = 0UL; - e820_mem.max_ram_blk_size = 0UL; - - for (i = 0U; i < e820_entries; i++) { - entry = &e820[i]; - if (e820_mem.mem_bottom > entry->baseaddr) { - e820_mem.mem_bottom = entry->baseaddr; - } - - if ((entry->baseaddr + entry->length) - > e820_mem.mem_top) { - e820_mem.mem_top = entry->baseaddr - + entry->length; - } - - if (entry->type == E820_TYPE_RAM) { - e820_mem.total_mem_size += entry->length; - if (entry->baseaddr == UOS_DEFAULT_START_ADDR) { - e820_mem.max_ram_blk_base = - entry->baseaddr; - e820_mem.max_ram_blk_size = entry->length; - } - } - } -} - -static void rebuild_vm0_e820(void) -{ - uint32_t i; - uint64_t entry_start; - uint64_t entry_end; - uint64_t hv_start_pa = get_hv_image_base(); - uint64_t hv_end_pa = hv_start_pa + CONFIG_HV_RAM_SIZE; - struct e820_entry *entry, new_entry = {0}; - - /* hypervisor mem need be filter out from e820 table - * it's hv itself + other hv reserved mem like vgt etc - */ - for (i = 0U; i < e820_entries; i++) { - entry = &e820[i]; - entry_start = entry->baseaddr; - entry_end = entry->baseaddr + entry->length; - - /* No need handle in these cases*/ - if ((entry->type != E820_TYPE_RAM) || (entry_end <= hv_start_pa) - || (entry_start >= hv_end_pa)) { - continue; - } - - /* filter out hv mem and adjust length of this entry*/ - if ((entry_start < hv_start_pa) && (entry_end <= hv_end_pa)) { - entry->length = hv_start_pa - entry_start; - continue; - } - /* filter out hv mem and need to create a new entry*/ - if ((entry_start < hv_start_pa) && (entry_end > hv_end_pa)) { - entry->length = hv_start_pa - entry_start; - new_entry.baseaddr = hv_end_pa; - new_entry.length = entry_end - hv_end_pa; - new_entry.type = E820_TYPE_RAM; - continue; - } - /* This entry is within the range of hv mem - * change to E820_TYPE_RESERVED - */ - if ((entry_start >= hv_start_pa) && (entry_end <= hv_end_pa)) { - entry->type = E820_TYPE_RESERVED; - continue; - } - - if ((entry_start >= hv_start_pa) && (entry_start < hv_end_pa) - && (entry_end > hv_end_pa)) { - entry->baseaddr = hv_end_pa; - entry->length = entry_end - hv_end_pa; - continue; - } - - } - - if (new_entry.length > 0UL) { - e820_entries++; - ASSERT(e820_entries <= E820_MAX_ENTRIES, - "e820 entry overflow"); - entry = &e820[e820_entries - 1]; - entry->baseaddr = new_entry.baseaddr; - entry->length = new_entry.length; - entry->type = new_entry.type; - } - - e820_mem.total_mem_size -= CONFIG_HV_RAM_SIZE; -} - /** * @param[inout] vm pointer to a vm descriptor * @@ -655,57 +509,3 @@ int prepare_vm0_memmap_and_e820(struct acrn_vm *vm) ept_mr_del(vm, pml4_page, hv_hpa, CONFIG_HV_RAM_SIZE); return 0; } - -uint64_t e820_alloc_low_memory(uint32_t size_arg) -{ - uint32_t i; - uint32_t size = size_arg; - struct e820_entry *entry, *new_entry; - - /* We want memory in page boundary and integral multiple of pages */ - size = (((size + PAGE_SIZE) - 1U) >> PAGE_SHIFT) << PAGE_SHIFT; - - for (i = 0U; i < e820_entries; i++) { - entry = &e820[i]; - uint64_t start, end, length; - - start = round_page_up(entry->baseaddr); - end = round_page_down(entry->baseaddr + entry->length); - length = end - start; - length = (end > start) ? (end - start) : 0; - - /* Search for available low memory */ - if ((entry->type != E820_TYPE_RAM) - || (length < size) - || ((start + size) > MEM_1M)) { - continue; - } - - /* found exact size of e820 entry */ - if (length == size) { - entry->type = E820_TYPE_RESERVED; - e820_mem.total_mem_size -= size; - return start; - } - - /* - * found entry with available memory larger than requested - * alocate memory from the end of this entry at page boundary - */ - new_entry = &e820[e820_entries]; - new_entry->type = E820_TYPE_RESERVED; - new_entry->baseaddr = end - size; - new_entry->length = (entry->baseaddr + - entry->length) - new_entry->baseaddr; - - /* Shrink the existing entry and total available memory */ - entry->length -= new_entry->length; - e820_mem.total_mem_size -= new_entry->length; - e820_entries++; - - return new_entry->baseaddr; - } - - pr_fatal("Can't allocate memory under 1M from E820\n"); - return ACRN_INVALID_HPA; -} diff --git a/hypervisor/arch/x86/guest/vm.c b/hypervisor/arch/x86/guest/vm.c index df8970a15..859557d21 100644 --- a/hypervisor/arch/x86/guest/vm.c +++ b/hypervisor/arch/x86/guest/vm.c @@ -7,6 +7,7 @@ #include #include #include +#include #include /* Local variables */ diff --git a/hypervisor/arch/x86/mmu.c b/hypervisor/arch/x86/mmu.c index f9fee0b38..ff630a00f 100644 --- a/hypervisor/arch/x86/mmu.c +++ b/hypervisor/arch/x86/mmu.c @@ -29,6 +29,7 @@ #include #include +#include static void *ppt_mmu_pml4_addr; static uint8_t sanitized_page[PAGE_SIZE] __aligned(PAGE_SIZE); diff --git a/hypervisor/arch/x86/trampoline.c b/hypervisor/arch/x86/trampoline.c index 8db2cd564..010ea22cf 100644 --- a/hypervisor/arch/x86/trampoline.c +++ b/hypervisor/arch/x86/trampoline.c @@ -7,6 +7,7 @@ #include #include #include +#include uint64_t trampoline_start16_paddr; diff --git a/hypervisor/common/vm_load.c b/hypervisor/common/vm_load.c index 8a6118823..cbbe39550 100644 --- a/hypervisor/common/vm_load.c +++ b/hypervisor/common/vm_load.c @@ -7,37 +7,7 @@ #include #include #include - -#ifdef CONFIG_PARTITION_MODE -static uint32_t create_e820_table(struct e820_entry *param_e820) -{ - uint32_t i; - - for (i = 0U; i < NUM_E820_ENTRIES; i++) { - param_e820[i].baseaddr = e820_default_entries[i].baseaddr; - param_e820[i].length = e820_default_entries[i].length; - param_e820[i].type = e820_default_entries[i].type; - } - - return NUM_E820_ENTRIES; -} -#else -static uint32_t create_e820_table(struct e820_entry *param_e820) -{ - uint32_t i; - - ASSERT(e820_entries > 0U, - "e820 should be inited"); - - for (i = 0U; i < e820_entries; i++) { - param_e820[i].baseaddr = e820[i].baseaddr; - param_e820[i].length = e820[i].length; - param_e820[i].type = e820[i].type; - } - - return e820_entries; -} -#endif +#include static void prepare_bsp_gdt(struct acrn_vm *vm) { diff --git a/hypervisor/include/arch/x86/e820.h b/hypervisor/include/arch/x86/e820.h new file mode 100644 index 000000000..85ebfedec --- /dev/null +++ b/hypervisor/include/arch/x86/e820.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2018 Intel Corporation. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +struct e820_mem_params { + uint64_t mem_bottom; + uint64_t mem_top; + uint64_t total_mem_size; + uint64_t max_ram_blk_base; /* used for the start address of UOS */ + uint64_t max_ram_blk_size; +}; + +void init_e820(void); +void obtain_e820_mem_info(void); +void rebuild_vm0_e820(void); +uint64_t e820_alloc_low_memory(uint32_t size_arg); + +extern uint32_t e820_entries; +extern struct e820_entry e820[E820_MAX_ENTRIES]; +extern struct e820_mem_params e820_mem; + +uint32_t create_e820_table(struct e820_entry *param_e820); + +#ifdef CONFIG_PARTITION_MODE +/* + * Default e820 mem map: + * + * Assumption is every VM launched by ACRN in partition mode uses 2G of RAM. + * there is reserved memory of 64K for MPtable and PCI hole of 512MB + */ +#define NUM_E820_ENTRIES 5U +extern const struct e820_entry e820_default_entries[NUM_E820_ENTRIES]; +#endif diff --git a/hypervisor/include/arch/x86/guest/guest.h b/hypervisor/include/arch/x86/guest/guest.h index 42b0bade1..55642ac71 100644 --- a/hypervisor/include/arch/x86/guest/guest.h +++ b/hypervisor/include/arch/x86/guest/guest.h @@ -70,17 +70,7 @@ #define LDTR_AR (0x0082U) /* LDT, type must be 2, refer to SDM Vol3 26.3.1.2 */ #define TR_AR (0x008bU) /* TSS (busy), refer to SDM Vol3 26.3.1.2 */ -struct e820_mem_params { - uint64_t mem_bottom; - uint64_t mem_top; - uint64_t total_mem_size; - uint64_t max_ram_blk_base; /* used for the start address of UOS */ - uint64_t max_ram_blk_size; -}; - int prepare_vm0_memmap_and_e820(struct acrn_vm *vm); -uint64_t e820_alloc_low_memory(uint32_t size_arg); - /* Definition for a mem map lookup */ struct vm_lu_mem_map { struct list_head list; /* EPT mem map lookup list*/ @@ -107,25 +97,6 @@ int gva2gpa(struct acrn_vcpu *vcpu, uint64_t gva, uint64_t *gpa, uint32_t *err_c enum vm_paging_mode get_vcpu_paging_mode(struct acrn_vcpu *vcpu); -void init_e820(void); -void obtain_e820_mem_info(void); -extern uint32_t e820_entries; -extern struct e820_entry e820[E820_MAX_ENTRIES]; - -#ifdef CONFIG_PARTITION_MODE -/* - * Default e820 mem map: - * - * Assumption is every VM launched by ACRN in partition mode uses 2G of RAM. - * there is reserved memory of 64K for MPtable and PCI hole of 512MB - */ -#define NUM_E820_ENTRIES 5U -extern const struct e820_entry e820_default_entries[NUM_E820_ENTRIES]; -#endif - -extern uint32_t boot_regs[2]; -extern struct e820_mem_params e820_mem; - int rdmsr_vmexit_handler(struct acrn_vcpu *vcpu); int wrmsr_vmexit_handler(struct acrn_vcpu *vcpu); void init_msr_emulation(struct acrn_vcpu *vcpu); diff --git a/hypervisor/include/arch/x86/multiboot.h b/hypervisor/include/arch/x86/multiboot.h index dae1ef544..e20656a0a 100644 --- a/hypervisor/include/arch/x86/multiboot.h +++ b/hypervisor/include/arch/x86/multiboot.h @@ -80,4 +80,7 @@ struct multiboot_module { int parse_hv_cmdline(void); int init_vm_boot_info(struct acrn_vm *vm); + +/* boot_regs store the multiboot header address */ +extern uint32_t boot_regs[2]; #endif diff --git a/hypervisor/partition/apl-mrb/vm_description.c b/hypervisor/partition/apl-mrb/vm_description.c index 230a0df16..9718d558a 100644 --- a/hypervisor/partition/apl-mrb/vm_description.c +++ b/hypervisor/partition/apl-mrb/vm_description.c @@ -5,6 +5,7 @@ */ #include +#include #define NUM_USER_VMS 2U diff --git a/hypervisor/partition/cb2_dnv/vm_description.c b/hypervisor/partition/cb2_dnv/vm_description.c index 6bc525bda..998f53b07 100644 --- a/hypervisor/partition/cb2_dnv/vm_description.c +++ b/hypervisor/partition/cb2_dnv/vm_description.c @@ -5,6 +5,7 @@ */ #include +#include #define NUM_USER_VMS 2U