hv: add method to deal with ">4G MMIO" BIOS option

1. Fixed the bug that HV may crush when ">4G MMIO"
BIOS option is disabled.
2. Fixed the bug that RTVM may encounter problems
at reboot time when it makes use of pSRAM
3. HV will skip PTCM initialization when it cannot
find PTCM.
4. Some codes are refined.

Tracked-On: #5330

Signed-off-by: Qian Wang <qian1.wang@intel.com>

hypervisor/arch/x86/cpu.c

@@ -28,7 +28,7 @@
 #include <sgx.h>
 #include <uart16550.h>
 #include <ivshmem.h>
-#include <ptct.h>
+#include <ptcm.h>
 
 #define CPU_UP_TIMEOUT		100U /* millisecond */
 #define CPU_DOWN_TIMEOUT	100U /* millisecond */

hypervisor/arch/x86/ptcm.c

@@ -6,6 +6,7 @@
 #include <types.h>
 #include <logmsg.h>
 #include <misc_cfg.h>
+#include <mmu.h>
 #include <ptcm.h>
 
 uint64_t psram_area_bottom;
@@ -26,29 +27,26 @@ static void parse_ptct(void)
 	struct ptcm_mem_region *p_mem_region;
 	struct ptct_entry_data_psram *psram_entry;
 	struct ptct_entry_data_ptcm_binary* ptcm_binary_entry;
-	struct acpi_table_ptct* acpi_ptct = ((struct acpi_table_ptct *)get_acpi_tbl(ACPI_SIG_PTCT));
-	pr_fatal("find PTCT subtable in HPA %llx, length: %d", acpi_ptct, acpi_ptct->header.length);
+	struct acpi_table_ptct* acpi_ptct = (struct acpi_table_ptct *)get_acpi_tbl(ACPI_SIG_PTCT);
+	pr_info("found PTCT subtable in HPA %llx, length: %d", acpi_ptct, acpi_ptct->header.length);
 
 	entry = &acpi_ptct->ptct_first_entry;	//&acpi_ptct->ptct_entries[0];
-	pr_fatal("find PTCT base entry, in HPA %llx", entry);
 
 	psram_area_bottom = PSRAM_BASE_HPA;
 
 	while (((uint64_t)entry - (uint64_t)acpi_ptct) < acpi_ptct->header.length) {
 		switch (entry->type) {
 		case PTCT_ENTRY_TYPE_PTCM_BINARY:
-			pr_fatal("PTCT_ENTRY_TYPE_PTCM_BINARY");
 			ptcm_binary_entry = (struct ptct_entry_data_ptcm_binary *)entry->data;
 			ptcm_binary.address = ptcm_binary_entry->address;
 			ptcm_binary.size = ptcm_binary_entry->size;
 			if (psram_area_top < ptcm_binary.address + ptcm_binary.size) {
 				psram_area_top = ptcm_binary.address + ptcm_binary.size;
 			}
-			pr_fatal("find PTCM bin, in HPA %llx, size %llx", ptcm_binary.address, ptcm_binary.size);
+			pr_info("found PTCM bin, in HPA %llx, size %llx", ptcm_binary.address, ptcm_binary.size);
 			break;
 		
 		case PTCT_ENTRY_TYPE_PSRAM:
-			pr_fatal("PTCT_ENTRY_TYPE_PSRAM");
 			psram_entry = (struct ptct_entry_data_psram *)entry->data;
 			if (psram_entry->apic_id_0 >= MAX_PCPU_NUM) {
 				break;
@@ -57,7 +55,7 @@ static void parse_ptct(void)
 			p_mem_region = &ptcm_mem_regions[psram_entry->apic_id_0];
 			if (psram_entry->cache_level == 3) {
 				if (l3_psram_regions_count >= MAX_L3_PSRAM_REGIONS){
-					pr_fatal("Too many L3 regions!");
+					pr_err("Too many L3 regions!");
 					break;
 				}
 
@@ -71,10 +69,10 @@ static void parse_ptct(void)
 					psram_area_top = p_mem_region->l3_base + p_mem_region->l3_size;
 				}
 				l3_psram_regions_count++;
-				pr_fatal("found L3 psram, in HPA %llx, size %llx", p_mem_region->l3_base, p_mem_region->l3_size);
+				pr_info("found L3 psram, in HPA %llx, size %llx", p_mem_region->l3_base, p_mem_region->l3_size);
 			} else if (psram_entry->cache_level == 2) {
 				if (l2_psram_regions_count >= MAX_L2_PSRAM_REGIONS){
-					pr_fatal("Too many L2 regions!");
+					pr_err("Too many L2 regions!");
 					break;
 				}
 				p_mem_region->l2_valid = true;
@@ -87,46 +85,57 @@ static void parse_ptct(void)
 					psram_area_top = p_mem_region->l2_base + p_mem_region->l2_size;
 				}
 				l2_psram_regions_count++;
-				pr_fatal("found L2 psram, in HPA %llx, size %llx", psram_entry->base, psram_entry->size);
+				pr_info("found L2 psram, in HPA %llx, size %llx", psram_entry->base, psram_entry->size);
 			}
 			break;
-
-		/* In current phase, we ignore other entries like gt_clos and wrc_close*/
+		/* In current phase, we ignore other entries like gt_clos and wrc_close */
 		default: 
 			break;
 		}
 		/* point to next ptct entry */
-		pr_fatal("%s entry: 0x%lx, size: %d\n", __func__, entry, entry->size);
 		entry = (struct ptct_entry *)((uint64_t)entry + entry->size);
 	}
+	psram_area_top = round_page_up(psram_area_top);
+	psram_area_bottom = round_page_down(psram_area_bottom);
 }
 
 static volatile uint64_t ptcm_command_interface_offset;
 static volatile struct ptct_entry* ptct_base_entry;
 static volatile ptcm_command_abi ptcm_command_interface = NULL;
+/* psram_is_initialized is used to tell whether psram is successfully initialized for all cores */
 static volatile bool psram_is_initialized = false;
 
-void init_psram(bool is_bsp)
+int32_t init_psram(bool is_bsp)
 {
 	uint32_t magic,version;
-	int ret;
+	int32_t ptcm_ret_code;
+	int ret = 0;
 
-	if (get_acpi_tbl(ACPI_SIG_PTCT) != NULL) {
+	/* When we shut down an RTVM, its pCPUs will be re-initialized
+	we must ensure init_psram() will only be executed at the first time when a pcpu is booted */
+	if (get_acpi_tbl(ACPI_SIG_PTCT) != NULL && psram_is_initialized == false) {
 		if (is_bsp) {
 			parse_ptct();
-			pr_fatal("PTCT is parsed by BSP");
+			pr_info("PTCT is parsed by BSP");
 			ptct_base_entry = (struct ptct_entry*)((uint64_t)get_acpi_tbl(ACPI_SIG_PTCT) + 0x24);
-			pr_fatal("ptct_base_entry is found by BSP at %llx", ptct_base_entry);
+			pr_info("ptct_base_entry is found by BSP at %llx", ptct_base_entry);
 			magic = ((uint32_t *)ptcm_binary.address)[0];
 			version = ((uint32_t *)ptcm_binary.address)[1];
-			ptcm_command_interface_offset =*(uint64_t *)(ptcm_binary.address + 0x8);
-			pr_fatal("ptcm_bin_address:%llx", ptcm_binary.address);
-			pr_fatal("ptcm_command_interface_offset is %llx", ptcm_command_interface_offset);
-			pr_fatal("magic:%x", magic);
-			pr_fatal("version:%x", version);
+			ptcm_command_interface_offset = *(uint64_t *)(ptcm_binary.address + 0x8);
+			pr_info("ptcm_bin_address:%llx", ptcm_binary.address);
+			pr_info("ptcm magic:%x", magic);
+			pr_info("ptcm version:%x", version);
 
-			ptcm_command_interface = (ptcm_command_abi)(ptcm_binary.address + ptcm_command_interface_offset);
-			pr_fatal("ptcm_command_interface is found at %llx",ptcm_command_interface);
+			if (magic != PTCM_MAGIC ){
+				pr_err("Incorrect PTCM magic! Please turn off 'Above 4G MMIO Assignment' option in BIOS");
+				psram_area_bottom = psram_area_top = 0;
+				ptcm_command_interface = (ptcm_command_abi)(-1);
+			}
+			else
+			{
+				ptcm_command_interface = (ptcm_command_abi)(ptcm_binary.address + ptcm_command_interface_offset);
+				pr_info("ptcm command interface is found at %llx",ptcm_command_interface);
+			}
 		} else {
 			//all AP should wait until BSP finishes parsing PTCT and finding the command interface.
 			while (!ptcm_command_interface) {
@@ -134,24 +143,31 @@ void init_psram(bool is_bsp)
 			}
 		}
 
-		ret = ptcm_command_interface(PTCM_CMD_INIT_PSRAM, (void *)ptct_base_entry);
-		pr_fatal("PTCM initialization for core %d with return code %d!!!!!!!!!!!!!", get_pcpu_id(), ret);
-		/* TODO: to handle the return errno gracefully */
-		ASSERT(ret == PTCM_STATUS_SUCCESS);
-
-		/* wait until all cores finishes pSRAM initialization*/
-		if (is_bsp){
-			psram_is_initialized = true;
-			pr_fatal("BSP pSRAM has been initialized\n");
-		} else{
-			while (psram_is_initialized) {
-				continue;
+		if ((int64_t)ptcm_command_interface != -1){ /* ptcm_command_interface is found */
+			ptcm_ret_code = ptcm_command_interface(PTCM_CMD_INIT_PSRAM, (void *)ptct_base_entry);
+			pr_info("ptcm initialization for core %d with return code %d", get_pcpu_id(), ptcm_ret_code);
+			/* TODO: to handle the return errno gracefully */
+			ASSERT(ptcm_ret_code == PTCM_STATUS_SUCCESS);
+			/* wait until all cores finishes pSRAM initialization*/
+			if (is_bsp){
+				psram_is_initialized = true;
+				pr_info("BSP pSRAM has been initialized\n");
+			}
+			else{
+				while (!psram_is_initialized) {
+					continue;
+				}
 			}
 		}
+		else {
+			ret = -1;
+		}
 	}
+	return ret;
 }
 #else
-void init_psram(__unused bool is_bsp)
+int32_t init_psram(__unused bool is_bsp)
 {
+	return -1;
 }
 #endif

hypervisor/include/arch/x86/ptcm.h

@@ -18,6 +18,8 @@ typedef int32_t MSABI (*ptcm_command_abi)(uint32_t command, void *command_struct
 #define PTCM_CMD_RDMSR (int32_t)3U
 #define PTCM_CMD_WRMSR (int32_t)4U
 
+#define PTCM_MAGIC 0x5054434dU
+
 #define PCTM_L2_CLOS_MASK_MAX_NUM 8U
 #define PCTM_L3_CLOS_MASK_MAX_NUM 4U
 
@@ -37,4 +39,5 @@ struct ptcm_header
     uint64_t command_interface_offset;
 };
 
+int32_t init_psram(bool is_bsp);
 #endif /* PTCM_H */

hypervisor/include/arch/x86/ptct.h

@@ -101,5 +101,4 @@ struct ptcm_mem_region
 
 extern uint64_t psram_area_bottom;
 extern uint64_t psram_area_top;
-void init_psram(bool is_bsp);
 #endif	/* PTCT_H */