hv: multi-arch: abstract some pcpu related interfaces and macros

The following pcpu related interfaces are moved into common:
- common/cpu.c::start_pcpus
- include/common/cpu.h::get_pcpu_id
- include/common/cpu.h::set_current_pcpu_id
Their arch specific implementations are moved into arch/$(ARCH):
- arch/$(ARCH)/cpu.c::arch_start_pcpu
- include/arch/$(ARCH)/asm/cpu.h::arch_get_pcpu_id
- include/arch/$(ARCH)/asm/cpu.h::arch_set_current_pcpu_id
The following interface is moved into common:
- pcpu_set_current_state (from arch/x86/cpu.c -> common/cpu.c)
The following MACROs are moved into include/common/cpu.h:
- CPU_UP_TIMEOUT
- CPU_DOWN_TIMEOUT
- BSP_CPU_ID
- INVALID_CPU_ID

Tracked-On: #8791
Signed-off-by: Jian Jun Chen <jian.jun.chen@intel.com>
Acked-by: Wang, Yu1 <yu1.wang@intel.com>

hypervisor/arch/x86/cpu.c

@@ -41,16 +41,12 @@
 #include <common/notify.h>
 #include <cpu.h>
 
-#define CPU_UP_TIMEOUT		100U /* millisecond */
-#define CPU_DOWN_TIMEOUT	100U /* millisecond */
-
 static uint16_t phys_cpu_num = 0U;
 static uint64_t pcpu_sync = 0UL;
 static uint64_t startup_paddr = 0UL;
 
 static void init_pcpu_xsave(void);
 static void init_keylocker(void);
-static void set_current_pcpu_id(uint16_t pcpu_id);
 static void print_hv_banner(void);
 static uint16_t get_pcpu_id_from_lapic_id(uint32_t lapic_id);
 static uint64_t start_tick __attribute__((__section__(".bss_noinit")));
@@ -86,19 +82,6 @@ static bool init_percpu_lapic_id(void)
 	return success;
 }
 
-static void pcpu_set_current_state(uint16_t pcpu_id, enum pcpu_boot_state state)
-{
-	/* Check if state is initializing */
-	if (state == PCPU_STATE_INITIALIZING) {
-
-		/* Save this CPU's logical ID to the TSC AUX MSR */
-		set_current_pcpu_id(pcpu_id);
-	}
-
-	/* Set state for the specified CPU */
-	per_cpu(boot_state, pcpu_id) = state;
-}
-
 static void enable_ac_for_splitlock(void)
 {
 #ifdef CONFIG_SPLIT_LOCK_DETECTION_ENABLED
@@ -341,10 +324,8 @@ static uint16_t get_pcpu_id_from_lapic_id(uint32_t lapic_id)
 	return pcpu_id;
 }
 
-static void start_pcpu(uint16_t pcpu_id)
+void arch_start_pcpu(uint16_t pcpu_id)
 {
-	uint32_t timeout;
-
 	/* Update the stack for pcpu */
 	stac();
 	write_trampoline_stack_sym(pcpu_id);
@@ -355,54 +336,6 @@ static void start_pcpu(uint16_t pcpu_id)
 	 */
 	cpu_memory_barrier();
 	send_startup_ipi(pcpu_id, startup_paddr);
-
-	/* Wait until the pcpu with pcpu_id is running and set the active bitmap or
-	 * configured time-out has expired
-	 */
-	timeout = CPU_UP_TIMEOUT * 1000U;
-	while (!is_pcpu_active(pcpu_id) && (timeout != 0U)) {
-		/* Delay 10us */
-		udelay(10U);
-
-		/* Decrement timeout value */
-		timeout -= 10U;
-	}
-
-	/* Check to see if expected CPU is actually up */
-	if (!is_pcpu_active(pcpu_id)) {
-		pr_fatal("Secondary CPU%hu failed to come up", pcpu_id);
-		pcpu_set_current_state(pcpu_id, PCPU_STATE_DEAD);
-	}
-}
-
-
-/**
- * @brief Start all cpus if the bit is set in mask except itself
- *
- * @param[in] mask bits mask of cpus which should be started
- *
- * @return true if all cpus set in mask are started
- * @return false if there are any cpus set in mask aren't started
- */
-bool start_pcpus(uint64_t mask)
-{
-	uint16_t i;
-	uint16_t pcpu_id = get_pcpu_id();
-	uint64_t expected_start_mask = mask;
-
-	i = ffs64(expected_start_mask);
-	while (i != INVALID_BIT_INDEX) {
-		bitmap_clear_non_atomic(i, &expected_start_mask);
-
-		if (pcpu_id == i) {
-			continue; /* Avoid start itself */
-		}
-
-		start_pcpu(i);
-		i = ffs64(expected_start_mask);
-	}
-
-	return check_pcpus_active(mask);
 }
 
 void make_pcpu_offline(uint16_t pcpu_id)
@@ -508,12 +441,6 @@ void cpu_dead(void)
 	}
 }
 
-static void set_current_pcpu_id(uint16_t pcpu_id)
-{
-	/* Write TSC AUX register */
-	msr_write(ACRN_PSEUDO_PCPUID_MSR, (uint32_t) pcpu_id);
-}
-
 static void print_hv_banner(void)
 {
 	const char *boot_msg = "ACRN Hypervisor\n\r";

hypervisor/arch/x86/exception.c

@@ -4,9 +4,9 @@
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
-#include <asm/cpu.h>
 #include <asm/irq.h>
 #include <debug/dump.h>
+#include <cpu.h>
 
 void dispatch_exception(struct intr_excp_ctx *ctx)
 {

hypervisor/arch/x86/hw_thermal.c

@@ -8,7 +8,7 @@
 #include <softirq.h>
 #include <irq.h>
 #include <logmsg.h>
-#include <asm/cpu.h>
+#include <cpu.h>
 #include <asm/msr.h>
 #include <asm/irq.h>
 #include <asm/apicreg.h>

hypervisor/arch/x86/nmi.c

@@ -9,6 +9,7 @@
 
 #include <asm/guest/vcpu.h>
 #include <asm/guest/virq.h>
+#include <cpu.h>
 
 void handle_nmi(__unused struct intr_excp_ctx *ctx)
 {

hypervisor/arch/x86/tsc_deadline_timer.c

@@ -9,7 +9,7 @@
 #include <irq.h>
 #include <logmsg.h>
 #include <timer.h>
-#include <asm/cpu.h>
+#include <cpu.h>
 #include <asm/msr.h>
 #include <asm/irq.h>
 #include <asm/apicreg.h>

hypervisor/common/cpu.c

@@ -3,7 +3,10 @@
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
+#include <per_cpu.h>
 #include <cpu.h>
+#include <delay.h>
+#include <logmsg.h>
 #include <bits.h>
 
 
@@ -47,3 +50,65 @@ uint64_t get_active_pcpu_bitmap(void)
 {
 	return pcpu_active_bitmap;
 }
+
+void pcpu_set_current_state(uint16_t pcpu_id, enum pcpu_boot_state state)
+{
+	/* Check if state is initializing */
+	if (state == PCPU_STATE_INITIALIZING) {
+
+		/* Save this CPU's logical ID to arch specific per-cpu reg */
+		set_current_pcpu_id(pcpu_id);
+	}
+
+	/* Set state for the specified CPU */
+	per_cpu(boot_state, pcpu_id) = state;
+}
+
+/**
+ * @brief Start all cpus if the bit is set in mask except itself
+ *
+ * @param[in] mask bits mask of cpus which should be started
+ *
+ * @return true if all cpus set in mask are started
+ * @return false if there are any cpus set in mask aren't started
+ */
+bool start_pcpus(uint64_t mask)
+{
+	uint16_t i;
+	uint16_t pcpu_id = get_pcpu_id();
+	uint64_t expected_start_mask = mask;
+	uint32_t timeout;
+
+	i = ffs64(expected_start_mask);
+	while (i != INVALID_BIT_INDEX) {
+		bitmap_clear_non_atomic(i, &expected_start_mask);
+
+		if (pcpu_id == i) {
+			continue; /* Avoid start itself */
+		}
+
+		arch_start_pcpu(i);
+
+		/* Wait until the pcpu with pcpu_id is running and set
+		 * the active bitmap or configured time-out has expired
+		 */
+		timeout = CPU_UP_TIMEOUT * 1000U;
+		while (!is_pcpu_active(i) && (timeout != 0U)) {
+			/* Delay 10us */
+			udelay(10U);
+
+			/* Decrement timeout value */
+			timeout -= 10U;
+		}
+
+		/* Check to see if expected CPU is actually up */
+		if (!is_pcpu_active(i)) {
+			pr_fatal("Secondary CPU%hu failed to come up", i);
+			pcpu_set_current_state(i, PCPU_STATE_DEAD);
+		}
+
+		i = ffs64(expected_start_mask);
+	}
+
+	return check_pcpus_active(mask);
+}

hypervisor/common/event.c

@@ -3,10 +3,10 @@
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
-#include <asm/cpu.h>
 #include <schedule.h>
 #include <event.h>
 #include <logmsg.h>
+#include <cpu.h>
 
 void init_event(struct sched_event *event)
 {

hypervisor/include/arch/x86/asm/cpu.h

@@ -287,16 +287,6 @@ extern uint64_t               secondary_cpu_stack[1];
  * to locate the per cpu data.
  */
 
-/* Boot CPU ID */
-#define BSP_CPU_ID             0U
-
-/**
- *The invalid cpu_id (INVALID_CPU_ID) is error
- *code for error handling, this means that
- *caller can't find a valid physical cpu
- *or virtual cpu.
- */
-#define INVALID_CPU_ID 0xffffU
 /**
  *The broadcast id (BROADCAST_CPU_ID)
  *used to notify all valid phyiscal cpu
@@ -447,7 +437,6 @@ void init_pcpu_pre(bool is_bsp);
  * hereby, pcpu_id is actually the current physcial cpu id.
  */
 void init_pcpu_post(uint16_t pcpu_id);
-bool start_pcpus(uint64_t mask);
 void wait_pcpus_offline(uint64_t mask);
 void stop_pcpus(void);
 void wait_sync_change(volatile const uint64_t *sync, uint64_t wake_sync);
@@ -651,11 +640,16 @@ cpu_rdtscp_execute(uint64_t *timestamp_ptr, uint32_t *cpu_id_ptr)
  * Macro to get CPU ID
  * @pre: the return CPU ID would never equal or large than phys_cpu_num.
  */
-static inline uint16_t get_pcpu_id(void)
+static inline uint16_t arch_get_pcpu_id(void)
 {
 	return (uint16_t)cpu_msr_read(ACRN_PSEUDO_PCPUID_MSR);
 }
 
+static inline void arch_set_current_pcpu_id(uint16_t pcpu_id)
+{
+	cpu_msr_write(ACRN_PSEUDO_PCPUID_MSR, (uint32_t)pcpu_id);
+}
+
 static inline uint64_t cpu_rsp_get(void)
 {
 	uint64_t ret;

hypervisor/include/arch/x86/asm/guest/vcpu.h

@@ -24,8 +24,8 @@
 #include <schedule.h>
 #include <event.h>
 #include <io_req.h>
+#include <cpu.h>
 #include <asm/msr.h>
-#include <asm/cpu.h>
 #include <asm/guest/instr_emul.h>
 #include <asm/guest/nested.h>
 #include <asm/vmx.h>

hypervisor/include/common/cpu.h

@@ -10,6 +10,17 @@
 #include <types.h>
 #include <asm/cpu.h>
 
+#define CPU_UP_TIMEOUT		100U /* millisecond */
+#define CPU_DOWN_TIMEOUT	100U /* millisecond */
+
+#define BSP_CPU_ID		0U /* Boot CPU ID */
+
+/**
+ * The invalid cpu_id (INVALID_CPU_ID) is error code for error handling,
+ * this means that caller can't find a valid physical cpu or virtual cpu.
+ */
+#define INVALID_CPU_ID		0xffffU
+
 /* CPU states defined */
 enum pcpu_boot_state {
         PCPU_STATE_RESET = 0U,
@@ -18,6 +29,10 @@ enum pcpu_boot_state {
         PCPU_STATE_HALTED,
         PCPU_STATE_DEAD,
 };
+
+static inline uint16_t arch_get_pcpu_id(void);
+static inline void arch_set_current_pcpu_id(uint16_t pcpu_id);
+void arch_start_pcpu(uint16_t pcpu_id);
 uint16_t arch_get_pcpu_num(void);
 uint16_t get_pcpu_nums(void);
 bool is_pcpu_active(uint16_t pcpu_id);
@@ -26,8 +41,20 @@ void clear_pcpu_active(uint16_t pcpu_id);
 bool check_pcpus_active(uint64_t mask);
 bool check_pcpus_inactive(uint64_t mask);
 uint64_t get_active_pcpu_bitmap(void);
+void pcpu_set_current_state(uint16_t pcpu_id, enum pcpu_boot_state state);
+bool start_pcpus(uint64_t mask);
 
 #define ALL_CPUS_MASK		((1UL << get_pcpu_nums()) - 1UL)
 #define AP_MASK			(ALL_CPUS_MASK & ~(1UL << BSP_CPU_ID))
 
+static inline uint16_t get_pcpu_id(void)
+{
+	return arch_get_pcpu_id();
+}
+
+static inline void set_current_pcpu_id(uint16_t pcpu_id)
+{
+	arch_set_current_pcpu_id(pcpu_id);
+}
+
 #endif /* COMMON_CPU_H */