hv: define posted interrupt IRQs/vectors

This is a preparation patch for adding support for VT-d PI
related vCPU scheduling.

ACRN does not support vCPU migration, one vCPU always runs on
the same pCPU, so PI's ndst is never changed after startup.

VCPUs of a VM won’t share same pCPU. So the maximum possible number
of VCPUs that can run on a pCPU is CONFIG_MAX_VM_NUM.

Allocate unique Activation Notification Vectors (ANV) for each vCPU
that belongs to the same pCPU, the ANVs need only be unique within each
pCPU, not across all vCPUs. This reduces # of pre-allocated ANVs for
posted interrupts to CONFIG_MAX_VM_NUM, and enables ACRN to avoid
switching between active and wake-up vector values in the posted
interrupt descriptor on vCPU scheduling state changes.

A total of CONFIG_MAX_VM_NUM consecutive IRQs/vectors are reserved
for posted interrupts use.

The code first initializes vcpu->arch.pid.control.bits.nv dynamically
(will be added in subsequent patch), the other code shall use
vcpu->arch.pid.control.bits.nv instead of the hard-coded notification vectors.

Rename some functions:
  apicv_post_intr --> apicv_trigger_pi_anv
  posted_intr_notification --> handle_pi_notification
  setup_posted_intr_notification --> setup_pi_notification

Tracked-On: #4506
Signed-off-by: dongshen <dongsheng.x.zhang@intel.com>
Reviewed-by: Eddie Dong <eddie.dong@Intel.com>

hypervisor/arch/x86/cpu.c

@@ -229,7 +229,7 @@ void init_pcpu_post(uint16_t pcpu_id)
 
 		timer_init();
 		setup_notification();
-		setup_posted_intr_notification();
+		setup_pi_notification();
 
 		if (init_iommu() != 0) {
 			panic("failed to initialize iommu!");

hypervisor/arch/x86/guest/vcpu.c

@@ -469,7 +469,7 @@ int32_t create_vcpu(uint16_t pcpu_id, struct acrn_vm *vm, struct acrn_vcpu **rtn
 		 * the "enable VPID" VM-execution control is 1, the current VPID
 		 * is the value of the VPID VM-execution control field in the VMCS.
 		 *
-		 * This assignment guarantees a unique non-zero per vcpu vpid in runtime.
+		 * This assignment guarantees a unique non-zero per vcpu vpid at runtime.
 		 */
 		vcpu->arch.vpid = 1U + (vm->vm_id * MAX_VCPUS_PER_VM) + vcpu->vcpu_id;
 

hypervisor/arch/x86/guest/vlapic.c

@@ -90,10 +90,9 @@ static inline void vlapic_dump_isr(__unused const struct acrn_vlapic *vlapic, __
 
 const struct acrn_apicv_ops *apicv_ops;
 
-static bool
+static bool apicv_set_intr_ready(struct acrn_vlapic *vlapic, uint32_t vector);
-apicv_set_intr_ready(struct acrn_vlapic *vlapic, uint32_t vector);
 
-static void apicv_post_intr(uint16_t dest_pcpu_id);
+static void apicv_trigger_pi_anv(uint16_t dest_pcpu_id, uint32_t anv);
 
 static void vlapic_x2apic_self_ipi_handler(struct acrn_vlapic *vlapic);
 
@@ -573,7 +572,7 @@ static void apicv_advanced_accept_intr(struct acrn_vlapic *vlapic, uint32_t vect
 		bitmap_set_lock(ACRN_REQUEST_EVENT, &vlapic->vcpu->arch.pending_req);
 
 		if (get_pcpu_id() != pcpuid_from_vcpu(vlapic->vcpu)) {
-			apicv_post_intr(pcpuid_from_vcpu(vlapic->vcpu));
+			apicv_trigger_pi_anv(pcpuid_from_vcpu(vlapic->vcpu), (uint32_t)(vlapic->vcpu->arch.pid.control.bits.nv));
 		}
 	}
 }
@@ -604,12 +603,13 @@ static void vlapic_accept_intr(struct acrn_vlapic *vlapic, uint32_t vector, bool
  * If pCPU in root-mode, virtual interrupt will be injected in next VM entry.
  *
  * @param[in] dest_pcpu_id Target CPU ID.
+ * @param[in] anv Activation Notification Vectors (ANV)
  *
  * @return None
  */
-static void apicv_post_intr(uint16_t dest_pcpu_id)
+static void apicv_trigger_pi_anv(uint16_t dest_pcpu_id, uint32_t anv)
 {
-	send_single_ipi(dest_pcpu_id, POSTED_INTR_VECTOR);
+	send_single_ipi(dest_pcpu_id, anv);
 }
 
 /**

hypervisor/arch/x86/guest/vmcs.c

@@ -352,7 +352,7 @@ static void init_exec_ctrl(struct acrn_vcpu *vcpu)
 		exec_vmwrite64(VMX_EOI_EXIT3_FULL, 0UL);
 
 		exec_vmwrite16(VMX_GUEST_INTR_STATUS, 0U);
-		exec_vmwrite16(VMX_POSTED_INTR_VECTOR, POSTED_INTR_VECTOR);
+		exec_vmwrite16(VMX_POSTED_INTR_VECTOR, (uint16_t)(vcpu->arch.pid.control.bits.nv));
 		exec_vmwrite64(VMX_PIR_DESC_ADDR_FULL, hva2hpa(get_pi_desc(vcpu)));
 	}
 

hypervisor/arch/x86/irq.c

@@ -37,8 +37,10 @@ struct static_mapping_table {
 static struct static_mapping_table irq_static_mappings[NR_STATIC_MAPPINGS] = {
 	{TIMER_IRQ, TIMER_VECTOR},
 	{NOTIFY_VCPU_IRQ, NOTIFY_VCPU_VECTOR},
-	{POSTED_INTR_IRQ, POSTED_INTR_VECTOR},
 	{PMI_IRQ, PMI_VECTOR},
+
+	/* To be initialized at runtime in init_irq_descs() */
+	[NR_STATIC_MAPPINGS_1 ... (NR_STATIC_MAPPINGS_1 + CONFIG_MAX_VM_NUM - 1U)] = {},
 };
 
 /*
@@ -421,6 +423,20 @@ static void init_irq_descs(void)
 {
 	uint32_t i;
 
+	/*
+	 * Fill in #CONFIG_MAX_VM_NUM posted interrupt specific irq and vector pairs
+	 * at runtime
+	 */
+	for (i = 0U; i < CONFIG_MAX_VM_NUM; i++) {
+		uint32_t idx = i + NR_STATIC_MAPPINGS_1;
+
+		ASSERT(irq_static_mappings[idx].irq == 0U, "");
+		ASSERT(irq_static_mappings[idx].vector == 0U, "");
+
+		irq_static_mappings[idx].irq = POSTED_INTR_IRQ + i;
+		irq_static_mappings[idx].vector = POSTED_INTR_VECTOR + i;
+	}
+
 	for (i = 0U; i < NR_IRQS; i++) {
 		irq_desc_array[i].irq = i;
 		irq_desc_array[i].vector = VECTOR_INVALID;

hypervisor/arch/x86/notify.c

@@ -98,21 +98,20 @@ void setup_notification(void)
 		notification_irq, irq_to_vector(notification_irq));
 }
 
-static void posted_intr_notification(__unused uint32_t irq, __unused void *data)
+static void handle_pi_notification(__unused uint32_t irq, __unused void *data)
 {
-	/* Dummy IRQ handler for case that Posted-Interrupt Notification
-	 * is sent to vCPU in root mode(isn't running),interrupt will be
-	 * picked up in next vmentry,do nothine here.
-	 */
 }
 
 /*pre-conditon: be called only by BSP initialization proccess*/
-void setup_posted_intr_notification(void)
+void setup_pi_notification(void)
 {
-	if (request_irq(POSTED_INTR_IRQ,
+	uint32_t i;
-			posted_intr_notification,
+
-			NULL, IRQF_NONE) < 0) {
+	for (i = 0U; i < CONFIG_MAX_VM_NUM; i++) {
-		pr_err("Failed to setup posted-intr notification");
+		if (request_irq(POSTED_INTR_IRQ + i, handle_pi_notification, NULL, IRQF_NONE) < 0) {
+			pr_err("Failed to setup pi notification");
+			break;
+		}
 	}
 }
 

hypervisor/include/arch/x86/irq.h

@@ -25,7 +25,24 @@
 #define NR_IRQS			256U
 #define IRQ_INVALID		0xffffffffU
 
-#define NR_STATIC_MAPPINGS     (4U)
+/* # of NR_STATIC_MAPPINGS_1 entries for timer, vcpu notify, and PMI */
+#define NR_STATIC_MAPPINGS_1	3U
+
+/*
+ * The static IRQ/Vector mapping table in irq.c consists of the following entries:
+ * # of NR_STATIC_MAPPINGS_1 entries for timer, vcpu notify, and PMI
+ *
+ * # of CONFIG_MAX_VM_NUM entries for posted interrupt notification, platform
+ * specific but known at build time:
+ * Allocate unique Activation Notification Vectors (ANV) for each vCPU that belongs
+ * to the same pCPU, the ANVs need only be unique within each pCPU, not across all
+ * vCPUs. The max numbers of vCPUs may be running on top of a pCPU is CONFIG_MAX_VM_NUM,
+ * since ACRN does not support 2 vCPUs of same VM running on top of same pCPU.
+ * This reduces # of pre-allocated ANVs for posted interrupts to CONFIG_MAX_VM_NUM,
+ * and enables ACRN to avoid switching between active and wake-up vector values
+ * in the posted interrupt descriptor on vCPU scheduling state changes.
+ */
+#define NR_STATIC_MAPPINGS	(NR_STATIC_MAPPINGS_1 + CONFIG_MAX_VM_NUM)
 
 #define HYPERVISOR_CALLBACK_VHM_VECTOR	0xF3U
 
@@ -39,13 +56,23 @@
 
 #define TIMER_VECTOR		(VECTOR_FIXED_START)
 #define NOTIFY_VCPU_VECTOR	(VECTOR_FIXED_START + 1U)
-#define POSTED_INTR_VECTOR	(VECTOR_FIXED_START + 2U)
+#define PMI_VECTOR		(VECTOR_FIXED_START + 2U)
-#define PMI_VECTOR		(VECTOR_FIXED_START + 3U)
+/*
+ * Starting vector for posted interrupts
+ * # of CONFIG_MAX_VM_NUM (POSTED_INTR_VECTOR ~ (POSTED_INTR_VECTOR + CONFIG_MAX_VM_NUM - 1U))
+ * consecutive vectors reserved for posted interrupts
+ */
+#define POSTED_INTR_VECTOR	(VECTOR_FIXED_START + NR_STATIC_MAPPINGS_1)
 
 #define TIMER_IRQ		(NR_IRQS - 1U)
 #define NOTIFY_VCPU_IRQ		(NR_IRQS - 2U)
-#define POSTED_INTR_IRQ		(NR_IRQS - 3U)
+#define PMI_IRQ			(NR_IRQS - 3U)
-#define PMI_IRQ			(NR_IRQS - 4U)
+/*
+ * Starting IRQ for posted interrupts
+ * # of CONFIG_MAX_VM_NUM (POSTED_INTR_IRQ ~ (POSTED_INTR_IRQ + CONFIG_MAX_VM_NUM - 1U))
+ * consecutive IRQs reserved for posted interrupts
+ */
+#define POSTED_INTR_IRQ	(NR_IRQS - NR_STATIC_MAPPINGS_1 - CONFIG_MAX_VM_NUM)
 
 /* the maximum number of msi entry is 2048 according to PCI
  * local bus specification
@@ -95,7 +122,7 @@ void init_default_irqs(uint16_t cpu_id);
 void dispatch_exception(struct intr_excp_ctx *ctx);
 
 void setup_notification(void);
-void setup_posted_intr_notification(void);
+void setup_pi_notification(void);
 
 typedef void (*spurious_handler_t)(uint32_t vector);
 extern spurious_handler_t spurious_handler;