hv: vm: properly reset pCPUs with LAPIC PT enabled during VM shutdown/reset

When a VM is configured with LAPIC PT mode and its vCPU is in x2APIC
mode, the corresponding pCPU needs to be reset during VM shutdown/reset
as its physical LAPIC was used by its guest.

This commit fixes an issue where this reset never happens.
is_lapic_pt_enabled() needs to be called before reset_vcpu() to be able
to correctly reflect a vCPU's APIC mode.

A vCPU with LAPIC PT mode but in xAPIC mode does not require such reset,
since its physical LAPIC was not touched by its guest directly.

v2 -> v3:
- refine edge case detection logic

v1 -> v2:
- use a separate function to return the bitmap of LAPIC PT enabled pCPUs

Tracked-On: #3708
Signed-off-by: Peter Fang <peter.fang@intel.com>
Reviewed-by: Eddie Dong <eddie.dong@intel.com>
Reviewed-by: Jack Ren <jack.ren@intel.com>

hypervisor/arch/x86/guest/vm.c

@@ -412,6 +412,31 @@ static void register_pm_io_handler(struct acrn_vm *vm)
 	}
 }
 
+/**
+ * @brief get bitmap of pCPUs whose vCPUs have LAPIC PT enabled
+ *
+ * @param[in] vm pointer to vm data structure
+ * @pre vm != NULL
+ *
+ * @return pCPU bitmap
+ */
+static uint64_t lapic_pt_enabled_pcpu_bitmap(struct acrn_vm *vm)
+{
+	uint16_t i;
+	struct acrn_vcpu *vcpu;
+	uint64_t bitmap = 0UL;
+
+	if (is_lapic_pt_configured(vm)) {
+		foreach_vcpu(i, vm, vcpu) {
+			if (is_lapic_pt_enabled(vcpu)) {
+				bitmap_set_nolock(vcpu->pcpu_id, &bitmap);
+			}
+		}
+	}
+
+	return bitmap;
+}
+
 /**
  * @pre vm_id < CONFIG_MAX_VM_NUM && vm_config != NULL && rtn_vm != NULL
  * @pre vm->state == VM_POWERED_OFF
@@ -562,7 +587,8 @@ int32_t create_vm(uint16_t vm_id, struct acrn_vm_config *vm_config, struct acrn_
 int32_t shutdown_vm(struct acrn_vm *vm)
 {
 	uint16_t i;
-	uint64_t mask = 0UL;
+	uint16_t this_pcpu_id;
+	uint64_t mask;
 	struct acrn_vcpu *vcpu = NULL;
 	struct acrn_vm_config *vm_config = NULL;
 	int32_t ret = 0;
@@ -572,20 +598,31 @@ int32_t shutdown_vm(struct acrn_vm *vm)
 	/* Only allow shutdown paused vm */
 	if (vm->state == VM_PAUSED) {
 		vm->state = VM_POWERED_OFF;
+		this_pcpu_id = get_pcpu_id();
+		mask = lapic_pt_enabled_pcpu_bitmap(vm);
+
+		/*
+		 * If the current pcpu needs to offline itself,
+		 * it will be done after shutdown_vm() completes
+		 * in the idle thread.
+		 */
+		if (bitmap_test(this_pcpu_id, &mask)) {
+			bitmap_clear_nolock(this_pcpu_id, &mask);
+			make_pcpu_offline(this_pcpu_id);
+		}
 
 		foreach_vcpu(i, vm, vcpu) {
 			reset_vcpu(vcpu);
 			offline_vcpu(vcpu);
 
-			if (is_lapic_pt_enabled(vcpu)) {
+			if (bitmap_test(vcpu->pcpu_id, &mask)) {
-				bitmap_set_nolock(vcpu->pcpu_id, &mask);
 				make_pcpu_offline(vcpu->pcpu_id);
 			}
 		}
 
 		wait_pcpus_offline(mask);
 
-		if (is_lapic_pt_configured(vm) && !start_pcpus(mask)) {
+		if ((mask != 0UL) && (!start_pcpus(mask))) {
 			pr_fatal("Failed to start all cpus in mask(0x%llx)", mask);
 			ret = -ETIMEDOUT;
 		}
@@ -608,10 +645,8 @@ int32_t shutdown_vm(struct acrn_vm *vm)
 
 		/* Free EPT allocated resources assigned to VM */
 		destroy_ept(vm);
-
-		ret = 0;
 	} else {
-	        ret = -EINVAL;
+		ret = -EINVAL;
 	}
 
 	/* Return status to caller */
@@ -638,17 +673,43 @@ void start_vm(struct acrn_vm *vm)
 int32_t reset_vm(struct acrn_vm *vm)
 {
 	uint16_t i;
+	uint16_t this_pcpu_id;
+	uint64_t mask;
 	struct acrn_vcpu *vcpu = NULL;
-	int32_t ret;
+	int32_t ret = 0;
 
 	if (vm->state == VM_PAUSED) {
+		this_pcpu_id = get_pcpu_id();
+		mask = lapic_pt_enabled_pcpu_bitmap(vm);
+
+		/*
+		 * The current pcpu can't reset itself
+		 */
+		if (bitmap_test(this_pcpu_id, &mask)) {
+			pr_warn("%s: cannot offline self(%u)",
+				__func__, this_pcpu_id);
+			bitmap_clear_nolock(this_pcpu_id, &mask);
+			ret = -EINVAL;
+		}
+
 		foreach_vcpu(i, vm, vcpu) {
 			reset_vcpu(vcpu);
+
+			if (bitmap_test(vcpu->pcpu_id, &mask)) {
+				make_pcpu_offline(vcpu->pcpu_id);
+			}
 		}
+
+		wait_pcpus_offline(mask);
+
+		if ((mask != 0UL) && (!start_pcpus(mask))) {
+			pr_fatal("Failed to start all cpus in mask(0x%llx)", mask);
+			ret = -ETIMEDOUT;
+		}
+
 		/*
 		 * Set VM vLAPIC state to VM_VLAPIC_XAPIC
 		 */
-
 		vm->arch_vm.vlapic_state = VM_VLAPIC_XAPIC;
 
 		if (is_sos_vm(vm)) {
@@ -660,10 +721,8 @@ int32_t reset_vm(struct acrn_vm *vm)
 		destroy_secure_world(vm, false);
 		vm->sworld_control.flag.active = 0UL;
 		vm->state = VM_CREATED;
-
-		ret = 0;
 	} else {
-		ret = -1;
+		ret = -EINVAL;
 	}
 
 	return ret;