hv: Add split-lock emulation for LOCK prefix instruction

This patch adds the split-lock emulation.
If a #AC is caused by instruction with LOCK prefix then
emulate it, otherwise, inject it back as it used to be.

 1. Kick other vcpus of the guest to stop execution
    and set the TF flag to have #DB if the guest has more
    than one vcpu.

 2. Skip over the LOCK prefix and resume the current
    vcpu back to guest for execution.

 3. Notify other vcpus to restart exception at the end
    of handling the #DB since we have completed
    the LOCK prefix instruction emulation.

Tracked-On: #5605
Signed-off-by: Jie Deng <jie.deng@intel.com>
Acked-by: Eddie Dong <eddie.dong@intel.com>

hypervisor/arch/x86/cpu_caps.c

@@ -119,6 +119,17 @@ bool has_core_cap(uint32_t bit_mask)
 	return ((cpu_caps.core_caps & bit_mask) != 0U);
 }
 
+bool is_ac_enabled(void)
+{
+	bool ac_enabled = false;
+
+	if (has_core_cap(1U << 5U) && (msr_read(MSR_TEST_CTL) & (1U << 29U))) {
+		ac_enabled = true;
+	}
+
+	return ac_enabled;
+}
+
 static void detect_ept_cap(void)
 {
 	uint64_t msr_val;

hypervisor/arch/x86/guest/vcpu.c

@@ -206,6 +206,7 @@ static void vcpu_reset_internal(struct acrn_vcpu *vcpu, enum reset_mode mode)
 	vcpu->arch.exception_info.exception = VECTOR_INVALID;
 	vcpu->arch.cur_context = NORMAL_WORLD;
 	vcpu->arch.irq_window_enabled = false;
+	vcpu->arch.emulating_lock = false;
 	(void)memset((void *)vcpu->arch.vmcs, 0U, PAGE_SIZE);
 
 	for (i = 0; i < NR_WORLD; i++) {

hypervisor/arch/x86/guest/virq.c

@@ -378,6 +378,10 @@ int32_t acrn_handle_pending_request(struct acrn_vcpu *vcpu)
 			wait_event(&vcpu->events[VCPU_EVENT_SYNC_WBINVD]);
 		}
 
+		if (bitmap_test_and_clear_lock(ACRN_REQUEST_SPLIT_LOCK, pending_req_bits)) {
+			wait_event(&vcpu->events[VCPU_EVENT_SPLIT_LOCK]);
+		}
+
 		if (bitmap_test_and_clear_lock(ACRN_REQUEST_EPT_FLUSH, pending_req_bits)) {
 			invept(vcpu->vm->arch_vm.nworld_eptp);
 			if (vcpu->vm->sworld_control.flag.active != 0UL) {
@@ -480,6 +484,143 @@ static inline void acrn_inject_pending_intr(struct acrn_vcpu *vcpu,
 	}
 }
 
+static void vcpu_kick_splitlock_emulation(struct acrn_vcpu *cur_vcpu)
+{
+	struct acrn_vcpu *other;
+	uint16_t i;
+
+	if (cur_vcpu->vm->hw.created_vcpus > 1U) {
+		foreach_vcpu(i, cur_vcpu->vm, other) {
+			if (other != cur_vcpu) {
+				vcpu_make_request(other, ACRN_REQUEST_SPLIT_LOCK);
+			}
+		}
+	}
+}
+
+static void vcpu_complete_splitlock_emulation(struct acrn_vcpu *cur_vcpu)
+{
+	struct acrn_vcpu *other;
+	uint16_t i;
+
+	if (cur_vcpu->vm->hw.created_vcpus > 1U) {
+		foreach_vcpu(i, cur_vcpu->vm, other) {
+			if (other != cur_vcpu) {
+				signal_event(&other->events[VCPU_EVENT_SPLIT_LOCK]);
+			}
+		}
+	}
+}
+
+static bool is_guest_ac_enabled(struct acrn_vcpu *vcpu)
+{
+	bool ret = false;
+
+	if ((vcpu_get_guest_msr(vcpu, MSR_TEST_CTL) & (1UL << 29UL)) != 0UL) {
+		ret = true;
+	}
+
+	return ret;
+}
+
+static int32_t emulate_splitlock(struct acrn_vcpu *vcpu, uint32_t exception_vector, bool *queue_exception)
+{
+	int32_t status = 0;
+	uint8_t inst[1];
+	uint32_t err_code = 0U;
+	uint64_t fault_addr;
+
+	/* Queue the exception by default if the exception cannot be handled. */
+	*queue_exception = true;
+
+	/*
+	 * The split-lock detection is enabled by default if the platform supports it.
+	 * Here, we check if the split-lock detection is really enabled or not. If the
+	 * split-lock detection is enabled in the platform but not enabled in the guest
+	 * then we try to emulate it, otherwise, inject the exception back.
+	 */
+	if (is_ac_enabled() && !is_guest_ac_enabled(vcpu)) {
+		switch (exception_vector) {
+		case IDT_AC:
+			status = copy_from_gva(vcpu, inst, vcpu_get_rip(vcpu), 1U, &err_code, &fault_addr);
+			if (status < 0) {
+				pr_err("Error copy instruction from Guest!");
+				if (status == -EFAULT) {
+					vcpu_inject_pf(vcpu, fault_addr, err_code);
+					status = 0;
+					/* For this case, inject #PF, not to queue #AC */
+					*queue_exception = false;
+				}
+			} else {
+				/*
+				 * If #AC is caused by instruction with LOCK prefix, then emulate it,
+				 * otherwise, inject it back.
+				 */
+				if (inst[0] == 0xf0U) {  /* This is LOCK prefix */
+					/*
+					 * Kick other vcpus of the guest to stop execution
+					 * until the split-lock emulation being completed.
+					 */
+					vcpu_kick_splitlock_emulation(vcpu);
+
+					/*
+					 * Skip the LOCK prefix and re-execute the instruction.
+					 */
+					vcpu->arch.inst_len = 1U;
+					if (vcpu->vm->hw.created_vcpus > 1U) {
+						/*
+						 * Set the TF to have a #DB after running the split-lock
+						 * instruction and tag the emulating_lock to be true.
+						 */
+						vcpu_set_rflags(vcpu, vcpu_get_rflags(vcpu) | HV_ARCH_VCPU_RFLAGS_TF);
+						vcpu->arch.emulating_lock = true;
+					}
+
+					/* Skip the #AC, we have emulated it. */
+					*queue_exception = false;
+				} else {
+					/*
+					 * TODO:
+					 * The xchg may also cause #AC for split-lock check.
+					 * Do xchg emulation here.
+					 */
+				}
+			}
+
+			break;
+		case IDT_DB:
+			/*
+			 * We only handle #DB caused by split-lock emulation,
+			 * otherwise, inject it back.
+			 */
+			if (vcpu->arch.emulating_lock) {
+				/*
+				 * The split-lock emulation has been completed, tag the emulating_lock
+				 * to be false, and clear the TF flag.
+				 */
+				vcpu->arch.emulating_lock = false;
+				vcpu_set_rflags(vcpu, vcpu_get_rflags(vcpu) & (~HV_ARCH_VCPU_RFLAGS_TF));
+				/*
+				 * Notify other vcpus of the guest to restart execution.
+				 */
+				vcpu_complete_splitlock_emulation(vcpu);
+
+				/* This #DB is for split-lock emulation, do not inject it */
+				*queue_exception = false;
+
+				/* This case we should not skip the instruction */
+				vcpu->arch.inst_len = 0U;
+			}
+
+			break;
+		default:
+			break;
+		}
+	}
+
+	return status;
+}
+
 /*
  * @pre vcpu != NULL
  */
@@ -489,6 +630,7 @@ int32_t exception_vmexit_handler(struct acrn_vcpu *vcpu)
 	uint32_t exception_vector = VECTOR_INVALID;
 	uint32_t cpl;
 	int32_t status = 0;
+	bool queue_exception;
 
 	pr_dbg(" Handling guest exception");
 
@@ -515,10 +657,11 @@ int32_t exception_vmexit_handler(struct acrn_vcpu *vcpu)
 		}
 	}
 
-	/* Handle all other exceptions */
-	vcpu_retain_rip(vcpu);
-
-	status = vcpu_queue_exception(vcpu, exception_vector, int_err_code);
+	status = emulate_splitlock(vcpu, exception_vector, &queue_exception);
+	if ((status == 0) && queue_exception) {
+		vcpu_retain_rip(vcpu);
+		status = vcpu_queue_exception(vcpu, exception_vector, int_err_code);
+	}
 
 	if (exception_vector == IDT_MC) {
 		/* just print error message for #MC, it then will be injected

hypervisor/arch/x86/guest/vmcs.c

@@ -367,9 +367,13 @@ static void init_exec_ctrl(struct acrn_vcpu *vcpu)
 
 	/* Set up guest exception mask bitmap setting a bit * causes a VM exit
 	 * on corresponding guest * exception - pg 2902 24.6.3
-	 * enable VM exit on MC only
+	 * enable VM exit on MC always
+	 * enable AC and DB for split lock emulation when split lock detection is enabled on physical platform.
 	 */
 	value32 = (1U << IDT_MC);
+	if (is_ac_enabled()) {
+		value32 = (value32 | (1U << IDT_AC) | (1U << IDT_DB));
+	}
 	exec_vmwrite32(VMX_EXCEPTION_BITMAP, value32);
 
 	/* Set up page fault error code mask - second paragraph * pg 2902

hypervisor/include/arch/x86/cpu_caps.h

@@ -53,6 +53,7 @@ bool pcpu_has_vmx_ept_cap(uint32_t bit_mask);
 bool pcpu_has_vmx_vpid_cap(uint32_t bit_mask);
 bool is_apl_platform(void);
 bool has_core_cap(uint32_t bit_mask);
+bool is_ac_enabled(void);
 void init_pcpu_capabilities(void);
 void init_pcpu_model_name(void);
 int32_t detect_hardware_support(void);

hypervisor/include/arch/x86/guest/vcpu.h

@@ -98,6 +98,11 @@
  */
 #define ACRN_REQUEST_WAIT_WBINVD		9U
 
+/**
+ * @brief Request for split lock operation
+ */
+#define ACRN_REQUEST_SPLIT_LOCK			10U
+
 /**
  * @}
  */
@@ -154,7 +159,9 @@ enum vm_cpu_mode {
 #define	VCPU_EVENT_IOREQ		0
 #define	VCPU_EVENT_VIRTUAL_INTERRUPT	1
 #define	VCPU_EVENT_SYNC_WBINVD		2
-#define	VCPU_EVENT_NUM			3
+#define VCPU_EVENT_SPLIT_LOCK		3
+#define	VCPU_EVENT_NUM			4
+
 
 enum reset_mode;
 
@@ -230,6 +237,7 @@ struct acrn_vcpu_arch {
 
 	uint8_t lapic_mask;
 	bool irq_window_enabled;
+	bool emulating_lock;
 	uint32_t nrexits;
 
 	/* VCPU context state information */

hypervisor/include/arch/x86/irq.h

@@ -130,6 +130,7 @@ uint32_t alloc_irq_num(uint32_t req_irq);
 uint32_t alloc_irq_vector(uint32_t irq);
 
 /* RFLAGS */
+#define HV_ARCH_VCPU_RFLAGS_TF              (1UL<<8U)
 #define HV_ARCH_VCPU_RFLAGS_IF              (1UL<<9U)
 #define HV_ARCH_VCPU_RFLAGS_RF              (1UL<<16U)