hv: nested: support for INVEPT and INVVPID emulation

invvpid and invept instructions cause VM exits unconditionally.
For initial support, we pass all the instruction operands as is
to the pCPU.

Tracked-On: #5923
Signed-off-by: Sainath Grandhi <sainath.grandhi@intel.com>
Signed-off-by: Zide Chen <zide.chen@intel.com>
Acked-by: Eddie Dong <eddie.dong@intel.com>

hypervisor/Makefile

@@ -281,6 +281,7 @@ VP_BASE_C_SRCS += arch/x86/guest/hyperv.c
 endif
 ifeq ($(CONFIG_NVMX_ENABLED),y)
 VP_BASE_C_SRCS += arch/x86/guest/nested.c
+VP_BASE_C_SRCS += arch/x86/guest/vept.c
 endif
 VP_BASE_C_SRCS += boot/guest/vboot_info.c
 VP_BASE_C_SRCS += common/hv_main.c

hypervisor/arch/x86/guest/nested.c

@@ -1315,6 +1315,78 @@ int32_t vmlaunch_vmexit_handler(struct acrn_vcpu *vcpu)
 	return 0;
 }
 
+/*
+ * @pre vcpu != NULL
+ * @pre desc != NULL
+ */
+int64_t get_invvpid_ept_operands(struct acrn_vcpu *vcpu, void *desc, size_t size)
+{
+	const uint32_t info = exec_vmread(VMX_INSTR_INFO);
+	uint64_t gpa;
+
+	gpa = get_vmx_memory_operand(vcpu, info);
+	(void)copy_from_gpa(vcpu->vm, desc, gpa, size);
+
+	return vcpu_get_gpreg(vcpu, VMX_II_REG2(info));
+}
+
+/*
+ * @pre vcpu != NULL
+ */
+static bool validate_canonical_addr(struct acrn_vcpu *vcpu, uint64_t va)
+{
+	uint32_t addr_width = 48U; /* linear address width */
+	uint64_t msb_mask;
+
+	if (vcpu_get_cr4(vcpu) & CR4_LA57) {
+		addr_width = 57U;
+	}
+
+	/*
+	 * In 64-bit mode, an address is considered to be in canonical form if address
+	 * bits 63 through to the most-significant implemented bit by the microarchitecture
+	 * are set to either all ones or all zeros.
+	 */
+
+	msb_mask = ~((1UL << addr_width) - 1UL);
+	return ((msb_mask & va) == 0UL) || ((msb_mask & va) == msb_mask);
+}
+
+/*
+ * @pre vcpu != NULL
+ */
+int32_t invvpid_vmexit_handler(struct acrn_vcpu *vcpu)
+{
+	uint32_t supported_types = (vcpu_get_guest_msr(vcpu, MSR_IA32_VMX_EPT_VPID_CAP) >> 40U) & 0xfU;
+	struct invvpid_operand desc;
+	uint64_t type;
+
+	if (check_vmx_permission(vcpu)) {
+		type = get_invvpid_ept_operands(vcpu, (void *)&desc, sizeof(desc));
+
+		if ((type > VMX_VPID_TYPE_SINGLE_NON_GLOBAL) || ((supported_types & (1U << type)) == 0)) {
+			nested_vmx_result(VMfailValid, VMXERR_INVEPT_INVVPID_INVALID_OPERAND);
+		} else if ((desc.rsvd1 != 0U) || (desc.rsvd2 != 0U)) {
+			nested_vmx_result(VMfailValid, VMXERR_INVEPT_INVVPID_INVALID_OPERAND);
+		} else if ((type != VMX_VPID_TYPE_ALL_CONTEXT) && (desc.vpid == 0U)) {
+			/* check VPID for type 0, 1, 3 */
+			nested_vmx_result(VMfailValid, VMXERR_INVEPT_INVVPID_INVALID_OPERAND);
+		} else if ((type == VMX_VPID_TYPE_INDIVIDUAL_ADDR) && !validate_canonical_addr(vcpu, desc.gva)) {
+			nested_vmx_result(VMfailValid, VMXERR_INVEPT_INVVPID_INVALID_OPERAND);
+		} else {
+			/*
+			 * VPIDs are pass-thru. Values programmed by L1 are used by L0.
+			 * INVVPID type, VPID and GLA, operands of INVVPID instruction, are
+			 * passed as is to the pCPU.
+			 */
+			asm_invvpid(desc, type);
+			nested_vmx_result(VMsucceed, 0);
+		}
+	}
+
+	return 0;
+}
+
 void init_nested_vmx(__unused struct acrn_vm *vm)
 {
 	static bool initialized = false;

hypervisor/arch/x86/guest/vept.c

@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2021 Intel Corporation. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <types.h>
+#include <logmsg.h>
+#include <asm/mmu.h>
+#include <asm/guest/vcpu.h>
+#include <asm/guest/vm.h>
+#include <asm/guest/vmexit.h>
+#include <asm/guest/ept.h>
+#include <asm/guest/nested.h>
+
+/**
+ * @pre vcpu != NULL
+ */
+int32_t invept_vmexit_handler(struct acrn_vcpu *vcpu)
+{
+	struct invept_desc operand_gla_ept;
+	uint64_t type;
+
+	if (check_vmx_permission(vcpu)) {
+		type = get_invvpid_ept_operands(vcpu, (void *)&operand_gla_ept, sizeof(operand_gla_ept));
+
+		if (type > INVEPT_TYPE_ALL_CONTEXTS) {
+			nested_vmx_result(VMfailValid, VMXERR_INVEPT_INVVPID_INVALID_OPERAND);
+		} else {
+			operand_gla_ept.eptp = gpa2hpa(vcpu->vm, operand_gla_ept.eptp);
+			asm_invept(type, operand_gla_ept);
+			nested_vmx_result(VMsucceed, 0);
+		}
+	}
+
+	return 0;
+}

hypervisor/arch/x86/guest/vmexit.c

@@ -16,6 +16,7 @@
 #include <asm/guest/vmx_io.h>
 #include <asm/guest/splitlock.h>
 #include <asm/guest/ept.h>
+#include <asm/guest/vept.h>
 #include <asm/vtd.h>
 #include <asm/cpuid.h>
 #include <asm/guest/vcpuid.h>
@@ -97,6 +98,10 @@ static const struct vm_exit_dispatch dispatch_table[NR_VMX_EXIT_REASONS] = {
 		.handler = undefined_vmexit_handler},
 	[VMX_EXIT_REASON_VMXON] = {
 		.handler = undefined_vmexit_handler},
+	[VMX_EXIT_REASON_INVEPT] = {
+		.handler = undefined_vmexit_handler},
+	[VMX_EXIT_REASON_INVVPID] = {
+		.handler = undefined_vmexit_handler},
 #else
 	[VMX_EXIT_REASON_VMLAUNCH] = {
 		.handler = vmlaunch_vmexit_handler},
@@ -119,6 +124,12 @@ static const struct vm_exit_dispatch dispatch_table[NR_VMX_EXIT_REASONS] = {
 	[VMX_EXIT_REASON_VMXON] = {
 		.handler = vmxon_vmexit_handler,
 		.need_exit_qualification = 1},
+	[VMX_EXIT_REASON_INVEPT] = {
+		.handler = invept_vmexit_handler,
+		.need_exit_qualification = 1},
+	[VMX_EXIT_REASON_INVVPID] = {
+		.handler = invvpid_vmexit_handler,
+		.need_exit_qualification = 1},
 #endif
 	[VMX_EXIT_REASON_CR_ACCESS] = {
 		.handler = cr_access_vmexit_handler,
@@ -165,14 +176,10 @@ static const struct vm_exit_dispatch dispatch_table[NR_VMX_EXIT_REASONS] = {
 	[VMX_EXIT_REASON_EPT_MISCONFIGURATION] = {
 		.handler = ept_misconfig_vmexit_handler,
 		.need_exit_qualification = 1},
-	[VMX_EXIT_REASON_INVEPT] = {
-		.handler = undefined_vmexit_handler},
 	[VMX_EXIT_REASON_RDTSCP] = {
 		.handler = unhandled_vmexit_handler},
 	[VMX_EXIT_REASON_VMX_PREEMPTION_TIMER_EXPIRED] = {
 		.handler = unhandled_vmexit_handler},
-	[VMX_EXIT_REASON_INVVPID] = {
-		.handler = undefined_vmexit_handler},
 	[VMX_EXIT_REASON_WBINVD] = {
 		.handler = wbinvd_vmexit_handler},
 	[VMX_EXIT_REASON_XSETBV] = {

hypervisor/include/arch/x86/asm/guest/nested.h

@@ -93,6 +93,7 @@ union value_64 {
 #define VMXERR_UNSUPPORTED_COMPONENT		(12)
 #define VMXERR_VMWRITE_RO_COMPONENT		(13)
 #define VMXERR_VMXON_IN_VMX_ROOT_OPERATION	(15)
+#define VMXERR_INVEPT_INVVPID_INVALID_OPERAND	(28)
 
 /*
  * This VMCS12 revision id is chosen arbitrarily.
@@ -314,6 +315,8 @@ enum VMXResult {
 	VMfailInvalid,
 };
 void nested_vmx_result(enum VMXResult, int error_number);
+int64_t get_invvpid_ept_operands(struct acrn_vcpu *vcpu, void *desc, size_t size);
+bool check_vmx_permission(struct acrn_vcpu *vcpu);
 int32_t vmxon_vmexit_handler(struct acrn_vcpu *vcpu);
 int32_t vmxoff_vmexit_handler(struct acrn_vcpu *vcpu);
 int32_t vmptrld_vmexit_handler(struct acrn_vcpu *vcpu);
@@ -322,6 +325,7 @@ int32_t vmread_vmexit_handler(struct acrn_vcpu *vcpu);
 int32_t vmwrite_vmexit_handler(struct acrn_vcpu *vcpu);
 int32_t vmresume_vmexit_handler(struct acrn_vcpu *vcpu);
 int32_t vmlaunch_vmexit_handler(struct acrn_vcpu *vcpu);
+int32_t invvpid_vmexit_handler(struct acrn_vcpu *vcpu);
 
 #ifdef CONFIG_NVMX_ENABLED
 struct acrn_nested {

hypervisor/include/arch/x86/asm/guest/vept.h

@@ -0,0 +1,12 @@
+/*
+ * Copyright (C) 2021 Intel Corporation. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+#ifndef VEPT_H
+#define VEPT_H
+
+#ifdef CONFIG_NVMX_ENABLED
+int32_t invept_vmexit_handler(struct acrn_vcpu *vcpu);
+#endif /* CONFIG_NVMX_ENABLED */
+#endif /* VEPT_H */