From 11e67f1c4af736dac1565720c23777d248c2890e Mon Sep 17 00:00:00 2001
From: "Yin, Fengwei" <fengwei.yin@intel.com>
Date: Tue, 9 Jul 2019 18:02:55 +0800
Subject: [PATCH] softirq: move softirq from hv_main to interrupt context

softirq shouldn't be bounded to vcpu thread. One issue for this
is shell (based on timer) can't work if we don't start any guest.

This change also is trying best to make softirq handler running
with irq enabled.

Also update the irq disable/enabel in vmexit handler to align
with the usage in vcpu_thread.

Tracked-On: #3387
Signed-off-by: Yin Fengwei <fengwei.yin@intel.com>
Acked-by: Anthony Xu <anthony.xu@intel.com>
---
 hypervisor/arch/x86/guest/vmexit.c    | 10 ++++++++--
 hypervisor/arch/x86/irq.c             |  2 ++
 hypervisor/common/hv_main.c           |  4 ----
 hypervisor/common/softirq.c           | 22 ++++++++++++++++++++--
 hypervisor/include/arch/x86/per_cpu.h |  1 +
 5 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/hypervisor/arch/x86/guest/vmexit.c b/hypervisor/arch/x86/guest/vmexit.c
index daec9c01e..f1af9d782 100644
--- a/hypervisor/arch/x86/guest/vmexit.c
+++ b/hypervisor/arch/x86/guest/vmexit.c
@@ -224,9 +224,15 @@ int32_t vmexit_handler(struct acrn_vcpu *vcpu)
 			/* exit dispatch handling */
 			if (basic_exit_reason == VMX_EXIT_REASON_EXTERNAL_INTERRUPT) {
 				/* Handling external_interrupt should disable intr */
-				CPU_IRQ_DISABLE();
+				if (!is_lapic_pt_enabled(vcpu)) {
+					CPU_IRQ_DISABLE();
+				}
+
 				ret = dispatch->handler(vcpu);
-				CPU_IRQ_ENABLE();
+
+				if (!is_lapic_pt_enabled(vcpu)) {
+					CPU_IRQ_ENABLE();
+				}
 			} else {
 				ret = dispatch->handler(vcpu);
 			}
diff --git a/hypervisor/arch/x86/irq.c b/hypervisor/arch/x86/irq.c
index 3d0cdcb5f..43f42b06c 100644
--- a/hypervisor/arch/x86/irq.c
+++ b/hypervisor/arch/x86/irq.c
@@ -361,6 +361,8 @@ void dispatch_interrupt(const struct intr_excp_ctx *ctx)
 	} else {
 		handle_spurious_interrupt(vr);
 	}
+
+	do_softirq();
 }
 
 void dispatch_exception(struct intr_excp_ctx *ctx)
diff --git a/hypervisor/common/hv_main.c b/hypervisor/common/hv_main.c
index ad4a6f44b..e1dfe07aa 100644
--- a/hypervisor/common/hv_main.c
+++ b/hypervisor/common/hv_main.c
@@ -28,11 +28,7 @@ void vcpu_thread(struct sched_object *obj)
 		}
 
 		if (!is_lapic_pt_enabled(vcpu)) {
-			/* handle pending softirq when irq enable*/
-			do_softirq();
 			CPU_IRQ_DISABLE();
-			/* handle risk softirq when disabling irq*/
-			do_softirq();
 		}
 
 		/* Don't open interrupt window between here and vmentry */
diff --git a/hypervisor/common/softirq.c b/hypervisor/common/softirq.c
index 258a9031c..aec994fed 100644
--- a/hypervisor/common/softirq.c
+++ b/hypervisor/common/softirq.c
@@ -32,9 +32,8 @@ void fire_softirq(uint16_t nr)
 	bitmap_set_lock(nr, &per_cpu(softirq_pending, get_pcpu_id()));
 }
 
-void do_softirq(void)
+static void do_softirq_internal(uint16_t cpu_id)
 {
-	uint16_t cpu_id = get_pcpu_id();
 	volatile uint64_t *softirq_pending_bitmap =
 			&per_cpu(softirq_pending, cpu_id);
 	uint16_t nr = ffs64(*softirq_pending_bitmap);
@@ -45,3 +44,22 @@ void do_softirq(void)
 		nr = ffs64(*softirq_pending_bitmap);
 	}
 }
+
+/*
+ * @pre: this function will only be called with irq disabled
+ */
+void do_softirq(void)
+{
+	uint16_t cpu_id = get_pcpu_id();
+
+	if (per_cpu(softirq_servicing, cpu_id) == 0U) {
+		per_cpu(softirq_servicing, cpu_id) = 1U;
+
+		CPU_IRQ_ENABLE();
+		do_softirq_internal(cpu_id);
+		CPU_IRQ_DISABLE();
+
+		do_softirq_internal(cpu_id);
+		per_cpu(softirq_servicing, cpu_id) = 0U;
+	}
+}
diff --git a/hypervisor/include/arch/x86/per_cpu.h b/hypervisor/include/arch/x86/per_cpu.h
index a3cb5d651..957e229a3 100644
--- a/hypervisor/include/arch/x86/per_cpu.h
+++ b/hypervisor/include/arch/x86/per_cpu.h
@@ -48,6 +48,7 @@ struct per_cpu_region {
 	uint8_t stack[CONFIG_STACK_SIZE] __aligned(16);
 	uint32_t lapic_id;
 	uint32_t lapic_ldr;
+	uint32_t softirq_servicing;
 	struct smp_call_info_data smp_call_info;
 #ifdef PROFILING_ON
 	struct profiling_info_wrapper profiling_info;