hv: sched: add sleep/wake for thread object

sleep one thread_object means to prevent it from being scheduled.
wake one thread_object is an opposite operation of sleep.
This patch also add notify_mode in thread_object to indicate how to
deliver the request.

Tracked-On: #3813
Signed-off-by: Jason Chen CJ <jason.cj.chen@intel.com>
Signed-off-by: Yu Wang <yu1.wang@intel.com>
Signed-off-by: Shuo A Liu <shuo.a.liu@intel.com>
Acked-by: Eddie Dong <eddie.dong@intel.com>

hypervisor/arch/x86/guest/vcpu.c

@@ -670,46 +670,25 @@ void pause_vcpu(struct acrn_vcpu *vcpu, enum vcpu_state new_state)
 	pr_dbg("vcpu%hu paused, new state: %d",
 		vcpu->vcpu_id, new_state);
 
-	get_schedule_lock(pcpu_id);
 	vcpu->prev_state = vcpu->state;
 	vcpu->state = new_state;
 
-	if (vcpu->running) {
+	sleep_thread(&vcpu->thread_obj);
-		remove_thread_obj(&vcpu->thread_obj, pcpu_id);
-
-		if (is_lapic_pt_enabled(vcpu)) {
-			make_reschedule_request(pcpu_id, DEL_MODE_INIT);
-		} else {
-			make_reschedule_request(pcpu_id, DEL_MODE_IPI);
-		}
-
-		release_schedule_lock(pcpu_id);
-
 	if (pcpu_id != get_pcpu_id()) {
 		while (vcpu->running) {
 			asm_pause();
 		}
 	}
-	} else {
-		remove_thread_obj(&vcpu->thread_obj, pcpu_id);
-		release_schedule_lock(pcpu_id);
-	}
 }
 
 void resume_vcpu(struct acrn_vcpu *vcpu)
 {
-	uint16_t pcpu_id = pcpuid_from_vcpu(vcpu);
-
 	pr_dbg("vcpu%hu resumed", vcpu->vcpu_id);
 
-	get_schedule_lock(pcpu_id);
 	vcpu->state = vcpu->prev_state;
-
 	if (vcpu->state == VCPU_RUNNING) {
-		insert_thread_obj(&vcpu->thread_obj, pcpu_id);
+		wake_thread(&vcpu->thread_obj);
-		make_reschedule_request(pcpu_id, DEL_MODE_IPI);
 	}
-	release_schedule_lock(pcpu_id);
 }
 
 /* TODO:
@@ -777,6 +756,8 @@ int32_t prepare_vcpu(struct acrn_vm *vm, uint16_t pcpu_id)
 		vcpu->thread_obj.sched_ctl = &per_cpu(sched_ctl, pcpu_id);
 		vcpu->thread_obj.thread_entry = vcpu_thread;
 		vcpu->thread_obj.pcpu_id = pcpu_id;
+		vcpu->thread_obj.notify_mode = is_lapic_pt_enabled(vcpu) ?
+			SCHED_NOTIFY_INIT : SCHED_NOTIFY_IPI;
 		vcpu->thread_obj.host_sp = build_stack_frame(vcpu);
 		vcpu->thread_obj.switch_out = context_switch_out;
 		vcpu->thread_obj.switch_in = context_switch_in;

hypervisor/common/schedule.c

@@ -164,6 +164,36 @@ void schedule(void)
 	}
 }
 
+void sleep_thread(struct thread_object *obj)
+{
+	uint16_t pcpu_id = obj->pcpu_id;
+
+	get_schedule_lock(pcpu_id);
+	remove_thread_obj(obj, pcpu_id);
+	if (is_running(obj)) {
+		if (obj->notify_mode == SCHED_NOTIFY_INIT) {
+			make_reschedule_request(pcpu_id, DEL_MODE_INIT);
+		} else {
+			make_reschedule_request(pcpu_id, DEL_MODE_IPI);
+		}
+	}
+	set_thread_status(obj, THREAD_STS_BLOCKED);
+	release_schedule_lock(pcpu_id);
+}
+
+void wake_thread(struct thread_object *obj)
+{
+	uint16_t pcpu_id = obj->pcpu_id;
+
+	get_schedule_lock(pcpu_id);
+	if (is_blocked(obj)) {
+		insert_thread_obj(obj, pcpu_id);
+		set_thread_status(obj, THREAD_STS_RUNNABLE);
+		make_reschedule_request(pcpu_id, DEL_MODE_IPI);
+	}
+	release_schedule_lock(pcpu_id);
+}
+
 void run_sched_thread(struct thread_object *obj)
 {
 	if (obj->thread_entry != NULL) {

hypervisor/include/common/schedule.h

@@ -19,6 +19,11 @@ enum thread_object_state {
 	THREAD_STS_BLOCKED
 };
 
+enum sched_notify_mode {
+	SCHED_NOTIFY_INIT,
+	SCHED_NOTIFY_IPI
+};
+
 struct thread_object;
 typedef void (*thread_entry_t)(struct thread_object *obj);
 typedef void (*switch_t)(struct thread_object *obj);
@@ -28,6 +33,7 @@ struct thread_object {
 	struct sched_control *sched_ctl;
 	thread_entry_t thread_entry;
 	volatile enum thread_object_state status;
+	enum sched_notify_mode notify_mode;
 
 	uint64_t host_sp;
 	switch_t switch_out;
@@ -57,6 +63,8 @@ void remove_thread_obj(struct thread_object *obj, uint16_t pcpu_id);
 void make_reschedule_request(uint16_t pcpu_id, uint16_t delmode);
 bool need_reschedule(uint16_t pcpu_id);
 
+void sleep_thread(struct thread_object *obj);
+void wake_thread(struct thread_object *obj);
 void schedule(void);
 void run_sched_thread(struct thread_object *obj);