From 064fd7647f1626dacaa1989080bf5339c841f295 Mon Sep 17 00:00:00 2001 From: Jie Deng Date: Thu, 9 Sep 2021 16:21:30 +0800 Subject: [PATCH] hv: add priority based scheduler This patch adds a new priority based scheduler to support vCPU scheduling based on their pre-configured priorities. A vCPU can be running only if there is no higher priority vCPU running on the same pCPU. Tracked-On: #6571 Signed-off-by: Jie Deng --- hypervisor/Makefile | 3 + hypervisor/arch/x86/guest/vcpu.c | 1 + hypervisor/common/hv_main.c | 1 + hypervisor/common/sched_prio.c | 98 +++++++++++++++++++ hypervisor/common/schedule.c | 3 + hypervisor/include/arch/x86/asm/per_cpu.h | 1 + hypervisor/include/arch/x86/asm/vm_config.h | 1 + hypervisor/include/common/schedule.h | 15 +++ misc/config_tools/library/hv_cfg_lib.py | 2 +- misc/config_tools/schema/config.xsd | 5 + misc/config_tools/schema/types.xsd | 19 +++- .../xforms/vm_configurations.c.xsl | 2 + 12 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 hypervisor/common/sched_prio.c diff --git a/hypervisor/Makefile b/hypervisor/Makefile index 3773da03e..50bb2891c 100644 --- a/hypervisor/Makefile +++ b/hypervisor/Makefile @@ -244,6 +244,9 @@ endif ifeq ($(CONFIG_SCHED_BVT),y) HW_C_SRCS += common/sched_bvt.c endif +ifeq ($(CONFIG_SCHED_PRIO),y) +HW_C_SRCS += common/sched_prio.c +endif HW_C_SRCS += hw/pci.c HW_C_SRCS += arch/x86/configs/vm_config.c HW_C_SRCS += boot/acpi_base.c diff --git a/hypervisor/arch/x86/guest/vcpu.c b/hypervisor/arch/x86/guest/vcpu.c index 73c4bf371..e40f1be72 100644 --- a/hypervisor/arch/x86/guest/vcpu.c +++ b/hypervisor/arch/x86/guest/vcpu.c @@ -963,6 +963,7 @@ int32_t prepare_vcpu(struct acrn_vm *vm, uint16_t pcpu_id) vcpu->thread_obj.host_sp = build_stack_frame(vcpu); vcpu->thread_obj.switch_out = context_switch_out; vcpu->thread_obj.switch_in = context_switch_in; + vcpu->thread_obj.priority = get_vm_config(vm->vm_id)->vm_prio; init_thread_data(&vcpu->thread_obj); for (i = 0; i < VCPU_EVENT_NUM; i++) { init_event(&vcpu->events[i]); diff --git a/hypervisor/common/hv_main.c b/hypervisor/common/hv_main.c index 073eb8c67..08e7daf77 100644 --- a/hypervisor/common/hv_main.c +++ b/hypervisor/common/hv_main.c @@ -105,6 +105,7 @@ void run_idle_thread(void) idle->thread_entry = default_idle; idle->switch_out = NULL; idle->switch_in = NULL; + idle->priority = PRIO_IDLE; run_thread(idle); diff --git a/hypervisor/common/sched_prio.c b/hypervisor/common/sched_prio.c new file mode 100644 index 000000000..8d16b452f --- /dev/null +++ b/hypervisor/common/sched_prio.c @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2021 Intel Corporation. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include + +struct sched_prio_data { + /* keep list as the first item */ + struct list_head list; +}; + +static int sched_prio_init(struct sched_control *ctl) +{ + struct sched_prio_control *prio_ctl = &per_cpu(sched_prio_ctl, ctl->pcpu_id); + + ASSERT(ctl->pcpu_id == get_pcpu_id(), "Init scheduler on wrong CPU!"); + + ctl->priv = prio_ctl; + INIT_LIST_HEAD(&prio_ctl->prio_queue); + + return 0; +} + +static void sched_prio_init_data(struct thread_object *obj) +{ + struct sched_prio_data *data; + + data = (struct sched_prio_data *)obj->data; + INIT_LIST_HEAD(&data->list); +} + +static struct thread_object *sched_prio_pick_next(struct sched_control *ctl) +{ + struct sched_prio_control *prio_ctl = (struct sched_prio_control *)ctl->priv; + struct thread_object *next = NULL; + + if (!list_empty(&prio_ctl->prio_queue)) { + next = get_first_item(&prio_ctl->prio_queue, struct thread_object, data); + } else { + next = &get_cpu_var(idle); + } + + return next; +} + +static void prio_queue_add(struct thread_object *obj) +{ + struct sched_prio_control *prio_ctl = + (struct sched_prio_control *)obj->sched_ctl->priv; + struct sched_prio_data *data = (struct sched_prio_data *)obj->data; + struct thread_object *iter_obj; + struct list_head *pos; + + if (list_empty(&prio_ctl->prio_queue)) { + list_add(&data->list, &prio_ctl->prio_queue); + } else { + list_for_each(pos, &prio_ctl->prio_queue) { + iter_obj = container_of(pos, struct thread_object, data); + if (iter_obj->priority < obj->priority) { + list_add_node(&data->list, pos->prev, pos); + break; + } + } + if (list_empty(&data->list)) { + list_add_tail(&data->list, &prio_ctl->prio_queue); + } + } +} + +static void prio_queue_remove(struct thread_object *obj) +{ + struct sched_prio_data *data = (struct sched_prio_data *)obj->data; + + list_del_init(&data->list); +} + +static void sched_prio_sleep(struct thread_object *obj) +{ + prio_queue_remove(obj); +} + +static void sched_prio_wake(struct thread_object *obj) +{ + prio_queue_add(obj); +} + +struct acrn_scheduler sched_prio = { + .name = "sched_prio", + .init = sched_prio_init, + .init_data = sched_prio_init_data, + .pick_next = sched_prio_pick_next, + .sleep = sched_prio_sleep, + .wake = sched_prio_wake, +}; diff --git a/hypervisor/common/schedule.c b/hypervisor/common/schedule.c index 1e88ff74a..1aabc9567 100644 --- a/hypervisor/common/schedule.c +++ b/hypervisor/common/schedule.c @@ -77,6 +77,9 @@ void init_sched(uint16_t pcpu_id) #endif #ifdef CONFIG_SCHED_BVT ctl->scheduler = &sched_bvt; +#endif +#ifdef CONFIG_SCHED_PRIO + ctl->scheduler = &sched_prio; #endif if (ctl->scheduler->init != NULL) { ctl->scheduler->init(ctl); diff --git a/hypervisor/include/arch/x86/asm/per_cpu.h b/hypervisor/include/arch/x86/asm/per_cpu.h index f9d2da7b9..25895bfce 100644 --- a/hypervisor/include/arch/x86/asm/per_cpu.h +++ b/hypervisor/include/arch/x86/asm/per_cpu.h @@ -41,6 +41,7 @@ struct per_cpu_region { struct sched_noop_control sched_noop_ctl; struct sched_iorr_control sched_iorr_ctl; struct sched_bvt_control sched_bvt_ctl; + struct sched_prio_control sched_prio_ctl; struct thread_object idle; struct host_gdt gdt; struct tss_64 tss; diff --git a/hypervisor/include/arch/x86/asm/vm_config.h b/hypervisor/include/arch/x86/asm/vm_config.h index 9bcfeeb4f..528210375 100644 --- a/hypervisor/include/arch/x86/asm/vm_config.h +++ b/hypervisor/include/arch/x86/asm/vm_config.h @@ -169,6 +169,7 @@ struct acrn_vm_config { * GUEST_FLAG_LAPIC_PASSTHROUGH * We could add more guest flags in future; */ + uint32_t vm_prio; /* The priority for VM vCPU scheduling */ struct acrn_vm_mem_config memory; /* memory configuration of VM */ struct epc_section epc; /* EPC memory configuration of VM */ uint16_t pci_dev_num; /* indicate how many PCI devices in VM */ diff --git a/hypervisor/include/common/schedule.h b/hypervisor/include/common/schedule.h index b848b4b42..bcebf2fa1 100644 --- a/hypervisor/include/common/schedule.h +++ b/hypervisor/include/common/schedule.h @@ -28,6 +28,14 @@ enum sched_notify_mode { SCHED_NOTIFY_IPI }; +/* Tools can configure a VM to use PRIO_LOW or PRIO_HIGH */ +enum thread_priority { + PRIO_IDLE = 0, + PRIO_LOW, + PRIO_HIGH, + PRIO_MAX +}; + struct thread_object; typedef void (*thread_entry_t)(struct thread_object *obj); typedef void (*switch_t)(struct thread_object *obj); @@ -44,6 +52,8 @@ struct thread_object { switch_t switch_out; switch_t switch_in; + int priority; + uint8_t data[THREAD_DATA_SIZE]; }; @@ -97,6 +107,11 @@ struct sched_bvt_control { struct hv_timer tick_timer; }; +extern struct acrn_scheduler sched_prio; +struct sched_prio_control { + struct list_head prio_queue; +}; + bool is_idle_thread(const struct thread_object *obj); uint16_t sched_get_pcpuid(const struct thread_object *obj); struct thread_object *sched_get_current(uint16_t pcpu_id); diff --git a/misc/config_tools/library/hv_cfg_lib.py b/misc/config_tools/library/hv_cfg_lib.py index c586712b9..a74b7adfe 100644 --- a/misc/config_tools/library/hv_cfg_lib.py +++ b/misc/config_tools/library/hv_cfg_lib.py @@ -12,7 +12,7 @@ import board_cfg_lib ERR_LIST = {} N_Y = ['n', 'y'] -SCHEDULER_TYPE = ['SCHED_NOOP', 'SCHED_IORR', 'SCHED_BVT'] +SCHEDULER_TYPE = ['SCHED_NOOP', 'SCHED_IORR', 'SCHED_BVT', 'SCHED_PRIO'] RANGE_DB = { 'LOG_LEVEL':{'min':0,'max':6}, diff --git a/misc/config_tools/schema/config.xsd b/misc/config_tools/schema/config.xsd index 367b5c92f..91ebcaa7d 100644 --- a/misc/config_tools/schema/config.xsd +++ b/misc/config_tools/schema/config.xsd @@ -400,6 +400,11 @@ Refer SDM 17.19.2 for details, and use with caution. Specify memory information for Service and User VMs. + + + Specify the VM vCPU priority for scheduling. + + General information for host kernel, boot diff --git a/misc/config_tools/schema/types.xsd b/misc/config_tools/schema/types.xsd index 3de5dc004..f00c7eb25 100644 --- a/misc/config_tools/schema/types.xsd +++ b/misc/config_tools/schema/types.xsd @@ -101,7 +101,7 @@ higher value (lower severity) are discarded. - Three scheduler options are supported: + Four scheduler options are supported: - ``SCHED_NOOP``: The NOOP (No-Operation) scheduler means there is a strict 1 to 1 mapping between vCPUs and pCPUs. @@ -113,6 +113,8 @@ higher value (lower severity) are discarded. earliest effective virtual time. *TODO: BVT scheduler will be built on top of a prioritized scheduling mechanism, i.e. higher priority threads get scheduled first, and same priority tasks are scheduled per BVT.* +- ``SCHED_PRIO``: The priority based scheduler. vCPU scheduling will be based on + their pre-configured priorities. Read more about the available scheduling options in :ref:`cpu_sharing`. @@ -120,6 +122,21 @@ Read more about the available scheduling options in :ref:`cpu_sharing`. + + + + + + + Two priorities are supported for priority based scheduler: + +- ``PRIO_LOW``: low priority for vCPU scheduling. +- ``PRIO_HIGH``: high priority for vCPU scheduling. + + + + + diff --git a/misc/config_tools/xforms/vm_configurations.c.xsl b/misc/config_tools/xforms/vm_configurations.c.xsl index 0c9b29652..24119aee7 100644 --- a/misc/config_tools/xforms/vm_configurations.c.xsl +++ b/misc/config_tools/xforms/vm_configurations.c.xsl @@ -18,6 +18,7 @@ + @@ -67,6 +68,7 @@ +