github/acrn-hypervisor
1
0
Fork 0
mirror of https://github.com/projectacrn/acrn-hypervisor.git synced 2025-06-23 22:18:17 +00:00
Code Issues Projects Releases Wiki Activity

hv: replace vcpu_affinity array with cpu_affinity_bitmap

Browse Source 
Currently the vcpu_affinity[] array fixes the vCPU to pCPU mapping.
While the new cpu_affinity_bitmap doesn't explicitly sepcify this
mapping, instead, it implicitly assumes that vCPU0 maps to the pCPU
with lowest pCPU ID, vCPU1 maps to the second lowest pCPU ID, and
so on.

This makes it possible for post-launched VM to run vCPUs on a subset of
these pCPUs only, and not all of them.

acrn-dm may launch post-launched VMs with the current approach: indicate
VM UUID and hypervisor launches all VCPUs from the PCPUs that are masked
in cpu_affinity_bitmap.

Also acrn-dm can choose to launch the VM on a subset of PCPUs that is
defined in cpu_affinity_bitmap. In this way, acrn-dm must specify the
subset of PCPUs in the CREATE_VM hypercall.

Additionally, with this change, a guest's vcpu_num can be easily calculated
from cpu_affinity_bitmap, so don't assign vcpu_num in vm_configuration.c.

Tracked-On: #4616
Signed-off-by: Zide Chen <zide.chen@intel.com>
Acked-by: Eddie Dong <eddie.dong@intel.com>
This commit is contained in:
Zide Chen 2020-04-08 12:24:11 -07:00 committed by wenlingz
parent 40ae32f172
commit 467539441b
13 changed files with 63 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
hypervisor/arch/x86/configs/vm_config.c
View File

@ -111,35 +111,24 @@ static bool check_vm_clos_config(uint16_t vm_id)
bool sanitize_vm_config(void)
{
bool ret = true;
uint16_t vm_id, vcpu_id, vuart_idx, nr;
uint64_t sos_pcpu_bitmap, pre_launch_pcpu_bitmap = 0U, vm_pcpu_bitmap;
uint16_t vm_id, vuart_idx;
uint64_t pre_launch_pcpu_bitmap = 0UL;
struct acrn_vm_config *vm_config;
sos_pcpu_bitmap = (uint64_t)((((uint64_t)1U) << get_pcpu_nums()) - 1U);
/* All physical CPUs except ocuppied by Pre-launched VMs are all
* belong to SOS_VM. i.e. The vm_pcpu_bitmap of a SOS_VM is decided
* by vm_pcpu_bitmap status in PRE_LAUNCHED_VMs.
* belong to SOS_VM. i.e. The cpu_affinity_bitmap of a SOS_VM is decided
* by cpu_affinity_bitmap status in PRE_LAUNCHED_VMs.
* We need to setup a rule, that the vm_configs[] array should follow
* the order of PRE_LAUNCHED_VM first, and then SOS_VM.
*/
for (vm_id = 0U; vm_id < CONFIG_MAX_VM_NUM; vm_id++) {
vm_config = get_vm_config(vm_id);
vm_pcpu_bitmap = 0U;
if (vm_config->load_order != SOS_VM) {
for (vcpu_id = 0U; vcpu_id < vm_config->vcpu_num; vcpu_id++) {
if (bitmap_weight(vm_config->vcpu_affinity[vcpu_id]) != 1U) {
pr_err("%s: vm%u vcpu%u should have only one prefer pcpu!",
__func__, vm_id, vcpu_id);
ret = false;
break;
}
vm_pcpu_bitmap |= vm_config->vcpu_affinity[vcpu_id];
}
vm_config->vcpu_num = bitmap_weight(vm_config->cpu_affinity_bitmap);
if ((bitmap_weight(vm_pcpu_bitmap) != vm_config->vcpu_num)) {
pr_err("%s: One VM cannot have multiple vcpus share one pcpu!", __func__);
ret = false;
}
if ((vm_config->cpu_affinity_bitmap & ~ALL_CPUS_MASK) != 0UL) {
pr_err("%s: vm%u assigns invalid PCPU (pcpu bitmap: 0x%llx)", __func__, vm_id,
vm_config->cpu_affinity_bitmap);
ret = false;
}
switch (vm_config->load_order) {
@ -155,27 +144,21 @@ bool sanitize_vm_config(void)
} else if (is_safety_vm_uuid(vm_config->uuid) && (vm_config->severity != (uint8_t)SEVERITY_SAFETY_VM)) {
ret = false;
} else {
pre_launch_pcpu_bitmap |= vm_pcpu_bitmap;
pre_launch_pcpu_bitmap |= vm_config->cpu_affinity_bitmap;
}
break;
case SOS_VM:
/* Deduct pcpus of PRE_LAUNCHED_VMs */
sos_pcpu_bitmap ^= pre_launch_pcpu_bitmap;
if ((sos_pcpu_bitmap == 0U) || ((vm_config->guest_flags & GUEST_FLAG_LAPIC_PASSTHROUGH) != 0U)) {
vm_config->cpu_affinity_bitmap = ALL_CPUS_MASK ^ pre_launch_pcpu_bitmap;
vm_config->vcpu_num = bitmap_weight(vm_config->cpu_affinity_bitmap);
if ((vm_config->vcpu_num == 0U) || (vm_config->severity != (uint8_t)SEVERITY_SOS) ||
((vm_config->guest_flags & GUEST_FLAG_LAPIC_PASSTHROUGH) != 0U)) {
ret = false;
} else if (vm_config->severity != (uint8_t)SEVERITY_SOS) {
ret = false;
} else {
vm_config->vcpu_num = bitmap_weight(sos_pcpu_bitmap);
for (vcpu_id = 0U; vcpu_id < vm_config->vcpu_num; vcpu_id++) {
nr = ffs64(sos_pcpu_bitmap);
vm_config->vcpu_affinity[vcpu_id] = 1U << nr;
bitmap_clear_nolock(nr, &sos_pcpu_bitmap);
}
}
break;
case POST_LAUNCHED_VM:
if ((vm_pcpu_bitmap == 0U) || ((vm_pcpu_bitmap & pre_launch_pcpu_bitmap) != 0U)) {
if ((vm_config->vcpu_num == 0U) ||
((vm_config->cpu_affinity_bitmap & pre_launch_pcpu_bitmap) != 0UL)) {
pr_err("%s: Post-launch VM has no pcpus or share pcpu with Pre-launch VM!", __func__);
ret = false;
}

12
hypervisor/arch/x86/guest/vm.c
View File

@ -212,7 +212,8 @@ static inline uint16_t get_vm_bsp_pcpu_id(const struct acrn_vm_config *vm_config
{
uint16_t cpu_id = INVALID_CPU_ID;
cpu_id = ffs64(vm_config->vcpu_affinity[BSP_CPU_ID]);
/* The set least significant bit represents the pCPU ID for BSP */
cpu_id = ffs64(vm_config->cpu_affinity_bitmap);
return (cpu_id < get_pcpu_nums()) ? cpu_id : INVALID_CPU_ID;
}
@ -395,7 +396,6 @@ int32_t create_vm(uint16_t vm_id, struct acrn_vm_config *vm_config, struct acrn_
{
struct acrn_vm *vm = NULL;
int32_t status = 0;
uint32_t i;
uint16_t pcpu_id;
/* Allocate memory for virtual machine */
@ -498,10 +498,12 @@ int32_t create_vm(uint16_t vm_id, struct acrn_vm_config *vm_config, struct acrn_
if (status == 0) {
/* We have assumptions:
* 1) vcpus used by SOS has been offlined by DM before UOS re-use it.
* 2) vcpu_affinity[] passed sanitization is OK for vcpu creating.
* 2) cpu_affinity_bitmap passed sanitization is OK for vcpu creating.
*/
for (i = 0U; i < vm_config->vcpu_num; i++) {
pcpu_id = ffs64(vm_config->vcpu_affinity[i]);
uint64_t pcpu_bitmap = vm_config->cpu_affinity_bitmap;
while (pcpu_bitmap != 0UL) {
pcpu_id = ffs64(pcpu_bitmap);
bitmap_clear_nolock(pcpu_id, &pcpu_bitmap);
status = prepare_vcpu(vm, pcpu_id);
if (status != 0) {
break;

3
hypervisor/include/arch/x86/cpu.h
View File

@ -189,7 +189,8 @@
#ifndef ASSEMBLER
#define AP_MASK (((1UL << get_pcpu_nums()) - 1UL) & ~(1UL << 0U))
#define ALL_CPUS_MASK ((1UL << get_pcpu_nums()) - 1UL)
#define AP_MASK (ALL_CPUS_MASK & ~(1UL << BSP_CPU_ID))
/**
*

1
hypervisor/include/arch/x86/irq.h
View File

@ -81,7 +81,6 @@
#define DEFAULT_DEST_MODE IOAPIC_RTE_DESTMODE_LOGICAL
#define DEFAULT_DELIVERY_MODE IOAPIC_RTE_DELMODE_LOPRI
#define ALL_CPUS_MASK (uint32_t) (((uint32_t)1U << (uint32_t) get_pcpu_nums()) - (uint32_t)1U)
#define IRQ_ALLOC_BITMAP_SIZE INT_DIV_ROUNDUP(NR_IRQS, 64U)

13
hypervisor/include/arch/x86/vm_config.h
View File

@ -17,7 +17,7 @@
#define CONFIG_MAX_VM_NUM (PRE_VM_NUM + SOS_VM_NUM + MAX_POST_VM_NUM)
#define AFFINITY_CPU(n) (1U << (n))
#define AFFINITY_CPU(n) (1UL << (n))
#define MAX_VCPUS_PER_VM MAX_PCPU_NUM
#define MAX_VUART_NUM_PER_VM 2U
#define MAX_VM_OS_NAME_LEN 32U
@ -147,8 +147,9 @@ struct acrn_vm_config {
const uint8_t uuid[16]; /* UUID of the VM */
uint16_t vcpu_num; /* Number of vCPUs for the VM */
uint8_t severity; /* severity of the VM */
uint64_t vcpu_affinity[MAX_VCPUS_PER_VM];/* bitmaps for vCPUs' affinity */
uint64_t cpu_affinity_bitmap; /* The set bits represent the pCPUs the vCPUs of
* the VM may run on.
*/
uint64_t guest_flags; /* VM flags that we want to configure for guest
* Now we have two flags:
* GUEST_FLAG_SECURE_WORLD_ENABLED
@ -160,6 +161,12 @@ struct acrn_vm_config {
uint16_t pci_dev_num; /* indicate how many PCI devices in VM */
struct acrn_vm_pci_dev_config *pci_devs; /* point to PCI devices BDF list */
struct acrn_vm_os_config os_config; /* OS information the VM */
/*
* below are varaible length members (per build).
* SOS can get the vm_configs[] array through hypercall, but SOS may not
* need to parse these members.
*/
uint16_t clos[MAX_VCPUS_PER_VM]; /* Class of Service, effective only if CONFIG_RDT_ENABLED
* is defined on CAT capable platforms
*/

6
hypervisor/scenarios/hybrid/vm_configurations.c
View File

@ -13,8 +13,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = {
CONFIG_SAFETY_VM(1),
.name = "ACRN PRE-LAUNCHED VM0",
.guest_flags = 0UL,
.vcpu_num = 1U,
.vcpu_affinity = VM0_CONFIG_VCPU_AFFINITY,
.cpu_affinity_bitmap = VM0_CONFIG_CPU_AFFINITY,
.memory = {
.start_hpa = VM0_CONFIG_MEM_START_HPA,
.size = VM0_CONFIG_MEM_SIZE,
@ -69,8 +68,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = {
},
{ /* VM2 */
CONFIG_POST_STD_VM(1),
.vcpu_num = 1U,
.vcpu_affinity = VM2_CONFIG_VCPU_AFFINITY,
.cpu_affinity_bitmap = VM2_CONFIG_CPU_AFFINITY,
.vuart[0] = {
.type = VUART_LEGACY_PIO,
.addr.port_base = COM1_BASE,

4
hypervisor/scenarios/hybrid/vm_configurations.h
View File

@ -22,7 +22,7 @@
#define MAX_POST_VM_NUM 1U
#define CONFIG_MAX_KATA_VM_NUM 0U
#define VM0_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(3U)}
#define VM0_CONFIG_CPU_AFFINITY (AFFINITY_CPU(3U))
#define VM0_CONFIG_MEM_START_HPA 0x100000000UL
#define VM0_CONFIG_MEM_SIZE 0x20000000UL
@ -40,5 +40,5 @@
SOS_IDLE \
SOS_BOOTARGS_DIFF
#define VM2_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(2U)}
#define VM2_CONFIG_CPU_AFFINITY (AFFINITY_CPU(2U))
#endif /* VM_CONFIGURATIONS_H */

21
hypervisor/scenarios/industry/vm_configurations.c
View File

@ -39,8 +39,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = {
},
{ /* VM1 */
CONFIG_POST_STD_VM(1),
.vcpu_num = 2U,
.vcpu_affinity = VM1_CONFIG_VCPU_AFFINITY,
.cpu_affinity_bitmap = VM1_CONFIG_CPU_AFFINITY,
.vuart[0] = {
.type = VUART_LEGACY_PIO,
.addr.port_base = COM1_BASE,
@ -54,8 +53,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = {
{ /* VM2 */
CONFIG_POST_RT_VM(1),
.guest_flags = 0UL,
.vcpu_num = 2U,
.vcpu_affinity = VM2_CONFIG_VCPU_AFFINITY,
.cpu_affinity_bitmap = VM2_CONFIG_CPU_AFFINITY,
.vuart[0] = {
.type = VUART_LEGACY_PIO,
.addr.port_base = COM1_BASE,
@ -71,8 +69,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = {
},
{ /* VM3 */
CONFIG_POST_STD_VM(2),
.vcpu_num = 1U,
.vcpu_affinity = VM3_CONFIG_VCPU_AFFINITY,
.cpu_affinity_bitmap = VM3_CONFIG_CPU_AFFINITY,
.vuart[0] = {
.type = VUART_LEGACY_PIO,
.addr.port_base = COM1_BASE,
@ -85,8 +82,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = {
},
{ /* VM4 */
CONFIG_POST_STD_VM(3),
.vcpu_num = 1U,
.vcpu_affinity = VM4_CONFIG_VCPU_AFFINITY,
.cpu_affinity_bitmap = VM4_CONFIG_CPU_AFFINITY,
.vuart[0] = {
.type = VUART_LEGACY_PIO,
.addr.port_base = COM1_BASE,
@ -99,8 +95,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = {
},
{ /* VM5 */
CONFIG_POST_STD_VM(4),
.vcpu_num = 1U,
.vcpu_affinity = VM5_CONFIG_VCPU_AFFINITY,
.cpu_affinity_bitmap = VM5_CONFIG_CPU_AFFINITY,
.vuart[0] = {
.type = VUART_LEGACY_PIO,
.addr.port_base = COM1_BASE,
@ -113,8 +108,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = {
},
{ /* VM6 */
CONFIG_POST_STD_VM(5),
.vcpu_num = 1U,
.vcpu_affinity = VM6_CONFIG_VCPU_AFFINITY,
.cpu_affinity_bitmap = VM6_CONFIG_CPU_AFFINITY,
.vuart[0] = {
.type = VUART_LEGACY_PIO,
.addr.port_base = COM1_BASE,
@ -127,8 +121,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = {
},
{ /* VM7 */
CONFIG_KATA_VM(1),
.vcpu_num = 1U,
.vcpu_affinity = VM7_CONFIG_VCPU_AFFINITY,
.cpu_affinity_bitmap = VM7_CONFIG_CPU_AFFINITY,
.vuart[0] = {
.type = VUART_LEGACY_PIO,
.addr.port_base = COM1_BASE,

14
hypervisor/scenarios/industry/vm_configurations.h
View File

@ -36,12 +36,12 @@
SOS_IDLE \
SOS_BOOTARGS_DIFF
#define VM1_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(0U), AFFINITY_CPU(1U)}
#define VM2_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(2U), AFFINITY_CPU(3U)}
#define VM3_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(1U)}
#define VM4_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(1U)}
#define VM5_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(1U)}
#define VM6_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(1U)}
#define VM7_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(1U)}
#define VM1_CONFIG_CPU_AFFINITY (AFFINITY_CPU(0U) | AFFINITY_CPU(1U))
#define VM2_CONFIG_CPU_AFFINITY (AFFINITY_CPU(2U) | AFFINITY_CPU(3U))
#define VM3_CONFIG_CPU_AFFINITY (AFFINITY_CPU(1U))
#define VM4_CONFIG_CPU_AFFINITY (AFFINITY_CPU(1U))
#define VM5_CONFIG_CPU_AFFINITY (AFFINITY_CPU(1U))
#define VM6_CONFIG_CPU_AFFINITY (AFFINITY_CPU(1U))
#define VM7_CONFIG_CPU_AFFINITY (AFFINITY_CPU(1U))
#endif /* VM_CONFIGURATIONS_H */

6
hypervisor/scenarios/logical_partition/vm_configurations.c
View File

@ -14,8 +14,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = {
{ /* VM0 */
CONFIG_PRE_STD_VM(1),
.name = "ACRN PRE-LAUNCHED VM0",
.vcpu_num = 2U,
.vcpu_affinity = VM0_CONFIG_VCPU_AFFINITY,
.cpu_affinity_bitmap = VM0_CONFIG_CPU_AFFINITY,
.memory = {
.start_hpa = VM0_CONFIG_MEM_START_HPA,
.size = VM0_CONFIG_MEM_SIZE,
@ -49,8 +48,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = {
{ /* VM1 */
CONFIG_PRE_STD_VM(2),
.name = "ACRN PRE-LAUNCHED VM1",
.vcpu_num = 2U,
.vcpu_affinity = VM1_CONFIG_VCPU_AFFINITY,
.cpu_affinity_bitmap = VM1_CONFIG_CPU_AFFINITY,
.guest_flags = (GUEST_FLAG_RT | GUEST_FLAG_LAPIC_PASSTHROUGH),
.memory = {
.start_hpa = VM1_CONFIG_MEM_START_HPA,

6
hypervisor/scenarios/logical_partition/vm_configurations.h
View File

@ -23,7 +23,7 @@
#define CONFIG_MAX_KATA_VM_NUM 0U
/* The VM CONFIGs like:
* VMX_CONFIG_VCPU_AFFINITY
* VMX_CONFIG_CPU_AFFINITY
* VMX_CONFIG_MEM_START_HPA
* VMX_CONFIG_MEM_SIZE
* VMX_CONFIG_OS_BOOTARG_ROOT
@ -32,14 +32,14 @@
* might be different on your board, please modify them per your needs.
*/
#define VM0_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(0U), AFFINITY_CPU(2U)}
#define VM0_CONFIG_CPU_AFFINITY (AFFINITY_CPU(0U) | AFFINITY_CPU(2U))
#define VM0_CONFIG_MEM_START_HPA 0x100000000UL
#define VM0_CONFIG_MEM_SIZE 0x20000000UL
#define VM0_CONFIG_OS_BOOTARG_ROOT ROOTFS_0
#define VM0_CONFIG_OS_BOOTARG_MAXCPUS "maxcpus=2 "
#define VM0_CONFIG_OS_BOOTARG_CONSOLE "console=ttyS0 "
#define VM1_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(1U), AFFINITY_CPU(3U)}
#define VM1_CONFIG_CPU_AFFINITY (AFFINITY_CPU(1U) | AFFINITY_CPU(3U))
#define VM1_CONFIG_MEM_START_HPA 0x120000000UL
#define VM1_CONFIG_MEM_SIZE 0x20000000UL
#define VM1_CONFIG_OS_BOOTARG_ROOT ROOTFS_0

6
hypervisor/scenarios/sdc/vm_configurations.c
View File

@ -37,8 +37,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = {
},
{ /* VM1 */
CONFIG_POST_STD_VM(1),
.vcpu_num = MAX_PCPU_NUM - CONFIG_MAX_KATA_VM_NUM - 1U,
.vcpu_affinity = VM1_CONFIG_VCPU_AFFINITY,
.cpu_affinity_bitmap = VM1_CONFIG_CPU_AFFINITY,
.vuart[0] = {
.type = VUART_LEGACY_PIO,
.addr.port_base = INVALID_COM_BASE,
@ -52,8 +51,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = {
#if CONFIG_MAX_KATA_VM_NUM > 0
{ /* VM2 */
CONFIG_KATA_VM(1),
.vcpu_num = 1U,
.vcpu_affinity = VM2_CONFIG_VCPU_AFFINITY,
.cpu_affinity_bitmap = VM2_CONFIG_CPU_AFFINITY,
.vuart[0] = {
.type = VUART_LEGACY_PIO,
.addr.port_base = INVALID_COM_BASE,

6
hypervisor/scenarios/sdc/vm_configurations.h
View File

@ -37,9 +37,9 @@
SOS_BOOTARGS_DIFF
#if CONFIG_MAX_KATA_VM_NUM > 0
#define VM1_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(1U), AFFINITY_CPU(2U)}
#define VM2_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(3U)}
#define VM1_CONFIG_CPU_AFFINITY (AFFINITY_CPU(1U) | AFFINITY_CPU(2U))
#define VM2_CONFIG_CPU_AFFINITY (AFFINITY_CPU(3U))
#else
#define VM1_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(1U), AFFINITY_CPU(2U), AFFINITY_CPU(3U)}
#define VM1_CONFIG_CPU_AFFINITY (AFFINITY_CPU(1U) | AFFINITY_CPU(2U) | AFFINITY_CPU(3U))
#endif
#endif /* VM_CONFIGURATIONS_H */