diff --git a/hypervisor/arch/x86/configs/vm_config.c b/hypervisor/arch/x86/configs/vm_config.c index 37da27c2b..9861b6318 100644 --- a/hypervisor/arch/x86/configs/vm_config.c +++ b/hypervisor/arch/x86/configs/vm_config.c @@ -69,23 +69,40 @@ static bool check_vm_uuid_collision(uint16_t vm_id) bool sanitize_vm_config(void) { bool ret = true; - uint16_t vm_id; - uint64_t sos_pcpu_bitmap, pre_launch_pcpu_bitmap; + uint16_t vm_id, vcpu_id, nr; + uint64_t sos_pcpu_bitmap, pre_launch_pcpu_bitmap = 0U, vm_pcpu_bitmap; struct acrn_vm_config *vm_config; sos_pcpu_bitmap = (uint64_t)((((uint64_t)1U) << get_pcpu_nums()) - 1U); - pre_launch_pcpu_bitmap = 0U; /* All physical CPUs except ocuppied by Pre-launched VMs are all - * belong to SOS_VM. i.e. The pcpu_bitmap of a SOS_VM is decided - * by pcpu_bitmap status in PRE_LAUNCHED_VMs. + * belong to SOS_VM. i.e. The vm_pcpu_bitmap of a SOS_VM is decided + * by vm_pcpu_bitmap status in PRE_LAUNCHED_VMs. * We need to setup a rule, that the vm_configs[] array should follow * the order of PRE_LAUNCHED_VM first, and then SOS_VM. */ for (vm_id = 0U; vm_id < CONFIG_MAX_VM_NUM; vm_id++) { vm_config = get_vm_config(vm_id); + vm_pcpu_bitmap = 0U; + if (vm_config->load_order != SOS_VM) { + for (vcpu_id = 0U; vcpu_id < vm_config->vcpu_num; vcpu_id++) { + if (bitmap_weight(vm_config->vcpu_affinity[vcpu_id]) != 1U) { + pr_err("%s: vm%u vcpu%u should have only one prefer pcpu!", + __func__, vm_id, vcpu_id); + ret = false; + break; + } + vm_pcpu_bitmap |= vm_config->vcpu_affinity[vcpu_id]; + } + + if ((bitmap_weight(vm_pcpu_bitmap) != vm_config->vcpu_num)) { + pr_err("%s: One VM cannot have multiple vcpus share one pcpu!", __func__); + ret = false; + } + } + switch (vm_config->load_order) { case PRE_LAUNCHED_VM: - if (vm_config->pcpu_bitmap == 0U) { + if (vm_config->vcpu_num == 0U) { ret = false; /* GUEST_FLAG_RT must be set if we have GUEST_FLAG_LAPIC_PASSTHROUGH set in guest_flags */ } else if (((vm_config->guest_flags & GUEST_FLAG_LAPIC_PASSTHROUGH) != 0U) @@ -94,7 +111,7 @@ bool sanitize_vm_config(void) }else if (vm_config->epc.size != 0UL) { ret = false; } else { - pre_launch_pcpu_bitmap |= vm_config->pcpu_bitmap; + pre_launch_pcpu_bitmap |= vm_pcpu_bitmap; } break; case SOS_VM: @@ -103,12 +120,19 @@ bool sanitize_vm_config(void) if ((sos_pcpu_bitmap == 0U) || ((vm_config->guest_flags & GUEST_FLAG_LAPIC_PASSTHROUGH) != 0U)) { ret = false; } else { - vm_config->pcpu_bitmap = sos_pcpu_bitmap; vm_config->vcpu_num = bitmap_weight(sos_pcpu_bitmap); + for (vcpu_id = 0U; vcpu_id < vm_config->vcpu_num; vcpu_id++) { + nr = ffs64(sos_pcpu_bitmap); + vm_config->vcpu_affinity[vcpu_id] = 1U << nr; + bitmap_clear_nolock(nr, &sos_pcpu_bitmap); + } } break; case POST_LAUNCHED_VM: - /* Nothing to do here for a POST_LAUNCHED_VM, break directly. */ + if ((vm_pcpu_bitmap == 0U) || ((vm_pcpu_bitmap & pre_launch_pcpu_bitmap) != 0U)) { + pr_err("%s: Post-launch VM has no pcpus or share pcpu with Pre-launch VM!", __func__); + ret = false; + } break; default: /* Nothing to do for a unknown VM, break directly. */ diff --git a/hypervisor/arch/x86/guest/pm.c b/hypervisor/arch/x86/guest/pm.c index 01021e52b..00dbef4b4 100644 --- a/hypervisor/arch/x86/guest/pm.c +++ b/hypervisor/arch/x86/guest/pm.c @@ -187,7 +187,6 @@ static inline void wait_for_other_vm_shutdown(const struct acrn_vm *self_vm) static inline void enter_s5(const struct acrn_vm *vm, uint32_t pm1a_cnt_val, uint32_t pm1b_cnt_val) { wait_for_other_vm_shutdown(vm); - pr_err("yfwyfw: before enter s5"); host_enter_s5(vm->pm.sx_state_data, pm1a_cnt_val, pm1b_cnt_val); } diff --git a/hypervisor/arch/x86/guest/vm.c b/hypervisor/arch/x86/guest/vm.c index 9f2d66dab..ccc25681d 100644 --- a/hypervisor/arch/x86/guest/vm.c +++ b/hypervisor/arch/x86/guest/vm.c @@ -177,7 +177,7 @@ static inline uint16_t get_vm_bsp_pcpu_id(const struct acrn_vm_config *vm_config { uint16_t cpu_id = INVALID_CPU_ID; - cpu_id = ffs64(vm_config->pcpu_bitmap); + cpu_id = ffs64(vm_config->vcpu_affinity[0]); return (cpu_id < get_pcpu_nums()) ? cpu_id : INVALID_CPU_ID; } @@ -724,18 +724,17 @@ void resume_vm_from_s3(struct acrn_vm *vm, uint32_t wakeup_vec) void prepare_vm(uint16_t vm_id, struct acrn_vm_config *vm_config) { int32_t err = 0; - uint16_t i; + uint16_t i, pcpu_id; struct acrn_vm *vm = NULL; err = create_vm(vm_id, vm_config, &vm); if (err == 0) { - for (i = 0U; i < get_pcpu_nums(); i++) { - if (bitmap_test(i, &vm_config->pcpu_bitmap)) { - err = prepare_vcpu(vm, i); - if (err != 0) { - break; - } + for (i = 0U; i < vm_config->vcpu_num; i++) { + pcpu_id = ffs64(vm_config->vcpu_affinity[i]); + err = prepare_vcpu(vm, pcpu_id); + if (err != 0) { + break; } } } diff --git a/hypervisor/include/arch/x86/vm_config.h b/hypervisor/include/arch/x86/vm_config.h index ab4dd8e71..ae9f7a146 100644 --- a/hypervisor/include/arch/x86/vm_config.h +++ b/hypervisor/include/arch/x86/vm_config.h @@ -15,7 +15,7 @@ #include #include -#define PLUG_CPU(n) (1U << (n)) +#define AFFINITY_CPU(n) (1U << (n)) #define MAX_VUART_NUM_PER_VM 2U #define MAX_VM_OS_NAME_LEN 32U #define MAX_MOD_TAG_LEN 32U @@ -95,8 +95,9 @@ struct acrn_vm_config { enum acrn_vm_load_order load_order; /* specify the load order of VM */ char name[MAX_VM_OS_NAME_LEN]; /* VM name identifier, useful for debug. */ const uint8_t uuid[16]; /* UUID of the VM */ - uint64_t pcpu_bitmap; /* from pcpu bitmap, we could know VM core number */ uint16_t vcpu_num; /* Number of vCPUs for the VM */ + + uint64_t vcpu_affinity[CONFIG_MAX_VCPUS_PER_VM];/* bitmaps for vCPUs' affinity */ uint64_t guest_flags; /* VM flags that we want to configure for guest * Now we have two flags: * GUEST_FLAG_SECURE_WORLD_ENABLED diff --git a/hypervisor/scenarios/hybrid/vm_configurations.c b/hypervisor/scenarios/hybrid/vm_configurations.c index af5a9ce71..4c00462d6 100644 --- a/hypervisor/scenarios/hybrid/vm_configurations.c +++ b/hypervisor/scenarios/hybrid/vm_configurations.c @@ -16,8 +16,8 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = { 0x8bU, 0x71U, 0x6eU, 0x31U, 0xdcU, 0x36U, 0xfaU, 0x47U}, /* fc836901-8685-4bc0-8b71-6e31dc36fa47 */ .guest_flags = GUEST_FLAG_HIGHEST_SEVERITY, - .pcpu_bitmap = VM0_CONFIG_PCPU_BITMAP, .vcpu_num = 1U, + .vcpu_affinity = VM0_CONFIG_VCPU_AFFINITY, .clos = 0U, .memory = { .start_hpa = VM0_CONFIG_MEM_START_HPA, @@ -84,6 +84,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = { 0x86U, 0x4eU, 0xcbU, 0x7aU, 0x18U, 0xb3U, 0x46U, 0x43U}, /* d2795438-25d6-11e8-864e-cb7a18b34643 */ .vcpu_num = 1U, + .vcpu_affinity = VM2_CONFIG_VCPU_AFFINITY, .vuart[0] = { .type = VUART_LEGACY_PIO, .addr.port_base = COM1_BASE, diff --git a/hypervisor/scenarios/hybrid/vm_configurations.h b/hypervisor/scenarios/hybrid/vm_configurations.h index db0a3dfa1..3909045be 100644 --- a/hypervisor/scenarios/hybrid/vm_configurations.h +++ b/hypervisor/scenarios/hybrid/vm_configurations.h @@ -15,7 +15,7 @@ #define CONFIG_MAX_VM_NUM (3U + CONFIG_MAX_KATA_VM_NUM) -#define VM0_CONFIG_PCPU_BITMAP (PLUG_CPU(3)) +#define VM0_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(3U)} #define VM0_CONFIG_MEM_START_HPA 0x100000000UL #define VM0_CONFIG_MEM_SIZE 0x20000000UL @@ -32,4 +32,5 @@ "i915.enable_gvt=1 " \ SOS_BOOTARGS_DIFF +#define VM2_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(2U)} #endif /* VM_CONFIGURATIONS_H */ diff --git a/hypervisor/scenarios/industry/vm_configurations.c b/hypervisor/scenarios/industry/vm_configurations.c index 780efe69f..b281c101d 100644 --- a/hypervisor/scenarios/industry/vm_configurations.c +++ b/hypervisor/scenarios/industry/vm_configurations.c @@ -48,6 +48,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = { 0x86U, 0x4eU, 0xcbU, 0x7aU, 0x18U, 0xb3U, 0x46U, 0x43U}, /* d2795438-25d6-11e8-864e-cb7a18b34643 */ .vcpu_num = 1U, + .vcpu_affinity = VM1_CONFIG_VCPU_AFFINITY, .vuart[0] = { .type = VUART_LEGACY_PIO, .addr.port_base = COM1_BASE, @@ -68,6 +69,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = { /* The hard RTVM must be launched as VM2 */ .guest_flags = GUEST_FLAG_HIGHEST_SEVERITY, .vcpu_num = 1U, + .vcpu_affinity = VM2_CONFIG_VCPU_AFFINITY, .vuart[0] = { .type = VUART_LEGACY_PIO, .addr.port_base = COM1_BASE, @@ -87,6 +89,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = { 0xb7U, 0x2aU, 0x8aU, 0x60U, 0x9eU, 0x41U, 0x90U, 0xd0U}, /* 38158821-5208-4005-b72a-8a609e4190d0 */ .vcpu_num = 1U, + .vcpu_affinity = VM3_CONFIG_VCPU_AFFINITY, .vuart[0] = { .type = VUART_LEGACY_PIO, .addr.port_base = COM1_BASE, diff --git a/hypervisor/scenarios/industry/vm_configurations.h b/hypervisor/scenarios/industry/vm_configurations.h index f68d3290e..f76646595 100644 --- a/hypervisor/scenarios/industry/vm_configurations.h +++ b/hypervisor/scenarios/industry/vm_configurations.h @@ -28,4 +28,8 @@ "i915.enable_gvt=1 " \ SOS_BOOTARGS_DIFF +#define VM1_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(1U)} +#define VM2_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(2U)} +#define VM3_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(3U)} + #endif /* VM_CONFIGURATIONS_H */ diff --git a/hypervisor/scenarios/logical_partition/vm_configurations.c b/hypervisor/scenarios/logical_partition/vm_configurations.c index 8d1106cc7..e12b49508 100644 --- a/hypervisor/scenarios/logical_partition/vm_configurations.c +++ b/hypervisor/scenarios/logical_partition/vm_configurations.c @@ -17,8 +17,8 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = { .uuid = {0x26U, 0xc5U, 0xe0U, 0xd8U, 0x8fU, 0x8aU, 0x47U, 0xd8U, \ 0x81U, 0x09U, 0xf2U, 0x01U, 0xebU, 0xd6U, 0x1aU, 0x5eU}, /* 26c5e0d8-8f8a-47d8-8109-f201ebd61a5e */ - .pcpu_bitmap = VM0_CONFIG_PCPU_BITMAP, .vcpu_num = 2U, + .vcpu_affinity = VM0_CONFIG_VCPU_AFFINITY, .clos = 0U, .memory = { .start_hpa = VM0_CONFIG_MEM_START_HPA, @@ -56,8 +56,8 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = { .uuid = {0xddU, 0x87U, 0xceU, 0x08U, 0x66U, 0xf9U, 0x47U, 0x3dU, \ 0xbcU, 0x58U, 0x76U, 0x05U, 0x83U, 0x7fU, 0x93U, 0x5eU}, /* dd87ce08-66f9-473d-bc58-7605837f935e */ - .pcpu_bitmap = VM1_CONFIG_PCPU_BITMAP, .vcpu_num = 2U, + .vcpu_affinity = VM1_CONFIG_VCPU_AFFINITY, .guest_flags = (GUEST_FLAG_RT | GUEST_FLAG_LAPIC_PASSTHROUGH), .clos = 0U, .memory = { diff --git a/hypervisor/scenarios/logical_partition/vm_configurations.h b/hypervisor/scenarios/logical_partition/vm_configurations.h index f46fd1e6b..b50c1f7c0 100644 --- a/hypervisor/scenarios/logical_partition/vm_configurations.h +++ b/hypervisor/scenarios/logical_partition/vm_configurations.h @@ -16,7 +16,7 @@ #define CONFIG_MAX_VM_NUM 2U /* The VM CONFIGs like: - * VMX_CONFIG_PCPU_BITMAP + * VMX_CONFIG_VCPU_AFFINITY * VMX_CONFIG_MEM_START_HPA * VMX_CONFIG_MEM_SIZE * VMX_CONFIG_OS_BOOTARG_ROOT @@ -25,14 +25,14 @@ * might be different on your board, please modify them per your needs. */ -#define VM0_CONFIG_PCPU_BITMAP (PLUG_CPU(0) | PLUG_CPU(2)) +#define VM0_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(0U), AFFINITY_CPU(2U)} #define VM0_CONFIG_MEM_START_HPA 0x100000000UL #define VM0_CONFIG_MEM_SIZE 0x20000000UL #define VM0_CONFIG_OS_BOOTARG_ROOT ROOTFS_0 #define VM0_CONFIG_OS_BOOTARG_MAXCPUS "maxcpus=2 " #define VM0_CONFIG_OS_BOOTARG_CONSOLE "console=ttyS0 " -#define VM1_CONFIG_PCPU_BITMAP (PLUG_CPU(1) | PLUG_CPU(3)) +#define VM1_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(1U), AFFINITY_CPU(3U)} #define VM1_CONFIG_MEM_START_HPA 0x120000000UL #define VM1_CONFIG_MEM_SIZE 0x20000000UL #define VM1_CONFIG_OS_BOOTARG_ROOT ROOTFS_0 diff --git a/hypervisor/scenarios/sdc/vm_configurations.c b/hypervisor/scenarios/sdc/vm_configurations.c index dbb41229d..ff16f6206 100644 --- a/hypervisor/scenarios/sdc/vm_configurations.c +++ b/hypervisor/scenarios/sdc/vm_configurations.c @@ -47,6 +47,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = { 0x86U, 0x4eU, 0xcbU, 0x7aU, 0x18U, 0xb3U, 0x46U, 0x43U}, /* d2795438-25d6-11e8-864e-cb7a18b34643 */ .vcpu_num = CONFIG_MAX_PCPU_NUM - CONFIG_MAX_KATA_VM_NUM - 1U, + .vcpu_affinity = VM1_CONFIG_VCPU_AFFINITY, .vuart[0] = { .type = VUART_LEGACY_PIO, .addr.port_base = INVALID_COM_BASE, @@ -64,6 +65,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = { 0xa0U, 0xdaU, 0xe5U, 0x13U, 0xcaU, 0x9bU, 0x8cU, 0x2fU}, /* a7ada506-1ab0-4b6b-a0da-e513ca9b8c2f */ .vcpu_num = 1U, + .vcpu_affinity = VM2_CONFIG_VCPU_AFFINITY, .vuart[0] = { .type = VUART_LEGACY_PIO, .addr.port_base = INVALID_COM_BASE, diff --git a/hypervisor/scenarios/sdc/vm_configurations.h b/hypervisor/scenarios/sdc/vm_configurations.h index c2c44b410..436e931a6 100644 --- a/hypervisor/scenarios/sdc/vm_configurations.h +++ b/hypervisor/scenarios/sdc/vm_configurations.h @@ -28,4 +28,10 @@ "i915.enable_gvt=1 " \ SOS_BOOTARGS_DIFF +#if CONFIG_MAX_KATA_VM_NUM > 0 + #define VM1_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(1U), AFFINITY_CPU(2U)} + #define VM2_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(3U)} +#else + #define VM1_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(1U), AFFINITY_CPU(2U), AFFINITY_CPU(3U)} +#endif #endif /* VM_CONFIGURATIONS_H */ diff --git a/hypervisor/scenarios/sdc2/vm_configurations.c b/hypervisor/scenarios/sdc2/vm_configurations.c index 634640a14..32df114e3 100644 --- a/hypervisor/scenarios/sdc2/vm_configurations.c +++ b/hypervisor/scenarios/sdc2/vm_configurations.c @@ -47,6 +47,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = { 0x86U, 0x4eU, 0xcbU, 0x7aU, 0x18U, 0xb3U, 0x46U, 0x43U}, /* d2795438-25d6-11e8-864e-cb7a18b34643 */ .vcpu_num = 1U, + .vcpu_affinity = VM1_CONFIG_VCPU_AFFINITY, .vuart[0] = { .type = VUART_LEGACY_PIO, .addr.port_base = INVALID_COM_BASE, @@ -63,6 +64,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = { 0xafU, 0x76U, 0xd4U, 0xbcU, 0x5aU, 0x8eU, 0xc0U, 0xe5U}, /* 495ae2e5-2603-4d64-af76-d4bc5a8ec0e5 */ .vcpu_num = 1U, + .vcpu_affinity = VM2_CONFIG_VCPU_AFFINITY, .vuart[0] = { .type = VUART_LEGACY_PIO, .addr.port_base = INVALID_COM_BASE, @@ -79,6 +81,7 @@ struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM] = { 0xb7U, 0x2aU, 0x8aU, 0x60U, 0x9eU, 0x41U, 0x90U, 0xd0U}, /* 38158821-5208-4005-b72a-8a609e4190d0 */ .vcpu_num = 1U, + .vcpu_affinity = VM3_CONFIG_VCPU_AFFINITY, .vuart[0] = { .type = VUART_LEGACY_PIO, .addr.port_base = INVALID_COM_BASE, diff --git a/hypervisor/scenarios/sdc2/vm_configurations.h b/hypervisor/scenarios/sdc2/vm_configurations.h index 27588fe04..96e069d9e 100644 --- a/hypervisor/scenarios/sdc2/vm_configurations.h +++ b/hypervisor/scenarios/sdc2/vm_configurations.h @@ -28,4 +28,8 @@ "i915.enable_gvt=1 " \ SOS_BOOTARGS_DIFF +#define VM1_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(1U)} +#define VM2_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(2U)} +#define VM3_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(3U)} + #endif /* VM_CONFIGURATIONS_H */