diff --git a/hypervisor/arch/x86/assign.c b/hypervisor/arch/x86/assign.c index 607cdb7a2..fc00998f5 100644 --- a/hypervisor/arch/x86/assign.c +++ b/hypervisor/arch/x86/assign.c @@ -53,10 +53,25 @@ static bool ptdev_hv_owned_intx(const struct vm *vm, union source_id *virt_sid) } } +static uint64_t calculate_logical_dest_mask(uint64_t pdmask) +{ + uint64_t dest_mask = 0UL; + uint64_t pcpu_mask = pdmask; + uint16_t pcpu_id; + + pcpu_id = ffs64(pcpu_mask); + while (pcpu_id != INVALID_BIT_INDEX) { + bitmap_clear_nolock(pcpu_id, &pcpu_mask); + dest_mask |= per_cpu(lapic_ldr, pcpu_id); + pcpu_id = ffs64(pcpu_mask); + } + return dest_mask; +} + static void ptdev_build_physical_msi(struct vm *vm, struct ptdev_msi_info *info, uint32_t vector) { - uint64_t vdmask, pdmask; + uint64_t vdmask, pdmask, dest_mask; uint32_t dest, delmode; bool phys; @@ -78,10 +93,11 @@ static void ptdev_build_physical_msi(struct vm *vm, struct ptdev_msi_info *info, info->pmsi_data &= ~0x7FFU; info->pmsi_data |= delmode | vector; + dest_mask = calculate_logical_dest_mask(pdmask); /* update physical dest mode & dest field */ info->pmsi_addr = info->vmsi_addr; info->pmsi_addr &= ~0xFF00CU; - info->pmsi_addr |= (uint32_t)(pdmask << 12U) | + info->pmsi_addr |= (uint32_t)(dest_mask << 12U) | MSI_ADDR_RH | MSI_ADDR_LOG; dev_dbg(ACRN_DBG_IRQ, "MSI addr:data = 0x%x:%x(V) -> 0x%x:%x(P)", @@ -99,7 +115,7 @@ ptdev_build_physical_rte(struct vm *vm, union source_id *virt_sid = &entry->virt_sid; if (virt_sid->intx_id.src == PTDEV_VPIN_IOAPIC) { - uint64_t vdmask, pdmask, delmode; + uint64_t vdmask, pdmask, delmode, dest_mask; uint32_t dest; union ioapic_rte virt_rte; bool phys; @@ -142,9 +158,10 @@ ptdev_build_physical_rte(struct vm *vm, IOAPIC_RTE_DELMOD | IOAPIC_RTE_INTVEC); rte.full |= IOAPIC_RTE_DESTLOG | delmode | (uint64_t)vector; + dest_mask = calculate_logical_dest_mask(pdmask); /* update physical dest field */ rte.full &= ~IOAPIC_RTE_DEST_MASK; - rte.full |= pdmask << IOAPIC_RTE_DEST_SHIFT; + rte.full |= dest_mask << IOAPIC_RTE_DEST_SHIFT; dev_dbg(ACRN_DBG_IRQ, "IOAPIC RTE = 0x%x:%x(V) -> 0x%x:%x(P)", virt_rte.u.hi_32, virt_rte.u.lo_32, diff --git a/hypervisor/arch/x86/cpu.c b/hypervisor/arch/x86/cpu.c index 7eaa897d2..f25ec4cce 100644 --- a/hypervisor/arch/x86/cpu.c +++ b/hypervisor/arch/x86/cpu.c @@ -54,7 +54,7 @@ static void cpu_cap_detect(void); static void cpu_xsave_init(void); static void set_current_cpu_id(uint16_t pcpu_id); static void print_hv_banner(void); -static uint16_t get_cpu_id_from_lapic_id(uint8_t lapic_id); +static uint16_t get_cpu_id_from_lapic_id(uint32_t lapic_id); int ibrs_type; static uint64_t start_tsc __attribute__((__section__(".bss_noinit"))); @@ -276,9 +276,9 @@ static void alloc_phy_cpu_data(uint16_t pcpu_num) ASSERT(per_cpu_data_base_ptr != NULL, ""); } -uint16_t __attribute__((weak)) parse_madt(uint8_t lapic_id_array[MAX_PCPU_NUM]) +uint16_t __attribute__((weak)) parse_madt(uint32_t lapic_id_array[MAX_PCPU_NUM]) { - static const uint8_t lapic_id[] = {0U, 2U, 4U, 6U}; + static const uint32_t lapic_id[] = {0U, 2U, 4U, 6U}; uint32_t i; for (i = 0U; i < ARRAY_SIZE(lapic_id); i++) { @@ -292,7 +292,7 @@ static void init_percpu_data_area(void) { uint16_t i; uint16_t pcpu_num = 0U; - uint8_t lapic_id_array[MAX_PCPU_NUM]; + uint32_t lapic_id_array[MAX_PCPU_NUM]; /* Save all lapic_id detected via parse_mdt in lapic_id_array */ pcpu_num = parse_madt(lapic_id_array); @@ -306,9 +306,6 @@ static void init_percpu_data_area(void) for (i = 0U; i < pcpu_num; i++) { per_cpu(lapic_id, i) = lapic_id_array[i]; } - - ASSERT(get_cpu_id_from_lapic_id(get_cur_lapic_id()) != INVALID_CPU_ID, - "fail to get phy cpu id"); } static void cpu_set_current_state(uint16_t pcpu_id, enum pcpu_boot_state state) @@ -406,6 +403,10 @@ void bsp_boot_init(void) /* Initialize the hypervisor paging */ init_paging(); + if (!cpu_has_cap(X86_FEATURE_X2APIC)) { + panic("x2APIC is not present!"); + } + early_init_lapic(); init_percpu_data_area(); @@ -581,7 +582,7 @@ static void cpu_secondary_post(void) cpu_dead(get_cpu_id()); } -static uint16_t get_cpu_id_from_lapic_id(uint8_t lapic_id) +static uint16_t get_cpu_id_from_lapic_id(uint32_t lapic_id) { uint16_t i; diff --git a/hypervisor/arch/x86/guest/vmsr.c b/hypervisor/arch/x86/guest/vmsr.c index ba1faa2d7..f68548f21 100644 --- a/hypervisor/arch/x86/guest/vmsr.c +++ b/hypervisor/arch/x86/guest/vmsr.c @@ -14,10 +14,10 @@ static const uint32_t emulated_msrs[] = { MSR_IA32_BIOS_SIGN_ID, /* Enable MSR_IA32_BIOS_SIGN_ID */ MSR_IA32_TIME_STAMP_COUNTER, MSR_IA32_PAT, + MSR_IA32_APIC_BASE, /* following MSR not emulated now */ /* - * MSR_IA32_APIC_BASE, * MSR_IA32_SYSENTER_CS, * MSR_IA32_SYSENTER_ESP, * MSR_IA32_SYSENTER_EIP, diff --git a/hypervisor/arch/x86/lapic.c b/hypervisor/arch/x86/lapic.c index 27e8dd5cc..2a5be0c12 100644 --- a/hypervisor/arch/x86/lapic.c +++ b/hypervisor/arch/x86/lapic.c @@ -6,7 +6,7 @@ #include -/* xAPIC/x2APIC Interrupt Command Register (ICR) structure */ +/* x2APIC Interrupt Command Register (ICR) structure */ union apic_icr { uint64_t value; struct { @@ -17,27 +17,16 @@ union apic_icr { uint32_t vector:8; uint32_t delivery_mode:3; uint32_t destination_mode:1; - uint32_t delivery_status:1; - uint32_t rsvd_1:1; + uint32_t rsvd_1:2; uint32_t level:1; uint32_t trigger_mode:1; uint32_t rsvd_2:2; uint32_t shorthand:2; uint32_t rsvd_3:12; - uint32_t rsvd_4:32; - } bits; - struct { - uint32_t rsvd_1:32; - uint32_t rsvd_2:24; - uint32_t dest_field:8; - } x_bits; - struct { - uint32_t rsvd_1:32; uint32_t dest_field:32; - } x2_bits; + } bits; }; -/* xAPIC/x2APIC Interrupt Command Register (ICR) structure */ union apic_lvt { uint32_t value; union { @@ -120,39 +109,12 @@ union lapic_base_msr { } fields; }; -struct lapic_info { - bool init_done; - struct { - uint64_t paddr; - void *vaddr; - } xapic; -}; - -static struct lapic_info lapic_info; static struct lapic_regs saved_lapic_regs; static union lapic_base_msr lapic_base_msr; -static inline uint32_t read_lapic_reg32(uint32_t offset) -{ - if ((offset < 0x20U) || (offset > 0x3ffU)) { - return 0; - } - - return mmio_read32(lapic_info.xapic.vaddr + offset); -} - -void write_lapic_reg32(uint32_t offset, uint32_t value) -{ - if ((offset < 0x20U) || (offset > 0x3ffU)) { - return; - } - - mmio_write32(value, lapic_info.xapic.vaddr + offset); -} - static void clear_lapic_isr(void) { - uint32_t isr_reg = LAPIC_IN_SERVICE_REGISTER_0; + uint32_t isr_reg = MSR_IA32_EXT_APIC_ISR0; /* This is a Intel recommended procedure and assures that the processor * does not get hung up due to already set "in-service" interrupts left @@ -160,20 +122,12 @@ static void clear_lapic_isr(void) * life, therefore we will ensure all the in-service bits are clear. */ do { - if (read_lapic_reg32(isr_reg) != 0U) { - write_lapic_reg32(LAPIC_EOI_REGISTER, 0U); + if (msr_read(isr_reg) != 0U) { + msr_write(MSR_IA32_EXT_APIC_EOI, 0U); continue; } - isr_reg += 0x10U; - } while (isr_reg <= LAPIC_IN_SERVICE_REGISTER_7); -} - -static void map_lapic(void) -{ - /* At some point we may need to translate this paddr to a vaddr. 1:1 - * mapping for now. - */ - lapic_info.xapic.vaddr = hpa2hva(lapic_info.xapic.paddr); + isr_reg += 0x1U; + } while (isr_reg <= MSR_IA32_EXT_APIC_ISR7); } void early_init_lapic(void) @@ -181,28 +135,17 @@ void early_init_lapic(void) /* Get local APIC base address */ lapic_base_msr.value = msr_read(MSR_IA32_APIC_BASE); - /* Initialize globals only 1 time */ - if (lapic_info.init_done == false) { - /* Get Local APIC physical address. */ - lapic_info.xapic.paddr = LAPIC_BASE; - - /* Map in the local xAPIC */ - map_lapic(); - - lapic_info.init_done = true; - } - - /* Check if xAPIC mode enabled */ - if (lapic_base_msr.fields.xAPIC_enable == 0U) { - /* Ensure in xAPIC mode */ - lapic_base_msr.fields.xAPIC_enable = 1U; - lapic_base_msr.fields.x2APIC_enable = 0U; - msr_write(MSR_IA32_APIC_BASE, lapic_base_msr.value); - } else { - /* Check if x2apic is disabled */ - ASSERT(lapic_base_msr.fields.x2APIC_enable == 0U, - "Disable X2APIC in BIOS"); - } + /* Enable LAPIC in x2APIC mode*/ + /* The following sequence of msr writes to enable x2APIC + * will work irrespective of the state of LAPIC + * left by BIOS + */ + /* Step1: Enable LAPIC in xAPIC mode */ + lapic_base_msr.fields.xAPIC_enable = 1U; + msr_write(MSR_IA32_APIC_BASE, lapic_base_msr.value); + /* Step2: Enable LAPIC in x2APIC mode */ + lapic_base_msr.fields.x2APIC_enable = 1U; + msr_write(MSR_IA32_APIC_BASE, lapic_base_msr.value); } /** @@ -210,25 +153,19 @@ void early_init_lapic(void) */ void init_lapic(uint16_t pcpu_id) { - /* Set the Logical Destination Register */ - write_lapic_reg32(LAPIC_LOGICAL_DESTINATION_REGISTER, - ((1U << pcpu_id) << 24U)); - - /* Set the Destination Format Register */ - write_lapic_reg32(LAPIC_DESTINATION_FORMAT_REGISTER, 0xfU << 28U); - + per_cpu(lapic_ldr, pcpu_id) = (uint32_t) msr_read(MSR_IA32_EXT_APIC_LDR); /* Mask all LAPIC LVT entries before enabling the local APIC */ - write_lapic_reg32(LAPIC_LVT_CMCI_REGISTER, LAPIC_LVT_MASK); - write_lapic_reg32(LAPIC_LVT_TIMER_REGISTER, LAPIC_LVT_MASK); - write_lapic_reg32(LAPIC_LVT_THERMAL_SENSOR_REGISTER, LAPIC_LVT_MASK); - write_lapic_reg32(LAPIC_LVT_PMC_REGISTER, LAPIC_LVT_MASK); - write_lapic_reg32(LAPIC_LVT_LINT0_REGISTER, LAPIC_LVT_MASK); - write_lapic_reg32(LAPIC_LVT_LINT1_REGISTER, LAPIC_LVT_MASK); - write_lapic_reg32(LAPIC_LVT_ERROR_REGISTER, LAPIC_LVT_MASK); + msr_write(MSR_IA32_EXT_APIC_LVT_CMCI, LAPIC_LVT_MASK); + msr_write(MSR_IA32_EXT_APIC_LVT_TIMER, LAPIC_LVT_MASK); + msr_write(MSR_IA32_EXT_APIC_LVT_THERMAL, LAPIC_LVT_MASK); + msr_write(MSR_IA32_EXT_APIC_LVT_PMI, LAPIC_LVT_MASK); + msr_write(MSR_IA32_EXT_APIC_LVT_LINT0, LAPIC_LVT_MASK); + msr_write(MSR_IA32_EXT_APIC_LVT_LINT1, LAPIC_LVT_MASK); + msr_write(MSR_IA32_EXT_APIC_LVT_ERROR, LAPIC_LVT_MASK); /* Enable Local APIC */ /* TODO: add spurious-interrupt handler */ - write_lapic_reg32(LAPIC_SPURIOUS_VECTOR_REGISTER, + msr_write(MSR_IA32_EXT_APIC_SIVR, LAPIC_SVR_APIC_ENABLE_MASK | LAPIC_SVR_VECTOR); /* Ensure there are no ISR bits set. */ @@ -237,68 +174,47 @@ void init_lapic(uint16_t pcpu_id) void save_lapic(struct lapic_regs *regs) { - regs->id.v = read_lapic_reg32(LAPIC_ID_REGISTER); - regs->tpr.v = read_lapic_reg32(LAPIC_TASK_PRIORITY_REGISTER); - regs->apr.v = read_lapic_reg32(LAPIC_ARBITRATION_PRIORITY_REGISTER); - regs->ppr.v = read_lapic_reg32(LAPIC_PROCESSOR_PRIORITY_REGISTER); - regs->ldr.v = read_lapic_reg32(LAPIC_LOGICAL_DESTINATION_REGISTER); - regs->dfr.v = read_lapic_reg32(LAPIC_DESTINATION_FORMAT_REGISTER); - regs->tmr[0].v = read_lapic_reg32(LAPIC_TRIGGER_MODE_REGISTER_0); - regs->tmr[1].v = read_lapic_reg32(LAPIC_TRIGGER_MODE_REGISTER_1); - regs->tmr[2].v = read_lapic_reg32(LAPIC_TRIGGER_MODE_REGISTER_2); - regs->tmr[3].v = read_lapic_reg32(LAPIC_TRIGGER_MODE_REGISTER_3); - regs->tmr[4].v = read_lapic_reg32(LAPIC_TRIGGER_MODE_REGISTER_4); - regs->tmr[5].v = read_lapic_reg32(LAPIC_TRIGGER_MODE_REGISTER_5); - regs->tmr[6].v = read_lapic_reg32(LAPIC_TRIGGER_MODE_REGISTER_6); - regs->tmr[7].v = read_lapic_reg32(LAPIC_TRIGGER_MODE_REGISTER_7); - regs->svr.v = read_lapic_reg32(LAPIC_SPURIOUS_VECTOR_REGISTER); + regs->tpr.v = (uint32_t) msr_read(MSR_IA32_EXT_APIC_TPR); + regs->ppr.v = (uint32_t) msr_read(MSR_IA32_EXT_APIC_PPR); + regs->tmr[0].v = (uint32_t) msr_read(MSR_IA32_EXT_APIC_TMR0); + regs->tmr[1].v = (uint32_t) msr_read(MSR_IA32_EXT_APIC_TMR1); + regs->tmr[2].v = (uint32_t) msr_read(MSR_IA32_EXT_APIC_TMR2); + regs->tmr[3].v = (uint32_t) msr_read(MSR_IA32_EXT_APIC_TMR3); + regs->tmr[4].v = (uint32_t) msr_read(MSR_IA32_EXT_APIC_TMR4); + regs->tmr[5].v = (uint32_t) msr_read(MSR_IA32_EXT_APIC_TMR5); + regs->tmr[6].v = (uint32_t) msr_read(MSR_IA32_EXT_APIC_TMR6); + regs->tmr[7].v = (uint32_t) msr_read(MSR_IA32_EXT_APIC_TMR7); + regs->svr.v = (uint32_t) msr_read(MSR_IA32_EXT_APIC_SIVR); regs->lvt[APIC_LVT_TIMER].v = - read_lapic_reg32(LAPIC_LVT_TIMER_REGISTER); + (uint32_t) msr_read(MSR_IA32_EXT_APIC_LVT_TIMER); regs->lvt[APIC_LVT_LINT0].v = - read_lapic_reg32(LAPIC_LVT_LINT0_REGISTER); + (uint32_t) msr_read(MSR_IA32_EXT_APIC_LVT_LINT0); regs->lvt[APIC_LVT_LINT1].v = - read_lapic_reg32(LAPIC_LVT_LINT1_REGISTER); + (uint32_t) msr_read(MSR_IA32_EXT_APIC_LVT_LINT1); regs->lvt[APIC_LVT_ERROR].v = - read_lapic_reg32(LAPIC_LVT_ERROR_REGISTER); - regs->icr_timer.v = read_lapic_reg32(LAPIC_INITIAL_COUNT_REGISTER); - regs->ccr_timer.v = read_lapic_reg32(LAPIC_CURRENT_COUNT_REGISTER); + (uint32_t) msr_read(MSR_IA32_EXT_APIC_LVT_ERROR); + regs->icr_timer.v = (uint32_t) msr_read(MSR_IA32_EXT_APIC_INIT_COUNT); + regs->ccr_timer.v = (uint32_t) msr_read(MSR_IA32_EXT_APIC_CUR_COUNT); regs->dcr_timer.v = - read_lapic_reg32(LAPIC_DIVIDE_CONFIGURATION_REGISTER); + (uint32_t) msr_read(MSR_IA32_EXT_APIC_DIV_CONF); } static void restore_lapic(struct lapic_regs *regs) { - write_lapic_reg32(LAPIC_ID_REGISTER, regs->id.v); - write_lapic_reg32(LAPIC_TASK_PRIORITY_REGISTER, regs->tpr.v); - write_lapic_reg32(LAPIC_LOGICAL_DESTINATION_REGISTER, regs->ldr.v); - write_lapic_reg32(LAPIC_DESTINATION_FORMAT_REGISTER, regs->dfr.v); - write_lapic_reg32(LAPIC_SPURIOUS_VECTOR_REGISTER, regs->svr.v); - write_lapic_reg32(LAPIC_LVT_TIMER_REGISTER, - regs->lvt[APIC_LVT_TIMER].v); + msr_write(MSR_IA32_EXT_APIC_TPR, (uint64_t) regs->tpr.v); + msr_write(MSR_IA32_EXT_APIC_SIVR, (uint64_t) regs->svr.v); + msr_write(MSR_IA32_EXT_APIC_LVT_TIMER, + (uint64_t) regs->lvt[APIC_LVT_TIMER].v); - write_lapic_reg32(LAPIC_LVT_LINT0_REGISTER, - regs->lvt[APIC_LVT_LINT0].v); - write_lapic_reg32(LAPIC_LVT_LINT1_REGISTER, - regs->lvt[APIC_LVT_LINT1].v); + msr_write(MSR_IA32_EXT_APIC_LVT_LINT0, + (uint64_t) regs->lvt[APIC_LVT_LINT0].v); + msr_write(MSR_IA32_EXT_APIC_LVT_LINT1, + (uint64_t) regs->lvt[APIC_LVT_LINT1].v); - write_lapic_reg32(LAPIC_LVT_ERROR_REGISTER, - regs->lvt[APIC_LVT_ERROR].v); - write_lapic_reg32(LAPIC_INITIAL_COUNT_REGISTER, regs->icr_timer.v); - write_lapic_reg32(LAPIC_DIVIDE_CONFIGURATION_REGISTER, - regs->dcr_timer.v); - - - write_lapic_reg32(LAPIC_ARBITRATION_PRIORITY_REGISTER, regs->apr.v); - write_lapic_reg32(LAPIC_PROCESSOR_PRIORITY_REGISTER, regs->ppr.v); - write_lapic_reg32(LAPIC_TRIGGER_MODE_REGISTER_0, regs->tmr[0].v); - write_lapic_reg32(LAPIC_TRIGGER_MODE_REGISTER_1, regs->tmr[1].v); - write_lapic_reg32(LAPIC_TRIGGER_MODE_REGISTER_2, regs->tmr[2].v); - write_lapic_reg32(LAPIC_TRIGGER_MODE_REGISTER_3, regs->tmr[3].v); - write_lapic_reg32(LAPIC_TRIGGER_MODE_REGISTER_4, regs->tmr[4].v); - write_lapic_reg32(LAPIC_TRIGGER_MODE_REGISTER_5, regs->tmr[5].v); - write_lapic_reg32(LAPIC_TRIGGER_MODE_REGISTER_6, regs->tmr[6].v); - write_lapic_reg32(LAPIC_TRIGGER_MODE_REGISTER_7, regs->tmr[7].v); - write_lapic_reg32(LAPIC_CURRENT_COUNT_REGISTER, regs->ccr_timer.v); + msr_write(MSR_IA32_EXT_APIC_LVT_ERROR, + (uint64_t) regs->lvt[APIC_LVT_ERROR].v); + msr_write(MSR_IA32_EXT_APIC_INIT_COUNT, (uint64_t) regs->icr_timer.v); + msr_write(MSR_IA32_EXT_APIC_DIV_CONF, (uint64_t) regs->dcr_timer.v); } void suspend_lapic(void) @@ -308,9 +224,9 @@ void suspend_lapic(void) save_lapic(&saved_lapic_regs); /* disable APIC with software flag */ - val = read_lapic_reg32(LAPIC_SPURIOUS_VECTOR_REGISTER); + val = (uint32_t) msr_read(MSR_IA32_EXT_APIC_SIVR); val = (~LAPIC_SVR_APIC_ENABLE_MASK) & val; - write_lapic_reg32(LAPIC_SPURIOUS_VECTOR_REGISTER, val); + msr_write(MSR_IA32_EXT_APIC_SIVR, (uint64_t) val); } void resume_lapic(void) @@ -323,26 +239,14 @@ void resume_lapic(void) void send_lapic_eoi(void) { - write_lapic_reg32(LAPIC_EOI_REGISTER, 0U); + msr_write(MSR_IA32_EXT_APIC_EOI, 0U); } -static void wait_for_delivery(void) +uint32_t get_cur_lapic_id(void) { - union apic_icr tmp; + uint32_t lapic_id; - do { - tmp.value_32.lo_32 = - read_lapic_reg32(LAPIC_INT_COMMAND_REGISTER_0); - } while (tmp.bits.delivery_status != 0U); -} - -uint8_t get_cur_lapic_id(void) -{ - uint32_t lapic_id_reg; - uint8_t lapic_id; - - lapic_id_reg = read_lapic_reg32(LAPIC_ID_REGISTER); - lapic_id = (uint8_t)(lapic_id_reg >> 24U); + lapic_id = (uint32_t) msr_read(MSR_IA32_EXT_XAPICID); return lapic_id; } @@ -362,20 +266,18 @@ send_startup_ipi(enum intr_cpu_startup_shorthand cpu_startup_shorthand, if (cpu_startup_shorthand == INTR_CPU_STARTUP_USE_DEST) { shorthand = INTR_LAPIC_ICR_USE_DEST_ARRAY; - icr.x_bits.dest_field = per_cpu(lapic_id, dest_pcpu_id); + icr.value_32.hi_32 = per_cpu(lapic_id, dest_pcpu_id); } else { /* Use destination shorthand */ shorthand = INTR_LAPIC_ICR_ALL_EX_SELF; icr.value_32.hi_32 = 0U; } /* Assert INIT IPI */ - write_lapic_reg32(LAPIC_INT_COMMAND_REGISTER_1, icr.value_32.hi_32); icr.bits.shorthand = shorthand; icr.bits.delivery_mode = INTR_LAPIC_ICR_INIT; icr.bits.level = INTR_LAPIC_ICR_ASSERT; icr.bits.trigger_mode = INTR_LAPIC_ICR_LEVEL; - write_lapic_reg32(LAPIC_INT_COMMAND_REGISTER_0, icr.value_32.lo_32); - wait_for_delivery(); + msr_write(MSR_IA32_EXT_APIC_ICR, icr.value); /* Give 10ms for INIT sequence to complete for old processors. * Modern processors (family == 6) don't need to wait here. @@ -386,19 +288,15 @@ send_startup_ipi(enum intr_cpu_startup_shorthand cpu_startup_shorthand, } /* De-assert INIT IPI */ - write_lapic_reg32(LAPIC_INT_COMMAND_REGISTER_1, icr.value_32.hi_32); icr.bits.level = INTR_LAPIC_ICR_DEASSERT; - write_lapic_reg32(LAPIC_INT_COMMAND_REGISTER_0, icr.value_32.lo_32); - wait_for_delivery(); + msr_write(MSR_IA32_EXT_APIC_ICR, icr.value); /* Send Start IPI with page number of secondary reset code */ - write_lapic_reg32(LAPIC_INT_COMMAND_REGISTER_1, icr.value_32.hi_32); icr.value_32.lo_32 = 0U; icr.bits.shorthand = shorthand; icr.bits.delivery_mode = INTR_LAPIC_ICR_STARTUP; icr.bits.vector = (uint8_t)(cpu_startup_start_address >> 12U); - write_lapic_reg32(LAPIC_INT_COMMAND_REGISTER_0, icr.value_32.lo_32); - wait_for_delivery(); + msr_write(MSR_IA32_EXT_APIC_ICR, icr.value); if (boot_cpu_data.family == 6U) { udelay(10U); /* 10us is enough for Modern processors */ @@ -407,64 +305,44 @@ send_startup_ipi(enum intr_cpu_startup_shorthand cpu_startup_shorthand, } /* Send another start IPI as per the Intel Arch specification */ - write_lapic_reg32(LAPIC_INT_COMMAND_REGISTER_1, icr.value_32.hi_32); - write_lapic_reg32(LAPIC_INT_COMMAND_REGISTER_0, icr.value_32.lo_32); - wait_for_delivery(); + msr_write(MSR_IA32_EXT_APIC_ICR, icr.value); } -/* dest_mode must be INTR_LAPIC_ICR_PHYSICAL(0x0U) or - * INTR_LAPIC_ICR_LOGICAL(0x1U) - */ -void send_dest_ipi(uint32_t dest, uint32_t vector, uint32_t dest_mode) +void send_dest_ipi_mask(uint32_t dest_mask, uint32_t vector) { - uint32_t hi_32, lo_32; + union apic_icr icr; + uint16_t pcpu_id; + uint32_t mask = dest_mask; - /* Set the target processor. */ - hi_32 = dest << 24U; + icr.value_32.lo_32 = vector | (INTR_LAPIC_ICR_PHYSICAL << 11U); - /* Set the vector ID. */ - lo_32 = vector | (dest_mode << 11U); + pcpu_id = ffs64(mask); - /* Set the destination field to the target processor. */ - write_lapic_reg32(LAPIC_INT_COMMAND_REGISTER_1, hi_32); - - /* Write the vector ID to ICR. */ - write_lapic_reg32(LAPIC_INT_COMMAND_REGISTER_0, lo_32); - - wait_for_delivery(); + while (pcpu_id != INVALID_BIT_INDEX) { + bitmap32_clear_nolock(pcpu_id, &mask); + if (bitmap_test(pcpu_id, &pcpu_active_bitmap)) { + icr.value_32.hi_32 = per_cpu(lapic_id, pcpu_id); + msr_write(MSR_IA32_EXT_APIC_ICR, icr.value); + } else { + pr_err("pcpu_id %d not in active!", pcpu_id); + } + pcpu_id = ffs64(mask); + } } void send_single_ipi(uint16_t pcpu_id, uint32_t vector) -{ - uint32_t dest_lapic_id; - - /* Get the lapic ID of the destination processor. */ - dest_lapic_id = per_cpu(lapic_id, pcpu_id); - - send_dest_ipi(dest_lapic_id, vector, INTR_LAPIC_ICR_PHYSICAL); -} - -int send_shorthand_ipi(uint8_t vector, - uint8_t shorthand, uint8_t delivery_mode) { union apic_icr icr; - int status = 0; - if ((shorthand < INTR_LAPIC_ICR_SELF) - || (shorthand > INTR_LAPIC_ICR_ALL_EX_SELF) - || (delivery_mode > INTR_LAPIC_ICR_NMI)) { - status = -EINVAL; + if (bitmap_test(pcpu_id, &pcpu_active_bitmap)) { + /* Set the destination field to the target processor. */ + icr.value_32.hi_32 = per_cpu(lapic_id, pcpu_id); + + /* Write the vector ID to ICR. */ + icr.value_32.lo_32 = vector | (INTR_LAPIC_ICR_PHYSICAL << 11U); + + msr_write(MSR_IA32_EXT_APIC_ICR, icr.value); + } else { + pr_err("pcpu_id %d not in active!", pcpu_id); } - - ASSERT(status == 0, "Incorrect arguments"); - - icr.value = 0U; - icr.bits.shorthand = shorthand; - icr.bits.delivery_mode = delivery_mode; - icr.bits.vector = vector; - write_lapic_reg32(LAPIC_INT_COMMAND_REGISTER_1, icr.value_32.hi_32); - write_lapic_reg32(LAPIC_INT_COMMAND_REGISTER_0, icr.value_32.lo_32); - wait_for_delivery(); - - return status; } diff --git a/hypervisor/arch/x86/notify.c b/hypervisor/arch/x86/notify.c index 7ca82e3b0..e49417148 100644 --- a/hypervisor/arch/x86/notify.c +++ b/hypervisor/arch/x86/notify.c @@ -51,8 +51,7 @@ void smp_call_function(uint64_t mask, smp_call_func_t func, void *data) } pcpu_id = ffs64(mask); } - send_dest_ipi((uint32_t)smp_call_mask, VECTOR_NOTIFY_VCPU, - INTR_LAPIC_ICR_LOGICAL); + send_dest_ipi_mask((uint32_t)smp_call_mask, VECTOR_NOTIFY_VCPU); /* wait for current smp call complete */ wait_sync_change(&smp_call_mask, 0UL); } diff --git a/hypervisor/arch/x86/timer.c b/hypervisor/arch/x86/timer.c index 0f66e6161..9bd2c3572 100644 --- a/hypervisor/arch/x86/timer.c +++ b/hypervisor/arch/x86/timer.c @@ -122,7 +122,7 @@ static void init_tsc_deadline_timer(void) val = VECTOR_TIMER; val |= APIC_LVTT_TM_TSCDLT; /* TSC deadline and unmask */ - write_lapic_reg32(LAPIC_LVT_TIMER_REGISTER, val); + msr_write(MSR_IA32_EXT_APIC_LVT_TIMER, val); asm volatile("mfence" : : : "memory"); /* disarm timer */ diff --git a/hypervisor/arch/x86/virq.c b/hypervisor/arch/x86/virq.c index 287ae0c78..278b50c2b 100644 --- a/hypervisor/arch/x86/virq.c +++ b/hypervisor/arch/x86/virq.c @@ -184,11 +184,11 @@ void dump_lapic(void) { dev_dbg(ACRN_DBG_INTR, "LAPIC: TIME %08x, init=0x%x cur=0x%x ISR=0x%x IRR=0x%x", - mmio_read32(hpa2hva(LAPIC_BASE + LAPIC_LVT_TIMER_REGISTER)), - mmio_read32(hpa2hva(LAPIC_BASE + LAPIC_INITIAL_COUNT_REGISTER)), - mmio_read32(hpa2hva(LAPIC_BASE + LAPIC_CURRENT_COUNT_REGISTER)), - mmio_read32(hpa2hva(LAPIC_BASE + LAPIC_IN_SERVICE_REGISTER_7)), - mmio_read32(hpa2hva(LAPIC_BASE + LAPIC_INT_REQUEST_REGISTER_7)) + msr_read(MSR_IA32_EXT_APIC_LVT_TIMER), + msr_read(MSR_IA32_EXT_APIC_INIT_COUNT), + msr_read(MSR_IA32_EXT_APIC_CUR_COUNT), + msr_read(MSR_IA32_EXT_APIC_ISR7), + msr_read(MSR_IA32_EXT_APIC_IRR7) ); } diff --git a/hypervisor/arch/x86/vtd.c b/hypervisor/arch/x86/vtd.c index 9addcc00a..318b56ca8 100644 --- a/hypervisor/arch/x86/vtd.c +++ b/hypervisor/arch/x86/vtd.c @@ -709,7 +709,7 @@ static void dmar_fault_msi_write(struct dmar_drhd_rt *dmar_uint, { uint32_t data; uint32_t addr_low; - uint8_t lapic_id = get_cur_lapic_id(); + uint32_t lapic_id = get_cur_lapic_id(); data = DMAR_MSI_DELIVERY_LOWPRI | vector; /* redirection hint: 0 diff --git a/hypervisor/boot/acpi.c b/hypervisor/boot/acpi.c index 696300464..ad38cf5a3 100644 --- a/hypervisor/boot/acpi.c +++ b/hypervisor/boot/acpi.c @@ -222,7 +222,12 @@ static void *get_acpi_tbl(const char *sig) return hpa2hva(addr); } -static uint16_t local_parse_madt(void *madt, uint8_t lapic_id_array[MAX_PCPU_NUM]) +/* TODO: As ACRN supports only x2APIC mode, we need to + * check upon using x2APIC APIC entries (Type 9) in MADT instead + * of Type 0 + */ +static uint16_t +local_parse_madt(void *madt, uint32_t lapic_id_array[MAX_PCPU_NUM]) { uint16_t pcpu_id = 0U; struct acpi_madt_local_apic *processor; @@ -265,7 +270,7 @@ static uint16_t local_parse_madt(void *madt, uint8_t lapic_id_array[MAX_PCPU_NUM } /* The lapic_id info gotten from madt will be returned in lapic_id_array */ -uint16_t parse_madt(uint8_t lapic_id_array[MAX_PCPU_NUM]) +uint16_t parse_madt(uint32_t lapic_id_array[MAX_PCPU_NUM]) { void *madt; diff --git a/hypervisor/boot/include/acpi.h b/hypervisor/boot/include/acpi.h index a185c99be..a88a423af 100644 --- a/hypervisor/boot/include/acpi.h +++ b/hypervisor/boot/include/acpi.h @@ -28,7 +28,7 @@ struct acpi_table_header { uint32_t asl_compiler_revision; }; -uint16_t parse_madt(uint8_t lapic_id_array[MAX_PCPU_NUM]); +uint16_t parse_madt(uint32_t lapic_id_array[MAX_PCPU_NUM]); void *get_dmar_table(void); #endif /* !ACPI_H */ diff --git a/hypervisor/include/arch/x86/cpufeatures.h b/hypervisor/include/arch/x86/cpufeatures.h index 7d3b21d2d..2226f9da0 100644 --- a/hypervisor/include/arch/x86/cpufeatures.h +++ b/hypervisor/include/arch/x86/cpufeatures.h @@ -27,7 +27,7 @@ #define X86_FEATURE_DCA ((FEAT_1_ECX << 5U) + 18U) #define X86_FEATURE_SSE4_1 ((FEAT_1_ECX << 5U) + 19U) #define X86_FEATURE_SSE4_2 ((FEAT_1_ECX << 5U) + 20U) -#define X86_FEATURE_x2APIC ((FEAT_1_ECX << 5U) + 21U) +#define X86_FEATURE_X2APIC ((FEAT_1_ECX << 5U) + 21U) #define X86_FEATURE_MOVBE ((FEAT_1_ECX << 5U) + 22U) #define X86_FEATURE_POPCNT ((FEAT_1_ECX << 5U) + 23U) #define X86_FEATURE_TSC_DEADLINE ((FEAT_1_ECX << 5U) + 24U) diff --git a/hypervisor/include/arch/x86/guest/guest.h b/hypervisor/include/arch/x86/guest/guest.h index 5efa0cb1a..8089b1647 100644 --- a/hypervisor/include/arch/x86/guest/guest.h +++ b/hypervisor/include/arch/x86/guest/guest.h @@ -30,7 +30,8 @@ #define IDX_BIOS_SIGN_ID (IDX_BIOS_UPDT_TRIG + 1U) #define IDX_TSC (IDX_BIOS_SIGN_ID + 1U) #define IDX_PAT (IDX_TSC + 1U) -#define IDX_MAX_MSR (IDX_PAT + 1U) +#define IDX_APIC_BASE (IDX_PAT + 1U) +#define IDX_MAX_MSR (IDX_APIC_BASE + 1U) /* * VCPU related APIs diff --git a/hypervisor/include/arch/x86/lapic.h b/hypervisor/include/arch/x86/lapic.h index 2651a3f62..e9de759f7 100644 --- a/hypervisor/include/arch/x86/lapic.h +++ b/hypervisor/include/arch/x86/lapic.h @@ -33,62 +33,6 @@ #define INTR_LAPIC_ICR_ALL_INC_SELF 0x2U #define INTR_LAPIC_ICR_ALL_EX_SELF 0x3U -/* LAPIC register offset for memory mapped IO access */ -#define LAPIC_ID_REGISTER 0x00000020U -#define LAPIC_VERSION_REGISTER 0x00000030U -#define LAPIC_TASK_PRIORITY_REGISTER 0x00000080U -#define LAPIC_ARBITRATION_PRIORITY_REGISTER 0x00000090U -#define LAPIC_PROCESSOR_PRIORITY_REGISTER 0x000000A0U -#define LAPIC_EOI_REGISTER 0x000000B0U -#define LAPIC_REMOTE_READ_REGISTER 0x000000C0U -#define LAPIC_LOGICAL_DESTINATION_REGISTER 0x000000D0U -#define LAPIC_DESTINATION_FORMAT_REGISTER 0x000000E0U -#define LAPIC_SPURIOUS_VECTOR_REGISTER 0x000000F0U -#define LAPIC_IN_SERVICE_REGISTER_0 0x00000100U -#define LAPIC_IN_SERVICE_REGISTER_1 0x00000110U -#define LAPIC_IN_SERVICE_REGISTER_2 0x00000120U -#define LAPIC_IN_SERVICE_REGISTER_3 0x00000130U -#define LAPIC_IN_SERVICE_REGISTER_4 0x00000140U -#define LAPIC_IN_SERVICE_REGISTER_5 0x00000150U -#define LAPIC_IN_SERVICE_REGISTER_6 0x00000160U -#define LAPIC_IN_SERVICE_REGISTER_7 0x00000170U -#define LAPIC_TRIGGER_MODE_REGISTER_0 0x00000180U -#define LAPIC_TRIGGER_MODE_REGISTER_1 0x00000190U -#define LAPIC_TRIGGER_MODE_REGISTER_2 0x000001A0U -#define LAPIC_TRIGGER_MODE_REGISTER_3 0x000001B0U -#define LAPIC_TRIGGER_MODE_REGISTER_4 0x000001C0U -#define LAPIC_TRIGGER_MODE_REGISTER_5 0x000001D0U -#define LAPIC_TRIGGER_MODE_REGISTER_6 0x000001E0U -#define LAPIC_TRIGGER_MODE_REGISTER_7 0x000001F0U -#define LAPIC_INT_REQUEST_REGISTER_0 0x00000200U -#define LAPIC_INT_REQUEST_REGISTER_1 0x00000210U -#define LAPIC_INT_REQUEST_REGISTER_2 0x00000220U -#define LAPIC_INT_REQUEST_REGISTER_3 0x00000230U -#define LAPIC_INT_REQUEST_REGISTER_4 0x00000240U -#define LAPIC_INT_REQUEST_REGISTER_5 0x00000250U -#define LAPIC_INT_REQUEST_REGISTER_6 0x00000260U -#define LAPIC_INT_REQUEST_REGISTER_7 0x00000270U -#define LAPIC_ERROR_STATUS_REGISTER 0x00000280U -#define LAPIC_LVT_CMCI_REGISTER 0x000002F0U -#define LAPIC_INT_COMMAND_REGISTER_0 0x00000300U -#define LAPIC_INT_COMMAND_REGISTER_1 0x00000310U -#define LAPIC_LVT_TIMER_REGISTER 0x00000320U -#define LAPIC_LVT_THERMAL_SENSOR_REGISTER 0x00000330U -#define LAPIC_LVT_PMC_REGISTER 0x00000340U -#define LAPIC_LVT_LINT0_REGISTER 0x00000350U -#define LAPIC_LVT_LINT1_REGISTER 0x00000360U -#define LAPIC_LVT_ERROR_REGISTER 0x00000370U -#define LAPIC_INITIAL_COUNT_REGISTER 0x00000380U -#define LAPIC_CURRENT_COUNT_REGISTER 0x00000390U -#define LAPIC_DIVIDE_CONFIGURATION_REGISTER 0x000003E0U - -/* LAPIC CPUID bit and bitmask definitions */ -#define CPUID_OUT_RDX_APIC_PRESENT ((uint64_t) 1UL << 9U) -#define CPUID_OUT_RCX_X2APIC_PRESENT ((uint64_t) 1UL << 21U) - -/* LAPIC MSR bit and bitmask definitions */ -#define MSR_01B_XAPIC_GLOBAL_ENABLE ((uint64_t) 1UL << 11U) - /* LAPIC register bit and bitmask definitions */ #define LAPIC_SVR_VECTOR 0x000000FFU #define LAPIC_SVR_APIC_ENABLE_MASK 0x00000100U @@ -107,38 +51,17 @@ enum intr_cpu_startup_shorthand { INTR_CPU_STARTUP_UNKNOWN, }; -union lapic_id_reg { - uint32_t value; - struct { - uint8_t xapic_id; - uint8_t rsvd[3]; - } xapic; - union { - uint32_t value; - struct { - uint8_t xapic_id; - uint8_t xapic_edid; - uint8_t rsvd[2]; - } ioxapic_view; - struct { - uint32_t x2apic_id:4; - uint32_t x2apic_cluster:28; - } ldr_view; - } x2apic; -}; - -void write_lapic_reg32(uint32_t offset, uint32_t value); void save_lapic(struct lapic_regs *regs); void early_init_lapic(void); void init_lapic(uint16_t pcpu_id); void send_lapic_eoi(void); -uint8_t get_cur_lapic_id(void); +uint32_t get_cur_lapic_id(void); void send_startup_ipi(enum intr_cpu_startup_shorthand cpu_startup_shorthand, uint16_t dest_pcpu_id, uint64_t cpu_startup_start_address); -/* API to send an IPI to dest guest */ -void send_dest_ipi(uint32_t dest, uint32_t vector, uint32_t dest_mode); -/* API to send an IPI to a single guest */ +/* API to send an IPI to multiple pCPUs */ +void send_dest_ipi_mask(uint32_t dest_mask, uint32_t vector); +/* API to send an IPI to a single pCPU */ void send_single_ipi(uint16_t pcpu_id, uint32_t vector); void suspend_lapic(void); diff --git a/hypervisor/include/arch/x86/per_cpu.h b/hypervisor/include/arch/x86/per_cpu.h index 0da28c9f1..aaca1cf7b 100644 --- a/hypervisor/include/arch/x86/per_cpu.h +++ b/hypervisor/include/arch/x86/per_cpu.h @@ -47,7 +47,8 @@ struct per_cpu_region { uint8_t sf_stack[CONFIG_STACK_SIZE] __aligned(16); uint8_t stack[CONFIG_STACK_SIZE] __aligned(16); char logbuf[LOG_MESSAGE_MAX_SIZE]; - uint8_t lapic_id; + uint32_t lapic_id; + uint32_t lapic_ldr; struct smp_call_info_data smp_call_info; } __aligned(CPU_PAGE_SIZE); /* per_cpu_region size aligned with CPU_PAGE_SIZE */