mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-05-11 18:05:43 +00:00
2d0ec00aff
Initially enable vcpu hotplug in qemu for arm base on Salli's work[1]. Fixes:#3280 Signed-off-by: Huang Shijie <shijie8@gmail.com> [1] https://github.com/salil-mehta/qemu/tree/virt-cpuhp-armv8/rfc-v1
244 lines
8.4 KiB
Diff
244 lines
8.4 KiB
Diff
From 1d9605a449833e464bd0388bb658d31a84ad4c7d Mon Sep 17 00:00:00 2001
|
|
From: Salil Mehta <salil.mehta@huawei.com>
|
|
Date: Wed, 24 Nov 2021 15:30:20 +0800
|
|
Subject: [PATCH 05/28] arm/cpuhp: Pre-create disabled possible vcpus @machine
|
|
init
|
|
|
|
In ARMv8 architecture, GIC needs all the vcpus to be created and present when
|
|
it is initialized. This is because:
|
|
1. GICC and MPIDR association must be fixed at the VM initialization time.
|
|
This is represented by register GIC_TYPER(mp_afffinity, proc_num)
|
|
2. GICC(cpu interfaces), GICR(redistributors) etc all must be initialized
|
|
at the boot time as well.
|
|
3. Memory regions associated with GICR etc. cannot be changed(add/del/mod)
|
|
after VM has inited.
|
|
|
|
This patch adds the support to pre-create all such possible vcpus within the
|
|
host using the KVM interface as part of the virt machine initialization. These
|
|
vcpus could later be attached to QOM/ACPI while they are actually hot plugged
|
|
and made present.
|
|
|
|
NOTE: There is some refactoring related to the kvm_destroy_vcpu/kvm_get_vcpu
|
|
(to make use of the common code) has been intentionaly left out in RFC
|
|
version to avoid obscuring the framework change of the cpu hotplug. The
|
|
existing code being presented in this patch could further be optimized
|
|
later.
|
|
|
|
Co-developed-by: Keqian Zhu <zhukeqian1@huawei.com>
|
|
Signed-off-by: Salil Mehta <salil.mehta@huawei.com>
|
|
Signed-off-by: Huang Shijie <shijie8@gmail.com>
|
|
---
|
|
accel/kvm/kvm-all.c | 31 +++++++++++++++++++++++++++
|
|
accel/kvm/kvm-cpus.h | 2 ++
|
|
hw/arm/virt.c | 50 +++++++++++++++++++++++++++++++++++++++++---
|
|
target/arm/kvm.c | 33 +++++++++++++++++++++++++++++
|
|
target/arm/kvm_arm.h | 11 ++++++++++
|
|
5 files changed, 124 insertions(+), 3 deletions(-)
|
|
|
|
diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c
|
|
index 0125c17edb..46717dad14 100644
|
|
--- a/accel/kvm/kvm-all.c
|
|
+++ b/accel/kvm/kvm-all.c
|
|
@@ -425,6 +425,37 @@ err:
|
|
return ret;
|
|
}
|
|
|
|
+void kvm_park_vcpu(CPUState *cs)
|
|
+{
|
|
+ unsigned long vcpu_id = cs->cpu_index;
|
|
+ struct KVMParkedVcpu *vcpu;
|
|
+
|
|
+ vcpu = g_malloc0(sizeof(*vcpu));
|
|
+ vcpu->vcpu_id = vcpu_id;
|
|
+ vcpu->kvm_fd = cs->kvm_fd;
|
|
+ QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
|
|
+}
|
|
+
|
|
+int kvm_create_vcpu(CPUState *cpu)
|
|
+{
|
|
+ unsigned long vcpu_id = cpu->cpu_index;
|
|
+ KVMState *s = kvm_state;
|
|
+ int ret = 0;
|
|
+
|
|
+ DPRINTF("kvm_create_vcpu\n");
|
|
+
|
|
+ ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id);
|
|
+ if (ret < 0) {
|
|
+ goto err;
|
|
+ }
|
|
+ cpu->kvm_fd = ret;
|
|
+ cpu->kvm_state = s;
|
|
+ cpu->vcpu_dirty = true;
|
|
+
|
|
+err:
|
|
+ return ret;
|
|
+}
|
|
+
|
|
void kvm_destroy_vcpu(CPUState *cpu)
|
|
{
|
|
if (do_kvm_destroy_vcpu(cpu) < 0) {
|
|
diff --git a/accel/kvm/kvm-cpus.h b/accel/kvm/kvm-cpus.h
|
|
index bf0bd1bee4..54a4fd19cf 100644
|
|
--- a/accel/kvm/kvm-cpus.h
|
|
+++ b/accel/kvm/kvm-cpus.h
|
|
@@ -14,6 +14,8 @@
|
|
|
|
int kvm_init_vcpu(CPUState *cpu, Error **errp);
|
|
int kvm_cpu_exec(CPUState *cpu);
|
|
+int kvm_create_vcpu(CPUState *cpu);
|
|
+void kvm_park_vcpu(CPUState *cs);
|
|
void kvm_destroy_vcpu(CPUState *cpu);
|
|
void kvm_cpu_synchronize_post_reset(CPUState *cpu);
|
|
void kvm_cpu_synchronize_post_init(CPUState *cpu);
|
|
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
|
|
index 81219d1d5a..853288b34a 100644
|
|
--- a/hw/arm/virt.c
|
|
+++ b/hw/arm/virt.c
|
|
@@ -1995,13 +1995,14 @@ static void machvirt_init(MachineState *machine)
|
|
assert(possible_cpus->len == max_cpus);
|
|
for (n = 0; n < possible_cpus->len; n++) {
|
|
Object *cpuobj;
|
|
-/* CPUState *cs;
|
|
+ CPUState *cs;
|
|
|
|
- if (n >= smp_cpus) {
|
|
+/* if (n >= smp_cpus) {
|
|
break;
|
|
}
|
|
*/
|
|
cpuobj = object_new(possible_cpus->cpus[n].type);
|
|
+ cs = CPU(cpuobj);
|
|
/* object_property_set_int(cpuobj, "mp-affinity",
|
|
possible_cpus->cpus[n].arch_id, NULL);
|
|
|
|
@@ -2096,9 +2097,52 @@ static void machvirt_init(MachineState *machine)
|
|
&error_abort);
|
|
}
|
|
}
|
|
-*/
|
|
+
|
|
qdev_realize(DEVICE(cpuobj), NULL, &error_fatal);
|
|
object_unref(cpuobj);
|
|
+*/
|
|
+ if (n < smp_cpus) {
|
|
+ char *core_id = g_strdup_printf("core%d", n);
|
|
+ qdev_set_id(DEVICE(cpuobj),core_id);
|
|
+ object_property_set_bool(cpuobj, "realized", true, &error_fatal);
|
|
+ g_free(core_id);
|
|
+ object_unref(OBJECT(cs));
|
|
+ } else {
|
|
+ CPUArchId *cpu_slot;
|
|
+ /* handling for vcpus which are yet to be hot-plugged */
|
|
+ cs->cpu_index = n; cs->disabled=true;
|
|
+ /* ARM host vcpu features need to be fixed at the boot time */
|
|
+ virt_cpu_set_properties(cpuobj, &possible_cpus->cpus[n]);
|
|
+ /*
|
|
+ * For KVM, we shall be pre-creating the now disabled/un-plugged
|
|
+ * possbile host vcpus and park them till the time they are
|
|
+ * actually hot plugged. This is required to pre-size the host
|
|
+ * GICC and GICR with the all possible vcpus for this VM.
|
|
+ */
|
|
+ if (kvm_enabled()) {
|
|
+ kvm_arm_create_host_vcpu(ARM_CPU(cs));
|
|
+ }
|
|
+ /*
|
|
+ * Add disabled vcpu to cpu slot during the init phase of the virt machine.
|
|
+ * 1. We need this ARMCPU object during the GIC init. This object
|
|
+ * will facilitate in pre-realizing the gic. Any info like
|
|
+ * mp-affinity(required to derive gicr_type) etc could still be
|
|
+ * fetched while preserving QOM abstraction akin to realized
|
|
+ * vcpus.
|
|
+ * 2. Now, after initialization of the virt machine is complete we could use
|
|
+ * two approaches to deal with this ARMCPU object:
|
|
+ * (i) re-use this ARMCPU object during hotplug of this vcpu.
|
|
+ * OR
|
|
+ * (ii) defer release this ARMCPU object after gic has been
|
|
+ * initialized or during pre-plug phase when a vcpu is
|
|
+ * hotplugged.
|
|
+ *
|
|
+ * We will use the (ii) approach and release the ARMCPU objects after GIC
|
|
+ * and machine has been initialized in machine_init_done() phase
|
|
+ */
|
|
+ cpu_slot = virt_find_cpu_slot(machine, cs->cpu_index);
|
|
+ cpu_slot->cpu = OBJECT(cs);
|
|
+ }
|
|
}
|
|
fdt_add_timer_nodes(vms);
|
|
fdt_add_cpu_nodes(vms);
|
|
diff --git a/target/arm/kvm.c b/target/arm/kvm.c
|
|
index d8381ba224..bb75beec9d 100644
|
|
--- a/target/arm/kvm.c
|
|
+++ b/target/arm/kvm.c
|
|
@@ -32,6 +32,7 @@
|
|
#include "hw/boards.h"
|
|
#include "hw/irq.h"
|
|
#include "qemu/log.h"
|
|
+#include "accel/kvm/kvm-cpus.h"
|
|
|
|
const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
|
|
KVM_CAP_LAST_INFO
|
|
@@ -627,6 +628,38 @@ void kvm_arm_reset_vcpu(ARMCPU *cpu)
|
|
write_list_to_cpustate(cpu);
|
|
}
|
|
|
|
+void kvm_arm_create_host_vcpu(ARMCPU *cpu)
|
|
+{
|
|
+ CPUState *cs = CPU(cpu);
|
|
+ unsigned long vcpu_id = cs->cpu_index;
|
|
+ int ret;
|
|
+
|
|
+ ret = kvm_create_vcpu(cs);
|
|
+ if (ret < 0) {
|
|
+ error_report("Failed to create host vcpu %ld", vcpu_id);
|
|
+ abort();
|
|
+ }
|
|
+
|
|
+ /*
|
|
+ * Initialize the vcpu in the host. This will reset the sys regs
|
|
+ * for this vcpu and related registers like MPIDR_EL1 etc. also
|
|
+ * gets programmed during this call to host. These are referred
|
|
+ * later while setting device attributes of the GICR during GICv3
|
|
+ * reset
|
|
+ */
|
|
+ ret = kvm_arch_init_vcpu(cs);
|
|
+ if (ret < 0) {
|
|
+ error_report("Failed to initialize host vcpu %ld", vcpu_id);
|
|
+ abort();
|
|
+ }
|
|
+
|
|
+ /*
|
|
+ * park the created vcpu. shall be used during kvm_get_vcpu() when
|
|
+ * threads are created during realization of ARM vcpus
|
|
+ */
|
|
+ kvm_park_vcpu(cs);
|
|
+}
|
|
+
|
|
/*
|
|
* Update KVM's MP_STATE based on what QEMU thinks it is
|
|
*/
|
|
diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
|
|
index 34f8daa377..e0490c97b5 100644
|
|
--- a/target/arm/kvm_arm.h
|
|
+++ b/target/arm/kvm_arm.h
|
|
@@ -155,6 +155,17 @@ void kvm_arm_cpu_post_load(ARMCPU *cpu);
|
|
*/
|
|
void kvm_arm_reset_vcpu(ARMCPU *cpu);
|
|
|
|
+/**
|
|
+ * kvm_arm_create_host_vcpu:
|
|
+ * @cpu: ARMCPU
|
|
+ *
|
|
+ * Called at to pre create all possible kvm vcpus within the the host at the
|
|
+ * virt machine init time. This will also init this pre-created vcpu and
|
|
+ * hence result in vcpu reset at host. These pre created and inited vcpus
|
|
+ * shall be parked for use when ARM vcpus are actually realized.
|
|
+ */
|
|
+void kvm_arm_create_host_vcpu(ARMCPU *cpu);
|
|
+
|
|
/**
|
|
* kvm_arm_init_serror_injection:
|
|
* @cs: CPUState
|
|
--
|
|
2.30.2
|
|
|