mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-10-31 17:37:20 +00:00
dragonball: add arm64 patches for upcall
The vcpu hotplug/hotunplug feature is implemented with upcall. This commit add three patches to support the feature on aarch64. Patches: > 0005: add support of upcall on aarch64 > 0006: skip activate offline cpus' MSI interrupt > 0007: set the correct boot cpu number Fixes: #6010 Signed-off-by: xuejun-xj <jiyunxue@linux.alibaba.com>
This commit is contained in:
xuejun-xj
@@ -1 +1 @@
|
|||||||
105
|
106
|
||||||
|
|||||||
@@ -0,0 +1,163 @@
|
|||||||
|
From 16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72 Mon Sep 17 00:00:00 2001
|
||||||
|
Message-Id: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com>
|
||||||
|
From: xuejun-xj <jiyunxue@linux.alibaba.com>
|
||||||
|
Date: Wed, 10 May 2023 13:55:43 +0800
|
||||||
|
Subject: [PATCH 1/3] upcall: dragonball-devmgr suppots cpu hotplug on arm64
|
||||||
|
|
||||||
|
Enable vcpuhotplug feature on aarch64 in guest kernel. It communicates
|
||||||
|
with dragonball by using upcall. This commit does these changes:
|
||||||
|
|
||||||
|
1. Wraps x86 related fields with CONFIG_X86_64.
|
||||||
|
2. Add "cpu_event_notification" for arm64.
|
||||||
|
3. Add "add_cpu_dev" and "del_cpu_dev" for arm64.
|
||||||
|
|
||||||
|
Signed-off-by: xuejun-xj <jiyunxue@linux.alibaba.com>
|
||||||
|
Reviewed-by : Chao Wu <chaowu@linux.alibaba.com>
|
||||||
|
Reviewed-by: Zizheng Bian <zizheng.bian@linux.alibaba.com>
|
||||||
|
Reviewed-by: Baolin Wang <baolin.wang@linux.alibaba.com>
|
||||||
|
---
|
||||||
|
.../upcall_srv/dragonball_device_manager.c | 84 ++++++++++++++++++-
|
||||||
|
1 file changed, 81 insertions(+), 3 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c b/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c
|
||||||
|
index 5a95b2ba63e8..088d38623b8d 100644
|
||||||
|
--- a/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c
|
||||||
|
+++ b/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c
|
||||||
|
@@ -85,15 +85,21 @@ struct devmgr_req {
|
||||||
|
#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU)
|
||||||
|
struct {
|
||||||
|
uint8_t count;
|
||||||
|
+#ifdef CONFIG_X86_64
|
||||||
|
uint8_t apic_ver;
|
||||||
|
uint8_t apic_ids[256];
|
||||||
|
+#endif
|
||||||
|
} cpu_dev_info;
|
||||||
|
#endif
|
||||||
|
} msg_load;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct cpu_dev_reply_info {
|
||||||
|
+#if defined(CONFIG_X86_64)
|
||||||
|
uint32_t apic_index;
|
||||||
|
+#elif defined(CONFIG_ARM64)
|
||||||
|
+ uint32_t cpu_id;
|
||||||
|
+#endif
|
||||||
|
};
|
||||||
|
|
||||||
|
struct devmgr_reply {
|
||||||
|
@@ -190,7 +196,8 @@ static void _fill_msg_header(struct devmgr_msg_header *msg, uint32_t msg_size,
|
||||||
|
msg->msg_flags = msg_flags;
|
||||||
|
}
|
||||||
|
|
||||||
|
-#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64)
|
||||||
|
+#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU)
|
||||||
|
+#if defined(CONFIG_X86_64)
|
||||||
|
static int get_cpu_id(int apic_id)
|
||||||
|
{
|
||||||
|
int i;
|
||||||
|
@@ -219,6 +226,24 @@ static void cpu_event_notification(
|
||||||
|
_fill_msg_header(&rep->msg_header,
|
||||||
|
sizeof(struct cpu_dev_reply_info), action_type, 0);
|
||||||
|
}
|
||||||
|
+#elif defined(CONFIG_ARM64)
|
||||||
|
+/**
|
||||||
|
+ * Return the first failed hotplug index of the cpu_id to dragonball.
|
||||||
|
+ * If hotplug/hotunplug succeeds, it will equals to the expected cpu count.
|
||||||
|
+ */
|
||||||
|
+static void cpu_event_notification(
|
||||||
|
+ uint8_t cpu_id,
|
||||||
|
+ int ret,
|
||||||
|
+ uint32_t action_type,
|
||||||
|
+ struct devmgr_reply *rep)
|
||||||
|
+{
|
||||||
|
+ pr_info("cpu event notification: cpu_id %d\n", cpu_id);
|
||||||
|
+ rep->msg_load.cpu_dev_info.cpu_id = cpu_id;
|
||||||
|
+ rep->ret = ret;
|
||||||
|
+ _fill_msg_header(&rep->msg_header,
|
||||||
|
+ sizeof(struct cpu_dev_reply_info), action_type, 0);
|
||||||
|
+}
|
||||||
|
+#endif
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#if defined(CONFIG_DRAGONBALL_HOTPLUG_VIRTIO_MMIO)
|
||||||
|
@@ -262,7 +287,8 @@ static int del_mmio_dev(struct devmgr_req *req,
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
|
-#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64)
|
||||||
|
+#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU)
|
||||||
|
+#if defined(CONFIG_X86_64)
|
||||||
|
static int add_cpu_upcall(int apic_id, uint8_t apic_ver)
|
||||||
|
{
|
||||||
|
int cpu_id, node_id;
|
||||||
|
@@ -430,6 +456,58 @@ static int del_cpu_dev(struct devmgr_req *req,
|
||||||
|
cpu_event_notification(i, ret, DEL_CPU, rep);
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
+#elif defined(CONFIG_ARM64)
|
||||||
|
+static int add_cpu_dev(struct devmgr_req *req, struct devmgr_reply *rep)
|
||||||
|
+{
|
||||||
|
+ int i, ret = 0;
|
||||||
|
+ unsigned int cpu_id, nr_online_cpus;
|
||||||
|
+ uint8_t count = req->msg_load.cpu_dev_info.count;
|
||||||
|
+
|
||||||
|
+ nr_online_cpus = num_online_cpus();
|
||||||
|
+
|
||||||
|
+ pr_info("Current vcpu number: %d, Add vcpu number: %d\n",
|
||||||
|
+ nr_online_cpus, count);
|
||||||
|
+
|
||||||
|
+ for (i = 0; i < count; ++i) {
|
||||||
|
+ cpu_id = nr_online_cpus + i;
|
||||||
|
+ ret = add_cpu(cpu_id);
|
||||||
|
+ if (ret != 0)
|
||||||
|
+ break;
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ cpu_event_notification(nr_online_cpus + i, ret, ADD_CPU, rep);
|
||||||
|
+ return ret;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+static int del_cpu_dev(struct devmgr_req *req, struct devmgr_reply *rep)
|
||||||
|
+{
|
||||||
|
+ int i, ret = 0;
|
||||||
|
+ unsigned int cpu_id, nr_online_cpus;
|
||||||
|
+ uint8_t count = req->msg_load.cpu_dev_info.count;
|
||||||
|
+
|
||||||
|
+ nr_online_cpus = num_online_cpus();
|
||||||
|
+
|
||||||
|
+ pr_info("Current vcpu number: %d, Delete vcpu number: %d\n",
|
||||||
|
+ nr_online_cpus, count);
|
||||||
|
+
|
||||||
|
+ if (count >= nr_online_cpus) {
|
||||||
|
+ pr_err("cpu del parameter check error: cannot remove all vcpus\n");
|
||||||
|
+ ret = -EINVAL;
|
||||||
|
+ cpu_event_notification(0, ret, DEL_CPU, rep);
|
||||||
|
+ return ret;
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for (i = 0; i < count; ++i) {
|
||||||
|
+ cpu_id = nr_online_cpus - i - 1;
|
||||||
|
+ ret = remove_cpu(cpu_id);
|
||||||
|
+ if (ret != 0)
|
||||||
|
+ break;
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ cpu_event_notification(nr_online_cpus - i, ret, DEL_CPU, rep);
|
||||||
|
+ return ret;
|
||||||
|
+}
|
||||||
|
+#endif
|
||||||
|
#endif
|
||||||
|
|
||||||
|
static struct {
|
||||||
|
@@ -440,7 +518,7 @@ static struct {
|
||||||
|
{ADD_MMIO, add_mmio_dev},
|
||||||
|
{DEL_MMIO, del_mmio_dev},
|
||||||
|
#endif
|
||||||
|
-#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64)
|
||||||
|
+#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU)
|
||||||
|
{ADD_CPU, add_cpu_dev},
|
||||||
|
{DEL_CPU, del_cpu_dev},
|
||||||
|
#endif
|
||||||
|
--
|
||||||
|
2.28.0
|
||||||
|
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
From 6e07ca77fe7b5c15e0e98d9e86294c7dd2553a5a Mon Sep 17 00:00:00 2001
|
||||||
|
Message-Id: <6e07ca77fe7b5c15e0e98d9e86294c7dd2553a5a.1685428663.git.jiyunxue@linux.alibaba.com>
|
||||||
|
In-Reply-To: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com>
|
||||||
|
References: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com>
|
||||||
|
From: xuejun-xj <jiyunxue@linux.alibaba.com>
|
||||||
|
Date: Wed, 10 May 2023 14:51:40 +0800
|
||||||
|
Subject: [PATCH 2/3] msi: control msi irq number activated
|
||||||
|
|
||||||
|
When passthroughing pci device, kernel will initialize and activate
|
||||||
|
(max_cpu_count+1) msi irq. However, in vcpu hotplugging situation,
|
||||||
|
because of vgic, max_cpu_count may be greater than online_cpu_count.
|
||||||
|
Those offline cpus will also be activated by kernel, which cause failure
|
||||||
|
of passthroughing pci device.
|
||||||
|
|
||||||
|
To solve this problem, this patch add a function
|
||||||
|
"check_affinity_mask_online" to check if msi_desc->affinity contains
|
||||||
|
online cpus. If current cpu is offline, it will continue the for loop to
|
||||||
|
skip activating related irq.
|
||||||
|
|
||||||
|
Signed-off-by: xuejun-xj <jiyunxue@linux.alibaba.com>
|
||||||
|
Reviewed-by: Shuo Tan <shuo.tan@linux.alibaba.com>
|
||||||
|
Reviewed-by: Baolin Wang <baolin.wang@linux.alibaba.com>
|
||||||
|
---
|
||||||
|
kernel/irq/msi.c | 20 ++++++++++++++++++++
|
||||||
|
1 file changed, 20 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c
|
||||||
|
index d924676c8781..d60a3fc654e6 100644
|
||||||
|
--- a/kernel/irq/msi.c
|
||||||
|
+++ b/kernel/irq/msi.c
|
||||||
|
@@ -395,6 +395,23 @@ static bool msi_check_reservation_mode(struct irq_domain *domain,
|
||||||
|
return desc->msi_attrib.is_msix || desc->msi_attrib.maskbit;
|
||||||
|
}
|
||||||
|
|
||||||
|
+/* This function is used for check whether the cpu affinity belongs to the
|
||||||
|
+ * online cpus. When we passthrough the nvme devices, the kernel will allocate
|
||||||
|
+ * maxcpus+1 MSI irqs and then activate them. In vcpu hotplug situations, it
|
||||||
|
+ * may happen that kernel activates the offline cpus when bootcpus < maxcpus.
|
||||||
|
+ * To avoid this conflict, this function check the affinities.
|
||||||
|
+ */
|
||||||
|
+static inline bool check_affinity_mask_online(struct irq_affinity_desc *affinity)
|
||||||
|
+{
|
||||||
|
+ int cpu;
|
||||||
|
+
|
||||||
|
+ for_each_cpu(cpu, &affinity->mask)
|
||||||
|
+ if (cpu_online(cpu))
|
||||||
|
+ return true;
|
||||||
|
+
|
||||||
|
+ return false;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
|
||||||
|
int nvec)
|
||||||
|
{
|
||||||
|
@@ -445,6 +462,9 @@ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
|
||||||
|
goto skip_activate;
|
||||||
|
|
||||||
|
for_each_msi_vector(desc, i, dev) {
|
||||||
|
+ if (desc->affinity
|
||||||
|
+ && !check_affinity_mask_online(desc->affinity))
|
||||||
|
+ continue;
|
||||||
|
if (desc->irq == i) {
|
||||||
|
virq = desc->irq;
|
||||||
|
dev_dbg(dev, "irq [%d-%d] for MSI\n",
|
||||||
|
--
|
||||||
|
2.28.0
|
||||||
|
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
From a05086142be13d43c7fc92500bcb870a2f37e485 Mon Sep 17 00:00:00 2001
|
||||||
|
Message-Id: <a05086142be13d43c7fc92500bcb870a2f37e485.1685428663.git.jiyunxue@linux.alibaba.com>
|
||||||
|
In-Reply-To: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com>
|
||||||
|
References: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com>
|
||||||
|
From: xuejun-xj <jiyunxue@linux.alibaba.com>
|
||||||
|
Date: Tue, 23 May 2023 09:43:02 +0800
|
||||||
|
Subject: [PATCH 3/3] smp: update bringup_nonboot_cpus parameters
|
||||||
|
|
||||||
|
On aarch64, kvm doesn't allow vmm to call KVM_CREATE_VCPU ioctls after
|
||||||
|
vm has already started, which is caused by vgic_initialized check in
|
||||||
|
kvm_arch_vcpu_precreate() function. Therefore, to support vcpu hotplug
|
||||||
|
feature on aarch64, all the vcpus should be created and configured ready
|
||||||
|
for start at booting procedure.
|
||||||
|
|
||||||
|
To solve the problem, dragonball will add a property in each cpu node,
|
||||||
|
called "boot-onlined". This property indicates whether this cpu should
|
||||||
|
be onlined at first boot. It has two values: 0 and 1. 0 means offline,
|
||||||
|
while 1 means online.
|
||||||
|
|
||||||
|
This commit also add a helper function called "of_get_cpu_boot_onlined",
|
||||||
|
which parse the cpu node and get the value of boot-onlined property.
|
||||||
|
Then update the global variable "boot_onlined_cpu".
|
||||||
|
|
||||||
|
When kernel calling smp_init(), bringup_nonboot_cpus will start all the
|
||||||
|
other cpus except cpu0. The activated cpu number equals setup_max_cpus.
|
||||||
|
In vcpu hotplug scenario, vmm will create all the vcpufd before vm is
|
||||||
|
initialized, while activating only a few vcpus at first boot. The
|
||||||
|
setup_max_cpus variable will be initialized as all vcpu count. This
|
||||||
|
cause that the other cpus cannot find enough cpu threads, and they will
|
||||||
|
wait for 5 seconds each cpu.
|
||||||
|
|
||||||
|
Therefore, we use boot_onlined_cpu instead of setup_max_cpus to give
|
||||||
|
"bringup_nonboot_cpus" correct cpu number it needs.
|
||||||
|
|
||||||
|
Signed-off-by: xuejun-xj <jiyunxue@linux.alibaba.com>
|
||||||
|
---
|
||||||
|
.../devicetree/bindings/arm/cpus.yaml | 11 +++++++++
|
||||||
|
arch/arm64/kernel/smp.c | 24 +++++++++++++++++++
|
||||||
|
kernel/smp.c | 10 +++++++-
|
||||||
|
3 files changed, 44 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/Documentation/devicetree/bindings/arm/cpus.yaml b/Documentation/devicetree/bindings/arm/cpus.yaml
|
||||||
|
index 14cd727d3c4b..691bb352d842 100644
|
||||||
|
--- a/Documentation/devicetree/bindings/arm/cpus.yaml
|
||||||
|
+++ b/Documentation/devicetree/bindings/arm/cpus.yaml
|
||||||
|
@@ -316,6 +316,17 @@ properties:
|
||||||
|
formed by encoding the target CPU id into the low bits of the
|
||||||
|
physical start address it should jump to.
|
||||||
|
|
||||||
|
+ boot-onlined:
|
||||||
|
+ $ref: '/schemas/types.yaml#/definitions/uint32'
|
||||||
|
+ description: |
|
||||||
|
+ The boot-onlined property is an optional u32 value that indicates
|
||||||
|
+ whether the cpu device should be activated at first boot. This is
|
||||||
|
+ useful in vcpu hotplug scenario to pass correct value of activated
|
||||||
|
+ cpu number.
|
||||||
|
+
|
||||||
|
+ This property has two values: 0 and 1. 1 means the cpu should be
|
||||||
|
+ activated while 0 means it shouldn't.
|
||||||
|
+
|
||||||
|
if:
|
||||||
|
# If the enable-method property contains one of those values
|
||||||
|
properties:
|
||||||
|
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
|
||||||
|
index 18e9727d3f64..5db8041929a6 100644
|
||||||
|
--- a/arch/arm64/kernel/smp.c
|
||||||
|
+++ b/arch/arm64/kernel/smp.c
|
||||||
|
@@ -464,6 +464,27 @@ void __init smp_prepare_boot_cpu(void)
|
||||||
|
init_gic_priority_masking();
|
||||||
|
}
|
||||||
|
|
||||||
|
+#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_ARM64)
|
||||||
|
+extern unsigned int boot_onlined_cpu;
|
||||||
|
+static void __init of_get_cpu_boot_onlined(struct device_node *dn)
|
||||||
|
+{
|
||||||
|
+ unsigned int boot_onlined;
|
||||||
|
+ int r;
|
||||||
|
+
|
||||||
|
+ r = of_property_read_u32(dn, "boot-onlined", &boot_onlined);
|
||||||
|
+ if (r) {
|
||||||
|
+ pr_err("%pOF: missing boot-onlined property\n", dn);
|
||||||
|
+ return;
|
||||||
|
+ }
|
||||||
|
+ /*
|
||||||
|
+ * Property boot-onlined has two values: 0 and 1.
|
||||||
|
+ * 0 means offline, and 1 means online.
|
||||||
|
+ * Here just count the number of boot_onlined_cpu.
|
||||||
|
+ */
|
||||||
|
+ boot_onlined_cpu += boot_onlined;
|
||||||
|
+}
|
||||||
|
+#endif
|
||||||
|
+
|
||||||
|
static u64 __init of_get_cpu_mpidr(struct device_node *dn)
|
||||||
|
{
|
||||||
|
const __be32 *cell;
|
||||||
|
@@ -654,6 +675,9 @@ static void __init of_parse_and_init_cpus(void)
|
||||||
|
struct device_node *dn;
|
||||||
|
|
||||||
|
for_each_of_cpu_node(dn) {
|
||||||
|
+#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_ARM64)
|
||||||
|
+ of_get_cpu_boot_onlined(dn);
|
||||||
|
+#endif
|
||||||
|
u64 hwid = of_get_cpu_mpidr(dn);
|
||||||
|
|
||||||
|
if (hwid == INVALID_HWID)
|
||||||
|
diff --git a/kernel/smp.c b/kernel/smp.c
|
||||||
|
index 25240fb2df94..567615b9a008 100644
|
||||||
|
--- a/kernel/smp.c
|
||||||
|
+++ b/kernel/smp.c
|
||||||
|
@@ -801,17 +801,25 @@ void __init setup_nr_cpu_ids(void)
|
||||||
|
nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
+/* Setup number of CPUs to activate */
|
||||||
|
+unsigned int boot_onlined_cpu = 0;
|
||||||
|
+
|
||||||
|
/* Called by boot processor to activate the rest. */
|
||||||
|
void __init smp_init(void)
|
||||||
|
{
|
||||||
|
int num_nodes, num_cpus;
|
||||||
|
+ int num_onlined_cpu = setup_max_cpus;
|
||||||
|
|
||||||
|
idle_threads_init();
|
||||||
|
cpuhp_threads_init();
|
||||||
|
|
||||||
|
pr_info("Bringing up secondary CPUs ...\n");
|
||||||
|
|
||||||
|
- bringup_nonboot_cpus(setup_max_cpus);
|
||||||
|
+#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_ARM64)
|
||||||
|
+ if (boot_onlined_cpu != 0)
|
||||||
|
+ num_onlined_cpu = boot_onlined_cpu;
|
||||||
|
+#endif
|
||||||
|
+ bringup_nonboot_cpus(num_onlined_cpu);
|
||||||
|
|
||||||
|
num_nodes = num_online_nodes();
|
||||||
|
num_cpus = num_online_cpus();
|
||||||
|
--
|
||||||
|
2.28.0
|
||||||
|
|
||||||
Reference in New Issue
Block a user