github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-04-30 04:34:27 +00:00
Code Issues Projects Releases Wiki Activity

Dragonball: introduce upcall

Browse Source 
Upcall is a direct communication tool between VMM and guest developed
upon vsock. The server side of the upcall is a driver in guest kernel
(kernel patches are needed for this feature) and it'll start to serve
the requests after the kernel starts. And the client side is in
Dragonball VMM , it'll be a thread that communicates with vsock through
uds.

We want to keep the lightweight of the VM through the implementation of
the upcall, through which we could achieve vCPU hotplug, virtio-mmio
hotplug without implementing complex and heavy virtualization features
such as ACPI virtualization.

fixes: #5642

Signed-off-by: Chao Wu <chaowu@linux.alibaba.com>
This commit is contained in:
Chao Wu 2022-11-21 22:49:35 +08:00
parent 7fabfb2cf0
commit f5c34ed088
4 changed files with 1566 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

520
tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0001-upcall-establish-upcall-server.patch Normal file
View File

@ -0,0 +1,520 @@
From 691186a091ecfc1777531a61594b88394d384cff Mon Sep 17 00:00:00 2001
From: Chao Wu <chaowu@linux.alibaba.com>
Date: Wed, 9 Nov 2022 11:38:36 +0800
Subject: [PATCH 1/4] upcall: establish upcall server
Upcall is a direct communication tool between hypervisor and guest. This
patch introduces the server side in the upcall system.
At the start of the upcall server, A kthread `db-vsock-srv` will be
created. In this kthread, a vsock listener is established upon specific
port(currently that port is 0xDB, DB refers to Dragonball). After socket
is created, it will start accepting the connection from the client side.
If the connection is established, upcall server will try to get cmd from
the client and that cmd could determine which upcall service will handle
the request from the client.
Besides, different service needs to be registered into upcall server so
that it could handle the request from the client. There is a
`register_db_vsock_service` in this commit provided for every service to
register service into service_entry list during initialization and we will
introduce device manager service in the following commits.
Signed-off-by: Liu Jiang <gerry@linux.alibaba.com>
Signed-off-by: Zizheng Bian <zizheng.bian@linux.alibaba.com>
Signed-off-by: Chao Wu <chaowu@linux.alibaba.com>
Signed-off-by: WangYu <WangYu@linux.alibaba.com>
Signed-off-by: Xingjun Liu <xingjun.liu@linux.alibaba.com>
---
drivers/misc/Kconfig | 1 +
drivers/misc/Makefile | 1 +
drivers/misc/dragonball/Kconfig | 21 ++
drivers/misc/dragonball/Makefile | 6 +
drivers/misc/dragonball/upcall_srv/Kconfig | 14 +
drivers/misc/dragonball/upcall_srv/Makefile | 13 +
.../upcall_srv/dragonball_upcall_srv.c | 323 ++++++++++++++++++
include/dragonball/upcall_srv.h | 42 +++
8 files changed, 421 insertions(+)
create mode 100644 drivers/misc/dragonball/Kconfig
create mode 100644 drivers/misc/dragonball/Makefile
create mode 100644 drivers/misc/dragonball/upcall_srv/Kconfig
create mode 100644 drivers/misc/dragonball/upcall_srv/Makefile
create mode 100644 drivers/misc/dragonball/upcall_srv/dragonball_upcall_srv.c
create mode 100644 include/dragonball/upcall_srv.h
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
index fafa8b0d8099..e05a14f77510 100644
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -481,4 +481,5 @@ source "drivers/misc/ocxl/Kconfig"
source "drivers/misc/cardreader/Kconfig"
source "drivers/misc/habanalabs/Kconfig"
source "drivers/misc/uacce/Kconfig"
+source "drivers/misc/dragonball/Kconfig"
endmenu
diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile
index d23231e73330..e0a1af9c05b2 100644
--- a/drivers/misc/Makefile
+++ b/drivers/misc/Makefile
@@ -57,3 +57,4 @@ obj-$(CONFIG_HABANA_AI) += habanalabs/
obj-$(CONFIG_UACCE) += uacce/
obj-$(CONFIG_XILINX_SDFEC) += xilinx_sdfec.o
obj-$(CONFIG_HISI_HIKEY_USB) += hisi_hikey_usb.o
+obj-$(CONFIG_DRAGONBALL_DRIVERS) += dragonball/
diff --git a/drivers/misc/dragonball/Kconfig b/drivers/misc/dragonball/Kconfig
new file mode 100644
index 000000000000..f81be3721908
--- /dev/null
+++ b/drivers/misc/dragonball/Kconfig
@@ -0,0 +1,21 @@
+#
+# Alibaba Dragonball Secure Container Runtime Drivers
+#
+
+menuconfig DRAGONBALL_DRIVERS
+ bool "Alibaba Dragonball Secure Container Runtime Drivers"
+ depends on X86_64 || ARM64
+ default n
+ help
+ Alibaba Dragonball is a secure container runtime with an embedded micro-vmm
+ to securely isolate container workloads.
+
+ Say Y here to get to see options for various misc drivers to support the
+ Alibaba Dragonball secure container runtime. This option alone does not
+ add any kernel code.
+
+ If unsure, say N.
+
+if DRAGONBALL_DRIVERS
+source "drivers/misc/dragonball/upcall_srv/Kconfig"
+endif # DRAGONBALL_DRIVERS
diff --git a/drivers/misc/dragonball/Makefile b/drivers/misc/dragonball/Makefile
new file mode 100644
index 000000000000..b7bd86d73ade
--- /dev/null
+++ b/drivers/misc/dragonball/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for Dragonball misc drivers
+#
+
+obj-$(CONFIG_DRAGONBALL_UPCALL_SRV) += upcall_srv/
diff --git a/drivers/misc/dragonball/upcall_srv/Kconfig b/drivers/misc/dragonball/upcall_srv/Kconfig
new file mode 100644
index 000000000000..b00bf1f8637d
--- /dev/null
+++ b/drivers/misc/dragonball/upcall_srv/Kconfig
@@ -0,0 +1,14 @@
+#
+# Alibaba Dragonball Secure Container Runtime Drivers for vsock
+#
+
+config DRAGONBALL_UPCALL_SRV
+ bool "Dragonball in-kernel Virtual Sockets Server"
+ depends on VIRTIO_VSOCKETS
+ default y
+ help
+ This configure implements an in-kernel vsock server to dispatch Dragonball
+ requests to registered service handlers, based on the reliable Virtual
+ Sockets communication channels between guest and host/vmm.
+
+ If unsure, say N.
diff --git a/drivers/misc/dragonball/upcall_srv/Makefile b/drivers/misc/dragonball/upcall_srv/Makefile
new file mode 100644
index 000000000000..4102e6c7edef
--- /dev/null
+++ b/drivers/misc/dragonball/upcall_srv/Makefile
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for the in-kernel vsock server.
+#
+# Copyright (C) 2022 Alibaba Cloud, Inc
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version
+# 2 of the License, or (at your option) any later version.
+#
+
+obj-$(CONFIG_DRAGONBALL_UPCALL_SRV) += dragonball_upcall_srv.o
diff --git a/drivers/misc/dragonball/upcall_srv/dragonball_upcall_srv.c b/drivers/misc/dragonball/upcall_srv/dragonball_upcall_srv.c
new file mode 100644
index 000000000000..1670bd8597f0
--- /dev/null
+++ b/drivers/misc/dragonball/upcall_srv/dragonball_upcall_srv.c
@@ -0,0 +1,323 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * drivers/misc/dragonball/upcall_srv/dragonball_upcall_srv.c
+ * Dragonball upcall server
+ *
+ * Copyright (C) 2022 Alibaba Cloud, Inc
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ */
+
+#define pr_fmt(fmt) "db-upcall-srv: " fmt
+
+#include <linux/kthread.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/net.h>
+#include <linux/vm_sockets.h>
+#include <net/net_namespace.h>
+#include <net/sock.h>
+#include <dragonball/upcall_srv.h>
+
+struct db_conn_info {
+ struct work_struct work;
+ struct socket *sock;
+};
+
+struct db_service_entry {
+ char cmd;
+ db_vsock_svc_handler_t handler;
+ struct list_head list;
+};
+
+/* Protects registered command. */
+static DEFINE_RWLOCK(db_service_lock);
+static LIST_HEAD(db_service_list);
+
+static struct task_struct *db_service_task;
+static unsigned int db_server_port = DB_SERVER_PORT;
+
+struct socket *db_create_vsock_listener(unsigned int port)
+{
+ struct socket *sock;
+ int ret = 0;
+
+ union {
+ struct sockaddr sa;
+ struct sockaddr_vm svm;
+ } addr = {
+ .svm = {
+ .svm_family = AF_VSOCK,
+ .svm_port = port,
+ .svm_cid = VMADDR_CID_ANY,
+ }
+ };
+
+ ret = sock_create_kern(&init_net, AF_VSOCK, SOCK_STREAM, 0, &sock);
+ if (ret) {
+ pr_err("Server vsock create failed, err: %d\n", ret);
+ return ERR_PTR(ret);
+ }
+
+ ret = sock->ops->bind(sock, &addr.sa, sizeof(addr.svm));
+ if (ret) {
+ pr_err("Server vsock bind failed, err: %d\n", ret);
+ goto err;
+ }
+ ret = sock->ops->listen(sock, 10);
+ if (ret < 0) {
+ pr_err("Server vsock listen error: %d\n", ret);
+ goto err;
+ }
+
+ return sock;
+err:
+ sock_release(sock);
+ return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(db_create_vsock_listener);
+
+int db_vsock_sendmsg(struct socket *sock, char *buf, size_t len)
+{
+ struct kvec vec;
+ struct msghdr msgh;
+
+ vec.iov_base = buf;
+ vec.iov_len = len;
+ memset(&msgh, 0, sizeof(msgh));
+
+ return kernel_sendmsg(sock, &msgh, &vec, 1, len);
+}
+EXPORT_SYMBOL_GPL(db_vsock_sendmsg);
+
+int db_vsock_recvmsg(struct socket *sock, char *buf, size_t len, int flags)
+{
+ struct kvec vec;
+ struct msghdr msgh;
+
+ memset(&vec, 0, sizeof(vec));
+ memset(&msgh, 0, sizeof(msgh));
+ vec.iov_base = buf;
+ vec.iov_len = len;
+
+ return kernel_recvmsg(sock, &msgh, &vec, 1, len, flags);
+}
+EXPORT_SYMBOL_GPL(db_vsock_recvmsg);
+
+static int db_vsock_recvcmd(struct socket *cli_socket, char *cmd)
+{
+ int ret;
+ char rcv;
+ long timeout;
+ struct kvec vec;
+ struct msghdr msg;
+
+ memset(&vec, 0, sizeof(vec));
+ memset(&msg, 0, sizeof(msg));
+ vec.iov_base = &rcv;
+ vec.iov_len = 1;
+
+ timeout = cli_socket->sk->sk_rcvtimeo;
+ cli_socket->sk->sk_rcvtimeo = DB_INIT_TIMEOUT * HZ;
+ ret = kernel_recvmsg(cli_socket, &msg, &vec, 1, 1, 0);
+ cli_socket->sk->sk_rcvtimeo = timeout;
+ *cmd = rcv;
+
+ return ret;
+}
+
+/*
+ * The workqueue handler for vsock work_struct.
+ *
+ * Each worker-pool bound to an actual CPU implements concurrency management
+ * by hooking into the scheduler. The worker-pool is notified whenever an
+ * active worker wakes up or sleeps and keeps track of the number of the
+ * currently runnable workers. Generally, work items are not expected to hog
+ * a CPU and consume many cycles. That means maintaining just enough concurrency
+ * to prevent work processing from stalling should be optimal.
+ *
+ * So it's OK to sleep in a workqueue handler, it won't cause too many worker
+ * threads.
+ */
+static void db_conn_service(struct work_struct *work)
+{
+ struct db_conn_info *conn_info =
+ container_of(work, struct db_conn_info, work);
+ struct db_service_entry *service_entry;
+ int len, ret = -1;
+ char cmd;
+
+ len = db_vsock_recvcmd(conn_info->sock, &cmd);
+ if (len <= 0)
+ goto recv_failed;
+
+ read_lock(&db_service_lock);
+ list_for_each_entry(service_entry, &db_service_list, list) {
+ if (cmd == service_entry->cmd) {
+ ret = service_entry->handler(conn_info->sock);
+ break;
+ }
+ }
+ read_unlock(&db_service_lock);
+
+recv_failed:
+ if (ret) {
+ sock_release(conn_info->sock);
+ pr_info("Client connection closed, error code: %d\n", ret);
+ }
+ kfree(conn_info);
+}
+
+static int db_create_cli_conn(struct socket *sock)
+{
+ struct db_conn_info *conn;
+
+ conn = kmalloc(sizeof(*conn), GFP_KERNEL);
+ if (!conn)
+ return -ENOMEM;
+
+ conn->sock = sock;
+ INIT_WORK(&conn->work, db_conn_service);
+ schedule_work(&conn->work);
+
+ return 0;
+}
+
+static int db_vsock_server(void *data)
+{
+ struct socket *sock;
+ int err;
+
+ sock = db_create_vsock_listener(db_server_port);
+ if (IS_ERR(sock)) {
+ err = PTR_ERR(sock);
+ pr_err("Init server err: %d\n", err);
+ return err;
+ }
+
+ while (!kthread_should_stop()) {
+ struct socket *conn;
+
+ conn = sock_alloc();
+ if (!conn)
+ return -ENOMEM;
+
+ conn->type = sock->type;
+ conn->ops = sock->ops;
+
+ /* 0:propotal 1:kernel */
+ err = sock->ops->accept(sock, conn, 0, 1);
+ if (err < 0) {
+ pr_err("Server accept err: %d\n", err);
+ sock_release(conn);
+ continue;
+ }
+
+ err = db_create_cli_conn(conn);
+ if (err)
+ pr_err("Create client connetion err: %d\n", err);
+ }
+
+ return 0;
+}
+
+static int db_create_service(void)
+{
+ struct task_struct *service;
+ int rc = 0;
+
+ service = kthread_create(db_vsock_server, NULL, "db-vsock-srv");
+ if (IS_ERR(service)) {
+ rc = PTR_ERR(service);
+ pr_err("Server task create failed, err: %d\n", rc);
+ } else {
+ db_service_task = service;
+ wake_up_process(service);
+ }
+ return rc;
+}
+
+static int db_vsock_srv_cmdline_set(const char *device,
+ const struct kernel_param *kp)
+{
+ unsigned int port = 0;
+ int processed, consumed = 0;
+
+ /* Get "@<port>" */
+ processed = sscanf(device, "@%u%n", &port, &consumed);
+ if (processed < 1 || device[consumed] || port == 0 || port > 1024) {
+ pr_err("Using @<port> format and port range (0, 1024].\n");
+ return -EINVAL;
+ }
+
+ db_server_port = port;
+ return 0;
+}
+
+static const struct kernel_param_ops db_vsock_srv_cmdline_param_ops = {
+ .set = db_vsock_srv_cmdline_set,
+};
+
+device_param_cb(port, &db_vsock_srv_cmdline_param_ops, NULL, 0400);
+
+int register_db_vsock_service(const char cmd, db_vsock_svc_handler_t handler)
+{
+ int rc = -EEXIST;
+ struct db_service_entry *service_entry;
+
+ write_lock(&db_service_lock);
+ list_for_each_entry(service_entry, &db_service_list, list) {
+ if (cmd == service_entry->cmd) {
+ rc = -EEXIST;
+ goto out;
+ }
+ }
+
+ service_entry = kzalloc(sizeof(*service_entry), GFP_KERNEL);
+ if (!service_entry) {
+ rc = -ENOMEM;
+ goto out;
+ }
+ service_entry->cmd = cmd;
+ service_entry->handler = handler;
+ list_add_tail(&service_entry->list, &db_service_list);
+ rc = 0;
+out:
+ write_unlock(&db_service_lock);
+ return rc;
+}
+EXPORT_SYMBOL_GPL(register_db_vsock_service);
+
+int unregister_db_vsock_service(const char cmd)
+{
+ int rc = -EEXIST;
+ struct db_service_entry *service_entry, *n;
+
+ write_lock(&db_service_lock);
+ list_for_each_entry_safe(service_entry, n, &db_service_list, list) {
+ if (cmd == service_entry->cmd) {
+ list_del(&service_entry->list);
+ rc = 0;
+ break;
+ }
+ }
+ write_unlock(&db_service_lock);
+
+ return rc;
+}
+EXPORT_SYMBOL_GPL(unregister_db_vsock_service);
+
+static int __init db_vsock_srv_init(void)
+{
+ return db_create_service();
+}
+
+late_initcall(db_vsock_srv_init);
+
+MODULE_AUTHOR("Alibaba, Inc.");
+MODULE_DESCRIPTION("Dragonball vsock server");
+MODULE_LICENSE("GPL v2");
diff --git a/include/dragonball/upcall_srv.h b/include/dragonball/upcall_srv.h
new file mode 100644
index 000000000000..1c733982cc30
--- /dev/null
+++ b/include/dragonball/upcall_srv.h
@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * db_upcall_srv.h Virtual Sockets Server for Dragonball
+ *
+ * Copyright (C) 2022 Alibaba Cloud, Inc
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#ifndef _DB_UPCALL_SRV_H
+#define _DB_UPCALL_SRV_H
+
+#include <linux/workqueue.h>
+#include <linux/net.h>
+
+/* Vsock port to listen for incoming connections. */
+#define DB_SERVER_PORT 0xDB
+#define DB_RECVBUF_SIZE 0x400
+#define DB_INIT_TIMEOUT 10
+
+/*
+ * Vsock service handler to handle new incoming connections.
+ *
+ * Return:
+ * 0: on success and the callback takes ownership of the sock.
+ * !0: on failure and the callback should keep the sock as is.
+ */
+typedef int (*db_vsock_svc_handler_t) (struct socket *sock);
+
+extern int register_db_vsock_service(const char cmd,
+ db_vsock_svc_handler_t handler);
+extern int unregister_db_vsock_service(const char cmd);
+
+extern struct socket *db_create_vsock_listener(unsigned int port);
+extern int db_vsock_sendmsg(struct socket *sock, char *buf, size_t len);
+extern int db_vsock_recvmsg(struct socket *sock, char *buf, size_t len,
+ int flags);
+
+#endif /* _DB_UPCALL_SRV_H */
--
2.19.1.6.gb485710b

330
tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0002-upcall-introduce-device-manager-upcall-service.patch Normal file
View File

@ -0,0 +1,330 @@
From 66e67cc3ebbebc9d138fff084c487d6b9d21f60c Mon Sep 17 00:00:00 2001
From: Chao Wu <chaowu@linux.alibaba.com>
Date: Mon, 21 Nov 2022 19:19:26 +0800
Subject: [PATCH 2/4] upcall: introduce device manager upcall service
Different services are registered into upcall server to handle the
request from the client side. This commit introduces devic manager
upcall service and when new message gets into upcall server, cmd `d` is
used for identifying the device manager service.
After a request is sent to device manager service, db_devmgr_handler
will start handle the request. A kthread `db_devmgr_server` will be
created and it will send CONNECT message to the client side to notify
the client start sending message for device management operations.
`db_devmgr_process` will be used for determining which device operations
will be triggered through msg_type. `get_action` will find out the
action for dealing with the operation and `action` fn will execute the
actual device management operation in the device manager service.
Signed-off-by: Liu Jiang <gerry@linux.alibaba.com>
Signed-off-by: Zizheng Bian <zizheng.bian@linux.alibaba.com>
Signed-off-by: Chao Wu <chaowu@linux.alibaba.com>
Signed-off-by: WangYu <WangYu@linux.alibaba.com>
Signed-off-by: Xingjun Liu <xingjun.liu@linux.alibaba.com>
---
drivers/misc/dragonball/upcall_srv/Kconfig | 12 +
drivers/misc/dragonball/upcall_srv/Makefile | 1 +
.../upcall_srv/dragonball_device_manager.c | 235 ++++++++++++++++++
include/dragonball/device_manager.h | 18 ++
4 files changed, 266 insertions(+)
create mode 100644 drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c
create mode 100644 include/dragonball/device_manager.h
diff --git a/drivers/misc/dragonball/upcall_srv/Kconfig b/drivers/misc/dragonball/upcall_srv/Kconfig
index b00bf1f8637d..6554a9741c00 100644
--- a/drivers/misc/dragonball/upcall_srv/Kconfig
+++ b/drivers/misc/dragonball/upcall_srv/Kconfig
@@ -12,3 +12,15 @@ config DRAGONBALL_UPCALL_SRV
Sockets communication channels between guest and host/vmm.
If unsure, say N.
+
+config DRAGONBALL_DEVICE_MANAGER
+ bool "Vsock Service to Handle Dragonball Device Management Requests"
+ depends on DRAGONBALL_UPCALL_SRV
+ depends on VIRTIO_VSOCKETS
+ default y
+ help
+ This configure implements a vsock service to handle Dragonball device
+ management requests, such as getting device information, hot-plugging
+ devices etc.
+
+ If unsure, say N.
diff --git a/drivers/misc/dragonball/upcall_srv/Makefile b/drivers/misc/dragonball/upcall_srv/Makefile
index 4102e6c7edef..409c0c11e2e6 100644
--- a/drivers/misc/dragonball/upcall_srv/Makefile
+++ b/drivers/misc/dragonball/upcall_srv/Makefile
@@ -11,3 +11,4 @@
#
obj-$(CONFIG_DRAGONBALL_UPCALL_SRV) += dragonball_upcall_srv.o
+obj-$(CONFIG_DRAGONBALL_DEVICE_MANAGER) += dragonball_device_manager.o
diff --git a/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c b/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c
new file mode 100644
index 000000000000..ebcb6ef74285
--- /dev/null
+++ b/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c
@@ -0,0 +1,235 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * drivers/misc/dragonball/vsock_srv/dragonball_device_manager.c
+ * vsock service for device management.
+ *
+ * Copyright (C) 2022 Alibaba Cloud, Inc
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ */
+
+#define pr_fmt(fmt) "db-dev-mgr: " fmt
+
+#include <linux/kthread.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/virtio_mmio.h>
+#include <linux/cpu.h>
+#include <linux/cpumask.h>
+#include <linux/cpuhotplug.h>
+#include <asm/cpu.h>
+#include <dragonball/upcall_srv.h>
+#include <dragonball/device_manager.h>
+#ifdef CONFIG_ARM64
+#include <linux/irqdomain.h>
+#include <linux/irq.h>
+#endif
+#include <linux/percpu.h>
+#include <linux/device.h>
+#include <asm/numa.h>
+
+/*
+ * Following designs are adopted to simplify implementation:
+ * 1) fix size messages with padding to ease receiving logic.
+ * 2) binary encoding instead of string encoding because it's on the same host.
+ * 3) synchronous communication in ping-pong mode, one in-fly request at most.
+ * 4) do not support module unloading
+ */
+
+/* These definitions are synchronized with dragonball */
+#define DEV_MGR_MSG_SIZE 0x400
+#define DEVMGR_CMD_BYTE 'd'
+#define DEVMGR_MAGIC_VERSION 0x444D0100 /* 'DM' + Version 1.0 */
+#define SHARED_IRQ_NO 5
+
+/* Type of request and reply messages. */
+enum devmgr_msg_type {
+ CONNECT = 0x00000000,
+ ADD_CPU = 0x00000001,
+ DEL_CPU = 0x00000002,
+ ADD_MEM = 0x00000003,
+ DEL_MEM = 0x00000004,
+ ADD_MMIO = 0x00000005,
+ DEL_MMIO = 0x00000006,
+ ADD_PCI = 0x00000007,
+ DEL_PCI = 0x00000008,
+};
+
+struct devmgr_msg_header {
+ /* magic version for identifying upcall */
+ uint32_t magic_version;
+ /* size of the upcall message */
+ uint32_t msg_size;
+ /* type for the message to identify its usage */
+ uint32_t msg_type;
+ /* flag for extra information */
+ uint32_t msg_flags;
+};
+
+struct devmgr_req {
+ struct devmgr_msg_header msg_header;
+ union {
+ char pad[DEV_MGR_MSG_SIZE - sizeof(struct devmgr_msg_header)];
+ } msg_load;
+};
+
+struct devmgr_reply {
+ struct devmgr_msg_header msg_header;
+ /*
+ * if ret is 0, it means the operation is successful.
+ * if ret is not 0, return value will be error code.
+ */
+ int32_t ret;
+ union {
+ char pad[DEV_MGR_MSG_SIZE - sizeof(struct devmgr_msg_header) - sizeof(int32_t)];
+ } msg_load;
+};
+
+struct task_res {
+ struct task_struct *task;
+ struct socket *sock;
+ struct devmgr_req req;
+ struct devmgr_reply reply;
+};
+
+typedef int (*action_route_t) (struct devmgr_req *req,
+ struct devmgr_reply *rep);
+
+static void _fill_msg_header(struct devmgr_msg_header *msg, uint32_t msg_size,
+ uint32_t msg_type, uint32_t msg_flags)
+{
+ msg->magic_version = DEVMGR_MAGIC_VERSION;
+ msg->msg_size = msg_size;
+ msg->msg_type = msg_type;
+ msg->msg_flags = msg_flags;
+}
+
+static struct {
+ enum devmgr_msg_type cmd;
+ action_route_t fn;
+} opt_map[] = {
+};
+
+static action_route_t get_action(struct devmgr_req *req)
+{
+ int i;
+ action_route_t action = NULL;
+ int size_opt = ARRAY_SIZE(opt_map);
+
+ for (i = 0; i < size_opt; i++) {
+ if (opt_map[i].cmd == req->msg_header.msg_type) {
+ action = opt_map[i].fn;
+ break;
+ }
+ }
+ return action;
+}
+
+static void db_devmgr_process(struct devmgr_req *req,
+ struct devmgr_reply *rep)
+{
+ int err;
+ action_route_t action;
+ struct devmgr_msg_header *req_mh = &req->msg_header;
+ struct devmgr_msg_header *rep_mh = &rep->msg_header;
+
+ if (req_mh->magic_version != DEVMGR_MAGIC_VERSION) {
+ _fill_msg_header(rep_mh, 0, req->msg_header.msg_type, 0);
+ return;
+ }
+
+ action = get_action(req);
+ if (action == NULL) {
+ pr_err("db_devmgr_process : Not found valid command");
+ rep->ret = -1;
+ _fill_msg_header(rep_mh, 0, req->msg_header.msg_type, 0);
+ return;
+ }
+
+ err = action(req, rep);
+ if (err) {
+ pr_err("db_devmgr_process : Command run failed, err: %d", err);
+ rep->ret = err;
+ _fill_msg_header(rep_mh, 0, req->msg_header.msg_type, 0);
+ return;
+ }
+}
+
+static int db_devmgr_server(void *data)
+{
+ struct task_res *res = (struct task_res *)data;
+ struct devmgr_msg_header *rep_mh = &res->reply.msg_header;
+ int len;
+
+ _fill_msg_header(rep_mh, 0, CONNECT, 0);
+ len = db_vsock_sendmsg(res->sock, (char *)&res->reply, DEV_MGR_MSG_SIZE);
+ if (len <= 0) {
+ pr_err("db_devmgr_server : Server send message failed, err: %d", len);
+ sock_release(res->sock);
+ kfree(res);
+ return len;
+ }
+
+ while (!kthread_should_stop()) {
+ len = db_vsock_recvmsg(res->sock, (char *)&res->req,
+ DEV_MGR_MSG_SIZE, 0);
+ if (len <= 0)
+ break;
+
+ /* The result(OK or Error) will fill into res->reply field */
+ db_devmgr_process(&res->req, &res->reply);
+
+ len = db_vsock_sendmsg(res->sock, (char *)&res->reply,
+ DEV_MGR_MSG_SIZE);
+ if (len <= 0)
+ break;
+ }
+
+ /* TODO: check who shutdown the socket, receiving or sending. */
+ sock_release(res->sock);
+ kfree(res);
+ return 0;
+}
+
+static int db_devmgr_handler(struct socket *sock)
+{
+ struct task_res *res;
+ struct task_struct *conn_task;
+
+ /* TODO: ensure singleton, only one server exists */
+ res = kzalloc(sizeof(*res), GFP_KERNEL);
+ if (!res)
+ return -ENOMEM;
+
+ res->sock = sock;
+ conn_task = kthread_create(db_devmgr_server, res, "db_dev_mgr");
+ if (IS_ERR(conn_task)) {
+ pr_err("db_devmgr_handler : Client process thread create failed, err: %d",
+ (int)PTR_ERR(conn_task));
+ goto failed;
+ } else {
+ res->task = conn_task;
+ wake_up_process(conn_task);
+ }
+
+ return 0;
+failed:
+ kfree(res);
+ return PTR_ERR(conn_task);
+}
+
+static int __init db_device_manager_init(void)
+{
+ return register_db_vsock_service(DEVMGR_CMD_BYTE, db_devmgr_handler);
+}
+
+late_initcall(db_device_manager_init);
+
+MODULE_AUTHOR("Alibaba, Inc.");
+MODULE_DESCRIPTION("Dragonball Device Manager");
+MODULE_LICENSE("GPL v2");
diff --git a/include/dragonball/device_manager.h b/include/dragonball/device_manager.h
new file mode 100644
index 000000000000..a1713e9f026d
--- /dev/null
+++ b/include/dragonball/device_manager.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * device_manager.h Device Manager for Dragonball
+ *
+ * Copyright (C) 2022 Alibaba Cloud, Inc
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#ifndef _DB_DEVICE_MANAGER_H
+#define _DB_DEVICE_MANAGER_H
+
+#include <linux/device.h>
+
+#endif /* _DB_DEVICE_MANAGER_H */
--
2.19.1.6.gb485710b

301
tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0003-upcall-add-cpu-hotplug-hot-unplug-into-device-manage.patch Normal file
View File

@ -0,0 +1,301 @@
From 901fb71c77144b170465611617f0f25099d8a780 Mon Sep 17 00:00:00 2001
From: Chao Wu <chaowu@linux.alibaba.com>
Date: Mon, 21 Nov 2022 19:44:50 +0800
Subject: [PATCH 3/4] upcall: add cpu hotplug/hot-unplug into device manager
service
Add cpu hotplug and hot-unplug support into device manager. In the
`devmgr_req` message, `msg_type` ADD_CPU in `msg_header` will trigger
`add_cpu_dev` action and DEL_CPU will trigger `del_cpu_dev` action, and
we use `apic_ids` and `count` delivered in `cpu_dev_info` to notify
which and how many cpus will be hotplugged / hot-unplugged.
`add_cpu_dev` and `del_cpu_dev` will eventually trigger `add_cpu_upcall`
and `del_cpu_upcall` to trigger the cpu hotplug / hot-unplug process in
the kernel. After the cpu hotplug / hot-unplug process,
`cpu_event_notification` will generate device manager reply to the
client side.
Signed-off-by: Liu Jiang <gerry@linux.alibaba.com>
Signed-off-by: Zizheng Bian <zizheng.bian@linux.alibaba.com>
Signed-off-by: Chao Wu <chaowu@linux.alibaba.com>
Signed-off-by: WangYu <WangYu@linux.alibaba.com>
Signed-off-by: Xingjun Liu <xingjun.liu@linux.alibaba.com>
---
drivers/misc/dragonball/upcall_srv/Kconfig | 11 +
.../upcall_srv/dragonball_device_manager.c | 219 ++++++++++++++++++
2 files changed, 230 insertions(+)
diff --git a/drivers/misc/dragonball/upcall_srv/Kconfig b/drivers/misc/dragonball/upcall_srv/Kconfig
index 6554a9741c00..b237882a2928 100644
--- a/drivers/misc/dragonball/upcall_srv/Kconfig
+++ b/drivers/misc/dragonball/upcall_srv/Kconfig
@@ -24,3 +24,14 @@ config DRAGONBALL_DEVICE_MANAGER
devices etc.
If unsure, say N.
+
+config DRAGONBALL_HOTPLUG_CPU
+ bool "CPU hotplug/hotunplug support"
+ depends on DRAGONBALL_DEVICE_MANAGER
+ default y
+ help
+ This configure implements a vCPU hotplug/hotunplug support, vmm
+ should send hotplug request by vsock which follow special data
+ structure with command and parameter to hot-pluging an vCPU.
+
+ If unsure, say N.
diff --git a/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c b/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c
index ebcb6ef74285..210ef5d6c9d5 100644
--- a/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c
+++ b/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c
@@ -75,9 +75,20 @@ struct devmgr_req {
struct devmgr_msg_header msg_header;
union {
char pad[DEV_MGR_MSG_SIZE - sizeof(struct devmgr_msg_header)];
+#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU)
+ struct {
+ uint8_t count;
+ uint8_t apic_ver;
+ uint8_t apic_ids[256];
+ } cpu_dev_info;
+#endif
} msg_load;
};
+struct cpu_dev_reply_info {
+ uint32_t apic_index;
+};
+
struct devmgr_reply {
struct devmgr_msg_header msg_header;
/*
@@ -87,6 +98,9 @@ struct devmgr_reply {
int32_t ret;
union {
char pad[DEV_MGR_MSG_SIZE - sizeof(struct devmgr_msg_header) - sizeof(int32_t)];
+#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU)
+ struct cpu_dev_reply_info cpu_dev_info;
+#endif
} msg_load;
};
@@ -109,10 +123,215 @@ static void _fill_msg_header(struct devmgr_msg_header *msg, uint32_t msg_size,
msg->msg_flags = msg_flags;
}
+#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64)
+static int get_cpu_id(int apic_id)
+{
+ int i;
+
+ for (i = 0; i < num_processors; i++) {
+ if (cpu_physical_id(i) == apic_id)
+ return i;
+ }
+ return -1;
+}
+
+/**
+ * Return the first failed hotplug index of the apic_ids to dragonball.
+ * If it is not equal to the count of all hotplug needed vcpus,
+ * we will rollback the vcpus from apics_ids[0] to apic_ids[i-1] in dragonball.
+ */
+static void cpu_event_notification(
+ uint8_t apic_ids_index,
+ int ret,
+ uint32_t action_type,
+ struct devmgr_reply *rep)
+{
+ pr_info("cpu event notification: apic ids index %d", apic_ids_index);
+ rep->msg_load.cpu_dev_info.apic_index = apic_ids_index;
+ rep->ret = ret;
+ _fill_msg_header(&rep->msg_header,
+ sizeof(struct cpu_dev_reply_info), action_type, 0);
+}
+#endif
+
+#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64)
+static int add_cpu_upcall(int apic_id, uint8_t apic_ver)
+{
+ int cpu_id, node_id;
+ int ret;
+
+ pr_info("adding vcpu apic_id %d", apic_id);
+
+ /**
+ * Get the mutex lock for hotplug and cpu update and cpu write lock.
+ * So that other threads won't influence the hotplug process.
+ */
+ lock_device_hotplug();
+ cpu_maps_update_begin();
+ cpu_hotplug_begin();
+
+ cpu_id = generic_processor_info(apic_id, apic_ver);
+ if (cpu_id < 0) {
+ pr_err("cpu (apic id %d) cannot be added, generic processor info failed", apic_id);
+ ret = -EINVAL;
+ goto rollback_generic_cpu;
+ }
+
+ /* update numa mapping for hot-plugged cpus. */
+ node_id = numa_cpu_node(cpu_id);
+ if (node_id != NUMA_NO_NODE)
+ numa_set_node(cpu_id, node_id);
+
+ ret = arch_register_cpu(cpu_id);
+ if (ret) {
+ pr_err("cpu %d cannot be added, register cpu failed %d", cpu_id, ret);
+ goto rollback_register_cpu;
+ }
+
+ cpu_hotplug_done();
+ cpu_maps_update_done();
+ unlock_device_hotplug();
+
+ ret = add_cpu(cpu_id);
+ if (ret) {
+ pr_err("cpu %d cannot be added, cpu up failed: %d", cpu_id, ret);
+ goto rollback_cpu_up;
+ }
+ return ret;
+
+rollback_cpu_up:
+ arch_unregister_cpu(cpu_id);
+ set_cpu_present(cpu_id, false);
+ per_cpu(x86_cpu_to_apicid, cpu_id) = -1;
+ num_processors--;
+ return ret;
+
+rollback_register_cpu:
+ set_cpu_present(cpu_id, false);
+ per_cpu(x86_cpu_to_apicid, cpu_id) = -1;
+ num_processors--;
+rollback_generic_cpu:
+ cpu_hotplug_done();
+ cpu_maps_update_done();
+ unlock_device_hotplug();
+ return ret;
+}
+
+static int del_cpu_upcall(int apic_id)
+{
+ int cpu_id = get_cpu_id(apic_id);
+ int ret;
+
+ if (cpu_id == 0) {
+ pr_err("cannot del bootstrap processor.");
+ return -EINVAL;
+ }
+ pr_info("deleting vcpu %d", cpu_id);
+ ret = remove_cpu(cpu_id);
+ if (ret) {
+ pr_err("del vcpu failed, err: %d", ret);
+ return ret;
+ }
+
+ lock_device_hotplug();
+ cpu_maps_update_begin();
+ cpu_hotplug_begin();
+
+ arch_unregister_cpu(cpu_id);
+ set_cpu_present(cpu_id, false);
+ per_cpu(x86_cpu_to_apicid, cpu_id) = -1;
+ num_processors--;
+
+ cpu_hotplug_done();
+ cpu_maps_update_done();
+ unlock_device_hotplug();
+
+ return ret;
+}
+
+static int add_cpu_dev(struct devmgr_req *req,
+ struct devmgr_reply *rep)
+{
+ int ret;
+ uint8_t i;
+ int apic_id;
+
+ uint8_t count = req->msg_load.cpu_dev_info.count;
+ uint8_t apic_ver = req->msg_load.cpu_dev_info.apic_ver;
+ uint8_t *apic_ids = req->msg_load.cpu_dev_info.apic_ids;
+
+ pr_info("add vcpu number: %d", count);
+
+ for (i = 0; i < count; ++i) {
+ apic_id = apic_ids[i];
+ if (get_cpu_id(apic_id) != -1) {
+ pr_err("cpu cannot be added: apci_id %d is already been used.", apic_id);
+ ret = -EINVAL;
+ return ret;
+ }
+ }
+
+ for (i = 0; i < count; ++i) {
+ apic_id = apic_ids[i];
+ ret = add_cpu_upcall(apic_id, apic_ver);
+ if (ret != 0)
+ break;
+ }
+
+ if (!ret)
+ cpu_event_notification(i, ret, ADD_CPU, rep);
+ return ret;
+}
+
+static int del_cpu_dev(struct devmgr_req *req,
+ struct devmgr_reply *rep)
+{
+ int ret;
+ uint8_t i;
+ int cpu_id;
+
+ uint8_t count = req->msg_load.cpu_dev_info.count;
+ uint8_t *apic_ids = req->msg_load.cpu_dev_info.apic_ids;
+
+ pr_info("del vcpu number : %d", count);
+
+ if (count >= num_processors) {
+ pr_err("cpu del parameter check error: cannot remove all vcpus");
+ ret = -EINVAL;
+ cpu_event_notification(0, ret, DEL_CPU, rep);
+ return ret;
+ }
+
+ for (i = 0; i < count; ++i) {
+ cpu_id = get_cpu_id(apic_ids[i]);
+ if (!cpu_possible(cpu_id)) {
+ pr_err("cpu %d cannot be deleted: cpu not possible", cpu_id);
+ ret = -EINVAL;
+ cpu_event_notification(0, ret, DEL_CPU, rep);
+ return ret;
+ }
+ }
+
+ for (i = 0; i < count; ++i) {
+ ret = del_cpu_upcall(apic_ids[i]);
+ if (ret != 0)
+ break;
+ }
+
+ if (!ret)
+ cpu_event_notification(i, ret, DEL_CPU, rep);
+ return ret;
+}
+#endif
+
static struct {
enum devmgr_msg_type cmd;
action_route_t fn;
} opt_map[] = {
+#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64)
+ {ADD_CPU, add_cpu_dev},
+ {DEL_CPU, del_cpu_dev},
+#endif
};
static action_route_t get_action(struct devmgr_req *req)
--
2.19.1.6.gb485710b

415
tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0004-upcall-add-virtio-mmio-hotplug-hot-unplug-into-devic.patch Normal file
View File

@ -0,0 +1,415 @@
From 90ced3463137076b9df2200b5c6ad720660c6bfc Mon Sep 17 00:00:00 2001
From: Chao Wu <chaowu@linux.alibaba.com>
Date: Wed, 23 Nov 2022 19:23:47 +0800
Subject: [PATCH 4/4] upcall: add virtio-mmio hotplug/hot-unplug into device
manager service
Add virtio-mmio hotplug/hot-unplug support into device manager. In the
`devmgr_req` message, `msg_type` ADD_MMIO in `msg_header` will trigger
`add_mmio_dev` action and DEL_MMIO will trigger `del_mmio_dev` action,
and we use `mmio_base`, `mmio_size` and `mmio_irq` delivered in
`add_mmio_dev` to notify how to hotplug the virtio-mmio device
Also `virtio_mmio_add_device` and `virtio_mmio_del_device` are
introduced under /drivers/virtio/virtio_mmio.c, and we extract
`vm_add_device` from `vm_cmdline_set` to help hotplug virtio-mmio
device.
Signed-off-by: Liu Jiang <gerry@linux.alibaba.com>
Signed-off-by: Zizheng Bian <zizheng.bian@linux.alibaba.com>
Signed-off-by: Chao Wu <chaowu@linux.alibaba.com>
Signed-off-by: WangYu <WangYu@linux.alibaba.com>
Signed-off-by: Xingjun Liu <xingjun.liu@linux.alibaba.com>
---
drivers/misc/dragonball/upcall_srv/Kconfig | 12 ++
.../upcall_srv/dragonball_device_manager.c | 112 ++++++++++++++
drivers/virtio/Kconfig | 14 ++
drivers/virtio/virtio_mmio.c | 138 +++++++++++++++---
include/dragonball/device_manager.h | 5 +
5 files changed, 259 insertions(+), 22 deletions(-)
diff --git a/drivers/misc/dragonball/upcall_srv/Kconfig b/drivers/misc/dragonball/upcall_srv/Kconfig
index b237882a2928..fc83f03c2edd 100644
--- a/drivers/misc/dragonball/upcall_srv/Kconfig
+++ b/drivers/misc/dragonball/upcall_srv/Kconfig
@@ -25,6 +25,18 @@ config DRAGONBALL_DEVICE_MANAGER
If unsure, say N.
+config DRAGONBALL_HOTPLUG_VIRTIO_MMIO
+ bool "Virtio-MMIO device hotplug/hotunplug support"
+ depends on DRAGONBALL_DEVICE_MANAGER
+ default y
+ help
+ This configure implements a Virtio-MMIO device hotplug/hotunplug
+ support, vmm should send hotplug request by vsock which follow
+ special data structure with command and parameter to hot-pluging
+ an MMIO device.
+
+ If unsure, say N.
+
config DRAGONBALL_HOTPLUG_CPU
bool "CPU hotplug/hotunplug support"
depends on DRAGONBALL_DEVICE_MANAGER
diff --git a/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c b/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c
index 210ef5d6c9d5..5a95b2ba63e8 100644
--- a/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c
+++ b/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c
@@ -75,6 +75,13 @@ struct devmgr_req {
struct devmgr_msg_header msg_header;
union {
char pad[DEV_MGR_MSG_SIZE - sizeof(struct devmgr_msg_header)];
+#if defined(CONFIG_DRAGONBALL_HOTPLUG_VIRTIO_MMIO)
+ struct {
+ uint64_t mmio_base;
+ uint64_t mmio_size;
+ uint32_t mmio_irq;
+ } add_mmio_dev;
+#endif
#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU)
struct {
uint8_t count;
@@ -98,6 +105,10 @@ struct devmgr_reply {
int32_t ret;
union {
char pad[DEV_MGR_MSG_SIZE - sizeof(struct devmgr_msg_header) - sizeof(int32_t)];
+#if defined(CONFIG_DRAGONBALL_HOTPLUG_VIRTIO_MMIO)
+ struct {
+ } add_mmio_dev;
+#endif
#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU)
struct cpu_dev_reply_info cpu_dev_info;
#endif
@@ -114,6 +125,62 @@ struct task_res {
typedef int (*action_route_t) (struct devmgr_req *req,
struct devmgr_reply *rep);
+#if defined(CONFIG_DRAGONBALL_HOTPLUG_VIRTIO_MMIO)
+#ifdef CONFIG_ARM64
+static uint32_t get_device_virq(uint32_t pin)
+{
+ uint32_t virq;
+ struct device_node *node;
+ struct irq_fwspec dummy_fwspec = {
+ .param_count = 3,
+ .param = {0, 0, IRQ_TYPE_EDGE_RISING}
+ };
+
+ node = of_find_node_by_name(NULL, "intc");
+ if (!node) {
+ pr_err("interrupt controller device node not found.");
+ return 0;
+ }
+ dummy_fwspec.param[1] = pin;
+ dummy_fwspec.fwnode = of_node_to_fwnode(node);
+ virq = irq_create_fwspec_mapping(&dummy_fwspec);
+ of_node_put(node);
+ return virq;
+}
+#elif defined(CONFIG_X86_64)
+static inline uint32_t get_device_virq(uint32_t irq)
+{
+ return irq;
+}
+#endif
+
+static int get_dev_resource(struct devmgr_req *req, struct resource *res)
+{
+ uint64_t base = req->msg_load.add_mmio_dev.mmio_base;
+ uint64_t size = req->msg_load.add_mmio_dev.mmio_size;
+ uint32_t irq = req->msg_load.add_mmio_dev.mmio_irq;
+ uint32_t virq;
+
+ if (req->msg_header.msg_size != sizeof(req->msg_load.add_mmio_dev))
+ return -EINVAL;
+
+ res[0].flags = IORESOURCE_MEM;
+ res[0].start = base;
+ res[0].end = base + size - 1;
+ res[1].flags = IORESOURCE_IRQ;
+ virq = get_device_virq(irq);
+ if (!virq)
+ return -EINVAL;
+ res[1].start = res[1].end = virq;
+
+ /* detect the irq sharing mode */
+ if (irq == SHARED_IRQ_NO)
+ res[1].flags |= IORESOURCE_IRQ_SHAREABLE;
+
+ return 0;
+}
+#endif
+
static void _fill_msg_header(struct devmgr_msg_header *msg, uint32_t msg_size,
uint32_t msg_type, uint32_t msg_flags)
{
@@ -154,6 +221,47 @@ static void cpu_event_notification(
}
#endif
+#if defined(CONFIG_DRAGONBALL_HOTPLUG_VIRTIO_MMIO)
+static int add_mmio_dev(struct devmgr_req *req,
+ struct devmgr_reply *rep)
+{
+ int ret;
+ struct resource res[2] = {};
+ struct devmgr_msg_header *rep_mh = &rep->msg_header;
+
+ ret = get_dev_resource(req, res);
+ if (ret)
+ return ret;
+
+ ret = virtio_mmio_add_device(res, ARRAY_SIZE(res));
+ if (!ret) {
+ rep->ret = ret;
+ _fill_msg_header(rep_mh, 0, ADD_MMIO, 0);
+ }
+ return ret;
+}
+
+static int del_mmio_dev(struct devmgr_req *req,
+ struct devmgr_reply *rep)
+{
+ int ret;
+ struct resource res[2] = {};
+ struct devmgr_msg_header *rep_mh = &rep->msg_header;
+
+ ret = get_dev_resource(req, res);
+ if (ret)
+ return ret;
+
+ ret = virtio_mmio_del_device(res, ARRAY_SIZE(res));
+ if (!ret) {
+ rep->ret = ret;
+ _fill_msg_header(rep_mh, 0, DEL_MMIO, 0);
+ }
+ return ret;
+}
+#endif
+
+
#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64)
static int add_cpu_upcall(int apic_id, uint8_t apic_ver)
{
@@ -328,6 +436,10 @@ static struct {
enum devmgr_msg_type cmd;
action_route_t fn;
} opt_map[] = {
+#if defined(CONFIG_DRAGONBALL_HOTPLUG_VIRTIO_MMIO)
+ {ADD_MMIO, add_mmio_dev},
+ {DEL_MMIO, del_mmio_dev},
+#endif
#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64)
{ADD_CPU, add_cpu_dev},
{DEL_CPU, del_cpu_dev},
diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig
index 7b41130d3f35..084de9827b5d 100644
--- a/drivers/virtio/Kconfig
+++ b/drivers/virtio/Kconfig
@@ -139,4 +139,18 @@ config VIRTIO_DMA_SHARED_BUFFER
This option adds a flavor of dma buffers that are backed by
virtio resources.
+config VIRTIO_MMIO_DRAGONBALL
+ bool "Enable features for Dragonball virtio MMIO devcies"
+ default n
+ depends on VIRTIO_MMIO
+ depends on X86_64 || ARM64
+ select X86_PLATFORM_MSI
+ select VIRTIO_MMIO_MSI
+ help
+ The Dragonball VMM implements several optimizations for MMIO virtio
+ devices. This option enables support of those optimization features:
+ - virtio-mmio hotplug through upcall
+
+ If unsure, say N
+
endif # VIRTIO_MENU
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
index 238383ff1064..c29d533bd4a8 100644
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -652,16 +652,41 @@ static struct device vm_cmdline_parent = {
static int vm_cmdline_parent_registered;
static int vm_cmdline_id;
+static int vm_add_device(struct resource *resources, size_t res_size)
+{
+ int err;
+ struct platform_device *pdev;
+
+ if (!vm_cmdline_parent_registered) {
+ err = device_register(&vm_cmdline_parent);
+ if (err) {
+ pr_err("Failed to register parent device!\n");
+ return err;
+ }
+ vm_cmdline_parent_registered = 1;
+ }
+
+ pr_info("Registering device virtio-mmio.%d at 0x%llx-0x%llx, IRQ %d.\n",
+ vm_cmdline_id,
+ (unsigned long long)resources[0].start,
+ (unsigned long long)resources[0].end,
+ (int)resources[1].start);
+
+ pdev = platform_device_register_resndata(&vm_cmdline_parent,
+ "virtio-mmio", vm_cmdline_id++,
+ resources, res_size, NULL, 0);
+
+ return PTR_ERR_OR_ZERO(pdev);
+}
+
static int vm_cmdline_set(const char *device,
const struct kernel_param *kp)
{
- int err;
struct resource resources[2] = {};
char *str;
long long int base, size;
unsigned int irq;
int processed, consumed = 0;
- struct platform_device *pdev;
/* Consume "size" part of the command line parameter */
size = memparse(device, &str);
@@ -686,26 +711,7 @@ static int vm_cmdline_set(const char *device,
resources[1].flags = IORESOURCE_IRQ;
resources[1].start = resources[1].end = irq;
- if (!vm_cmdline_parent_registered) {
- err = device_register(&vm_cmdline_parent);
- if (err) {
- pr_err("Failed to register parent device!\n");
- return err;
- }
- vm_cmdline_parent_registered = 1;
- }
-
- pr_info("Registering device virtio-mmio.%d at 0x%llx-0x%llx, IRQ %d.\n",
- vm_cmdline_id,
- (unsigned long long)resources[0].start,
- (unsigned long long)resources[0].end,
- (int)resources[1].start);
-
- pdev = platform_device_register_resndata(&vm_cmdline_parent,
- "virtio-mmio", vm_cmdline_id++,
- resources, ARRAY_SIZE(resources), NULL, 0);
-
- return PTR_ERR_OR_ZERO(pdev);
+ return vm_add_device(resources, ARRAY_SIZE(resources));
}
static int vm_cmdline_get_device(struct device *dev, void *data)
@@ -755,6 +761,94 @@ static void vm_unregister_cmdline_devices(void)
}
}
+#ifdef CONFIG_DRAGONBALL_DEVICE_MANAGER
+static int vm_match_device(struct device *dev, void *data)
+{
+ struct resource *resource = (struct resource *)data;
+ struct platform_device *pdev = to_platform_device(dev);
+
+ if ((pdev->resource[0].start == resource[0].start) &&
+ (pdev->resource[0].end == resource[0].end) &&
+ (pdev->resource[1].start == resource[1].start))
+ return 1;
+ return 0;
+}
+
+static struct device *vm_find_device(struct resource *res)
+{
+ return device_find_child(&vm_cmdline_parent, res, vm_match_device);
+}
+
+static int vm_device_overlap(struct device *dev, void *data)
+{
+ struct resource *res = (struct resource *)data;
+ struct platform_device *pdev = to_platform_device(dev);
+
+ /* Detect IRQ number conflicts except shared IRQs. */
+ if (!(res[1].flags & IORESOURCE_IRQ_SHAREABLE) &&
+ (pdev->resource[1].start == res[1].start)) {
+ return 1;
+ }
+
+ /* Detect device MMIO addresses overlapping */
+ if ((pdev->resource[0].start < res[0].end) &&
+ (pdev->resource[0].end > res[0].start)) {
+ return 1;
+ }
+
+ return 0;
+}
+
+static struct device *vm_detect_resource(struct resource *res)
+{
+ /* return NULL if no resource overlapped */
+ return device_find_child(&vm_cmdline_parent, res, vm_device_overlap);
+}
+
+int virtio_mmio_add_device(struct resource *resources, size_t res_size)
+{
+ int err;
+ struct device *dev;
+
+ if (res_size < 2 || !resources)
+ return -EINVAL;
+
+ dev = vm_detect_resource(resources);
+ if (dev) {
+ put_device(dev);
+ return -EEXIST;
+ }
+
+ lock_device_hotplug();
+ err = vm_add_device(resources, res_size);
+ unlock_device_hotplug();
+
+ return err;
+}
+EXPORT_SYMBOL_GPL(virtio_mmio_add_device);
+
+int virtio_mmio_del_device(struct resource *resources, size_t res_size)
+{
+ int ret;
+ struct device *dev;
+
+ if (res_size < 2 || !resources)
+ return -EINVAL;
+
+ dev = vm_find_device(resources);
+ if (!dev)
+ return -ENODEV;
+
+ put_device(dev);
+ lock_device_hotplug();
+ ret = vm_unregister_cmdline_device(dev, NULL);
+ unlock_device_hotplug();
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(virtio_mmio_del_device);
+#endif /* CONFIG_DRAGONBALL_DEVICE_MANAGER */
+
#else
static void vm_unregister_cmdline_devices(void)
diff --git a/include/dragonball/device_manager.h b/include/dragonball/device_manager.h
index a1713e9f026d..785761c47f97 100644
--- a/include/dragonball/device_manager.h
+++ b/include/dragonball/device_manager.h
@@ -15,4 +15,9 @@
#include <linux/device.h>
+#if defined(CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES)
+int virtio_mmio_add_device(struct resource *resources, size_t res_size);
+int virtio_mmio_del_device(struct resource *resources, size_t res_size);
+#endif
+
#endif /* _DB_DEVICE_MANAGER_H */
--
2.19.1.6.gb485710b