dragonball: add upcall support

Upcall is a direct communication tool between VMM and guest developed
upon vsock. It is used to implement device hotplug.

Signed-off-by: Liu Jiang <gerry@linux.alibaba.com>
Signed-off-by: jingshan <jingshan@linux.alibaba.com>
Signed-off-by: Chao Wu <chaowu@linux.alibaba.com>
Signed-off-by: wllenyj <wllenyj@linux.alibaba.com>
Signed-off-by: Zizheng Bian <zizheng.bian@linux.alibaba.com>

src/dragonball/Cargo.toml

@@ -19,6 +19,7 @@ dbs-boot = "0.2.0"
 dbs-device = "0.1.0"
 dbs-interrupt = { version = "0.1.0", features = ["kvm-irq"] }
 dbs-legacy-devices = "0.1.0"
+dbs-upcall = { version = "0.1.0", optional = true }
 dbs-utils = "0.1.0"
 dbs-virtio-devices = { version = "0.1.0", optional = true, features = ["virtio-mmio"] }
 kvm-bindings = "0.5.0"
@@ -45,12 +46,13 @@ slog-async = "2.7.0"
 
 [features]
 atomic-guest-memory = []
+hotplug = ["dbs-upcall", "virtio-vsock"]
 virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"]
 
 [patch.'crates-io']
-dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" }
+dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" }
-dbs-interrupt = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" }
+dbs-interrupt = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" }
-dbs-legacy-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" }
+dbs-legacy-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" }
-dbs-utils = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" }
+dbs-upcall = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" }
-dbs-virtio-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" }
+dbs-utils = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" }
-dbs-upcall = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" }
+dbs-virtio-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" }

src/dragonball/src/device_manager/mod.rs

@@ -29,6 +29,12 @@ use dbs_virtio_devices::{
     VirtioDevice,
 };
 
+#[cfg(feature = "hotplug")]
+use dbs_upcall::{
+    MmioDevRequest, DevMgrRequest, DevMgrService, UpcallClient, UpcallClientError,
+    UpcallClientRequest, UpcallClientResponse
+};
+
 use crate::address_space_manager::GuestAddressSpaceImpl;
 use crate::error::StartMicrovmError;
 use crate::resource_manager::ResourceManager;
@@ -83,6 +89,11 @@ pub enum DeviceMgrError {
     /// Error from Virtio subsystem.
     #[error(transparent)]
     Virtio(virtio::Error),
+
+    #[cfg(feature = "hotplug")]
+    /// Failed to hotplug the device.
+    #[error("failed to hotplug virtual device")]
+    HotplugDevice(#[source] UpcallClientError),
 }
 
 /// Specialized version of `std::result::Result` for device manager operations.
@@ -188,6 +199,8 @@ pub struct DeviceOpContext {
     logger: slog::Logger,
     is_hotplug: bool,
 
+    #[cfg(feature = "hotplug")]
+    upcall_client: Option<Arc<UpcallClient<DevMgrService>>>,
     #[cfg(feature = "dbs-virtio-devices")]
     virtio_devices: Vec<Arc<DbsMmioV2Device>>,
 }
@@ -220,6 +233,8 @@ impl DeviceOpContext {
             address_space,
             logger,
             is_hotplug,
+            #[cfg(feature = "hotplug")]
+            upcall_client: None,
             #[cfg(feature = "dbs-virtio-devices")]
             virtio_devices: Vec::new(),
         }
@@ -265,6 +280,86 @@ impl DeviceOpContext {
     }
 }
 
+#[cfg(not(feature = "hotplug"))]
+impl DeviceOpContext {
+    pub(crate) fn insert_hotplug_mmio_device(
+        &self,
+        _dev: &Arc<dyn DeviceIo>,
+        _callback: Option<()>,
+    ) -> Result<()> {
+        Err(DeviceMgrError::InvalidOperation)
+    }
+
+    pub(crate) fn remove_hotplug_mmio_device(
+        &self,
+        _dev: &Arc<dyn DeviceIo>,
+        _callback: Option<()>,
+    ) -> Result<()> {
+        Err(DeviceMgrError::InvalidOperation)
+    }
+}
+
+#[cfg(feature = "hotplug")]
+impl DeviceOpContext {
+    fn call_hotplug_device(
+        &self,
+        req: DevMgrRequest,
+        callback: Option<Box<dyn Fn(UpcallClientResponse) + Send>>,
+    ) -> Result<()> {
+        if let Some(upcall_client) = self.upcall_client.as_ref() {
+            if let Some(cb) = callback {
+                upcall_client
+                    .send_request(UpcallClientRequest::DevMgr(req), cb)
+                    .map_err(DeviceMgrError::HotplugDevice)?;
+            } else {
+                upcall_client
+                    .send_request_without_result(UpcallClientRequest::DevMgr(req))
+                    .map_err(DeviceMgrError::HotplugDevice)?;
+            }
+            Ok(())
+        } else {
+            Err(DeviceMgrError::InvalidOperation)
+        }
+    }
+
+    pub(crate) fn insert_hotplug_mmio_device(
+        &self,
+        dev: &Arc<DbsMmioV2Device>,
+        callback: Option<Box<dyn Fn(UpcallClientResponse) + Send>>,
+    ) -> Result<()> {
+        if !self.is_hotplug {
+            return Err(DeviceMgrError::InvalidOperation);
+        }
+
+        let (mmio_base, mmio_size, mmio_irq) = DeviceManager::get_virtio_device_info(dev)?;
+        let req = DevMgrRequest::AddMmioDev(MmioDevRequest {
+            mmio_base,
+            mmio_size,
+            mmio_irq,
+        });
+
+        self.call_hotplug_device(req, callback)
+    }
+
+    pub(crate) fn remove_hotplug_mmio_device(
+        &self,
+        dev: &Arc<DbsMmioV2Device>,
+        callback: Option<Box<dyn Fn(UpcallClientResponse) + Send>>,
+    ) -> Result<()> {
+        if !self.is_hotplug {
+            return Err(DeviceMgrError::InvalidOperation);
+        }
+        let (mmio_base, mmio_size, mmio_irq) = DeviceManager::get_virtio_device_info(dev)?;
+        let req = DevMgrRequest::DelMmioDev(MmioDevRequest {
+            mmio_base,
+            mmio_size,
+            mmio_irq,
+        });
+
+        self.call_hotplug_device(req, callback)
+    }
+}
+
 /// Device manager for virtual machines, which manages all device for a virtual machine.
 pub struct DeviceManager {
     io_manager: Arc<ArcSwap<IoManager>>,