runtime-rs: Support VhostUserEndpoint

This commit introduces VhostUserEndpoint and supports relative to
vhost-user-net devices for device manager. For now, Dragonball is able to
attach vhost-user-net devices.

Fixes: #8625

Signed-off-by: Xuewei Niu <niuxuewei.nxw@antgroup.com>

src/runtime-rs/Cargo.lock

@@ -808,6 +808,7 @@ dependencies = [
  "byteorder",
  "caps",
  "dbs-address-space",
+ "dbs-boot",
  "dbs-device",
  "dbs-interrupt",
  "dbs-utils",

src/runtime-rs/crates/hypervisor/Cargo.toml

@@ -35,7 +35,7 @@ kata-types = { path = "../../../libs/kata-types" }
 logging = { path = "../../../libs/logging" }
 shim-interface = { path = "../../../libs/shim-interface" }
 
-dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "vhost-net", "dbs-upcall","virtio-mem", "virtio-balloon"] }
+dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "vhost-net", "dbs-upcall","virtio-mem", "virtio-balloon", "vhost-user-net"] }
 
 ch-config = { path = "ch-config", optional = true }
 tests_utils = { path = "../../tests/utils" }

src/runtime-rs/crates/hypervisor/src/device/device_manager.rs

@@ -12,8 +12,9 @@ use tokio::sync::{Mutex, RwLock};
 
 use crate::{
     vhost_user_blk::VhostUserBlkDevice, BlockConfig, BlockDevice, HybridVsockDevice, Hypervisor,
-    NetworkDevice, ShareFsDevice, VfioDevice, VhostUserConfig, VsockDevice, KATA_BLK_DEV_TYPE,
-    KATA_MMIO_BLK_DEV_TYPE, KATA_NVDIMM_DEV_TYPE, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM,
+    NetworkDevice, ShareFsDevice, VfioDevice, VhostUserConfig, VhostUserNetDevice, VsockDevice,
+    KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, KATA_NVDIMM_DEV_TYPE, VIRTIO_BLOCK_MMIO,
+    VIRTIO_BLOCK_PCI, VIRTIO_PMEM,
 };
 
 use super::{
@@ -231,6 +232,11 @@ impl DeviceManager {
                         return Some(device_id.to_string());
                     }
                 }
+                DeviceType::VhostUserNetwork(device) => {
+                    if device.config.socket_path == host_path {
+                        return Some(device_id.to_string());
+                    }
+                }
                 _ => {
                     // TODO: support find other device type
                     continue;
@@ -326,6 +332,22 @@ impl DeviceManager {
 
                 Arc::new(Mutex::new(NetworkDevice::new(device_id.clone(), config)))
             }
+            DeviceConfig::VhostUserNetworkCfg(config) => {
+                if let Some(dev_id) = self.find_device(config.socket_path.clone()).await {
+                    info!(
+                        sl!(),
+                        "vhost-user-net device {} found, just return device id {}",
+                        config.socket_path,
+                        dev_id
+                    );
+                    return Ok(dev_id);
+                }
+
+                Arc::new(Mutex::new(VhostUserNetDevice::new(
+                    device_id.clone(),
+                    config.clone(),
+                )))
+            }
             DeviceConfig::HybridVsockCfg(hvconfig) => {
                 // No need to do find device for hybrid vsock device.
                 Arc::new(Mutex::new(HybridVsockDevice::new(&device_id, hvconfig)))

src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs

@@ -6,6 +6,8 @@
 
 mod vfio;
 mod vhost_user;
+pub mod vhost_user_blk;
+mod vhost_user_net;
 mod virtio_blk;
 mod virtio_fs;
 mod virtio_net;
@@ -15,6 +17,8 @@ pub use vfio::{
     bind_device_to_host, bind_device_to_vfio, get_vfio_device, HostDevice, VfioBusMode, VfioConfig,
     VfioDevice,
 };
+pub use vhost_user::{VhostUserConfig, VhostUserDevice, VhostUserType};
+pub use vhost_user_net::VhostUserNetDevice;
 pub use virtio_blk::{
     BlockConfig, BlockDevice, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, KATA_NVDIMM_DEV_TYPE,
     VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM,
@@ -26,6 +30,3 @@ pub use virtio_net::{Address, NetworkConfig, NetworkDevice};
 pub use virtio_vsock::{
     HybridVsockConfig, HybridVsockDevice, VsockConfig, VsockDevice, DEFAULT_GUEST_VSOCK_CID,
 };
-
-pub mod vhost_user_blk;
-pub use vhost_user::{VhostUserConfig, VhostUserDevice, VhostUserType};

src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user_net.rs

@@ -0,0 +1,59 @@
+// Copyright (C) 2019-2023 Ant Group. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use anyhow::{Context, Result};
+use async_trait::async_trait;
+
+use crate::device::{Device, DeviceType};
+use crate::{Hypervisor, VhostUserConfig};
+
+#[derive(Debug, Clone, Default)]
+/// Vhost-user-net device for device manager.
+pub struct VhostUserNetDevice {
+    pub device_id: String,
+    pub config: VhostUserConfig,
+}
+
+impl VhostUserNetDevice {
+    pub fn new(device_id: String, config: VhostUserConfig) -> Self {
+        Self { device_id, config }
+    }
+}
+
+#[async_trait]
+impl Device for VhostUserNetDevice {
+    async fn attach(&mut self, h: &dyn Hypervisor) -> Result<()> {
+        h.add_device(DeviceType::VhostUserNetwork(self.clone()))
+            .await
+            .context("add vhost-user-net device to hypervisor")?;
+        Ok(())
+    }
+
+    async fn detach(&mut self, h: &dyn Hypervisor) -> Result<Option<u64>> {
+        h.remove_device(DeviceType::VhostUserNetwork(self.clone()))
+            .await
+            .context("remove vhost-user-net device from hypervisor")?;
+        Ok(Some(self.config.index))
+    }
+
+    async fn update(&mut self, _h: &dyn Hypervisor) -> Result<()> {
+        // There's no need to do update for vhost-user-net
+        Ok(())
+    }
+
+    async fn get_device_info(&self) -> DeviceType {
+        DeviceType::VhostUserNetwork(self.clone())
+    }
+
+    async fn increase_attach_count(&mut self) -> Result<bool> {
+        // Vhost-user-net devices will not be attached multiple times, just
+        // return Ok(false)
+        Ok(false)
+    }
+
+    async fn decrease_attach_count(&mut self) -> Result<bool> {
+        // Vhost-user-net devices will not be detached multiple times, just
+        // return Ok(false)
+        Ok(false)
+    }
+}

src/runtime-rs/crates/hypervisor/src/device/mod.rs

@@ -10,7 +10,7 @@ use crate::device::driver::vhost_user_blk::VhostUserBlkDevice;
 use crate::{
     BlockConfig, BlockDevice, HybridVsockConfig, HybridVsockDevice, Hypervisor as hypervisor,
     NetworkConfig, NetworkDevice, ShareFsConfig, ShareFsDevice, VfioConfig, VfioDevice,
-    VhostUserConfig, VsockConfig, VsockDevice,
+    VhostUserConfig, VhostUserNetDevice, VsockConfig, VsockDevice,
 };
 use anyhow::Result;
 use async_trait::async_trait;
@@ -25,6 +25,7 @@ pub enum DeviceConfig {
     BlockCfg(BlockConfig),
     VhostUserBlkCfg(VhostUserConfig),
     NetworkCfg(NetworkConfig),
+    VhostUserNetworkCfg(VhostUserConfig),
     ShareFsCfg(ShareFsConfig),
     VfioCfg(VfioConfig),
     VsockCfg(VsockConfig),
@@ -37,6 +38,7 @@ pub enum DeviceType {
     VhostUserBlk(VhostUserBlkDevice),
     Vfio(VfioDevice),
     Network(NetworkDevice),
+    VhostUserNetwork(VhostUserNetDevice),
     ShareFs(ShareFsDevice),
     HybridVsock(HybridVsockDevice),
     Vsock(VsockDevice),

src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs

@@ -7,12 +7,16 @@
 use std::path::PathBuf;
 
 use anyhow::{anyhow, Context, Result};
+use dbs_utils::net::MacAddr;
+use dragonball::api::v1::VhostUserConfig as DragonballVhostUserConfig;
 use dragonball::api::v1::{
-    BlockDeviceConfigInfo, FsDeviceConfigInfo, FsMountConfigInfo, VsockDeviceConfigInfo,
+    BlockDeviceConfigInfo, FsDeviceConfigInfo, FsMountConfigInfo, NetworkInterfaceConfig,
+    VsockDeviceConfigInfo,
 };
 use dragonball::device_manager::blk_dev_mgr::BlockDeviceType;
 
 use super::{build_dragonball_network_config, DragonballInner};
+use crate::VhostUserConfig;
 use crate::{
     device::DeviceType, HybridVsockConfig, NetworkConfig, ShareFsConfig, ShareFsMountConfig,
     ShareFsMountOperation, ShareFsMountType, VfioBusMode, VfioDevice, VmmState, JAILER_ROOT,
@@ -67,6 +71,9 @@ impl DragonballInner {
             DeviceType::ShareFs(sharefs) => self
                 .add_share_fs_device(&sharefs.config)
                 .context("add share fs device"),
+            DeviceType::VhostUserNetwork(dev) => self
+                .add_vhost_user_net_device(&dev.config)
+                .context("add vhost-user-net device"),
             DeviceType::Vsock(_) => todo!(),
         }
     }
@@ -216,6 +223,25 @@ impl DragonballInner {
             .context("insert network device")
     }
 
+    /// Add vhost-user-net deivce to Dragonball
+    fn add_vhost_user_net_device(&mut self, config: &VhostUserConfig) -> Result<()> {
+        let guest_mac = MacAddr::parse_str(&config.mac_address).ok();
+        let net_cfg = NetworkInterfaceConfig {
+            num_queues: Some(config.num_queues),
+            queue_size: Some(config.queue_size as u16),
+            backend: dragonball::api::v1::Backend::VhostUser(DragonballVhostUserConfig {
+                sock_path: config.socket_path.clone(),
+            }),
+            guest_mac,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        self.vmm_instance
+            .insert_network_device(net_cfg)
+            .context("insert vhost-user-net device")
+    }
+
     fn add_hvsock(&mut self, config: &HybridVsockConfig) -> Result<()> {
         let vsock_cfg = VsockDeviceConfigInfo {
             id: String::from(JAILER_ROOT),

src/runtime-rs/crates/resource/src/network/endpoint/endpoint_persist.rs

@@ -44,6 +44,12 @@ pub struct TapEndpointState {
     pub if_name: String,
 }
 
+#[derive(Serialize, Deserialize, Clone, Default)]
+pub struct VhostUserEndpointState {
+    pub if_name: String,
+    pub socket_path: String,
+}
+
 #[derive(Serialize, Deserialize, Clone, Default)]
 pub struct EndpointState {
     pub physical_endpoint: Option<PhysicalEndpointState>,
@@ -52,5 +58,6 @@ pub struct EndpointState {
     pub macvlan_endpoint: Option<MacvlanEndpointState>,
     pub vlan_endpoint: Option<VlanEndpointState>,
     pub tap_endpoint: Option<TapEndpointState>,
+    pub vhost_user_endpoint: Option<VhostUserEndpointState>,
     // TODO : other endpoint
 }

src/runtime-rs/crates/resource/src/network/endpoint/mod.rs

@@ -18,6 +18,8 @@ pub mod endpoint_persist;
 mod endpoints_test;
 mod tap_endpoint;
 pub use tap_endpoint::TapEndpoint;
+mod vhost_user_endpoint;
+pub use vhost_user_endpoint::VhostUserEndpoint;
 
 use anyhow::Result;
 use async_trait::async_trait;

src/runtime-rs/crates/resource/src/network/endpoint/vhost_user_endpoint.rs

@@ -0,0 +1,114 @@
+// Copyright (C) 2019-2023 Ant Group. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::path::Path;
+use std::sync::Arc;
+
+use anyhow::{anyhow, Context, Result};
+use async_trait::async_trait;
+use hypervisor::device::device_manager::{do_handle_device, DeviceManager};
+use hypervisor::device::{DeviceConfig, DeviceType};
+use hypervisor::{Hypervisor, VhostUserConfig, VhostUserNetDevice, VhostUserType};
+use tokio::sync::RwLock;
+
+use super::endpoint_persist::VhostUserEndpointState;
+use super::Endpoint;
+use crate::network::EndpointState;
+
+/// VhostUserEndpoint uses vhost-user-net device, which supports DPDK, etc.
+#[derive(Debug)]
+pub struct VhostUserEndpoint {
+    // Endpoint index
+    #[allow(dead_code)]
+    index: u32,
+    // Name of virt interface
+    name: String,
+    // Hardware address of virt interface
+    guest_mac: String,
+    // Vhost-user-net device's socket path
+    socket_path: String,
+    // Device manager
+    dev_mgr: Arc<RwLock<DeviceManager>>,
+    // Virtio queue num
+    queue_num: usize,
+    // Virtio queue size
+    queue_size: usize,
+}
+
+impl VhostUserEndpoint {
+    pub async fn new(
+        dev_mgr: &Arc<RwLock<DeviceManager>>,
+        index: u32,
+        name: &str,
+        guest_mac: &str,
+        socket_path: &str,
+        queue_num: usize,
+        queue_size: usize,
+    ) -> Result<Self> {
+        let sk_path = Path::new(socket_path);
+        if sk_path.exists() {
+            return Err(anyhow!("vhost-user-net socket path {} exists", socket_path));
+        }
+
+        Ok(VhostUserEndpoint {
+            index,
+            name: name.to_string(),
+            guest_mac: guest_mac.to_string(),
+            socket_path: socket_path.to_string(),
+            dev_mgr: dev_mgr.clone(),
+            queue_num,
+            queue_size,
+        })
+    }
+
+    fn get_network_config(&self) -> VhostUserConfig {
+        VhostUserConfig {
+            socket_path: self.socket_path.clone(),
+            mac_address: self.guest_mac.clone(),
+            device_type: VhostUserType::Net,
+            queue_size: self.queue_size as u32,
+            num_queues: self.queue_num,
+            ..Default::default()
+        }
+    }
+}
+
+#[async_trait]
+impl Endpoint for VhostUserEndpoint {
+    async fn name(&self) -> String {
+        self.name.clone()
+    }
+
+    async fn hardware_addr(&self) -> String {
+        self.guest_mac.clone()
+    }
+
+    async fn attach(&self) -> Result<()> {
+        let config = self.get_network_config();
+        do_handle_device(&self.dev_mgr, &DeviceConfig::VhostUserNetworkCfg(config))
+            .await
+            .context("handle device")?;
+        Ok(())
+    }
+
+    async fn detach(&self, h: &dyn Hypervisor) -> Result<()> {
+        let config = self.get_network_config();
+        h.remove_device(DeviceType::VhostUserNetwork(VhostUserNetDevice {
+            config,
+            ..Default::default()
+        }))
+        .await
+        .context("remove device")?;
+        Ok(())
+    }
+
+    async fn save(&self) -> Option<EndpointState> {
+        Some(EndpointState {
+            vhost_user_endpoint: Some(VhostUserEndpointState {
+                if_name: self.name.clone(),
+                socket_path: self.socket_path.clone(),
+            }),
+            ..Default::default()
+        })
+    }
+}