dragonball: Support vhost-user-net devices

This PR introduces vhost-user-net devices to Dragonball. The devices are
allowed to run as server on the VMM side.

Fixes: #8502

Signed-off-by: Eric Ren <renzhen@linux.alibaba.com>
Signed-off-by: Liu Jiang <gerry@linux.alibaba.com>
Signed-off-by: Zha Bin <zhabin@linux.alibaba.com>
Signed-off-by: Chao Wu <chaowu@linux.alibaba.com>
Signed-off-by: Zizheng Bian <zizheng.bian@linux.alibaba.com>
Signed-off-by: Xuewei Niu <niuxuewei.nxw@antgroup.com>

src/dragonball/Cargo.toml

@@ -65,3 +65,4 @@ virtio-mem = ["dbs-virtio-devices/virtio-mem", "virtio-queue", "atomic-guest-mem
 virtio-balloon = ["dbs-virtio-devices/virtio-balloon", "virtio-queue"]
 vhost-net = ["dbs-virtio-devices/vhost-net"]
 vhost-user-fs = ["dbs-virtio-devices/vhost-user-fs"]
+vhost-user-net = ["dbs-virtio-devices/vhost-user-net"]

src/dragonball/src/api/v1/mod.rs

@@ -19,7 +19,17 @@ mod machine_config;
 pub use self::machine_config::{VmConfigError, MAX_SUPPORTED_VCPUS};
 
 /// Wrapper for configuring the virtio networking
-#[cfg(any(feature = "virtio-net", feature = "vhost-net"))]
+#[cfg(any(
+    feature = "virtio-net",
+    feature = "vhost-net",
+    feature = "vhost-user-net"
+))]
 mod virtio_net;
 #[cfg(any(feature = "virtio-net", feature = "vhost-net"))]
-pub use virtio_net::{Backend, NetworkInterfaceConfig, NetworkInterfaceUpdateConfig, VirtioConfig};
+pub use virtio_net::VirtioConfig;
+#[cfg(any(
+    feature = "virtio-net",
+    feature = "vhost-net",
+    feature = "vhost-user-net"
+))]
+pub use virtio_net::{Backend, NetworkInterfaceConfig, NetworkInterfaceUpdateConfig};

src/dragonball/src/api/v1/virtio_net.rs

@@ -12,7 +12,13 @@ use serde::{Deserialize, Serialize};
 use super::{VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo};
 use crate::config_manager::RateLimiterConfigInfo;
 #[cfg(feature = "vhost-net")]
-use crate::device_manager::vhost_net_dev_mgr::{self, VhostNetDeviceConfigInfo};
+use crate::device_manager::vhost_net_dev_mgr;
+#[cfg(feature = "vhost-net")]
+use crate::device_manager::vhost_net_dev_mgr::VhostNetDeviceConfigInfo;
+#[cfg(feature = "vhost-user-net")]
+use crate::device_manager::vhost_user_net_dev_mgr;
+#[cfg(feature = "vhost-user-net")]
+use crate::device_manager::vhost_user_net_dev_mgr::VhostUserNetDeviceConfigInfo;
 #[cfg(feature = "virtio-net")]
 use crate::device_manager::virtio_net_dev_mgr;
 
@@ -28,6 +34,10 @@ pub enum Backend {
     #[cfg(feature = "vhost-net")]
     /// Vhost-net
     Vhost(VirtioConfig),
+    #[serde(rename = "vhost-user")]
+    #[cfg(feature = "vhost-user-net")]
+    /// Vhost-user-net
+    VhostUser(VhostUserConfig),
 }
 
 impl Default for Backend {
@@ -43,6 +53,7 @@ impl Default for Backend {
 }
 
 /// Virtio network config, working for virtio-net and vhost-net.
+#[cfg(any(feature = "virtio-net", feature = "vhost-net"))]
 #[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
 pub struct VirtioConfig {
     /// ID of the guest network interface.
@@ -57,6 +68,13 @@ pub struct VirtioConfig {
     pub allow_duplicate_mac: bool,
 }
 
+#[cfg(feature = "vhost-user-net")]
+#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
+pub struct VhostUserConfig {
+    /// Vhost-user socket path.
+    pub sock_path: String,
+}
+
 /// This struct represents the strongly typed equivalent of the json body from
 /// net iface related requests.
 /// This struct works with virtio-net devices and vhost-net devices.
@@ -168,6 +186,39 @@ impl From<&NetworkInterfaceConfig> for VhostNetDeviceConfigInfo {
     }
 }
 
+#[cfg(feature = "vhost-user-net")]
+impl From<NetworkInterfaceConfig> for VhostUserNetDeviceConfigInfo {
+    fn from(value: NetworkInterfaceConfig) -> Self {
+        let self_ref = &value;
+        self_ref.into()
+    }
+}
+#[cfg(feature = "vhost-user-net")]
+impl From<&NetworkInterfaceConfig> for VhostUserNetDeviceConfigInfo {
+    fn from(value: &NetworkInterfaceConfig) -> Self {
+        let num_queues = value
+            .num_queues
+            .unwrap_or(vhost_user_net_dev_mgr::DEFAULT_NUM_QUEUES);
+        let queue_size = value
+            .queue_size
+            .unwrap_or(vhost_user_net_dev_mgr::DEFAULT_QUEUE_SIZE);
+        // It is safe because we tested the type of config before.
+        #[allow(unreachable_patterns)]
+        let config = match &value.backend {
+            Backend::VhostUser(config) => config,
+            _ => panic!("The virtio backend config is invalid: {:?}", value),
+        };
+        Self {
+            sock_path: config.sock_path.clone(),
+            num_queues,
+            queue_size,
+            guest_mac: value.guest_mac,
+            use_shared_irq: value.use_shared_irq,
+            use_generic_irq: value.use_generic_irq,
+        }
+    }
+}
+
 /// The data fed into a network iface update request. Currently, only the RX and
 /// TX rate limiters can be updated.
 #[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize, Default)]
@@ -202,13 +253,13 @@ impl From<&NetworkInterfaceUpdateConfig> for VirtioNetDeviceConfigUpdateInfo {
     }
 }
 
+#[cfg(feature = "virtio-net")]
 #[cfg(test)]
 mod tests {
     use dbs_utils::net::MacAddr;
 
-    use crate::api::v1::Backend;
-
     use super::NetworkInterfaceConfig;
+    use crate::api::v1::Backend;
 
     #[test]
     fn test_network_interface_config() {

src/dragonball/src/api/v1/vmm_action.rs

@@ -40,6 +40,10 @@ pub use crate::device_manager::mem_dev_mgr::{MemDeviceConfigInfo, MemDeviceError
 pub use crate::device_manager::vhost_net_dev_mgr::{
     VhostNetDeviceConfigInfo, VhostNetDeviceError, VhostNetDeviceMgr,
 };
+#[cfg(feature = "vhost-user-net")]
+use crate::device_manager::vhost_user_net_dev_mgr::{
+    VhostUserNetDeviceConfigInfo, VhostUserNetDeviceError, VhostUserNetDeviceMgr,
+};
 #[cfg(feature = "virtio-net")]
 pub use crate::device_manager::virtio_net_dev_mgr::{
     VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo, VirtioNetDeviceError,
@@ -110,6 +114,11 @@ pub enum VmmActionError {
     /// Vhost-net device relared errors.
     VhostNet(#[source] VhostNetDeviceError),
 
+    #[error("vhost-user-net device error: {0:?}")]
+    #[cfg(feature = "vhost-user-net")]
+    /// Vhost-user-net device relared errors.
+    VhostUserNet(#[source] VhostUserNetDeviceError),
+
     #[cfg(any(feature = "virtio-fs", feature = "vhost-user-fs"))]
     /// The action `InsertFsDevice` failed either because of bad user input or an internal error.
     #[error("virtio-fs device error: {0}")]
@@ -191,7 +200,11 @@ pub enum VmmAction {
     /// are the RX and TX rate limiters.
     UpdateBlockDevice(BlockDeviceConfigUpdateInfo),
 
-    #[cfg(any(feature = "virtio-net", feature = "vhost-net"))]
+    #[cfg(any(
+        feature = "virtio-net",
+        feature = "vhost-net",
+        feature = "vhost-user-net"
+    ))]
     /// Add a new network interface config or update one that already exists using the
     /// `NetworkInterfaceConfig` as input. This action can only be called before the microVM has
     /// booted. The response is sent using the `OutcomeSender`.
@@ -320,12 +333,20 @@ impl VmmService {
             VmmAction::RemoveBlockDevice(drive_id) => {
                 self.remove_block_device(vmm, event_mgr, &drive_id)
             }
-            #[cfg(any(feature = "virtio-net", feature = "vhost-net"))]
+            #[cfg(any(
+                feature = "virtio-net",
+                feature = "vhost-net",
+                feature = "vhost-user-net"
+            ))]
             VmmAction::InsertNetworkDevice(config) => match config.backend {
                 #[cfg(feature = "virtio-net")]
                 Backend::Virtio(_) => self.add_virtio_net_device(vmm, event_mgr, config.into()),
                 #[cfg(feature = "vhost-net")]
                 Backend::Vhost(_) => self.add_vhost_net_device(vmm, event_mgr, config.into()),
+                #[cfg(feature = "vhost-user-net")]
+                Backend::VhostUser(_) => {
+                    self.add_vhost_user_net_device(vmm, event_mgr, config.into())
+                }
             },
             #[cfg(feature = "virtio-net")]
             VmmAction::UpdateNetworkInterface(netif_update) => {
@@ -712,6 +733,30 @@ impl VmmService {
             .map_err(VmmActionError::VhostNet)
     }
 
+    #[cfg(feature = "vhost-user-net")]
+    fn add_vhost_user_net_device(
+        &mut self,
+        vmm: &mut Vmm,
+        event_mgr: &mut EventManager,
+        config: VhostUserNetDeviceConfigInfo,
+    ) -> VmmRequestResult {
+        let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?;
+        let ctx = vm
+            .create_device_op_context(Some(event_mgr.epoll_manager()))
+            .map_err(|err| {
+                if let StartMicroVmError::MicroVMAlreadyRunning = err {
+                    VmmActionError::VhostUserNet(VhostUserNetDeviceError::UpdateNotAllowedPostBoot)
+                } else if let StartMicroVmError::UpcallServerNotReady = err {
+                    VmmActionError::UpcallServerNotReady
+                } else {
+                    VmmActionError::StartMicroVm(err)
+                }
+            })?;
+        VhostUserNetDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config)
+            .map(|_| VmmData::Empty)
+            .map_err(VmmActionError::VhostUserNet)
+    }
+
     #[cfg(any(feature = "virtio-fs", feature = "vhost-user-fs"))]
     #[instrument(skip(self))]
     fn add_fs_device(&mut self, vmm: &mut Vmm, config: FsDeviceConfigInfo) -> VmmRequestResult {

src/dragonball/src/dbs_virtio_devices/Cargo.toml

@@ -57,3 +57,4 @@ vhost = ["virtio-mmio", "vhost-rs/vhost-user-master", "vhost-rs/vhost-kern"]
 vhost-net = ["vhost", "vhost-rs/vhost-net"]
 vhost-user = ["vhost"]
 vhost-user-fs = ["vhost-user"]
+vhost-user-net = ["vhost-user"]

src/dragonball/src/dbs_virtio_devices/src/vhost/mod.rs

@@ -4,15 +4,19 @@
 
 //! Vhost-based virtio device backend implementations.
 
-#[cfg(feature = "vhost-net")]
+#[cfg(feature = "vhost")]
 pub mod vhost_kern;
 
-pub use vhost_rs::vhost_user::Error as VhostUserError;
-pub use vhost_rs::Error as VhostError;
-
 #[cfg(feature = "vhost-user")]
 pub mod vhost_user;
 
+/// Common code for vhost-based network device
+#[cfg(any(feature = "vhost-net", feature = "vhost-user-net"))]
+mod net;
+
+pub use vhost_rs::vhost_user::Error as VhostUserError;
+pub use vhost_rs::Error as VhostError;
+
 impl std::convert::From<VhostError> for super::Error {
     fn from(e: VhostError) -> Self {
         super::Error::VhostError(e)

src/dragonball/src/dbs_virtio_devices/src/vhost/net.rs

@@ -0,0 +1,74 @@
+// Copyright (C) 2019-2023 Alibaba Cloud. All rights reserved.
+// Copyright (C) 2019-2023 Ant Group. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use log::{debug, error, warn};
+use virtio_bindings::bindings::virtio_net::{
+    virtio_net_ctrl_hdr, virtio_net_ctrl_mq, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
+};
+use virtio_queue::{Descriptor, DescriptorChain};
+use vm_memory::{Bytes, GuestMemory};
+
+use crate::{DbsGuestAddressSpace, Error as VirtioError, Result as VirtioResult};
+
+pub(crate) trait FromNetCtrl<T> {
+    fn from_net_ctrl_st<M: GuestMemory>(mem: &M, desc: &Descriptor) -> VirtioResult<T> {
+        let mut buf = vec![0u8; std::mem::size_of::<T>()];
+        match mem.read_slice(&mut buf, desc.addr()) {
+            Ok(_) => unsafe { Ok(std::ptr::read_volatile(&buf[..] as *const _ as *const T)) },
+            Err(err) => {
+                error!("Failed to read from memory, {}", err);
+                Err(VirtioError::InternalError)
+            }
+        }
+    }
+}
+
+impl FromNetCtrl<virtio_net_ctrl_hdr> for virtio_net_ctrl_hdr {}
+impl FromNetCtrl<virtio_net_ctrl_mq> for virtio_net_ctrl_mq {}
+
+pub(crate) fn virtio_handle_ctrl_mq<AS, F>(
+    desc_chain: &mut DescriptorChain<&AS::M>,
+    cmd: u8,
+    mem: &AS::M,
+    ctrl_mq_vq_pairs_set: F,
+) -> VirtioResult<()>
+where
+    AS: DbsGuestAddressSpace,
+    F: FnOnce(u16) -> VirtioResult<()>,
+{
+    if cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET as u8 {
+        if let Some(next) = desc_chain.next() {
+            if let Ok(ctrl_mq) = virtio_net_ctrl_mq::from_net_ctrl_st(mem, &next) {
+                let curr_queues = ctrl_mq.virtqueue_pairs;
+                ctrl_mq_vq_pairs_set(curr_queues)?;
+            }
+        }
+    }
+    Ok(())
+}
+
+pub(crate) fn virtio_handle_ctrl_status<AS>(
+    driver_name: &str,
+    desc_chain: &mut DescriptorChain<&AS::M>,
+    status: u8,
+    mem: &AS::M,
+) -> VirtioResult<u32>
+where
+    AS: DbsGuestAddressSpace,
+{
+    let buf = vec![status];
+    let mut total = 0;
+    for next in desc_chain {
+        if next.is_write_only() {
+            match mem.write_slice(&buf, next.addr()) {
+                Ok(_) => {
+                    debug!("{}: succeed to update virtio ctrl status!", driver_name);
+                    total += 1;
+                }
+                Err(_) => warn!("{}: failed to update ctrl status!", driver_name),
+            }
+        }
+    }
+    Ok(total)
+}

src/dragonball/src/dbs_virtio_devices/src/vhost/vhost_kern/net.rs

@@ -28,9 +28,10 @@ use vhost_rs::VhostBackend;
 use vhost_rs::{VhostUserMemoryRegionInfo, VringConfigData};
 use virtio_bindings::bindings::virtio_net::*;
 use virtio_bindings::bindings::virtio_ring::*;
-use virtio_queue::{Descriptor, DescriptorChain, QueueT};
+use virtio_queue::{DescriptorChain, QueueT};
-use vm_memory::{Address, Bytes, GuestMemory, GuestMemoryRegion, MemoryRegionAddress};
+use vm_memory::{Address, GuestMemory, GuestMemoryRegion, MemoryRegionAddress};
 
+use crate::vhost::net::{virtio_handle_ctrl_mq, virtio_handle_ctrl_status, FromNetCtrl};
 #[cfg(test)]
 use crate::vhost::vhost_kern::test_utils::{
     MockVhostBackend as VhostBackend, MockVhostNet as VhostNet,
@@ -592,22 +593,6 @@ where
     }
 }
 
-trait FromNetCtrl<T> {
-    fn from_net_ctrl_st<M: GuestMemory>(mem: &M, desc: &Descriptor) -> VirtioResult<T> {
-        let mut buf = vec![0u8; std::mem::size_of::<T>()];
-        match mem.read_slice(&mut buf, desc.addr()) {
-            Ok(_) => unsafe { Ok(std::ptr::read_volatile(&buf[..] as *const _ as *const T)) },
-            Err(err) => {
-                error!("Failed to read from memory, {}", err);
-                Err(VirtioError::InternalError)
-            }
-        }
-    }
-}
-
-impl FromNetCtrl<virtio_net_ctrl_hdr> for virtio_net_ctrl_hdr {}
-impl FromNetCtrl<virtio_net_ctrl_mq> for virtio_net_ctrl_mq {}
-
 pub(crate) struct NetEpollHandler<AS, Q, R>
 where
     AS: DbsGuestAddressSpace,
@@ -627,6 +612,8 @@ where
     fn process_ctrl_request(&mut self) -> VirtioResult<()> {
         let guard = self.config.lock_guest_memory();
         let mem = guard.deref();
+        // It is safty to unwrap here as the value of ctrl_queue is
+        // confirmed in `CTRL_SLOT`.
         let cvq = self.config.ctrl_queue.as_mut().unwrap();
 
         while let Some(mut desc_chain) = cvq.get_next_descriptor(mem)? {
@@ -657,8 +644,16 @@ where
             let ctrl_hdr = virtio_net_ctrl_hdr::from_net_ctrl_st(mem, &header)?;
             match ctrl_hdr.class as u32 {
                 VIRTIO_NET_CTRL_MQ => {
-                    Self::virtio_handle_ctrl_mq(desc_chain, ctrl_hdr.cmd, mem)?;
+                    virtio_handle_ctrl_mq::<AS, _>(desc_chain, ctrl_hdr.cmd, mem, |curr_queues| {
-                    return Self::virtio_handle_ctrl_status(desc_chain, VIRTIO_NET_OK as u8, mem);
+                        info!("{}: vq pairs: {}", NET_DRIVER_NAME, curr_queues);
+                        Ok(())
+                    })?;
+                    return virtio_handle_ctrl_status::<AS>(
+                        NET_DRIVER_NAME,
+                        desc_chain,
+                        VIRTIO_NET_OK as u8,
+                        mem,
+                    );
                 }
                 _ => error!(
                     "{}: unknown net control request class: 0x{:x}",
@@ -668,45 +663,6 @@ where
         }
         Ok(0)
     }
-
-    fn virtio_handle_ctrl_mq(
-        desc_chain: &mut DescriptorChain<&AS::M>,
-        cmd: u8,
-        mem: &AS::M,
-    ) -> VirtioResult<()> {
-        if cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET as u8 {
-            if let Some(next) = desc_chain.next() {
-                if let Ok(ctrl_mq) = virtio_net_ctrl_mq::from_net_ctrl_st(mem, &next) {
-                    let curr_queues = ctrl_mq.virtqueue_pairs;
-                    info!("{}: vq pairs: {}", NET_DRIVER_NAME, curr_queues);
-                }
-            }
-        }
-        Ok(())
-    }
-
-    fn virtio_handle_ctrl_status(
-        desc_chain: &mut DescriptorChain<&AS::M>,
-        status: u8,
-        mem: &AS::M,
-    ) -> VirtioResult<u32> {
-        let buf = vec![status];
-        let mut total = 0;
-
-        for next in desc_chain {
-            if next.is_write_only() {
-                match mem.write_slice(&buf, next.addr()) {
-                    Ok(_) => {
-                        debug!("{}: succeed to update virtio ctrl status!", NET_DRIVER_NAME);
-                        total += 1;
-                    }
-                    Err(_) => warn!("{}: failed to update ctrl status!", NET_DRIVER_NAME),
-                }
-            }
-        }
-
-        Ok(total)
-    }
 }
 
 impl<AS, Q, R> MutEventSubscriber for NetEpollHandler<AS, Q, R>

src/dragonball/src/dbs_virtio_devices/src/vhost/vhost_user/connection.rs

@@ -12,7 +12,7 @@ use vhost_rs::vhost_user::message::{VhostUserProtocolFeatures, VhostUserVringAdd
 use vhost_rs::vhost_user::{
     Error as VhostUserError, Listener as VhostUserListener, Master, VhostUserMaster,
 };
-use vhost_rs::{VhostBackend, VhostUserMemoryRegionInfo, VringConfigData};
+use vhost_rs::{Error as VhostError, VhostBackend, VhostUserMemoryRegionInfo, VringConfigData};
 use virtio_bindings::bindings::virtio_net::VIRTIO_F_RING_PACKED;
 use virtio_queue::QueueT;
 use vm_memory::{
@@ -20,9 +20,8 @@ use vm_memory::{
 };
 use vmm_sys_util::eventfd::EventFd;
 
-use super::super::super::device::VirtioDeviceConfig;
+use crate::device::VirtioDeviceConfig;
-use super::super::super::{Error as VirtioError, Result as VirtioResult};
+use crate::{Error as VirtioError, Result as VirtioResult};
-use super::VhostError;
 
 enum EndpointProtocolFlags {
     ProtocolMq = 1,
@@ -100,7 +99,9 @@ impl Listener {
     }
 }
 
+/// TODO: Mark as unused as it is used by vhost-user-block devices only.
 /// Struct to pass info to vhost user backend
+#[allow(dead_code)]
 #[derive(Clone)]
 pub struct BackendInfo {
     /// -1 means to tell backend to destroy corresponding
@@ -123,6 +124,8 @@ pub(super) struct EndpointParam<'a, AS: GuestAddressSpace, Q: QueueT, R: GuestMe
     pub protocol_flag: u16,
     pub dev_protocol_features: VhostUserProtocolFeatures,
     pub reconnect: bool,
+    // TODO: Mark as unused as it is used by vhost-user-block only.
+    #[allow(dead_code)]
     pub backend: Option<BackendInfo>,
     pub init_queues: u32,
     pub slave_req_fd: Option<RawFd>,
@@ -195,6 +198,9 @@ impl Endpoint {
         Ok(Some(features))
     }
 
+    // TODO: Remove this after enabling vhost-user-fs on the runtime-rs. Issue:
+    // https://github.com/kata-containers/kata-containers/issues/8691
+    #[allow(dead_code)]
     pub fn update_memory<AS: GuestAddressSpace>(&mut self, vm_as: &AS) -> VirtioResult<()> {
         let master = match self.conn.as_mut() {
             Some(conn) => conn,

src/dragonball/src/dbs_virtio_devices/src/vhost/vhost_user/mod.rs

@@ -3,11 +3,12 @@
 
 //! Vhost-based virtio device backend implementations.
 
-use super::VhostError;
-
 pub mod connection;
 #[cfg(feature = "vhost-user-fs")]
 pub mod fs;
 
+#[cfg(feature = "vhost-user-net")]
+pub mod net;
+
 #[cfg(test)]
 mod test_utils;

src/dragonball/src/dbs_virtio_devices/src/vhost/vhost_user/net.rs

@@ -0,0 +1,795 @@
+// Copyright (C) 2019-2023 Alibaba Cloud. All rights reserved.
+// Copyright (C) 2019-2023 Ant Group. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::marker::PhantomData;
+use std::ops::Deref;
+use std::sync::{Arc, Mutex, MutexGuard};
+use std::time::Duration;
+
+use dbs_device::resources::ResourceConstraint;
+use dbs_utils::epoll_manager::{EpollManager, EventOps, Events, MutEventSubscriber, SubscriberId};
+use dbs_utils::net::MacAddr;
+use log::{debug, error, info, trace, warn};
+use vhost_rs::vhost_user::{
+    Error as VhostUserError, Master, VhostUserProtocolFeatures, VhostUserVirtioFeatures,
+};
+use vhost_rs::Error as VhostError;
+use virtio_bindings::bindings::virtio_net::{
+    virtio_net_ctrl_hdr, VIRTIO_NET_CTRL_MQ, VIRTIO_NET_F_CTRL_MAC_ADDR, VIRTIO_NET_F_CTRL_RX,
+    VIRTIO_NET_F_CTRL_VLAN, VIRTIO_NET_F_CTRL_VQ, VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MAC,
+    VIRTIO_NET_F_MQ, VIRTIO_NET_F_MTU, VIRTIO_NET_OK, VIRTIO_NET_S_LINK_UP,
+};
+use virtio_queue::{DescriptorChain, QueueT};
+use vm_memory::GuestMemoryRegion;
+use vmm_sys_util::epoll::EventSet;
+
+use super::connection::{Endpoint, Listener};
+use crate::vhost::net::{virtio_handle_ctrl_mq, virtio_handle_ctrl_status, FromNetCtrl};
+use crate::vhost::vhost_user::connection::EndpointParam;
+use crate::{
+    ActivateResult, ConfigResult, DbsGuestAddressSpace, Error as VirtioError,
+    Result as VirtioResult, VirtioDevice, VirtioDeviceConfig, VirtioDeviceInfo, TYPE_NET,
+};
+
+const NET_DRIVER_NAME: &str = "vhost-user-net";
+// Epoll token for incoming connection on the Unix Domain Socket listener.
+const LISTENER_SLOT: u32 = 0;
+// Epoll token for monitoring the Unix Domain Socket between the master and
+// the slave.
+const MASTER_SLOT: u32 = 1;
+// Epoll token for control queue
+const CTRL_SLOT: u32 = 2;
+// Control queue count
+const CTRL_QUEUE_NUM: u16 = 64;
+
+/// An implementation of vhost-user-net device
+struct VhostUserNetDevice {
+    /// Fixed value: "vhost-user-net".
+    id: String,
+    device_info: VirtioDeviceInfo,
+    /// Unix domain socket connecting to the vhost-user slave.
+    endpoint: Endpoint,
+    /// Unix domain socket listener to accept incoming connection from the slave.
+    listener: Listener,
+    /// current enabled queues with vhost-user slave
+    curr_queues: u32,
+}
+
+impl VhostUserNetDevice {
+    fn new(
+        master: Master,
+        mut avail_features: u64,
+        listener: Listener,
+        guest_mac: Option<&MacAddr>,
+        queue_sizes: Arc<Vec<u16>>,
+        epoll_mgr: EpollManager,
+    ) -> VirtioResult<Self> {
+        // hard-coding MTU
+        info!(
+            "{}: slave support features 0x{:x}",
+            NET_DRIVER_NAME, avail_features
+        );
+        avail_features |= (1 << VIRTIO_NET_F_MTU) as u64;
+        // All these features depends on availability of control
+        // channel (VIRTIO_NET_F_CTRL_VQ).
+        avail_features &= !(1 << VIRTIO_NET_F_CTRL_VQ
+            | 1 << VIRTIO_NET_F_CTRL_RX
+            | 1 << VIRTIO_NET_F_CTRL_VLAN
+            | 1 << VIRTIO_NET_F_GUEST_ANNOUNCE
+            | 1 << VIRTIO_NET_F_MQ
+            | 1 << VIRTIO_NET_F_CTRL_MAC_ADDR) as u64;
+        // Multi-queue features
+        if queue_sizes.len() > 2 {
+            avail_features |= (1 << VIRTIO_NET_F_MQ | 1 << VIRTIO_NET_F_CTRL_VQ) as u64;
+        }
+        // Network device configuration layout:
+        // https://docs.oasis-open.org/virtio/virtio/v1.1/csprd01/virtio-v1.1-csprd01.html#x1-2000004
+        // - [u8; 6]: mac address
+        // - u16: status
+        // - u16: max_virtqueue_pairs
+        // - u16: mtu
+        // - u32: speed
+        // - u8: duplex
+        let mut config_space = vec![0u8; 17];
+        if let Some(mac) = guest_mac {
+            // When this feature isn't available, the driver generates a random
+            // MAC address. Otherwise, it should attempt to read the device MAC
+            // address from the config space.
+            avail_features |= 1u64 << VIRTIO_NET_F_MAC;
+            config_space[0..6].copy_from_slice(mac.get_bytes());
+        } else {
+            avail_features &= !(1 << VIRTIO_NET_F_MAC) as u64;
+        }
+        // status: mark link as up
+        config_space[6] = VIRTIO_NET_S_LINK_UP as u8;
+        config_space[7] = 0;
+        // max_virtqueue_pairs: only support one rx/tx pair
+        config_space[8] = (queue_sizes.len() / 2) as u8;
+        config_space[9] = 0;
+        // mtu: 1500 = 1536 - vxlan header?
+        config_space[10] = 220;
+        config_space[11] = 5;
+        // speed: 1000Mb
+        config_space[12] = 232;
+        config_space[13] = 3;
+        config_space[14] = 0;
+        config_space[15] = 0;
+        // duplex: full duplex: 0x01
+        config_space[16] = 1;
+        Ok(VhostUserNetDevice {
+            id: NET_DRIVER_NAME.to_owned(),
+            device_info: VirtioDeviceInfo::new(
+                NET_DRIVER_NAME.to_owned(),
+                avail_features,
+                queue_sizes,
+                config_space,
+                epoll_mgr,
+            ),
+            endpoint: Endpoint::new(master, MASTER_SLOT, NET_DRIVER_NAME.to_owned()),
+            listener,
+            curr_queues: 2,
+        })
+    }
+
+    /// Create a vhost-user-net server instance.
+    /// The function will hang on until a connection is established with a slave.
+    fn new_server(
+        path: &str,
+        guest_mac: Option<&MacAddr>,
+        queue_sizes: Arc<Vec<u16>>,
+        epoll_mgr: EpollManager,
+    ) -> VirtioResult<Self> {
+        info!(
+            "{}: creating Unix Domain Socket listener...",
+            NET_DRIVER_NAME
+        );
+
+        let listener = Listener::new(
+            NET_DRIVER_NAME.to_string(),
+            path.to_string(),
+            true,
+            LISTENER_SLOT,
+        )?;
+
+        info!(
+            "{}: waiting for incoming connection from the slave...",
+            NET_DRIVER_NAME
+        );
+        let (master, avail_features) = listener.accept()?;
+        info!("{}: connection to slave is ready.", NET_DRIVER_NAME);
+
+        Self::new(
+            master,
+            avail_features,
+            listener,
+            guest_mac,
+            queue_sizes,
+            epoll_mgr,
+        )
+    }
+
+    fn activate_slave<AS, Q, R>(
+        &mut self,
+        handler: &VhostUserNetHandler<AS, Q, R>,
+    ) -> ActivateResult
+    where
+        AS: DbsGuestAddressSpace,
+        Q: QueueT + Send + 'static,
+        R: GuestMemoryRegion + Sync + Send + 'static,
+    {
+        trace!(target: "vhost-net", "{}: VhostUserNetDevice::activate_slave()", self.id);
+        let mut config = EndpointParam {
+            virtio_config: &handler.config,
+            intr_evts: handler.config.get_queue_interrupt_eventfds(),
+            queue_sizes: &self.device_info.queue_sizes,
+            features: self.get_acked_features(),
+            protocol_flag: 0,
+            dev_protocol_features: Self::get_dev_protocol_features(),
+            reconnect: false,
+            backend: None,
+            init_queues: self.curr_queues,
+            slave_req_fd: None,
+        };
+        config.set_protocol_mq();
+        // Do negotiate with the vhost-user slave
+        loop {
+            match self.endpoint.negotiate(&config, None) {
+                Ok(_) => break,
+                Err(VirtioError::VhostError(VhostError::VhostUserProtocol(err))) => {
+                    if err.should_reconnect() {
+                        // Fall through to rebuild the connection.
+                        warn!(
+                            "{}: socket disconnected while initializing the connnection, {}",
+                            self.id, err
+                        );
+                    } else {
+                        error!("{}: failed to setup connection, {}", self.id, err);
+                        return Err(VhostError::VhostUserProtocol(err).into());
+                    }
+                }
+                Err(err) => {
+                    error!("{}: failed to setup connection, {}", self.id, err);
+                    return Err(err.into());
+                }
+            }
+            // Do reconnect
+            // Wait 100ms for the next connection
+            let delay = Duration::from_millis(100);
+            std::thread::sleep(delay);
+            // The underlying communication channel has been disconnected,
+            // recreate it again.
+            let (master, avail_features) = self.listener.accept()?;
+            if !avail_features & self.device_info.acked_features() != 0 {
+                error!("{}: Virtio features changed when reconnecting, avail features: 0x{:X}, acked features: 0x{:X}.", 
+                    self.id, avail_features, self.device_info.acked_features());
+                return Err(VhostError::VhostUserProtocol(VhostUserError::FeatureMismatch).into());
+            }
+            self.endpoint.set_master(master);
+        }
+        Ok(())
+    }
+
+    fn get_acked_features(&self) -> u64 {
+        // typical features: 0x17c6effcb
+        // typical acked_features: 0x17000FF8B
+        // typical protocol features: 0x37
+        let mut features = self.device_info.acked_features();
+        // Enable support of vhost-user protocol features if available
+        features |=
+            self.device_info.avail_features() & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
+        features
+    }
+
+    /// Vhost-user protocol features this device supports
+    fn get_dev_protocol_features() -> VhostUserProtocolFeatures {
+        VhostUserProtocolFeatures::MQ
+    }
+
+    fn handle_connect<AS, Q, R>(
+        &mut self,
+        ops: &mut EventOps,
+        _events: Events,
+        handler: &VhostUserNetHandler<AS, Q, R>,
+    ) -> VirtioResult<()>
+    where
+        AS: DbsGuestAddressSpace,
+        Q: QueueT + Send + 'static,
+        R: GuestMemoryRegion + Sync + Send + 'static,
+    {
+        info!("{}: try to accept new socket for reconnect...", self.id);
+        match self.listener.try_accept() {
+            Ok(Some((master, _avail_features))) => {
+                let mut config = EndpointParam {
+                    virtio_config: &handler.config,
+                    intr_evts: handler.config.get_queue_interrupt_eventfds(),
+                    queue_sizes: &self.device_info.queue_sizes,
+                    features: self.get_acked_features(),
+                    protocol_flag: 0,
+                    dev_protocol_features: VhostUserNetDevice::get_dev_protocol_features(),
+                    reconnect: true,
+                    backend: None,
+                    init_queues: self.curr_queues,
+                    slave_req_fd: None,
+                };
+                config.set_protocol_mq();
+                self.endpoint.reconnect(master, &config, ops)?;
+                info!("{}: communication channel has been recovered.", self.id);
+                Ok(())
+            }
+            Ok(None) => {
+                warn!(
+                    "{}: no incoming connection available when handle incoming connection",
+                    self.id
+                );
+                Ok(())
+            }
+            Err(_) => {
+                warn!("{}: no incoming connection available", self.id);
+                Err(VirtioError::InternalError)
+            }
+        }
+    }
+
+    fn handle_disconnect(&mut self, ops: &mut EventOps) -> VirtioResult<()> {
+        trace!(target: "vhost-net", "{}: VhostUserNetDevice::handle_disconnect()", self.id);
+        self.endpoint.disconnect(ops)
+    }
+
+    fn handle_set_queues(&mut self, queue_pairs: u32) -> VirtioResult<()> {
+        trace!(target: "vhost-net", "{}: VhostUserNetDevice::handle_set_queues({})", self.id, queue_pairs);
+        self.curr_queues = queue_pairs * 2;
+        debug!("{}: set multi-queue to {}", self.id, self.curr_queues);
+        loop {
+            match self.endpoint.set_queues_attach(self.curr_queues) {
+                Ok(_) => break,
+                Err(VirtioError::VhostError(VhostError::VhostUserProtocol(err))) => {
+                    if err.should_reconnect() {
+                        warn!(
+                            "{}: socket disconnected while initializing the connnection: {}",
+                            self.id, err
+                        );
+                    } else {
+                        error!("{}: failed to setup connection: {}", self.id, err);
+                        return Err(VhostError::VhostUserProtocol(err).into());
+                    }
+                }
+                Err(err) => {
+                    error!("{}: failed to setup connection: {}", self.id, err);
+                    return Err(err);
+                }
+            }
+            // Do reconnect
+            // Wait 100ms for the next connection
+            let delay = Duration::from_millis(100);
+            std::thread::sleep(delay);
+            // The underlying communication channel has been disconnected,
+            // recreate it again.
+            let (master, avail_features) = self.listener.accept()?;
+            if !avail_features & self.device_info.acked_features() != 0 {
+                error!("{}: Virtio features changed when reconnecting, avail features: 0x{:X}, acked features: 0x{:X}.", 
+                    self.id, avail_features, self.device_info.acked_features());
+                return Err(VhostError::VhostUserProtocol(VhostUserError::FeatureMismatch).into());
+            }
+            self.endpoint.set_master(master);
+        }
+        Ok(())
+    }
+}
+
+#[derive(Clone)]
+pub struct VhostUserNet<AS>
+where
+    AS: DbsGuestAddressSpace,
+{
+    id: String,
+    device: Arc<Mutex<VhostUserNetDevice>>,
+    queue_sizes: Arc<Vec<u16>>,
+    ctrl_queue_sizes: u16,
+    subscriber_id: Option<SubscriberId>,
+    phantom: PhantomData<AS>,
+}
+
+impl<AS> VhostUserNet<AS>
+where
+    AS: DbsGuestAddressSpace,
+{
+    /// Create a new vhost-user net device.
+    ///
+    /// Create a Unix Domain Socket listener and wait until the the first incoming connection is
+    /// ready. The listener will be used to accept new incoming connections when the current
+    /// connection gets broken.
+    pub fn new_server(
+        path: &str,
+        guest_mac: Option<&MacAddr>,
+        queue_sizes: Arc<Vec<u16>>,
+        epoll_mgr: EpollManager,
+    ) -> VirtioResult<Self> {
+        let device =
+            VhostUserNetDevice::new_server(path, guest_mac, queue_sizes.clone(), epoll_mgr)?;
+        let ctrl_queue_sizes = if queue_sizes.len() > 2 {
+            CTRL_QUEUE_NUM
+        } else {
+            0
+        };
+        let id = device.device_info.driver_name.clone();
+        Ok(VhostUserNet {
+            id,
+            device: Arc::new(Mutex::new(device)),
+            queue_sizes,
+            ctrl_queue_sizes,
+            subscriber_id: None,
+            phantom: PhantomData,
+        })
+    }
+
+    fn device(&self) -> MutexGuard<VhostUserNetDevice> {
+        // Do not expect poisoned lock.
+        self.device.lock().unwrap()
+    }
+}
+
+impl<AS, Q, R> VirtioDevice<AS, Q, R> for VhostUserNet<AS>
+where
+    AS: DbsGuestAddressSpace,
+    Q: QueueT + Send + 'static,
+    R: GuestMemoryRegion + Sync + Send + 'static,
+{
+    fn device_type(&self) -> u32 {
+        TYPE_NET
+    }
+
+    fn queue_max_sizes(&self) -> &[u16] {
+        &self.queue_sizes
+    }
+
+    fn ctrl_queue_max_sizes(&self) -> u16 {
+        self.ctrl_queue_sizes
+    }
+
+    fn get_avail_features(&self, page: u32) -> u32 {
+        self.device().device_info.get_avail_features(page)
+    }
+
+    fn set_acked_features(&mut self, page: u32, value: u32) {
+        trace!(target: "vhost-net", "{}: VirtioDevice::set_acked_features({}, 0x{:x})",
+               self.id, page, value);
+        self.device().device_info.set_acked_features(page, value)
+    }
+
+    fn read_config(&mut self, offset: u64, data: &mut [u8]) -> ConfigResult {
+        trace!(target: "vhost-net", "{}: VirtioDevice::read_config(0x{:x}, {:?})",
+               self.id, offset, data);
+        self.device().device_info.read_config(offset, data)?;
+        Ok(())
+    }
+
+    fn write_config(&mut self, offset: u64, data: &[u8]) -> ConfigResult {
+        trace!(target: "vhost-net", "{}: VirtioDevice::write_config(0x{:x}, {:?})",
+               self.id, offset, data);
+        self.device().device_info.write_config(offset, data)?;
+        Ok(())
+    }
+
+    fn activate(&mut self, config: VirtioDeviceConfig<AS, Q, R>) -> ActivateResult {
+        trace!(target: "vhost-net", "{}: VirtioDevice::activate()", self.id);
+        let mut device = self.device();
+        device.device_info.check_queue_sizes(&config.queues)?;
+        let handler = VhostUserNetHandler {
+            device: self.device.clone(),
+            config,
+            id: self.id.clone(),
+        };
+        device.activate_slave(&handler)?;
+        let epoll_mgr = device.device_info.epoll_manager.clone();
+        drop(device);
+        self.subscriber_id = Some(epoll_mgr.add_subscriber(Box::new(handler)));
+        Ok(())
+    }
+
+    fn get_resource_requirements(
+        &self,
+        requests: &mut Vec<dbs_device::resources::ResourceConstraint>,
+        use_generic_irq: bool,
+    ) {
+        trace!(target: "vhost-net", "{}: VirtioDevice::get_resource_requirements()", self.id);
+        requests.push(ResourceConstraint::LegacyIrq { irq: None });
+        if use_generic_irq {
+            // Allocate one irq for device configuration change events, and
+            // one irq for each queue.
+            requests.push(ResourceConstraint::GenericIrq {
+                size: (self.queue_sizes.len() + 1) as u32,
+            });
+        }
+    }
+
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn as_any_mut(&mut self) -> &mut dyn std::any::Any {
+        self
+    }
+}
+
+pub struct VhostUserNetHandler<AS, Q, R>
+where
+    AS: DbsGuestAddressSpace,
+    Q: QueueT + Send + 'static,
+    R: GuestMemoryRegion + Sync + Send + 'static,
+{
+    device: Arc<Mutex<VhostUserNetDevice>>,
+    config: VirtioDeviceConfig<AS, Q, R>,
+    id: String,
+}
+
+impl<AS, Q, R> VhostUserNetHandler<AS, Q, R>
+where
+    AS: DbsGuestAddressSpace,
+    Q: QueueT + Send + 'static,
+    R: GuestMemoryRegion + Sync + Send + 'static,
+{
+    fn device(&self) -> MutexGuard<VhostUserNetDevice> {
+        // Do not expect poisoned lock here
+        self.device.lock().unwrap()
+    }
+
+    fn process_ctrl_request(&mut self) -> VirtioResult<()> {
+        let guard = self.config.lock_guest_memory();
+        let mem = guard.deref();
+        // It is safty to unwrap here as the value of ctrl_queue is
+        // confirmed in `CTRL_SLOT`.
+        let cvq = self.config.ctrl_queue.as_mut().unwrap();
+        let device = self.device.clone();
+        while let Some(mut desc_chain) = cvq.get_next_descriptor(mem)? {
+            let len = match Self::process_ctrl_desc(&mut desc_chain, &device, mem) {
+                Ok(len) => {
+                    debug!("{}: process ctrl desc succeed!", self.id);
+                    len
+                }
+                Err(e) => {
+                    debug!(
+                        "{}: failed to process control queue request, {}",
+                        self.id, e
+                    );
+                    0
+                }
+            };
+            cvq.add_used(mem, desc_chain.head_index(), len);
+        }
+        Ok(())
+    }
+
+    fn process_ctrl_desc(
+        desc_chain: &mut DescriptorChain<&AS::M>,
+        device: &Arc<Mutex<VhostUserNetDevice>>,
+        mem: &AS::M,
+    ) -> VirtioResult<u32> {
+        if let Some(header) = desc_chain.next() {
+            let ctrl_hdr = virtio_net_ctrl_hdr::from_net_ctrl_st(mem, &header)?;
+            match ctrl_hdr.class as u32 {
+                VIRTIO_NET_CTRL_MQ => {
+                    virtio_handle_ctrl_mq::<AS, _>(desc_chain, ctrl_hdr.cmd, mem, |curr_queues| {
+                        device.lock().unwrap().handle_set_queues(curr_queues as u32)
+                    })?;
+                    return virtio_handle_ctrl_status::<AS>(
+                        NET_DRIVER_NAME,
+                        desc_chain,
+                        VIRTIO_NET_OK as u8,
+                        mem,
+                    );
+                }
+                _ => error!(
+                    "{}: unknown net control request class: 0x{:x}",
+                    NET_DRIVER_NAME, ctrl_hdr.class
+                ),
+            }
+        }
+        Ok(0)
+    }
+}
+
+impl<AS, Q, R> MutEventSubscriber for VhostUserNetHandler<AS, Q, R>
+where
+    AS: DbsGuestAddressSpace,
+    Q: QueueT + Send + 'static,
+    R: GuestMemoryRegion + Sync + Send + 'static,
+{
+    fn process(&mut self, events: Events, ops: &mut EventOps) {
+        match events.data() {
+            LISTENER_SLOT => {
+                if let Err(err) = self.device().handle_connect(ops, events, self) {
+                    warn!(
+                        "{}: failed to accept incoming connection, {:?}",
+                        self.id, err
+                    );
+                }
+            }
+            MASTER_SLOT => {
+                if let Err(e) = self.device().handle_disconnect(ops) {
+                    warn!("{}: failed to handle disconnect event, {:?}", self.id, e);
+                }
+            }
+            CTRL_SLOT => {
+                if let Some(config) = self.config.ctrl_queue.as_ref() {
+                    if let Err(err) = config.consume_event() {
+                        error!("{}: failed to read eventfd, {:?}", self.id, err);
+                    } else if let Err(err) = self.process_ctrl_request() {
+                        error!(
+                            "{}: failed to handle control queue request, {:?}",
+                            self.id, err
+                        );
+                    }
+                }
+            }
+            _ => warn!("{}: unknown epoll event slot {}", self.id, events.data()),
+        }
+    }
+
+    fn init(&mut self, ops: &mut EventOps) {
+        trace!(target: "vhost-net", "{}: VhostUserFsHandler::init()", self.id);
+        // Do not detect socket disconnection between the time window to
+        // register epoll events. Though it has dependency on the presence
+        // of self.connection, but it doesn't really send/receive data
+        // through the socket, so delay the detection of disconnect to the
+        // registered connection monitor handler.
+        let device = self.device();
+        if let Err(err) = device.endpoint.register_epoll_event(ops) {
+            error!(
+                "{}: failed to register epoll event for endpoint, {:?}",
+                self.id, err
+            );
+        }
+        if let Err(e) = device.listener.register_epoll_event(ops) {
+            error!(
+                "{}: failed to register epoll event for listener, {:?}",
+                self.id, e
+            );
+        }
+        if let Some(config) = self.config.ctrl_queue.as_ref() {
+            let event = Events::with_data(&config.eventfd, CTRL_SLOT, EventSet::IN);
+            if let Err(err) = ops.add(event) {
+                error!(
+                    "{}: failed to register epoll event for control queue, {:?}",
+                    self.id, err
+                );
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+    use std::thread;
+
+    use dbs_device::resources::DeviceResources;
+    use dbs_interrupt::{InterruptManager, InterruptSourceType, MsiNotifier, NoopNotifier};
+    use dbs_utils::epoll_manager::EpollManager;
+    use kvm_ioctls::Kvm;
+    use vhost_rs::vhost_user::message::VhostUserU64;
+    use vhost_rs::vhost_user::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
+    use virtio_queue::QueueSync;
+    use vm_memory::{FileOffset, GuestAddress, GuestMemoryMmap, GuestRegionMmap};
+    use vmm_sys_util::tempfile::TempFile;
+
+    use crate::tests::create_address_space;
+    use crate::vhost::vhost_user::net::VhostUserNet;
+    use crate::vhost::vhost_user::test_utils::{
+        negotiate_slave, Endpoint, MasterReq, VhostUserMsgHeader,
+    };
+    use crate::{VirtioDevice, VirtioDeviceConfig, VirtioQueueConfig, TYPE_NET};
+
+    fn connect_slave(path: &str) -> Option<Endpoint<MasterReq>> {
+        let mut retry_count = 5;
+        loop {
+            match Endpoint::<MasterReq>::connect(path) {
+                Ok(endpoint) => return Some(endpoint),
+                Err(_) => {
+                    if retry_count > 0 {
+                        std::thread::sleep(std::time::Duration::from_millis(100));
+                        retry_count -= 1;
+                        continue;
+                    } else {
+                        return None;
+                    }
+                }
+            }
+        }
+    }
+
+    fn create_vhost_user_net_slave(slave: &mut Endpoint<MasterReq>) {
+        let (hdr, rfds) = slave.recv_header().unwrap();
+        assert_eq!(hdr.get_code(), MasterReq::GET_FEATURES);
+        assert!(rfds.is_none());
+        let vfeatures = 0x15 | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
+        let hdr = VhostUserMsgHeader::new(MasterReq::GET_FEATURES, 0x4, 8);
+        let msg = VhostUserU64::new(vfeatures);
+        slave.send_message(&hdr, &msg, None).unwrap();
+    }
+
+    #[test]
+    fn test_vhost_user_net_virtio_device_normal() {
+        let device_socket = "/tmp/vhost.1";
+        let queue_sizes = Arc::new(vec![128]);
+        let epoll_mgr = EpollManager::default();
+        let handler = thread::spawn(move || {
+            let mut slave = connect_slave(device_socket).unwrap();
+            create_vhost_user_net_slave(&mut slave);
+        });
+        let mut dev: VhostUserNet<Arc<GuestMemoryMmap>> =
+            VhostUserNet::new_server(device_socket, None, queue_sizes, epoll_mgr).unwrap();
+        assert_eq!(
+            VirtioDevice::<Arc<GuestMemoryMmap<()>>, QueueSync, GuestRegionMmap>::device_type(&dev),
+            TYPE_NET
+        );
+        let queue_size = vec![128];
+        assert_eq!(
+            VirtioDevice::<Arc<GuestMemoryMmap<()>>, QueueSync, GuestRegionMmap>::queue_max_sizes(
+                &dev
+            ),
+            &queue_size[..]
+        );
+        assert_eq!(
+            VirtioDevice::<Arc<GuestMemoryMmap<()>>, QueueSync, GuestRegionMmap>::get_avail_features(&dev, 0),
+            dev.device().device_info.get_avail_features(0)
+        );
+        assert_eq!(
+            VirtioDevice::<Arc<GuestMemoryMmap<()>>, QueueSync, GuestRegionMmap>::get_avail_features(&dev, 1),
+            dev.device().device_info.get_avail_features(1)
+        );
+        assert_eq!(
+            VirtioDevice::<Arc<GuestMemoryMmap<()>>, QueueSync, GuestRegionMmap>::get_avail_features(&dev, 2),
+            dev.device().device_info.get_avail_features(2)
+        );
+        VirtioDevice::<Arc<GuestMemoryMmap<()>>, QueueSync, GuestRegionMmap>::set_acked_features(
+            &mut dev, 2, 0,
+        );
+        assert_eq!(VirtioDevice::<Arc<GuestMemoryMmap<()>>, QueueSync, GuestRegionMmap>::get_avail_features(&dev, 2), 0);
+        let config: [u8; 8] = [0; 8];
+        VirtioDevice::<Arc<GuestMemoryMmap<()>>, QueueSync, GuestRegionMmap>::write_config(
+            &mut dev, 0, &config,
+        );
+        let mut data: [u8; 8] = [1; 8];
+        VirtioDevice::<Arc<GuestMemoryMmap<()>>, QueueSync, GuestRegionMmap>::read_config(
+            &mut dev, 0, &mut data,
+        );
+        assert_eq!(config, data);
+        handler.join().unwrap();
+    }
+
+    #[test]
+    fn test_vhost_user_net_virtio_device_activate() {
+        let device_socket = "/tmp/vhost.1";
+        let queue_sizes = Arc::new(vec![128]);
+        let epoll_mgr = EpollManager::default();
+        let handler = thread::spawn(move || {
+            let mut slave = connect_slave(device_socket).unwrap();
+            create_vhost_user_net_slave(&mut slave);
+            let mut pfeatures = VhostUserProtocolFeatures::all();
+            // A workaround for no support for `INFLIGHT_SHMFD`. File an issue to track
+            // this: https://github.com/kata-containers/kata-containers/issues/8705.
+            pfeatures -= VhostUserProtocolFeatures::INFLIGHT_SHMFD;
+            negotiate_slave(&mut slave, pfeatures, true, 1);
+        });
+        let mut dev: VhostUserNet<Arc<GuestMemoryMmap>> =
+            VhostUserNet::new_server(device_socket, None, queue_sizes, epoll_mgr).unwrap();
+        // invalid queue size
+        {
+            let kvm = Kvm::new().unwrap();
+            let vm_fd = Arc::new(kvm.create_vm().unwrap());
+            let mem = GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
+            let resources = DeviceResources::new();
+            let queues = vec![
+                VirtioQueueConfig::create(128, 0).unwrap(),
+                VirtioQueueConfig::create(128, 0).unwrap(),
+            ];
+            let address_space = create_address_space();
+            let config =
+                VirtioDeviceConfig::<Arc<GuestMemoryMmap>, QueueSync, GuestRegionMmap>::new(
+                    Arc::new(mem),
+                    address_space,
+                    vm_fd,
+                    resources,
+                    queues,
+                    None,
+                    Arc::new(NoopNotifier::default()),
+                );
+            assert!(dev.activate(config).is_err());
+        }
+        // success
+        {
+            let kvm = Kvm::new().unwrap();
+            let vm_fd = Arc::new(kvm.create_vm().unwrap());
+            let (_vmfd, irq_manager) = crate::tests::create_vm_and_irq_manager();
+            let group = irq_manager
+                .create_group(InterruptSourceType::MsiIrq, 0, 3)
+                .unwrap();
+            let notifier = MsiNotifier::new(group.clone(), 1);
+            let mut queues = vec![VirtioQueueConfig::create(128, 0).unwrap()];
+            queues[0].set_interrupt_notifier(Arc::new(notifier));
+            let f = TempFile::new().unwrap().into_file();
+            f.set_len(0x400).unwrap();
+            let mem = GuestMemoryMmap::from_ranges_with_files(&[(
+                GuestAddress(0),
+                0x400,
+                Some(FileOffset::new(f, 0)),
+            )])
+            .unwrap();
+            let resources = DeviceResources::new();
+            let address_space = create_address_space();
+            let config =
+                VirtioDeviceConfig::<Arc<GuestMemoryMmap>, QueueSync, GuestRegionMmap>::new(
+                    Arc::new(mem),
+                    address_space,
+                    vm_fd,
+                    resources,
+                    queues,
+                    None,
+                    Arc::new(NoopNotifier::default()),
+                );
+            dev.activate(config).unwrap();
+        }
+        handler.join().unwrap();
+    }
+}

src/dragonball/src/device_manager/mod.rs

@@ -108,6 +108,12 @@ pub mod vhost_net_dev_mgr;
 #[cfg(feature = "vhost-net")]
 use self::vhost_net_dev_mgr::VhostNetDeviceMgr;
 
+#[cfg(feature = "vhost-user-net")]
+/// Device manager for vhost-user-net devices.
+pub mod vhost_user_net_dev_mgr;
+#[cfg(feature = "vhost-user-net")]
+use self::vhost_user_net_dev_mgr::VhostUserNetDeviceMgr;
+
 macro_rules! info(
     ($l:expr, $($args:tt)+) => {
         slog::info!($l, $($args)+; slog::o!("subsystem" => "device_manager"))
@@ -546,6 +552,9 @@ pub struct DeviceManager {
 
     #[cfg(feature = "vhost-net")]
     vhost_net_manager: VhostNetDeviceMgr,
+
+    #[cfg(feature = "vhost-user-net")]
+    vhost_user_net_manager: VhostUserNetDeviceMgr,
 }
 
 impl DeviceManager {
@@ -584,6 +593,8 @@ impl DeviceManager {
             balloon_manager: BalloonDeviceMgr::default(),
             #[cfg(feature = "vhost-net")]
             vhost_net_manager: VhostNetDeviceMgr::default(),
+            #[cfg(feature = "vhost-user-net")]
+            vhost_user_net_manager: VhostUserNetDeviceMgr::default(),
         }
     }
 
@@ -759,6 +770,11 @@ impl DeviceManager {
             .attach_devices(&mut ctx)
             .map_err(StartMicroVmError::VhostNetDeviceError)?;
 
+        #[cfg(feature = "vhost-user-net")]
+        self.vhost_user_net_manager
+            .attach_devices(&mut ctx)
+            .map_err(StartMicroVmError::VhostUserNetDeviceError)?;
+
         // Ensure that all devices are attached before kernel boot args are
         // generated.
         ctx.generate_kernel_boot_args(kernel_config)
@@ -1184,6 +1200,8 @@ mod tests {
                 mmio_device_info: HashMap::new(),
                 #[cfg(feature = "vhost-net")]
                 vhost_net_manager: VhostNetDeviceMgr::default(),
+                #[cfg(feature = "vhost-user-net")]
+                vhost_user_net_manager: VhostUserNetDeviceMgr::default(),
 
                 logger,
                 shared_info,

src/dragonball/src/device_manager/vhost_user_net_dev_mgr.rs

@@ -0,0 +1,325 @@
+// Copyright (C) 2019-2023 Alibaba Cloud. All rights reserved.
+// Copyright (C) 2019-2023 Ant Group. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::result::Result;
+use std::sync::Arc;
+
+use dbs_utils::net::MacAddr;
+use dbs_virtio_devices::vhost::vhost_user::net::VhostUserNet;
+use dbs_virtio_devices::Error as VirtioError;
+use serde::{Deserialize, Serialize};
+
+use super::{DeviceManager, DeviceMgrError, DeviceOpContext};
+use crate::address_space_manager::GuestAddressSpaceImpl;
+use crate::config_manager::{ConfigItem, DeviceConfigInfos};
+
+/// Default number of virtio queues, one rx/tx pair.
+pub const DEFAULT_NUM_QUEUES: usize = 2;
+/// Default size of virtio queues.
+pub const DEFAULT_QUEUE_SIZE: u16 = 256;
+// The flag of whether to use the shared irq.
+const USE_SHARED_IRQ: bool = true;
+// The flag of whether to use the generic irq.
+const USE_GENERIC_IRQ: bool = false;
+
+/// Errors associated with vhost user net devices.
+#[derive(Debug, thiserror::Error)]
+pub enum VhostUserNetDeviceError {
+    /// The virtual machine instance ID is invalid.
+    #[error("the virtual machine instance ID is invalid")]
+    InvalidVmId,
+    /// Internal error. Invalid queue number configuration for vhost_user_net device.
+    #[error("invalid queue number {0} for vhost_user_net device")]
+    InvalidQueueNum(usize),
+    /// Failure from device manager,
+    #[error("failure in device manager operations: {0:?}")]
+    DeviceManager(DeviceMgrError),
+    /// Duplicated Unix domain socket path for vhost_user_net device.
+    #[error("duplicated Unix domain socket path {0} for vhost_user_net device")]
+    DuplicatedUdsPath(String),
+    /// Failure from Virtio subsystem.
+    #[error(transparent)]
+    Virtio(VirtioError),
+    /// The update is not allowed after booting the microvm.
+    #[error("update operation is not allowed after boot")]
+    UpdateNotAllowedPostBoot,
+    /// Split this at some point.
+    /// Internal errors are due to resource exhaustion.
+    /// Users errors are due to invalid permissions.
+    #[error("cannot create a vhost-user-net device: {0:?}")]
+    CreateNetDevice(VirtioError),
+    /// Cannot initialize a MMIO Network Device or add a device to the MMIO Bus.
+    #[error("failure while registering network device: {0:?}")]
+    RegisterNetDevice(DeviceMgrError),
+}
+/// Configuration information for vhost user net devices.
+#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
+pub struct VhostUserNetDeviceConfigInfo {
+    /// vhost-user socket path.
+    pub sock_path: String,
+    /// Number of virtqueues to use.
+    pub num_queues: usize,
+    /// Size of each virtqueue.
+    pub queue_size: u16,
+    /// mac of the interface.
+    pub guest_mac: Option<MacAddr>,
+    /// Use shared irq
+    pub use_shared_irq: Option<bool>,
+    /// Use generic irq
+    pub use_generic_irq: Option<bool>,
+}
+
+impl VhostUserNetDeviceConfigInfo {
+    /// Returns a reference to the mac address. If the mac address is not configured, it
+    /// returns None.
+    pub fn guest_mac(&self) -> Option<&MacAddr> {
+        self.guest_mac.as_ref()
+    }
+
+    ///Rx and Tx queue and max queue sizes
+    pub fn queue_sizes(&self) -> Vec<u16> {
+        let mut queue_size = self.queue_size;
+        if queue_size == 0 {
+            queue_size = DEFAULT_QUEUE_SIZE;
+        }
+        let num_queues = if self.num_queues > 0 {
+            self.num_queues
+        } else {
+            DEFAULT_NUM_QUEUES
+        };
+        (0..num_queues).map(|_| queue_size).collect::<Vec<u16>>()
+    }
+}
+
+impl ConfigItem for VhostUserNetDeviceConfigInfo {
+    type Err = VhostUserNetDeviceError;
+
+    fn check_conflicts(&self, _other: &Self) -> std::result::Result<(), Self::Err> {
+        Ok(())
+    }
+
+    fn id(&self) -> &str {
+        &self.sock_path
+    }
+}
+
+/// Device manager to manage all vhost user net devices.
+pub struct VhostUserNetDeviceMgr {
+    pub(crate) configs: DeviceConfigInfos<VhostUserNetDeviceConfigInfo>,
+}
+
+impl VhostUserNetDeviceMgr {
+    fn create_device(
+        cfg: &VhostUserNetDeviceConfigInfo,
+        ctx: &mut DeviceOpContext,
+    ) -> Result<Box<VhostUserNet<GuestAddressSpaceImpl>>, VirtioError> {
+        match ctx.epoll_mgr.as_ref() {
+            Some(epoll_mgr) => Ok(Box::new(VhostUserNet::new_server(
+                &cfg.sock_path,
+                cfg.guest_mac(),
+                Arc::new(cfg.queue_sizes()),
+                epoll_mgr.clone(),
+            )?)),
+            None => Err(VirtioError::InvalidInput),
+        }
+    }
+
+    /// Insert or update a vhost user net device into the manager.
+    pub fn insert_device(
+        device_mgr: &mut DeviceManager,
+        mut ctx: DeviceOpContext,
+        config: VhostUserNetDeviceConfigInfo,
+    ) -> Result<(), VhostUserNetDeviceError> {
+        // Validate device configuration first.
+        if config.num_queues % 2 != 0 {
+            return Err(VhostUserNetDeviceError::InvalidQueueNum(config.num_queues));
+        }
+        if !cfg!(feature = "hotplug") && ctx.is_hotplug {
+            return Err(VhostUserNetDeviceError::UpdateNotAllowedPostBoot);
+        }
+        slog::info!(
+            ctx.logger(),
+            "add vhost-user-net device configuration";
+            "subsystem" => "vhost_net_dev_mgr",
+            "sock_path" => &config.sock_path,
+        );
+        let device_index = device_mgr
+            .vhost_user_net_manager
+            .configs
+            .insert_or_update(&config)?;
+        if ctx.is_hotplug {
+            slog::info!(
+                ctx.logger(),
+                "attach virtio-net device";
+                "subsystem" => "vhost_net_dev_mgr",
+                "sock_path" => &config.sock_path,
+            );
+            match Self::create_device(&config, &mut ctx) {
+                Ok(device) => {
+                    let dev = DeviceManager::create_mmio_virtio_device(
+                        device,
+                        &mut ctx,
+                        config.use_shared_irq.unwrap_or(USE_SHARED_IRQ),
+                        config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
+                    )
+                    .map_err(VhostUserNetDeviceError::DeviceManager)?;
+                    ctx.insert_hotplug_mmio_device(&dev, None)
+                        .map_err(VhostUserNetDeviceError::DeviceManager)?;
+                }
+                Err(err) => {
+                    device_mgr
+                        .vhost_user_net_manager
+                        .configs
+                        .remove(device_index);
+                    return Err(VhostUserNetDeviceError::Virtio(err));
+                }
+            }
+        }
+        Ok(())
+    }
+
+    /// Attach all configured vhost user net device to the virtual machine
+    /// instance.
+    pub fn attach_devices(
+        &mut self,
+        ctx: &mut DeviceOpContext,
+    ) -> Result<(), VhostUserNetDeviceError> {
+        for info in self.configs.iter() {
+            slog::info!(
+                ctx.logger(),
+                "attach vhost-user-net device";
+                "subsystem" => "vhost_net_dev_mgr",
+                "sock_path" => &info.config.sock_path,
+            );
+            let device = Self::create_device(&info.config, ctx)
+                .map_err(VhostUserNetDeviceError::CreateNetDevice)?;
+            DeviceManager::create_mmio_virtio_device(
+                device,
+                ctx,
+                info.config.use_shared_irq.unwrap_or(USE_SHARED_IRQ),
+                info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
+            )
+            .map_err(VhostUserNetDeviceError::RegisterNetDevice)?;
+        }
+        Ok(())
+    }
+}
+
+impl Default for VhostUserNetDeviceMgr {
+    /// Create a new vhost user net device manager.
+    fn default() -> Self {
+        VhostUserNetDeviceMgr {
+            configs: DeviceConfigInfos::<VhostUserNetDeviceConfigInfo>::new(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_utils::tests::create_vm_for_test;
+
+    #[test]
+    fn test_create_vhost_user_net_device() {
+        let vm = create_vm_for_test();
+        let mgr = DeviceManager::new_test_mgr();
+        let sock_1 = String::from("id_1");
+        let guest_mac_1 = "01:23:45:67:89:0a";
+        let netif_1 = VhostUserNetDeviceConfigInfo {
+            sock_path: sock_1,
+            num_queues: 2,
+            queue_size: 128,
+            guest_mac: Some(MacAddr::parse_str(guest_mac_1).unwrap()),
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+        // no epoll manager
+        let mut ctx = DeviceOpContext::new(
+            None,
+            &mgr,
+            None,
+            None,
+            false,
+            Some(vm.vm_config().clone()),
+            vm.shared_info().clone(),
+        );
+        assert!(VhostUserNetDeviceMgr::create_device(&netif_1, &mut ctx).is_err());
+    }
+
+    #[test]
+    fn test_insert_vhost_user_net_device() {
+        let vm = create_vm_for_test();
+        let mut mgr = DeviceManager::new_test_mgr();
+        let sock_1 = String::from("id_1");
+        let guest_mac_1 = "01:23:45:67:89:0a";
+        // Test create.
+        let netif_1 = VhostUserNetDeviceConfigInfo {
+            sock_path: sock_1,
+            num_queues: 2,
+            queue_size: 128,
+            guest_mac: Some(MacAddr::parse_str(guest_mac_1).unwrap()),
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+        let ctx = DeviceOpContext::new(
+            None,
+            &mgr,
+            None,
+            None,
+            false,
+            Some(vm.vm_config().clone()),
+            vm.shared_info().clone(),
+        );
+        assert!(VhostUserNetDeviceMgr::insert_device(&mut mgr, ctx, netif_1).is_ok());
+        assert_eq!(mgr.vhost_user_net_manager.configs.len(), 1);
+    }
+
+    #[test]
+    fn test_vhost_user_net_insert_error_cases() {
+        let vm = create_vm_for_test();
+        let mut mgr = DeviceManager::new_test_mgr();
+        let sock_1 = String::from("id_1");
+        let guest_mac_1 = "01:23:45:67:89:0a";
+        // invalid queue num.
+        let netif_1 = VhostUserNetDeviceConfigInfo {
+            sock_path: sock_1,
+            num_queues: 1,
+            queue_size: 128,
+            guest_mac: Some(MacAddr::parse_str(guest_mac_1).unwrap()),
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+        let ctx = DeviceOpContext::new(
+            None,
+            &mgr,
+            None,
+            None,
+            false,
+            Some(vm.vm_config().clone()),
+            vm.shared_info().clone(),
+        );
+        let res = VhostUserNetDeviceMgr::insert_device(&mut mgr, ctx, netif_1);
+        if let Err(VhostUserNetDeviceError::InvalidQueueNum(1)) = res {
+            assert_eq!(mgr.vhost_user_net_manager.configs.len(), 0);
+        } else {
+            panic!()
+        }
+    }
+
+    #[test]
+    fn test_vhost_user_net_error_display() {
+        let err = VhostUserNetDeviceError::InvalidVmId;
+        let _ = format!("{}{:?}", err, err);
+        let err = VhostUserNetDeviceError::InvalidQueueNum(0);
+        let _ = format!("{}{:?}", err, err);
+        let err = VhostUserNetDeviceError::DeviceManager(DeviceMgrError::GetDeviceResource);
+        let _ = format!("{}{:?}", err, err);
+        let err = VhostUserNetDeviceError::DuplicatedUdsPath(String::from("1"));
+        let _ = format!("{}{:?}", err, err);
+        let err = VhostUserNetDeviceError::Virtio(VirtioError::DescriptorChainTooShort);
+        let _ = format!("{}{:?}", err, err);
+        let err = VhostUserNetDeviceError::UpdateNotAllowedPostBoot;
+        let _ = format!("{}{:?}", err, err);
+    }
+}

src/dragonball/src/error.rs

@@ -198,6 +198,13 @@ pub enum StartMicroVmError {
     #[cfg(feature = "vhost-net")]
     #[error("vhost-net errors: {0:?}")]
     VhostNetDeviceError(#[source] device_manager::vhost_net_dev_mgr::VhostNetDeviceError),
+
+    /// Vhost-user-net device errors.
+    #[cfg(feature = "vhost-user-net")]
+    #[error("vhost-user-net errors: {0:?}")]
+    VhostUserNetDeviceError(
+        #[source] device_manager::vhost_user_net_dev_mgr::VhostUserNetDeviceError,
+    ),
 }
 
 /// Errors associated with starting the instance.

src/runtime-rs/Cargo.lock

@@ -807,6 +807,7 @@ version = "0.3.1"
 dependencies = [
  "byteorder",
  "caps",
+ "dbs-address-space",
  "dbs-device",
  "dbs-interrupt",
  "dbs-utils",