diff --git a/src/dragonball/src/api/v1/mod.rs b/src/dragonball/src/api/v1/mod.rs index f9975bdd95..2db7891de1 100644 --- a/src/dragonball/src/api/v1/mod.rs +++ b/src/dragonball/src/api/v1/mod.rs @@ -24,14 +24,15 @@ pub use self::machine_config::{VmConfigError, MAX_SUPPORTED_VCPUS}; feature = "vhost-net", feature = "vhost-user-net" ))] -mod virtio_net; #[cfg(feature = "vhost-user-net")] -pub use virtio_net::VhostUserConfig; +pub use crate::device_manager::net_dev_mgr::VhostUserConfig; #[cfg(any(feature = "virtio-net", feature = "vhost-net"))] -pub use virtio_net::VirtioConfig; +pub use crate::device_manager::net_dev_mgr::VirtioConfig; #[cfg(any( feature = "virtio-net", feature = "vhost-net", feature = "vhost-user-net" ))] -pub use virtio_net::{Backend, NetworkInterfaceConfig, NetworkInterfaceUpdateConfig}; +pub use crate::device_manager::net_dev_mgr::{ + Backend, NetworkInterfaceConfig, NetworkInterfaceUpdateConfig, +}; diff --git a/src/dragonball/src/api/v1/virtio_net.rs b/src/dragonball/src/api/v1/virtio_net.rs deleted file mode 100644 index bfd0841453..0000000000 --- a/src/dragonball/src/api/v1/virtio_net.rs +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright (C) 2019-2023 Alibaba Cloud. All rights reserved. -// Copyright (C) 2019-2023 Ant Group. All rights reserved. -// -// SPDX-License-Identifier: Apache-2.0 - -use core::panic; - -use dbs_utils::net::MacAddr; -use serde::{Deserialize, Serialize}; - -#[cfg(feature = "virtio-net")] -use super::{VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo}; -use crate::config_manager::RateLimiterConfigInfo; -#[cfg(feature = "vhost-net")] -use crate::device_manager::vhost_net_dev_mgr; -#[cfg(feature = "vhost-net")] -use crate::device_manager::vhost_net_dev_mgr::VhostNetDeviceConfigInfo; -#[cfg(feature = "vhost-user-net")] -use crate::device_manager::vhost_user_net_dev_mgr; -#[cfg(feature = "vhost-user-net")] -use crate::device_manager::vhost_user_net_dev_mgr::VhostUserNetDeviceConfigInfo; -#[cfg(feature = "virtio-net")] -use crate::device_manager::virtio_net_dev_mgr; - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] -#[serde(tag = "type", deny_unknown_fields)] -/// An enum to specify a backend of Virtio network -pub enum Backend { - #[serde(rename = "virtio")] - #[cfg(feature = "virtio-net")] - /// Virtio-net - Virtio(VirtioConfig), - #[serde(rename = "vhost")] - #[cfg(feature = "vhost-net")] - /// Vhost-net - Vhost(VirtioConfig), - #[serde(rename = "vhost-user")] - #[cfg(feature = "vhost-user-net")] - /// Vhost-user-net - VhostUser(VhostUserConfig), -} - -impl Default for Backend { - #[allow(unreachable_code)] - fn default() -> Self { - #[cfg(feature = "virtio-net")] - return Self::Virtio(VirtioConfig::default()); - #[cfg(feature = "vhost-net")] - return Self::Vhost(VirtioConfig::default()); - - panic!("no available default network backend") - } -} - -/// Virtio network config, working for virtio-net and vhost-net. -#[cfg(any(feature = "virtio-net", feature = "vhost-net"))] -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] -pub struct VirtioConfig { - /// ID of the guest network interface. - pub iface_id: String, - /// Host level path for the guest network interface. - pub host_dev_name: String, - /// Rate Limiter for received packages. - pub rx_rate_limiter: Option, - /// Rate Limiter for transmitted packages. - pub tx_rate_limiter: Option, - /// Allow duplicate mac - pub allow_duplicate_mac: bool, -} - -/// Config for vhost-user-net device -#[cfg(feature = "vhost-user-net")] -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] -pub struct VhostUserConfig { - /// Vhost-user socket path. - pub sock_path: String, -} - -/// This struct represents the strongly typed equivalent of the json body from -/// net iface related requests. -/// This struct works with virtio-net devices and vhost-net devices. -#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize, Default)] -#[serde(deny_unknown_fields)] -pub struct NetworkInterfaceConfig { - /// Number of virtqueue pairs to use. (https://www.linux-kvm.org/page/Multiqueue) - pub num_queues: Option, - /// Size of each virtqueue. - pub queue_size: Option, - /// Net backend driver. - #[serde(default = "Backend::default")] - pub backend: Backend, - /// mac of the interface. - pub guest_mac: Option, - /// Use shared irq - pub use_shared_irq: Option, - /// Use generic irq - pub use_generic_irq: Option, -} - -#[cfg(feature = "virtio-net")] -impl From for VirtioNetDeviceConfigInfo { - fn from(value: NetworkInterfaceConfig) -> Self { - let self_ref = &value; - self_ref.into() - } -} - -#[cfg(feature = "virtio-net")] -impl From<&NetworkInterfaceConfig> for VirtioNetDeviceConfigInfo { - fn from(value: &NetworkInterfaceConfig) -> Self { - let queue_size = value - .queue_size - .unwrap_or(virtio_net_dev_mgr::DEFAULT_QUEUE_SIZE); - - // It is safe because we tested the type of config before. - #[allow(unreachable_patterns)] - let config = match &value.backend { - Backend::Virtio(config) => config, - _ => panic!("The virtio backend config is invalid: {:?}", value), - }; - - Self { - iface_id: config.iface_id.clone(), - host_dev_name: config.host_dev_name.clone(), - num_queues: virtio_net_dev_mgr::DEFAULT_NUM_QUEUES, - queue_size, - guest_mac: value.guest_mac, - rx_rate_limiter: config.rx_rate_limiter.clone(), - tx_rate_limiter: config.tx_rate_limiter.clone(), - allow_duplicate_mac: config.allow_duplicate_mac, - use_shared_irq: value.use_shared_irq, - use_generic_irq: value.use_generic_irq, - } - } -} - -#[cfg(feature = "vhost-net")] -impl From for VhostNetDeviceConfigInfo { - fn from(value: NetworkInterfaceConfig) -> Self { - let self_ref = &value; - self_ref.into() - } -} - -#[cfg(feature = "vhost-net")] -impl From<&NetworkInterfaceConfig> for VhostNetDeviceConfigInfo { - fn from(value: &NetworkInterfaceConfig) -> Self { - let num_queues = value - .num_queues - .map(|nq| { - if nq == 0 { - vhost_net_dev_mgr::DEFAULT_NUM_QUEUES - } else { - nq - } - }) - .unwrap_or(vhost_net_dev_mgr::DEFAULT_NUM_QUEUES); - let queue_size = value - .queue_size - .map(|qs| { - if qs == 0 { - vhost_net_dev_mgr::DEFAULT_QUEUE_SIZE - } else { - qs - } - }) - .unwrap_or(vhost_net_dev_mgr::DEFAULT_QUEUE_SIZE); - - // It is safe because we tested the type of config before. - #[allow(unreachable_patterns)] - let config = match &value.backend { - Backend::Vhost(config) => config, - _ => panic!("The virtio backend config is invalid: {:?}", value), - }; - - Self { - iface_id: config.iface_id.clone(), - host_dev_name: config.host_dev_name.clone(), - num_queues, - queue_size, - guest_mac: value.guest_mac, - allow_duplicate_mac: config.allow_duplicate_mac, - use_shared_irq: value.use_shared_irq, - use_generic_irq: value.use_generic_irq, - } - } -} - -#[cfg(feature = "vhost-user-net")] -impl From for VhostUserNetDeviceConfigInfo { - fn from(value: NetworkInterfaceConfig) -> Self { - let self_ref = &value; - self_ref.into() - } -} -#[cfg(feature = "vhost-user-net")] -impl From<&NetworkInterfaceConfig> for VhostUserNetDeviceConfigInfo { - fn from(value: &NetworkInterfaceConfig) -> Self { - let num_queues = value - .num_queues - .map(|nq| { - if nq == 0 { - vhost_user_net_dev_mgr::DEFAULT_NUM_QUEUES - } else { - nq - } - }) - .unwrap_or(vhost_user_net_dev_mgr::DEFAULT_NUM_QUEUES); - let queue_size = value - .queue_size - .map(|qs| { - if qs == 0 { - vhost_user_net_dev_mgr::DEFAULT_QUEUE_SIZE - } else { - qs - } - }) - .unwrap_or(vhost_user_net_dev_mgr::DEFAULT_QUEUE_SIZE); - // It is safe because we tested the type of config before. - #[allow(unreachable_patterns)] - let config = match &value.backend { - Backend::VhostUser(config) => config, - _ => panic!("The virtio backend config is invalid: {:?}", value), - }; - Self { - sock_path: config.sock_path.clone(), - num_queues, - queue_size, - guest_mac: value.guest_mac, - use_shared_irq: value.use_shared_irq, - use_generic_irq: value.use_generic_irq, - } - } -} - -/// The data fed into a network iface update request. Currently, only the RX and -/// TX rate limiters can be updated. -#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize, Default)] -#[serde(deny_unknown_fields)] -pub struct NetworkInterfaceUpdateConfig { - /// ID of the guest network interface. - pub iface_id: String, - /// New RX rate limiter config. Only provided data will be updated. I.e. if any optional data - /// is missing, it will not be nullified, but left unchanged. - pub rx_rate_limiter: Option, - /// New TX rate limiter config. Only provided data will be updated. I.e. if any optional data - /// is missing, it will not be nullified, but left unchanged. - pub tx_rate_limiter: Option, -} - -#[cfg(feature = "virtio-net")] -impl From for VirtioNetDeviceConfigUpdateInfo { - fn from(value: NetworkInterfaceUpdateConfig) -> Self { - let self_ref = &value; - self_ref.into() - } -} - -#[cfg(feature = "virtio-net")] -impl From<&NetworkInterfaceUpdateConfig> for VirtioNetDeviceConfigUpdateInfo { - fn from(value: &NetworkInterfaceUpdateConfig) -> Self { - Self { - iface_id: value.iface_id.clone(), - rx_rate_limiter: value.rx_rate_limiter.clone(), - tx_rate_limiter: value.tx_rate_limiter.clone(), - } - } -} - -#[cfg(feature = "virtio-net")] -#[cfg(test)] -mod tests { - use dbs_utils::net::MacAddr; - - use super::NetworkInterfaceConfig; - use crate::api::v1::Backend; - - #[test] - fn test_network_interface_config() { - let json_str = r#"{ - "num_queues": 4, - "queue_size": 512, - "backend": { - "type": "virtio", - "iface_id": "eth0", - "host_dev_name": "tap0", - "allow_duplicate_mac": true - }, - "guest_mac": "81:87:1D:00:08:A9" - }"#; - let net_config: NetworkInterfaceConfig = serde_json::from_str(json_str).unwrap(); - assert_eq!(net_config.num_queues, Some(4)); - assert_eq!(net_config.queue_size, Some(512)); - assert_eq!( - net_config.guest_mac, - Some(MacAddr::from_bytes(&[129, 135, 29, 0, 8, 169]).unwrap()) - ); - if let Backend::Virtio(config) = net_config.backend { - assert_eq!(config.iface_id, "eth0"); - assert_eq!(config.host_dev_name, "tap0"); - assert!(config.allow_duplicate_mac); - } else { - panic!("Unexpected backend type"); - } - } -} diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 2f2a317abf..f9096f0b9e 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -37,21 +37,16 @@ pub use crate::device_manager::fs_dev_mgr::{ }; #[cfg(feature = "virtio-mem")] pub use crate::device_manager::mem_dev_mgr::{MemDeviceConfigInfo, MemDeviceError}; +#[cfg(any( + feature = "virtio-net", + feature = "vhost-net", + feature = "vhost-user-net" +))] +pub use crate::device_manager::net_dev_mgr::{ + NetworkDeviceError, NetworkDeviceMgr, NetworkInterfaceConfig, NetworkInterfaceUpdateConfig, +}; #[cfg(feature = "host-device")] use crate::device_manager::vfio_dev_mgr::{HostDeviceConfig, VfioDeviceError}; -#[cfg(feature = "vhost-net")] -pub use crate::device_manager::vhost_net_dev_mgr::{ - VhostNetDeviceConfigInfo, VhostNetDeviceError, VhostNetDeviceMgr, -}; -#[cfg(feature = "vhost-user-net")] -use crate::device_manager::vhost_user_net_dev_mgr::{ - VhostUserNetDeviceConfigInfo, VhostUserNetDeviceError, VhostUserNetDeviceMgr, -}; -#[cfg(feature = "virtio-net")] -pub use crate::device_manager::virtio_net_dev_mgr::{ - VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo, VirtioNetDeviceError, - VirtioNetDeviceMgr, -}; #[cfg(feature = "virtio-vsock")] pub use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError}; #[cfg(feature = "host-device")] @@ -109,20 +104,14 @@ pub enum VmmActionError { #[error("virtio-blk device error: {0}")] Block(#[source] BlockDeviceError), - #[cfg(feature = "virtio-net")] - /// Net device related errors. - #[error("virtio-net device error: {0}")] - VirtioNet(#[source] VirtioNetDeviceError), - - #[cfg(feature = "vhost-net")] - #[error("vhost-net device error: {0:?}")] - /// Vhost-net device relared errors. - VhostNet(#[source] VhostNetDeviceError), - - #[error("vhost-user-net device error: {0:?}")] - #[cfg(feature = "vhost-user-net")] - /// Vhost-user-net device relared errors. - VhostUserNet(#[source] VhostUserNetDeviceError), + #[cfg(any( + feature = "virtio-net", + feature = "vhost-net", + feature = "vhost-user-net" + ))] + /// Network device related errors. + #[error("network device error: {0}")] + Network(#[source] NetworkDeviceError), #[cfg(any(feature = "virtio-fs", feature = "vhost-user-fs"))] /// The action `InsertFsDevice` failed either because of bad user input or an internal error. @@ -233,7 +222,7 @@ pub enum VmmAction { /// are the RX and TX rate limiters. /// TODO: vhost-net rate limiters aren't implemented, see: /// https://github.com/kata-containers/kata-containers/issues/8327 - UpdateNetworkInterface(VirtioNetDeviceConfigUpdateInfo), + UpdateNetworkInterface(NetworkInterfaceUpdateConfig), #[cfg(any(feature = "virtio-fs", feature = "vhost-user-fs"))] /// Add a new shared fs device or update one that already exists using the @@ -376,16 +365,9 @@ impl VmmService { feature = "vhost-net", feature = "vhost-user-net" ))] - VmmAction::InsertNetworkDevice(config) => match config.backend { - #[cfg(feature = "virtio-net")] - Backend::Virtio(_) => self.add_virtio_net_device(vmm, event_mgr, config.into()), - #[cfg(feature = "vhost-net")] - Backend::Vhost(_) => self.add_vhost_net_device(vmm, event_mgr, config.into()), - #[cfg(feature = "vhost-user-net")] - Backend::VhostUser(_) => { - self.add_vhost_user_net_device(vmm, event_mgr, config.into()) - } - }, + VmmAction::InsertNetworkDevice(config) => { + self.add_network_device(vmm, event_mgr, config) + } #[cfg(feature = "virtio-net")] VmmAction::UpdateNetworkInterface(netif_update) => { self.update_net_rate_limiters(vmm, netif_update) @@ -764,20 +746,24 @@ impl VmmService { .map_err(VmmActionError::Block) } - #[cfg(feature = "virtio-net")] + #[cfg(any( + feature = "virtio-net", + feature = "vhost-net", + feature = "vhost-user-net" + ))] #[instrument(skip(self, event_mgr))] - fn add_virtio_net_device( + fn add_network_device( &mut self, vmm: &mut Vmm, event_mgr: &mut EventManager, - config: VirtioNetDeviceConfigInfo, + config: NetworkInterfaceConfig, ) -> VmmRequestResult { let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; let ctx = vm .create_device_op_context(Some(event_mgr.epoll_manager())) .map_err(|e| { if let StartMicroVmError::MicroVMAlreadyRunning = e { - VmmActionError::VirtioNet(VirtioNetDeviceError::UpdateNotAllowedPostBoot) + VmmActionError::Network(NetworkDeviceError::UpdateNotAllowedPostBoot) } else if let StartMicroVmError::UpcallServerNotReady = e { VmmActionError::UpcallServerNotReady } else { @@ -786,10 +772,10 @@ impl VmmService { })?; vm.device_manager_mut() - .virtio_net_manager + .net_manager .insert_device(ctx, config) .map(|_| VmmData::Empty) - .map_err(VmmActionError::VirtioNet) + .map_err(VmmActionError::Network) } #[cfg(feature = "virtio-net")] @@ -797,61 +783,15 @@ impl VmmService { fn update_net_rate_limiters( &mut self, vmm: &mut Vmm, - config: VirtioNetDeviceConfigUpdateInfo, + config: NetworkInterfaceUpdateConfig, ) -> VmmRequestResult { let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; vm.device_manager_mut() - .virtio_net_manager + .net_manager .update_device_ratelimiters(config) .map(|_| VmmData::Empty) - .map_err(VmmActionError::VirtioNet) - } - - #[cfg(feature = "vhost-net")] - fn add_vhost_net_device( - &mut self, - vmm: &mut Vmm, - event_mgr: &mut EventManager, - config: VhostNetDeviceConfigInfo, - ) -> VmmRequestResult { - let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; - let ctx = vm - .create_device_op_context(Some(event_mgr.epoll_manager())) - .map_err(|err| match err { - StartMicroVmError::MicroVMAlreadyRunning => { - VmmActionError::VhostNet(VhostNetDeviceError::UpdateNotAllowedPostBoot) - } - StartMicroVmError::UpcallServerNotReady => VmmActionError::UpcallServerNotReady, - _ => VmmActionError::StartMicroVm(err), - })?; - VhostNetDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config) - .map(|_| VmmData::Empty) - .map_err(VmmActionError::VhostNet) - } - - #[cfg(feature = "vhost-user-net")] - fn add_vhost_user_net_device( - &mut self, - vmm: &mut Vmm, - event_mgr: &mut EventManager, - config: VhostUserNetDeviceConfigInfo, - ) -> VmmRequestResult { - let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; - let ctx = vm - .create_device_op_context(Some(event_mgr.epoll_manager())) - .map_err(|err| { - if let StartMicroVmError::MicroVMAlreadyRunning = err { - VmmActionError::VhostUserNet(VhostUserNetDeviceError::UpdateNotAllowedPostBoot) - } else if let StartMicroVmError::UpcallServerNotReady = err { - VmmActionError::UpcallServerNotReady - } else { - VmmActionError::StartMicroVm(err) - } - })?; - VhostUserNetDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config) - .map(|_| VmmData::Empty) - .map_err(VmmActionError::VhostUserNet) + .map_err(VmmActionError::Network) } #[cfg(any(feature = "virtio-fs", feature = "vhost-user-fs"))] @@ -1772,10 +1712,20 @@ mod tests { fn test_vmm_action_insert_network_device() { skip_if_kvm_unaccessable!(); + // A device must be named: the id identifies it for conflict detection + // and for later requests, so the manager refuses an unnamed one. + let named = || NetworkInterfaceConfig { + backend: Backend::Virtio(VirtioConfig { + iface_id: String::from("eth0"), + ..Default::default() + }), + ..Default::default() + }; + let tests = &mut [ // hotplug unready TestData::new( - VmmAction::InsertNetworkDevice(NetworkInterfaceConfig::default()), + VmmAction::InsertNetworkDevice(named()), InstanceState::Running, &|result| { assert!(matches!( @@ -1794,7 +1744,7 @@ mod tests { ), // success TestData::new( - VmmAction::InsertNetworkDevice(NetworkInterfaceConfig::default()), + VmmAction::InsertNetworkDevice(named()), InstanceState::Uninitialized, &|result| { assert!(result.is_ok()); @@ -1815,7 +1765,7 @@ mod tests { let tests = &mut [ // invalid id TestData::new( - VmmAction::UpdateNetworkInterface(VirtioNetDeviceConfigUpdateInfo { + VmmAction::UpdateNetworkInterface(NetworkInterfaceUpdateConfig { iface_id: String::from("1"), rx_rate_limiter: None, tx_rate_limiter: None, @@ -1824,14 +1774,14 @@ mod tests { &|result| { assert!(matches!( result, - Err(VmmActionError::VirtioNet( - VirtioNetDeviceError::InvalidIfaceId(_) - )) + Err(VmmActionError::Network(NetworkDeviceError::InvalidIfaceId( + _ + ))) )); let err_string = format!("{}", result.unwrap_err()); let expected_err = String::from( - "virtio-net device error: \ - invalid virtio-net iface id '1'", + "network device error: \ + invalid network iface id '1'", ); assert_eq!(err_string, expected_err); }, diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 1dcda5d8c8..02728d0ceb 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -89,11 +89,19 @@ pub mod blk_dev_mgr; #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] use self::blk_dev_mgr::BlockDeviceMgr; -#[cfg(feature = "virtio-net")] -/// Device manager for virtio-net devices. -pub mod virtio_net_dev_mgr; -#[cfg(feature = "virtio-net")] -use self::virtio_net_dev_mgr::VirtioNetDeviceMgr; +#[cfg(any( + feature = "virtio-net", + feature = "vhost-net", + feature = "vhost-user-net" +))] +/// Device manager for all network device backends. +pub mod net_dev_mgr; +#[cfg(any( + feature = "virtio-net", + feature = "vhost-net", + feature = "vhost-user-net" +))] +use self::net_dev_mgr::NetworkDeviceMgr; #[cfg(any(feature = "virtio-fs", feature = "vhost-user-fs"))] /// virtio-block device manager @@ -117,23 +125,12 @@ pub mod balloon_dev_mgr; #[cfg(feature = "virtio-balloon")] use self::balloon_dev_mgr::BalloonDeviceMgr; -#[cfg(feature = "vhost-net")] -/// Device manager for vhost-net devices. -pub mod vhost_net_dev_mgr; -#[cfg(feature = "vhost-net")] -use self::vhost_net_dev_mgr::VhostNetDeviceMgr; #[cfg(feature = "host-device")] /// Device manager for PCI/MMIO VFIO devices. pub mod vfio_dev_mgr; #[cfg(feature = "host-device")] use self::vfio_dev_mgr::VfioDeviceMgr; -#[cfg(feature = "vhost-user-net")] -/// Device manager for vhost-user-net devices. -pub mod vhost_user_net_dev_mgr; -#[cfg(feature = "vhost-user-net")] -use self::vhost_user_net_dev_mgr::VhostUserNetDeviceMgr; - macro_rules! info( ($l:expr, $($args:tt)+) => { slog::info!($l, $($args)+; slog::o!("subsystem" => "device_manager")) @@ -656,8 +653,12 @@ pub struct DeviceManager { // This is necessary because we want the root to always be mounted on /dev/vda. pub(crate) block_manager: BlockDeviceMgr, - #[cfg(feature = "virtio-net")] - pub(crate) virtio_net_manager: VirtioNetDeviceMgr, + #[cfg(any( + feature = "virtio-net", + feature = "vhost-net", + feature = "vhost-user-net" + ))] + pub(crate) net_manager: NetworkDeviceMgr, #[cfg(any(feature = "virtio-fs", feature = "vhost-user-fs"))] fs_manager: Arc>, @@ -667,12 +668,6 @@ pub struct DeviceManager { #[cfg(feature = "virtio-balloon")] pub(crate) balloon_manager: BalloonDeviceMgr, - - #[cfg(feature = "vhost-net")] - vhost_net_manager: VhostNetDeviceMgr, - - #[cfg(feature = "vhost-user-net")] - vhost_user_net_manager: VhostUserNetDeviceMgr, #[cfg(feature = "host-device")] pub(crate) vfio_manager: Arc>, #[cfg(feature = "host-device")] @@ -737,18 +732,18 @@ impl DeviceManager { vsock_manager: VsockDeviceMgr::default(), #[cfg(any(feature = "virtio-blk", feature = "vhost-user-blk"))] block_manager: BlockDeviceMgr::default(), - #[cfg(feature = "virtio-net")] - virtio_net_manager: VirtioNetDeviceMgr::default(), + #[cfg(any( + feature = "virtio-net", + feature = "vhost-net", + feature = "vhost-user-net" + ))] + net_manager: NetworkDeviceMgr::default(), #[cfg(any(feature = "virtio-fs", feature = "vhost-user-fs"))] fs_manager: Arc::new(Mutex::new(FsDeviceMgr::default())), #[cfg(feature = "virtio-mem")] mem_manager: MemDeviceMgr::default(), #[cfg(feature = "virtio-balloon")] balloon_manager: BalloonDeviceMgr::default(), - #[cfg(feature = "vhost-net")] - vhost_net_manager: VhostNetDeviceMgr::default(), - #[cfg(feature = "vhost-user-net")] - vhost_user_net_manager: VhostUserNetDeviceMgr::default(), #[cfg(feature = "host-device")] vfio_manager: Arc::new(Mutex::new(VfioDeviceMgr::new( vm_fd, @@ -919,10 +914,14 @@ impl DeviceManager { .map_err(StartMicroVmError::FsDeviceError)?; } - #[cfg(feature = "virtio-net")] - self.virtio_net_manager + #[cfg(any( + feature = "virtio-net", + feature = "vhost-net", + feature = "vhost-user-net" + ))] + self.net_manager .attach_devices(&mut ctx) - .map_err(StartMicroVmError::VirtioNetDeviceError)?; + .map_err(StartMicroVmError::NetworkDeviceError)?; #[cfg(feature = "virtio-vsock")] self.vsock_manager.attach_devices(&mut ctx)?; @@ -932,16 +931,6 @@ impl DeviceManager { .generate_kernel_boot_args(kernel_config) .map_err(StartMicroVmError::DeviceManager)?; - #[cfg(feature = "vhost-net")] - self.vhost_net_manager - .attach_devices(&mut ctx) - .map_err(StartMicroVmError::VhostNetDeviceError)?; - - #[cfg(feature = "vhost-user-net")] - self.vhost_user_net_manager - .attach_devices(&mut ctx) - .map_err(StartMicroVmError::VhostUserNetDeviceError)?; - #[cfg(feature = "host-device")] { // It is safe bacause we don't expect poison lock. @@ -1005,14 +994,15 @@ impl DeviceManager { // FIXME: To acquire the full abilities for gracefully removing // virtio-net and virtio-vsock devices, updating dragonball-sandbox // is required. - #[cfg(feature = "virtio-net")] - self.virtio_net_manager.remove_devices(&mut ctx)?; + #[cfg(any( + feature = "virtio-net", + feature = "vhost-net", + feature = "vhost-user-net" + ))] + self.net_manager.remove_devices(&mut ctx)?; #[cfg(feature = "virtio-vsock")] self.vsock_manager.remove_devices(&mut ctx)?; - #[cfg(feature = "vhost-net")] - self.vhost_net_manager.remove_devices(&mut ctx)?; - Ok(()) } } @@ -1663,8 +1653,12 @@ mod tests { block_manager: BlockDeviceMgr::default(), #[cfg(any(feature = "virtio-fs", feature = "vhost-user-fs"))] fs_manager: Arc::new(Mutex::new(FsDeviceMgr::default())), - #[cfg(feature = "virtio-net")] - virtio_net_manager: VirtioNetDeviceMgr::default(), + #[cfg(any( + feature = "virtio-net", + feature = "vhost-net", + feature = "vhost-user-net" + ))] + net_manager: NetworkDeviceMgr::default(), #[cfg(feature = "virtio-vsock")] vsock_manager: VsockDeviceMgr::default(), #[cfg(feature = "virtio-mem")] @@ -1673,10 +1667,6 @@ mod tests { balloon_manager: BalloonDeviceMgr::default(), #[cfg(target_arch = "aarch64")] mmio_device_info: HashMap::new(), - #[cfg(feature = "vhost-net")] - vhost_net_manager: VhostNetDeviceMgr::default(), - #[cfg(feature = "vhost-user-net")] - vhost_user_net_manager: VhostUserNetDeviceMgr::default(), #[cfg(feature = "host-device")] vfio_manager: Arc::new(Mutex::new(VfioDeviceMgr::new( vm_fd, diff --git a/src/dragonball/src/device_manager/net_dev_mgr.rs b/src/dragonball/src/device_manager/net_dev_mgr.rs new file mode 100644 index 0000000000..be0fce3e62 --- /dev/null +++ b/src/dragonball/src/device_manager/net_dev_mgr.rs @@ -0,0 +1,1123 @@ +// Copyright (C) 2019-2023 Alibaba Cloud. All rights reserved. +// Copyright (C) 2019-2026 Ant Group. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +//! Device manager for all network device backends. +//! +//! virtio-net, vhost-net and vhost-user-net are all network interfaces of the +//! guest and are configured through the same API request, so they share one +//! manager and one `info_list`, and the backend is selected per device by +//! [`Backend`]. This mirrors how the block manager keeps its backends in one +//! list keyed by `BlockDeviceType`. + +use std::convert::TryInto; +use std::sync::Arc; + +use dbs_utils::net::MacAddr; +use dbs_utils::rate_limiter::BucketUpdate; +use dbs_virtio_devices as virtio; +use dbs_virtio_devices::Error as VirtioError; +use serde::{Deserialize, Serialize}; + +use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::config_manager::{ConfigItem, DeviceConfigInfos, RateLimiterConfigInfo}; +use crate::device_manager::{DbsVirtioDevice, DeviceManager, DeviceMgrError, DeviceOpContext}; +use crate::get_bucket_update; + +use super::DbsMmioV2Device; + +/// Default number of virtio queues, one rx/tx pair. +pub const DEFAULT_NUM_QUEUES: usize = 2; +/// Default size of virtio queues. +pub const DEFAULT_QUEUE_SIZE: u16 = 256; +// The flag of whether to use the shared irq. +const USE_SHARED_IRQ: bool = true; +// The flag of whether to use the generic irq, for the in-VMM backends. +const USE_GENERIC_IRQ: bool = true; +// vhost-user-net does not use the generic irq. +#[cfg(feature = "vhost-user-net")] +const VHOST_USER_USE_GENERIC_IRQ: bool = false; + +/// Errors associated with network device operations. +#[derive(Debug, thiserror::Error)] +pub enum NetworkDeviceError { + /// The virtual machine instance ID is invalid. + #[error("the virtual machine instance ID is invalid")] + InvalidVMID, + + /// The iface ID is invalid. + #[error("invalid network iface id '{0}'")] + InvalidIfaceId(String), + + /// No iface ID was supplied for the device. + #[error("no iface id supplied for the network device")] + MissingIfaceId, + + /// Invalid queue number configuration for the device. + #[error("invalid queue number {0} for network device")] + InvalidQueueNum(usize), + + /// Failure from device manager. + #[error("failure in device manager operations, {0}")] + DeviceManager(#[source] DeviceMgrError), + + /// The device ID is already in use. + #[error("the device ID {0} already exists")] + DeviceIDAlreadyExist(String), + + /// The MAC address is already in use. + #[error("the guest MAC address {0} is already in use")] + GuestMacAddressInUse(String), + + /// The host device name is already in use. + #[error("the host device name {0} is already in use")] + HostDeviceNameInUse(String), + + /// Duplicated Unix domain socket path for a vhost-user-net device. + #[error("duplicated Unix domain socket path {0} for vhost-user-net device")] + DuplicatedUdsPath(String), + + /// Failure from virtio subsystem. + #[error(transparent)] + Virtio(VirtioError), + + /// Failed to send patch message to the net epoll handler. + #[error("could not send patch message to the net epoll handler")] + NetEpollHandlerSendFail, + + /// The update is not allowed after booting the microvm. + #[error("update operation is not allowed after boot")] + UpdateNotAllowedPostBoot, + + /// The requested operation is not supported by this backend. + #[error("network device '{iface_id}': operation not supported by the {backend} backend")] + UnsupportedBackend { + /// Identifier of the device. + iface_id: String, + /// Backend that does not support the operation. + backend: &'static str, + }, + + /// Split this at some point. + /// Internal errors are due to resource exhaustion. + /// Users errors are due to invalid permissions. + #[error("cannot create network device: {0}")] + CreateNetDevice(#[source] VirtioError), + + /// Cannot initialize a MMIO Network Device or add a device to the MMIO Bus. + #[error("failure while registering network device: {0}")] + RegisterNetDevice(#[source] DeviceMgrError), +} + +/// Virtio network config, working for virtio-net and vhost-net. +#[cfg(any(feature = "virtio-net", feature = "vhost-net"))] +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +pub struct VirtioConfig { + /// ID of the guest network interface. + pub iface_id: String, + /// Host level path for the guest network interface. + pub host_dev_name: String, + /// Rate Limiter for received packages. + pub rx_rate_limiter: Option, + /// Rate Limiter for transmitted packages. + pub tx_rate_limiter: Option, + /// Allow duplicate mac + pub allow_duplicate_mac: bool, +} + +/// Config for a vhost-user-net device. +#[cfg(feature = "vhost-user-net")] +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +pub struct VhostUserConfig { + /// ID of the guest network interface. + #[serde(default)] + pub iface_id: String, + /// Vhost-user socket path. + pub sock_path: String, +} + +/// An enum to specify a backend of a network device. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[serde(tag = "type", deny_unknown_fields)] +pub enum Backend { + #[serde(rename = "virtio")] + #[cfg(feature = "virtio-net")] + /// Virtio-net + Virtio(VirtioConfig), + #[serde(rename = "vhost")] + #[cfg(feature = "vhost-net")] + /// Vhost-net + Vhost(VirtioConfig), + #[serde(rename = "vhost-user")] + #[cfg(feature = "vhost-user-net")] + /// Vhost-user-net + VhostUser(VhostUserConfig), +} + +impl Backend { + /// Name of the backend, for diagnostics. + pub fn name(&self) -> &'static str { + #[allow(unreachable_patterns)] + match self { + #[cfg(feature = "virtio-net")] + Backend::Virtio(_) => "virtio-net", + #[cfg(feature = "vhost-net")] + Backend::Vhost(_) => "vhost-net", + #[cfg(feature = "vhost-user-net")] + Backend::VhostUser(_) => "vhost-user-net", + _ => "unknown", + } + } +} + +impl Default for Backend { + #[allow(unreachable_code)] + fn default() -> Self { + #[cfg(feature = "virtio-net")] + return Self::Virtio(VirtioConfig::default()); + #[cfg(feature = "vhost-net")] + return Self::Vhost(VirtioConfig::default()); + + panic!("no available default network backend") + } +} + +/// This struct represents the strongly typed equivalent of the json body from +/// net iface related requests, and is the configuration the manager stores for +/// every network device regardless of its backend. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize, Default)] +#[serde(deny_unknown_fields)] +pub struct NetworkInterfaceConfig { + /// Number of virtqueue pairs to use. (https://www.linux-kvm.org/page/Multiqueue) + pub num_queues: Option, + /// Size of each virtqueue. + pub queue_size: Option, + /// Net backend driver. + #[serde(default = "Backend::default")] + pub backend: Backend, + /// mac of the interface. + pub guest_mac: Option, + /// Use shared irq + pub use_shared_irq: Option, + /// Use generic irq + pub use_generic_irq: Option, +} + +impl NetworkInterfaceConfig { + /// Returns a reference to the mac address. If the mac address is not + /// configured, it returns None. + pub fn guest_mac(&self) -> Option<&MacAddr> { + self.guest_mac.as_ref() + } + + /// Number of virtqueues to use. + /// + /// The virtio-net backend has always ignored the requested queue count and + /// used the default; that behaviour is preserved here rather than changed + /// as a side effect of merging the managers. + pub fn num_queues(&self) -> usize { + #[cfg(feature = "virtio-net")] + if matches!(self.backend, Backend::Virtio(_)) { + return DEFAULT_NUM_QUEUES; + } + + match self.num_queues { + Some(0) | None => DEFAULT_NUM_QUEUES, + Some(num_queues) => num_queues, + } + } + + /// Size of each virtqueue. + pub fn queue_size(&self) -> u16 { + match self.queue_size { + Some(0) | None => DEFAULT_QUEUE_SIZE, + Some(queue_size) => queue_size, + } + } + + /// Rx and Tx queue and max queue sizes. + pub fn queue_sizes(&self) -> Vec { + let queue_size = self.queue_size(); + (0..self.num_queues()).map(|_| queue_size).collect() + } + + /// Whether the device uses the generic irq, defaulting per backend. + fn use_generic_irq(&self) -> bool { + #[allow(unused_mut, unused_assignments)] + let mut default = USE_GENERIC_IRQ; + #[cfg(feature = "vhost-user-net")] + if matches!(self.backend, Backend::VhostUser(_)) { + default = VHOST_USER_USE_GENERIC_IRQ; + } + self.use_generic_irq.unwrap_or(default) + } + + /// Host level path for the guest network interface, for the backends that + /// are backed by a tap device. `None` for backends that are not. + fn host_dev_name(&self) -> Option<&str> { + #[allow(unreachable_patterns)] + match &self.backend { + #[cfg(feature = "virtio-net")] + Backend::Virtio(config) => Some(&config.host_dev_name), + #[cfg(feature = "vhost-net")] + Backend::Vhost(config) => Some(&config.host_dev_name), + _ => None, + } + } + + /// Vhost-user socket path, for the backends that have one. `None` for + /// backends that are not vhost-user. + fn sock_path(&self) -> Option<&str> { + #[allow(unreachable_patterns)] + match &self.backend { + #[cfg(feature = "vhost-user-net")] + Backend::VhostUser(config) => Some(&config.sock_path), + _ => None, + } + } + + /// Whether a duplicate guest MAC address is allowed for this device. + fn allow_duplicate_mac(&self) -> bool { + #[allow(unreachable_patterns)] + match &self.backend { + #[cfg(feature = "virtio-net")] + Backend::Virtio(config) => config.allow_duplicate_mac, + #[cfg(feature = "vhost-net")] + Backend::Vhost(config) => config.allow_duplicate_mac, + _ => false, + } + } +} + +impl ConfigItem for NetworkInterfaceConfig { + type Err = NetworkDeviceError; + + /// Identity of the device: the interface id, as for every other device + /// class, and never a host resource such as a tap name or a socket path. + /// Supplied by the caller; an unnamed device is refused on insertion. + fn id(&self) -> &str { + #[allow(unreachable_patterns)] + match &self.backend { + #[cfg(feature = "virtio-net")] + Backend::Virtio(config) => &config.iface_id, + #[cfg(feature = "vhost-net")] + Backend::Vhost(config) => &config.iface_id, + #[cfg(feature = "vhost-user-net")] + Backend::VhostUser(config) => &config.iface_id, + _ => "", + } + } + + fn check_conflicts(&self, other: &Self) -> Result<(), NetworkDeviceError> { + if let Some(mac) = self.guest_mac.as_ref() { + if !other.allow_duplicate_mac() && Some(mac) == other.guest_mac.as_ref() { + return Err(NetworkDeviceError::GuestMacAddressInUse(mac.to_string())); + } + } + + if let (Some(name), Some(other_name)) = (self.host_dev_name(), other.host_dev_name()) { + if name == other_name { + return Err(NetworkDeviceError::HostDeviceNameInUse(name.to_owned())); + } + } + + // A vhost-user socket is served by one backend process for one + // device, so two devices may not share one. + if let (Some(path), Some(other_path)) = (self.sock_path(), other.sock_path()) { + if path == other_path { + return Err(NetworkDeviceError::DuplicatedUdsPath(path.to_owned())); + } + } + + Ok(()) + } +} + +/// The data fed into a network iface update request. Currently, only the RX and +/// TX rate limiters can be updated, and only for the virtio-net backend. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize, Default)] +#[serde(deny_unknown_fields)] +pub struct NetworkInterfaceUpdateConfig { + /// ID of the guest network interface. + pub iface_id: String, + /// New RX rate limiter config. Only provided data will be updated. I.e. if any optional data + /// is missing, it will not be nullified, but left unchanged. + pub rx_rate_limiter: Option, + /// New TX rate limiter config. Only provided data will be updated. I.e. if any optional data + /// is missing, it will not be nullified, but left unchanged. + pub tx_rate_limiter: Option, +} + +impl NetworkInterfaceUpdateConfig { + /// Provides a `BucketUpdate` description for the RX bandwidth rate limiter. + pub fn rx_bytes(&self) -> BucketUpdate { + get_bucket_update!(self, rx_rate_limiter, bandwidth) + } + /// Provides a `BucketUpdate` description for the RX ops rate limiter. + pub fn rx_ops(&self) -> BucketUpdate { + get_bucket_update!(self, rx_rate_limiter, ops) + } + /// Provides a `BucketUpdate` description for the TX bandwidth rate limiter. + pub fn tx_bytes(&self) -> BucketUpdate { + get_bucket_update!(self, tx_rate_limiter, bandwidth) + } + /// Provides a `BucketUpdate` description for the TX ops rate limiter. + pub fn tx_ops(&self) -> BucketUpdate { + get_bucket_update!(self, tx_rate_limiter, ops) + } +} + +/// Device manager to manage all network devices, of every backend. +pub struct NetworkDeviceMgr { + pub(crate) info_list: DeviceConfigInfos, + pub(crate) use_shared_irq: bool, +} + +impl NetworkDeviceMgr { + /// Gets the index of the device with the specified `iface_id` if it exists + /// in the list. + pub fn get_index_of_iface_id(&self, if_id: &str) -> Option { + self.info_list + .iter() + .position(|info| info.config.id() == if_id) + } + + /// Insert or update a network device into the manager. + pub fn insert_device( + &mut self, + mut ctx: DeviceOpContext, + config: NetworkInterfaceConfig, + ) -> std::result::Result<(), NetworkDeviceError> { + // The id is the name the interface is given inside the guest, so an + // unnamed device is not a device the guest can have. Refusing it here + // fails VM start, rather than letting it quietly take the place of + // another unnamed device and booting with one interface missing. + if config.id().is_empty() { + return Err(NetworkDeviceError::MissingIfaceId); + } + + if !config.num_queues().is_multiple_of(2) { + return Err(NetworkDeviceError::InvalidQueueNum(config.num_queues())); + } + if !cfg!(feature = "hotplug") && ctx.is_hotplug { + return Err(NetworkDeviceError::UpdateNotAllowedPostBoot); + } + // Before boot a repeated id reconfigures the device, as it does for + // every other device class. Once the device is attached that is no + // longer possible: the config would be replaced while the live device + // stayed on the bus with no handle left to remove it, and the guest + // would end up with two interfaces sharing one name. + if ctx.is_hotplug + && self + .info_list + .iter() + .any(|info| info.config.id() == config.id()) + { + return Err(NetworkDeviceError::DeviceIDAlreadyExist( + config.id().to_owned(), + )); + } + + slog::info!( + ctx.logger(), + "add network device configuration"; + "subsystem" => "net_dev_mgr", + "backend" => config.backend.name(), + "id" => config.id(), + ); + + let device_index = self.info_list.insert_or_update(&config)?; + + if ctx.is_hotplug { + slog::info!( + ctx.logger(), + "attach network device"; + "subsystem" => "net_dev_mgr", + "backend" => config.backend.name(), + "id" => config.id(), + ); + + match Self::create_device(&config, &mut ctx) { + Ok(device) => { + let dev = DeviceManager::create_mmio_virtio_device( + device, + &mut ctx, + config.use_shared_irq.unwrap_or(self.use_shared_irq), + config.use_generic_irq(), + ) + .map_err(NetworkDeviceError::DeviceManager)?; + ctx.insert_hotplug_mmio_device(&dev, None) + .map_err(NetworkDeviceError::DeviceManager)?; + // live-upgrade need save/restore device from info.device. + self.info_list[device_index].set_device(dev); + } + Err(e) => { + self.info_list.remove(device_index); + return Err(e); + } + } + } + + Ok(()) + } + + /// Update the ratelimiter settings of a virtio-net device. + /// + /// Only the virtio-net backend implements rate limiting; the request is + /// refused for the others rather than silently ignored. + pub fn update_device_ratelimiters( + &mut self, + new_cfg: NetworkInterfaceUpdateConfig, + ) -> std::result::Result<(), NetworkDeviceError> { + match self.get_index_of_iface_id(&new_cfg.iface_id) { + Some(index) => { + let config = &mut self.info_list[index].config; + #[allow(unreachable_patterns)] + match &mut config.backend { + #[cfg(feature = "virtio-net")] + Backend::Virtio(ref mut virtio_config) => { + // Patch semantics: an omitted (None) limiter is left + // unchanged rather than nulled, matching this struct's + // doc and the live-device patch below, where a None + // field maps to BucketUpdate::None ("no update"). A + // limiter is cleared by sending a zero-size bucket. + if new_cfg.rx_rate_limiter.is_some() { + virtio_config.rx_rate_limiter = new_cfg.rx_rate_limiter.clone(); + } + if new_cfg.tx_rate_limiter.is_some() { + virtio_config.tx_rate_limiter = new_cfg.tx_rate_limiter.clone(); + } + } + backend => { + let backend = backend.name(); + return Err(NetworkDeviceError::UnsupportedBackend { + iface_id: new_cfg.iface_id.clone(), + backend, + }); + } + } + let device = self.info_list[index] + .device + .as_mut() + .ok_or_else(|| NetworkDeviceError::InvalidIfaceId(new_cfg.iface_id.clone()))?; + + // Only the in-VMM virtio-net device implements rate limiting, + // so the patch path exists only with that feature. + #[cfg(feature = "virtio-net")] + if let Some(mmio_dev) = device.as_any().downcast_ref::() { + let guard = mmio_dev.state(); + let inner_dev = guard.get_inner_device(); + if let Some(net_dev) = inner_dev + .as_any() + .downcast_ref::>() + { + return net_dev + .set_patch_rate_limiters( + new_cfg.rx_bytes(), + new_cfg.rx_ops(), + new_cfg.tx_bytes(), + new_cfg.tx_ops(), + ) + .map(|_p| ()) + .map_err(|_e| NetworkDeviceError::NetEpollHandlerSendFail); + } + } + Ok(()) + } + None => Err(NetworkDeviceError::InvalidIfaceId(new_cfg.iface_id.clone())), + } + } + + /// Attach all configured network devices to the virtual machine instance. + pub fn attach_devices( + &mut self, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), NetworkDeviceError> { + for info in self.info_list.iter_mut() { + slog::info!( + ctx.logger(), + "attach network device"; + "subsystem" => "net_dev_mgr", + "backend" => info.config.backend.name(), + "id" => info.config.id(), + ); + + let device = Self::create_device(&info.config, ctx)?; + let device = DeviceManager::create_mmio_virtio_device( + device, + ctx, + info.config.use_shared_irq.unwrap_or(self.use_shared_irq), + info.config.use_generic_irq(), + ) + .map_err(NetworkDeviceError::RegisterNetDevice)?; + info.set_device(device); + } + + Ok(()) + } + + /// Create the backing device for `cfg`, dispatching on its backend. + fn create_device( + cfg: &NetworkInterfaceConfig, + ctx: &mut DeviceOpContext, + ) -> std::result::Result { + #[allow(unreachable_patterns)] + match &cfg.backend { + #[cfg(feature = "virtio-net")] + Backend::Virtio(config) => Self::create_virtio_net_device(cfg, config, ctx), + #[cfg(feature = "vhost-net")] + Backend::Vhost(config) => Self::create_vhost_net_device(cfg, config, ctx), + #[cfg(feature = "vhost-user-net")] + Backend::VhostUser(config) => Self::create_vhost_user_net_device(cfg, config, ctx), + _ => Err(NetworkDeviceError::UnsupportedBackend { + iface_id: cfg.id().to_owned(), + backend: cfg.backend.name(), + }), + } + } + + #[cfg(feature = "virtio-net")] + fn create_virtio_net_device( + cfg: &NetworkInterfaceConfig, + config: &VirtioConfig, + ctx: &mut DeviceOpContext, + ) -> std::result::Result { + let epoll_mgr = ctx + .epoll_mgr + .clone() + .ok_or(NetworkDeviceError::Virtio(VirtioError::InvalidInput))?; + let rx_rate_limiter = match config.rx_rate_limiter.as_ref() { + Some(rl) => Some( + rl.try_into() + .map_err(|e| NetworkDeviceError::CreateNetDevice(VirtioError::IOError(e)))?, + ), + None => None, + }; + let tx_rate_limiter = match config.tx_rate_limiter.as_ref() { + Some(rl) => Some( + rl.try_into() + .map_err(|e| NetworkDeviceError::CreateNetDevice(VirtioError::IOError(e)))?, + ), + None => None, + }; + + let net_device = virtio::net::Net::new( + config.host_dev_name.clone(), + cfg.guest_mac(), + Arc::new(cfg.queue_sizes()), + epoll_mgr, + rx_rate_limiter, + tx_rate_limiter, + ) + .map_err(NetworkDeviceError::CreateNetDevice)?; + + Ok(Box::new(net_device)) + } + + #[cfg(feature = "vhost-net")] + fn create_vhost_net_device( + cfg: &NetworkInterfaceConfig, + config: &VirtioConfig, + ctx: &mut DeviceOpContext, + ) -> std::result::Result { + slog::info!( + ctx.logger(), + "create a vhost-net device"; + "subsystem" => "net_dev_mgr", + "id" => &config.iface_id, + "host_dev_name" => &config.host_dev_name, + ); + let epoll_mgr = ctx + .epoll_mgr + .clone() + .ok_or(NetworkDeviceError::Virtio(VirtioError::InvalidInput))?; + + Ok(Box::new( + virtio::vhost::vhost_kern::net::Net::new( + config.host_dev_name.clone(), + cfg.guest_mac(), + Arc::new(cfg.queue_sizes()), + epoll_mgr, + ) + .map_err(NetworkDeviceError::CreateNetDevice)?, + )) + } + + #[cfg(feature = "vhost-user-net")] + fn create_vhost_user_net_device( + cfg: &NetworkInterfaceConfig, + config: &VhostUserConfig, + ctx: &mut DeviceOpContext, + ) -> std::result::Result { + let epoll_mgr = ctx + .epoll_mgr + .clone() + .ok_or(NetworkDeviceError::Virtio(VirtioError::InvalidInput))?; + + Ok(Box::new( + virtio::vhost::vhost_user::net::VhostUserNet::new_server( + &config.sock_path, + cfg.guest_mac(), + Arc::new(cfg.queue_sizes()), + epoll_mgr, + ) + .map_err(NetworkDeviceError::CreateNetDevice)?, + )) + } + + /// Remove all network devices. + /// + /// A device that fails to be destroyed is logged and skipped rather than + /// aborting the teardown: this runs while the VM is being torn down, and + /// one broken device (e.g. a vhost-user device whose backend process is + /// already gone) must not leave every remaining device attached. + pub fn remove_devices(&mut self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> { + while let Some(mut info) = self.info_list.pop() { + slog::info!(ctx.logger(), "remove network device: {}", info.config.id()); + if let Some(device) = info.device.take() { + if let Err(e) = DeviceManager::destroy_mmio_device(device, ctx) { + slog::error!( + ctx.logger(), + "failed to destroy network device, continuing teardown"; + "subsystem" => "net_dev_mgr", + "id" => info.config.id(), + "error" => format!("{e:?}"), + ); + } + } + } + Ok(()) + } +} + +impl Default for NetworkDeviceMgr { + /// Create a new network device manager. + fn default() -> Self { + NetworkDeviceMgr { + info_list: DeviceConfigInfos::new(), + use_shared_irq: USE_SHARED_IRQ, + } + } +} + +#[cfg(test)] +mod tests { + use dbs_utils::net::MacAddr; + use test_utils::{skip_if_kvm_unaccessable, skip_if_not_root}; + + use super::*; + use crate::device_manager::DeviceManager; + use crate::test_utils::tests::create_vm_for_test; + use crate::vm::VmConfigInfo; + + fn ctx_without_epoll_mgr(mgr: &DeviceManager, vm: &crate::vm::Vm) -> DeviceOpContext { + DeviceOpContext::new( + None, + mgr, + None, + None, + false, + Some(VmConfigInfo::default()), + vm.shared_info().clone(), + ) + } + + #[cfg(any(feature = "virtio-net", feature = "vhost-net"))] + fn virtio_backend_config(iface_id: &str, host_dev_name: &str) -> VirtioConfig { + VirtioConfig { + iface_id: String::from(iface_id), + host_dev_name: String::from(host_dev_name), + rx_rate_limiter: None, + tx_rate_limiter: None, + allow_duplicate_mac: false, + } + } + + fn net_config(backend: Backend, guest_mac: &str, num_queues: usize) -> NetworkInterfaceConfig { + NetworkInterfaceConfig { + num_queues: Some(num_queues), + queue_size: Some(128), + backend, + guest_mac: Some(MacAddr::parse_str(guest_mac).unwrap()), + use_shared_irq: None, + use_generic_irq: None, + } + } + + #[cfg(feature = "virtio-net")] + #[test] + fn test_network_interface_config_from_json() { + let json_str = r#"{ + "num_queues": 4, + "queue_size": 512, + "backend": { + "type": "virtio", + "iface_id": "eth0", + "host_dev_name": "tap0", + "allow_duplicate_mac": true + }, + "guest_mac": "81:87:1D:00:08:A9" + }"#; + let net_config: NetworkInterfaceConfig = serde_json::from_str(json_str).unwrap(); + assert_eq!(net_config.num_queues, Some(4)); + assert_eq!(net_config.queue_size, Some(512)); + assert_eq!( + net_config.guest_mac, + Some(MacAddr::from_bytes(&[129, 135, 29, 0, 8, 169]).unwrap()) + ); + if let Backend::Virtio(config) = &net_config.backend { + assert_eq!(config.iface_id, "eth0"); + assert_eq!(config.host_dev_name, "tap0"); + assert!(config.allow_duplicate_mac); + } else { + panic!("Unexpected backend type"); + } + // The virtio backend has always used the default queue count. + assert_eq!(net_config.num_queues(), DEFAULT_NUM_QUEUES); + assert_eq!(net_config.queue_size(), 512); + assert_eq!(net_config.id(), "eth0"); + } + + #[test] + fn test_backend_identity_and_name() { + #[cfg(feature = "virtio-net")] + { + let cfg = net_config( + Backend::Virtio(virtio_backend_config("id_1", "dev1")), + "01:23:45:67:89:0a", + 2, + ); + assert_eq!(cfg.id(), "id_1"); + assert_eq!(cfg.backend.name(), "virtio-net"); + } + #[cfg(feature = "vhost-net")] + { + let cfg = net_config( + Backend::Vhost(virtio_backend_config("id_2", "dev2")), + "01:23:45:67:89:0b", + 2, + ); + assert_eq!(cfg.id(), "id_2"); + assert_eq!(cfg.backend.name(), "vhost-net"); + } + #[cfg(feature = "vhost-user-net")] + { + // Supplied id is the identity, exactly as for the other backends; + // the socket path is a resource, not an identity. + let cfg = net_config( + Backend::VhostUser(VhostUserConfig { + iface_id: String::from("id_3"), + sock_path: String::from("/tmp/sock_1"), + }), + "01:23:45:67:89:0c", + 2, + ); + assert_eq!(cfg.id(), "id_3"); + assert_eq!(cfg.backend.name(), "vhost-user-net"); + + // With no id supplied, id() is empty until a caller sets one. + let cfg = net_config( + Backend::VhostUser(VhostUserConfig { + iface_id: String::new(), + sock_path: String::from("/tmp/sock_2"), + }), + "01:23:45:67:89:0d", + 2, + ); + assert_eq!(cfg.id(), ""); + } + } + + #[cfg(feature = "vhost-net")] + #[test] + fn test_create_vhost_net_device() { + skip_if_kvm_unaccessable!(); + let vm = create_vm_for_test(); + let mgr = DeviceManager::new_test_mgr(); + let netif_1 = net_config( + Backend::Vhost(virtio_backend_config("id_1", "dev1")), + "01:23:45:67:89:0a", + 2, + ); + + // no epoll manager + let mut ctx = ctx_without_epoll_mgr(&mgr, &vm); + assert!(NetworkDeviceMgr::create_device(&netif_1, &mut ctx).is_err()); + } + + #[cfg(feature = "vhost-user-net")] + #[test] + fn test_create_vhost_user_net_device() { + skip_if_kvm_unaccessable!(); + let vm = create_vm_for_test(); + let mgr = DeviceManager::new_test_mgr(); + let netif_1 = net_config( + Backend::VhostUser(VhostUserConfig { + iface_id: String::new(), + sock_path: String::from("/tmp/vhost_user_net_test"), + }), + "01:23:45:67:89:0a", + 2, + ); + + // no epoll manager + let mut ctx = ctx_without_epoll_mgr(&mgr, &vm); + assert!(NetworkDeviceMgr::create_device(&netif_1, &mut ctx).is_err()); + } + + #[cfg(feature = "vhost-net")] + #[test] + fn test_attach_vhost_net_device() { + skip_if_kvm_unaccessable!(); + // Attaching the device opens the tap named by the config, and + // TUNSETIFF creates it when absent, which needs CAP_NET_ADMIN. + skip_if_not_root!(); + let mut vm = create_vm_for_test(); + let device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + vm.vm_address_space().cloned(), + false, + Some(VmConfigInfo::default()), + vm.shared_info().clone(), + ); + + let netif_1 = net_config( + Backend::Vhost(virtio_backend_config("id_1", "dev1")), + "01:23:45:67:89:0a", + 2, + ); + + assert!(vm + .device_manager_mut() + .net_manager + .insert_device(device_op_ctx, netif_1) + .is_ok()); + assert_eq!(vm.device_manager().net_manager.info_list.len(), 1); + + let mut device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + vm.vm_address_space().cloned(), + false, + Some(VmConfigInfo::default()), + vm.shared_info().clone(), + ); + + assert!(vm + .device_manager_mut() + .net_manager + .attach_devices(&mut device_op_ctx) + .is_ok()); + } + + #[cfg(feature = "vhost-net")] + #[test] + fn test_insert_vhost_net_device() { + skip_if_kvm_unaccessable!(); + let vm = create_vm_for_test(); + let mut mgr = DeviceManager::new_test_mgr(); + + let mut netif_1 = net_config( + Backend::Vhost(virtio_backend_config("id_1", "dev1")), + "01:23:45:67:89:0a", + 2, + ); + + let ctx = ctx_without_epoll_mgr(&mgr, &vm); + assert!(mgr.net_manager.insert_device(ctx, netif_1.clone()).is_ok()); + assert_eq!(mgr.net_manager.info_list.len(), 1); + + // Before boot, repeating an id reconfigures that device in place, as + // it does for every other device class. + netif_1.guest_mac = Some(MacAddr::parse_str("01:23:45:67:89:0b").unwrap()); + let ctx = ctx_without_epoll_mgr(&mgr, &vm); + assert!(mgr.net_manager.insert_device(ctx, netif_1.clone()).is_ok()); + assert_eq!(mgr.net_manager.info_list.len(), 1); + + netif_1.backend = Backend::Vhost(virtio_backend_config("id_1", "dev2")); + let ctx = ctx_without_epoll_mgr(&mgr, &vm); + assert!(mgr.net_manager.insert_device(ctx, netif_1).is_ok()); + assert_eq!(mgr.net_manager.info_list.len(), 1); + } + + #[cfg(feature = "vhost-user-net")] + #[test] + fn test_insert_vhost_user_net_device() { + skip_if_kvm_unaccessable!(); + let vm = create_vm_for_test(); + let mut mgr = DeviceManager::new_test_mgr(); + + let netif_1 = net_config( + Backend::VhostUser(VhostUserConfig { + iface_id: String::from("id_1"), + sock_path: String::from("/tmp/vhost_user_net_insert"), + }), + "01:23:45:67:89:0a", + 2, + ); + + let ctx = ctx_without_epoll_mgr(&mgr, &vm); + assert!(mgr.net_manager.insert_device(ctx, netif_1).is_ok()); + assert_eq!(mgr.net_manager.info_list.len(), 1); + } + + #[cfg(feature = "vhost-net")] + #[test] + fn test_net_insert_error_cases() { + skip_if_kvm_unaccessable!(); + let vm = create_vm_for_test(); + let mut mgr = DeviceManager::new_test_mgr(); + + // invalid queue num + let mut netif_1 = net_config( + Backend::Vhost(virtio_backend_config("id_1", "dev_1")), + "01:23:45:67:89:0a", + 1, + ); + let ctx = ctx_without_epoll_mgr(&mgr, &vm); + let res = mgr.net_manager.insert_device(ctx, netif_1.clone()); + assert!(matches!(res, Err(NetworkDeviceError::InvalidQueueNum(1)))); + assert_eq!(mgr.net_manager.info_list.len(), 0); + + // Adding the first valid network config. + netif_1.num_queues = Some(2); + let ctx = ctx_without_epoll_mgr(&mgr, &vm); + assert!(mgr.net_manager.insert_device(ctx, netif_1.clone()).is_ok()); + assert_eq!(mgr.net_manager.info_list.len(), 1); + + // Error Case: add a new device sharing the same host_dev_name. + let mut netif_2 = netif_1.clone(); + netif_2.backend = Backend::Vhost(virtio_backend_config("id_2", "dev_1")); + netif_2.guest_mac = Some(MacAddr::parse_str("11:45:45:67:89:0b").unwrap()); + let ctx = ctx_without_epoll_mgr(&mgr, &vm); + let res = mgr.net_manager.insert_device(ctx, netif_2.clone()); + assert!(matches!( + res, + Err(NetworkDeviceError::HostDeviceNameInUse(_)) + )); + + // Error Case: add a new device sharing the same guest MAC address. + netif_2.backend = Backend::Vhost(virtio_backend_config("id_2", "dev_2")); + netif_2.guest_mac = netif_1.guest_mac; + let ctx = ctx_without_epoll_mgr(&mgr, &vm); + let res = mgr.net_manager.insert_device(ctx, netif_2); + assert!(matches!( + res, + Err(NetworkDeviceError::GuestMacAddressInUse(_)) + )); + } + + #[cfg(feature = "vhost-user-net")] + #[test] + fn test_vhost_user_net_insert_error_cases() { + skip_if_kvm_unaccessable!(); + let vm = create_vm_for_test(); + let mut mgr = DeviceManager::new_test_mgr(); + + // invalid queue num + let netif_1 = net_config( + Backend::VhostUser(VhostUserConfig { + iface_id: String::from("id_1"), + sock_path: String::from("/tmp/vhost_user_net_err"), + }), + "01:23:45:67:89:0a", + 1, + ); + let ctx = ctx_without_epoll_mgr(&mgr, &vm); + let res = mgr.net_manager.insert_device(ctx, netif_1); + assert!(matches!(res, Err(NetworkDeviceError::InvalidQueueNum(1)))); + assert_eq!(mgr.net_manager.info_list.len(), 0); + } + + #[cfg(feature = "vhost-user-net")] + #[test] + fn test_vhost_user_net_duplicate_sock_path() { + skip_if_kvm_unaccessable!(); + let vm = create_vm_for_test(); + let mut mgr = DeviceManager::new_test_mgr(); + + // Distinct MACs, so it is the socket clash that is reported rather + // than the MAC one. + let mk = |iface_id: &str, mac: &str| { + net_config( + Backend::VhostUser(VhostUserConfig { + iface_id: String::from(iface_id), + sock_path: String::from("/tmp/vhost_user_net_dup"), + }), + mac, + 2, + ) + }; + + let ctx = ctx_without_epoll_mgr(&mgr, &vm); + assert!(mgr + .net_manager + .insert_device(ctx, mk("id_1", "01:23:45:67:89:0a")) + .is_ok()); + assert_eq!(mgr.net_manager.info_list.len(), 1); + + // A second device with its own id but the same socket is refused: + // one vhost-user socket is served by one backend for one device. + let ctx = ctx_without_epoll_mgr(&mgr, &vm); + let res = mgr + .net_manager + .insert_device(ctx, mk("id_2", "01:23:45:67:89:0b")); + assert!(matches!(res, Err(NetworkDeviceError::DuplicatedUdsPath(_)))); + assert_eq!(mgr.net_manager.info_list.len(), 1); + + // Re-inserting the same id before boot reconfigures that device: its + // own socket is not a clash with itself. + let ctx = ctx_without_epoll_mgr(&mgr, &vm); + assert!(mgr + .net_manager + .insert_device(ctx, mk("id_1", "01:23:45:67:89:0c")) + .is_ok()); + assert_eq!(mgr.net_manager.info_list.len(), 1); + } + + #[cfg(feature = "vhost-user-net")] + #[test] + fn test_unnamed_device_is_refused() { + skip_if_kvm_unaccessable!(); + let vm = create_vm_for_test(); + let mut mgr = DeviceManager::new_test_mgr(); + + // The id becomes the interface name in the guest, so an unnamed + // device is refused instead of silently replacing another one. + let cfg = net_config( + Backend::VhostUser(VhostUserConfig { + iface_id: String::new(), + sock_path: String::from("/tmp/sock_a"), + }), + "01:23:45:67:89:0a", + 2, + ); + let ctx = ctx_without_epoll_mgr(&mgr, &vm); + let res = mgr.net_manager.insert_device(ctx, cfg); + assert!(matches!(res, Err(NetworkDeviceError::MissingIfaceId))); + assert_eq!(mgr.net_manager.info_list.len(), 0); + } + + #[test] + fn test_net_device_error_display() { + let err = NetworkDeviceError::DuplicatedUdsPath(String::from("1")); + let _ = format!("{err}{err:?}"); + + let err = NetworkDeviceError::InvalidQueueNum(1); + let _ = format!("{err}{err:?}"); + + let err = NetworkDeviceError::UnsupportedBackend { + iface_id: String::from("id_1"), + backend: "vhost-user-net", + }; + let _ = format!("{err}{err:?}"); + } +} diff --git a/src/dragonball/src/device_manager/vhost_net_dev_mgr.rs b/src/dragonball/src/device_manager/vhost_net_dev_mgr.rs deleted file mode 100644 index c296a6d9b4..0000000000 --- a/src/dragonball/src/device_manager/vhost_net_dev_mgr.rs +++ /dev/null @@ -1,643 +0,0 @@ -// Copyright (C) 2019-2023 Alibaba Cloud. All rights reserved. -// Copyright (C) 2019-2023 Ant Group. All rights reserved. -// -// SPDX-License-Identifier: Apache-2.0 - -use std::result::Result; -use std::sync::Arc; - -use dbs_utils::net::MacAddr; -use dbs_virtio_devices::vhost::vhost_kern::net::Net; -use dbs_virtio_devices::Error as VirtioError; -use serde::{Deserialize, Serialize}; -use virtio_queue::QueueSync; - -use super::{DeviceManager, DeviceMgrError, DeviceOpContext}; -use crate::address_space_manager::{GuestAddressSpaceImpl, GuestRegionImpl}; -use crate::config_manager::{ConfigItem, DeviceConfigInfos}; - -/// Default number of virtio queues, one rx/tx pair. -pub const DEFAULT_NUM_QUEUES: usize = 2; -/// Default size of virtio queues. -pub const DEFAULT_QUEUE_SIZE: u16 = 256; -// The flag of whether to use the shared irq. -const USE_SHARED_IRQ: bool = true; -// The flag of whether to use the generic irq. -const USE_GENERIC_IRQ: bool = true; - -#[derive(Debug, thiserror::Error)] -/// Errors associated with vhost-net device operations -pub enum VhostNetDeviceError { - /// The Context Identifier is already in use. - #[error("the device id {0} already exists")] - DeviceIdAlreadyExist(String), - /// The MAC address is already in use. - #[error("the guest Mac address {0} is already in use")] - GuestMacAddressInUse(String), - /// The host device name is already in use. - #[error("the host device name {0} is already in use")] - HostDeviceNameInUse(String), - /// The update isn't allowed after booting the mircovm. - #[error("update operation is not allowed after booting")] - UpdateNotAllowedPostBoot, - /// Invalid queue number for vhost-net device. - #[error("invalid queue number {0} for vhost-net device")] - InvalidQueueNum(usize), - /// Failure from device manager. - #[error("failure in device manager operations: {0:?}")] - DeviceManager(#[source] DeviceMgrError), - /// Failure from virtio subsystem. - #[error("virtio error: {0:?}")] - Virtio(VirtioError), - /// Split this at some point. - /// Internal errors are due to resource exhaustion. - /// Users errors are due to invalid permissions. - #[error("cannot create a vhost-net device: {0}")] - CreateNetDevice(#[source] VirtioError), - /// Cannot initialize a MMIO Network Device or add a device to the MMIO Bus. - #[error("failure while registering vhost-net device: {0}")] - RegisterNetDevice(#[source] DeviceMgrError), -} - -/// Configuration information for vhost net devices. -/// TODO: https://github.com/kata-containers/kata-containers/issues/8382. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct VhostNetDeviceConfigInfo { - /// Id of the guest network interface. - pub iface_id: String, - /// Host level path for the guest network interface. - pub host_dev_name: String, - /// Number of virtqueues to use. - pub num_queues: usize, - /// Size of each virtqueue. - pub queue_size: u16, - /// Guest MAC address. - pub guest_mac: Option, - /// allow duplicate mac - pub allow_duplicate_mac: bool, - /// Use shared irq - pub use_shared_irq: Option, - /// Use shared irq - pub use_generic_irq: Option, -} - -impl VhostNetDeviceConfigInfo { - /// Returns a reference to the mac address. Its mac address is not - /// configured, it returns None. - pub fn guest_mac(&self) -> Option<&MacAddr> { - self.guest_mac.as_ref() - } - - /// Returns rx and tx queue sizes, the length is num_queues, each value is - /// queue_size. - pub fn queue_sizes(&self) -> Vec { - let queue_size = if self.queue_size > 0 { - self.queue_size - } else { - DEFAULT_QUEUE_SIZE - }; - let num_queues = if self.num_queues > 0 { - self.num_queues - } else { - DEFAULT_NUM_QUEUES - }; - - (0..num_queues).map(|_| queue_size).collect() - } -} - -impl ConfigItem for VhostNetDeviceConfigInfo { - type Err = VhostNetDeviceError; - - fn id(&self) -> &str { - &self.iface_id - } - - fn check_conflicts(&self, other: &Self) -> Result<(), Self::Err> { - if self.iface_id == other.iface_id { - Err(VhostNetDeviceError::DeviceIdAlreadyExist( - self.iface_id.clone(), - )) - } else if !other.allow_duplicate_mac - && self.guest_mac.is_some() - && self.guest_mac == other.guest_mac - { - Err(VhostNetDeviceError::GuestMacAddressInUse( - self.guest_mac.as_ref().unwrap().to_string(), - )) - } else if self.host_dev_name == other.host_dev_name { - Err(VhostNetDeviceError::HostDeviceNameInUse( - self.host_dev_name.clone(), - )) - } else { - Ok(()) - } - } -} - -/// Device manager to manage all vhost net devices. -pub struct VhostNetDeviceMgr { - info_list: DeviceConfigInfos, - use_shared_irq: bool, -} - -impl VhostNetDeviceMgr { - /// Create a `vhost_kern::net::Net` struct representing a vhost-net device. - fn create_device( - cfg: &VhostNetDeviceConfigInfo, - ctx: &mut DeviceOpContext, - ) -> Result>, VirtioError> { - slog::info!( - ctx.logger(), - "create a vhost-net device"; - "subsystem" => "vhost_net_dev_mgr", - "id" => &cfg.iface_id, - "host_dev_name" => &cfg.host_dev_name, - ); - let epoll_mgr = ctx.epoll_mgr.clone().ok_or(VirtioError::InvalidInput)?; - Ok(Box::new(Net::new( - cfg.host_dev_name.clone(), - cfg.guest_mac(), - Arc::new(cfg.queue_sizes()), - epoll_mgr, - )?)) - } - - /// Insert or update a vhost-net device into the device manager. If it is a - /// hotplug device, then it will be attached to the hypervisor. - pub fn insert_device( - device_mgr: &mut DeviceManager, - mut ctx: DeviceOpContext, - config: VhostNetDeviceConfigInfo, - ) -> Result<(), VhostNetDeviceError> { - if !config.num_queues.is_multiple_of(2) { - return Err(VhostNetDeviceError::InvalidQueueNum(config.num_queues)); - } - if !cfg!(feature = "hotplug") && ctx.is_hotplug { - return Err(VhostNetDeviceError::UpdateNotAllowedPostBoot); - } - - slog::info!( - ctx.logger(), - "add vhost-net device configuration"; - "subsystem" => "vhost_net_dev_mgr", - "id" => &config.iface_id, - "host_dev_name" => &config.host_dev_name, - ); - - let mgr = &mut device_mgr.vhost_net_manager; - let device_index = mgr.info_list.insert_or_update(&config)?; - - // If it is a hotplug device, then it will be attached immediately. - if ctx.is_hotplug { - slog::info!( - ctx.logger(), - "attach vhost-net device"; - "subsystem" => "vhost_net_dev_mgr", - "id" => &config.iface_id, - "host_dev_name" => &config.host_dev_name, - ); - - match Self::create_device(&config, &mut ctx) { - Ok(device) => { - let mmio_dev = DeviceManager::create_mmio_virtio_device( - device, - &mut ctx, - config.use_shared_irq.unwrap_or(mgr.use_shared_irq), - config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), - ) - .map_err(VhostNetDeviceError::RegisterNetDevice)?; - ctx.insert_hotplug_mmio_device(&mmio_dev, None) - .map_err(VhostNetDeviceError::DeviceManager)?; - // live-upgrade need save/restore device from info.device. - mgr.info_list[device_index].set_device(mmio_dev); - } - Err(err) => { - mgr.info_list.remove(device_index); - return Err(VhostNetDeviceError::Virtio(err)); - } - } - } - - Ok(()) - } - - /// Attach all configured vhost-net device to the virtual machine instance. - pub fn attach_devices(&mut self, ctx: &mut DeviceOpContext) -> Result<(), VhostNetDeviceError> { - for info in self.info_list.iter_mut() { - slog::info!( - ctx.logger(), - "attach vhost-net device"; - "subsystem" => "vhost_net_dev_mgr", - "id" => &info.config.iface_id, - "host_dev_name" => &info.config.host_dev_name, - ); - - let device = Self::create_device(&info.config, ctx) - .map_err(VhostNetDeviceError::CreateNetDevice)?; - let mmio_dev = DeviceManager::create_mmio_virtio_device( - device, - ctx, - info.config.use_shared_irq.unwrap_or(self.use_shared_irq), - info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), - ) - .map_err(VhostNetDeviceError::RegisterNetDevice)?; - info.set_device(mmio_dev); - } - Ok(()) - } - - /// Remove all vhost-net devices. - pub fn remove_devices(&mut self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> { - while let Some(mut info) = self.info_list.pop() { - slog::info!( - ctx.logger(), - "remove virtio-net device: {}", - info.config.iface_id - ); - if let Some(device) = info.device.take() { - DeviceManager::destroy_mmio_device(device, ctx)?; - } - } - - Ok(()) - } -} - -impl Default for VhostNetDeviceMgr { - fn default() -> Self { - Self { - info_list: DeviceConfigInfos::new(), - use_shared_irq: USE_SHARED_IRQ, - } - } -} - -#[cfg(test)] -mod tests { - use dbs_utils::net::MacAddr; - use dbs_virtio_devices::Error as VirtioError; - use test_utils::skip_if_kvm_unaccessable; - - use crate::{ - device_manager::{ - vhost_net_dev_mgr::{VhostNetDeviceConfigInfo, VhostNetDeviceError, VhostNetDeviceMgr}, - DeviceManager, DeviceMgrError, DeviceOpContext, - }, - test_utils::tests::create_vm_for_test, - vm::VmConfigInfo, - }; - - #[test] - fn test_create_vhost_net_device() { - skip_if_kvm_unaccessable!(); - let vm = create_vm_for_test(); - let mgr = DeviceManager::new_test_mgr(); - let id_1 = String::from("id_1"); - let host_dev_name_1 = String::from("dev1"); - let guest_mac_1 = "01:23:45:67:89:0a"; - - let netif_1 = VhostNetDeviceConfigInfo { - iface_id: id_1, - host_dev_name: host_dev_name_1, - num_queues: 2, - queue_size: 128, - guest_mac: Some(MacAddr::parse_str(guest_mac_1).unwrap()), - allow_duplicate_mac: false, - use_shared_irq: None, - use_generic_irq: None, - }; - - // no epoll manager - let mut ctx = DeviceOpContext::new( - None, - &mgr, - None, - None, - false, - Some(VmConfigInfo::default()), - vm.shared_info().clone(), - ); - assert!(VhostNetDeviceMgr::create_device(&netif_1, &mut ctx).is_err()); - } - - #[test] - fn test_attach_vhost_net_device() { - skip_if_kvm_unaccessable!(); - // Init vm for test. - let mut vm = create_vm_for_test(); - let device_op_ctx = DeviceOpContext::new( - Some(vm.epoll_manager().clone()), - vm.device_manager(), - Some(vm.vm_as().unwrap().clone()), - vm.vm_address_space().cloned(), - false, - Some(VmConfigInfo::default()), - vm.shared_info().clone(), - ); - - let id_1 = String::from("id_1"); - let host_dev_name_1 = String::from("dev1"); - let guest_mac_1 = "01:23:45:67:89:0a"; - - let netif_1 = VhostNetDeviceConfigInfo { - iface_id: id_1, - host_dev_name: host_dev_name_1, - num_queues: 2, - queue_size: 128, - guest_mac: Some(MacAddr::parse_str(guest_mac_1).unwrap()), - allow_duplicate_mac: false, - use_shared_irq: None, - use_generic_irq: None, - }; - - assert!( - VhostNetDeviceMgr::insert_device(vm.device_manager_mut(), device_op_ctx, netif_1) - .is_ok() - ); - assert_eq!(vm.device_manager().vhost_net_manager.info_list.len(), 1); - - let mut device_op_ctx = DeviceOpContext::new( - Some(vm.epoll_manager().clone()), - vm.device_manager(), - Some(vm.vm_as().unwrap().clone()), - vm.vm_address_space().cloned(), - false, - Some(VmConfigInfo::default()), - vm.shared_info().clone(), - ); - - assert!(vm - .device_manager_mut() - .vhost_net_manager - .attach_devices(&mut device_op_ctx) - .is_ok()); - } - - #[test] - fn test_insert_vhost_net_device() { - skip_if_kvm_unaccessable!(); - let vm = create_vm_for_test(); - let mut mgr = DeviceManager::new_test_mgr(); - - let id_1 = String::from("id_1"); - let mut host_dev_name_1 = String::from("dev1"); - let mut guest_mac_1 = "01:23:45:67:89:0a"; - - // Test create. - let mut netif_1 = VhostNetDeviceConfigInfo { - iface_id: id_1, - host_dev_name: host_dev_name_1, - num_queues: 2, - queue_size: 128, - guest_mac: Some(MacAddr::parse_str(guest_mac_1).unwrap()), - allow_duplicate_mac: false, - use_shared_irq: None, - use_generic_irq: None, - }; - - let ctx = DeviceOpContext::new( - None, - &mgr, - None, - None, - false, - Some(VmConfigInfo::default()), - vm.shared_info().clone(), - ); - assert!(VhostNetDeviceMgr::insert_device(&mut mgr, ctx, netif_1.clone()).is_ok()); - assert_eq!(mgr.vhost_net_manager.info_list.len(), 1); - - // Test update mac address (this test does not modify the tap). - guest_mac_1 = "01:23:45:67:89:0b"; - netif_1.guest_mac = Some(MacAddr::parse_str(guest_mac_1).unwrap()); - let ctx = DeviceOpContext::new( - None, - &mgr, - None, - None, - false, - Some(VmConfigInfo::default()), - vm.shared_info().clone(), - ); - assert!(VhostNetDeviceMgr::insert_device(&mut mgr, ctx, netif_1.clone()).is_ok()); - assert_eq!(mgr.vhost_net_manager.info_list.len(), 1); - - // Test update host_dev_name (the tap will be updated). - host_dev_name_1 = String::from("dev2"); - netif_1.host_dev_name = host_dev_name_1; - let ctx = DeviceOpContext::new( - None, - &mgr, - None, - None, - false, - Some(VmConfigInfo::default()), - vm.shared_info().clone(), - ); - assert!(VhostNetDeviceMgr::insert_device(&mut mgr, ctx, netif_1).is_ok()); - assert_eq!(mgr.vhost_net_manager.info_list.len(), 1); - } - - #[test] - fn test_vhost_net_insert_error_cases() { - skip_if_kvm_unaccessable!(); - let vm = create_vm_for_test(); - let mut mgr = DeviceManager::new_test_mgr(); - - let guest_mac_1 = "01:23:45:67:89:0a"; - let guest_mac_2 = "11:45:45:67:89:0b"; - - let ctx = DeviceOpContext::new( - None, - &mgr, - None, - None, - false, - Some(VmConfigInfo::default()), - vm.shared_info().clone(), - ); - - // invalid queue num - let mut netif_1 = VhostNetDeviceConfigInfo { - iface_id: String::from("id_1"), - host_dev_name: String::from("dev_1"), - num_queues: 1, - queue_size: 128, - guest_mac: Some(MacAddr::parse_str(guest_mac_1).unwrap()), - allow_duplicate_mac: false, - use_shared_irq: None, - use_generic_irq: None, - }; - - let res = VhostNetDeviceMgr::insert_device(&mut mgr, ctx, netif_1.clone()); - if let Err(VhostNetDeviceError::InvalidQueueNum(1)) = res { - assert_eq!(mgr.vhost_net_manager.info_list.len(), 0); - } else { - panic!(); - } - - // Adding the first valid network config. - netif_1.num_queues = 2; - let ctx = DeviceOpContext::new( - None, - &mgr, - None, - None, - false, - Some(VmConfigInfo::default()), - vm.shared_info().clone(), - ); - assert!(VhostNetDeviceMgr::insert_device(&mut mgr, ctx, netif_1.clone()).is_ok()); - assert_eq!(mgr.vhost_net_manager.info_list.len(), 1); - - // Error Cases for CREATE - // Error Case: Add new network config with the same host_dev_name - netif_1.iface_id = String::from("id_2"); - netif_1.guest_mac = Some(MacAddr::parse_str(guest_mac_2).unwrap()); - let ctx = DeviceOpContext::new( - None, - &mgr, - None, - None, - false, - Some(VmConfigInfo::default()), - vm.shared_info().clone(), - ); - let res = VhostNetDeviceMgr::insert_device(&mut mgr, ctx, netif_1.clone()); - if let Err(VhostNetDeviceError::HostDeviceNameInUse(_)) = res { - assert_eq!(mgr.vhost_net_manager.info_list.len(), 1); - } else { - panic!(); - } - - // Error Cases for CREATE - // Error Case: Add new network config with the same guest_address - netif_1.iface_id = String::from("id_2"); - netif_1.host_dev_name = String::from("dev_2"); - netif_1.guest_mac = Some(MacAddr::parse_str(guest_mac_1).unwrap()); - let ctx = DeviceOpContext::new( - None, - &mgr, - None, - None, - false, - Some(VmConfigInfo::default()), - vm.shared_info().clone(), - ); - let res = VhostNetDeviceMgr::insert_device(&mut mgr, ctx, netif_1); - if let Err(VhostNetDeviceError::GuestMacAddressInUse(_)) = res { - assert_eq!(mgr.vhost_net_manager.info_list.len(), 1); - } else { - panic!(); - } - - // Adding the second valid network config. - let mut netif_2 = VhostNetDeviceConfigInfo { - iface_id: String::from("id_2"), - host_dev_name: String::from("dev_2"), - num_queues: 2, - queue_size: 128, - guest_mac: Some(MacAddr::parse_str(guest_mac_2).unwrap()), - allow_duplicate_mac: false, - use_shared_irq: None, - use_generic_irq: None, - }; - let ctx = DeviceOpContext::new( - None, - &mgr, - None, - None, - false, - Some(VmConfigInfo::default()), - vm.shared_info().clone(), - ); - assert!(VhostNetDeviceMgr::insert_device(&mut mgr, ctx, netif_2.clone()).is_ok()); - assert_eq!(mgr.vhost_net_manager.info_list.len(), 2); - - // Error Cases for UPDATE - // Error Case: update netif_2 network config with the same host_dev_name as netif_1 - netif_2.host_dev_name = String::from("dev_1"); - let ctx = DeviceOpContext::new( - None, - &mgr, - None, - None, - false, - Some(VmConfigInfo::default()), - vm.shared_info().clone(), - ); - let res = VhostNetDeviceMgr::insert_device(&mut mgr, ctx, netif_2.clone()); - if let Err(VhostNetDeviceError::HostDeviceNameInUse(_)) = res { - assert_eq!(mgr.vhost_net_manager.info_list.len(), 2); - } else { - panic!(); - } - - // Error Cases for UPDATE - // Error Case: update netif_2 network config with the same guest_address as netif_ - netif_2.guest_mac = Some(MacAddr::parse_str(guest_mac_1).unwrap()); - let ctx = DeviceOpContext::new( - None, - &mgr, - None, - None, - false, - Some(VmConfigInfo::default()), - vm.shared_info().clone(), - ); - let res = VhostNetDeviceMgr::insert_device(&mut mgr, ctx, netif_2); - if let Err(VhostNetDeviceError::GuestMacAddressInUse(_)) = res { - assert_eq!(mgr.vhost_net_manager.info_list.len(), 2); - } else { - panic!(); - } - - // Adding the third valid network config with same mac. - let netif_3 = VhostNetDeviceConfigInfo { - iface_id: String::from("id_3"), - host_dev_name: String::from("dev_3"), - num_queues: 2, - queue_size: 128, - guest_mac: Some(MacAddr::parse_str(guest_mac_1).unwrap()), - allow_duplicate_mac: true, - use_shared_irq: None, - use_generic_irq: None, - }; - let ctx = DeviceOpContext::new( - None, - &mgr, - None, - None, - false, - Some(VmConfigInfo::default()), - vm.shared_info().clone(), - ); - assert!(VhostNetDeviceMgr::insert_device(&mut mgr, ctx, netif_3).is_ok()); - assert_eq!(mgr.vhost_net_manager.info_list.len(), 3); - } - - #[test] - fn test_vhost_net_error_display() { - let err = VhostNetDeviceError::InvalidQueueNum(0); - let _ = format!("{err}{err:?}"); - - let err = VhostNetDeviceError::DeviceManager(DeviceMgrError::GetDeviceResource); - let _ = format!("{err}{err:?}"); - - let err = VhostNetDeviceError::DeviceIdAlreadyExist(String::from("1")); - let _ = format!("{err}{err:?}"); - - let err = VhostNetDeviceError::GuestMacAddressInUse(String::from("1")); - let _ = format!("{err}{err:?}"); - - let err = VhostNetDeviceError::HostDeviceNameInUse(String::from("1")); - let _ = format!("{err}{err:?}"); - - let err = VhostNetDeviceError::Virtio(VirtioError::DescriptorChainTooShort); - let _ = format!("{err}{err:?}"); - - let err = VhostNetDeviceError::UpdateNotAllowedPostBoot; - let _ = format!("{err}{err:?}"); - } -} diff --git a/src/dragonball/src/device_manager/vhost_user_net_dev_mgr.rs b/src/dragonball/src/device_manager/vhost_user_net_dev_mgr.rs deleted file mode 100644 index 75e1af690e..0000000000 --- a/src/dragonball/src/device_manager/vhost_user_net_dev_mgr.rs +++ /dev/null @@ -1,329 +0,0 @@ -// Copyright (C) 2019-2023 Alibaba Cloud. All rights reserved. -// Copyright (C) 2019-2023 Ant Group. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use std::result::Result; -use std::sync::Arc; - -use dbs_utils::net::MacAddr; -use dbs_virtio_devices::vhost::vhost_user::net::VhostUserNet; -use dbs_virtio_devices::Error as VirtioError; -use serde::{Deserialize, Serialize}; - -use super::{DeviceManager, DeviceMgrError, DeviceOpContext}; -use crate::address_space_manager::GuestAddressSpaceImpl; -use crate::config_manager::{ConfigItem, DeviceConfigInfos}; - -/// Default number of virtio queues, one rx/tx pair. -pub const DEFAULT_NUM_QUEUES: usize = 2; -/// Default size of virtio queues. -pub const DEFAULT_QUEUE_SIZE: u16 = 256; -// The flag of whether to use the shared irq. -const USE_SHARED_IRQ: bool = true; -// The flag of whether to use the generic irq. -const USE_GENERIC_IRQ: bool = false; - -/// Errors associated with vhost user net devices. -#[derive(Debug, thiserror::Error)] -pub enum VhostUserNetDeviceError { - /// The virtual machine instance ID is invalid. - #[error("the virtual machine instance ID is invalid")] - InvalidVmId, - /// Internal error. Invalid queue number configuration for vhost_user_net device. - #[error("invalid queue number {0} for vhost_user_net device")] - InvalidQueueNum(usize), - /// Failure from device manager, - #[error("failure in device manager operations: {0:?}")] - DeviceManager(DeviceMgrError), - /// Duplicated Unix domain socket path for vhost_user_net device. - #[error("duplicated Unix domain socket path {0} for vhost_user_net device")] - DuplicatedUdsPath(String), - /// Failure from Virtio subsystem. - #[error(transparent)] - Virtio(VirtioError), - /// The update is not allowed after booting the microvm. - #[error("update operation is not allowed after boot")] - UpdateNotAllowedPostBoot, - /// Split this at some point. - /// Internal errors are due to resource exhaustion. - /// Users errors are due to invalid permissions. - #[error("cannot create a vhost-user-net device: {0:?}")] - CreateNetDevice(VirtioError), - /// Cannot initialize a MMIO Network Device or add a device to the MMIO Bus. - #[error("failure while registering network device: {0:?}")] - RegisterNetDevice(DeviceMgrError), -} -/// Configuration information for vhost user net devices. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct VhostUserNetDeviceConfigInfo { - /// vhost-user socket path. - pub sock_path: String, - /// Number of virtqueues to use. - pub num_queues: usize, - /// Size of each virtqueue. - pub queue_size: u16, - /// mac of the interface. - pub guest_mac: Option, - /// Use shared irq - pub use_shared_irq: Option, - /// Use generic irq - pub use_generic_irq: Option, -} - -impl VhostUserNetDeviceConfigInfo { - /// Returns a reference to the mac address. If the mac address is not configured, it - /// returns None. - pub fn guest_mac(&self) -> Option<&MacAddr> { - self.guest_mac.as_ref() - } - - ///Rx and Tx queue and max queue sizes - pub fn queue_sizes(&self) -> Vec { - let mut queue_size = self.queue_size; - if queue_size == 0 { - queue_size = DEFAULT_QUEUE_SIZE; - } - let num_queues = if self.num_queues > 0 { - self.num_queues - } else { - DEFAULT_NUM_QUEUES - }; - (0..num_queues).map(|_| queue_size).collect::>() - } -} - -impl ConfigItem for VhostUserNetDeviceConfigInfo { - type Err = VhostUserNetDeviceError; - - fn check_conflicts(&self, _other: &Self) -> std::result::Result<(), Self::Err> { - Ok(()) - } - - fn id(&self) -> &str { - &self.sock_path - } -} - -/// Device manager to manage all vhost user net devices. -pub struct VhostUserNetDeviceMgr { - pub(crate) configs: DeviceConfigInfos, -} - -impl VhostUserNetDeviceMgr { - fn create_device( - cfg: &VhostUserNetDeviceConfigInfo, - ctx: &mut DeviceOpContext, - ) -> Result>, VirtioError> { - match ctx.epoll_mgr.as_ref() { - Some(epoll_mgr) => Ok(Box::new(VhostUserNet::new_server( - &cfg.sock_path, - cfg.guest_mac(), - Arc::new(cfg.queue_sizes()), - epoll_mgr.clone(), - )?)), - None => Err(VirtioError::InvalidInput), - } - } - - /// Insert or update a vhost user net device into the manager. - pub fn insert_device( - device_mgr: &mut DeviceManager, - mut ctx: DeviceOpContext, - config: VhostUserNetDeviceConfigInfo, - ) -> Result<(), VhostUserNetDeviceError> { - // Validate device configuration first. - if !config.num_queues.is_multiple_of(2) { - return Err(VhostUserNetDeviceError::InvalidQueueNum(config.num_queues)); - } - if !cfg!(feature = "hotplug") && ctx.is_hotplug { - return Err(VhostUserNetDeviceError::UpdateNotAllowedPostBoot); - } - slog::info!( - ctx.logger(), - "add vhost-user-net device configuration"; - "subsystem" => "vhost_net_dev_mgr", - "sock_path" => &config.sock_path, - ); - let device_index = device_mgr - .vhost_user_net_manager - .configs - .insert_or_update(&config)?; - if ctx.is_hotplug { - slog::info!( - ctx.logger(), - "attach virtio-net device"; - "subsystem" => "vhost_net_dev_mgr", - "sock_path" => &config.sock_path, - ); - match Self::create_device(&config, &mut ctx) { - Ok(device) => { - let dev = DeviceManager::create_mmio_virtio_device( - device, - &mut ctx, - config.use_shared_irq.unwrap_or(USE_SHARED_IRQ), - config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), - ) - .map_err(VhostUserNetDeviceError::DeviceManager)?; - ctx.insert_hotplug_mmio_device(&dev, None) - .map_err(VhostUserNetDeviceError::DeviceManager)?; - } - Err(err) => { - device_mgr - .vhost_user_net_manager - .configs - .remove(device_index); - return Err(VhostUserNetDeviceError::Virtio(err)); - } - } - } - Ok(()) - } - - /// Attach all configured vhost user net device to the virtual machine - /// instance. - pub fn attach_devices( - &mut self, - ctx: &mut DeviceOpContext, - ) -> Result<(), VhostUserNetDeviceError> { - for info in self.configs.iter() { - slog::info!( - ctx.logger(), - "attach vhost-user-net device"; - "subsystem" => "vhost_net_dev_mgr", - "sock_path" => &info.config.sock_path, - ); - let device = Self::create_device(&info.config, ctx) - .map_err(VhostUserNetDeviceError::CreateNetDevice)?; - DeviceManager::create_mmio_virtio_device( - device, - ctx, - info.config.use_shared_irq.unwrap_or(USE_SHARED_IRQ), - info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), - ) - .map_err(VhostUserNetDeviceError::RegisterNetDevice)?; - } - Ok(()) - } -} - -impl Default for VhostUserNetDeviceMgr { - /// Create a new vhost user net device manager. - fn default() -> Self { - VhostUserNetDeviceMgr { - configs: DeviceConfigInfos::::new(), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::tests::create_vm_for_test; - use test_utils::skip_if_kvm_unaccessable; - - #[test] - fn test_create_vhost_user_net_device() { - skip_if_kvm_unaccessable!(); - let vm = create_vm_for_test(); - let mgr = DeviceManager::new_test_mgr(); - let sock_1 = String::from("id_1"); - let guest_mac_1 = "01:23:45:67:89:0a"; - let netif_1 = VhostUserNetDeviceConfigInfo { - sock_path: sock_1, - num_queues: 2, - queue_size: 128, - guest_mac: Some(MacAddr::parse_str(guest_mac_1).unwrap()), - use_shared_irq: None, - use_generic_irq: None, - }; - // no epoll manager - let mut ctx = DeviceOpContext::new( - None, - &mgr, - None, - None, - false, - Some(vm.vm_config().clone()), - vm.shared_info().clone(), - ); - assert!(VhostUserNetDeviceMgr::create_device(&netif_1, &mut ctx).is_err()); - } - - #[test] - fn test_insert_vhost_user_net_device() { - skip_if_kvm_unaccessable!(); - let vm = create_vm_for_test(); - let mut mgr = DeviceManager::new_test_mgr(); - let sock_1 = String::from("id_1"); - let guest_mac_1 = "01:23:45:67:89:0a"; - // Test create. - let netif_1 = VhostUserNetDeviceConfigInfo { - sock_path: sock_1, - num_queues: 2, - queue_size: 128, - guest_mac: Some(MacAddr::parse_str(guest_mac_1).unwrap()), - use_shared_irq: None, - use_generic_irq: None, - }; - let ctx = DeviceOpContext::new( - None, - &mgr, - None, - None, - false, - Some(vm.vm_config().clone()), - vm.shared_info().clone(), - ); - assert!(VhostUserNetDeviceMgr::insert_device(&mut mgr, ctx, netif_1).is_ok()); - assert_eq!(mgr.vhost_user_net_manager.configs.len(), 1); - } - - #[test] - fn test_vhost_user_net_insert_error_cases() { - skip_if_kvm_unaccessable!(); - let vm = create_vm_for_test(); - let mut mgr = DeviceManager::new_test_mgr(); - let sock_1 = String::from("id_1"); - let guest_mac_1 = "01:23:45:67:89:0a"; - // invalid queue num. - let netif_1 = VhostUserNetDeviceConfigInfo { - sock_path: sock_1, - num_queues: 1, - queue_size: 128, - guest_mac: Some(MacAddr::parse_str(guest_mac_1).unwrap()), - use_shared_irq: None, - use_generic_irq: None, - }; - let ctx = DeviceOpContext::new( - None, - &mgr, - None, - None, - false, - Some(vm.vm_config().clone()), - vm.shared_info().clone(), - ); - let res = VhostUserNetDeviceMgr::insert_device(&mut mgr, ctx, netif_1); - if let Err(VhostUserNetDeviceError::InvalidQueueNum(1)) = res { - assert_eq!(mgr.vhost_user_net_manager.configs.len(), 0); - } else { - panic!() - } - } - - #[test] - fn test_vhost_user_net_error_display() { - let err = VhostUserNetDeviceError::InvalidVmId; - let _ = format!("{err}{err:?}"); - let err = VhostUserNetDeviceError::InvalidQueueNum(0); - let _ = format!("{err}{err:?}"); - let err = VhostUserNetDeviceError::DeviceManager(DeviceMgrError::GetDeviceResource); - let _ = format!("{err}{err:?}"); - let err = VhostUserNetDeviceError::DuplicatedUdsPath(String::from("1")); - let _ = format!("{err}{err:?}"); - let err = VhostUserNetDeviceError::Virtio(VirtioError::DescriptorChainTooShort); - let _ = format!("{err}{err:?}"); - let err = VhostUserNetDeviceError::UpdateNotAllowedPostBoot; - let _ = format!("{err}{err:?}"); - } -} diff --git a/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs b/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs deleted file mode 100644 index d64f38d2b9..0000000000 --- a/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs +++ /dev/null @@ -1,400 +0,0 @@ -// Copyright 2020-2022 Alibaba, Inc. or its affiliates. All Rights Reserved. -// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -// -// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the THIRD-PARTY file. - -use std::convert::TryInto; -use std::sync::Arc; - -use dbs_utils::net::{MacAddr, Tap, TapError}; -use dbs_utils::rate_limiter::BucketUpdate; -use dbs_virtio_devices as virtio; -use dbs_virtio_devices::net::Net; -use dbs_virtio_devices::Error as VirtioError; -use serde_derive::{Deserialize, Serialize}; - -use crate::address_space_manager::GuestAddressSpaceImpl; -use crate::config_manager::{ - ConfigItem, DeviceConfigInfo, DeviceConfigInfos, RateLimiterConfigInfo, -}; -use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; -use crate::get_bucket_update; - -use super::DbsMmioV2Device; - -/// Default number of virtio queues, one rx/tx pair. -pub const DEFAULT_NUM_QUEUES: usize = 2; -/// Default size of virtio queues. -pub const DEFAULT_QUEUE_SIZE: u16 = 256; -// The flag of whether to use the shared irq. -const USE_SHARED_IRQ: bool = true; -// The flag of whether to use the generic irq. -const USE_GENERIC_IRQ: bool = true; - -/// Errors associated with virtio net device operations. -#[derive(Debug, thiserror::Error)] -pub enum VirtioNetDeviceError { - /// The virtual machine instance ID is invalid. - #[error("the virtual machine instance ID is invalid")] - InvalidVMID, - - /// The iface ID is invalid. - #[error("invalid virtio-net iface id '{0}'")] - InvalidIfaceId(String), - - /// Invalid queue number configuration for virtio_net device. - #[error("invalid queue number {0} for virtio-net device")] - InvalidQueueNum(usize), - - /// Failure from device manager, - #[error("failure in device manager operations, {0}")] - DeviceManager(#[source] DeviceMgrError), - - /// The Context Identifier is already in use. - #[error("the device ID {0} already exists")] - DeviceIDAlreadyExist(String), - - /// The MAC address is already in use. - #[error("the guest MAC address {0} is already in use")] - GuestMacAddressInUse(String), - - /// The host device name is already in use. - #[error("the host device name {0} is already in use")] - HostDeviceNameInUse(String), - - /// Cannot open/create tap device. - #[error("cannot open TAP device")] - OpenTap(#[source] TapError), - - /// Failure from virtio subsystem. - #[error(transparent)] - Virtio(VirtioError), - - /// Failed to send patch message to net epoll handler. - #[error("could not send patch message to the net epoll handler")] - NetEpollHanderSendFail, - - /// The update is not allowed after booting the microvm. - #[error("update operation is not allowed after boot")] - UpdateNotAllowedPostBoot, - - /// Split this at some point. - /// Internal errors are due to resource exhaustion. - /// Users errors are due to invalid permissions. - #[error("cannot create network device: {0}")] - CreateNetDevice(#[source] VirtioError), - - /// Cannot initialize a MMIO Network Device or add a device to the MMIO Bus. - #[error("failure while registering network device: {0}")] - RegisterNetDevice(#[source] DeviceMgrError), -} - -/// Configuration information for virtio net devices. -#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] -pub struct VirtioNetDeviceConfigUpdateInfo { - /// ID of the guest network interface. - pub iface_id: String, - /// Rate Limiter for received packages. - pub rx_rate_limiter: Option, - /// Rate Limiter for transmitted packages. - pub tx_rate_limiter: Option, -} - -impl VirtioNetDeviceConfigUpdateInfo { - /// Provides a `BucketUpdate` description for the RX bandwidth rate limiter. - pub fn rx_bytes(&self) -> BucketUpdate { - get_bucket_update!(self, rx_rate_limiter, bandwidth) - } - /// Provides a `BucketUpdate` description for the RX ops rate limiter. - pub fn rx_ops(&self) -> BucketUpdate { - get_bucket_update!(self, rx_rate_limiter, ops) - } - /// Provides a `BucketUpdate` description for the TX bandwidth rate limiter. - pub fn tx_bytes(&self) -> BucketUpdate { - get_bucket_update!(self, tx_rate_limiter, bandwidth) - } - /// Provides a `BucketUpdate` description for the TX ops rate limiter. - pub fn tx_ops(&self) -> BucketUpdate { - get_bucket_update!(self, tx_rate_limiter, ops) - } -} - -/// Configuration information for virtio net devices. -/// TODO: https://github.com/kata-containers/kata-containers/issues/8382. -#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize, Default)] -pub struct VirtioNetDeviceConfigInfo { - /// ID of the guest network interface. - pub iface_id: String, - /// Host level path for the guest network interface. - pub host_dev_name: String, - /// Number of virtqueues to use. - pub num_queues: usize, - /// Size of each virtqueue. Unit: byte. - pub queue_size: u16, - /// Guest MAC address. - pub guest_mac: Option, - /// Rate Limiter for received packages. - pub rx_rate_limiter: Option, - /// Rate Limiter for transmitted packages. - pub tx_rate_limiter: Option, - /// allow duplicate mac - pub allow_duplicate_mac: bool, - /// Use shared irq - pub use_shared_irq: Option, - /// Use generic irq - pub use_generic_irq: Option, -} - -impl VirtioNetDeviceConfigInfo { - /// Returns the tap device that `host_dev_name` refers to. - pub fn open_tap(&self) -> std::result::Result { - Tap::open_named(self.host_dev_name.as_str(), false).map_err(VirtioNetDeviceError::OpenTap) - } - - /// Returns a reference to the mac address. It the mac address is not configured, it - /// return None. - pub fn guest_mac(&self) -> Option<&MacAddr> { - self.guest_mac.as_ref() - } - - ///Rx and Tx queue and max queue sizes - pub fn queue_sizes(&self) -> Vec { - let mut queue_size = self.queue_size; - if queue_size == 0 { - queue_size = DEFAULT_QUEUE_SIZE; - } - let num_queues = if self.num_queues > 0 { - self.num_queues - } else { - DEFAULT_NUM_QUEUES - }; - - (0..num_queues).map(|_| queue_size).collect::>() - } -} - -impl ConfigItem for VirtioNetDeviceConfigInfo { - type Err = VirtioNetDeviceError; - - fn id(&self) -> &str { - &self.iface_id - } - - fn check_conflicts(&self, other: &Self) -> Result<(), VirtioNetDeviceError> { - if self.iface_id == other.iface_id { - Err(VirtioNetDeviceError::DeviceIDAlreadyExist( - self.iface_id.clone(), - )) - } else if !other.allow_duplicate_mac - && self.guest_mac.is_some() - && self.guest_mac == other.guest_mac - { - Err(VirtioNetDeviceError::GuestMacAddressInUse( - self.guest_mac.as_ref().unwrap().to_string(), - )) - } else if self.host_dev_name == other.host_dev_name { - Err(VirtioNetDeviceError::HostDeviceNameInUse( - self.host_dev_name.clone(), - )) - } else { - Ok(()) - } - } -} - -/// Virtio Net Device Info -pub type VirtioNetDeviceInfo = DeviceConfigInfo; - -/// Device manager to manage all virtio net devices. -pub struct VirtioNetDeviceMgr { - pub(crate) info_list: DeviceConfigInfos, - pub(crate) use_shared_irq: bool, -} - -impl VirtioNetDeviceMgr { - /// Gets the index of the device with the specified `drive_id` if it exists in the list. - pub fn get_index_of_iface_id(&self, if_id: &str) -> Option { - self.info_list - .iter() - .position(|info| info.config.iface_id.eq(if_id)) - } - - /// Insert or update a virtio net device into the manager. - pub fn insert_device( - &mut self, - mut ctx: DeviceOpContext, - config: VirtioNetDeviceConfigInfo, - ) -> std::result::Result<(), VirtioNetDeviceError> { - if !config.num_queues.is_multiple_of(2) { - return Err(VirtioNetDeviceError::InvalidQueueNum(config.num_queues)); - } - if !cfg!(feature = "hotplug") && ctx.is_hotplug { - return Err(VirtioNetDeviceError::UpdateNotAllowedPostBoot); - } - - slog::info!( - ctx.logger(), - "add virtio-net device configuration"; - "subsystem" => "net_dev_mgr", - "id" => &config.iface_id, - "host_dev_name" => &config.host_dev_name, - ); - - let device_index = self.info_list.insert_or_update(&config)?; - - if ctx.is_hotplug { - slog::info!( - ctx.logger(), - "attach virtio-net device"; - "subsystem" => "net_dev_mgr", - "id" => &config.iface_id, - "host_dev_name" => &config.host_dev_name, - ); - - match Self::create_device(&config, &mut ctx) { - Ok(device) => { - let dev = DeviceManager::create_mmio_virtio_device( - device, - &mut ctx, - config.use_shared_irq.unwrap_or(self.use_shared_irq), - config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), - ) - .map_err(VirtioNetDeviceError::DeviceManager)?; - ctx.insert_hotplug_mmio_device(&dev, None) - .map_err(VirtioNetDeviceError::DeviceManager)?; - // live-upgrade need save/restore device from info.device. - self.info_list[device_index].set_device(dev); - } - Err(e) => { - self.info_list.remove(device_index); - return Err(VirtioNetDeviceError::Virtio(e)); - } - } - } - - Ok(()) - } - - /// Update the ratelimiter settings of a virtio net device. - pub fn update_device_ratelimiters( - &mut self, - new_cfg: VirtioNetDeviceConfigUpdateInfo, - ) -> std::result::Result<(), VirtioNetDeviceError> { - match self.get_index_of_iface_id(&new_cfg.iface_id) { - Some(index) => { - let config = &mut self.info_list[index].config; - config.rx_rate_limiter = new_cfg.rx_rate_limiter.clone(); - config.tx_rate_limiter = new_cfg.tx_rate_limiter.clone(); - let device = self.info_list[index].device.as_mut().ok_or_else(|| { - VirtioNetDeviceError::InvalidIfaceId(new_cfg.iface_id.clone()) - })?; - - if let Some(mmio_dev) = device.as_any().downcast_ref::() { - let guard = mmio_dev.state(); - let inner_dev = guard.get_inner_device(); - if let Some(net_dev) = inner_dev - .as_any() - .downcast_ref::>() - { - return net_dev - .set_patch_rate_limiters( - new_cfg.rx_bytes(), - new_cfg.rx_ops(), - new_cfg.tx_bytes(), - new_cfg.tx_ops(), - ) - .map(|_p| ()) - .map_err(|_e| VirtioNetDeviceError::NetEpollHanderSendFail); - } - } - Ok(()) - } - None => Err(VirtioNetDeviceError::InvalidIfaceId( - new_cfg.iface_id.clone(), - )), - } - } - - /// Attach all configured net device to the virtual machine instance. - pub fn attach_devices( - &mut self, - ctx: &mut DeviceOpContext, - ) -> std::result::Result<(), VirtioNetDeviceError> { - for info in self.info_list.iter_mut() { - slog::info!( - ctx.logger(), - "attach virtio-net device"; - "subsystem" => "net_dev_mgr", - "id" => &info.config.iface_id, - "host_dev_name" => &info.config.host_dev_name, - ); - - let device = Self::create_device(&info.config, ctx) - .map_err(VirtioNetDeviceError::CreateNetDevice)?; - let device = DeviceManager::create_mmio_virtio_device( - device, - ctx, - info.config.use_shared_irq.unwrap_or(self.use_shared_irq), - info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), - ) - .map_err(VirtioNetDeviceError::RegisterNetDevice)?; - info.set_device(device); - } - - Ok(()) - } - - fn create_device( - cfg: &VirtioNetDeviceConfigInfo, - ctx: &mut DeviceOpContext, - ) -> std::result::Result>, virtio::Error> { - let epoll_mgr = ctx.epoll_mgr.clone().ok_or(virtio::Error::InvalidInput)?; - let rx_rate_limiter = match cfg.rx_rate_limiter.as_ref() { - Some(rl) => Some(rl.try_into().map_err(virtio::Error::IOError)?), - None => None, - }; - let tx_rate_limiter = match cfg.tx_rate_limiter.as_ref() { - Some(rl) => Some(rl.try_into().map_err(virtio::Error::IOError)?), - None => None, - }; - - let net_device = Net::new( - cfg.host_dev_name.clone(), - cfg.guest_mac(), - Arc::new(cfg.queue_sizes()), - epoll_mgr, - rx_rate_limiter, - tx_rate_limiter, - )?; - - Ok(Box::new(net_device)) - } - - /// Remove all virtio-net devices. - pub fn remove_devices(&mut self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> { - while let Some(mut info) = self.info_list.pop() { - slog::info!( - ctx.logger(), - "remove virtio-net device: {}", - info.config.iface_id - ); - if let Some(device) = info.device.take() { - DeviceManager::destroy_mmio_device(device, ctx)?; - } - } - Ok(()) - } -} - -impl Default for VirtioNetDeviceMgr { - /// Create a new virtio net device manager. - fn default() -> Self { - VirtioNetDeviceMgr { - info_list: DeviceConfigInfos::new(), - use_shared_irq: USE_SHARED_IRQ, - } - } -} diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index 2c1ae7a935..67754b3f82 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -195,10 +195,14 @@ pub enum StartMicroVmError { #[error("virtio-blk errors: {0}")] BlockDeviceError(#[source] device_manager::blk_dev_mgr::BlockDeviceError), - #[cfg(feature = "virtio-net")] - /// Virtio-net errors. - #[error("virtio-net errors: {0}")] - VirtioNetDeviceError(#[source] device_manager::virtio_net_dev_mgr::VirtioNetDeviceError), + #[cfg(any( + feature = "virtio-net", + feature = "vhost-net", + feature = "vhost-user-net" + ))] + /// Network device errors. + #[error("network device errors: {0}")] + NetworkDeviceError(#[source] device_manager::net_dev_mgr::NetworkDeviceError), #[cfg(any(feature = "virtio-fs", feature = "vhost-user-fs"))] /// Virtio-fs errors. @@ -209,18 +213,6 @@ pub enum StartMicroVmError { /// Virtio-balloon errors. #[error("virtio-balloon errors: {0}")] BalloonDeviceError(#[source] device_manager::balloon_dev_mgr::BalloonDeviceError), - - /// Vhost-net device errors. - #[cfg(feature = "vhost-net")] - #[error("vhost-net errors: {0:?}")] - VhostNetDeviceError(#[source] device_manager::vhost_net_dev_mgr::VhostNetDeviceError), - - /// Vhost-user-net device errors. - #[cfg(feature = "vhost-user-net")] - #[error("vhost-user-net errors: {0:?}")] - VhostUserNetDeviceError( - #[source] device_manager::vhost_user_net_dev_mgr::VhostUserNetDeviceError, - ), #[cfg(feature = "host-device")] /// Failed to create VFIO device #[error("cannot create VFIO device {0:?}")] diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs index 4e16cd070e..64822ff01e 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs @@ -358,6 +358,7 @@ impl DragonballInner { num_queues: Some(num_queues), queue_size: Some(config.queue_size as u16), backend: dragonball::api::v1::Backend::VhostUser(DragonballVhostUserConfig { + iface_id: config.dev_id.clone(), sock_path: config.socket_path.clone(), }), guest_mac, diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/vhost_user_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/vhost_user_endpoint.rs index 15d47f46c1..4ecdda99d9 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/vhost_user_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/vhost_user_endpoint.rs @@ -58,6 +58,10 @@ impl VhostUserEndpoint { fn get_network_config(&self) -> VhostUserConfig { VhostUserConfig { + // Name the device after the virt interface, as the virtio-net and + // vhost-net paths do, so the hypervisor identifies it by a name + // the runtime knows rather than by its socket path. + dev_id: self.name.clone(), socket_path: self.socket_path.clone(), mac_address: self.guest_mac.clone(), device_type: VhostUserType::Net,