diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index fcedd492ce..d44e115ca5 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -50,6 +50,7 @@ atomic-guest-memory = [] hotplug = ["virtio-vsock"] virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"] virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"] +virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"] [patch.'crates-io'] dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 253a6854ce..0a021341ac 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -21,6 +21,11 @@ use crate::vmm::Vmm; use crate::device_manager::blk_dev_mgr::{ BlockDeviceConfigInfo, BlockDeviceConfigUpdateInfo, BlockDeviceError, BlockDeviceMgr, }; +#[cfg(feature = "virtio-net")] +use crate::device_manager::virtio_net_dev_mgr::{ + VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo, VirtioNetDeviceError, + VirtioNetDeviceMgr, +}; #[cfg(feature = "virtio-vsock")] use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError}; @@ -64,6 +69,11 @@ pub enum VmmActionError { /// Block device related errors. #[error("virtio-blk device error: {0}")] Block(#[source] BlockDeviceError), + + #[cfg(feature = "virtio-net")] + /// Net device related errors. + #[error("virtio-net device error: {0}")] + VirtioNet(#[source] VirtioNetDeviceError), } /// This enum represents the public interface of the VMM. Each action contains various @@ -108,6 +118,17 @@ pub enum VmmAction { /// Update a block device, after microVM start. Currently, the only updatable properties /// are the RX and TX rate limiters. UpdateBlockDevice(BlockDeviceConfigUpdateInfo), + + #[cfg(feature = "virtio-net")] + /// Add a new network interface config or update one that already exists using the + /// `NetworkInterfaceConfig` as input. This action can only be called before the microVM has + /// booted. The response is sent using the `OutcomeSender`. + InsertNetworkDevice(VirtioNetDeviceConfigInfo), + + #[cfg(feature = "virtio-net")] + /// Update a network interface, after microVM start. Currently, the only updatable properties + /// are the RX and TX rate limiters. + UpdateNetworkInterface(VirtioNetDeviceConfigUpdateInfo), } /// The enum represents the response sent by the VMM in case of success. The response is either @@ -189,6 +210,14 @@ impl VmmService { VmmAction::RemoveBlockDevice(drive_id) => { self.remove_block_device(vmm, event_mgr, &drive_id) } + #[cfg(feature = "virtio-net")] + VmmAction::InsertNetworkDevice(virtio_net_cfg) => { + self.add_virtio_net_device(vmm, event_mgr, virtio_net_cfg) + } + #[cfg(feature = "virtio-net")] + VmmAction::UpdateNetworkInterface(netif_update) => { + self.update_net_rate_limiters(vmm, netif_update) + } }; debug!("send vmm response: {:?}", response); @@ -495,4 +524,46 @@ impl VmmService { .map(|_| VmmData::Empty) .map_err(VmmActionError::Block) } + + #[cfg(feature = "virtio-net")] + fn add_virtio_net_device( + &mut self, + vmm: &mut Vmm, + event_mgr: &mut EventManager, + config: VirtioNetDeviceConfigInfo, + ) -> VmmRequestResult { + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + let ctx = vm + .create_device_op_context(Some(event_mgr.epoll_manager())) + .map_err(|e| { + if let StartMicrovmError::MicroVMAlreadyRunning = e { + VmmActionError::VirtioNet(VirtioNetDeviceError::UpdateNotAllowedPostBoot) + } else if let StartMicrovmError::UpcallNotReady = e { + VmmActionError::UpcallNotReady + } else { + VmmActionError::StartMicrovm(e) + } + })?; + + VirtioNetDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::VirtioNet) + } + + #[cfg(feature = "virtio-net")] + fn update_net_rate_limiters( + &mut self, + vmm: &mut Vmm, + config: VirtioNetDeviceConfigUpdateInfo, + ) -> VmmRequestResult { + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + + VirtioNetDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::VirtioNet) + } } diff --git a/src/dragonball/src/config_manager.rs b/src/dragonball/src/config_manager.rs index e25b792e7b..cbfb790413 100644 --- a/src/dragonball/src/config_manager.rs +++ b/src/dragonball/src/config_manager.rs @@ -10,6 +10,28 @@ use dbs_device::DeviceIo; use dbs_utils::rate_limiter::{RateLimiter, TokenBucket}; use serde_derive::{Deserialize, Serialize}; +macro_rules! get_bucket_update { + ($self:ident, $rate_limiter: ident, $metric: ident) => {{ + match &$self.$rate_limiter { + Some(rl_cfg) => { + let tb_cfg = &rl_cfg.$metric; + dbs_utils::rate_limiter::RateLimiter::make_bucket( + tb_cfg.size, + tb_cfg.one_time_burst, + tb_cfg.refill_time, + ) + // Updated active rate-limiter. + .map(dbs_utils::rate_limiter::BucketUpdate::Update) + // Updated/deactivated rate-limiter + .unwrap_or(dbs_utils::rate_limiter::BucketUpdate::Disabled) + } + // No update to the rate-limiter. + None => dbs_utils::rate_limiter::BucketUpdate::None, + } + }}; +} +pub(crate) use get_bucket_update; + /// Trait for generic configuration information. pub trait ConfigItem { /// Related errors. diff --git a/src/dragonball/src/device_manager/blk_dev_mgr.rs b/src/dragonball/src/device_manager/blk_dev_mgr.rs index 8a1b2d02fc..38e46cd594 100644 --- a/src/dragonball/src/device_manager/blk_dev_mgr.rs +++ b/src/dragonball/src/device_manager/blk_dev_mgr.rs @@ -20,7 +20,9 @@ use dbs_virtio_devices::block::{aio::Aio, io_uring::IoUring, Block, LocalFile, U use serde_derive::{Deserialize, Serialize}; use crate::address_space_manager::GuestAddressSpaceImpl; -use crate::config_manager::{ConfigItem, DeviceConfigInfo, RateLimiterConfigInfo}; +use crate::config_manager::{ + get_bucket_update, ConfigItem, DeviceConfigInfo, RateLimiterConfigInfo, +}; use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; use crate::vm::KernelConfigInfo; @@ -43,27 +45,6 @@ macro_rules! error( }; ); -macro_rules! get_bucket_update { - ($self:ident, $rate_limiter: ident, $metric: ident) => {{ - match &$self.$rate_limiter { - Some(rl_cfg) => { - let tb_cfg = &rl_cfg.$metric; - dbs_utils::rate_limiter::RateLimiter::make_bucket( - tb_cfg.size, - tb_cfg.one_time_burst, - tb_cfg.refill_time, - ) - // Updated active rate-limiter. - .map(dbs_utils::rate_limiter::BucketUpdate::Update) - // Updated/deactivated rate-limiter - .unwrap_or(dbs_utils::rate_limiter::BucketUpdate::Disabled) - } - // No update to the rate-limiter. - None => dbs_utils::rate_limiter::BucketUpdate::None, - } - }}; -} - /// Default queue size for VirtIo block devices. pub const QUEUE_SIZE: u16 = 128; diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 11a742bbba..9313919f85 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -65,6 +65,12 @@ pub mod blk_dev_mgr; #[cfg(feature = "virtio-blk")] use self::blk_dev_mgr::BlockDeviceMgr; +#[cfg(feature = "virtio-net")] +/// Device manager for virtio-net devices. +pub mod virtio_net_dev_mgr; +#[cfg(feature = "virtio-net")] +use self::virtio_net_dev_mgr::VirtioNetDeviceMgr; + macro_rules! info( ($l:expr, $($args:tt)+) => { slog::info!($l, $($args)+; slog::o!("subsystem" => "device_manager")) @@ -426,6 +432,9 @@ pub struct DeviceManager { // If there is a Root Block Device, this should be added as the first element of the list. // This is necessary because we want the root to always be mounted on /dev/vda. pub(crate) block_manager: BlockDeviceMgr, + + #[cfg(feature = "virtio-net")] + pub(crate) virtio_net_manager: VirtioNetDeviceMgr, } impl DeviceManager { @@ -452,6 +461,8 @@ impl DeviceManager { vsock_manager: VsockDeviceMgr::default(), #[cfg(feature = "virtio-blk")] block_manager: BlockDeviceMgr::default(), + #[cfg(feature = "virtio-net")] + virtio_net_manager: VirtioNetDeviceMgr::default(), } } @@ -577,6 +588,11 @@ impl DeviceManager { .attach_devices(&mut ctx) .map_err(StartMicrovmError::BlockDeviceError)?; + #[cfg(feature = "virtio-net")] + self.virtio_net_manager + .attach_devices(&mut ctx) + .map_err(StartMicrovmError::VirtioNetDeviceError)?; + #[cfg(feature = "virtio-vsock")] self.vsock_manager.attach_devices(&mut ctx)?; diff --git a/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs b/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs new file mode 100644 index 0000000000..65f29fbcd0 --- /dev/null +++ b/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs @@ -0,0 +1,386 @@ +// Copyright 2020-2022 Alibaba, Inc. or its affiliates. All Rights Reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +use std::convert::TryInto; +use std::sync::Arc; + +use dbs_utils::net::{MacAddr, Tap, TapError}; +use dbs_utils::rate_limiter::BucketUpdate; +use dbs_virtio_devices as virtio; +use dbs_virtio_devices::net::Net; +use dbs_virtio_devices::Error as VirtioError; +use serde_derive::{Deserialize, Serialize}; + +use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::config_manager::{ + get_bucket_update, ConfigItem, DeviceConfigInfo, DeviceConfigInfos, RateLimiterConfigInfo, +}; +use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; + +use super::DbsMmioV2Device; + +/// Default number of virtio queues, one rx/tx pair. +pub const NUM_QUEUES: usize = 2; +/// Default size of virtio queues. +pub const QUEUE_SIZE: u16 = 256; +// The flag of whether to use the shared irq. +const USE_SHARED_IRQ: bool = true; +// The flag of whether to use the generic irq. +const USE_GENERIC_IRQ: bool = true; + +/// Errors associated with virtio net device operations. +#[derive(Debug, thiserror::Error)] +pub enum VirtioNetDeviceError { + /// The virtual machine instance ID is invalid. + #[error("the virtual machine instance ID is invalid")] + InvalidVMID, + + /// The iface ID is invalid. + #[error("invalid virtio-net iface id '{0}'")] + InvalidIfaceId(String), + + /// Invalid queue number configuration for virtio_net device. + #[error("invalid queue number {0} for virtio-net device")] + InvalidQueueNum(usize), + + /// Failure from device manager, + #[error("failure in device manager operations, {0}")] + DeviceManager(#[source] DeviceMgrError), + + /// The Context Identifier is already in use. + #[error("the device ID {0} already exists")] + DeviceIDAlreadyExist(String), + + /// The MAC address is already in use. + #[error("the guest MAC address {0} is already in use")] + GuestMacAddressInUse(String), + + /// The host device name is already in use. + #[error("the host device name {0} is already in use")] + HostDeviceNameInUse(String), + + /// Cannot open/create tap device. + #[error("cannot open TAP device")] + OpenTap(#[source] TapError), + + /// Failure from virtio subsystem. + #[error(transparent)] + Virtio(VirtioError), + + /// Failed to send patch message to net epoll handler. + #[error("could not send patch message to the net epoll handler")] + NetEpollHanderSendFail, + + /// The update is not allowed after booting the microvm. + #[error("update operation is not allowed after boot")] + UpdateNotAllowedPostBoot, + + /// Split this at some point. + /// Internal errors are due to resource exhaustion. + /// Users errors are due to invalid permissions. + #[error("cannot create network device: {0}")] + CreateNetDevice(#[source] VirtioError), + + /// Cannot initialize a MMIO Network Device or add a device to the MMIO Bus. + #[error("failure while registering network device: {0}")] + RegisterNetDevice(#[source] DeviceMgrError), +} + +/// Configuration information for virtio net devices. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct VirtioNetDeviceConfigUpdateInfo { + /// ID of the guest network interface. + pub iface_id: String, + /// Rate Limiter for received packages. + pub rx_rate_limiter: Option, + /// Rate Limiter for transmitted packages. + pub tx_rate_limiter: Option, +} + +impl VirtioNetDeviceConfigUpdateInfo { + /// Provides a `BucketUpdate` description for the RX bandwidth rate limiter. + pub fn rx_bytes(&self) -> BucketUpdate { + get_bucket_update!(self, rx_rate_limiter, bandwidth) + } + /// Provides a `BucketUpdate` description for the RX ops rate limiter. + pub fn rx_ops(&self) -> BucketUpdate { + get_bucket_update!(self, rx_rate_limiter, ops) + } + /// Provides a `BucketUpdate` description for the TX bandwidth rate limiter. + pub fn tx_bytes(&self) -> BucketUpdate { + get_bucket_update!(self, tx_rate_limiter, bandwidth) + } + /// Provides a `BucketUpdate` description for the TX ops rate limiter. + pub fn tx_ops(&self) -> BucketUpdate { + get_bucket_update!(self, tx_rate_limiter, ops) + } +} + +/// Configuration information for virtio net devices. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, Default)] +pub struct VirtioNetDeviceConfigInfo { + /// ID of the guest network interface. + pub iface_id: String, + /// Host level path for the guest network interface. + pub host_dev_name: String, + /// Number of virtqueues to use. + pub num_queues: usize, + /// Size of each virtqueue. Unit: byte. + pub queue_size: u16, + /// Guest MAC address. + pub guest_mac: Option, + /// Rate Limiter for received packages. + pub rx_rate_limiter: Option, + /// Rate Limiter for transmitted packages. + pub tx_rate_limiter: Option, + /// allow duplicate mac + pub allow_duplicate_mac: bool, + /// Use shared irq + pub use_shared_irq: Option, + /// Use generic irq + pub use_generic_irq: Option, +} + +impl VirtioNetDeviceConfigInfo { + /// Returns the tap device that `host_dev_name` refers to. + pub fn open_tap(&self) -> std::result::Result { + Tap::open_named(self.host_dev_name.as_str(), false).map_err(VirtioNetDeviceError::OpenTap) + } + + /// Returns a reference to the mac address. It the mac address is not configured, it + /// return None. + pub fn guest_mac(&self) -> Option<&MacAddr> { + self.guest_mac.as_ref() + } + + ///Rx and Tx queue and max queue sizes + pub fn queue_sizes(&self) -> Vec { + let mut queue_size = self.queue_size; + if queue_size == 0 { + queue_size = QUEUE_SIZE; + } + let num_queues = if self.num_queues > 0 { + self.num_queues + } else { + NUM_QUEUES + }; + + (0..num_queues).map(|_| queue_size).collect::>() + } +} + +impl ConfigItem for VirtioNetDeviceConfigInfo { + type Err = VirtioNetDeviceError; + + fn id(&self) -> &str { + &self.iface_id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), VirtioNetDeviceError> { + if self.iface_id == other.iface_id { + Err(VirtioNetDeviceError::DeviceIDAlreadyExist( + self.iface_id.clone(), + )) + } else if !other.allow_duplicate_mac + && self.guest_mac.is_some() + && self.guest_mac == other.guest_mac + { + Err(VirtioNetDeviceError::GuestMacAddressInUse( + self.guest_mac.as_ref().unwrap().to_string(), + )) + } else if self.host_dev_name == other.host_dev_name { + Err(VirtioNetDeviceError::HostDeviceNameInUse( + self.host_dev_name.clone(), + )) + } else { + Ok(()) + } + } +} + +/// Virtio Net Device Info +pub type VirtioNetDeviceInfo = DeviceConfigInfo; + +/// Device manager to manage all virtio net devices. +pub struct VirtioNetDeviceMgr { + pub(crate) info_list: DeviceConfigInfos, + pub(crate) use_shared_irq: bool, +} + +impl VirtioNetDeviceMgr { + /// Gets the index of the device with the specified `drive_id` if it exists in the list. + pub fn get_index_of_iface_id(&self, if_id: &str) -> Option { + self.info_list + .iter() + .position(|info| info.config.iface_id.eq(if_id)) + } + + /// Insert or update a virtio net device into the manager. + pub fn insert_device( + device_mgr: &mut DeviceManager, + mut ctx: DeviceOpContext, + config: VirtioNetDeviceConfigInfo, + ) -> std::result::Result<(), VirtioNetDeviceError> { + if config.num_queues % 2 != 0 { + return Err(VirtioNetDeviceError::InvalidQueueNum(config.num_queues)); + } + if !cfg!(feature = "hotplug") && ctx.is_hotplug { + return Err(VirtioNetDeviceError::UpdateNotAllowedPostBoot); + } + + let mgr = &mut device_mgr.virtio_net_manager; + + slog::info!( + ctx.logger(), + "add virtio-net device configuration"; + "subsystem" => "net_dev_mgr", + "id" => &config.iface_id, + "host_dev_name" => &config.host_dev_name, + ); + + let device_index = mgr.info_list.insert_or_update(&config)?; + + if ctx.is_hotplug { + slog::info!( + ctx.logger(), + "attach virtio-net device"; + "subsystem" => "net_dev_mgr", + "id" => &config.iface_id, + "host_dev_name" => &config.host_dev_name, + ); + + match Self::create_device(&config, &mut ctx) { + Ok(device) => { + let dev = DeviceManager::create_mmio_virtio_device( + device, + &mut ctx, + config.use_shared_irq.unwrap_or(mgr.use_shared_irq), + config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(VirtioNetDeviceError::DeviceManager)?; + ctx.insert_hotplug_mmio_device(&dev.clone(), None) + .map_err(VirtioNetDeviceError::DeviceManager)?; + // live-upgrade need save/restore device from info.device. + mgr.info_list[device_index].set_device(dev); + } + Err(e) => { + mgr.info_list.remove(device_index); + return Err(VirtioNetDeviceError::Virtio(e)); + } + } + } + + Ok(()) + } + + /// Update the ratelimiter settings of a virtio net device. + pub fn update_device_ratelimiters( + device_mgr: &mut DeviceManager, + new_cfg: VirtioNetDeviceConfigUpdateInfo, + ) -> std::result::Result<(), VirtioNetDeviceError> { + let mgr = &mut device_mgr.virtio_net_manager; + match mgr.get_index_of_iface_id(&new_cfg.iface_id) { + Some(index) => { + let config = &mut mgr.info_list[index].config; + config.rx_rate_limiter = new_cfg.rx_rate_limiter.clone(); + config.tx_rate_limiter = new_cfg.tx_rate_limiter.clone(); + let device = mgr.info_list[index].device.as_mut().ok_or_else(|| { + VirtioNetDeviceError::InvalidIfaceId(new_cfg.iface_id.clone()) + })?; + + if let Some(mmio_dev) = device.as_any().downcast_ref::() { + let guard = mmio_dev.state(); + let inner_dev = guard.get_inner_device(); + if let Some(net_dev) = inner_dev + .as_any() + .downcast_ref::>() + { + return net_dev + .set_patch_rate_limiters( + new_cfg.rx_bytes(), + new_cfg.rx_ops(), + new_cfg.tx_bytes(), + new_cfg.tx_ops(), + ) + .map(|_p| ()) + .map_err(|_e| VirtioNetDeviceError::NetEpollHanderSendFail); + } + } + Ok(()) + } + None => Err(VirtioNetDeviceError::InvalidIfaceId( + new_cfg.iface_id.clone(), + )), + } + } + + /// Attach all configured vsock device to the virtual machine instance. + pub fn attach_devices( + &mut self, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), VirtioNetDeviceError> { + for info in self.info_list.iter_mut() { + slog::info!( + ctx.logger(), + "attach virtio-net device"; + "subsystem" => "net_dev_mgr", + "id" => &info.config.iface_id, + "host_dev_name" => &info.config.host_dev_name, + ); + + let device = Self::create_device(&info.config, ctx) + .map_err(VirtioNetDeviceError::CreateNetDevice)?; + let device = DeviceManager::create_mmio_virtio_device( + device, + ctx, + info.config.use_shared_irq.unwrap_or(self.use_shared_irq), + info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(VirtioNetDeviceError::RegisterNetDevice)?; + info.set_device(device); + } + + Ok(()) + } + + fn create_device( + cfg: &VirtioNetDeviceConfigInfo, + ctx: &mut DeviceOpContext, + ) -> std::result::Result>, virtio::Error> { + let epoll_mgr = ctx.epoll_mgr.clone().ok_or(virtio::Error::InvalidInput)?; + let rx_rate_limiter = match cfg.rx_rate_limiter.as_ref() { + Some(rl) => Some(rl.try_into().map_err(virtio::Error::IOError)?), + None => None, + }; + let tx_rate_limiter = match cfg.tx_rate_limiter.as_ref() { + Some(rl) => Some(rl.try_into().map_err(virtio::Error::IOError)?), + None => None, + }; + + let net_device = Net::new( + cfg.host_dev_name.clone(), + cfg.guest_mac(), + Arc::new(cfg.queue_sizes()), + epoll_mgr, + rx_rate_limiter, + tx_rate_limiter, + )?; + + Ok(Box::new(net_device)) + } +} + +impl Default for VirtioNetDeviceMgr { + /// Create a new virtio net device manager. + fn default() -> Self { + VirtioNetDeviceMgr { + info_list: DeviceConfigInfos::new(), + use_shared_irq: USE_SHARED_IRQ, + } + } +} diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index 9c9c46564c..667405c5ec 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -163,6 +163,11 @@ pub enum StartMicroVmError { /// Virtio-blk errors. #[error("virtio-blk errors: {0}")] BlockDeviceError(#[source] device_manager::blk_dev_mgr::BlockDeviceError), + + #[cfg(feature = "virtio-net")] + /// Virtio-net errors. + #[error("virtio-net errors: {0}")] + VirtioNetDeviceError(#[source] device_manager::virtio_net_dev_mgr::VirtioNetDeviceError), } /// Errors associated with starting the instance.