dragonball: add virtio-net device support

Virtio-net devices are supported.

Signed-off-by: wllenyj <wllenyj@linux.alibaba.com>
This commit is contained in:
wllenyj 2022-05-16 18:08:24 +08:00 committed by Chao Wu
parent 3d20387a25
commit 948381bdbe
7 changed files with 504 additions and 22 deletions

View File

@ -50,6 +50,7 @@ atomic-guest-memory = []
hotplug = ["virtio-vsock"]
virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"]
virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"]
virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"]
[patch.'crates-io']
dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" }

View File

@ -21,6 +21,11 @@ use crate::vmm::Vmm;
use crate::device_manager::blk_dev_mgr::{
BlockDeviceConfigInfo, BlockDeviceConfigUpdateInfo, BlockDeviceError, BlockDeviceMgr,
};
#[cfg(feature = "virtio-net")]
use crate::device_manager::virtio_net_dev_mgr::{
VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo, VirtioNetDeviceError,
VirtioNetDeviceMgr,
};
#[cfg(feature = "virtio-vsock")]
use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError};
@ -64,6 +69,11 @@ pub enum VmmActionError {
/// Block device related errors.
#[error("virtio-blk device error: {0}")]
Block(#[source] BlockDeviceError),
#[cfg(feature = "virtio-net")]
/// Net device related errors.
#[error("virtio-net device error: {0}")]
VirtioNet(#[source] VirtioNetDeviceError),
}
/// This enum represents the public interface of the VMM. Each action contains various
@ -108,6 +118,17 @@ pub enum VmmAction {
/// Update a block device, after microVM start. Currently, the only updatable properties
/// are the RX and TX rate limiters.
UpdateBlockDevice(BlockDeviceConfigUpdateInfo),
#[cfg(feature = "virtio-net")]
/// Add a new network interface config or update one that already exists using the
/// `NetworkInterfaceConfig` as input. This action can only be called before the microVM has
/// booted. The response is sent using the `OutcomeSender`.
InsertNetworkDevice(VirtioNetDeviceConfigInfo),
#[cfg(feature = "virtio-net")]
/// Update a network interface, after microVM start. Currently, the only updatable properties
/// are the RX and TX rate limiters.
UpdateNetworkInterface(VirtioNetDeviceConfigUpdateInfo),
}
/// The enum represents the response sent by the VMM in case of success. The response is either
@ -189,6 +210,14 @@ impl VmmService {
VmmAction::RemoveBlockDevice(drive_id) => {
self.remove_block_device(vmm, event_mgr, &drive_id)
}
#[cfg(feature = "virtio-net")]
VmmAction::InsertNetworkDevice(virtio_net_cfg) => {
self.add_virtio_net_device(vmm, event_mgr, virtio_net_cfg)
}
#[cfg(feature = "virtio-net")]
VmmAction::UpdateNetworkInterface(netif_update) => {
self.update_net_rate_limiters(vmm, netif_update)
}
};
debug!("send vmm response: {:?}", response);
@ -495,4 +524,46 @@ impl VmmService {
.map(|_| VmmData::Empty)
.map_err(VmmActionError::Block)
}
#[cfg(feature = "virtio-net")]
fn add_virtio_net_device(
&mut self,
vmm: &mut Vmm,
event_mgr: &mut EventManager,
config: VirtioNetDeviceConfigInfo,
) -> VmmRequestResult {
let vm = vmm
.get_vm_by_id_mut("")
.ok_or(VmmActionError::InvalidVMID)?;
let ctx = vm
.create_device_op_context(Some(event_mgr.epoll_manager()))
.map_err(|e| {
if let StartMicrovmError::MicroVMAlreadyRunning = e {
VmmActionError::VirtioNet(VirtioNetDeviceError::UpdateNotAllowedPostBoot)
} else if let StartMicrovmError::UpcallNotReady = e {
VmmActionError::UpcallNotReady
} else {
VmmActionError::StartMicrovm(e)
}
})?;
VirtioNetDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config)
.map(|_| VmmData::Empty)
.map_err(VmmActionError::VirtioNet)
}
#[cfg(feature = "virtio-net")]
fn update_net_rate_limiters(
&mut self,
vmm: &mut Vmm,
config: VirtioNetDeviceConfigUpdateInfo,
) -> VmmRequestResult {
let vm = vmm
.get_vm_by_id_mut("")
.ok_or(VmmActionError::InvalidVMID)?;
VirtioNetDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), config)
.map(|_| VmmData::Empty)
.map_err(VmmActionError::VirtioNet)
}
}

View File

@ -10,6 +10,28 @@ use dbs_device::DeviceIo;
use dbs_utils::rate_limiter::{RateLimiter, TokenBucket};
use serde_derive::{Deserialize, Serialize};
macro_rules! get_bucket_update {
($self:ident, $rate_limiter: ident, $metric: ident) => {{
match &$self.$rate_limiter {
Some(rl_cfg) => {
let tb_cfg = &rl_cfg.$metric;
dbs_utils::rate_limiter::RateLimiter::make_bucket(
tb_cfg.size,
tb_cfg.one_time_burst,
tb_cfg.refill_time,
)
// Updated active rate-limiter.
.map(dbs_utils::rate_limiter::BucketUpdate::Update)
// Updated/deactivated rate-limiter
.unwrap_or(dbs_utils::rate_limiter::BucketUpdate::Disabled)
}
// No update to the rate-limiter.
None => dbs_utils::rate_limiter::BucketUpdate::None,
}
}};
}
pub(crate) use get_bucket_update;
/// Trait for generic configuration information.
pub trait ConfigItem {
/// Related errors.

View File

@ -20,7 +20,9 @@ use dbs_virtio_devices::block::{aio::Aio, io_uring::IoUring, Block, LocalFile, U
use serde_derive::{Deserialize, Serialize};
use crate::address_space_manager::GuestAddressSpaceImpl;
use crate::config_manager::{ConfigItem, DeviceConfigInfo, RateLimiterConfigInfo};
use crate::config_manager::{
get_bucket_update, ConfigItem, DeviceConfigInfo, RateLimiterConfigInfo,
};
use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext};
use crate::vm::KernelConfigInfo;
@ -43,27 +45,6 @@ macro_rules! error(
};
);
macro_rules! get_bucket_update {
($self:ident, $rate_limiter: ident, $metric: ident) => {{
match &$self.$rate_limiter {
Some(rl_cfg) => {
let tb_cfg = &rl_cfg.$metric;
dbs_utils::rate_limiter::RateLimiter::make_bucket(
tb_cfg.size,
tb_cfg.one_time_burst,
tb_cfg.refill_time,
)
// Updated active rate-limiter.
.map(dbs_utils::rate_limiter::BucketUpdate::Update)
// Updated/deactivated rate-limiter
.unwrap_or(dbs_utils::rate_limiter::BucketUpdate::Disabled)
}
// No update to the rate-limiter.
None => dbs_utils::rate_limiter::BucketUpdate::None,
}
}};
}
/// Default queue size for VirtIo block devices.
pub const QUEUE_SIZE: u16 = 128;

View File

@ -65,6 +65,12 @@ pub mod blk_dev_mgr;
#[cfg(feature = "virtio-blk")]
use self::blk_dev_mgr::BlockDeviceMgr;
#[cfg(feature = "virtio-net")]
/// Device manager for virtio-net devices.
pub mod virtio_net_dev_mgr;
#[cfg(feature = "virtio-net")]
use self::virtio_net_dev_mgr::VirtioNetDeviceMgr;
macro_rules! info(
($l:expr, $($args:tt)+) => {
slog::info!($l, $($args)+; slog::o!("subsystem" => "device_manager"))
@ -426,6 +432,9 @@ pub struct DeviceManager {
// If there is a Root Block Device, this should be added as the first element of the list.
// This is necessary because we want the root to always be mounted on /dev/vda.
pub(crate) block_manager: BlockDeviceMgr,
#[cfg(feature = "virtio-net")]
pub(crate) virtio_net_manager: VirtioNetDeviceMgr,
}
impl DeviceManager {
@ -452,6 +461,8 @@ impl DeviceManager {
vsock_manager: VsockDeviceMgr::default(),
#[cfg(feature = "virtio-blk")]
block_manager: BlockDeviceMgr::default(),
#[cfg(feature = "virtio-net")]
virtio_net_manager: VirtioNetDeviceMgr::default(),
}
}
@ -577,6 +588,11 @@ impl DeviceManager {
.attach_devices(&mut ctx)
.map_err(StartMicrovmError::BlockDeviceError)?;
#[cfg(feature = "virtio-net")]
self.virtio_net_manager
.attach_devices(&mut ctx)
.map_err(StartMicrovmError::VirtioNetDeviceError)?;
#[cfg(feature = "virtio-vsock")]
self.vsock_manager.attach_devices(&mut ctx)?;

View File

@ -0,0 +1,386 @@
// Copyright 2020-2022 Alibaba, Inc. or its affiliates. All Rights Reserved.
// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the THIRD-PARTY file.
use std::convert::TryInto;
use std::sync::Arc;
use dbs_utils::net::{MacAddr, Tap, TapError};
use dbs_utils::rate_limiter::BucketUpdate;
use dbs_virtio_devices as virtio;
use dbs_virtio_devices::net::Net;
use dbs_virtio_devices::Error as VirtioError;
use serde_derive::{Deserialize, Serialize};
use crate::address_space_manager::GuestAddressSpaceImpl;
use crate::config_manager::{
get_bucket_update, ConfigItem, DeviceConfigInfo, DeviceConfigInfos, RateLimiterConfigInfo,
};
use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext};
use super::DbsMmioV2Device;
/// Default number of virtio queues, one rx/tx pair.
pub const NUM_QUEUES: usize = 2;
/// Default size of virtio queues.
pub const QUEUE_SIZE: u16 = 256;
// The flag of whether to use the shared irq.
const USE_SHARED_IRQ: bool = true;
// The flag of whether to use the generic irq.
const USE_GENERIC_IRQ: bool = true;
/// Errors associated with virtio net device operations.
#[derive(Debug, thiserror::Error)]
pub enum VirtioNetDeviceError {
/// The virtual machine instance ID is invalid.
#[error("the virtual machine instance ID is invalid")]
InvalidVMID,
/// The iface ID is invalid.
#[error("invalid virtio-net iface id '{0}'")]
InvalidIfaceId(String),
/// Invalid queue number configuration for virtio_net device.
#[error("invalid queue number {0} for virtio-net device")]
InvalidQueueNum(usize),
/// Failure from device manager,
#[error("failure in device manager operations, {0}")]
DeviceManager(#[source] DeviceMgrError),
/// The Context Identifier is already in use.
#[error("the device ID {0} already exists")]
DeviceIDAlreadyExist(String),
/// The MAC address is already in use.
#[error("the guest MAC address {0} is already in use")]
GuestMacAddressInUse(String),
/// The host device name is already in use.
#[error("the host device name {0} is already in use")]
HostDeviceNameInUse(String),
/// Cannot open/create tap device.
#[error("cannot open TAP device")]
OpenTap(#[source] TapError),
/// Failure from virtio subsystem.
#[error(transparent)]
Virtio(VirtioError),
/// Failed to send patch message to net epoll handler.
#[error("could not send patch message to the net epoll handler")]
NetEpollHanderSendFail,
/// The update is not allowed after booting the microvm.
#[error("update operation is not allowed after boot")]
UpdateNotAllowedPostBoot,
/// Split this at some point.
/// Internal errors are due to resource exhaustion.
/// Users errors are due to invalid permissions.
#[error("cannot create network device: {0}")]
CreateNetDevice(#[source] VirtioError),
/// Cannot initialize a MMIO Network Device or add a device to the MMIO Bus.
#[error("failure while registering network device: {0}")]
RegisterNetDevice(#[source] DeviceMgrError),
}
/// Configuration information for virtio net devices.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
pub struct VirtioNetDeviceConfigUpdateInfo {
/// ID of the guest network interface.
pub iface_id: String,
/// Rate Limiter for received packages.
pub rx_rate_limiter: Option<RateLimiterConfigInfo>,
/// Rate Limiter for transmitted packages.
pub tx_rate_limiter: Option<RateLimiterConfigInfo>,
}
impl VirtioNetDeviceConfigUpdateInfo {
/// Provides a `BucketUpdate` description for the RX bandwidth rate limiter.
pub fn rx_bytes(&self) -> BucketUpdate {
get_bucket_update!(self, rx_rate_limiter, bandwidth)
}
/// Provides a `BucketUpdate` description for the RX ops rate limiter.
pub fn rx_ops(&self) -> BucketUpdate {
get_bucket_update!(self, rx_rate_limiter, ops)
}
/// Provides a `BucketUpdate` description for the TX bandwidth rate limiter.
pub fn tx_bytes(&self) -> BucketUpdate {
get_bucket_update!(self, tx_rate_limiter, bandwidth)
}
/// Provides a `BucketUpdate` description for the TX ops rate limiter.
pub fn tx_ops(&self) -> BucketUpdate {
get_bucket_update!(self, tx_rate_limiter, ops)
}
}
/// Configuration information for virtio net devices.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, Default)]
pub struct VirtioNetDeviceConfigInfo {
/// ID of the guest network interface.
pub iface_id: String,
/// Host level path for the guest network interface.
pub host_dev_name: String,
/// Number of virtqueues to use.
pub num_queues: usize,
/// Size of each virtqueue. Unit: byte.
pub queue_size: u16,
/// Guest MAC address.
pub guest_mac: Option<MacAddr>,
/// Rate Limiter for received packages.
pub rx_rate_limiter: Option<RateLimiterConfigInfo>,
/// Rate Limiter for transmitted packages.
pub tx_rate_limiter: Option<RateLimiterConfigInfo>,
/// allow duplicate mac
pub allow_duplicate_mac: bool,
/// Use shared irq
pub use_shared_irq: Option<bool>,
/// Use generic irq
pub use_generic_irq: Option<bool>,
}
impl VirtioNetDeviceConfigInfo {
/// Returns the tap device that `host_dev_name` refers to.
pub fn open_tap(&self) -> std::result::Result<Tap, VirtioNetDeviceError> {
Tap::open_named(self.host_dev_name.as_str(), false).map_err(VirtioNetDeviceError::OpenTap)
}
/// Returns a reference to the mac address. It the mac address is not configured, it
/// return None.
pub fn guest_mac(&self) -> Option<&MacAddr> {
self.guest_mac.as_ref()
}
///Rx and Tx queue and max queue sizes
pub fn queue_sizes(&self) -> Vec<u16> {
let mut queue_size = self.queue_size;
if queue_size == 0 {
queue_size = QUEUE_SIZE;
}
let num_queues = if self.num_queues > 0 {
self.num_queues
} else {
NUM_QUEUES
};
(0..num_queues).map(|_| queue_size).collect::<Vec<u16>>()
}
}
impl ConfigItem for VirtioNetDeviceConfigInfo {
type Err = VirtioNetDeviceError;
fn id(&self) -> &str {
&self.iface_id
}
fn check_conflicts(&self, other: &Self) -> Result<(), VirtioNetDeviceError> {
if self.iface_id == other.iface_id {
Err(VirtioNetDeviceError::DeviceIDAlreadyExist(
self.iface_id.clone(),
))
} else if !other.allow_duplicate_mac
&& self.guest_mac.is_some()
&& self.guest_mac == other.guest_mac
{
Err(VirtioNetDeviceError::GuestMacAddressInUse(
self.guest_mac.as_ref().unwrap().to_string(),
))
} else if self.host_dev_name == other.host_dev_name {
Err(VirtioNetDeviceError::HostDeviceNameInUse(
self.host_dev_name.clone(),
))
} else {
Ok(())
}
}
}
/// Virtio Net Device Info
pub type VirtioNetDeviceInfo = DeviceConfigInfo<VirtioNetDeviceConfigInfo>;
/// Device manager to manage all virtio net devices.
pub struct VirtioNetDeviceMgr {
pub(crate) info_list: DeviceConfigInfos<VirtioNetDeviceConfigInfo>,
pub(crate) use_shared_irq: bool,
}
impl VirtioNetDeviceMgr {
/// Gets the index of the device with the specified `drive_id` if it exists in the list.
pub fn get_index_of_iface_id(&self, if_id: &str) -> Option<usize> {
self.info_list
.iter()
.position(|info| info.config.iface_id.eq(if_id))
}
/// Insert or update a virtio net device into the manager.
pub fn insert_device(
device_mgr: &mut DeviceManager,
mut ctx: DeviceOpContext,
config: VirtioNetDeviceConfigInfo,
) -> std::result::Result<(), VirtioNetDeviceError> {
if config.num_queues % 2 != 0 {
return Err(VirtioNetDeviceError::InvalidQueueNum(config.num_queues));
}
if !cfg!(feature = "hotplug") && ctx.is_hotplug {
return Err(VirtioNetDeviceError::UpdateNotAllowedPostBoot);
}
let mgr = &mut device_mgr.virtio_net_manager;
slog::info!(
ctx.logger(),
"add virtio-net device configuration";
"subsystem" => "net_dev_mgr",
"id" => &config.iface_id,
"host_dev_name" => &config.host_dev_name,
);
let device_index = mgr.info_list.insert_or_update(&config)?;
if ctx.is_hotplug {
slog::info!(
ctx.logger(),
"attach virtio-net device";
"subsystem" => "net_dev_mgr",
"id" => &config.iface_id,
"host_dev_name" => &config.host_dev_name,
);
match Self::create_device(&config, &mut ctx) {
Ok(device) => {
let dev = DeviceManager::create_mmio_virtio_device(
device,
&mut ctx,
config.use_shared_irq.unwrap_or(mgr.use_shared_irq),
config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
)
.map_err(VirtioNetDeviceError::DeviceManager)?;
ctx.insert_hotplug_mmio_device(&dev.clone(), None)
.map_err(VirtioNetDeviceError::DeviceManager)?;
// live-upgrade need save/restore device from info.device.
mgr.info_list[device_index].set_device(dev);
}
Err(e) => {
mgr.info_list.remove(device_index);
return Err(VirtioNetDeviceError::Virtio(e));
}
}
}
Ok(())
}
/// Update the ratelimiter settings of a virtio net device.
pub fn update_device_ratelimiters(
device_mgr: &mut DeviceManager,
new_cfg: VirtioNetDeviceConfigUpdateInfo,
) -> std::result::Result<(), VirtioNetDeviceError> {
let mgr = &mut device_mgr.virtio_net_manager;
match mgr.get_index_of_iface_id(&new_cfg.iface_id) {
Some(index) => {
let config = &mut mgr.info_list[index].config;
config.rx_rate_limiter = new_cfg.rx_rate_limiter.clone();
config.tx_rate_limiter = new_cfg.tx_rate_limiter.clone();
let device = mgr.info_list[index].device.as_mut().ok_or_else(|| {
VirtioNetDeviceError::InvalidIfaceId(new_cfg.iface_id.clone())
})?;
if let Some(mmio_dev) = device.as_any().downcast_ref::<DbsMmioV2Device>() {
let guard = mmio_dev.state();
let inner_dev = guard.get_inner_device();
if let Some(net_dev) = inner_dev
.as_any()
.downcast_ref::<virtio::net::Net<GuestAddressSpaceImpl>>()
{
return net_dev
.set_patch_rate_limiters(
new_cfg.rx_bytes(),
new_cfg.rx_ops(),
new_cfg.tx_bytes(),
new_cfg.tx_ops(),
)
.map(|_p| ())
.map_err(|_e| VirtioNetDeviceError::NetEpollHanderSendFail);
}
}
Ok(())
}
None => Err(VirtioNetDeviceError::InvalidIfaceId(
new_cfg.iface_id.clone(),
)),
}
}
/// Attach all configured vsock device to the virtual machine instance.
pub fn attach_devices(
&mut self,
ctx: &mut DeviceOpContext,
) -> std::result::Result<(), VirtioNetDeviceError> {
for info in self.info_list.iter_mut() {
slog::info!(
ctx.logger(),
"attach virtio-net device";
"subsystem" => "net_dev_mgr",
"id" => &info.config.iface_id,
"host_dev_name" => &info.config.host_dev_name,
);
let device = Self::create_device(&info.config, ctx)
.map_err(VirtioNetDeviceError::CreateNetDevice)?;
let device = DeviceManager::create_mmio_virtio_device(
device,
ctx,
info.config.use_shared_irq.unwrap_or(self.use_shared_irq),
info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
)
.map_err(VirtioNetDeviceError::RegisterNetDevice)?;
info.set_device(device);
}
Ok(())
}
fn create_device(
cfg: &VirtioNetDeviceConfigInfo,
ctx: &mut DeviceOpContext,
) -> std::result::Result<Box<Net<GuestAddressSpaceImpl>>, virtio::Error> {
let epoll_mgr = ctx.epoll_mgr.clone().ok_or(virtio::Error::InvalidInput)?;
let rx_rate_limiter = match cfg.rx_rate_limiter.as_ref() {
Some(rl) => Some(rl.try_into().map_err(virtio::Error::IOError)?),
None => None,
};
let tx_rate_limiter = match cfg.tx_rate_limiter.as_ref() {
Some(rl) => Some(rl.try_into().map_err(virtio::Error::IOError)?),
None => None,
};
let net_device = Net::new(
cfg.host_dev_name.clone(),
cfg.guest_mac(),
Arc::new(cfg.queue_sizes()),
epoll_mgr,
rx_rate_limiter,
tx_rate_limiter,
)?;
Ok(Box::new(net_device))
}
}
impl Default for VirtioNetDeviceMgr {
/// Create a new virtio net device manager.
fn default() -> Self {
VirtioNetDeviceMgr {
info_list: DeviceConfigInfos::new(),
use_shared_irq: USE_SHARED_IRQ,
}
}
}

View File

@ -163,6 +163,11 @@ pub enum StartMicroVmError {
/// Virtio-blk errors.
#[error("virtio-blk errors: {0}")]
BlockDeviceError(#[source] device_manager::blk_dev_mgr::BlockDeviceError),
#[cfg(feature = "virtio-net")]
/// Virtio-net errors.
#[error("virtio-net errors: {0}")]
VirtioNetDeviceError(#[source] device_manager::virtio_net_dev_mgr::VirtioNetDeviceError),
}
/// Errors associated with starting the instance.