diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index f0fc11dda4..e7b1da0c06 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -42,7 +42,11 @@ slog-async = "2.7.0" [features] atomic-guest-memory = [] +virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"] [patch.'crates-io'] +dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } +dbs-interrupt = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } dbs-legacy-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } +dbs-utils = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } dbs-virtio-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 5ffe5ed3ab..5691690ea8 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -16,6 +16,19 @@ use dbs_legacy_devices::ConsoleHandler; use dbs_utils::epoll_manager::EpollManager; use kvm_ioctls::VmFd; +#[cfg(feature = "dbs-virtio-devices")] +use dbs_device::resources::ResourceConstraint; +#[cfg(feature = "dbs-virtio-devices")] +use dbs_virtio_devices as virtio; +#[cfg(feature = "dbs-virtio-devices")] +use dbs_virtio_devices::{ + mmio::{ + MmioV2Device, DRAGONBALL_FEATURE_INTR_USED, DRAGONBALL_FEATURE_PER_QUEUE_NOTIFY, + DRAGONBALL_MMIO_DOORBELL_SIZE, MMIO_DEFAULT_CFG_SIZE, + }, + VirtioDevice, +}; + use crate::address_space_manager::GuestAddressSpaceImpl; use crate::error::StartMicrovmError; use crate::resource_manager::ResourceManager; @@ -29,6 +42,18 @@ pub use self::console_manager::ConsoleManager; mod legacy; pub use self::legacy::{Error as LegacyDeviceError, LegacyDeviceManager}; +#[cfg(feature = "virtio-vsock")] +/// Device manager for user-space vsock devices. +pub mod vsock_dev_mgr; +#[cfg(feature = "virtio-vsock")] +use self::vsock_dev_mgr::VsockDeviceMgr; + +macro_rules! info( + ($l:expr, $($args:tt)+) => { + slog::info!($l, $($args)+; slog::o!("subsystem" => "device_manager")) + }; +); + /// Errors related to device manager operations. #[derive(Debug, thiserror::Error)] pub enum DeviceMgrError { @@ -53,11 +78,22 @@ pub enum DeviceMgrError { /// Failure from legacy device manager. #[error(transparent)] LegacyManager(legacy::Error), + + #[cfg(feature = "dbs-virtio-devices")] + /// Error from Virtio subsystem. + #[error(transparent)] + Virtio(virtio::Error), } /// Specialized version of `std::result::Result` for device manager operations. pub type Result = ::std::result::Result; +/// Type of the dragonball virtio devices. +#[cfg(feature = "dbs-virtio-devices")] +pub type DbsVirtioDevice = Box< + dyn VirtioDevice, +>; + /// Type of the dragonball virtio mmio devices. #[cfg(feature = "dbs-virtio-devices")] pub type DbsMmioV2Device = @@ -240,6 +276,8 @@ pub struct DeviceManager { pub(crate) con_manager: ConsoleManager, pub(crate) legacy_manager: Option, + #[cfg(feature = "virtio-vsock")] + pub(crate) vsock_manager: VsockDeviceMgr, } impl DeviceManager { @@ -259,6 +297,8 @@ impl DeviceManager { logger: logger.new(slog::o!()), con_manager: ConsoleManager::new(epoll_manager, logger), legacy_manager: None, + #[cfg(feature = "virtio-vsock")] + vsock_manager: VsockDeviceMgr::default(), } } @@ -386,6 +426,9 @@ impl DeviceManager { self.create_legacy_devices(&mut ctx)?; self.init_legacy_devices(dmesg_fifo, com1_sock_path, &mut ctx)?; + #[cfg(feature = "virtio-vsock")] + self.vsock_manager.attach_devices(&mut ctx)?; + ctx.generate_kernel_boot_args(kernel_config) .map_err(StartMicrovmError::DeviceManager)?; @@ -424,4 +467,136 @@ impl DeviceManager { Err(DeviceMgrError::GetDeviceResource) } } + + /// Create an Virtio MMIO transport layer device for the virtio backend device. + pub fn create_mmio_virtio_device( + device: DbsVirtioDevice, + ctx: &mut DeviceOpContext, + use_shared_irq: bool, + use_generic_irq: bool, + ) -> std::result::Result, DeviceMgrError> { + let features = DRAGONBALL_FEATURE_INTR_USED | DRAGONBALL_FEATURE_PER_QUEUE_NOTIFY; + DeviceManager::create_mmio_virtio_device_with_features( + device, + ctx, + Some(features), + use_shared_irq, + use_generic_irq, + ) + } + + /// Create an Virtio MMIO transport layer device for the virtio backend device with specified + /// features. + pub fn create_mmio_virtio_device_with_features( + device: DbsVirtioDevice, + ctx: &mut DeviceOpContext, + features: Option, + use_shared_irq: bool, + use_generic_irq: bool, + ) -> std::result::Result, DeviceMgrError> { + // Every emulated Virtio MMIO device needs a 4K configuration space, + // and another 4K space for per queue notification. + const MMIO_ADDRESS_DEFAULT: ResourceConstraint = ResourceConstraint::MmioAddress { + range: None, + align: 0, + size: MMIO_DEFAULT_CFG_SIZE + DRAGONBALL_MMIO_DOORBELL_SIZE, + }; + let mut requests = vec![MMIO_ADDRESS_DEFAULT]; + device.get_resource_requirements(&mut requests, use_generic_irq); + let resources = ctx + .res_manager + .allocate_device_resources(&requests, use_shared_irq) + .map_err(|_| DeviceMgrError::GetDeviceResource)?; + + let virtio_dev = match MmioV2Device::new( + ctx.vm_fd.clone(), + ctx.get_vm_as()?, + ctx.irq_manager.clone(), + device, + resources, + features, + ) { + Ok(d) => d, + Err(e) => return Err(DeviceMgrError::Virtio(e)), + }; + + Self::register_mmio_virtio_device(Arc::new(virtio_dev), ctx) + } + + /// Teardown the Virtio MMIO transport layer device associated with the virtio backend device. + pub fn destroy_mmio_virtio_device( + device: Arc, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), DeviceMgrError> { + Self::destroy_mmio_device(device.clone(), ctx)?; + + let mmio_dev = device + .as_any() + .downcast_ref::() + .ok_or(DeviceMgrError::InvalidOperation)?; + + mmio_dev.remove(); + + Ok(()) + } + + fn destroy_mmio_device( + device: Arc, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), DeviceMgrError> { + // unregister IoManager + Self::deregister_mmio_virtio_device(&device, ctx)?; + + // unregister Resource manager + let resources = device.get_assigned_resources(); + ctx.res_manager.free_device_resources(&resources); + + Ok(()) + } + + /// Create an Virtio MMIO transport layer device for the virtio backend device. + pub fn register_mmio_virtio_device( + device: Arc, + ctx: &mut DeviceOpContext, + ) -> std::result::Result, DeviceMgrError> { + let (mmio_base, mmio_size, irq) = Self::get_virtio_device_info(&device)?; + info!( + ctx.logger(), + "create virtio mmio device 0x{:x}@0x{:x}, irq: 0x{:x}", mmio_size, mmio_base, irq + ); + let resources = device.get_trapped_io_resources(); + + let mut tx = ctx.io_context.begin_tx(); + if let Err(e) = ctx + .io_context + .register_device_io(&mut tx, device.clone(), &resources) + { + ctx.io_context.cancel_tx(tx); + Err(DeviceMgrError::IoManager(e)) + } else { + ctx.virtio_devices.push(device.clone()); + ctx.io_context.commit_tx(tx); + Ok(device) + } + } + + /// Deregister a Virtio MMIO device from IoManager + pub fn deregister_mmio_virtio_device( + device: &Arc, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), DeviceMgrError> { + let resources = device.get_trapped_io_resources(); + info!( + ctx.logger(), + "unregister mmio virtio device: {:?}", resources + ); + let mut tx = ctx.io_context.begin_tx(); + if let Err(e) = ctx.io_context.unregister_device_io(&mut tx, &resources) { + ctx.io_context.cancel_tx(tx); + Err(DeviceMgrError::IoManager(e)) + } else { + ctx.io_context.commit_tx(tx); + Ok(()) + } + } } diff --git a/src/dragonball/src/device_manager/vsock_dev_mgr.rs b/src/dragonball/src/device_manager/vsock_dev_mgr.rs new file mode 100644 index 0000000000..cec58d7de1 --- /dev/null +++ b/src/dragonball/src/device_manager/vsock_dev_mgr.rs @@ -0,0 +1,285 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +use std::sync::Arc; + +use dbs_virtio_devices as virtio; +use dbs_virtio_devices::mmio::DRAGONBALL_FEATURE_INTR_USED; +use dbs_virtio_devices::vsock::backend::{ + VsockInnerBackend, VsockInnerConnector, VsockTcpBackend, VsockUnixStreamBackend, +}; +use dbs_virtio_devices::vsock::Vsock; +use dbs_virtio_devices::Error as VirtioError; +use serde_derive::{Deserialize, Serialize}; + +use super::StartMicrovmError; +use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos}; +use crate::device_manager::{DeviceManager, DeviceOpContext}; + +pub use dbs_virtio_devices::vsock::QUEUE_SIZES; + +const SUBSYSTEM: &str = "vsock_dev_mgr"; +// The flag of whether to use the shared irq. +const USE_SHARED_IRQ: bool = true; +// The flag of whether to use the generic irq. +const USE_GENERIC_IRQ: bool = true; + +/// Errors associated with `VsockDeviceConfigInfo`. +#[derive(Debug, thiserror::Error)] +pub enum VsockDeviceError { + /// The virtual machine instance ID is invalid. + #[error("the virtual machine instance ID is invalid")] + InvalidVMID, + + /// The Context Identifier is already in use. + #[error("the device ID {0} already exists")] + DeviceIDAlreadyExist(String), + + /// The Context Identifier is invalid. + #[error("the guest CID {0} is invalid")] + GuestCIDInvalid(u32), + + /// The Context Identifier is already in use. + #[error("the guest CID {0} is already in use")] + GuestCIDAlreadyInUse(u32), + + /// The Unix Domain Socket path is already in use. + #[error("the Unix Domain Socket path {0} is already in use")] + UDSPathAlreadyInUse(String), + + /// The net address is already in use. + #[error("the net address {0} is already in use")] + NetAddrAlreadyInUse(String), + + /// The update is not allowed after booting the microvm. + #[error("update operation is not allowed after boot")] + UpdateNotAllowedPostBoot, + + /// The VsockId Already Exists + #[error("vsock id {0} already exists")] + VsockIdAlreadyExists(String), + + /// Inner backend create error + #[error("vsock inner backend create error: {0}")] + CreateInnerBackend(#[source] std::io::Error), +} + +/// Configuration information for a vsock device. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct VsockDeviceConfigInfo { + /// ID of the vsock device. + pub id: String, + /// A 32-bit Context Identifier (CID) used to identify the guest. + pub guest_cid: u32, + /// unix domain socket path. + pub uds_path: Option, + /// tcp socket address. + pub tcp_addr: Option, + /// Virtio queue size. + pub queue_size: Vec, + /// Use shared irq + pub use_shared_irq: Option, + /// Use generic irq + pub use_generic_irq: Option, +} + +impl VsockDeviceConfigInfo { + /// Get number and size of queues supported. + pub fn queue_sizes(&self) -> Vec { + self.queue_size.clone() + } +} + +impl ConfigItem for VsockDeviceConfigInfo { + type Err = VsockDeviceError; + + fn id(&self) -> &str { + &self.id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), VsockDeviceError> { + if self.id == other.id { + return Err(VsockDeviceError::DeviceIDAlreadyExist(self.id.clone())); + } + if self.guest_cid == other.guest_cid { + return Err(VsockDeviceError::GuestCIDAlreadyInUse(self.guest_cid)); + } + if let (Some(self_uds_path), Some(other_uds_path)) = + (self.uds_path.as_ref(), other.uds_path.as_ref()) + { + if self_uds_path == other_uds_path { + return Err(VsockDeviceError::UDSPathAlreadyInUse(self_uds_path.clone())); + } + } + if let (Some(self_net_addr), Some(other_net_addr)) = + (self.tcp_addr.as_ref(), other.tcp_addr.as_ref()) + { + if self_net_addr == other_net_addr { + return Err(VsockDeviceError::NetAddrAlreadyInUse(self_net_addr.clone())); + } + } + + Ok(()) + } +} + +/// Vsock Device Info +pub type VsockDeviceInfo = DeviceConfigInfo; + +/// Device manager to manage all vsock devices. +pub struct VsockDeviceMgr { + pub(crate) info_list: DeviceConfigInfos, + pub(crate) default_inner_backend: Option, + pub(crate) default_inner_connector: Option, + pub(crate) use_shared_irq: bool, +} + +impl VsockDeviceMgr { + /// Insert or update a vsock device into the manager. + pub fn insert_device( + &mut self, + ctx: DeviceOpContext, + config: VsockDeviceConfigInfo, + ) -> std::result::Result<(), VsockDeviceError> { + if ctx.is_hotplug { + slog::error!( + ctx.logger(), + "no support of virtio-vsock device hotplug"; + "subsystem" => SUBSYSTEM, + "id" => &config.id, + "uds_path" => &config.uds_path, + ); + + return Err(VsockDeviceError::UpdateNotAllowedPostBoot); + } + + // VMADDR_CID_ANY (-1U) means any address for binding; + // VMADDR_CID_HYPERVISOR (0) is reserved for services built into the hypervisor; + // VMADDR_CID_RESERVED (1) must not be used; + // VMADDR_CID_HOST (2) is the well-known address of the host. + if config.guest_cid <= 2 { + return Err(VsockDeviceError::GuestCIDInvalid(config.guest_cid)); + } + + slog::info!( + ctx.logger(), + "add virtio-vsock device configuration"; + "subsystem" => SUBSYSTEM, + "id" => &config.id, + "uds_path" => &config.uds_path, + ); + + self.lazy_make_default_connector()?; + + self.info_list.insert_or_update(&config)?; + + Ok(()) + } + + /// Attach all configured vsock device to the virtual machine instance. + pub fn attach_devices( + &mut self, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), StartMicrovmError> { + let epoll_mgr = ctx + .epoll_mgr + .clone() + .ok_or(StartMicrovmError::CreateVsockDevice( + virtio::Error::InvalidInput, + ))?; + + for info in self.info_list.iter_mut() { + slog::info!( + ctx.logger(), + "attach virtio-vsock device"; + "subsystem" => SUBSYSTEM, + "id" => &info.config.id, + "uds_path" => &info.config.uds_path, + ); + + let mut device = Box::new( + Vsock::new( + info.config.guest_cid as u64, + Arc::new(info.config.queue_sizes()), + epoll_mgr.clone(), + ) + .map_err(VirtioError::VirtioVsockError) + .map_err(StartMicrovmError::CreateVsockDevice)?, + ); + if let Some(uds_path) = info.config.uds_path.as_ref() { + let unix_backend = VsockUnixStreamBackend::new(uds_path.clone()) + .map_err(VirtioError::VirtioVsockError) + .map_err(StartMicrovmError::CreateVsockDevice)?; + device + .add_backend(Box::new(unix_backend), true) + .map_err(VirtioError::VirtioVsockError) + .map_err(StartMicrovmError::CreateVsockDevice)?; + } + if let Some(tcp_addr) = info.config.tcp_addr.as_ref() { + let tcp_backend = VsockTcpBackend::new(tcp_addr.clone()) + .map_err(VirtioError::VirtioVsockError) + .map_err(StartMicrovmError::CreateVsockDevice)?; + device + .add_backend(Box::new(tcp_backend), false) + .map_err(VirtioError::VirtioVsockError) + .map_err(StartMicrovmError::CreateVsockDevice)?; + } + // add inner backend to the the first added vsock device + if let Some(inner_backend) = self.default_inner_backend.take() { + device + .add_backend(Box::new(inner_backend), false) + .map_err(VirtioError::VirtioVsockError) + .map_err(StartMicrovmError::CreateVsockDevice)?; + } + let device = DeviceManager::create_mmio_virtio_device_with_features( + device, + ctx, + Some(DRAGONBALL_FEATURE_INTR_USED), + info.config.use_shared_irq.unwrap_or(self.use_shared_irq), + info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(StartMicrovmError::RegisterVsockDevice)?; + info.device = Some(device); + } + + Ok(()) + } + + // check the default connector is present, or build it. + fn lazy_make_default_connector(&mut self) -> std::result::Result<(), VsockDeviceError> { + if self.default_inner_connector.is_none() { + let inner_backend = + VsockInnerBackend::new().map_err(VsockDeviceError::CreateInnerBackend)?; + self.default_inner_connector = Some(inner_backend.get_connector()); + self.default_inner_backend = Some(inner_backend); + } + Ok(()) + } + + /// Get the default vsock inner connector. + pub fn get_default_connector( + &mut self, + ) -> std::result::Result { + self.lazy_make_default_connector()?; + + // safe to unwrap, because we created the inner connector before + Ok(self.default_inner_connector.clone().unwrap()) + } +} + +impl Default for VsockDeviceMgr { + /// Create a new Vsock device manager. + fn default() -> Self { + VsockDeviceMgr { + info_list: DeviceConfigInfos::new(), + default_inner_backend: None, + default_inner_connector: None, + use_shared_irq: USE_SHARED_IRQ, + } + } +} diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index af54810d24..5a497abb6b 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -9,18 +9,33 @@ //! Error codes for the virtual machine monitor subsystem. +#[cfg(feature = "dbs-virtio-devices")] +use dbs_virtio_devices::Error as VirtIoError; + +use crate::device_manager; + /// Errors associated with starting the instance. #[derive(Debug, thiserror::Error)] pub enum StartMicrovmError { - /// The device manager was not configured. - #[error("the device manager failed to manage devices: {0}")] - DeviceManager(#[source] crate::device_manager::DeviceMgrError), - - /// Cannot add devices to the Legacy I/O Bus. - #[error("failure in managing legacy device: {0}")] - LegacyDevice(#[source] crate::device_manager::LegacyDeviceError), - /// Cannot read from an Event file descriptor. #[error("failure while reading from EventFd file descriptor")] EventFd, + + /// The device manager was not configured. + #[error("the device manager failed to manage devices: {0}")] + DeviceManager(#[source] device_manager::DeviceMgrError), + + /// Cannot add devices to the Legacy I/O Bus. + #[error("failure in managing legacy device: {0}")] + LegacyDevice(#[source] device_manager::LegacyDeviceError), + + #[cfg(feature = "virtio-vsock")] + /// Failed to create the vsock device. + #[error("cannot create virtio-vsock device: {0}")] + CreateVsockDevice(#[source] VirtIoError), + + #[cfg(feature = "virtio-vsock")] + /// Cannot initialize a MMIO Vsock Device or add a device to the MMIO Bus. + #[error("failure while registering virtio-vsock device: {0}")] + RegisterVsockDevice(#[source] device_manager::DeviceMgrError), }