From a047331a340f4b51d919f50455b842a30d170148 Mon Sep 17 00:00:00 2001 From: Xuewei Niu Date: Tue, 22 Aug 2023 15:14:43 +0800 Subject: [PATCH] runtime-rs: Network config distinguishes backends Network backends determine the virtio dataplane implementations. Common protocols include virtio-net, vhost-net and vhost-user-net, etc. Network config has a new field named `backend` to specify which protocol to use. Signed-off-by: Xuewei Niu --- src/dragonball/src/api/v1/mod.rs | 2 +- src/dragonball/src/api/v1/virtio_net.rs | 1 + src/dragonball/src/api/v1/vmm_action.rs | 10 ++- src/runtime-rs/Cargo.lock | 13 +++ src/runtime-rs/crates/hypervisor/Cargo.toml | 2 +- .../hypervisor/src/device/device_manager.rs | 27 ++++-- .../hypervisor/src/device/driver/mod.rs | 1 + .../src/device/driver/virtio_net.rs | 86 +++++++++++++++++-- .../hypervisor/src/dragonball/inner_device.rs | 31 ++----- .../hypervisor/src/dragonball/vmm_instance.rs | 4 +- .../src/network/endpoint/ipvlan_endpoint.rs | 9 +- .../src/network/endpoint/macvlan_endpoint.rs | 9 +- .../src/network/endpoint/tap_endpoint.rs | 9 +- .../src/network/endpoint/veth_endpoint.rs | 9 +- .../src/network/endpoint/vlan_endpoint.rs | 9 +- 15 files changed, 157 insertions(+), 65 deletions(-) diff --git a/src/dragonball/src/api/v1/mod.rs b/src/dragonball/src/api/v1/mod.rs index fd1cc434a2..6b6ed0645f 100644 --- a/src/dragonball/src/api/v1/mod.rs +++ b/src/dragonball/src/api/v1/mod.rs @@ -20,4 +20,4 @@ pub use self::machine_config::{VmConfigError, MAX_SUPPORTED_VCPUS}; /// Wrapper for configuring the virtio networking mod virtio_net; -pub use virtio_net::{Backend, NetworkInterfaceConfig, NetworkInterfaceUpdateConfig}; +pub use virtio_net::{Backend, NetworkInterfaceConfig, NetworkInterfaceUpdateConfig, VirtioConfig}; diff --git a/src/dragonball/src/api/v1/virtio_net.rs b/src/dragonball/src/api/v1/virtio_net.rs index c4f04e09b8..dce9e831f9 100644 --- a/src/dragonball/src/api/v1/virtio_net.rs +++ b/src/dragonball/src/api/v1/virtio_net.rs @@ -32,6 +32,7 @@ impl Default for Backend { } } +/// Virtio network config, working for virtio-net and vhost-net. #[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] pub struct VirtioConfig { /// ID of the guest network interface. diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index dfa8869cae..48260bda23 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -320,13 +320,15 @@ impl VmmService { } #[cfg(feature = "virtio-net")] VmmAction::InsertNetworkDevice(config) => match config.backend { - Backend::Virtio(_) => - { + Backend::Virtio(_) => { + #[cfg(not(feature = "virtio-net"))] + panic!("virtio-net feature is not enabled"); #[cfg(feature = "virtio-net")] self.add_virtio_net_device(vmm, event_mgr, config.into()) } - Backend::Vhost(_) => - { + Backend::Vhost(_) => { + #[cfg(not(feature = "vhost-net"))] + panic!("vhost-net feature is not enabled"); #[cfg(feature = "vhost-net")] self.add_vhost_net_device(vmm, event_mgr, config.into()) } diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 3e0b303104..de25176850 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -797,6 +797,7 @@ dependencies = [ "serde_json", "thiserror", "threadpool", + "vhost", "virtio-bindings", "virtio-queue", "vm-memory", @@ -4036,6 +4037,18 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "vhost" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6769e8dbf5276b4376439fbf36bb880d203bf614bf7ef444198edc24b5a9f35" +dependencies = [ + "bitflags 1.3.2", + "libc", + "vm-memory", + "vmm-sys-util 0.11.1", +] + [[package]] name = "virt_container" version = "0.1.0" diff --git a/src/runtime-rs/crates/hypervisor/Cargo.toml b/src/runtime-rs/crates/hypervisor/Cargo.toml index 4c1c5f9249..e277e1decd 100644 --- a/src/runtime-rs/crates/hypervisor/Cargo.toml +++ b/src/runtime-rs/crates/hypervisor/Cargo.toml @@ -36,7 +36,7 @@ kata-types = { path = "../../../libs/kata-types" } logging = { path = "../../../libs/logging" } shim-interface = { path = "../../../libs/shim-interface" } -dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "dbs-upcall"] } +dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "vhost-net", "dbs-upcall"] } ch-config = { path = "ch-config", optional = true } tests_utils = { path = "../../tests/utils" } diff --git a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs index 09971b0a02..967bc42d86 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs @@ -12,8 +12,8 @@ use tokio::sync::{Mutex, RwLock}; use crate::{ vhost_user_blk::VhostUserBlkDevice, BlockConfig, BlockDevice, HybridVsockDevice, Hypervisor, - NetworkDevice, VfioDevice, VhostUserConfig, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, - KATA_NVDIMM_DEV_TYPE, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM, + NetworkBackend, NetworkDevice, VfioDevice, VhostUserConfig, KATA_BLK_DEV_TYPE, + KATA_MMIO_BLK_DEV_TYPE, KATA_NVDIMM_DEV_TYPE, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM, }; use super::{ @@ -221,11 +221,18 @@ impl DeviceManager { return Some(device_id.to_string()); } } - DeviceType::Network(device) => { - if device.config.host_dev_name == host_path { - return Some(device_id.to_string()); + DeviceType::Network(device) => match device.config.backend { + NetworkBackend::Virtio(config) => { + if config.host_dev_name == host_path { + return Some(device_id.to_string()); + } } - } + NetworkBackend::Vhost(config) => { + if config.host_dev_name == host_path { + return Some(device_id.to_string()); + } + } + }, _ => { // TODO: support find other device type continue; @@ -307,11 +314,15 @@ impl DeviceManager { } DeviceConfig::NetworkCfg(config) => { // try to find the device, found and just return id. - if let Some(dev_id_matched) = self.find_device(config.host_dev_name.clone()).await { + let host_path = match &config.backend { + NetworkBackend::Virtio(config) => &config.host_dev_name, + NetworkBackend::Vhost(config) => &config.host_dev_name, + }; + if let Some(dev_id_matched) = self.find_device(host_path.to_owned()).await { info!( sl!(), "network device with path:{:?} found. return network device id: {:?}", - config.host_dev_name.clone(), + host_path, dev_id_matched ); diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs index acc6c4b6ed..22b833205a 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs @@ -23,6 +23,7 @@ pub use virtio_fs::{ ShareFsDevice, ShareFsDeviceConfig, ShareFsMountConfig, ShareFsMountDevice, ShareFsMountType, ShareFsOperation, }; +pub use virtio_net::{Address, NetworkBackend, NetworkConfig, NetworkDevice, VirtioConfig}; pub use virtio_net::{Address, NetworkConfig, NetworkDevice}; pub use virtio_vsock::{ HybridVsockConfig, HybridVsockDevice, VsockConfig, VsockDevice, DEFAULT_GUEST_VSOCK_CID, diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs index 3d19625b9f..467a8351af 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs @@ -8,6 +8,11 @@ use std::fmt; use anyhow::{Context, Result}; use async_trait::async_trait; +use dbs_utils::net::MacAddr as DragonballMacAddr; +use dragonball::api::v1::{ + Backend as DragonballNetworkBackend, NetworkInterfaceConfig as DragonballNetworkConfig, + VirtioConfig as DragonballVirtioConfig, +}; use crate::{ device::{Device, DeviceType}, @@ -28,25 +33,90 @@ impl fmt::Debug for Address { } } +#[derive(Clone, Debug)] +pub enum NetworkBackend { + Virtio(VirtioConfig), + Vhost(VirtioConfig), +} + +impl Default for NetworkBackend { + fn default() -> Self { + Self::Virtio(VirtioConfig::default()) + } +} + +impl From for DragonballNetworkBackend { + fn from(value: NetworkBackend) -> Self { + match value { + NetworkBackend::Virtio(config) => Self::Virtio(DragonballVirtioConfig { + iface_id: config.virt_iface_name.clone(), + host_dev_name: config.host_dev_name.clone(), + rx_rate_limiter: None, + tx_rate_limiter: None, + allow_duplicate_mac: config.allow_duplicate_mac, + }), + NetworkBackend::Vhost(config) => Self::Vhost(DragonballVirtioConfig { + iface_id: config.virt_iface_name.clone(), + host_dev_name: config.host_dev_name.clone(), + rx_rate_limiter: None, + tx_rate_limiter: None, + allow_duplicate_mac: config.allow_duplicate_mac, + }), + } + } +} + +/// Virtio network backend config +#[derive(Clone, Debug, Default)] +pub struct VirtioConfig { + /// Host level path for the guest network interface. + pub host_dev_name: String, + /// Guest iface name for the guest network interface. + pub virt_iface_name: String, + /// Allow duplicate mac + pub allow_duplicate_mac: bool, +} + #[derive(Clone, Debug, Default)] pub struct NetworkConfig { /// for detach, now it's default value 0. pub index: u64, - /// Host level path for the guest network interface. - pub host_dev_name: String, - - /// Guest iface name for the guest network interface. - pub virt_iface_name: String, - + /// Network device backend + pub backend: NetworkBackend, /// Guest MAC address. pub guest_mac: Option
, - /// Virtio queue size pub queue_size: usize, - /// Virtio queue num pub queue_num: usize, + /// Use shared irq + pub use_shared_irq: Option, + /// Use generic irq + pub use_generic_irq: Option, +} + +impl From for DragonballNetworkConfig { + fn from(value: NetworkConfig) -> Self { + let r = &value; + r.into() + } +} + +impl From<&NetworkConfig> for DragonballNetworkConfig { + fn from(value: &NetworkConfig) -> Self { + Self { + num_queues: Some(value.queue_num), + queue_size: Some(value.queue_size as u16), + backend: value.backend.clone().into(), + guest_mac: value.guest_mac.clone().map(|mac| { + // We are safety since mac address is checked by endpoints. + DragonballMacAddr::from_bytes(&mac.0).unwrap() + }), + use_shared_irq: value.use_shared_irq, + use_generic_irq: value.use_generic_irq, + } + } } #[derive(Clone, Debug, Default)] diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs index 1904f2e31f..90c1ae3169 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs @@ -7,14 +7,10 @@ use std::path::PathBuf; use anyhow::{anyhow, Context, Result}; -use dbs_utils::net::MacAddr; -use dragonball::{ - api::v1::{ - BlockDeviceConfigInfo, FsDeviceConfigInfo, FsMountConfigInfo, VirtioNetDeviceConfigInfo, - VsockDeviceConfigInfo, - }, - device_manager::blk_dev_mgr::BlockDeviceType, +use dragonball::api::v1::{ + BlockDeviceConfigInfo, FsDeviceConfigInfo, FsMountConfigInfo, VsockDeviceConfigInfo, }; +use dragonball::device_manager::blk_dev_mgr::BlockDeviceType; use super::DragonballInner; use crate::{ @@ -85,7 +81,7 @@ impl DragonballInner { // Dragonball doesn't support remove network device, just print message. info!( sl!(), - "dragonball remove network device: {:?}.", network.config.virt_iface_name + "dragonball remove network device: {:?}.", network.config ); Ok(()) @@ -204,25 +200,8 @@ impl DragonballInner { } fn add_net_device(&mut self, config: &NetworkConfig) -> Result<()> { - let iface_cfg = VirtioNetDeviceConfigInfo { - iface_id: config.virt_iface_name.clone(), - host_dev_name: config.host_dev_name.clone(), - guest_mac: match &config.guest_mac { - Some(mac) => MacAddr::from_bytes(&mac.0).ok(), - None => None, - }, - num_queues: config.queue_num, - queue_size: config.queue_size as u16, - ..Default::default() - }; - - info!( - sl!(), - "add {} endpoint to {}", iface_cfg.host_dev_name, iface_cfg.iface_id - ); - self.vmm_instance - .insert_network_device(iface_cfg) + .insert_network_device(config.into()) .context("insert network device") } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs index 68dfe41041..ec3b7b6fbd 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs @@ -16,7 +16,7 @@ use crossbeam_channel::{unbounded, Receiver, Sender}; use dragonball::{ api::v1::{ BlockDeviceConfigInfo, BootSourceConfig, FsDeviceConfigInfo, FsMountConfigInfo, - InstanceInfo, InstanceState, VcpuResizeInfo, VirtioNetDeviceConfigInfo, VmmAction, + InstanceInfo, InstanceState, NetworkInterfaceConfig, VcpuResizeInfo, VmmAction, VmmActionError, VmmData, VmmRequest, VmmResponse, VmmService, VsockDeviceConfigInfo, }, vm::VmConfigInfo, @@ -216,7 +216,7 @@ impl VmmInstance { Ok(()) } - pub fn insert_network_device(&self, net_cfg: VirtioNetDeviceConfigInfo) -> Result<()> { + pub fn insert_network_device(&self, net_cfg: NetworkInterfaceConfig) -> Result<()> { self.handle_request_with_retry(Request::Sync(VmmAction::InsertNetworkDevice( net_cfg.clone(), ))) diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs index 7039275e86..13300c0f86 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs @@ -19,7 +19,7 @@ use hypervisor::{ driver::NetworkConfig, DeviceConfig, DeviceType, }, - Hypervisor, NetworkDevice, + Hypervisor, NetworkBackend, NetworkDevice, VirtioConfig, }; use super::{ @@ -64,8 +64,11 @@ impl IPVlanEndpoint { })?; Ok(NetworkConfig { - host_dev_name: iface.name.clone(), - virt_iface_name: self.net_pair.virt_iface.name.clone(), + backend: NetworkBackend::Virtio(VirtioConfig { + host_dev_name: iface.name.clone(), + virt_iface_name: self.net_pair.virt_iface.name.clone(), + ..Default::default() + }), guest_mac: Some(guest_mac), ..Default::default() }) diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs index ad390973fb..d5db2df2b5 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs @@ -19,7 +19,7 @@ use hypervisor::{ driver::NetworkConfig, DeviceConfig, DeviceType, }, - Hypervisor, NetworkDevice, + Hypervisor, NetworkBackend, NetworkDevice, VirtioConfig, }; use super::{ @@ -63,8 +63,11 @@ impl MacVlanEndpoint { })?; Ok(NetworkConfig { - host_dev_name: iface.name.clone(), - virt_iface_name: self.net_pair.virt_iface.name.clone(), + backend: NetworkBackend::Virtio(VirtioConfig { + host_dev_name: iface.name.clone(), + virt_iface_name: self.net_pair.virt_iface.name.clone(), + ..Default::default() + }), guest_mac: Some(guest_mac), ..Default::default() }) diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/tap_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/tap_endpoint.rs index e22a91d922..0b5db23f96 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/tap_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/tap_endpoint.rs @@ -10,7 +10,7 @@ use anyhow::{Context, Result}; use async_trait::async_trait; use hypervisor::device::device_manager::{do_handle_device, DeviceManager}; use hypervisor::device::{DeviceConfig, DeviceType}; -use hypervisor::{Hypervisor, NetworkConfig, NetworkDevice}; +use hypervisor::{Hypervisor, NetworkBackend, NetworkConfig, NetworkDevice, VirtioConfig}; use tokio::sync::RwLock; use super::endpoint_persist::TapEndpointState; @@ -74,8 +74,11 @@ impl TapEndpoint { fn get_network_config(&self) -> Result { let guest_mac = utils::parse_mac(&self.guest_mac).context("Parse mac address")?; Ok(NetworkConfig { - host_dev_name: self.tap_iface.name.clone(), - virt_iface_name: self.name.clone(), + backend: NetworkBackend::Virtio(VirtioConfig { + host_dev_name: self.tap_iface.name.clone(), + virt_iface_name: self.name.clone(), + ..Default::default() + }), guest_mac: Some(guest_mac), queue_num: self.queue_num, queue_size: self.queue_size, diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs index b24b5cf31c..131de67362 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs @@ -19,7 +19,7 @@ use hypervisor::{ driver::NetworkConfig, DeviceConfig, DeviceType, }, - Hypervisor, NetworkDevice, + Hypervisor, NetworkBackend, NetworkDevice, VirtioConfig, }; use super::{ @@ -63,8 +63,11 @@ impl VethEndpoint { })?; Ok(NetworkConfig { - host_dev_name: iface.name.clone(), - virt_iface_name: self.net_pair.virt_iface.name.clone(), + backend: NetworkBackend::Virtio(VirtioConfig { + host_dev_name: iface.name.clone(), + virt_iface_name: self.net_pair.virt_iface.name.clone(), + ..Default::default() + }), guest_mac: Some(guest_mac), ..Default::default() }) diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs index bfc852d39b..770390b568 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs @@ -19,7 +19,7 @@ use hypervisor::{ driver::NetworkConfig, DeviceConfig, DeviceType, }, - Hypervisor, NetworkDevice, + Hypervisor, NetworkBackend, NetworkDevice, VirtioConfig, }; use super::{ @@ -62,8 +62,11 @@ impl VlanEndpoint { })?; Ok(NetworkConfig { - host_dev_name: iface.name.clone(), - virt_iface_name: self.net_pair.virt_iface.name.clone(), + backend: NetworkBackend::Virtio(VirtioConfig { + host_dev_name: iface.name.clone(), + virt_iface_name: self.net_pair.virt_iface.name.clone(), + ..Default::default() + }), guest_mac: Some(guest_mac), ..Default::default() })