github/kata-containers
1
0
Fork 2
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-09-17 23:07:55 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #8626 from justxuewei/vhost-user-endpoint

Browse Source
This commit is contained in:
Xuewei Niu
2023-12-26 12:52:21 +08:00
committed by GitHub
parent 38eb4077a6 36a4cbccf6
commit 1065ca6fa7
16 changed files with 281 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/dragonball/src/api/v1/mod.rs
View File

@@ -32,4 +32,6 @@ pub use virtio_net::VirtioConfig;
feature = "vhost-net",
feature = "vhost-user-net"
))]
pub use virtio_net::{Backend, NetworkInterfaceConfig, NetworkInterfaceUpdateConfig};
pub use virtio_net::{
Backend, NetworkInterfaceConfig, NetworkInterfaceUpdateConfig, VhostUserConfig,
};

15
src/dragonball/src/api/v1/virtio_net.rs
View File

@@ -68,6 +68,7 @@ pub struct VirtioConfig {
pub allow_duplicate_mac: bool,
}
/// Config for vhost-user-net device
#[cfg(feature = "vhost-user-net")]
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
pub struct VhostUserConfig {
@@ -197,9 +198,23 @@ impl From<&NetworkInterfaceConfig> for VhostUserNetDeviceConfigInfo {
fn from(value: &NetworkInterfaceConfig) -> Self {
let num_queues = value
.num_queues
.map(|nq| {
if nq == 0 {
vhost_user_net_dev_mgr::DEFAULT_NUM_QUEUES
} else {
nq
}
})
.unwrap_or(vhost_user_net_dev_mgr::DEFAULT_NUM_QUEUES);
let queue_size = value
.queue_size
.map(|qs| {
if qs == 0 {
vhost_user_net_dev_mgr::DEFAULT_QUEUE_SIZE
} else {
qs
}
})
.unwrap_or(vhost_user_net_dev_mgr::DEFAULT_QUEUE_SIZE);
// It is safe because we tested the type of config before.
#[allow(unreachable_patterns)]

1
src/runtime-rs/Cargo.lock generated
View File

@@ -808,6 +808,7 @@ dependencies = [
"byteorder",
"caps",
"dbs-address-space",
"dbs-boot",
"dbs-device",
"dbs-interrupt",
"dbs-utils",

2
src/runtime-rs/crates/hypervisor/Cargo.toml
View File

@@ -35,7 +35,7 @@ kata-types = { path = "../../../libs/kata-types" }
logging = { path = "../../../libs/logging" }
shim-interface = { path = "../../../libs/shim-interface" }
dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "vhost-net", "dbs-upcall","virtio-mem", "virtio-balloon"] }
dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "vhost-net", "dbs-upcall","virtio-mem", "virtio-balloon", "vhost-user-net"] }
ch-config = { path = "ch-config", optional = true }
tests_utils = { path = "../../tests/utils" }

29
src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
View File

@@ -12,8 +12,9 @@ use tokio::sync::{Mutex, RwLock};
use crate::{
vhost_user_blk::VhostUserBlkDevice, BlockConfig, BlockDevice, HybridVsockDevice, Hypervisor,
NetworkDevice, ShareFsDevice, VfioDevice, VhostUserConfig, VsockDevice, KATA_BLK_DEV_TYPE,
KATA_MMIO_BLK_DEV_TYPE, KATA_NVDIMM_DEV_TYPE, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM,
NetworkDevice, ShareFsDevice, VfioDevice, VhostUserConfig, VhostUserNetDevice, VsockDevice,
KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, KATA_NVDIMM_DEV_TYPE, VIRTIO_BLOCK_MMIO,
VIRTIO_BLOCK_PCI, VIRTIO_PMEM,
};
use super::{
@@ -231,8 +232,12 @@ impl DeviceManager {
return Some(device_id.to_string());
}
}
_ => {
// TODO: support find other device type
DeviceType::VhostUserNetwork(device) => {
if device.config.socket_path == host_path {
return Some(device_id.to_string());
}
}
DeviceType::HybridVsock(_) | DeviceType::Vsock(_) => {
continue;
}
}
@@ -326,6 +331,22 @@ impl DeviceManager {
Arc::new(Mutex::new(NetworkDevice::new(device_id.clone(), config)))
}
DeviceConfig::VhostUserNetworkCfg(config) => {
if let Some(dev_id) = self.find_device(config.socket_path.clone()).await {
info!(
sl!(),
"vhost-user-net device {} found, just return device id {}",
config.socket_path,
dev_id
);
return Ok(dev_id);
}
Arc::new(Mutex::new(VhostUserNetDevice::new(
device_id.clone(),
config.clone(),
)))
}
DeviceConfig::HybridVsockCfg(hvconfig) => {
// No need to do find device for hybrid vsock device.
Arc::new(Mutex::new(HybridVsockDevice::new(&device_id, hvconfig)))

7
src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs
View File

@@ -6,6 +6,8 @@
mod vfio;
mod vhost_user;
pub mod vhost_user_blk;
mod vhost_user_net;
mod virtio_blk;
mod virtio_fs;
mod virtio_net;
@@ -15,6 +17,8 @@ pub use vfio::{
bind_device_to_host, bind_device_to_vfio, get_vfio_device, HostDevice, VfioBusMode, VfioConfig,
VfioDevice,
};
pub use vhost_user::{VhostUserConfig, VhostUserDevice, VhostUserType};
pub use vhost_user_net::VhostUserNetDevice;
pub use virtio_blk::{
BlockConfig, BlockDevice, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, KATA_NVDIMM_DEV_TYPE,
VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM,
@@ -26,6 +30,3 @@ pub use virtio_net::{Address, NetworkConfig, NetworkDevice};
pub use virtio_vsock::{
HybridVsockConfig, HybridVsockDevice, VsockConfig, VsockDevice, DEFAULT_GUEST_VSOCK_CID,
};
pub mod vhost_user_blk;
pub use vhost_user::{VhostUserConfig, VhostUserDevice, VhostUserType};

21
src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs
View File

@@ -6,29 +6,20 @@
use crate::device::pci_path::PciPath;
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Default)]
pub enum VhostUserType {
/// Blk - represents a block vhostuser device type
/// "vhost-user-blk-pci"
Blk(String),
#[default]
Blk,
/// SCSI - represents SCSI based vhost-user type
/// "vhost-user-scsi-pci"
SCSI(String),
SCSI,
/// Net - represents Net based vhost-user type
/// "virtio-net-pci"
Net(String),
Net,
/// FS - represents a virtio-fs vhostuser device type
/// "vhost-user-fs-pci"
FS(String),
}
impl Default for VhostUserType {
fn default() -> Self {
VhostUserType::Blk("vhost-user-blk-pci".to_owned())
}
FS,
}
#[derive(Debug, Clone, Default)]

59
src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user_net.rs Normal file
View File

@@ -0,0 +1,59 @@
// Copyright (C) 2019-2023 Ant Group. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
use anyhow::{Context, Result};
use async_trait::async_trait;
use crate::device::{Device, DeviceType};
use crate::{Hypervisor, VhostUserConfig};
#[derive(Debug, Clone, Default)]
/// Vhost-user-net device for device manager.
pub struct VhostUserNetDevice {
pub device_id: String,
pub config: VhostUserConfig,
}
impl VhostUserNetDevice {
pub fn new(device_id: String, config: VhostUserConfig) -> Self {
Self { device_id, config }
}
}
#[async_trait]
impl Device for VhostUserNetDevice {
async fn attach(&mut self, h: &dyn Hypervisor) -> Result<()> {
h.add_device(DeviceType::VhostUserNetwork(self.clone()))
.await
.context("add vhost-user-net device to hypervisor")?;
Ok(())
}
async fn detach(&mut self, h: &dyn Hypervisor) -> Result<Option<u64>> {
h.remove_device(DeviceType::VhostUserNetwork(self.clone()))
.await
.context("remove vhost-user-net device from hypervisor")?;
Ok(Some(self.config.index))
}
async fn update(&mut self, _h: &dyn Hypervisor) -> Result<()> {
// There's no need to do update for vhost-user-net
Ok(())
}
async fn get_device_info(&self) -> DeviceType {
DeviceType::VhostUserNetwork(self.clone())
}
async fn increase_attach_count(&mut self) -> Result<bool> {
// Vhost-user-net devices will not be attached multiple times, just
// return Ok(false)
Ok(false)
}
async fn decrease_attach_count(&mut self) -> Result<bool> {
// Vhost-user-net devices will not be detached multiple times, just
// return Ok(false)
Ok(false)
}
}

4
src/runtime-rs/crates/hypervisor/src/device/mod.rs
View File

@@ -10,7 +10,7 @@ use crate::device::driver::vhost_user_blk::VhostUserBlkDevice;
use crate::{
BlockConfig, BlockDevice, HybridVsockConfig, HybridVsockDevice, Hypervisor as hypervisor,
NetworkConfig, NetworkDevice, ShareFsConfig, ShareFsDevice, VfioConfig, VfioDevice,
VhostUserConfig, VsockConfig, VsockDevice,
VhostUserConfig, VhostUserNetDevice, VsockConfig, VsockDevice,
};
use anyhow::Result;
use async_trait::async_trait;
@@ -25,6 +25,7 @@ pub enum DeviceConfig {
BlockCfg(BlockConfig),
VhostUserBlkCfg(VhostUserConfig),
NetworkCfg(NetworkConfig),
VhostUserNetworkCfg(VhostUserConfig),
ShareFsCfg(ShareFsConfig),
VfioCfg(VfioConfig),
VsockCfg(VsockConfig),
@@ -37,6 +38,7 @@ pub enum DeviceType {
VhostUserBlk(VhostUserBlkDevice),
Vfio(VfioDevice),
Network(NetworkDevice),
VhostUserNetwork(VhostUserNetDevice),
ShareFs(ShareFsDevice),
HybridVsock(HybridVsockDevice),
Vsock(VsockDevice),

28
src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs
View File

@@ -7,12 +7,16 @@
use std::path::PathBuf;
use anyhow::{anyhow, Context, Result};
use dbs_utils::net::MacAddr;
use dragonball::api::v1::VhostUserConfig as DragonballVhostUserConfig;
use dragonball::api::v1::{
BlockDeviceConfigInfo, FsDeviceConfigInfo, FsMountConfigInfo, VsockDeviceConfigInfo,
BlockDeviceConfigInfo, FsDeviceConfigInfo, FsMountConfigInfo, NetworkInterfaceConfig,
VsockDeviceConfigInfo,
};
use dragonball::device_manager::blk_dev_mgr::BlockDeviceType;
use super::{build_dragonball_network_config, DragonballInner};
use crate::VhostUserConfig;
use crate::{
device::DeviceType, HybridVsockConfig, NetworkConfig, ShareFsConfig, ShareFsMountConfig,
ShareFsMountOperation, ShareFsMountType, VfioBusMode, VfioDevice, VmmState, JAILER_ROOT,
@@ -67,6 +71,9 @@ impl DragonballInner {
DeviceType::ShareFs(sharefs) => self
.add_share_fs_device(&sharefs.config)
.context("add share fs device"),
DeviceType::VhostUserNetwork(dev) => self
.add_vhost_user_net_device(&dev.config)
.context("add vhost-user-net device"),
DeviceType::Vsock(_) => todo!(),
}
}
@@ -216,6 +223,25 @@ impl DragonballInner {
.context("insert network device")
}
/// Add vhost-user-net deivce to Dragonball
fn add_vhost_user_net_device(&mut self, config: &VhostUserConfig) -> Result<()> {
let guest_mac = MacAddr::parse_str(&config.mac_address).ok();
let net_cfg = NetworkInterfaceConfig {
num_queues: Some(config.num_queues),
queue_size: Some(config.queue_size as u16),
backend: dragonball::api::v1::Backend::VhostUser(DragonballVhostUserConfig {
sock_path: config.socket_path.clone(),
}),
guest_mac,
use_shared_irq: None,
use_generic_irq: None,
};
self.vmm_instance
.insert_network_device(net_cfg)
.context("insert vhost-user-net device")
}
fn add_hvsock(&mut self, config: &HybridVsockConfig) -> Result<()> {
let vsock_cfg = VsockDeviceConfigInfo {
id: String::from(JAILER_ROOT),

29
src/runtime-rs/crates/resource/src/network/dan.rs
View File

@@ -33,9 +33,10 @@ use tokio::sync::RwLock;
use super::network_entity::NetworkEntity;
use super::utils::address::{ip_family_from_ip_addr, parse_ip_cidr};
use super::{EndpointState, NetnsGuard, Network};
use crate::network::endpoint::TapEndpoint;
use crate::network::endpoint::{TapEndpoint, VhostUserEndpoint};
use crate::network::network_info::network_info_from_dan::NetworkInfoFromDan;
use crate::network::utils::generate_private_mac_addr;
use crate::network::Endpoint;
/// Directly attachable network
pub struct Dan {
@@ -78,16 +79,23 @@ impl DanInner {
let mut entity_list = Vec::with_capacity(config.devices.len());
for (idx, device) in config.devices.iter().enumerate() {
let name = format!("eth{}", idx);
let endpoint = match &device.device {
// TODO: Support VhostUserNet protocol
let endpoint: Arc<dyn Endpoint> = match &device.device {
Device::VhostUser {
path,
queue_num: _,
queue_size: _,
} => {
warn!(sl!(), "A DAN device whose type is \"vhost-user\" and socket path is {} is ignored.", path);
continue;
}
queue_num,
queue_size,
} => Arc::new(
VhostUserEndpoint::new(
dev_mgr,
&name,
&device.guest_mac,
path,
*queue_num,
*queue_size,
)
.await
.with_context(|| format!("create a vhost user endpoint, path: {}", path))?,
),
Device::HostTap {
tap_name,
queue_num,
@@ -95,7 +103,6 @@ impl DanInner {
} => Arc::new(
TapEndpoint::new(
&handle,
idx as u32,
&name,
tap_name,
&device.guest_mac,
@@ -104,7 +111,7 @@ impl DanInner {
dev_mgr,
)
.await
.with_context(|| format!("New a {} tap endpoint", tap_name))?,
.with_context(|| format!("create a {} tap endpoint", tap_name))?,
),
};

7
src/runtime-rs/crates/resource/src/network/endpoint/endpoint_persist.rs
View File

@@ -44,6 +44,12 @@ pub struct TapEndpointState {
pub if_name: String,
}
#[derive(Serialize, Deserialize, Clone, Default)]
pub struct VhostUserEndpointState {
pub if_name: String,
pub socket_path: String,
}
#[derive(Serialize, Deserialize, Clone, Default)]
pub struct EndpointState {
pub physical_endpoint: Option<PhysicalEndpointState>,
@@ -52,5 +58,6 @@ pub struct EndpointState {
pub macvlan_endpoint: Option<MacvlanEndpointState>,
pub vlan_endpoint: Option<VlanEndpointState>,
pub tap_endpoint: Option<TapEndpointState>,
pub vhost_user_endpoint: Option<VhostUserEndpointState>,
// TODO : other endpoint
}

2
src/runtime-rs/crates/resource/src/network/endpoint/mod.rs
View File

@@ -18,6 +18,8 @@ pub mod endpoint_persist;
mod endpoints_test;
mod tap_endpoint;
pub use tap_endpoint::TapEndpoint;
mod vhost_user_endpoint;
pub use vhost_user_endpoint::VhostUserEndpoint;
use anyhow::Result;
use async_trait::async_trait;

5
src/runtime-rs/crates/resource/src/network/endpoint/tap_endpoint.rs
View File

@@ -21,9 +21,6 @@ use crate::network::{utils, EndpointState};
/// TapEndpoint is used to attach to the hypervisor directly
#[derive(Debug)]
pub struct TapEndpoint {
// Index
#[allow(dead_code)]
index: u32,
// Name of virt interface
name: String,
// Hardware address of virt interface
@@ -42,7 +39,6 @@ impl TapEndpoint {
#[allow(clippy::too_many_arguments)]
pub async fn new(
handle: &rtnetlink::Handle,
index: u32,
name: &str,
tap_name: &str,
guest_mac: &str,
@@ -57,7 +53,6 @@ impl TapEndpoint {
utils::get_mac_addr(&tap_link.attrs().hardware_addr).context("Get mac addr of tap")?;
Ok(TapEndpoint {
index,
name: name.to_owned(),
guest_mac: guest_mac.to_owned(),
tap_iface: NetworkInterface {

109
src/runtime-rs/crates/resource/src/network/endpoint/vhost_user_endpoint.rs Normal file
View File

@@ -0,0 +1,109 @@
// Copyright (C) 2019-2023 Ant Group. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
use std::path::Path;
use std::sync::Arc;
use anyhow::{anyhow, Context, Result};
use async_trait::async_trait;
use hypervisor::device::device_manager::{do_handle_device, DeviceManager};
use hypervisor::device::{DeviceConfig, DeviceType};
use hypervisor::{Hypervisor, VhostUserConfig, VhostUserNetDevice, VhostUserType};
use tokio::sync::RwLock;
use super::endpoint_persist::VhostUserEndpointState;
use super::Endpoint;
use crate::network::EndpointState;
/// VhostUserEndpoint uses vhost-user-net device, which supports DPDK, etc.
#[derive(Debug)]
pub struct VhostUserEndpoint {
// Name of virt interface
name: String,
// Hardware address of virt interface
guest_mac: String,
// Vhost-user-net device's socket path
socket_path: String,
// Device manager
dev_mgr: Arc<RwLock<DeviceManager>>,
// Virtio queue num
queue_num: usize,
// Virtio queue size
queue_size: usize,
}
impl VhostUserEndpoint {
pub async fn new(
dev_mgr: &Arc<RwLock<DeviceManager>>,
name: &str,
guest_mac: &str,
socket_path: &str,
queue_num: usize,
queue_size: usize,
) -> Result<Self> {
let sk_path = Path::new(socket_path);
if sk_path.exists() {
return Err(anyhow!("vhost-user-net socket path {} exists", socket_path));
}
Ok(VhostUserEndpoint {
name: name.to_string(),
guest_mac: guest_mac.to_string(),
socket_path: socket_path.to_string(),
dev_mgr: dev_mgr.clone(),
queue_num,
queue_size,
})
}
fn get_network_config(&self) -> VhostUserConfig {
VhostUserConfig {
socket_path: self.socket_path.clone(),
mac_address: self.guest_mac.clone(),
device_type: VhostUserType::Net,
queue_size: self.queue_size as u32,
num_queues: self.queue_num,
..Default::default()
}
}
}
#[async_trait]
impl Endpoint for VhostUserEndpoint {
async fn name(&self) -> String {
self.name.clone()
}
async fn hardware_addr(&self) -> String {
self.guest_mac.clone()
}
async fn attach(&self) -> Result<()> {
let config = self.get_network_config();
do_handle_device(&self.dev_mgr, &DeviceConfig::VhostUserNetworkCfg(config))
.await
.context("handle device")?;
Ok(())
}
async fn detach(&self, h: &dyn Hypervisor) -> Result<()> {
let config = self.get_network_config();
h.remove_device(DeviceType::VhostUserNetwork(VhostUserNetDevice {
config,
..Default::default()
}))
.await
.context("remove device")?;
Ok(())
}
async fn save(&self) -> Option<EndpointState> {
Some(EndpointState {
vhost_user_endpoint: Some(VhostUserEndpointState {
if_name: self.name.clone(),
socket_path: self.socket_path.clone(),
}),
..Default::default()
})
}
}

2
src/runtime-rs/crates/resource/src/volume/direct_volumes/spdk_volume.rs
View File

@@ -75,7 +75,7 @@ impl SPDKVolume {
let vhu_blk_config = &mut VhostUserConfig {
socket_path: device,
device_type: VhostUserType::Blk("vhost-user-blk-pci".to_owned()),
device_type: VhostUserType::Blk,
driver_option: block_driver,
..Default::default()
};