diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index e5d9cb160e..b82c108c44 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -94,7 +94,7 @@ checksum = "1b827f9d9f6c2fff719d25f5d44cbc8d2ef6df1ef00d055c5c14d5dc25529579" dependencies = [ "libc", "log", - "nix", + "nix 0.23.1", "regex", ] @@ -376,7 +376,7 @@ dependencies = [ "kata-types", "lazy_static", "libc", - "nix", + "nix 0.24.2", "num_cpus", "oci", "once_cell", @@ -508,6 +508,18 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nix" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" +dependencies = [ + "bitflags", + "cfg-if", + "libc", + "memoffset", +] + [[package]] name = "ntapi" version = "0.3.7" @@ -1108,7 +1120,7 @@ dependencies = [ "futures", "libc", "log", - "nix", + "nix 0.23.1", "protobuf", "protobuf-codegen-pure", "thiserror", @@ -1162,7 +1174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e32675ee2b3ce5df274c0ab52d19b28789632406277ca26bffee79a8e27dc133" dependencies = [ "libc", - "nix", + "nix 0.23.1", ] [[package]] diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/endpoints_test.rs b/src/runtime-rs/crates/resource/src/network/endpoint/endpoints_test.rs index aeb7e0e3e9..90623d59cb 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/endpoints_test.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/endpoints_test.rs @@ -7,21 +7,267 @@ #[cfg(test)] mod tests { use anyhow::Context; + use netlink_packet_route::MACVLAN_MODE_PRIVATE; use scopeguard::defer; use std::sync::Arc; use crate::network::{ - endpoint::IPVlanEndpoint, + endpoint::{IPVlanEndpoint, MacVlanEndpoint, VlanEndpoint}, network_model::{ + self, tc_filter_model::{fetch_index, TcFilterModel}, - NetworkModelType, + NetworkModelType, TC_FILTER_NET_MODEL_STR, }, network_pair::{NetworkInterface, NetworkPair, TapInterface}, }; + // this unit test tests the integrity of MacVlanEndpoint::new() + #[actix_rt::test] + async fn test_vlan_construction() { + let idx = 8193; + let mac_addr = String::from("02:78:CA:FE:00:04"); + let manual_vlan_iface_name = format!("eth{}", idx); + let tap_iface_name = format!("tap{}_kata", idx); // create by NetworkPair::new() + let dummy_name = format!("dummy{}", idx); + let vlanid = 123; + + if let Ok((conn, handle, _)) = + rtnetlink::new_connection().context("failed to create netlink connection") + { + let thread_handler = tokio::spawn(conn); + defer!({ + thread_handler.abort(); + }); + + if let Ok(()) = handle + .link() + .add() + .dummy(dummy_name.clone()) + .execute() + .await + .context("failed to create dummy link") + { + let dummy_index = fetch_index(&handle, dummy_name.clone().as_str()) + .await + .expect("failed to get the index of dummy link"); + + // since IPVlanEndpoint::new() needs an EXISTING virt_iface (which is created + // by containerd normally), we have to manually create a virt_iface. + if let Ok(()) = handle + .link() + .add() + .vlan(manual_vlan_iface_name.clone(), dummy_index, vlanid) + .execute() + .await + .context("failed to create manual veth pair") + { + if let Ok(mut result) = VlanEndpoint::new(&handle, "", idx, 5) + .await + .context("failed to create new ipvlan endpoint") + { + let manual = VlanEndpoint { + net_pair: NetworkPair { + tap: TapInterface { + id: String::from("uniqueTestID_kata"), + name: format!("br{}_kata", idx), + tap_iface: NetworkInterface { + name: tap_iface_name.clone(), + ..Default::default() + }, + }, + virt_iface: NetworkInterface { + name: manual_vlan_iface_name.clone(), + hard_addr: mac_addr.clone(), + ..Default::default() + }, + model: Arc::new(TcFilterModel::new().unwrap()), // impossible to panic + network_qos: false, + }, + }; + + result.net_pair.tap.id = String::from("uniqueTestID_kata"); + result.net_pair.tap.tap_iface.hard_addr = String::from(""); + result.net_pair.virt_iface.hard_addr = mac_addr.clone(); + + // check the integrity by compare all variables + assert_eq!(manual.net_pair.tap.id, result.net_pair.tap.id); + assert_eq!(manual.net_pair.tap.name, result.net_pair.tap.name); + assert_eq!( + manual.net_pair.tap.tap_iface.name, + result.net_pair.tap.tap_iface.name + ); + assert_eq!( + manual.net_pair.tap.tap_iface.hard_addr, + result.net_pair.tap.tap_iface.hard_addr + ); + assert_eq!( + manual.net_pair.tap.tap_iface.addrs, + result.net_pair.tap.tap_iface.addrs + ); + assert_eq!( + manual.net_pair.virt_iface.name, + result.net_pair.virt_iface.name + ); + assert_eq!( + manual.net_pair.virt_iface.hard_addr, + result.net_pair.virt_iface.hard_addr + ); + // using match branch to avoid deriving PartialEq trait + match manual.net_pair.model.model_type() { + NetworkModelType::TcFilter => {} // ok + _ => unreachable!(), + } + match result.net_pair.model.model_type() { + NetworkModelType::TcFilter => {} + _ => unreachable!(), + } + assert_eq!(manual.net_pair.network_qos, result.net_pair.network_qos); + } + let link_index = fetch_index(&handle, manual_vlan_iface_name.as_str()) + .await + .expect("failed to fetch index"); + assert!(handle.link().del(link_index).execute().await.is_ok()); + let link_index = fetch_index(&handle, tap_iface_name.as_str()) + .await + .expect("failed to fetch index"); + assert!(handle.link().del(link_index).execute().await.is_ok()); + assert!(handle.link().del(dummy_index).execute().await.is_ok()); + } + } + } + } + + // this unit test tests the integrity of VlanEndpoint::new() + #[actix_rt::test] + async fn test_macvlan_construction() { + let idx = 8194; + let mac_addr = String::from("02:25:CA:FE:00:04"); + let manual_macvlan_iface_name = format!("eth{}", idx); + let tap_iface_name = format!("tap{}_kata", idx); // create by NetworkPair::new() + let model_str = TC_FILTER_NET_MODEL_STR; + let dummy_name = format!("dummy{}", idx); + + if let Ok((conn, handle, _)) = + rtnetlink::new_connection().context("failed to create netlink connection") + { + let thread_handler = tokio::spawn(conn); + defer!({ + thread_handler.abort(); + }); + + if let Ok(()) = handle + .link() + .add() + .dummy(dummy_name.clone()) + .execute() + .await + .context("failed to create dummy link") + { + let dummy_index = fetch_index(&handle, dummy_name.clone().as_str()) + .await + .expect("failed to get the index of dummy link"); + + // the mode here does not matter, could be any of available modes + if let Ok(()) = handle + .link() + .add() + .macvlan( + manual_macvlan_iface_name.clone(), + dummy_index, + MACVLAN_MODE_PRIVATE, + ) + .execute() + .await + .context("failed to create manual macvlan pair") + { + // model here does not matter, could be any of supported models + if let Ok(mut result) = MacVlanEndpoint::new( + &handle, + manual_macvlan_iface_name.clone().as_str(), + idx, + model_str, + 5, + ) + .await + .context("failed to create new macvlan endpoint") + { + let manual = MacVlanEndpoint { + net_pair: NetworkPair { + tap: TapInterface { + id: String::from("uniqueTestID_kata"), + name: format!("br{}_kata", idx), + tap_iface: NetworkInterface { + name: tap_iface_name.clone(), + ..Default::default() + }, + }, + virt_iface: NetworkInterface { + name: manual_macvlan_iface_name.clone(), + hard_addr: mac_addr.clone(), + ..Default::default() + }, + model: network_model::new(model_str) + .expect("failed to create new network model"), + network_qos: false, + }, + }; + + result.net_pair.tap.id = String::from("uniqueTestID_kata"); + result.net_pair.tap.tap_iface.hard_addr = String::from(""); + result.net_pair.virt_iface.hard_addr = mac_addr.clone(); + + // check the integrity by compare all variables + assert_eq!(manual.net_pair.tap.id, result.net_pair.tap.id); + assert_eq!(manual.net_pair.tap.name, result.net_pair.tap.name); + assert_eq!( + manual.net_pair.tap.tap_iface.name, + result.net_pair.tap.tap_iface.name + ); + assert_eq!( + manual.net_pair.tap.tap_iface.hard_addr, + result.net_pair.tap.tap_iface.hard_addr + ); + assert_eq!( + manual.net_pair.tap.tap_iface.addrs, + result.net_pair.tap.tap_iface.addrs + ); + assert_eq!( + manual.net_pair.virt_iface.name, + result.net_pair.virt_iface.name + ); + assert_eq!( + manual.net_pair.virt_iface.hard_addr, + result.net_pair.virt_iface.hard_addr + ); + // using match branch to avoid deriving PartialEq trait + // TcFilter model is hard-coded "model_str" variable + match manual.net_pair.model.model_type() { + NetworkModelType::TcFilter => {} // ok + _ => unreachable!(), + } + match result.net_pair.model.model_type() { + NetworkModelType::TcFilter => {} + _ => unreachable!(), + } + assert_eq!(manual.net_pair.network_qos, result.net_pair.network_qos); + } + // delete the manually created links + let link_index = fetch_index(&handle, manual_macvlan_iface_name.as_str()) + .await + .expect("failed to fetch index"); + assert!(handle.link().del(link_index).execute().await.is_ok()); + let link_index = fetch_index(&handle, tap_iface_name.as_str()) + .await + .expect("failed to fetch index"); + assert!(handle.link().del(link_index).execute().await.is_ok()); + assert!(handle.link().del(dummy_index).execute().await.is_ok()); + } + } + } + } + // this unit test tests the integrity of IPVlanEndpoint::new() - // by comparing the manual constructed object with object constructed by new() #[actix_rt::test] async fn test_ipvlan_construction() { let idx = 8192; @@ -30,7 +276,7 @@ mod tests { let tap_iface_name = format!("tap{}_kata", idx); // create by kata if let Ok((conn, handle, _)) = - rtnetlink::new_connection().context("failed to create new netlink connection") + rtnetlink::new_connection().context("failed to create netlink connection") { let thread_handler = tokio::spawn(conn); defer!({ @@ -45,11 +291,11 @@ mod tests { .veth("foo".to_string(), manual_virt_iface_name.clone()) .execute() .await - .context("failed to create virt_iface") + .context("failed to create manual veth pair") { if let Ok(mut result) = IPVlanEndpoint::new(&handle, "", idx, 5) .await - .context("failed to create new IPVlan Endpoint") + .context("failed to create new ipvlan endpoint") { let manual = IPVlanEndpoint { net_pair: NetworkPair { @@ -98,10 +344,6 @@ mod tests { manual.net_pair.virt_iface.hard_addr, result.net_pair.virt_iface.hard_addr ); - assert_eq!( - manual.net_pair.virt_iface.addrs, - result.net_pair.virt_iface.addrs - ); // using match branch to avoid deriving PartialEq trait match manual.net_pair.model.model_type() { NetworkModelType::TcFilter => {} // ok @@ -113,13 +355,14 @@ mod tests { } assert_eq!(manual.net_pair.network_qos, result.net_pair.network_qos); } - if let Ok(link_index) = fetch_index(&handle, manual_virt_iface_name.as_str()).await - { - assert!(handle.link().del(link_index).execute().await.is_ok()) - } - if let Ok(link_index) = fetch_index(&handle, tap_iface_name.as_str()).await { - assert!(handle.link().del(link_index).execute().await.is_ok()) - } + let link_index = fetch_index(&handle, manual_virt_iface_name.as_str()) + .await + .expect("failed to fetch index"); + assert!(handle.link().del(link_index).execute().await.is_ok()); + let link_index = fetch_index(&handle, tap_iface_name.as_str()) + .await + .expect("failed to fetch index"); + assert!(handle.link().del(link_index).execute().await.is_ok()); } } } diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs new file mode 100644 index 0000000000..21f22345d8 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs @@ -0,0 +1,84 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::io::{self, Error}; + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use hypervisor::{device::NetworkConfig, Device, Hypervisor}; + +use super::Endpoint; +use crate::network::{utils, NetworkPair}; + +#[derive(Debug)] +pub struct MacVlanEndpoint { + pub(crate) net_pair: NetworkPair, +} + +impl MacVlanEndpoint { + pub async fn new( + handle: &rtnetlink::Handle, + name: &str, + idx: u32, + model: &str, + queues: usize, + ) -> Result { + let net_pair = NetworkPair::new(handle, idx, name, model, queues) + .await + .context("error creating new networkInterfacePair")?; + Ok(MacVlanEndpoint { net_pair }) + } + + fn get_network_config(&self) -> Result { + let iface = &self.net_pair.tap.tap_iface; + let guest_mac = utils::parse_mac(&iface.hard_addr).ok_or_else(|| { + Error::new( + io::ErrorKind::InvalidData, + format!("hard_addr {}", &iface.hard_addr), + ) + })?; + Ok(NetworkConfig { + id: self.net_pair.virt_iface.name.clone(), + host_dev_name: iface.name.clone(), + guest_mac: Some(guest_mac), + }) + } +} + +#[async_trait] +impl Endpoint for MacVlanEndpoint { + async fn name(&self) -> String { + self.net_pair.virt_iface.name.clone() + } + + async fn hardware_addr(&self) -> String { + self.net_pair.tap.tap_iface.hard_addr.clone() + } + + async fn attach(&self, h: &dyn Hypervisor) -> Result<()> { + self.net_pair + .add_network_model() + .await + .context("add network model")?; + let config = self.get_network_config().context("get network config")?; + h.add_device(Device::Network(config)) + .await + .context("Error add device")?; + Ok(()) + } + + async fn detach(&self, h: &dyn Hypervisor) -> Result<()> { + self.net_pair + .del_network_model() + .await + .context("del network model")?; + let config = self.get_network_config().context("get network config")?; + h.remove_device(Device::Network(config)) + .await + .context("remove device")?; + Ok(()) + } +} diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs b/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs index b8e4557b53..9e5e841c82 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs @@ -10,6 +10,10 @@ mod veth_endpoint; pub use veth_endpoint::VethEndpoint; mod ipvlan_endpoint; pub use ipvlan_endpoint::IPVlanEndpoint; +mod vlan_endpoint; +pub use vlan_endpoint::VlanEndpoint; +mod macvlan_endpoint; +pub use macvlan_endpoint::MacVlanEndpoint; mod endpoints_test; use anyhow::Result; diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs new file mode 100644 index 0000000000..14626318cf --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs @@ -0,0 +1,88 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::io::{self, Error}; + +use anyhow::{Context, Result}; +use async_trait::async_trait; + +use super::Endpoint; +use crate::network::network_model::TC_FILTER_NET_MODEL_STR; +use crate::network::{utils, NetworkPair}; +use hypervisor::{device::NetworkConfig, Device, Hypervisor}; + +#[derive(Debug)] +pub struct VlanEndpoint { + pub(crate) net_pair: NetworkPair, +} + +impl VlanEndpoint { + pub async fn new( + handle: &rtnetlink::Handle, + name: &str, + idx: u32, + queues: usize, + ) -> Result { + let net_pair = NetworkPair::new(handle, idx, name, TC_FILTER_NET_MODEL_STR, queues) + .await + .context("error creating networkInterfacePair")?; + Ok(VlanEndpoint { net_pair }) + } + + fn get_network_config(&self) -> Result { + let iface = &self.net_pair.tap.tap_iface; + let guest_mac = utils::parse_mac(&iface.hard_addr).ok_or_else(|| { + Error::new( + io::ErrorKind::InvalidData, + format!("hard_addr {}", &iface.hard_addr), + ) + })?; + Ok(NetworkConfig { + id: self.net_pair.virt_iface.name.clone(), + host_dev_name: iface.name.clone(), + guest_mac: Some(guest_mac), + }) + } +} + +#[async_trait] +impl Endpoint for VlanEndpoint { + async fn name(&self) -> String { + self.net_pair.virt_iface.name.clone() + } + + async fn hardware_addr(&self) -> String { + self.net_pair.tap.tap_iface.hard_addr.clone() + } + + async fn attach(&self, h: &dyn Hypervisor) -> Result<()> { + self.net_pair + .add_network_model() + .await + .context("error adding network model")?; + let config = self.get_network_config().context("get network config")?; + h.add_device(Device::Network(config)) + .await + .context("error adding device by hypervisor")?; + + Ok(()) + } + + async fn detach(&self, h: &dyn Hypervisor) -> Result<()> { + self.net_pair + .del_network_model() + .await + .context("error deleting network model")?; + let config = self + .get_network_config() + .context("error getting network config")?; + h.remove_device(Device::Network(config)) + .await + .context("error removing device by hypervisor")?; + + Ok(()) + } +} diff --git a/src/runtime-rs/crates/resource/src/network/network_with_netns.rs b/src/runtime-rs/crates/resource/src/network/network_with_netns.rs index 1d2a1e55f9..d89e8ab7e6 100644 --- a/src/runtime-rs/crates/resource/src/network/network_with_netns.rs +++ b/src/runtime-rs/crates/resource/src/network/network_with_netns.rs @@ -17,7 +17,9 @@ use scopeguard::defer; use tokio::sync::RwLock; use super::{ - endpoint::{Endpoint, IPVlanEndpoint, PhysicalEndpoint, VethEndpoint}, + endpoint::{ + Endpoint, IPVlanEndpoint, MacVlanEndpoint, PhysicalEndpoint, VethEndpoint, VlanEndpoint, + }, network_entity::NetworkEntity, network_info::network_info_from_link::NetworkInfoFromLink, utils::{link, netns}, @@ -186,12 +188,30 @@ async fn create_endpoint( .context("veth endpoint")?; Arc::new(ret) } + "vlan" => { + let ret = VlanEndpoint::new(handle, &attrs.name, idx, config.queues) + .await + .context("vlan endpoint")?; + Arc::new(ret) + } "ipvlan" => { let ret = IPVlanEndpoint::new(handle, &attrs.name, idx, config.queues) .await .context("ipvlan endpoint")?; Arc::new(ret) } + "macvlan" => { + let ret = MacVlanEndpoint::new( + handle, + &attrs.name, + idx, + &config.network_model, + config.queues, + ) + .await + .context("macvlan endpoint")?; + Arc::new(ret) + } _ => return Err(anyhow!("unsupported link type: {}", link_type)), } }; diff --git a/src/runtime-rs/crates/resource/src/network/utils/link/manager.rs b/src/runtime-rs/crates/resource/src/network/utils/link/manager.rs index fd4a8b1c99..efc43bb704 100644 --- a/src/runtime-rs/crates/resource/src/network/utils/link/manager.rs +++ b/src/runtime-rs/crates/resource/src/network/utils/link/manager.rs @@ -103,6 +103,11 @@ fn link_info(mut infos: Vec) -> Box { link = Some(Box::new(IpVlan::default())); } } + InfoKind::MacVlan => { + if link.is_none() { + link = Some(Box::new(MacVlan::default())); + } + } InfoKind::Vlan => { if link.is_none() { link = Some(Box::new(Vlan::default())); @@ -129,6 +134,9 @@ fn link_info(mut infos: Vec) -> Box { InfoData::IpVlan(_) => { link = Some(Box::new(IpVlan::default())); } + InfoData::MacVlan(_) => { + link = Some(Box::new(MacVlan::default())); + } InfoData::Vlan(_) => { link = Some(Box::new(Vlan::default())); } @@ -247,6 +255,26 @@ impl Link for IpVlan { } } +#[derive(Debug, PartialEq, Eq, Clone, Default)] +pub struct MacVlan { + attrs: Option, + + /// on create only + pub peer_name: String, +} + +impl Link for MacVlan { + fn attrs(&self) -> &LinkAttrs { + self.attrs.as_ref().unwrap() + } + fn set_attrs(&mut self, attr: LinkAttrs) { + self.attrs = Some(attr) + } + fn r#type(&self) -> &'static str { + "macvlan" + } +} + #[derive(Debug, PartialEq, Eq, Clone, Default)] pub struct Vlan { attrs: Option,