runtime-rs: add functionalities support for macvlan and vlan endpoints

Add macvlan and vlan support to runtime-rs code and corresponding unit
tests.

Fixes: #4701
Signed-off-by: Ji-Xinyou <jerryji0414@outlook.com>

src/libs/Cargo.lock

@@ -94,7 +94,7 @@ checksum = "1b827f9d9f6c2fff719d25f5d44cbc8d2ef6df1ef00d055c5c14d5dc25529579"
 dependencies = [
  "libc",
  "log",
- "nix",
+ "nix 0.23.1",
  "regex",
 ]
 
@@ -376,7 +376,7 @@ dependencies = [
  "kata-types",
  "lazy_static",
  "libc",
- "nix",
+ "nix 0.24.2",
  "num_cpus",
  "oci",
  "once_cell",
@@ -508,6 +508,18 @@ dependencies = [
  "memoffset",
 ]
 
+[[package]]
+name = "nix"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "libc",
+ "memoffset",
+]
+
 [[package]]
 name = "ntapi"
 version = "0.3.7"
@@ -1108,7 +1120,7 @@ dependencies = [
  "futures",
  "libc",
  "log",
- "nix",
+ "nix 0.23.1",
  "protobuf",
  "protobuf-codegen-pure",
  "thiserror",
@@ -1162,7 +1174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e32675ee2b3ce5df274c0ab52d19b28789632406277ca26bffee79a8e27dc133"
 dependencies = [
  "libc",
- "nix",
+ "nix 0.23.1",
 ]
 
 [[package]]

src/runtime-rs/crates/resource/src/network/endpoint/endpoints_test.rs

@@ -7,21 +7,267 @@
 #[cfg(test)]
 mod tests {
     use anyhow::Context;
+    use netlink_packet_route::MACVLAN_MODE_PRIVATE;
     use scopeguard::defer;
 
     use std::sync::Arc;
 
     use crate::network::{
-        endpoint::IPVlanEndpoint,
+        endpoint::{IPVlanEndpoint, MacVlanEndpoint, VlanEndpoint},
         network_model::{
+            self,
             tc_filter_model::{fetch_index, TcFilterModel},
-            NetworkModelType,
+            NetworkModelType, TC_FILTER_NET_MODEL_STR,
         },
         network_pair::{NetworkInterface, NetworkPair, TapInterface},
     };
 
+    // this unit test tests the integrity of MacVlanEndpoint::new()
+    #[actix_rt::test]
+    async fn test_vlan_construction() {
+        let idx = 8193;
+        let mac_addr = String::from("02:78:CA:FE:00:04");
+        let manual_vlan_iface_name = format!("eth{}", idx);
+        let tap_iface_name = format!("tap{}_kata", idx); // create by NetworkPair::new()
+        let dummy_name = format!("dummy{}", idx);
+        let vlanid = 123;
+
+        if let Ok((conn, handle, _)) =
+            rtnetlink::new_connection().context("failed to create netlink connection")
+        {
+            let thread_handler = tokio::spawn(conn);
+            defer!({
+                thread_handler.abort();
+            });
+
+            if let Ok(()) = handle
+                .link()
+                .add()
+                .dummy(dummy_name.clone())
+                .execute()
+                .await
+                .context("failed to create dummy link")
+            {
+                let dummy_index = fetch_index(&handle, dummy_name.clone().as_str())
+                    .await
+                    .expect("failed to get the index of dummy link");
+
+                // since IPVlanEndpoint::new() needs an EXISTING virt_iface (which is created
+                // by containerd normally), we have to manually create a virt_iface.
+                if let Ok(()) = handle
+                    .link()
+                    .add()
+                    .vlan(manual_vlan_iface_name.clone(), dummy_index, vlanid)
+                    .execute()
+                    .await
+                    .context("failed to create manual veth pair")
+                {
+                    if let Ok(mut result) = VlanEndpoint::new(&handle, "", idx, 5)
+                        .await
+                        .context("failed to create new ipvlan endpoint")
+                    {
+                        let manual = VlanEndpoint {
+                            net_pair: NetworkPair {
+                                tap: TapInterface {
+                                    id: String::from("uniqueTestID_kata"),
+                                    name: format!("br{}_kata", idx),
+                                    tap_iface: NetworkInterface {
+                                        name: tap_iface_name.clone(),
+                                        ..Default::default()
+                                    },
+                                },
+                                virt_iface: NetworkInterface {
+                                    name: manual_vlan_iface_name.clone(),
+                                    hard_addr: mac_addr.clone(),
+                                    ..Default::default()
+                                },
+                                model: Arc::new(TcFilterModel::new().unwrap()), // impossible to panic
+                                network_qos: false,
+                            },
+                        };
+
+                        result.net_pair.tap.id = String::from("uniqueTestID_kata");
+                        result.net_pair.tap.tap_iface.hard_addr = String::from("");
+                        result.net_pair.virt_iface.hard_addr = mac_addr.clone();
+
+                        // check the integrity by compare all variables
+                        assert_eq!(manual.net_pair.tap.id, result.net_pair.tap.id);
+                        assert_eq!(manual.net_pair.tap.name, result.net_pair.tap.name);
+                        assert_eq!(
+                            manual.net_pair.tap.tap_iface.name,
+                            result.net_pair.tap.tap_iface.name
+                        );
+                        assert_eq!(
+                            manual.net_pair.tap.tap_iface.hard_addr,
+                            result.net_pair.tap.tap_iface.hard_addr
+                        );
+                        assert_eq!(
+                            manual.net_pair.tap.tap_iface.addrs,
+                            result.net_pair.tap.tap_iface.addrs
+                        );
+                        assert_eq!(
+                            manual.net_pair.virt_iface.name,
+                            result.net_pair.virt_iface.name
+                        );
+                        assert_eq!(
+                            manual.net_pair.virt_iface.hard_addr,
+                            result.net_pair.virt_iface.hard_addr
+                        );
+                        // using match branch to avoid deriving PartialEq trait
+                        match manual.net_pair.model.model_type() {
+                            NetworkModelType::TcFilter => {} // ok
+                            _ => unreachable!(),
+                        }
+                        match result.net_pair.model.model_type() {
+                            NetworkModelType::TcFilter => {}
+                            _ => unreachable!(),
+                        }
+                        assert_eq!(manual.net_pair.network_qos, result.net_pair.network_qos);
+                    }
+                    let link_index = fetch_index(&handle, manual_vlan_iface_name.as_str())
+                        .await
+                        .expect("failed to fetch index");
+                    assert!(handle.link().del(link_index).execute().await.is_ok());
+                    let link_index = fetch_index(&handle, tap_iface_name.as_str())
+                        .await
+                        .expect("failed to fetch index");
+                    assert!(handle.link().del(link_index).execute().await.is_ok());
+                    assert!(handle.link().del(dummy_index).execute().await.is_ok());
+                }
+            }
+        }
+    }
+
+    // this unit test tests the integrity of VlanEndpoint::new()
+    #[actix_rt::test]
+    async fn test_macvlan_construction() {
+        let idx = 8194;
+        let mac_addr = String::from("02:25:CA:FE:00:04");
+        let manual_macvlan_iface_name = format!("eth{}", idx);
+        let tap_iface_name = format!("tap{}_kata", idx); // create by NetworkPair::new()
+        let model_str = TC_FILTER_NET_MODEL_STR;
+        let dummy_name = format!("dummy{}", idx);
+
+        if let Ok((conn, handle, _)) =
+            rtnetlink::new_connection().context("failed to create netlink connection")
+        {
+            let thread_handler = tokio::spawn(conn);
+            defer!({
+                thread_handler.abort();
+            });
+
+            if let Ok(()) = handle
+                .link()
+                .add()
+                .dummy(dummy_name.clone())
+                .execute()
+                .await
+                .context("failed to create dummy link")
+            {
+                let dummy_index = fetch_index(&handle, dummy_name.clone().as_str())
+                    .await
+                    .expect("failed to get the index of dummy link");
+
+                // the mode here does not matter, could be any of available modes
+                if let Ok(()) = handle
+                    .link()
+                    .add()
+                    .macvlan(
+                        manual_macvlan_iface_name.clone(),
+                        dummy_index,
+                        MACVLAN_MODE_PRIVATE,
+                    )
+                    .execute()
+                    .await
+                    .context("failed to create manual macvlan pair")
+                {
+                    // model here does not matter, could be any of supported models
+                    if let Ok(mut result) = MacVlanEndpoint::new(
+                        &handle,
+                        manual_macvlan_iface_name.clone().as_str(),
+                        idx,
+                        model_str,
+                        5,
+                    )
+                    .await
+                    .context("failed to create new macvlan endpoint")
+                    {
+                        let manual = MacVlanEndpoint {
+                            net_pair: NetworkPair {
+                                tap: TapInterface {
+                                    id: String::from("uniqueTestID_kata"),
+                                    name: format!("br{}_kata", idx),
+                                    tap_iface: NetworkInterface {
+                                        name: tap_iface_name.clone(),
+                                        ..Default::default()
+                                    },
+                                },
+                                virt_iface: NetworkInterface {
+                                    name: manual_macvlan_iface_name.clone(),
+                                    hard_addr: mac_addr.clone(),
+                                    ..Default::default()
+                                },
+                                model: network_model::new(model_str)
+                                    .expect("failed to create new network model"),
+                                network_qos: false,
+                            },
+                        };
+
+                        result.net_pair.tap.id = String::from("uniqueTestID_kata");
+                        result.net_pair.tap.tap_iface.hard_addr = String::from("");
+                        result.net_pair.virt_iface.hard_addr = mac_addr.clone();
+
+                        // check the integrity by compare all variables
+                        assert_eq!(manual.net_pair.tap.id, result.net_pair.tap.id);
+                        assert_eq!(manual.net_pair.tap.name, result.net_pair.tap.name);
+                        assert_eq!(
+                            manual.net_pair.tap.tap_iface.name,
+                            result.net_pair.tap.tap_iface.name
+                        );
+                        assert_eq!(
+                            manual.net_pair.tap.tap_iface.hard_addr,
+                            result.net_pair.tap.tap_iface.hard_addr
+                        );
+                        assert_eq!(
+                            manual.net_pair.tap.tap_iface.addrs,
+                            result.net_pair.tap.tap_iface.addrs
+                        );
+                        assert_eq!(
+                            manual.net_pair.virt_iface.name,
+                            result.net_pair.virt_iface.name
+                        );
+                        assert_eq!(
+                            manual.net_pair.virt_iface.hard_addr,
+                            result.net_pair.virt_iface.hard_addr
+                        );
+                        // using match branch to avoid deriving PartialEq trait
+                        // TcFilter model is hard-coded "model_str" variable
+                        match manual.net_pair.model.model_type() {
+                            NetworkModelType::TcFilter => {} // ok
+                            _ => unreachable!(),
+                        }
+                        match result.net_pair.model.model_type() {
+                            NetworkModelType::TcFilter => {}
+                            _ => unreachable!(),
+                        }
+                        assert_eq!(manual.net_pair.network_qos, result.net_pair.network_qos);
+                    }
+                    // delete the manually created links
+                    let link_index = fetch_index(&handle, manual_macvlan_iface_name.as_str())
+                        .await
+                        .expect("failed to fetch index");
+                    assert!(handle.link().del(link_index).execute().await.is_ok());
+                    let link_index = fetch_index(&handle, tap_iface_name.as_str())
+                        .await
+                        .expect("failed to fetch index");
+                    assert!(handle.link().del(link_index).execute().await.is_ok());
+                    assert!(handle.link().del(dummy_index).execute().await.is_ok());
+                }
+            }
+        }
+    }
+
     // this unit test tests the integrity of IPVlanEndpoint::new()
-    // by comparing the manual constructed object with object constructed by new()
     #[actix_rt::test]
     async fn test_ipvlan_construction() {
         let idx = 8192;
@@ -30,7 +276,7 @@ mod tests {
         let tap_iface_name = format!("tap{}_kata", idx); // create by kata
 
         if let Ok((conn, handle, _)) =
-            rtnetlink::new_connection().context("failed to create new netlink connection")
+            rtnetlink::new_connection().context("failed to create netlink connection")
         {
             let thread_handler = tokio::spawn(conn);
             defer!({
@@ -45,11 +291,11 @@ mod tests {
                 .veth("foo".to_string(), manual_virt_iface_name.clone())
                 .execute()
                 .await
-                .context("failed to create virt_iface")
+                .context("failed to create manual veth pair")
             {
                 if let Ok(mut result) = IPVlanEndpoint::new(&handle, "", idx, 5)
                     .await
-                    .context("failed to create new IPVlan Endpoint")
+                    .context("failed to create new ipvlan endpoint")
                 {
                     let manual = IPVlanEndpoint {
                         net_pair: NetworkPair {
@@ -98,10 +344,6 @@ mod tests {
                         manual.net_pair.virt_iface.hard_addr,
                         result.net_pair.virt_iface.hard_addr
                     );
-                    assert_eq!(
-                        manual.net_pair.virt_iface.addrs,
-                        result.net_pair.virt_iface.addrs
-                    );
                     // using match branch to avoid deriving PartialEq trait
                     match manual.net_pair.model.model_type() {
                         NetworkModelType::TcFilter => {} // ok
@@ -113,13 +355,14 @@ mod tests {
                     }
                     assert_eq!(manual.net_pair.network_qos, result.net_pair.network_qos);
                 }
-                if let Ok(link_index) = fetch_index(&handle, manual_virt_iface_name.as_str()).await
+                let link_index = fetch_index(&handle, manual_virt_iface_name.as_str())
-                {
+                    .await
-                    assert!(handle.link().del(link_index).execute().await.is_ok())
+                    .expect("failed to fetch index");
-                }
+                assert!(handle.link().del(link_index).execute().await.is_ok());
-                if let Ok(link_index) = fetch_index(&handle, tap_iface_name.as_str()).await {
+                let link_index = fetch_index(&handle, tap_iface_name.as_str())
-                    assert!(handle.link().del(link_index).execute().await.is_ok())
+                    .await
-                }
+                    .expect("failed to fetch index");
+                assert!(handle.link().del(link_index).execute().await.is_ok());
             }
         }
     }

src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs

@@ -0,0 +1,84 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::io::{self, Error};
+
+use anyhow::{Context, Result};
+use async_trait::async_trait;
+use hypervisor::{device::NetworkConfig, Device, Hypervisor};
+
+use super::Endpoint;
+use crate::network::{utils, NetworkPair};
+
+#[derive(Debug)]
+pub struct MacVlanEndpoint {
+    pub(crate) net_pair: NetworkPair,
+}
+
+impl MacVlanEndpoint {
+    pub async fn new(
+        handle: &rtnetlink::Handle,
+        name: &str,
+        idx: u32,
+        model: &str,
+        queues: usize,
+    ) -> Result<Self> {
+        let net_pair = NetworkPair::new(handle, idx, name, model, queues)
+            .await
+            .context("error creating new networkInterfacePair")?;
+        Ok(MacVlanEndpoint { net_pair })
+    }
+
+    fn get_network_config(&self) -> Result<NetworkConfig> {
+        let iface = &self.net_pair.tap.tap_iface;
+        let guest_mac = utils::parse_mac(&iface.hard_addr).ok_or_else(|| {
+            Error::new(
+                io::ErrorKind::InvalidData,
+                format!("hard_addr {}", &iface.hard_addr),
+            )
+        })?;
+        Ok(NetworkConfig {
+            id: self.net_pair.virt_iface.name.clone(),
+            host_dev_name: iface.name.clone(),
+            guest_mac: Some(guest_mac),
+        })
+    }
+}
+
+#[async_trait]
+impl Endpoint for MacVlanEndpoint {
+    async fn name(&self) -> String {
+        self.net_pair.virt_iface.name.clone()
+    }
+
+    async fn hardware_addr(&self) -> String {
+        self.net_pair.tap.tap_iface.hard_addr.clone()
+    }
+
+    async fn attach(&self, h: &dyn Hypervisor) -> Result<()> {
+        self.net_pair
+            .add_network_model()
+            .await
+            .context("add network model")?;
+        let config = self.get_network_config().context("get network config")?;
+        h.add_device(Device::Network(config))
+            .await
+            .context("Error add device")?;
+        Ok(())
+    }
+
+    async fn detach(&self, h: &dyn Hypervisor) -> Result<()> {
+        self.net_pair
+            .del_network_model()
+            .await
+            .context("del network model")?;
+        let config = self.get_network_config().context("get network config")?;
+        h.remove_device(Device::Network(config))
+            .await
+            .context("remove device")?;
+        Ok(())
+    }
+}

src/runtime-rs/crates/resource/src/network/endpoint/mod.rs

@@ -10,6 +10,10 @@ mod veth_endpoint;
 pub use veth_endpoint::VethEndpoint;
 mod ipvlan_endpoint;
 pub use ipvlan_endpoint::IPVlanEndpoint;
+mod vlan_endpoint;
+pub use vlan_endpoint::VlanEndpoint;
+mod macvlan_endpoint;
+pub use macvlan_endpoint::MacVlanEndpoint;
 mod endpoints_test;
 
 use anyhow::Result;

src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs

@@ -0,0 +1,88 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::io::{self, Error};
+
+use anyhow::{Context, Result};
+use async_trait::async_trait;
+
+use super::Endpoint;
+use crate::network::network_model::TC_FILTER_NET_MODEL_STR;
+use crate::network::{utils, NetworkPair};
+use hypervisor::{device::NetworkConfig, Device, Hypervisor};
+
+#[derive(Debug)]
+pub struct VlanEndpoint {
+    pub(crate) net_pair: NetworkPair,
+}
+
+impl VlanEndpoint {
+    pub async fn new(
+        handle: &rtnetlink::Handle,
+        name: &str,
+        idx: u32,
+        queues: usize,
+    ) -> Result<Self> {
+        let net_pair = NetworkPair::new(handle, idx, name, TC_FILTER_NET_MODEL_STR, queues)
+            .await
+            .context("error creating networkInterfacePair")?;
+        Ok(VlanEndpoint { net_pair })
+    }
+
+    fn get_network_config(&self) -> Result<NetworkConfig> {
+        let iface = &self.net_pair.tap.tap_iface;
+        let guest_mac = utils::parse_mac(&iface.hard_addr).ok_or_else(|| {
+            Error::new(
+                io::ErrorKind::InvalidData,
+                format!("hard_addr {}", &iface.hard_addr),
+            )
+        })?;
+        Ok(NetworkConfig {
+            id: self.net_pair.virt_iface.name.clone(),
+            host_dev_name: iface.name.clone(),
+            guest_mac: Some(guest_mac),
+        })
+    }
+}
+
+#[async_trait]
+impl Endpoint for VlanEndpoint {
+    async fn name(&self) -> String {
+        self.net_pair.virt_iface.name.clone()
+    }
+
+    async fn hardware_addr(&self) -> String {
+        self.net_pair.tap.tap_iface.hard_addr.clone()
+    }
+
+    async fn attach(&self, h: &dyn Hypervisor) -> Result<()> {
+        self.net_pair
+            .add_network_model()
+            .await
+            .context("error adding network model")?;
+        let config = self.get_network_config().context("get network config")?;
+        h.add_device(Device::Network(config))
+            .await
+            .context("error adding device by hypervisor")?;
+
+        Ok(())
+    }
+
+    async fn detach(&self, h: &dyn Hypervisor) -> Result<()> {
+        self.net_pair
+            .del_network_model()
+            .await
+            .context("error deleting network model")?;
+        let config = self
+            .get_network_config()
+            .context("error getting network config")?;
+        h.remove_device(Device::Network(config))
+            .await
+            .context("error removing device by hypervisor")?;
+
+        Ok(())
+    }
+}

src/runtime-rs/crates/resource/src/network/network_with_netns.rs

@@ -17,7 +17,9 @@ use scopeguard::defer;
 use tokio::sync::RwLock;
 
 use super::{
-    endpoint::{Endpoint, IPVlanEndpoint, PhysicalEndpoint, VethEndpoint},
+    endpoint::{
+        Endpoint, IPVlanEndpoint, MacVlanEndpoint, PhysicalEndpoint, VethEndpoint, VlanEndpoint,
+    },
     network_entity::NetworkEntity,
     network_info::network_info_from_link::NetworkInfoFromLink,
     utils::{link, netns},
@@ -186,12 +188,30 @@ async fn create_endpoint(
                 .context("veth endpoint")?;
                 Arc::new(ret)
             }
+            "vlan" => {
+                let ret = VlanEndpoint::new(handle, &attrs.name, idx, config.queues)
+                    .await
+                    .context("vlan endpoint")?;
+                Arc::new(ret)
+            }
             "ipvlan" => {
                 let ret = IPVlanEndpoint::new(handle, &attrs.name, idx, config.queues)
                     .await
                     .context("ipvlan endpoint")?;
                 Arc::new(ret)
             }
+            "macvlan" => {
+                let ret = MacVlanEndpoint::new(
+                    handle,
+                    &attrs.name,
+                    idx,
+                    &config.network_model,
+                    config.queues,
+                )
+                .await
+                .context("macvlan endpoint")?;
+                Arc::new(ret)
+            }
             _ => return Err(anyhow!("unsupported link type: {}", link_type)),
         }
     };

src/runtime-rs/crates/resource/src/network/utils/link/manager.rs

@@ -103,6 +103,11 @@ fn link_info(mut infos: Vec<Info>) -> Box<dyn Link> {
                         link = Some(Box::new(IpVlan::default()));
                     }
                 }
+                InfoKind::MacVlan => {
+                    if link.is_none() {
+                        link = Some(Box::new(MacVlan::default()));
+                    }
+                }
                 InfoKind::Vlan => {
                     if link.is_none() {
                         link = Some(Box::new(Vlan::default()));
@@ -129,6 +134,9 @@ fn link_info(mut infos: Vec<Info>) -> Box<dyn Link> {
                 InfoData::IpVlan(_) => {
                     link = Some(Box::new(IpVlan::default()));
                 }
+                InfoData::MacVlan(_) => {
+                    link = Some(Box::new(MacVlan::default()));
+                }
                 InfoData::Vlan(_) => {
                     link = Some(Box::new(Vlan::default()));
                 }
@@ -247,6 +255,26 @@ impl Link for IpVlan {
     }
 }
 
+#[derive(Debug, PartialEq, Eq, Clone, Default)]
+pub struct MacVlan {
+    attrs: Option<LinkAttrs>,
+
+    /// on create only
+    pub peer_name: String,
+}
+
+impl Link for MacVlan {
+    fn attrs(&self) -> &LinkAttrs {
+        self.attrs.as_ref().unwrap()
+    }
+    fn set_attrs(&mut self, attr: LinkAttrs) {
+        self.attrs = Some(attr)
+    }
+    fn r#type(&self) -> &'static str {
+        "macvlan"
+    }
+}
+
 #[derive(Debug, PartialEq, Eq, Clone, Default)]
 pub struct Vlan {
     attrs: Option<LinkAttrs>,