diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs index f9c1a91a2a..134e54d529 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs @@ -10,6 +10,7 @@ use anyhow::{Context, Result}; use async_trait::async_trait; use crate::device::topology::PCIeTopology; +use crate::device::pci_path::PciPath; use crate::device::{Device, DeviceType}; use crate::Hypervisor as hypervisor; @@ -48,6 +49,8 @@ pub struct NetworkConfig { pub use_generic_irq: Option, /// Allow duplicate mac pub allow_duplicate_mac: bool, + /// Guest PCI path after hot-plug (used by the agent to wait for uevents). + pub pci_path: Option, } #[derive(Clone, Debug, Default)] @@ -76,11 +79,16 @@ impl Device for NetworkDevice { _pcie_topo: &mut Option<&mut PCIeTopology>, h: &dyn hypervisor, ) -> Result<()> { - h.add_device(DeviceType::Network(self.clone())) + let updated = h + .add_device(DeviceType::Network(self.clone())) .await .context("add network device.")?; - return Ok(()); + if let DeviceType::Network(net) = updated { + self.config = net.config; + } + + Ok(()) } async fn detach( diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index 4d1536685b..fc3b6476c5 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -18,7 +18,7 @@ use crate::{ use crate::utils::{ bytes_to_megs, create_dir_all_with_inherit_owner, enter_netns, get_jailer_root, megs_to_bytes, - set_groups, vm_cleanup, + set_groups, uses_native_ccw_bus, vm_cleanup, }; use anyhow::{anyhow, Context, Result}; @@ -1016,14 +1016,29 @@ impl QemuInner { }; match device { - DeviceType::Network(ref network_device) => { + DeviceType::Network(mut network_device) => { let (netdev, virtio_net_device) = get_network_device( &self.config, &network_device.config.host_dev_name, network_device.config.guest_mac.clone().unwrap(), &mut None, )?; - qmp.hotplug_network_device(&netdev, &virtio_net_device)? + qmp.hotplug_network_device(&netdev, &virtio_net_device)?; + + // On s390x the NIC is plugged onto the virtual channel subsystem + // (virtio-net-ccw) and has no PCI address, so the PCI-path lookup + // below does not apply. The agent matches the CCW interface via + // its uevent instead; forcing a PCI-path resolution here would + // fail and abort the whole NIC attach (breaking guest networking). + if !uses_native_ccw_bus() { + let frontend_id = format!("frontend-{}", virtio_net_device.get_netdev_id()); + let pci_path = qmp + .get_device_by_qdev_id(&frontend_id) + .context("get network device pci path")?; + network_device.config.pci_path = Some(pci_path); + } + + return Ok(DeviceType::Network(network_device)); } DeviceType::Block(mut block_device) => { let block_driver = &self.config.blockdev_info.block_device_driver; diff --git a/src/runtime-rs/crates/resource/src/network/dan.rs b/src/runtime-rs/crates/resource/src/network/dan.rs index e27c0f9115..e17a557886 100644 --- a/src/runtime-rs/crates/resource/src/network/dan.rs +++ b/src/runtime-rs/crates/resource/src/network/dan.rs @@ -144,7 +144,12 @@ impl Network for Dan { _netns_guard = NetnsGuard::new(netns).context("New netns guard")?; } for e in inner.entity_list.iter() { - e.endpoint.attach().await.context("Attach")?; + if let Some(device_path) = e.endpoint.attach().await.context("Attach")? { + e.network_info + .set_device_path(device_path) + .await + .context("set device path")?; + } } Ok(()) } diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs index bc075186d1..77c8db93af 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs @@ -9,14 +9,14 @@ use std::sync::Arc; use anyhow::{Context, Result}; use async_trait::async_trait; -use hypervisor::device::device_manager::{do_handle_device, DeviceManager}; +use hypervisor::device::device_manager::DeviceManager; use hypervisor::device::driver::NetworkConfig; -use hypervisor::device::{DeviceConfig, DeviceType}; +use hypervisor::device::DeviceType; use hypervisor::{Hypervisor, NetworkDevice}; use tokio::sync::RwLock; use super::endpoint_persist::{EndpointState, IpVlanEndpointState}; -use super::Endpoint; +use super::{attach_network_device, Endpoint}; use crate::network::{network_model::TC_FILTER_NET_MODEL_STR, utils, NetworkPair}; // IPVlanEndpoint is the endpoint bridged to VM @@ -73,18 +73,16 @@ impl Endpoint for IPVlanEndpoint { self.net_pair.tap.tap_iface.hard_addr.clone() } - async fn attach(&self) -> Result<()> { + async fn attach(&self) -> Result> { self.net_pair .add_network_model() .await .context("error adding network model")?; let config = self.get_network_config().context("get network config")?; - do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config)) + attach_network_device(&self.d, config) .await - .context("do handle network IPVlan endpoint device failed.")?; - - Ok(()) + .context("do handle network IPVlan endpoint device failed.") } async fn detach(&self, h: &dyn Hypervisor) -> Result<()> { diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs index 633db428f0..fe68f91021 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs @@ -9,14 +9,14 @@ use std::sync::Arc; use anyhow::{Context, Result}; use async_trait::async_trait; -use hypervisor::device::device_manager::{do_handle_device, DeviceManager}; +use hypervisor::device::device_manager::DeviceManager; use hypervisor::device::driver::NetworkConfig; -use hypervisor::device::{DeviceConfig, DeviceType}; +use hypervisor::device::DeviceType; use hypervisor::{Hypervisor, NetworkDevice}; use tokio::sync::RwLock; use super::endpoint_persist::{EndpointState, MacvlanEndpointState}; -use super::Endpoint; +use super::{attach_network_device, Endpoint}; use crate::network::{utils, NetworkPair}; #[derive(Debug)] @@ -72,18 +72,16 @@ impl Endpoint for MacVlanEndpoint { self.net_pair.tap.tap_iface.hard_addr.clone() } - async fn attach(&self) -> Result<()> { + async fn attach(&self) -> Result> { self.net_pair .add_network_model() .await .context("add network model")?; let config = self.get_network_config().context("get network config")?; - do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config)) + attach_network_device(&self.d, config) .await - .context("do handle network MacVlan endpoint device failed.")?; - - Ok(()) + .context("do handle network MacVlan endpoint device failed.") } async fn detach(&self, h: &dyn Hypervisor) -> Result<()> { diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs b/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs index ac1e1993aa..32139085cc 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs @@ -23,15 +23,31 @@ pub use vhost_user_endpoint::VhostUserEndpoint; use anyhow::Result; use async_trait::async_trait; +use hypervisor::device::device_manager::{do_handle_device, DeviceManager}; +use hypervisor::device::driver::NetworkConfig; +use hypervisor::device::{DeviceConfig, DeviceType}; use hypervisor::Hypervisor; +use tokio::sync::RwLock; +use std::sync::Arc; use super::EndpointState; +pub(crate) async fn attach_network_device( + d: &Arc>, + config: NetworkConfig, +) -> Result> { + let device_info = do_handle_device(d, &DeviceConfig::NetworkCfg(config)).await?; + match device_info { + DeviceType::Network(net) => Ok(net.config.pci_path.map(|p| p.to_string())), + _ => Ok(None), + } +} + #[async_trait] pub trait Endpoint: std::fmt::Debug + Send + Sync { async fn name(&self) -> String; async fn hardware_addr(&self) -> String; - async fn attach(&self) -> Result<()>; + async fn attach(&self) -> Result>; async fn detach(&self, hypervisor: &dyn Hypervisor) -> Result<()>; async fn save(&self) -> Option; /// Returns the guest PCI path for this endpoint if it is a cold-plugged diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs index e786cb92e8..6f3abf9e09 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs @@ -117,7 +117,7 @@ impl Endpoint for PhysicalEndpoint { self.hard_addr.clone() } - async fn attach(&self) -> Result<()> { + async fn attach(&self) -> Result> { // Push the desired netdev MAC down to the VF as an "admin MAC" via the // PF before we rebind to vfio-pci. Without this the guest mlx5_core // inherits the VF firmware default MAC, which differs from the @@ -181,7 +181,7 @@ impl Endpoint for PhysicalEndpoint { } } - Ok(()) + Ok(None) } // detach for physical endpoint unbinds the physical network interface from vfio-pci diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/tap_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/tap_endpoint.rs index 490eca204f..ec03b38c91 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/tap_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/tap_endpoint.rs @@ -8,13 +8,14 @@ use std::sync::Arc; use anyhow::{Context, Result}; use async_trait::async_trait; -use hypervisor::device::device_manager::{do_handle_device, DeviceManager}; -use hypervisor::device::{DeviceConfig, DeviceType}; -use hypervisor::{Hypervisor, NetworkConfig, NetworkDevice}; +use hypervisor::device::device_manager::DeviceManager; +use hypervisor::device::driver::NetworkConfig; +use hypervisor::device::DeviceType; +use hypervisor::{Hypervisor, NetworkDevice}; use tokio::sync::RwLock; use super::endpoint_persist::TapEndpointState; -use super::Endpoint; +use super::{attach_network_device, Endpoint}; use crate::network::network_pair::{get_link_by_name, NetworkInterface}; use crate::network::{utils, EndpointState}; @@ -89,12 +90,11 @@ impl Endpoint for TapEndpoint { self.guest_mac.clone() } - async fn attach(&self) -> Result<()> { + async fn attach(&self) -> Result> { let config = self.get_network_config().context("Get network config")?; - do_handle_device(&self.dev_mgr, &DeviceConfig::NetworkCfg(config)) + attach_network_device(&self.dev_mgr, config) .await - .context("Handle device")?; - Ok(()) + .context("Handle device") } async fn detach(&self, h: &dyn Hypervisor) -> Result<()> { diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs index e7c758c206..73187c5360 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs @@ -9,14 +9,14 @@ use std::sync::Arc; use anyhow::{Context, Result}; use async_trait::async_trait; -use hypervisor::device::device_manager::{do_handle_device, DeviceManager}; +use hypervisor::device::device_manager::DeviceManager; use hypervisor::device::driver::NetworkConfig; -use hypervisor::device::{DeviceConfig, DeviceType}; +use hypervisor::device::DeviceType; use hypervisor::{Hypervisor, NetworkDevice}; use tokio::sync::RwLock; use super::endpoint_persist::{EndpointState, VethEndpointState}; -use super::Endpoint; +use super::{attach_network_device, Endpoint}; use crate::network::{utils, NetworkPair}; #[derive(Debug)] @@ -72,18 +72,16 @@ impl Endpoint for VethEndpoint { self.net_pair.tap.tap_iface.hard_addr.clone() } - async fn attach(&self) -> Result<()> { + async fn attach(&self) -> Result> { self.net_pair .add_network_model() .await .context("add network model")?; let config = self.get_network_config().context("get network config")?; - do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config)) + attach_network_device(&self.d, config) .await - .context("do handle network Veth endpoint device failed.")?; - - Ok(()) + .context("do handle network Veth endpoint device failed.") } async fn detach(&self, h: &dyn Hypervisor) -> Result<()> { diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/vhost_user_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/vhost_user_endpoint.rs index 480ce6b383..15d47f46c1 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/vhost_user_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/vhost_user_endpoint.rs @@ -78,12 +78,12 @@ impl Endpoint for VhostUserEndpoint { self.guest_mac.clone() } - async fn attach(&self) -> Result<()> { + async fn attach(&self) -> Result> { let config = self.get_network_config(); do_handle_device(&self.dev_mgr, &DeviceConfig::VhostUserNetworkCfg(config)) .await .context("handle device")?; - Ok(()) + Ok(None) } async fn detach(&self, h: &dyn Hypervisor) -> Result<()> { diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs index 9617c029fe..37c89cdd63 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs @@ -9,14 +9,14 @@ use std::sync::Arc; use anyhow::{Context, Result}; use async_trait::async_trait; -use hypervisor::device::device_manager::{do_handle_device, DeviceManager}; +use hypervisor::device::device_manager::DeviceManager; use hypervisor::device::driver::NetworkConfig; -use hypervisor::device::{DeviceConfig, DeviceType}; +use hypervisor::device::DeviceType; use hypervisor::{Hypervisor, NetworkDevice}; use tokio::sync::RwLock; use super::endpoint_persist::{EndpointState, VlanEndpointState}; -use super::Endpoint; +use super::{attach_network_device, Endpoint}; use crate::network::network_model::TC_FILTER_NET_MODEL_STR; use crate::network::{utils, NetworkPair}; @@ -72,18 +72,16 @@ impl Endpoint for VlanEndpoint { self.net_pair.tap.tap_iface.hard_addr.clone() } - async fn attach(&self) -> Result<()> { + async fn attach(&self) -> Result> { self.net_pair .add_network_model() .await .context("add network model failed.")?; let config = self.get_network_config().context("get network config")?; - do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config)) + attach_network_device(&self.d, config) .await - .context("do handle network Vlan endpoint device failed.")?; - - Ok(()) + .context("do handle network Vlan endpoint device failed.") } async fn detach(&self, h: &dyn Hypervisor) -> Result<()> { diff --git a/src/runtime-rs/crates/resource/src/network/network_info/mod.rs b/src/runtime-rs/crates/resource/src/network/network_info/mod.rs index a0e896bb3a..3a8b8deb67 100644 --- a/src/runtime-rs/crates/resource/src/network/network_info/mod.rs +++ b/src/runtime-rs/crates/resource/src/network/network_info/mod.rs @@ -16,4 +16,7 @@ pub trait NetworkInfo: std::fmt::Debug + Send + Sync { async fn interface(&self) -> Result; async fn routes(&self) -> Result>; async fn neighs(&self) -> Result>; + async fn set_device_path(&self, _path: String) -> Result<()> { + Ok(()) + } } diff --git a/src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs b/src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs index f3f23f2673..6543a5704f 100644 --- a/src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs +++ b/src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs @@ -7,6 +7,7 @@ use std::{ convert::TryFrom, net::{IpAddr, Ipv4Addr, Ipv6Addr}, + sync::Arc, }; use agent::{ARPNeighbor, IPAddress, IPFamily, Interface, Route}; @@ -19,6 +20,7 @@ use netlink_packet_route::{ route::{RouteAddress, RouteAttribute, RouteMessage, RouteMetric}, }; use rtnetlink::{IpVersion, RouteMessageBuilder}; +use tokio::sync::RwLock; use super::NetworkInfo; use crate::network::utils::{ @@ -29,6 +31,7 @@ use crate::network::utils::{ #[derive(Debug)] pub(crate) struct NetworkInfoFromLink { interface: Interface, + device_path: Arc>, neighs: Vec, routes: Vec, } @@ -54,6 +57,7 @@ impl NetworkInfoFromLink { field_type: link.r#type().to_string(), raw_flags: attrs.flags & libc::IFF_NOARP as u32, }, + device_path: Arc::new(RwLock::new(String::new())), neighs: handle_neighbors(handle, attrs) .await .context("handle neighbours")?, @@ -258,7 +262,9 @@ async fn handle_routes(handle: &rtnetlink::Handle, attrs: &LinkAttrs) -> Result< #[async_trait] impl NetworkInfo for NetworkInfoFromLink { async fn interface(&self) -> Result { - Ok(self.interface.clone()) + let mut iface = self.interface.clone(); + iface.device_path = self.device_path.read().await.clone(); + Ok(iface) } async fn routes(&self) -> Result> { @@ -268,6 +274,11 @@ impl NetworkInfo for NetworkInfoFromLink { async fn neighs(&self) -> Result> { Ok(self.neighs.clone()) } + + async fn set_device_path(&self, path: String) -> Result<()> { + *self.device_path.write().await = path; + Ok(()) + } } fn parse_route_addr(ra: &RouteAddress) -> Result { diff --git a/src/runtime-rs/crates/resource/src/network/network_with_netns.rs b/src/runtime-rs/crates/resource/src/network/network_with_netns.rs index 0ce1a93ee8..a7747a0f0e 100644 --- a/src/runtime-rs/crates/resource/src/network/network_with_netns.rs +++ b/src/runtime-rs/crates/resource/src/network/network_with_netns.rs @@ -93,7 +93,12 @@ impl Network for NetworkWithNetns { let inner = self.inner.read().await; let _netns_guard = netns::NetnsGuard::new(&inner.netns_path).context("net netns guard")?; for e in &inner.entity_list { - e.endpoint.attach().await.context("attach")?; + if let Some(device_path) = e.endpoint.attach().await.context("attach")? { + e.network_info + .set_device_path(device_path) + .await + .context("set device path")?; + } } Ok(()) }