runtime-rs: pass NIC PCI path to agent on hot-plug

Persist and forward the hot-plugged NIC guest PCI path so the agent can
resolve sysfs location and update interfaces reliably.

Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>
Assisted-by: OpenAI Codex <codex@openai.com>

src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs

@@ -10,6 +10,7 @@ use anyhow::{Context, Result};
 use async_trait::async_trait;
 
 use crate::device::topology::PCIeTopology;
+use crate::device::pci_path::PciPath;
 use crate::device::{Device, DeviceType};
 use crate::Hypervisor as hypervisor;
 
@@ -48,6 +49,8 @@ pub struct NetworkConfig {
     pub use_generic_irq: Option<bool>,
     /// Allow duplicate mac
     pub allow_duplicate_mac: bool,
+    /// Guest PCI path after hot-plug (used by the agent to wait for uevents).
+    pub pci_path: Option<PciPath>,
 }
 
 #[derive(Clone, Debug, Default)]
@@ -76,11 +79,16 @@ impl Device for NetworkDevice {
         _pcie_topo: &mut Option<&mut PCIeTopology>,
         h: &dyn hypervisor,
     ) -> Result<()> {
-        h.add_device(DeviceType::Network(self.clone()))
+        let updated = h
+            .add_device(DeviceType::Network(self.clone()))
             .await
             .context("add network device.")?;
 
-        return Ok(());
+        if let DeviceType::Network(net) = updated {
+            self.config = net.config;
+        }
+
+        Ok(())
     }
 
     async fn detach(

src/runtime-rs/crates/hypervisor/src/qemu/inner.rs

@@ -18,7 +18,7 @@ use crate::{
 
 use crate::utils::{
     bytes_to_megs, create_dir_all_with_inherit_owner, enter_netns, get_jailer_root, megs_to_bytes,
-    set_groups, vm_cleanup,
+    set_groups, uses_native_ccw_bus, vm_cleanup,
 };
 
 use anyhow::{anyhow, Context, Result};
@@ -1016,14 +1016,29 @@ impl QemuInner {
         };
 
         match device {
-            DeviceType::Network(ref network_device) => {
+            DeviceType::Network(mut network_device) => {
                 let (netdev, virtio_net_device) = get_network_device(
                     &self.config,
                     &network_device.config.host_dev_name,
                     network_device.config.guest_mac.clone().unwrap(),
                     &mut None,
                 )?;
-                qmp.hotplug_network_device(&netdev, &virtio_net_device)?
+                qmp.hotplug_network_device(&netdev, &virtio_net_device)?;
+
+                // On s390x the NIC is plugged onto the virtual channel subsystem
+                // (virtio-net-ccw) and has no PCI address, so the PCI-path lookup
+                // below does not apply.  The agent matches the CCW interface via
+                // its uevent instead; forcing a PCI-path resolution here would
+                // fail and abort the whole NIC attach (breaking guest networking).
+                if !uses_native_ccw_bus() {
+                    let frontend_id = format!("frontend-{}", virtio_net_device.get_netdev_id());
+                    let pci_path = qmp
+                        .get_device_by_qdev_id(&frontend_id)
+                        .context("get network device pci path")?;
+                    network_device.config.pci_path = Some(pci_path);
+                }
+
+                return Ok(DeviceType::Network(network_device));
             }
             DeviceType::Block(mut block_device) => {
                 let block_driver = &self.config.blockdev_info.block_device_driver;

src/runtime-rs/crates/resource/src/network/dan.rs

@@ -144,7 +144,12 @@ impl Network for Dan {
             _netns_guard = NetnsGuard::new(netns).context("New netns guard")?;
         }
         for e in inner.entity_list.iter() {
-            e.endpoint.attach().await.context("Attach")?;
+            if let Some(device_path) = e.endpoint.attach().await.context("Attach")? {
+                e.network_info
+                    .set_device_path(device_path)
+                    .await
+                    .context("set device path")?;
+            }
         }
         Ok(())
     }

src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs

@@ -9,14 +9,14 @@ use std::sync::Arc;
 
 use anyhow::{Context, Result};
 use async_trait::async_trait;
-use hypervisor::device::device_manager::{do_handle_device, DeviceManager};
+use hypervisor::device::device_manager::DeviceManager;
 use hypervisor::device::driver::NetworkConfig;
-use hypervisor::device::{DeviceConfig, DeviceType};
+use hypervisor::device::DeviceType;
 use hypervisor::{Hypervisor, NetworkDevice};
 use tokio::sync::RwLock;
 
 use super::endpoint_persist::{EndpointState, IpVlanEndpointState};
-use super::Endpoint;
+use super::{attach_network_device, Endpoint};
 use crate::network::{network_model::TC_FILTER_NET_MODEL_STR, utils, NetworkPair};
 
 // IPVlanEndpoint is the endpoint bridged to VM
@@ -73,18 +73,16 @@ impl Endpoint for IPVlanEndpoint {
         self.net_pair.tap.tap_iface.hard_addr.clone()
     }
 
-    async fn attach(&self) -> Result<()> {
+    async fn attach(&self) -> Result<Option<String>> {
         self.net_pair
             .add_network_model()
             .await
             .context("error adding network model")?;
 
         let config = self.get_network_config().context("get network config")?;
-        do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config))
+        attach_network_device(&self.d, config)
             .await
-            .context("do handle network IPVlan endpoint device failed.")?;
-
-        Ok(())
+            .context("do handle network IPVlan endpoint device failed.")
     }
 
     async fn detach(&self, h: &dyn Hypervisor) -> Result<()> {

src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs

@@ -9,14 +9,14 @@ use std::sync::Arc;
 
 use anyhow::{Context, Result};
 use async_trait::async_trait;
-use hypervisor::device::device_manager::{do_handle_device, DeviceManager};
+use hypervisor::device::device_manager::DeviceManager;
 use hypervisor::device::driver::NetworkConfig;
-use hypervisor::device::{DeviceConfig, DeviceType};
+use hypervisor::device::DeviceType;
 use hypervisor::{Hypervisor, NetworkDevice};
 use tokio::sync::RwLock;
 
 use super::endpoint_persist::{EndpointState, MacvlanEndpointState};
-use super::Endpoint;
+use super::{attach_network_device, Endpoint};
 use crate::network::{utils, NetworkPair};
 
 #[derive(Debug)]
@@ -72,18 +72,16 @@ impl Endpoint for MacVlanEndpoint {
         self.net_pair.tap.tap_iface.hard_addr.clone()
     }
 
-    async fn attach(&self) -> Result<()> {
+    async fn attach(&self) -> Result<Option<String>> {
         self.net_pair
             .add_network_model()
             .await
             .context("add network model")?;
 
         let config = self.get_network_config().context("get network config")?;
-        do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config))
+        attach_network_device(&self.d, config)
             .await
-            .context("do handle network MacVlan endpoint device failed.")?;
-
-        Ok(())
+            .context("do handle network MacVlan endpoint device failed.")
     }
 
     async fn detach(&self, h: &dyn Hypervisor) -> Result<()> {

src/runtime-rs/crates/resource/src/network/endpoint/mod.rs

@@ -23,15 +23,31 @@ pub use vhost_user_endpoint::VhostUserEndpoint;
 
 use anyhow::Result;
 use async_trait::async_trait;
+use hypervisor::device::device_manager::{do_handle_device, DeviceManager};
+use hypervisor::device::driver::NetworkConfig;
+use hypervisor::device::{DeviceConfig, DeviceType};
 use hypervisor::Hypervisor;
+use tokio::sync::RwLock;
+use std::sync::Arc;
 
 use super::EndpointState;
 
+pub(crate) async fn attach_network_device(
+    d: &Arc<RwLock<DeviceManager>>,
+    config: NetworkConfig,
+) -> Result<Option<String>> {
+    let device_info = do_handle_device(d, &DeviceConfig::NetworkCfg(config)).await?;
+    match device_info {
+        DeviceType::Network(net) => Ok(net.config.pci_path.map(|p| p.to_string())),
+        _ => Ok(None),
+    }
+}
+
 #[async_trait]
 pub trait Endpoint: std::fmt::Debug + Send + Sync {
     async fn name(&self) -> String;
     async fn hardware_addr(&self) -> String;
-    async fn attach(&self) -> Result<()>;
+    async fn attach(&self) -> Result<Option<String>>;
     async fn detach(&self, hypervisor: &dyn Hypervisor) -> Result<()>;
     async fn save(&self) -> Option<EndpointState>;
     /// Returns the guest PCI path for this endpoint if it is a cold-plugged

src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs

@@ -117,7 +117,7 @@ impl Endpoint for PhysicalEndpoint {
         self.hard_addr.clone()
     }
 
-    async fn attach(&self) -> Result<()> {
+    async fn attach(&self) -> Result<Option<String>> {
         // Push the desired netdev MAC down to the VF as an "admin MAC" via the
         // PF before we rebind to vfio-pci. Without this the guest mlx5_core
         // inherits the VF firmware default MAC, which differs from the
@@ -181,7 +181,7 @@ impl Endpoint for PhysicalEndpoint {
             }
         }
 
-        Ok(())
+        Ok(None)
     }
 
     // detach for physical endpoint unbinds the physical network interface from vfio-pci

src/runtime-rs/crates/resource/src/network/endpoint/tap_endpoint.rs

@@ -8,13 +8,14 @@ use std::sync::Arc;
 
 use anyhow::{Context, Result};
 use async_trait::async_trait;
-use hypervisor::device::device_manager::{do_handle_device, DeviceManager};
-use hypervisor::device::{DeviceConfig, DeviceType};
-use hypervisor::{Hypervisor, NetworkConfig, NetworkDevice};
+use hypervisor::device::device_manager::DeviceManager;
+use hypervisor::device::driver::NetworkConfig;
+use hypervisor::device::DeviceType;
+use hypervisor::{Hypervisor, NetworkDevice};
 use tokio::sync::RwLock;
 
 use super::endpoint_persist::TapEndpointState;
-use super::Endpoint;
+use super::{attach_network_device, Endpoint};
 use crate::network::network_pair::{get_link_by_name, NetworkInterface};
 use crate::network::{utils, EndpointState};
 
@@ -89,12 +90,11 @@ impl Endpoint for TapEndpoint {
         self.guest_mac.clone()
     }
 
-    async fn attach(&self) -> Result<()> {
+    async fn attach(&self) -> Result<Option<String>> {
         let config = self.get_network_config().context("Get network config")?;
-        do_handle_device(&self.dev_mgr, &DeviceConfig::NetworkCfg(config))
+        attach_network_device(&self.dev_mgr, config)
             .await
-            .context("Handle device")?;
-        Ok(())
+            .context("Handle device")
     }
 
     async fn detach(&self, h: &dyn Hypervisor) -> Result<()> {

src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs

@@ -9,14 +9,14 @@ use std::sync::Arc;
 
 use anyhow::{Context, Result};
 use async_trait::async_trait;
-use hypervisor::device::device_manager::{do_handle_device, DeviceManager};
+use hypervisor::device::device_manager::DeviceManager;
 use hypervisor::device::driver::NetworkConfig;
-use hypervisor::device::{DeviceConfig, DeviceType};
+use hypervisor::device::DeviceType;
 use hypervisor::{Hypervisor, NetworkDevice};
 use tokio::sync::RwLock;
 
 use super::endpoint_persist::{EndpointState, VethEndpointState};
-use super::Endpoint;
+use super::{attach_network_device, Endpoint};
 use crate::network::{utils, NetworkPair};
 
 #[derive(Debug)]
@@ -72,18 +72,16 @@ impl Endpoint for VethEndpoint {
         self.net_pair.tap.tap_iface.hard_addr.clone()
     }
 
-    async fn attach(&self) -> Result<()> {
+    async fn attach(&self) -> Result<Option<String>> {
         self.net_pair
             .add_network_model()
             .await
             .context("add network model")?;
 
         let config = self.get_network_config().context("get network config")?;
-        do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config))
+        attach_network_device(&self.d, config)
             .await
-            .context("do handle network Veth endpoint device failed.")?;
-
-        Ok(())
+            .context("do handle network Veth endpoint device failed.")
     }
 
     async fn detach(&self, h: &dyn Hypervisor) -> Result<()> {

src/runtime-rs/crates/resource/src/network/endpoint/vhost_user_endpoint.rs

@@ -78,12 +78,12 @@ impl Endpoint for VhostUserEndpoint {
         self.guest_mac.clone()
     }
 
-    async fn attach(&self) -> Result<()> {
+    async fn attach(&self) -> Result<Option<String>> {
         let config = self.get_network_config();
         do_handle_device(&self.dev_mgr, &DeviceConfig::VhostUserNetworkCfg(config))
             .await
             .context("handle device")?;
-        Ok(())
+        Ok(None)
     }
 
     async fn detach(&self, h: &dyn Hypervisor) -> Result<()> {

src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs

@@ -9,14 +9,14 @@ use std::sync::Arc;
 
 use anyhow::{Context, Result};
 use async_trait::async_trait;
-use hypervisor::device::device_manager::{do_handle_device, DeviceManager};
+use hypervisor::device::device_manager::DeviceManager;
 use hypervisor::device::driver::NetworkConfig;
-use hypervisor::device::{DeviceConfig, DeviceType};
+use hypervisor::device::DeviceType;
 use hypervisor::{Hypervisor, NetworkDevice};
 use tokio::sync::RwLock;
 
 use super::endpoint_persist::{EndpointState, VlanEndpointState};
-use super::Endpoint;
+use super::{attach_network_device, Endpoint};
 use crate::network::network_model::TC_FILTER_NET_MODEL_STR;
 use crate::network::{utils, NetworkPair};
 
@@ -72,18 +72,16 @@ impl Endpoint for VlanEndpoint {
         self.net_pair.tap.tap_iface.hard_addr.clone()
     }
 
-    async fn attach(&self) -> Result<()> {
+    async fn attach(&self) -> Result<Option<String>> {
         self.net_pair
             .add_network_model()
             .await
             .context("add network model failed.")?;
 
         let config = self.get_network_config().context("get network config")?;
-        do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config))
+        attach_network_device(&self.d, config)
             .await
-            .context("do handle network Vlan endpoint device failed.")?;
-
-        Ok(())
+            .context("do handle network Vlan endpoint device failed.")
     }
 
     async fn detach(&self, h: &dyn Hypervisor) -> Result<()> {

src/runtime-rs/crates/resource/src/network/network_info/mod.rs

@@ -16,4 +16,7 @@ pub trait NetworkInfo: std::fmt::Debug + Send + Sync {
     async fn interface(&self) -> Result<Interface>;
     async fn routes(&self) -> Result<Vec<Route>>;
     async fn neighs(&self) -> Result<Vec<ARPNeighbor>>;
+    async fn set_device_path(&self, _path: String) -> Result<()> {
+        Ok(())
+    }
 }

src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs

@@ -7,6 +7,7 @@
 use std::{
     convert::TryFrom,
     net::{IpAddr, Ipv4Addr, Ipv6Addr},
+    sync::Arc,
 };
 
 use agent::{ARPNeighbor, IPAddress, IPFamily, Interface, Route};
@@ -19,6 +20,7 @@ use netlink_packet_route::{
     route::{RouteAddress, RouteAttribute, RouteMessage, RouteMetric},
 };
 use rtnetlink::{IpVersion, RouteMessageBuilder};
+use tokio::sync::RwLock;
 
 use super::NetworkInfo;
 use crate::network::utils::{
@@ -29,6 +31,7 @@ use crate::network::utils::{
 #[derive(Debug)]
 pub(crate) struct NetworkInfoFromLink {
     interface: Interface,
+    device_path: Arc<RwLock<String>>,
     neighs: Vec<ARPNeighbor>,
     routes: Vec<Route>,
 }
@@ -54,6 +57,7 @@ impl NetworkInfoFromLink {
                 field_type: link.r#type().to_string(),
                 raw_flags: attrs.flags & libc::IFF_NOARP as u32,
             },
+            device_path: Arc::new(RwLock::new(String::new())),
             neighs: handle_neighbors(handle, attrs)
                 .await
                 .context("handle neighbours")?,
@@ -258,7 +262,9 @@ async fn handle_routes(handle: &rtnetlink::Handle, attrs: &LinkAttrs) -> Result<
 #[async_trait]
 impl NetworkInfo for NetworkInfoFromLink {
     async fn interface(&self) -> Result<Interface> {
-        Ok(self.interface.clone())
+        let mut iface = self.interface.clone();
+        iface.device_path = self.device_path.read().await.clone();
+        Ok(iface)
     }
 
     async fn routes(&self) -> Result<Vec<Route>> {
@@ -268,6 +274,11 @@ impl NetworkInfo for NetworkInfoFromLink {
     async fn neighs(&self) -> Result<Vec<ARPNeighbor>> {
         Ok(self.neighs.clone())
     }
+
+    async fn set_device_path(&self, path: String) -> Result<()> {
+        *self.device_path.write().await = path;
+        Ok(())
+    }
 }
 
 fn parse_route_addr(ra: &RouteAddress) -> Result<IpAddr> {

src/runtime-rs/crates/resource/src/network/network_with_netns.rs

@@ -93,7 +93,12 @@ impl Network for NetworkWithNetns {
         let inner = self.inner.read().await;
         let _netns_guard = netns::NetnsGuard::new(&inner.netns_path).context("net netns guard")?;
         for e in &inner.entity_list {
-            e.endpoint.attach().await.context("attach")?;
+            if let Some(device_path) = e.endpoint.attach().await.context("attach")? {
+                e.network_info
+                    .set_device_path(device_path)
+                    .await
+                    .context("set device path")?;
+            }
         }
         Ok(())
     }