Merge pull request #10213 from ChengyuZhu6/device

Refine device management for kata-agent

src/agent/src/device/block_device_handler.rs

@@ -0,0 +1,342 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2024 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#[cfg(target_arch = "s390x")]
+use crate::ccw;
+use crate::device::{
+    pcipath_to_sysfs, DeviceContext, DeviceHandler, DeviceInfo, SpecUpdate, BLOCK,
+};
+#[cfg(target_arch = "s390x")]
+use crate::linux_abi::CCW_ROOT_BUS_PATH;
+use crate::linux_abi::{create_pci_root_bus_path, SYSFS_DIR, SYSTEM_DEV_PATH};
+use crate::pci;
+use crate::sandbox::Sandbox;
+use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher};
+use anyhow::{anyhow, Context, Result};
+use kata_types::device::{DRIVER_BLK_CCW_TYPE, DRIVER_BLK_MMIO_TYPE, DRIVER_BLK_PCI_TYPE};
+use protocols::agent::Device;
+use regex::Regex;
+use std::path::Path;
+use std::str::FromStr;
+use std::sync::Arc;
+use tokio::sync::Mutex;
+use tracing::instrument;
+
+#[derive(Debug)]
+pub struct VirtioBlkPciDeviceHandler {}
+
+#[derive(Debug)]
+pub struct VirtioBlkCcwDeviceHandler {}
+
+#[derive(Debug)]
+pub struct VirtioBlkMmioDeviceHandler {}
+
+#[async_trait::async_trait]
+impl DeviceHandler for VirtioBlkPciDeviceHandler {
+    #[instrument]
+    fn driver_types(&self) -> &[&str] {
+        &[DRIVER_BLK_PCI_TYPE]
+    }
+
+    #[instrument]
+    async fn device_handler(&self, device: &Device, ctx: &mut DeviceContext) -> Result<SpecUpdate> {
+        let pcipath = pci::Path::from_str(&device.id)?;
+        let vm_path = get_virtio_blk_pci_device_name(ctx.sandbox, &pcipath).await?;
+
+        Ok(DeviceInfo::new(&vm_path, true)
+            .context("New device info")?
+            .into())
+    }
+}
+
+#[async_trait::async_trait]
+impl DeviceHandler for VirtioBlkCcwDeviceHandler {
+    #[instrument]
+    fn driver_types(&self) -> &[&str] {
+        &[DRIVER_BLK_CCW_TYPE]
+    }
+
+    #[cfg(target_arch = "s390x")]
+    #[instrument]
+    async fn device_handler(&self, device: &Device, ctx: &mut DeviceContext) -> Result<SpecUpdate> {
+        let ccw_device = ccw::Device::from_str(&device.id)?;
+        let vm_path = get_virtio_blk_ccw_device_name(ctx.sandbox, &ccw_device).await?;
+
+        Ok(DeviceInfo::new(&vm_path, true)
+            .context("New device info")?
+            .into())
+    }
+
+    #[cfg(not(target_arch = "s390x"))]
+    async fn device_handler(
+        &self,
+        _device: &Device,
+        _ctx: &mut DeviceContext,
+    ) -> Result<SpecUpdate> {
+        Err(anyhow!("CCW is only supported on s390x"))
+    }
+}
+
+#[async_trait::async_trait]
+impl DeviceHandler for VirtioBlkMmioDeviceHandler {
+    #[instrument]
+    fn driver_types(&self) -> &[&str] {
+        &[DRIVER_BLK_MMIO_TYPE]
+    }
+
+    #[instrument]
+    async fn device_handler(&self, device: &Device, ctx: &mut DeviceContext) -> Result<SpecUpdate> {
+        if device.vm_path.is_empty() {
+            return Err(anyhow!("Invalid path for virtio mmio blk device"));
+        }
+        if !Path::new(&device.vm_path).exists() {
+            get_virtio_blk_mmio_device_name(ctx.sandbox, &device.vm_path.to_string())
+                .await
+                .context("failed to get mmio device name")?;
+        }
+
+        Ok(DeviceInfo::new(device.vm_path(), true)
+            .context("New device info")?
+            .into())
+    }
+}
+
+#[instrument]
+pub async fn get_virtio_blk_pci_device_name(
+    sandbox: &Arc<Mutex<Sandbox>>,
+    pcipath: &pci::Path,
+) -> Result<String> {
+    let root_bus_sysfs = format!("{}{}", SYSFS_DIR, create_pci_root_bus_path());
+    let sysfs_rel_path = pcipath_to_sysfs(&root_bus_sysfs, pcipath)?;
+    let matcher = VirtioBlkPciMatcher::new(&sysfs_rel_path);
+
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+    Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
+}
+
+#[instrument]
+pub async fn get_virtio_blk_mmio_device_name(
+    sandbox: &Arc<Mutex<Sandbox>>,
+    devpath: &str,
+) -> Result<()> {
+    let devname = devpath
+        .strip_prefix("/dev/")
+        .ok_or_else(|| anyhow!("Storage source '{}' must start with /dev/", devpath))?;
+
+    let matcher = VirtioBlkMmioMatcher::new(devname);
+    let uev = wait_for_uevent(sandbox, matcher)
+        .await
+        .context("failed to wait for uevent")?;
+    if uev.devname != devname {
+        return Err(anyhow!(
+            "Unexpected device name {} for mmio device (expected {})",
+            uev.devname,
+            devname
+        ));
+    }
+    Ok(())
+}
+
+#[cfg(target_arch = "s390x")]
+#[instrument]
+pub async fn get_virtio_blk_ccw_device_name(
+    sandbox: &Arc<Mutex<Sandbox>>,
+    device: &ccw::Device,
+) -> Result<String> {
+    let matcher = VirtioBlkCCWMatcher::new(CCW_ROOT_BUS_PATH, device);
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+    let devname = uev.devname;
+    Path::new(SYSTEM_DEV_PATH)
+        .join(&devname)
+        .to_str()
+        .map(String::from)
+        .ok_or_else(|| anyhow!("CCW device name {} is not valid UTF-8", &devname))
+}
+
+#[derive(Debug)]
+pub struct VirtioBlkPciMatcher {
+    rex: Regex,
+}
+
+impl VirtioBlkPciMatcher {
+    pub fn new(relpath: &str) -> VirtioBlkPciMatcher {
+        let root_bus = create_pci_root_bus_path();
+        let re = format!(r"^{}{}/virtio[0-9]+/block/", root_bus, relpath);
+
+        VirtioBlkPciMatcher {
+            rex: Regex::new(&re).expect("BUG: failed to compile VirtioBlkPciMatcher regex"),
+        }
+    }
+}
+
+impl UeventMatcher for VirtioBlkPciMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.subsystem == BLOCK && self.rex.is_match(&uev.devpath) && !uev.devname.is_empty()
+    }
+}
+
+#[derive(Debug)]
+pub struct VirtioBlkMmioMatcher {
+    suffix: String,
+}
+
+impl VirtioBlkMmioMatcher {
+    pub fn new(devname: &str) -> VirtioBlkMmioMatcher {
+        VirtioBlkMmioMatcher {
+            suffix: format!(r"/block/{}", devname),
+        }
+    }
+}
+
+impl UeventMatcher for VirtioBlkMmioMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.subsystem == BLOCK && uev.devpath.ends_with(&self.suffix) && !uev.devname.is_empty()
+    }
+}
+
+#[cfg(target_arch = "s390x")]
+#[derive(Debug)]
+pub struct VirtioBlkCCWMatcher {
+    rex: Regex,
+}
+
+#[cfg(target_arch = "s390x")]
+impl VirtioBlkCCWMatcher {
+    pub fn new(root_bus_path: &str, device: &ccw::Device) -> Self {
+        let re = format!(
+            r"^{}/0\.[0-3]\.[0-9a-f]{{1,4}}/{}/virtio[0-9]+/block/",
+            root_bus_path, device
+        );
+        VirtioBlkCCWMatcher {
+            rex: Regex::new(&re).expect("BUG: failed to compile VirtioBlkCCWMatcher regex"),
+        }
+    }
+}
+
+#[cfg(target_arch = "s390x")]
+impl UeventMatcher for VirtioBlkCCWMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.action == "add" && self.rex.is_match(&uev.devpath) && !uev.devname.is_empty()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    #[allow(clippy::redundant_clone)]
+    async fn test_virtio_blk_matcher() {
+        let root_bus = create_pci_root_bus_path();
+        let devname = "vda";
+
+        let mut uev_a = crate::uevent::Uevent::default();
+        let relpath_a = "/0000:00:0a.0";
+        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev_a.subsystem = BLOCK.to_string();
+        uev_a.devname = devname.to_string();
+        uev_a.devpath = format!("{}{}/virtio4/block/{}", root_bus, relpath_a, devname);
+        let matcher_a = VirtioBlkPciMatcher::new(relpath_a);
+
+        let mut uev_b = uev_a.clone();
+        let relpath_b = "/0000:00:0a.0/0000:00:0b.0";
+        uev_b.devpath = format!("{}{}/virtio0/block/{}", root_bus, relpath_b, devname);
+        let matcher_b = VirtioBlkPciMatcher::new(relpath_b);
+
+        assert!(matcher_a.is_match(&uev_a));
+        assert!(matcher_b.is_match(&uev_b));
+        assert!(!matcher_b.is_match(&uev_a));
+        assert!(!matcher_a.is_match(&uev_b));
+    }
+
+    #[cfg(target_arch = "s390x")]
+    #[tokio::test]
+    async fn test_virtio_blk_ccw_matcher() {
+        let root_bus = CCW_ROOT_BUS_PATH;
+        let subsystem = "block";
+        let devname = "vda";
+        let relpath = "0.0.0002";
+
+        let mut uev = crate::uevent::Uevent::default();
+        uev.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev.subsystem = subsystem.to_string();
+        uev.devname = devname.to_string();
+        uev.devpath = format!(
+            "{}/0.0.0001/{}/virtio1/{}/{}",
+            root_bus, relpath, subsystem, devname
+        );
+
+        // Valid path
+        let device = ccw::Device::from_str(relpath).unwrap();
+        let matcher = VirtioBlkCCWMatcher::new(root_bus, &device);
+        assert!(matcher.is_match(&uev));
+
+        // Invalid paths
+        uev.devpath = format!(
+            "{}/0.0.0001/0.0.0003/virtio1/{}/{}",
+            root_bus, subsystem, devname
+        );
+        assert!(!matcher.is_match(&uev));
+
+        uev.devpath = format!("0.0.0001/{}/virtio1/{}/{}", relpath, subsystem, devname);
+        assert!(!matcher.is_match(&uev));
+
+        uev.devpath = format!(
+            "{}/0.0.0001/{}/virtio/{}/{}",
+            root_bus, relpath, subsystem, devname
+        );
+        assert!(!matcher.is_match(&uev));
+
+        uev.devpath = format!("{}/0.0.0001/{}/virtio1", root_bus, relpath);
+        assert!(!matcher.is_match(&uev));
+
+        uev.devpath = format!(
+            "{}/1.0.0001/{}/virtio1/{}/{}",
+            root_bus, relpath, subsystem, devname
+        );
+        assert!(!matcher.is_match(&uev));
+
+        uev.devpath = format!(
+            "{}/0.4.0001/{}/virtio1/{}/{}",
+            root_bus, relpath, subsystem, devname
+        );
+        assert!(!matcher.is_match(&uev));
+
+        uev.devpath = format!(
+            "{}/0.0.10000/{}/virtio1/{}/{}",
+            root_bus, relpath, subsystem, devname
+        );
+        assert!(!matcher.is_match(&uev));
+    }
+
+    #[tokio::test]
+    #[allow(clippy::redundant_clone)]
+    async fn test_virtio_blk_mmio_matcher() {
+        let devname_a = "vda";
+        let devname_b = "vdb";
+        let mut uev_a = crate::uevent::Uevent::default();
+        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev_a.subsystem = BLOCK.to_string();
+        uev_a.devname = devname_a.to_string();
+        uev_a.devpath = format!(
+            "/sys/devices/virtio-mmio-cmdline/virtio-mmio.0/virtio0/block/{}",
+            devname_a
+        );
+        let matcher_a = VirtioBlkMmioMatcher::new(devname_a);
+
+        let mut uev_b = uev_a.clone();
+        uev_b.devpath = format!(
+            "/sys/devices/virtio-mmio-cmdline/virtio-mmio.4/virtio4/block/{}",
+            devname_b
+        );
+        let matcher_b = VirtioBlkMmioMatcher::new(devname_b);
+
+        assert!(matcher_a.is_match(&uev_a));
+        assert!(matcher_b.is_match(&uev_b));
+        assert!(!matcher_b.is_match(&uev_a));
+        assert!(!matcher_a.is_match(&uev_b));
+    }
+}

src/agent/src/device/network_device_handler.rs

@@ -0,0 +1,114 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2024 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+use crate::device::pcipath_to_sysfs;
+use crate::linux_abi::*;
+use crate::pci;
+use crate::sandbox::Sandbox;
+use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher};
+use anyhow::{anyhow, Result};
+use regex::Regex;
+use std::fs;
+use std::sync::Arc;
+use tokio::sync::Mutex;
+
+pub async fn wait_for_net_interface(
+    sandbox: &Arc<Mutex<Sandbox>>,
+    pcipath: &pci::Path,
+) -> Result<()> {
+    let root_bus_sysfs = format!("{}{}", SYSFS_DIR, create_pci_root_bus_path());
+    let sysfs_rel_path = pcipath_to_sysfs(&root_bus_sysfs, pcipath)?;
+
+    let matcher = NetPciMatcher::new(&sysfs_rel_path);
+
+    // Check if the interface is already added in case network is cold-plugged
+    // or the uevent loop is started before network is added.
+    // We check for the pci deive in the sysfs directory for network devices.
+    let pattern = format!(
+        r"[./]+{}/[a-z0-9/]*net/[a-z0-9/]*",
+        matcher.devpath.as_str()
+    );
+    let re = Regex::new(&pattern).expect("BUG: Failed to compile regex for NetPciMatcher");
+
+    for entry in fs::read_dir(SYSFS_NET_PATH)? {
+        let entry = entry?;
+        let path = entry.path();
+        let target_path = fs::read_link(path)?;
+        let target_path_str = target_path
+            .to_str()
+            .ok_or_else(|| anyhow!("Expected symlink in dir {}", SYSFS_NET_PATH))?;
+
+        if re.is_match(target_path_str) {
+            return Ok(());
+        }
+    }
+    let _uev = wait_for_uevent(sandbox, matcher).await?;
+
+    Ok(())
+}
+
+#[derive(Debug)]
+pub struct NetPciMatcher {
+    devpath: String,
+}
+
+impl NetPciMatcher {
+    pub fn new(relpath: &str) -> NetPciMatcher {
+        let root_bus = create_pci_root_bus_path();
+
+        NetPciMatcher {
+            devpath: format!("{}{}", root_bus, relpath),
+        }
+    }
+}
+
+impl UeventMatcher for NetPciMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.devpath.starts_with(self.devpath.as_str())
+            && uev.subsystem == "net"
+            && !uev.interface.is_empty()
+            && uev.action == "add"
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    #[allow(clippy::redundant_clone)]
+    async fn test_net_pci_matcher() {
+        let root_bus = create_pci_root_bus_path();
+        let relpath_a = "/0000:00:02.0/0000:01:01.0";
+
+        let mut uev_a = crate::uevent::Uevent::default();
+        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev_a.devpath = format!("{}{}", root_bus, relpath_a);
+        uev_a.subsystem = String::from("net");
+        uev_a.interface = String::from("eth0");
+        let matcher_a = NetPciMatcher::new(relpath_a);
+        println!("Matcher a : {}", matcher_a.devpath);
+
+        let relpath_b = "/0000:00:02.0/0000:01:02.0";
+        let mut uev_b = uev_a.clone();
+        uev_b.devpath = format!("{}{}", root_bus, relpath_b);
+        let matcher_b = NetPciMatcher::new(relpath_b);
+
+        assert!(matcher_a.is_match(&uev_a));
+        assert!(matcher_b.is_match(&uev_b));
+        assert!(!matcher_b.is_match(&uev_a));
+        assert!(!matcher_a.is_match(&uev_b));
+
+        let relpath_c = "/0000:00:02.0/0000:01:03.0";
+        let net_substr = "/net/eth0";
+        let mut uev_c = uev_a.clone();
+        uev_c.devpath = format!("{}{}{}", root_bus, relpath_c, net_substr);
+        let matcher_c = NetPciMatcher::new(relpath_c);
+
+        assert!(matcher_c.is_match(&uev_c));
+        assert!(!matcher_a.is_match(&uev_c));
+        assert!(!matcher_b.is_match(&uev_c));
+    }
+}

src/agent/src/device/nvdimm_device_handler.rs

@@ -0,0 +1,83 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2024 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use crate::device::{DeviceContext, DeviceHandler, DeviceInfo, SpecUpdate, BLOCK};
+use crate::linux_abi::ACPI_DEV_PATH;
+use crate::sandbox::Sandbox;
+use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher};
+use anyhow::{anyhow, Context, Result};
+use kata_types::device::DRIVER_NVDIMM_TYPE;
+use protocols::agent::Device;
+use std::sync::Arc;
+use tokio::sync::Mutex;
+use tracing::instrument;
+
+#[derive(Debug)]
+pub struct VirtioNvdimmDeviceHandler {}
+
+#[async_trait::async_trait]
+impl DeviceHandler for VirtioNvdimmDeviceHandler {
+    #[instrument]
+    fn driver_types(&self) -> &[&str] {
+        &[DRIVER_NVDIMM_TYPE]
+    }
+
+    #[instrument]
+    async fn device_handler(&self, device: &Device, ctx: &mut DeviceContext) -> Result<SpecUpdate> {
+        if device.vm_path.is_empty() {
+            return Err(anyhow!("Invalid path for nvdimm device"));
+        }
+        Ok(DeviceInfo::new(device.vm_path(), true)
+            .context("New device info")?
+            .into())
+    }
+}
+
+#[instrument]
+pub async fn wait_for_pmem_device(sandbox: &Arc<Mutex<Sandbox>>, devpath: &str) -> Result<()> {
+    let devname = match devpath.strip_prefix("/dev/") {
+        Some(dev) => dev,
+        None => {
+            return Err(anyhow!(
+                "Storage source '{}' must start with /dev/",
+                devpath
+            ))
+        }
+    };
+
+    let matcher = PmemBlockMatcher::new(devname);
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+    if uev.devname != devname {
+        return Err(anyhow!(
+            "Unexpected device name {} for pmem device (expected {})",
+            uev.devname,
+            devname
+        ));
+    }
+    Ok(())
+}
+
+#[derive(Debug)]
+pub struct PmemBlockMatcher {
+    suffix: String,
+}
+
+impl PmemBlockMatcher {
+    pub fn new(devname: &str) -> PmemBlockMatcher {
+        let suffix = format!(r"/block/{}", devname);
+
+        PmemBlockMatcher { suffix }
+    }
+}
+
+impl UeventMatcher for PmemBlockMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.subsystem == BLOCK
+            && uev.devpath.starts_with(ACPI_DEV_PATH)
+            && uev.devpath.ends_with(&self.suffix)
+            && !uev.devname.is_empty()
+    }
+}

src/agent/src/device/scsi_device_handler.rs

@@ -0,0 +1,140 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2024 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use crate::device::{DeviceContext, DeviceHandler, DeviceInfo, SpecUpdate, BLOCK};
+use crate::linux_abi::*;
+use crate::sandbox::Sandbox;
+use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher};
+use anyhow::{anyhow, Context, Result};
+use kata_types::device::DRIVER_SCSI_TYPE;
+use protocols::agent::Device;
+use std::fs;
+use std::path::PathBuf;
+use std::sync::Arc;
+use tokio::sync::Mutex;
+use tracing::instrument;
+
+#[derive(Debug)]
+pub struct ScsiDeviceHandler {}
+
+#[async_trait::async_trait]
+impl DeviceHandler for ScsiDeviceHandler {
+    #[instrument]
+    fn driver_types(&self) -> &[&str] {
+        &[DRIVER_SCSI_TYPE]
+    }
+
+    #[instrument]
+    async fn device_handler(&self, device: &Device, ctx: &mut DeviceContext) -> Result<SpecUpdate> {
+        let vm_path = get_scsi_device_name(ctx.sandbox, &device.id).await?;
+
+        Ok(DeviceInfo::new(&vm_path, true)
+            .context("New device info")?
+            .into())
+    }
+}
+
+#[instrument]
+pub async fn get_scsi_device_name(
+    sandbox: &Arc<Mutex<Sandbox>>,
+    scsi_addr: &str,
+) -> Result<String> {
+    let matcher = ScsiBlockMatcher::new(scsi_addr);
+
+    scan_scsi_bus(scsi_addr)?;
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+    Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
+}
+
+// FIXME: This matcher is only correct if the guest has at most one
+// SCSI host.
+#[derive(Debug)]
+pub struct ScsiBlockMatcher {
+    search: String,
+}
+
+impl ScsiBlockMatcher {
+    pub fn new(scsi_addr: &str) -> ScsiBlockMatcher {
+        let search = format!(r"/0:0:{}/block/", scsi_addr);
+
+        ScsiBlockMatcher { search }
+    }
+}
+
+impl UeventMatcher for ScsiBlockMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.subsystem == BLOCK && uev.devpath.contains(&self.search) && !uev.devname.is_empty()
+    }
+}
+
+/// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN)
+#[instrument]
+fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
+    let tokens: Vec<&str> = scsi_addr.split(':').collect();
+    if tokens.len() != 2 {
+        return Err(anyhow!(
+            "Unexpected format for SCSI Address: {}, expect SCSIID:LUA",
+            scsi_addr
+        ));
+    }
+
+    // Scan scsi host passing in the channel, SCSI id and LUN.
+    // Channel is always 0 because we have only one SCSI controller.
+    let scan_data = &format!("0 {} {}", tokens[0], tokens[1]);
+
+    for entry in fs::read_dir(SYSFS_SCSI_HOST_PATH)? {
+        let host = entry?.file_name();
+
+        let host_str = host.to_str().ok_or_else(|| {
+            anyhow!(
+                "failed to convert directory entry to unicode for file {:?}",
+                host
+            )
+        })?;
+
+        let scan_path = PathBuf::from(&format!("{}/{}/{}", SYSFS_SCSI_HOST_PATH, host_str, "scan"));
+
+        fs::write(scan_path, scan_data)?;
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    #[allow(clippy::redundant_clone)]
+    async fn test_scsi_block_matcher() {
+        let root_bus = create_pci_root_bus_path();
+        let devname = "sda";
+
+        let mut uev_a = crate::uevent::Uevent::default();
+        let addr_a = "0:0";
+        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev_a.subsystem = BLOCK.to_string();
+        uev_a.devname = devname.to_string();
+        uev_a.devpath = format!(
+            "{}/0000:00:00.0/virtio0/host0/target0:0:0/0:0:{}/block/sda",
+            root_bus, addr_a
+        );
+        let matcher_a = ScsiBlockMatcher::new(addr_a);
+
+        let mut uev_b = uev_a.clone();
+        let addr_b = "2:0";
+        uev_b.devpath = format!(
+            "{}/0000:00:00.0/virtio0/host0/target0:0:2/0:0:{}/block/sdb",
+            root_bus, addr_b
+        );
+        let matcher_b = ScsiBlockMatcher::new(addr_b);
+
+        assert!(matcher_a.is_match(&uev_a));
+        assert!(matcher_b.is_match(&uev_b));
+        assert!(!matcher_b.is_match(&uev_a));
+        assert!(!matcher_a.is_match(&uev_b));
+    }
+}

src/agent/src/device/vfio_device_handler.rs

@@ -0,0 +1,455 @@
+// Copyright (c) 2019 Ant Financial
+// Copyright (c) 2024 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#[cfg(target_arch = "s390x")]
+use crate::ap;
+use crate::device::{pcipath_to_sysfs, DevUpdate, DeviceContext, DeviceHandler, SpecUpdate};
+use crate::linux_abi::*;
+use crate::pci;
+use crate::sandbox::Sandbox;
+use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher};
+use anyhow::{anyhow, Context, Result};
+use kata_types::device::{DRIVER_VFIO_AP_TYPE, DRIVER_VFIO_PCI_GK_TYPE, DRIVER_VFIO_PCI_TYPE};
+use protocols::agent::Device;
+use slog::Logger;
+use std::ffi::OsStr;
+use std::fmt;
+use std::fs;
+use std::os::unix::ffi::OsStrExt;
+use std::path::Path;
+use std::str::FromStr;
+use std::sync::Arc;
+use tokio::sync::Mutex;
+use tracing::instrument;
+
+#[derive(Debug)]
+pub struct VfioPciDeviceHandler {}
+
+#[derive(Debug)]
+pub struct VfioApDeviceHandler {}
+
+#[async_trait::async_trait]
+impl DeviceHandler for VfioPciDeviceHandler {
+    #[instrument]
+    fn driver_types(&self) -> &[&str] {
+        &[DRIVER_VFIO_PCI_GK_TYPE, DRIVER_VFIO_PCI_TYPE]
+    }
+
+    #[instrument]
+    async fn device_handler(&self, device: &Device, ctx: &mut DeviceContext) -> Result<SpecUpdate> {
+        let vfio_in_guest = device.type_ != DRIVER_VFIO_PCI_GK_TYPE;
+        let mut pci_fixups = Vec::<(pci::Address, pci::Address)>::new();
+        let mut group = None;
+
+        for opt in device.options.iter() {
+            let (host, pcipath) = split_vfio_pci_option(opt)
+                .ok_or_else(|| anyhow!("Malformed VFIO PCI option {:?}", opt))?;
+            let host =
+                pci::Address::from_str(host).context("Bad host PCI address in VFIO option {:?}")?;
+            let pcipath = pci::Path::from_str(pcipath)?;
+
+            let guestdev = wait_for_pci_device(ctx.sandbox, &pcipath).await?;
+            if vfio_in_guest {
+                pci_driver_override(ctx.logger, SYSFS_BUS_PCI_PATH, guestdev, "vfio-pci")?;
+
+                // Devices must have an IOMMU group to be usable via VFIO
+                let devgroup = pci_iommu_group(SYSFS_BUS_PCI_PATH, guestdev)?
+                    .ok_or_else(|| anyhow!("{} has no IOMMU group", guestdev))?;
+
+                if let Some(g) = group {
+                    if g != devgroup {
+                        return Err(anyhow!("{} is not in guest IOMMU group {}", guestdev, g));
+                    }
+                }
+
+                group = Some(devgroup);
+            }
+
+            // collect PCI address mapping for both vfio-pci-gk and vfio-pci device
+            pci_fixups.push((host, guestdev));
+        }
+
+        let dev_update = if vfio_in_guest {
+            // If there are any devices at all, logic above ensures that group is not None
+            let group = group.ok_or_else(|| anyhow!("failed to get VFIO group"))?;
+
+            let vm_path = get_vfio_pci_device_name(group, ctx.sandbox).await?;
+
+            Some(DevUpdate::new(&vm_path, &vm_path)?)
+        } else {
+            None
+        };
+
+        Ok(SpecUpdate {
+            dev: dev_update,
+            pci: pci_fixups,
+        })
+    }
+}
+
+#[async_trait::async_trait]
+impl DeviceHandler for VfioApDeviceHandler {
+    #[instrument]
+    fn driver_types(&self) -> &[&str] {
+        &[DRIVER_VFIO_AP_TYPE]
+    }
+
+    #[cfg(target_arch = "s390x")]
+    #[instrument]
+    async fn device_handler(&self, device: &Device, ctx: &mut DeviceContext) -> Result<SpecUpdate> {
+        // Force AP bus rescan
+        fs::write(AP_SCANS_PATH, "1")?;
+        for apqn in device.options.iter() {
+            wait_for_ap_device(ctx.sandbox, ap::Address::from_str(apqn)?).await?;
+        }
+        let dev_update = Some(DevUpdate::new(Z9_CRYPT_DEV_PATH, Z9_CRYPT_DEV_PATH)?);
+        Ok(SpecUpdate {
+            dev: dev_update,
+            pci: Vec::new(),
+        })
+    }
+
+    #[cfg(not(target_arch = "s390x"))]
+    #[instrument]
+    async fn device_handler(&self, _: &Device, _: &mut DeviceContext) -> Result<SpecUpdate> {
+        Err(anyhow!("VFIO-AP is only supported on s390x"))
+    }
+}
+
+async fn get_vfio_pci_device_name(
+    grp: IommuGroup,
+    sandbox: &Arc<Mutex<Sandbox>>,
+) -> Result<String> {
+    let matcher = VfioMatcher::new(grp);
+
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+    Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
+}
+
+#[derive(Debug)]
+pub struct VfioMatcher {
+    syspath: String,
+}
+
+impl VfioMatcher {
+    pub fn new(grp: IommuGroup) -> VfioMatcher {
+        VfioMatcher {
+            syspath: format!("/devices/virtual/vfio/{}", grp),
+        }
+    }
+}
+
+impl UeventMatcher for VfioMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.devpath == self.syspath
+    }
+}
+
+#[cfg(target_arch = "s390x")]
+#[derive(Debug)]
+pub struct ApMatcher {
+    syspath: String,
+}
+
+#[cfg(target_arch = "s390x")]
+impl ApMatcher {
+    pub fn new(address: ap::Address) -> ApMatcher {
+        ApMatcher {
+            syspath: format!(
+                "{}/card{:02x}/{}",
+                AP_ROOT_BUS_PATH, address.adapter_id, address
+            ),
+        }
+    }
+}
+
+#[cfg(target_arch = "s390x")]
+impl UeventMatcher for ApMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.action == "add" && uev.devpath == self.syspath
+    }
+}
+
+#[derive(Debug)]
+pub struct PciMatcher {
+    devpath: String,
+}
+
+impl PciMatcher {
+    pub fn new(relpath: &str) -> Result<PciMatcher> {
+        let root_bus = create_pci_root_bus_path();
+        Ok(PciMatcher {
+            devpath: format!("{}{}", root_bus, relpath),
+        })
+    }
+}
+
+impl UeventMatcher for PciMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.devpath == self.devpath
+    }
+}
+
+#[cfg(target_arch = "s390x")]
+#[instrument]
+async fn wait_for_ap_device(sandbox: &Arc<Mutex<Sandbox>>, address: ap::Address) -> Result<()> {
+    let matcher = ApMatcher::new(address);
+    wait_for_uevent(sandbox, matcher).await?;
+    Ok(())
+}
+
+pub async fn wait_for_pci_device(
+    sandbox: &Arc<Mutex<Sandbox>>,
+    pcipath: &pci::Path,
+) -> Result<pci::Address> {
+    let root_bus_sysfs = format!("{}{}", SYSFS_DIR, create_pci_root_bus_path());
+    let sysfs_rel_path = pcipath_to_sysfs(&root_bus_sysfs, pcipath)?;
+    let matcher = PciMatcher::new(&sysfs_rel_path)?;
+
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+
+    let addr = uev
+        .devpath
+        .rsplit('/')
+        .next()
+        .ok_or_else(|| anyhow!("Bad device path {:?} in uevent", &uev.devpath))?;
+    let addr = pci::Address::from_str(addr)?;
+    Ok(addr)
+}
+
+// Represents an IOMMU group
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct IommuGroup(u32);
+
+impl fmt::Display for IommuGroup {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        write!(f, "{}", self.0)
+    }
+}
+
+// Determine the IOMMU group of a PCI device
+#[instrument]
+fn pci_iommu_group<T>(syspci: T, dev: pci::Address) -> Result<Option<IommuGroup>>
+where
+    T: AsRef<OsStr> + std::fmt::Debug,
+{
+    let syspci = Path::new(&syspci);
+    let grouppath = syspci
+        .join("devices")
+        .join(dev.to_string())
+        .join("iommu_group");
+
+    match fs::read_link(&grouppath) {
+        // Device has no group
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
+        Err(e) => Err(anyhow!("Error reading link {:?}: {}", &grouppath, e)),
+        Ok(group) => {
+            if let Some(group) = group.file_name() {
+                if let Some(group) = group.to_str() {
+                    if let Ok(group) = group.parse::<u32>() {
+                        return Ok(Some(IommuGroup(group)));
+                    }
+                }
+            }
+            Err(anyhow!(
+                "Unexpected IOMMU group link {:?} => {:?}",
+                grouppath,
+                group
+            ))
+        }
+    }
+}
+
+fn split_vfio_pci_option(opt: &str) -> Option<(&str, &str)> {
+    let mut tokens = opt.split('=');
+    let hostbdf = tokens.next()?;
+    let path = tokens.next()?;
+    if tokens.next().is_some() {
+        None
+    } else {
+        Some((hostbdf, path))
+    }
+}
+
+// Force a given PCI device to bind to the given driver, does
+// basically the same thing as
+//    driverctl set-override <PCI address> <driver>
+#[instrument]
+pub fn pci_driver_override<T, U>(
+    logger: &Logger,
+    syspci: T,
+    dev: pci::Address,
+    drv: U,
+) -> Result<()>
+where
+    T: AsRef<OsStr> + std::fmt::Debug,
+    U: AsRef<OsStr> + std::fmt::Debug,
+{
+    let syspci = Path::new(&syspci);
+    let drv = drv.as_ref();
+    info!(logger, "rebind_pci_driver: {} => {:?}", dev, drv);
+
+    let devpath = syspci.join("devices").join(dev.to_string());
+    let overridepath = &devpath.join("driver_override");
+
+    fs::write(overridepath, drv.as_bytes())?;
+
+    let drvpath = &devpath.join("driver");
+    let need_unbind = match fs::read_link(drvpath) {
+        Ok(d) if d.file_name() == Some(drv) => return Ok(()), // Nothing to do
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, // No current driver
+        Err(e) => return Err(anyhow!("Error checking driver on {}: {}", dev, e)),
+        Ok(_) => true, // Current driver needs unbinding
+    };
+    if need_unbind {
+        let unbindpath = &drvpath.join("unbind");
+        fs::write(unbindpath, dev.to_string())?;
+    }
+    let probepath = syspci.join("drivers_probe");
+    fs::write(probepath, dev.to_string())?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::tempdir;
+
+    #[tokio::test]
+    #[allow(clippy::redundant_clone)]
+    async fn test_vfio_matcher() {
+        let grpa = IommuGroup(1);
+        let grpb = IommuGroup(22);
+
+        let mut uev_a = crate::uevent::Uevent::default();
+        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev_a.devname = format!("vfio/{}", grpa);
+        uev_a.devpath = format!("/devices/virtual/vfio/{}", grpa);
+        let matcher_a = VfioMatcher::new(grpa);
+
+        let mut uev_b = uev_a.clone();
+        uev_b.devpath = format!("/devices/virtual/vfio/{}", grpb);
+        let matcher_b = VfioMatcher::new(grpb);
+
+        assert!(matcher_a.is_match(&uev_a));
+        assert!(matcher_b.is_match(&uev_b));
+        assert!(!matcher_b.is_match(&uev_a));
+        assert!(!matcher_a.is_match(&uev_b));
+    }
+    #[test]
+    fn test_split_vfio_pci_option() {
+        assert_eq!(
+            split_vfio_pci_option("0000:01:00.0=02/01"),
+            Some(("0000:01:00.0", "02/01"))
+        );
+        assert_eq!(split_vfio_pci_option("0000:01:00.0=02/01=rubbish"), None);
+        assert_eq!(split_vfio_pci_option("0000:01:00.0"), None);
+    }
+
+    #[test]
+    fn test_pci_driver_override() {
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let testdir = tempdir().expect("failed to create tmpdir");
+        let syspci = testdir.path(); // Path to mock /sys/bus/pci
+
+        let dev0 = pci::Address::new(0, 0, pci::SlotFn::new(0, 0).unwrap());
+        let dev0path = syspci.join("devices").join(dev0.to_string());
+        let dev0drv = dev0path.join("driver");
+        let dev0override = dev0path.join("driver_override");
+
+        let drvapath = syspci.join("drivers").join("drv_a");
+        let drvaunbind = drvapath.join("unbind");
+
+        let probepath = syspci.join("drivers_probe");
+
+        // Start mocking dev0 as being unbound
+        fs::create_dir_all(&dev0path).unwrap();
+
+        pci_driver_override(&logger, syspci, dev0, "drv_a").unwrap();
+        assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_a");
+        assert_eq!(fs::read_to_string(&probepath).unwrap(), dev0.to_string());
+
+        // Now mock dev0 already being attached to drv_a
+        fs::create_dir_all(&drvapath).unwrap();
+        std::os::unix::fs::symlink(&drvapath, dev0drv).unwrap();
+        std::fs::remove_file(&probepath).unwrap();
+
+        pci_driver_override(&logger, syspci, dev0, "drv_a").unwrap(); // no-op
+        assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_a");
+        assert!(!probepath.exists());
+
+        // Now try binding to a different driver
+        pci_driver_override(&logger, syspci, dev0, "drv_b").unwrap();
+        assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_b");
+        assert_eq!(fs::read_to_string(&probepath).unwrap(), dev0.to_string());
+        assert_eq!(fs::read_to_string(drvaunbind).unwrap(), dev0.to_string());
+    }
+
+    #[test]
+    fn test_pci_iommu_group() {
+        let testdir = tempdir().expect("failed to create tmpdir"); // mock /sys
+        let syspci = testdir.path().join("bus").join("pci");
+
+        // Mock dev0, which has no group
+        let dev0 = pci::Address::new(0, 0, pci::SlotFn::new(0, 0).unwrap());
+        let dev0path = syspci.join("devices").join(dev0.to_string());
+
+        fs::create_dir_all(dev0path).unwrap();
+
+        // Test dev0
+        assert!(pci_iommu_group(&syspci, dev0).unwrap().is_none());
+
+        // Mock dev1, which is in group 12
+        let dev1 = pci::Address::new(0, 1, pci::SlotFn::new(0, 0).unwrap());
+        let dev1path = syspci.join("devices").join(dev1.to_string());
+        let dev1group = dev1path.join("iommu_group");
+
+        fs::create_dir_all(&dev1path).unwrap();
+        std::os::unix::fs::symlink("../../../kernel/iommu_groups/12", dev1group).unwrap();
+
+        // Test dev1
+        assert_eq!(
+            pci_iommu_group(&syspci, dev1).unwrap(),
+            Some(IommuGroup(12))
+        );
+
+        // Mock dev2, which has a bogus group (dir instead of symlink)
+        let dev2 = pci::Address::new(0, 2, pci::SlotFn::new(0, 0).unwrap());
+        let dev2path = syspci.join("devices").join(dev2.to_string());
+        let dev2group = dev2path.join("iommu_group");
+
+        fs::create_dir_all(dev2group).unwrap();
+
+        // Test dev2
+        assert!(pci_iommu_group(&syspci, dev2).is_err());
+    }
+
+    #[cfg(target_arch = "s390x")]
+    #[tokio::test]
+    async fn test_vfio_ap_matcher() {
+        let subsystem = "ap";
+        let card = "0a";
+        let relpath = format!("{}.0001", card);
+
+        let mut uev = Uevent::default();
+        uev.action = U_EVENT_ACTION_ADD.to_string();
+        uev.subsystem = subsystem.to_string();
+        uev.devpath = format!("{}/card{}/{}", AP_ROOT_BUS_PATH, card, relpath);
+
+        let ap_address = ap::Address::from_str(&relpath).unwrap();
+        let matcher = ApMatcher::new(ap_address);
+
+        assert!(matcher.is_match(&uev));
+
+        let mut uev_remove = uev.clone();
+        uev_remove.action = U_EVENT_ACTION_REMOVE.to_string();
+        assert!(!matcher.is_match(&uev_remove));
+
+        let mut uev_other_device = uev.clone();
+        uev_other_device.devpath = format!("{}/card{}/{}.0002", AP_ROOT_BUS_PATH, card, card);
+        assert!(!matcher.is_match(&uev_other_device));
+    }
+}

src/agent/src/rpc.rs

@@ -56,9 +56,9 @@ use nix::unistd::{self, Pid};
 use rustjail::process::ProcessOperations;
 
 use crate::cdh;
-use crate::device::{
-    add_devices, get_virtio_blk_pci_device_name, update_env_pci, wait_for_net_interface,
-};
+use crate::device::block_device_handler::get_virtio_blk_pci_device_name;
+use crate::device::network_device_handler::wait_for_net_interface;
+use crate::device::{add_devices, update_env_pci};
 use crate::features::get_build_features;
 use crate::image::KATA_IMAGE_WORK_DIR;
 use crate::linux_abi::*;
@@ -222,7 +222,7 @@ impl AgentService {
         // updates the devices listed in the OCI spec, so that they actually
         // match real devices inside the VM. This step is necessary since we
         // cannot predict everything from the caller.
-        add_devices(&req.devices, &mut oci, &self.sandbox).await?;
+        add_devices(&sl(), &req.devices, &mut oci, &self.sandbox).await?;
 
         let process = oci
             .process_mut()

src/agent/src/storage/bind_watcher_handler.rs

@@ -4,9 +4,9 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use crate::device::DRIVER_WATCHABLE_BIND_TYPE;
 use crate::storage::{new_device, StorageContext, StorageHandler};
 use anyhow::Result;
+use kata_types::device::DRIVER_WATCHABLE_BIND_TYPE;
 use kata_types::mount::StorageDevice;
 use protocols::agent::Storage;
 use std::iter;

src/agent/src/storage/block_handler.rs

@@ -10,23 +10,26 @@ use std::path::Path;
 use std::str::FromStr;
 use std::sync::Arc;
 
-use crate::device::{
+use anyhow::{anyhow, Context, Result};
+use kata_types::device::{
     DRIVER_BLK_CCW_TYPE, DRIVER_BLK_MMIO_TYPE, DRIVER_BLK_PCI_TYPE, DRIVER_NVDIMM_TYPE,
     DRIVER_SCSI_TYPE,
 };
-use anyhow::{anyhow, Context, Result};
 use kata_types::mount::StorageDevice;
 use protocols::agent::Storage;
 use tracing::instrument;
 
-use crate::device::{
-    get_scsi_device_name, get_virtio_blk_pci_device_name, get_virtio_mmio_device_name,
-    wait_for_pmem_device,
+#[cfg(target_arch = "s390x")]
+use crate::ccw;
+#[cfg(target_arch = "s390x")]
+use crate::device::block_device_handler::get_virtio_blk_ccw_device_name;
+use crate::device::block_device_handler::{
+    get_virtio_blk_mmio_device_name, get_virtio_blk_pci_device_name,
 };
+use crate::device::nvdimm_device_handler::wait_for_pmem_device;
+use crate::device::scsi_device_handler::get_scsi_device_name;
 use crate::pci;
 use crate::storage::{common_storage_handler, new_device, StorageContext, StorageHandler};
-#[cfg(target_arch = "s390x")]
-use crate::{ccw, device::get_virtio_blk_ccw_device_name};
 
 #[derive(Debug)]
 pub struct VirtioBlkMmioHandler {}
@@ -45,7 +48,7 @@ impl StorageHandler for VirtioBlkMmioHandler {
         ctx: &mut StorageContext,
     ) -> Result<Arc<dyn StorageDevice>> {
         if !Path::new(&storage.source).exists() {
-            get_virtio_mmio_device_name(ctx.sandbox, &storage.source)
+            get_virtio_blk_mmio_device_name(ctx.sandbox, &storage.source)
                 .await
                 .context("failed to get mmio device name")?;
         }

src/agent/src/storage/ephemeral_handler.rs

@@ -20,13 +20,14 @@ use slog::Logger;
 use tokio::sync::Mutex;
 use tracing::instrument;
 
-use crate::device::{DRIVER_EPHEMERAL_TYPE, FS_TYPE_HUGETLB};
 use crate::mount::baremount;
 use crate::sandbox::Sandbox;
 use crate::storage::{
     common_storage_handler, new_device, parse_options, StorageContext, StorageHandler, MODE_SETGID,
 };
+use kata_types::device::DRIVER_EPHEMERAL_TYPE;
 
+const FS_TYPE_HUGETLB: &str = "hugetlbfs";
 const FS_GID_EQ: &str = "fsgid=";
 const SYS_FS_HUGEPAGES_PREFIX: &str = "/sys/kernel/mm/hugepages";
 

src/agent/src/storage/fs_handler.rs

@@ -8,9 +8,9 @@ use std::fs;
 use std::path::Path;
 use std::sync::Arc;
 
-use crate::device::{DRIVER_9P_TYPE, DRIVER_OVERLAYFS_TYPE, DRIVER_VIRTIOFS_TYPE};
 use crate::storage::{common_storage_handler, new_device, StorageContext, StorageHandler};
 use anyhow::{anyhow, Context, Result};
+use kata_types::device::{DRIVER_9P_TYPE, DRIVER_OVERLAYFS_TYPE, DRIVER_VIRTIOFS_TYPE};
 use kata_types::mount::StorageDevice;
 use protocols::agent::Storage;
 use tracing::instrument;

src/agent/src/storage/local_handler.rs

@@ -8,15 +8,14 @@ use std::fs;
 use std::os::unix::fs::PermissionsExt;
 use std::sync::Arc;
 
-use crate::device::DRIVER_LOCAL_TYPE;
+use crate::storage::{new_device, parse_options, StorageContext, StorageHandler, MODE_SETGID};
 use anyhow::{Context, Result};
+use kata_types::device::DRIVER_LOCAL_TYPE;
 use kata_types::mount::{StorageDevice, KATA_MOUNT_OPTION_FS_GID};
 use nix::unistd::Gid;
 use protocols::agent::Storage;
 use tracing::instrument;
 
-use crate::storage::{new_device, parse_options, StorageContext, StorageHandler, MODE_SETGID};
-
 #[derive(Debug)]
 pub struct LocalHandler {}
 

src/libs/kata-types/src/device.rs

@@ -0,0 +1,42 @@
+// Copyright (c) 2024 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use crate::handler::HandlerManager;
+
+/// DRIVER_BLK_PCI_TYPE is the device driver for virtio-blk
+pub const DRIVER_BLK_PCI_TYPE: &str = "blk";
+/// DRIVER_BLK_CCW_TYPE is the device driver for virtio-blk-ccw
+pub const DRIVER_BLK_CCW_TYPE: &str = "blk-ccw";
+/// DRIVER_BLK_MMIO_TYPE is the device driver for virtio-mmio
+pub const DRIVER_BLK_MMIO_TYPE: &str = "mmioblk";
+/// DRIVER_SCSI_TYPE is the device driver for virtio-scsi
+pub const DRIVER_SCSI_TYPE: &str = "scsi";
+/// DRIVER_NVDIMM_TYPE is the device driver for nvdimm
+pub const DRIVER_NVDIMM_TYPE: &str = "nvdimm";
+/// DRIVER_VFIO_PCI_GK_TYPE is the device driver for vfio-pci 
+/// while the device will be bound to a guest kernel driver
+pub const DRIVER_VFIO_PCI_GK_TYPE: &str = "vfio-pci-gk";
+/// DRIVER_VFIO_PCI_TYPE is the device driver for vfio-pci
+/// VFIO PCI device to be bound to vfio-pci and made available inside the
+/// container as a VFIO device node
+pub const DRIVER_VFIO_PCI_TYPE: &str = "vfio-pci";
+/// DRIVER_VFIO_AP_TYPE is the device driver for vfio-ap.
+pub const DRIVER_VFIO_AP_TYPE: &str = "vfio-ap";
+
+/// DRIVER_9P_TYPE is the driver for 9pfs volume.
+pub const DRIVER_9P_TYPE: &str = "9p";
+/// DRIVER_EPHEMERAL_TYPE is the driver for ephemeral volume.
+pub const DRIVER_EPHEMERAL_TYPE: &str = "ephemeral";
+/// DRIVER_LOCAL_TYPE is the driver for local volume.
+pub const DRIVER_LOCAL_TYPE: &str = "local";
+/// DRIVER_OVERLAYFS_TYPE is the driver for overlayfs volume.
+pub const DRIVER_OVERLAYFS_TYPE: &str = "overlayfs";
+/// DRIVER_VIRTIOFS_TYPE is the driver for virtio-fs volume.
+pub const DRIVER_VIRTIOFS_TYPE: &str = "virtio-fs";
+/// DRIVER_VIRTIOFS_TYPE is the driver for Bind watch volume.
+pub const DRIVER_WATCHABLE_BIND_TYPE: &str = "watchable-bind";
+
+/// Manager to manage registered device handlers.
+pub type DeviceHandlerManager<H> = HandlerManager<H>;

src/libs/kata-types/src/handler.rs

@@ -0,0 +1,59 @@
+// Copyright (c) 2024 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::collections::HashMap;
+use anyhow::{Result, anyhow};
+use std::collections::hash_map::Entry;
+
+/// Generic manager to manage registered handlers.
+pub struct HandlerManager<H> {
+    handlers: HashMap<String, H>,
+}
+
+impl<H> Default for HandlerManager<H> 
+where
+    H: Clone,
+{
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<H> HandlerManager<H> 
+where
+    H: Clone,
+{
+    /// Create a new instance of `HandlerManager`.
+    pub fn new() -> Self {
+        Self {
+            handlers: HashMap::new(),
+        }
+    }
+
+    /// Register a handler.
+    pub fn add_handler(&mut self, ids: &[&str], handler: H) -> Result<()> {
+        for &id in ids {
+            match self.handlers.entry(id.to_string()) {
+                Entry::Occupied(_) => {
+                    return Err(anyhow!("handler for {} already exists", id));
+                }
+                Entry::Vacant(entry) => {
+                    entry.insert(handler.clone());
+                }
+            }
+        }
+        Ok(())
+    }
+
+    /// Get handler with specified `id`.
+    pub fn handler(&self, id: &str) -> Option<&H> {
+        self.handlers.get(id)
+    }
+
+    /// Get names of registered handlers.
+    pub fn get_handlers(&self) -> Vec<String> {
+        self.handlers.keys().map(|v| v.to_string()).collect()
+    }
+}

src/libs/kata-types/src/lib.rs

@@ -23,6 +23,12 @@ pub mod container;
 /// Constants and data types related to CPU.
 pub mod cpu;
 
+/// Contants and data types related to device.
+pub mod device;
+
+/// Constants and data types related to handler.
+pub mod handler;
+
 /// Constants and data types related to Kubernetes/kubelet.
 pub mod k8s;
 

src/libs/kata-types/src/mount.rs

@@ -5,10 +5,11 @@
 //
 
 use anyhow::{anyhow, Context, Error, Result};
-use std::collections::hash_map::Entry;
 use std::convert::TryFrom;
 use std::{collections::HashMap, fs, path::PathBuf};
 
+use crate::handler::HandlerManager;
+
 /// Prefix to mark a volume as Kata special.
 pub const KATA_VOLUME_TYPE_PREFIX: &str = "kata:";
 
@@ -59,6 +60,9 @@ pub const KATA_VIRTUAL_VOLUME_LAYER_NYDUS_FS: &str = "layer_nydus_fs";
 /// Download and extra container image inside guest vm.
 pub const KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL: &str = "image_guest_pull";
 
+/// Manager to manage registered storage device handlers.
+pub type StorageHandlerManager<H> = HandlerManager<H>;
+
 /// Information about a mount.
 #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
 pub struct Mount {
@@ -441,57 +445,6 @@ pub trait StorageDevice: Send + Sync {
     fn cleanup(&self) -> Result<()>;
 }
 
-/// Manager to manage registered storage device handlers.
-pub struct StorageHandlerManager<H> {
-    handlers: HashMap<String, H>,
-}
-
-impl<H> Default for StorageHandlerManager<H> 
-where
-    H: Clone,
-{
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl<H> StorageHandlerManager<H> 
-where
-    H: Clone,
-{
-    /// Create a new instance of `StorageHandlerManager`.
-    pub fn new() -> Self {
-        Self {
-            handlers: HashMap::new(),
-        }
-    }
-
-    /// Register a storage device handler.
-    pub fn add_handler(&mut self, ids: &[&str], handler: H) -> Result<()> {
-        for &id in ids {
-            match self.handlers.entry(id.to_string()) {
-                Entry::Occupied(_) => {
-                    return Err(anyhow!("storage handler for {} already exists", id));
-                }
-                Entry::Vacant(entry) => {
-                    entry.insert(handler.clone());
-                }
-            }
-        }
-        Ok(())
-    }
-
-    /// Get storage handler with specified `id`.
-    pub fn handler(&self, id: &str) -> Option<&H> {
-        self.handlers.get(id)
-    }
-
-    /// Get names of registered handlers.
-    pub fn get_handlers(&self) -> Vec<String> {
-        self.handlers.keys().map(|v| v.to_string()).collect()
-    }
-}
-
 /// Join user provided volume path with kata direct-volume root path.
 ///
 /// The `volume_path` is base64-url-encoded and then safely joined to the `prefix`