agent: introduce DmVerityHandler to support dm-verity volume

We utilize the KataVirtualVolume which storing the dm-verity info
and the path of disk image on the host supplied by snapshotter as an integral part of `CreateContainer`.
Within this process, we copy the verity info and the disk image path to mount slice to create a block device by virtio-blk.
Then storing the `lowerdir` in rootfs.storage which is the mountpoint of the verity path through `CreateContainerRequest`.
To maintain clarity and avoid any need for modification to the `VirtioBlkPciHandler`,we introduce the `DmVerityHandler`.
This dedicated handler is responsible for calling image-rs to create verity device and mount the device to the `lowerdir` within the guest environment.

Signed-off-by: Jiang Liu <gerry@linux.alibaba.com>
Signed-off-by: ChengyuZhu6 <chengyu.zhu@intel.com>

src/agent/src/storage/block_handler.rs

@@ -27,19 +27,29 @@ use crate::{ccw, device::get_virtio_blk_ccw_device_name};
 #[derive(Debug)]
 pub struct VirtioBlkMmioHandler {}
 
-#[async_trait::async_trait]
+impl VirtioBlkMmioHandler {
-impl StorageHandler for VirtioBlkMmioHandler {
+    pub async fn update_device_path(
-    #[instrument]
+        storage: &mut Storage,
-    async fn create_device(
+        ctx: &mut StorageContext<'_>,
-        &self,
+    ) -> Result<()> {
-        storage: Storage,
-        ctx: &mut StorageContext,
-    ) -> Result<Arc<dyn StorageDevice>> {
         if !Path::new(&storage.source).exists() {
             get_virtio_mmio_device_name(ctx.sandbox, &storage.source)
                 .await
                 .context("failed to get mmio device name")?;
         }
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+impl StorageHandler for VirtioBlkMmioHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        Self::update_device_path(&mut storage, ctx).await?;
         let path = common_storage_handler(ctx.logger, &storage)?;
         new_device(path)
     }
@@ -48,14 +58,11 @@ impl StorageHandler for VirtioBlkMmioHandler {
 #[derive(Debug)]
 pub struct VirtioBlkPciHandler {}
 
-#[async_trait::async_trait]
+impl VirtioBlkPciHandler {
-impl StorageHandler for VirtioBlkPciHandler {
+    pub async fn update_device_path(
-    #[instrument]
+        storage: &mut Storage,
-    async fn create_device(
+        ctx: &mut StorageContext<'_>,
-        &self,
+    ) -> Result<()> {
-        mut storage: Storage,
-        ctx: &mut StorageContext,
-    ) -> Result<Arc<dyn StorageDevice>> {
         // If hot-plugged, get the device node path based on the PCI path
         // otherwise use the virt path provided in Storage Source
         if storage.source.starts_with("/dev") {
@@ -71,6 +78,19 @@ impl StorageHandler for VirtioBlkPciHandler {
             storage.source = dev_path;
         }
 
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+impl StorageHandler for VirtioBlkPciHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        Self::update_device_path(&mut storage, ctx).await?;
         let path = common_storage_handler(ctx.logger, &storage)?;
         new_device(path)
     }
@@ -79,6 +99,21 @@ impl StorageHandler for VirtioBlkPciHandler {
 #[derive(Debug)]
 pub struct VirtioBlkCcwHandler {}
 
+impl VirtioBlkCcwHandler {
+    pub async fn update_device_path(
+        _storage: &mut Storage,
+        _ctx: &mut StorageContext<'_>,
+    ) -> Result<()> {
+        #[cfg(target_arch = "s390x")]
+        {
+            let ccw_device = ccw::Device::from_str(&_storage.source)?;
+            let dev_path = get_virtio_blk_ccw_device_name(_ctx.sandbox, &ccw_device).await?;
+            _storage.source = dev_path;
+        }
+        Ok(())
+    }
+}
+
 #[async_trait::async_trait]
 impl StorageHandler for VirtioBlkCcwHandler {
     #[cfg(target_arch = "s390x")]
@@ -88,9 +123,7 @@ impl StorageHandler for VirtioBlkCcwHandler {
         mut storage: Storage,
         ctx: &mut StorageContext,
     ) -> Result<Arc<dyn StorageDevice>> {
-        let ccw_device = ccw::Device::from_str(&storage.source)?;
+        Self::update_device_path(&mut storage, ctx).await?;
-        let dev_path = get_virtio_blk_ccw_device_name(ctx.sandbox, &ccw_device).await?;
-        storage.source = dev_path;
         let path = common_storage_handler(ctx.logger, &storage)?;
         new_device(path)
     }
@@ -109,6 +142,18 @@ impl StorageHandler for VirtioBlkCcwHandler {
 #[derive(Debug)]
 pub struct ScsiHandler {}
 
+impl ScsiHandler {
+    pub async fn update_device_path(
+        storage: &mut Storage,
+        ctx: &mut StorageContext<'_>,
+    ) -> Result<()> {
+        // Retrieve the device path from SCSI address.
+        let dev_path = get_scsi_device_name(ctx.sandbox, &storage.source).await?;
+        storage.source = dev_path;
+        Ok(())
+    }
+}
+
 #[async_trait::async_trait]
 impl StorageHandler for ScsiHandler {
     #[instrument]
@@ -117,10 +162,7 @@ impl StorageHandler for ScsiHandler {
         mut storage: Storage,
         ctx: &mut StorageContext,
     ) -> Result<Arc<dyn StorageDevice>> {
-        // Retrieve the device path from SCSI address.
+        Self::update_device_path(&mut storage, ctx).await?;
-        let dev_path = get_scsi_device_name(ctx.sandbox, &storage.source).await?;
-        storage.source = dev_path;
-
         let path = common_storage_handler(ctx.logger, &storage)?;
         new_device(path)
     }
@@ -129,17 +171,26 @@ impl StorageHandler for ScsiHandler {
 #[derive(Debug)]
 pub struct PmemHandler {}
 
+impl PmemHandler {
+    pub async fn update_device_path(
+        storage: &mut Storage,
+        ctx: &mut StorageContext<'_>,
+    ) -> Result<()> {
+        // Retrieve the device for pmem storage
+        wait_for_pmem_device(ctx.sandbox, &storage.source).await?;
+        Ok(())
+    }
+}
+
 #[async_trait::async_trait]
 impl StorageHandler for PmemHandler {
     #[instrument]
     async fn create_device(
         &self,
-        storage: Storage,
+        mut storage: Storage,
         ctx: &mut StorageContext,
     ) -> Result<Arc<dyn StorageDevice>> {
-        // Retrieve the device for pmem storage
+        Self::update_device_path(&mut storage, ctx).await?;
-        wait_for_pmem_device(ctx.sandbox, &storage.source).await?;
-
         let path = common_storage_handler(ctx.logger, &storage)?;
         new_device(path)
     }

src/agent/src/storage/dm_verity.rs

@@ -0,0 +1,165 @@
+// Copyright (c) 2023 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::path::Path;
+use std::sync::Arc;
+
+use anyhow::{anyhow, Context, Result};
+use image_rs::verity::{create_dmverity_device, destroy_dmverity_device};
+use kata_sys_util::mount::create_mount_destination;
+use kata_types::mount::{DmVerityInfo, StorageDevice};
+use kata_types::volume::{
+    KATA_VOLUME_DMVERITY_OPTION_SOURCE_TYPE, KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO,
+    KATA_VOLUME_DMVERITY_SOURCE_TYPE_PMEM, KATA_VOLUME_DMVERITY_SOURCE_TYPE_SCSI,
+    KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_CCW, KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_MMIO,
+    KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_PCI,
+};
+use protocols::agent::Storage;
+use slog::Logger;
+use tracing::instrument;
+
+use crate::storage::block_handler::{
+    PmemHandler, ScsiHandler, VirtioBlkCcwHandler, VirtioBlkMmioHandler, VirtioBlkPciHandler,
+};
+use crate::storage::{common_storage_handler, StorageContext, StorageHandler};
+
+use super::StorageDeviceGeneric;
+
+#[derive(Debug)]
+pub struct DmVerityHandler {}
+
+impl DmVerityHandler {
+    fn get_dm_verity_info(storage: &Storage) -> Result<DmVerityInfo> {
+        for option in storage.driver_options.iter() {
+            if let Some((key, value)) = option.split_once('=') {
+                if key == KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO {
+                    let verity_info: DmVerityInfo = serde_json::from_str(value)?;
+                    return Ok(verity_info);
+                }
+            }
+        }
+
+        Err(anyhow!("missing DmVerity information for DmVerity volume"))
+    }
+
+    async fn update_source_device(
+        storage: &mut Storage,
+        ctx: &mut StorageContext<'_>,
+    ) -> Result<()> {
+        for option in storage.driver_options.clone() {
+            if let Some((key, value)) = option.split_once('=') {
+                if key == KATA_VOLUME_DMVERITY_OPTION_SOURCE_TYPE {
+                    match value {
+                        KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_PCI => {
+                            VirtioBlkPciHandler::update_device_path(storage, ctx).await?;
+                        }
+                        KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_MMIO => {
+                            VirtioBlkMmioHandler::update_device_path(storage, ctx).await?;
+                        }
+                        KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_CCW => {
+                            VirtioBlkCcwHandler::update_device_path(storage, ctx).await?;
+                        }
+                        KATA_VOLUME_DMVERITY_SOURCE_TYPE_SCSI => {
+                            ScsiHandler::update_device_path(storage, ctx).await?;
+                        }
+                        KATA_VOLUME_DMVERITY_SOURCE_TYPE_PMEM => {
+                            PmemHandler::update_device_path(storage, ctx).await?;
+                        }
+                        _ => {}
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+impl StorageHandler for DmVerityHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        Self::update_source_device(&mut storage, ctx).await?;
+        create_mount_destination(&storage.source, &storage.mount_point, "", &storage.fstype)
+            .context("Could not create mountpoint")?;
+
+        let verity_info = Self::get_dm_verity_info(&storage)?;
+        let verity_info = serde_json::to_string(&verity_info)
+            .map_err(|e| anyhow!("failed to serialize dm_verity info, {}", e))?;
+        let verity_device_path = create_dmverity_device(&verity_info, Path::new(storage.source()))
+            .context("create device with dm-verity enabled")?;
+        storage.source = verity_device_path;
+        common_storage_handler(ctx.logger, &storage)?;
+
+        Ok(Arc::new(DmVerityDevice {
+            common: StorageDeviceGeneric::new(storage.mount_point),
+            verity_device_path: storage.source,
+            logger: ctx.logger.clone(),
+        }))
+    }
+}
+
+struct DmVerityDevice {
+    common: StorageDeviceGeneric,
+    verity_device_path: String,
+    logger: Logger,
+}
+
+impl StorageDevice for DmVerityDevice {
+    fn path(&self) -> Option<&str> {
+        self.common.path()
+    }
+
+    fn cleanup(&self) -> Result<()> {
+        self.common.cleanup().context("clean up dm-verity volume")?;
+        let device_path = &self.verity_device_path;
+        debug!(
+            self.logger,
+            "destroy verity device path = {:?}", device_path
+        );
+        destroy_dmverity_device(device_path)?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use kata_types::{mount::DmVerityInfo, volume::KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO};
+    use protocols::agent::Storage;
+
+    use crate::storage::dm_verity::DmVerityHandler;
+
+    #[test]
+    fn test_get_dm_verity_info() {
+        let verity_info = DmVerityInfo {
+            hashtype: "sha256".to_string(),
+            hash: "d86104eee715a1b59b62148641f4ca73edf1be3006c4d481f03f55ac05640570".to_string(),
+            blocknum: 2361,
+            blocksize: 512,
+            hashsize: 4096,
+            offset: 1212416,
+        };
+
+        let verity_info_str = serde_json::to_string(&verity_info);
+        assert!(verity_info_str.is_ok());
+
+        let storage = Storage {
+            driver: KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO.to_string(),
+            driver_options: vec![format!("verity_info={}", verity_info_str.ok().unwrap())],
+            ..Default::default()
+        };
+
+        match DmVerityHandler::get_dm_verity_info(&storage) {
+            Ok(result) => {
+                assert_eq!(verity_info, result);
+            }
+            Err(e) => panic!("err = {}", e),
+        }
+    }
+}

src/agent/src/storage/mod.rs

@@ -13,6 +13,7 @@ use std::sync::Arc;
 use anyhow::{anyhow, Context, Result};
 use kata_sys_util::mount::{create_mount_destination, parse_mount_options};
 use kata_types::mount::{StorageDevice, StorageHandlerManager, KATA_SHAREDFS_GUEST_PREMOUNT_TAG};
+use kata_types::volume::KATA_VOLUME_TYPE_DMVERITY;
 use nix::unistd::{Gid, Uid};
 use protocols::agent::Storage;
 use protocols::types::FSGroupChangePolicy;
@@ -22,6 +23,7 @@ use tracing::instrument;
 
 use self::bind_watcher_handler::BindWatcherHandler;
 use self::block_handler::{PmemHandler, ScsiHandler, VirtioBlkMmioHandler, VirtioBlkPciHandler};
+use self::dm_verity::DmVerityHandler;
 use self::ephemeral_handler::EphemeralHandler;
 use self::fs_handler::{OverlayfsHandler, Virtio9pHandler, VirtioFsHandler};
 use self::local_handler::LocalHandler;
@@ -37,6 +39,7 @@ pub use self::ephemeral_handler::update_ephemeral_mounts;
 
 mod bind_watcher_handler;
 mod block_handler;
+mod dm_verity;
 mod ephemeral_handler;
 mod fs_handler;
 mod local_handler;
@@ -145,6 +148,7 @@ lazy_static! {
         manager.add_handler(DRIVER_SCSI_TYPE, Arc::new(ScsiHandler{})).unwrap();
         manager.add_handler(DRIVER_VIRTIOFS_TYPE, Arc::new(VirtioFsHandler{})).unwrap();
         manager.add_handler(DRIVER_WATCHABLE_BIND_TYPE, Arc::new(BindWatcherHandler{})).unwrap();
+        manager.add_handler(KATA_VOLUME_TYPE_DMVERITY, Arc::new(DmVerityHandler{})).unwrap();
         manager
     };
 }

src/libs/kata-types/src/lib.rs

@@ -29,6 +29,9 @@ pub mod k8s;
 /// Constants and data types related to mount point.
 pub mod mount;
 
+/// Constants and data types related to data volumes.
+pub mod volume;
+
 pub(crate) mod utils;
 
 /// hypervisor capabilities

src/libs/kata-types/src/volume.rs

@@ -0,0 +1,22 @@
+// Copyright (c) 2023 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+/// Volume to support dm-verity over block devices.
+pub const KATA_VOLUME_TYPE_DMVERITY: &str = "dmverity";
+
+/// Key to identify dmverity information in `Storage.driver_options`
+pub const KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO: &str = "verity_info";
+/// Key to identify type of source device in `Storage.driver_options`
+pub const KATA_VOLUME_DMVERITY_OPTION_SOURCE_TYPE: &str = "source_type";
+/// Source device of dmverity volume is a Virtio PCI device
+pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_PCI: &str = "virtio_pci";
+/// Source device of dmverity volume is a Virtio MMIO device
+pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_MMIO: &str = "virtio_mmio";
+/// Source device of dmverity volume is a Virtio CCW device
+pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_CCW: &str = "virtio_ccw";
+/// Source device of dmverity volume is a SCSI disk.
+pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_SCSI: &str = "scsi";
+/// Source device of dmverity volume is a pmem disk.
+pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_PMEM: &str = "pmem";