From 72c9f62b706bc8476c6df17b3cc5c102a71a0a61 Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Fri, 1 Sep 2023 00:50:40 +0800 Subject: [PATCH] agent: introduce DmVerityHandler to support dm-verity volume We utilize the KataVirtualVolume which storing the dm-verity info and the path of disk image on the host supplied by snapshotter as an integral part of `CreateContainer`. Within this process, we copy the verity info and the disk image path to mount slice to create a block device by virtio-blk. Then storing the `lowerdir` in rootfs.storage which is the mountpoint of the verity path through `CreateContainerRequest`. To maintain clarity and avoid any need for modification to the `VirtioBlkPciHandler`,we introduce the `DmVerityHandler`. This dedicated handler is responsible for calling image-rs to create verity device and mount the device to the `lowerdir` within the guest environment. Signed-off-by: Jiang Liu Signed-off-by: ChengyuZhu6 --- src/agent/src/storage/block_handler.rs | 105 ++++++++++++---- src/agent/src/storage/dm_verity.rs | 165 +++++++++++++++++++++++++ src/agent/src/storage/mod.rs | 4 + src/libs/kata-types/src/lib.rs | 3 + src/libs/kata-types/src/volume.rs | 22 ++++ 5 files changed, 272 insertions(+), 27 deletions(-) create mode 100644 src/agent/src/storage/dm_verity.rs create mode 100644 src/libs/kata-types/src/volume.rs diff --git a/src/agent/src/storage/block_handler.rs b/src/agent/src/storage/block_handler.rs index 60330253ce..6af326a555 100644 --- a/src/agent/src/storage/block_handler.rs +++ b/src/agent/src/storage/block_handler.rs @@ -27,19 +27,29 @@ use crate::{ccw, device::get_virtio_blk_ccw_device_name}; #[derive(Debug)] pub struct VirtioBlkMmioHandler {} -#[async_trait::async_trait] -impl StorageHandler for VirtioBlkMmioHandler { - #[instrument] - async fn create_device( - &self, - storage: Storage, - ctx: &mut StorageContext, - ) -> Result> { +impl VirtioBlkMmioHandler { + pub async fn update_device_path( + storage: &mut Storage, + ctx: &mut StorageContext<'_>, + ) -> Result<()> { if !Path::new(&storage.source).exists() { get_virtio_mmio_device_name(ctx.sandbox, &storage.source) .await .context("failed to get mmio device name")?; } + Ok(()) + } +} + +#[async_trait::async_trait] +impl StorageHandler for VirtioBlkMmioHandler { + #[instrument] + async fn create_device( + &self, + mut storage: Storage, + ctx: &mut StorageContext, + ) -> Result> { + Self::update_device_path(&mut storage, ctx).await?; let path = common_storage_handler(ctx.logger, &storage)?; new_device(path) } @@ -48,14 +58,11 @@ impl StorageHandler for VirtioBlkMmioHandler { #[derive(Debug)] pub struct VirtioBlkPciHandler {} -#[async_trait::async_trait] -impl StorageHandler for VirtioBlkPciHandler { - #[instrument] - async fn create_device( - &self, - mut storage: Storage, - ctx: &mut StorageContext, - ) -> Result> { +impl VirtioBlkPciHandler { + pub async fn update_device_path( + storage: &mut Storage, + ctx: &mut StorageContext<'_>, + ) -> Result<()> { // If hot-plugged, get the device node path based on the PCI path // otherwise use the virt path provided in Storage Source if storage.source.starts_with("/dev") { @@ -71,6 +78,19 @@ impl StorageHandler for VirtioBlkPciHandler { storage.source = dev_path; } + Ok(()) + } +} + +#[async_trait::async_trait] +impl StorageHandler for VirtioBlkPciHandler { + #[instrument] + async fn create_device( + &self, + mut storage: Storage, + ctx: &mut StorageContext, + ) -> Result> { + Self::update_device_path(&mut storage, ctx).await?; let path = common_storage_handler(ctx.logger, &storage)?; new_device(path) } @@ -79,6 +99,21 @@ impl StorageHandler for VirtioBlkPciHandler { #[derive(Debug)] pub struct VirtioBlkCcwHandler {} +impl VirtioBlkCcwHandler { + pub async fn update_device_path( + _storage: &mut Storage, + _ctx: &mut StorageContext<'_>, + ) -> Result<()> { + #[cfg(target_arch = "s390x")] + { + let ccw_device = ccw::Device::from_str(&_storage.source)?; + let dev_path = get_virtio_blk_ccw_device_name(_ctx.sandbox, &ccw_device).await?; + _storage.source = dev_path; + } + Ok(()) + } +} + #[async_trait::async_trait] impl StorageHandler for VirtioBlkCcwHandler { #[cfg(target_arch = "s390x")] @@ -88,9 +123,7 @@ impl StorageHandler for VirtioBlkCcwHandler { mut storage: Storage, ctx: &mut StorageContext, ) -> Result> { - let ccw_device = ccw::Device::from_str(&storage.source)?; - let dev_path = get_virtio_blk_ccw_device_name(ctx.sandbox, &ccw_device).await?; - storage.source = dev_path; + Self::update_device_path(&mut storage, ctx).await?; let path = common_storage_handler(ctx.logger, &storage)?; new_device(path) } @@ -109,6 +142,18 @@ impl StorageHandler for VirtioBlkCcwHandler { #[derive(Debug)] pub struct ScsiHandler {} +impl ScsiHandler { + pub async fn update_device_path( + storage: &mut Storage, + ctx: &mut StorageContext<'_>, + ) -> Result<()> { + // Retrieve the device path from SCSI address. + let dev_path = get_scsi_device_name(ctx.sandbox, &storage.source).await?; + storage.source = dev_path; + Ok(()) + } +} + #[async_trait::async_trait] impl StorageHandler for ScsiHandler { #[instrument] @@ -117,10 +162,7 @@ impl StorageHandler for ScsiHandler { mut storage: Storage, ctx: &mut StorageContext, ) -> Result> { - // Retrieve the device path from SCSI address. - let dev_path = get_scsi_device_name(ctx.sandbox, &storage.source).await?; - storage.source = dev_path; - + Self::update_device_path(&mut storage, ctx).await?; let path = common_storage_handler(ctx.logger, &storage)?; new_device(path) } @@ -129,17 +171,26 @@ impl StorageHandler for ScsiHandler { #[derive(Debug)] pub struct PmemHandler {} +impl PmemHandler { + pub async fn update_device_path( + storage: &mut Storage, + ctx: &mut StorageContext<'_>, + ) -> Result<()> { + // Retrieve the device for pmem storage + wait_for_pmem_device(ctx.sandbox, &storage.source).await?; + Ok(()) + } +} + #[async_trait::async_trait] impl StorageHandler for PmemHandler { #[instrument] async fn create_device( &self, - storage: Storage, + mut storage: Storage, ctx: &mut StorageContext, ) -> Result> { - // Retrieve the device for pmem storage - wait_for_pmem_device(ctx.sandbox, &storage.source).await?; - + Self::update_device_path(&mut storage, ctx).await?; let path = common_storage_handler(ctx.logger, &storage)?; new_device(path) } diff --git a/src/agent/src/storage/dm_verity.rs b/src/agent/src/storage/dm_verity.rs new file mode 100644 index 0000000000..e177538fc4 --- /dev/null +++ b/src/agent/src/storage/dm_verity.rs @@ -0,0 +1,165 @@ +// Copyright (c) 2023 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::path::Path; +use std::sync::Arc; + +use anyhow::{anyhow, Context, Result}; +use image_rs::verity::{create_dmverity_device, destroy_dmverity_device}; +use kata_sys_util::mount::create_mount_destination; +use kata_types::mount::{DmVerityInfo, StorageDevice}; +use kata_types::volume::{ + KATA_VOLUME_DMVERITY_OPTION_SOURCE_TYPE, KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO, + KATA_VOLUME_DMVERITY_SOURCE_TYPE_PMEM, KATA_VOLUME_DMVERITY_SOURCE_TYPE_SCSI, + KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_CCW, KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_MMIO, + KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_PCI, +}; +use protocols::agent::Storage; +use slog::Logger; +use tracing::instrument; + +use crate::storage::block_handler::{ + PmemHandler, ScsiHandler, VirtioBlkCcwHandler, VirtioBlkMmioHandler, VirtioBlkPciHandler, +}; +use crate::storage::{common_storage_handler, StorageContext, StorageHandler}; + +use super::StorageDeviceGeneric; + +#[derive(Debug)] +pub struct DmVerityHandler {} + +impl DmVerityHandler { + fn get_dm_verity_info(storage: &Storage) -> Result { + for option in storage.driver_options.iter() { + if let Some((key, value)) = option.split_once('=') { + if key == KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO { + let verity_info: DmVerityInfo = serde_json::from_str(value)?; + return Ok(verity_info); + } + } + } + + Err(anyhow!("missing DmVerity information for DmVerity volume")) + } + + async fn update_source_device( + storage: &mut Storage, + ctx: &mut StorageContext<'_>, + ) -> Result<()> { + for option in storage.driver_options.clone() { + if let Some((key, value)) = option.split_once('=') { + if key == KATA_VOLUME_DMVERITY_OPTION_SOURCE_TYPE { + match value { + KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_PCI => { + VirtioBlkPciHandler::update_device_path(storage, ctx).await?; + } + KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_MMIO => { + VirtioBlkMmioHandler::update_device_path(storage, ctx).await?; + } + KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_CCW => { + VirtioBlkCcwHandler::update_device_path(storage, ctx).await?; + } + KATA_VOLUME_DMVERITY_SOURCE_TYPE_SCSI => { + ScsiHandler::update_device_path(storage, ctx).await?; + } + KATA_VOLUME_DMVERITY_SOURCE_TYPE_PMEM => { + PmemHandler::update_device_path(storage, ctx).await?; + } + _ => {} + } + } + } + } + Ok(()) + } +} + +#[async_trait::async_trait] +impl StorageHandler for DmVerityHandler { + #[instrument] + async fn create_device( + &self, + mut storage: Storage, + ctx: &mut StorageContext, + ) -> Result> { + Self::update_source_device(&mut storage, ctx).await?; + create_mount_destination(&storage.source, &storage.mount_point, "", &storage.fstype) + .context("Could not create mountpoint")?; + + let verity_info = Self::get_dm_verity_info(&storage)?; + let verity_info = serde_json::to_string(&verity_info) + .map_err(|e| anyhow!("failed to serialize dm_verity info, {}", e))?; + let verity_device_path = create_dmverity_device(&verity_info, Path::new(storage.source())) + .context("create device with dm-verity enabled")?; + storage.source = verity_device_path; + common_storage_handler(ctx.logger, &storage)?; + + Ok(Arc::new(DmVerityDevice { + common: StorageDeviceGeneric::new(storage.mount_point), + verity_device_path: storage.source, + logger: ctx.logger.clone(), + })) + } +} + +struct DmVerityDevice { + common: StorageDeviceGeneric, + verity_device_path: String, + logger: Logger, +} + +impl StorageDevice for DmVerityDevice { + fn path(&self) -> Option<&str> { + self.common.path() + } + + fn cleanup(&self) -> Result<()> { + self.common.cleanup().context("clean up dm-verity volume")?; + let device_path = &self.verity_device_path; + debug!( + self.logger, + "destroy verity device path = {:?}", device_path + ); + destroy_dmverity_device(device_path)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + + use kata_types::{mount::DmVerityInfo, volume::KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO}; + use protocols::agent::Storage; + + use crate::storage::dm_verity::DmVerityHandler; + + #[test] + fn test_get_dm_verity_info() { + let verity_info = DmVerityInfo { + hashtype: "sha256".to_string(), + hash: "d86104eee715a1b59b62148641f4ca73edf1be3006c4d481f03f55ac05640570".to_string(), + blocknum: 2361, + blocksize: 512, + hashsize: 4096, + offset: 1212416, + }; + + let verity_info_str = serde_json::to_string(&verity_info); + assert!(verity_info_str.is_ok()); + + let storage = Storage { + driver: KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO.to_string(), + driver_options: vec![format!("verity_info={}", verity_info_str.ok().unwrap())], + ..Default::default() + }; + + match DmVerityHandler::get_dm_verity_info(&storage) { + Ok(result) => { + assert_eq!(verity_info, result); + } + Err(e) => panic!("err = {}", e), + } + } +} diff --git a/src/agent/src/storage/mod.rs b/src/agent/src/storage/mod.rs index f312bbd83b..a8a3b3717a 100644 --- a/src/agent/src/storage/mod.rs +++ b/src/agent/src/storage/mod.rs @@ -13,6 +13,7 @@ use std::sync::Arc; use anyhow::{anyhow, Context, Result}; use kata_sys_util::mount::{create_mount_destination, parse_mount_options}; use kata_types::mount::{StorageDevice, StorageHandlerManager, KATA_SHAREDFS_GUEST_PREMOUNT_TAG}; +use kata_types::volume::KATA_VOLUME_TYPE_DMVERITY; use nix::unistd::{Gid, Uid}; use protocols::agent::Storage; use protocols::types::FSGroupChangePolicy; @@ -22,6 +23,7 @@ use tracing::instrument; use self::bind_watcher_handler::BindWatcherHandler; use self::block_handler::{PmemHandler, ScsiHandler, VirtioBlkMmioHandler, VirtioBlkPciHandler}; +use self::dm_verity::DmVerityHandler; use self::ephemeral_handler::EphemeralHandler; use self::fs_handler::{OverlayfsHandler, Virtio9pHandler, VirtioFsHandler}; use self::local_handler::LocalHandler; @@ -37,6 +39,7 @@ pub use self::ephemeral_handler::update_ephemeral_mounts; mod bind_watcher_handler; mod block_handler; +mod dm_verity; mod ephemeral_handler; mod fs_handler; mod local_handler; @@ -145,6 +148,7 @@ lazy_static! { manager.add_handler(DRIVER_SCSI_TYPE, Arc::new(ScsiHandler{})).unwrap(); manager.add_handler(DRIVER_VIRTIOFS_TYPE, Arc::new(VirtioFsHandler{})).unwrap(); manager.add_handler(DRIVER_WATCHABLE_BIND_TYPE, Arc::new(BindWatcherHandler{})).unwrap(); + manager.add_handler(KATA_VOLUME_TYPE_DMVERITY, Arc::new(DmVerityHandler{})).unwrap(); manager }; } diff --git a/src/libs/kata-types/src/lib.rs b/src/libs/kata-types/src/lib.rs index 5eb407561c..57090028a3 100644 --- a/src/libs/kata-types/src/lib.rs +++ b/src/libs/kata-types/src/lib.rs @@ -29,6 +29,9 @@ pub mod k8s; /// Constants and data types related to mount point. pub mod mount; +/// Constants and data types related to data volumes. +pub mod volume; + pub(crate) mod utils; /// hypervisor capabilities diff --git a/src/libs/kata-types/src/volume.rs b/src/libs/kata-types/src/volume.rs new file mode 100644 index 0000000000..2a6de62238 --- /dev/null +++ b/src/libs/kata-types/src/volume.rs @@ -0,0 +1,22 @@ +// Copyright (c) 2023 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +/// Volume to support dm-verity over block devices. +pub const KATA_VOLUME_TYPE_DMVERITY: &str = "dmverity"; + +/// Key to identify dmverity information in `Storage.driver_options` +pub const KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO: &str = "verity_info"; +/// Key to identify type of source device in `Storage.driver_options` +pub const KATA_VOLUME_DMVERITY_OPTION_SOURCE_TYPE: &str = "source_type"; +/// Source device of dmverity volume is a Virtio PCI device +pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_PCI: &str = "virtio_pci"; +/// Source device of dmverity volume is a Virtio MMIO device +pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_MMIO: &str = "virtio_mmio"; +/// Source device of dmverity volume is a Virtio CCW device +pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_CCW: &str = "virtio_ccw"; +/// Source device of dmverity volume is a SCSI disk. +pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_SCSI: &str = "scsi"; +/// Source device of dmverity volume is a pmem disk. +pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_PMEM: &str = "pmem";