From b2d0e5b712eb070051a8012c4b329fd0682b4e27 Mon Sep 17 00:00:00 2001 From: Alex Lyn Date: Tue, 16 Jun 2026 16:28:53 +0800 Subject: [PATCH] kata-agent: Use kata-types dmverity with optional devicemapper support Replace the agent's inline devicemapper implementation with the libs kata-types::dmverity module. The agent's devicemapper Cargo feature now forwards to kata-types/devicemapper, removing the direct libdevmapper link dependency from the agent crate. Gate all dm-verity imports, constants, and call sites behind libdevmapper. Add USE_DEVMAPPER Makefile variable (default no) that appends the devicemapper feature flag and forces LIBC=gnu when enabled. Signed-off-by: Alex Lyn --- Cargo.lock | 1 - src/agent/Cargo.toml | 2 +- src/agent/Makefile | 9 + src/agent/src/rpc.rs | 7 +- src/agent/src/storage/multi_layer_erofs.rs | 270 +++------------------ 5 files changed, 43 insertions(+), 246 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4da6b73b83..a3f10c2432 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3591,7 +3591,6 @@ dependencies = [ "const_format", "container-device-interface", "derivative", - "devicemapper", "futures", "ipnetwork", "kata-agent-policy", diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 0596701a7b..b4c567607b 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -60,7 +60,6 @@ tracing-subscriber.workspace = true # TODO: bump tracing-opentelemetry to sync with version in workspace tracing-opentelemetry = "0.17.0" opentelemetry.workspace = true -devicemapper.workspace = true # Configuration serde.workspace = true @@ -107,3 +106,4 @@ lto = true seccomp = ["rustjail/seccomp"] agent-policy = ["kata-agent-policy"] init-data = [] +devicemapper = ["kata-types/devicemapper"] diff --git a/src/agent/Makefile b/src/agent/Makefile index 79e59907a8..6f79ab69e2 100644 --- a/src/agent/Makefile +++ b/src/agent/Makefile @@ -49,6 +49,15 @@ ifeq ($(INIT_DATA),yes) override EXTRA_RUSTFEATURES += init-data endif +##VAR USE_DEVMAPPER=yes|no define if agent enables dm-verity (devicemapper) support +USE_DEVMAPPER ?= no + +# Enable the devicemapper feature (links libdevmapper for dm-verity) with GNU libc +ifeq ($(USE_DEVMAPPER),yes) + override EXTRA_RUSTFEATURES += devicemapper + override LIBC = gnu +endif + include ../../utils.mk ifneq ($(EXTRA_RUSTFEATURES),) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 88d645250d..d3876cc814 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -87,12 +87,13 @@ use crate::passfd_io; use crate::pci; use crate::random; use crate::sandbox::{Sandbox, SandboxError}; -use crate::storage::multi_layer_erofs::cleanup_dmverity_devices; use crate::storage::{add_storages, update_ephemeral_mounts, STORAGE_HANDLERS}; use crate::util; use crate::version::{AGENT_VERSION, API_VERSION}; use crate::AGENT_CONFIG; use crate::{confidential_data_hub, linux_abi::*}; +#[cfg(feature = "devicemapper")] +use kata_types::dmverity::cleanup_dmverity_devices; use crate::trace_rpc_call; use crate::tracer::extract_carrier_from_ttrpc; @@ -2100,11 +2101,11 @@ async fn remove_container_resources(sandbox: &mut Sandbox, cid: &str) -> Result< // Cleanup dm-verity devices for this container (after all mounts are unmounted) if let Some(verity_devices) = sandbox.container_verity_devices.remove(cid) { + #[cfg(feature = "devicemapper")] if !verity_devices.is_empty() { - // Cleanup dm-verity devices for this container, ignoring any errors - // since we want to proceed with cleanup as much as possible cleanup_dmverity_devices(&verity_devices, &sandbox.logger); } + let _ = verity_devices; } sandbox.container_mounts.remove(cid); diff --git a/src/agent/src/storage/multi_layer_erofs.rs b/src/agent/src/storage/multi_layer_erofs.rs index 8d51271281..1ddb3a8033 100644 --- a/src/agent/src/storage/multi_layer_erofs.rs +++ b/src/agent/src/storage/multi_layer_erofs.rs @@ -13,7 +13,6 @@ //! - Supports X-kata.mkdir.path options to create directories in upper layer before overlay mount //! - Supports GPT-partitioned disks with dm-verity integrity verification for each partition -use nix::sys::stat::{self, Mode, SFlag}; use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; @@ -31,16 +30,14 @@ use crate::storage::{StorageContext, StorageHandler}; use anyhow::{anyhow, Context, Result}; use kata_sys_util::mount::{create_mount_destination, Mounter}; use kata_types::device::{DRIVER_BLK_PCI_TYPE, DRIVER_SCSI_TYPE}; +#[cfg(feature = "devicemapper")] +use kata_types::dmverity::{cleanup_dmverity_devices, create_dmverity_device, DmVerityInfo}; use kata_types::mount::StorageDevice; use protocols::agent::Storage; use safe_path::scoped_join; use slog::Logger; use tokio::sync::Mutex; -// dm-verity support imports -use devicemapper::{DevId, DmFlags, DmName, DmOptions, DmUdevFlags, DM}; -use kata_types::mount::DmVerityInfo; - /// EROFS Type const EROFS_TYPE: &str = "erofs"; /// ext4 Type (upper virtio disk based rw layer) @@ -62,94 +59,23 @@ const OPT_PARTITION_NUMBER: &str = "X-kata.partition-number="; /// dm-verity related storage options #[allow(dead_code)] const OPT_DMVERITY_ENABLED: &str = "X-kata.dmverity-enabled=true"; +#[cfg(feature = "devicemapper")] const OPT_DMVERITY_ROOT_HASH: &str = "X-kata.dmverity.roothash="; +#[cfg(feature = "devicemapper")] const OPT_DMVERITY_HASH_OFFSET: &str = "X-kata.dmverity.hashoffset="; +#[cfg(feature = "devicemapper")] const OPT_DMVERITY_BLOCK_SIZE: &str = "X-kata.dmverity.blocksize="; +#[cfg(feature = "devicemapper")] const OPT_DMVERITY_HASH_SIZE: &str = "X-kata.dmverity.hashsize="; +#[cfg(feature = "devicemapper")] const OPT_DMVERITY_HASH_ALGORITHM: &str = "X-kata.dmverity.algorithm="; +#[cfg(feature = "devicemapper")] const OPT_DMVERITY_SALT: &str = "X-kata.dmverity.salt="; +#[cfg(feature = "devicemapper")] const OPT_DMVERITY_HASH_TYPE: &str = "X-kata.dmverity.hashtype="; +#[cfg(feature = "devicemapper")] const OPT_DMVERITY_NO_SUPERBLOCK: &str = "X-kata.dmverity.no-superblock="; -/// Build DmOptions that fully disable udev synchronization. -fn no_udev_dm_options() -> DmOptions { - DmOptions::default().set_udev_flags( - DmUdevFlags::DM_UDEV_DISABLE_LIBRARY_FALLBACK - | DmUdevFlags::DM_UDEV_DISABLE_SUBSYSTEM_RULES_FLAG - | DmUdevFlags::DM_UDEV_DISABLE_DISK_RULES_FLAG - | DmUdevFlags::DM_UDEV_DISABLE_OTHER_RULES_FLAG - | DmUdevFlags::DM_UDEV_DISABLE_DM_RULES_FLAG, - ) -} - -/// Build DmOptions for read-only device removal in a no-udev environment. -fn dm_opts_readonly() -> DmOptions { - no_udev_dm_options().set_flags(DmFlags::DM_READONLY) -} - -/// Build DmOptions for deferred device removal in a no-udev environment. -fn dm_opts_deferred_remove() -> DmOptions { - no_udev_dm_options().set_flags(DmFlags::DM_DEFERRED_REMOVE) -} - -/// Create a block device node for a dm-verity device using mknod(2). -fn create_dm_dev_node(name: &str, dev: devicemapper::Device) -> Result { - // Ensure /dev/mapper exists. - let mapper_dir = Path::new("/dev/mapper"); - if !mapper_dir.exists() { - std::fs::create_dir_all(mapper_dir) - .with_context(|| format!("failed to create directory {}", mapper_dir.display()))?; - } - - let dev_path = format!("/dev/mapper/{}", name); - // Remove stale node from a previous failed run, if any - if Path::new(&dev_path).exists() { - std::fs::remove_file(&dev_path) - .with_context(|| format!("failed to remove stale device node {}", dev_path))?; - } - - // Use Device -> dev_t conversion from the crate instead of raw makedev. - let dev_t: nix::libc::dev_t = dev.into(); - stat::mknod( - dev_path.as_str(), - SFlag::S_IFBLK, - Mode::from_bits_truncate(0o600), - dev_t, - ) - .with_context(|| format!("failed to mknod block device {}", dev_path))?; - - Ok(dev_path) -} - -/// Remove a device node that was created by create_dm_dev_node. -fn remove_dm_dev_node(dev_path: &str) { - if dev_path.starts_with("/dev/mapper/") && Path::new(dev_path).exists() { - if let Err(e) = std::fs::remove_file(dev_path) { - slog::warn!( - slog_scope::logger(), - "failed to remove dm device node"; - "path" => dev_path, - "error" => %e, - ); - } - } -} - -/// Generate a unique dm-verity device name based on the source device path and verity hash. -fn build_dmverity_device_name(source_device_path: &Path, verity_info: &DmVerityInfo) -> String { - let source_short = source_device_path - .file_name() - .map(|f| f.to_string_lossy()) - .unwrap_or_default(); - let hash_prefix = &verity_info.hash[..verity_info.hash.len().min(32)]; - let mut name = format!( - "kata-verity-{}-off{}-{}", - source_short, verity_info.offset, hash_prefix - ); - name.truncate(128); - name -} - #[derive(Debug)] pub struct MultiLayerErofsHandler {} @@ -387,8 +313,8 @@ pub async fn handle_multi_layer_erofs_group( } // Concurrently create dm-verity devices and mount layers. - // Each layer has independent device nodes and mount points, so there is no - // ordering dependency during creation — only the final lowerdir list must + // No worry about that because each layer has independent device nodes and mount points, + // so there is no ordering dependency during creation — only the final lowerdir list must // preserve the original sort order. let futures: Vec<_> = erofs_storages .iter() @@ -451,6 +377,7 @@ pub async fn handle_multi_layer_erofs_group( } } + #[cfg(feature = "devicemapper")] if !failed_verity_devices.is_empty() { cleanup_dmverity_devices(&failed_verity_devices, &logger); } @@ -622,7 +549,8 @@ fn is_dmverity_enabled(storage: &Storage) -> bool { } /// Parse dm-verity configuration from storage options -fn parse_dmverity_options(storage: &Storage) -> Result { +#[cfg(feature = "devicemapper")] +pub fn parse_dmverity_options(storage: &Storage) -> Result { let mut hashtype = String::from("sha256"); let mut hash = String::new(); let mut blocknum: u64 = 0; @@ -702,8 +630,8 @@ fn parse_dmverity_options(storage: &Storage) -> Result { } /// Create dm-verity device for a partition and return the verity device path -#[allow(dead_code)] -fn create_partition_dmverity_device( +#[cfg(feature = "devicemapper")] +async fn create_partition_dmverity_device( partition_path: &str, storage: &Storage, logger: &Logger, @@ -721,6 +649,7 @@ fn create_partition_dmverity_device( // Create dm-verity device let verity_device_path = create_dmverity_device(&verity_info, Path::new(partition_path)) + .await .context("failed to create dm-verity device")?; info!( @@ -733,157 +662,15 @@ fn create_partition_dmverity_device( Ok(verity_device_path) } -/// Create a dm-verity device using devicemapper -fn create_dmverity_device(verity_info: &DmVerityInfo, source_device_path: &Path) -> Result { - let dm = DM::new()?; - let verity_name_string = build_dmverity_device_name(source_device_path, verity_info); - let verity_name = DmName::new(&verity_name_string)?; - let id = DevId::Name(verity_name); - - let opts = no_udev_dm_options(); - let ro_opts = dm_opts_readonly(); - - // Step 0: Remove stale device if it already exists - if dm.device_remove(&id, dm_opts_deferred_remove()).is_ok() { - // Stale device removed; continue with creation. - } - - // Step 1: Create device as read-only with no-udev flags - dm.device_create(verity_name, None, ro_opts)?; - - // Calculate hash start block. - // - // The `offset` field (from X-kata.dmverity.hashoffset) is the byte offset - // of the dm-verity superblock from the start of the device, as stored in - // the containerd .erofs.dmverity JSON. It equals data_blocks * data_block_size. - // - // In the dm-verity table, `hash_start_block` is the block number (in - // hash_block_size units) where the hash TREE DATA begins — NOT where the - // superblock begins. When version=1 (with superblock), the superblock - // occupies one hash-block-aligned region at `offset`, and the actual hash - // tree starts after it. The kernel never reads the superblock; it relies - // entirely on the table parameters. - // - // Therefore, when no_superblock=false: - // hash_start_block = (offset / hashsize) + superblock_blocks - // where superblock_blocks = ceil(512 / hashsize) = 1 (for hashsize >= 512) - // - // When no_superblock=true (version=0, no superblock): - // hash_start_block = offset / hashsize - let hash_start_block: u64 = if verity_info.no_superblock { - verity_info.offset / verity_info.hashsize - } else { - // dm-verity v1 superblock is 512 bytes, aligned up to hash block size - let superblock_blocks = 512_u64.div_ceil(verity_info.hashsize); - (verity_info.offset / verity_info.hashsize) + superblock_blocks - }; - - // Use provided salt or default to "-" (no salt) - let salt = verity_info.salt.as_deref().unwrap_or("-"); - let verity_params = format!( - "{} {} {} {} {} {} {} {} {} {}", // 10 parameters - verity_info.hash_type, // version: "1" for verity v1.0 - source_device_path.display(), // data device - source_device_path.display(), // hash device (usually same as data) - verity_info.blocksize, // data block size - verity_info.hashsize, // hash block size - verity_info.blocknum, // number of data blocks - hash_start_block, // hash start block - verity_info.hashtype, // hash algorithm ("sha256", "sha1", etc.) - verity_info.hash, // root hash (hex encoded) - salt // salt (hex encoded or "-" for none) - ); - - let verity_table = vec![( - 0, - verity_info.blocknum * verity_info.blocksize / 512, - "verity".into(), - verity_params.clone(), - )]; - - info!( - slog_scope::logger(), - "dm-verity table parameters"; - "device" => source_device_path.display(), - "data_blocks" => verity_info.blocknum, - "data_block_size" => verity_info.blocksize, - "hash_block_size" => verity_info.hashsize, - "hash_start_block" => hash_start_block, - "hash_algorithm" => &verity_info.hashtype, - "hash_type" => verity_info.hash_type, - "no_superblock" => verity_info.no_superblock, - "salt" => salt, - "table_params" => &verity_params, - ); - - // Step 2: Load table and resume (activate) with read-only + no-udev flags - dm.table_load(&id, verity_table.as_slice(), ro_opts)?; - dm.device_suspend(&id, opts)?; - - // Step 3: Get device info and create the device node via mknod. - // In a udev-less guest VM, /dev/block/M:N and /dev/mapper/ are - // never created by udev. We must create the node ourselves using the - // major:minor numbers returned by the device-mapper ioctl. - let device_info = dm.device_info(&id)?; - let dev_path = create_dm_dev_node(&verity_name_string, device_info.device())?; - - Ok(dev_path) -} - -/// Destroy a dm-verity device -fn destroy_dmverity_device(verity_device_name: &str) -> Result<()> { - let dm = devicemapper::DM::new()?; - let name = devicemapper::DmName::new(verity_device_name)?; - - dm.device_remove(&devicemapper::DevId::Name(name), dm_opts_deferred_remove()) - .context(format!("remove DmverityDevice {}", verity_device_name))?; - - Ok(()) -} - -/// Destroy dm-verity device by path -fn destroy_partition_dmverity_device(verity_device_path: &str, logger: &Logger) -> Result<()> { - // The verity device path is /dev/mapper/ (as created by create_dm_dev_node). - // Extract the DM device name for removal. Also remove the mknod-created device node. - let device_name = verity_device_path - .strip_prefix("/dev/mapper/") - .unwrap_or(verity_device_path) - .to_string(); - - destroy_dmverity_device(&device_name).context("Failed to destroy dm-verity device")?; - info!( - logger, - "Destroying dm-verity device"; - "device-name" => &device_name, - ); - - // Remove the device node we created with mknod. - remove_dm_dev_node(verity_device_path); - - Ok(()) -} - -/// Cleanup all dm-verity devices for a multi-layer EROFS mount -pub fn cleanup_dmverity_devices(verity_devices: &[String], logger: &Logger) { - info!( - logger, - "Cleaning up {} dm-verity devices", - verity_devices.len() - ); - - // Destroy in reverse order - for verity_device in verity_devices.iter().rev() { - if let Err(e) = destroy_partition_dmverity_device(verity_device, logger) { - warn!( - logger, - "Failed to destroy dm-verity device"; - "device-path" => verity_device, - "error" => format!("{:#}", e), - ); - } - } - - info!(logger, "dm-verity device cleanup completed"); +#[cfg(not(feature = "devicemapper"))] +async fn create_partition_dmverity_device( + _partition_path: &str, + _storage: &Storage, + _logger: &Logger, +) -> Result { + Err(anyhow!( + "dm-verity support not compiled in: build with `--features devicemapper` to enable", + )) } /// Validate that a container ID does not contain path traversal sequences. @@ -1054,7 +841,7 @@ async fn wait_and_mount_layer( })?; // Create dm-verity device - let verity_device = create_partition_dmverity_device(partition, layer, logger)?; + let verity_device = create_partition_dmverity_device(partition, layer, logger).await?; info!( logger, "Using dm-verity device for mount"; @@ -1420,6 +1207,7 @@ mod tests { // --- parse_dmverity_options --- + #[cfg(feature = "devicemapper")] #[test] fn test_parse_dmverity_options_required_fields_and_blocknum() { // Test required fields and blocknum calculation.