From c74bddddaf6a3e7296c96bf8f7091c2c9124d3a3 Mon Sep 17 00:00:00 2001 From: Alex Lyn Date: Tue, 16 Jun 2026 14:46:24 +0800 Subject: [PATCH] kata-types: Add dmverity module with optional devicemapper support Introduce a new `dmverity` module in kata-types that provides dm-verity device creation, destruction and lifecycle management via devicemapper ioctls. The module is conditionally compiled behind the `devicemapper` feature flag, which also pulls in tokio for async device-node polling. The workspace devicemapper dependency is pinned to a specific git revision for reproducible builds. Signed-off-by: Alex Lyn --- .github/workflows/build-checks.yaml | 1 + Cargo.lock | 34 +-- Cargo.toml | 2 +- src/libs/kata-types/Cargo.toml | 4 +- src/libs/kata-types/src/dmverity.rs | 339 ++++++++++++++++++++++++++++ src/libs/kata-types/src/lib.rs | 4 + 6 files changed, 365 insertions(+), 19 deletions(-) create mode 100644 src/libs/kata-types/src/dmverity.rs diff --git a/.github/workflows/build-checks.yaml b/.github/workflows/build-checks.yaml index d732b81f0b..de8d556b8f 100644 --- a/.github/workflows/build-checks.yaml +++ b/.github/workflows/build-checks.yaml @@ -58,6 +58,7 @@ jobs: path: src/libs needs: - rust + - libdevmapper - protobuf-compiler - name: agent-ctl path: src/tools/agent-ctl diff --git a/Cargo.lock b/Cargo.lock index ba0f14aabf..4da6b73b83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,7 +139,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -150,7 +150,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1830,8 +1830,7 @@ checksum = "f18f717c5c7c2e3483feb64cccebd077245ad6d19007c2db0fd341d38595353c" [[package]] name = "devicemapper" version = "0.34.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "607791a4633fca6e032a66614f4fe96a721dd8641ebe98438283e53d361503cd" +source = "git+https://github.com/stratis-storage/devicemapper-rs?rev=078e70ceecf0d08eb0c800626827c9bc0a9a8adc#078e70ceecf0d08eb0c800626827c9bc0a9a8adc" dependencies = [ "bitflags 2.11.1", "cfg-if 1.0.4", @@ -1848,8 +1847,7 @@ dependencies = [ [[package]] name = "devicemapper-sys" version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06421aaad10b53bd5d1fe004c26efddfaaeaa4438ff52b84a0f660b3c87d63e6" +source = "git+https://github.com/stratis-storage/devicemapper-rs?rev=078e70ceecf0d08eb0c800626827c9bc0a9a8adc#078e70ceecf0d08eb0c800626827c9bc0a9a8adc" dependencies = [ "bindgen", "pkg-config", @@ -2114,7 +2112,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3326,7 +3324,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3838,6 +3836,7 @@ dependencies = [ "bitmask-enum", "byte-unit", "crc", + "devicemapper", "flate2", "glob", "gpt", @@ -3860,6 +3859,7 @@ dependencies = [ "tempfile", "test-utils", "thiserror 1.0.69", + "tokio", "toml", ] @@ -4619,7 +4619,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6828,7 +6828,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6929,7 +6929,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -7624,7 +7624,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -7877,10 +7877,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand 2.4.1", - "getrandom 0.4.2", + "getrandom 0.3.4", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -7889,7 +7889,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8610,7 +8610,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset 0.9.1", "tempfile", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -9194,7 +9194,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6d96c07191..d5bd783028 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -211,7 +211,7 @@ ttrpc = "0.8.4" url = "2.5.4" which = "4.3.0" gpt = "4.1.0" -devicemapper = { version = "0.34", default-features = false } +devicemapper = { git = "https://github.com/stratis-storage/devicemapper-rs", rev = "078e70ceecf0d08eb0c800626827c9bc0a9a8adc" } # Per-package release profile overrides for kata-deploy. The kata-deploy # binary runs once at pod start and then idles waiting for SIGTERM, so we diff --git a/src/libs/kata-types/Cargo.toml b/src/libs/kata-types/Cargo.toml index 4c47ea9dba..6713bd8663 100644 --- a/src/libs/kata-types/Cargo.toml +++ b/src/libs/kata-types/Cargo.toml @@ -35,7 +35,8 @@ gpt = "4.1.0" scopeguard = "1.0.0" crc = "3.4.0" safe-path = { path = "../safe-path", optional = true } - +devicemapper = { workspace = true, optional = true } +tokio = { workspace = true, optional = true, features = ["time", "rt"] } [target.'cfg(target_os = "macos")'.dependencies] sysctl = "0.7.1" @@ -49,3 +50,4 @@ rstest = "0.18" default = [] enable-vendor = [] safe-path = ["dep:safe-path"] # safe-path is platform-specific +devicemapper = ["dep:devicemapper", "dep:tokio"] diff --git a/src/libs/kata-types/src/dmverity.rs b/src/libs/kata-types/src/dmverity.rs new file mode 100644 index 0000000000..2cb9022737 --- /dev/null +++ b/src/libs/kata-types/src/dmverity.rs @@ -0,0 +1,339 @@ +// Copyright (c) 2026 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// +// + +use anyhow::{anyhow, Context, Result}; +use devicemapper::{DevId, DmFlags, DmName, DmOptions, DmUdevFlags, DM}; +use nix::sys::stat::{self, Mode, SFlag}; +use slog::Logger; +use std::path::Path; +use std::sync::OnceLock; +use std::time::Duration; +use tokio::time::sleep; + +pub use crate::mount::DmVerityInfo; + +/// Detect whether udevd is running in the guest. +/// +/// Checks for the udevd control socket — its presence reliably indicates a +/// running udevd. The result is cached for the process lifetime since udev +/// availability does not change after boot. +pub fn has_udev() -> bool { + static UDEV_AVAILABLE: OnceLock = OnceLock::new(); + *UDEV_AVAILABLE.get_or_init(|| Path::new("/run/udev/control").exists()) +} + +/// DmOptions with all udev interactions disabled, for use when udev is not running. +fn no_udev_dm_options() -> DmOptions { + DmOptions::default().set_udev_flags( + DmUdevFlags::DM_UDEV_DISABLE_LIBRARY_FALLBACK + | DmUdevFlags::DM_UDEV_DISABLE_SUBSYSTEM_RULES_FLAG + | DmUdevFlags::DM_UDEV_DISABLE_DISK_RULES_FLAG + | DmUdevFlags::DM_UDEV_DISABLE_OTHER_RULES_FLAG + | DmUdevFlags::DM_UDEV_DISABLE_DM_RULES_FLAG, + ) +} + +/// DmOptions for creating a read-only dm-verity device: udev-aware. +fn dm_opts_readonly() -> DmOptions { + no_udev_dm_options().set_flags(DmFlags::DM_READONLY) +} + +/// DmOptions for deferred device removal: udev-aware. +fn dm_opts_deferred_remove() -> DmOptions { + no_udev_dm_options().set_flags(DmFlags::DM_DEFERRED_REMOVE) +} + +/// DmOptions for creating a dm-verity device, with appropriate flags based on udev availability. +#[allow(dead_code)] +fn dm_create_options() -> DmOptions { + if has_udev() { + DmOptions::default().set_flags(DmFlags::DM_READONLY) + } else { + dm_opts_readonly() + } +} + +/// DmOptions for device suspend/resume: udev-aware. +#[allow(dead_code)] +fn dm_suspend_options() -> DmOptions { + if has_udev() { + DmOptions::default() + } else { + no_udev_dm_options() + } +} + +/// DmOptions for deferred device removal: udev-aware. +fn dm_remove_options() -> DmOptions { + if has_udev() { + DmOptions::default().set_flags(DmFlags::DM_DEFERRED_REMOVE) + } else { + dm_opts_deferred_remove() + } +} + +/// Create a block device node for a dm-verity device using mknod(2). +pub fn create_dm_dev_node(name: &str, dev: devicemapper::Device) -> Result { + let mapper_dir = Path::new("/dev/mapper"); + if !mapper_dir.exists() { + std::fs::create_dir_all(mapper_dir) + .with_context(|| format!("failed to create directory {}", mapper_dir.display()))?; + } + + let dev_path = format!("/dev/mapper/{}", name); + if Path::new(&dev_path).exists() { + std::fs::remove_file(&dev_path) + .with_context(|| format!("failed to remove stale device node {}", dev_path))?; + } + + let dev_t: nix::libc::dev_t = dev.into(); + stat::mknod( + dev_path.as_str(), + SFlag::S_IFBLK, + Mode::from_bits_truncate(0o600), + dev_t, + ) + .with_context(|| format!("failed to mknod block device {}", dev_path))?; + + Ok(dev_path) +} + +/// Remove a device node created by `create_dm_dev_node`. +pub fn remove_dm_dev_node(dev_path: &str) { + if dev_path.starts_with("/dev/mapper/") && Path::new(dev_path).exists() { + if let Err(e) = std::fs::remove_file(dev_path) { + slog::warn!( + slog_scope::logger(), + "failed to remove dm device node"; + "path" => dev_path, + "error" => %e, + ); + } + } +} + +/// Generate a unique dm-verity device name from source path and verity hash. +pub fn build_dmverity_device_name(source_device_path: &Path, verity_info: &DmVerityInfo) -> String { + let source_short = source_device_path + .file_name() + .map(|f| f.to_string_lossy()) + .unwrap_or_default(); + let hash_prefix = &verity_info.hash[..verity_info.hash.len().min(32)]; + let mut name = format!( + "kata-verity-{}-off{}-{}", + source_short, verity_info.offset, hash_prefix + ); + name.truncate(128); + name +} + +/// Result of dm-verity device setup, indicating whether the device node is ready or if we need to wait for udev. +enum DmSetupResult { + Ready(String), + NeedUdevWait, +} + +/// Destroy a dm-verity device by name. +pub fn destroy_dmverity_device(verity_device_name: &str) -> Result<()> { + let dm = devicemapper::DM::new()?; + let name = devicemapper::DmName::new(verity_device_name)?; + + dm.device_remove(&devicemapper::DevId::Name(name), dm_remove_options()) + .context(format!("remove DmverityDevice {}", verity_device_name))?; + + Ok(()) +} + +/// Destroy a dm-verity device by its `/dev/mapper/` path. +pub fn destroy_partition_dmverity_device(verity_device_path: &str, logger: &Logger) -> Result<()> { + // The verity device path is /dev/mapper/ (as created by create_dm_dev_node). + // Extract the DM device name for removal. Also remove the mknod-created device node. + let device_name = verity_device_path + .strip_prefix("/dev/mapper/") + .unwrap_or(verity_device_path) + .to_string(); + + destroy_dmverity_device(&device_name).context("Failed to destroy dm-verity device")?; + info!( + logger, + "Destroying dm-verity device"; + "device-name" => &device_name, + ); + + // Only remove the device node manually if we created it via mknod. + // When udev is running, it handles node lifecycle automatically. + if !has_udev() { + remove_dm_dev_node(verity_device_path); + } + + Ok(()) +} + +/// Clean up all dm-verity devices for a multi-layer EROFS mount. +pub fn cleanup_dmverity_devices(verity_devices: &[String], logger: &Logger) { + info!( + logger, + "Cleaning up {} dm-verity devices", + verity_devices.len() + ); + + // Destroy in reverse order + for verity_device in verity_devices.iter().rev() { + if let Err(e) = destroy_partition_dmverity_device(verity_device, logger) { + warn!( + logger, + "Failed to destroy dm-verity device"; + "device-path" => verity_device, + "error" => format!("{:#}", e), + ); + } + } + + info!(logger, "dm-verity device cleanup completed"); +} + +/// Wait for udev to create a device-mapper node under `/dev/mapper/`. +pub async fn wait_for_dm_dev_node(name: &str) -> Result { + let dev_path = format!("/dev/mapper/{}", name); + let path = Path::new(&dev_path); + + if path.exists() { + return Ok(dev_path); + } + + const MAX_WAIT_MS: u64 = 2000; + const POLL_INTERVAL_MS: u64 = 50; + + for _attempt in 0..(MAX_WAIT_MS / POLL_INTERVAL_MS) { + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + if path.exists() { + return Ok(dev_path); + } + } + + Err(anyhow!( + "udev did not create dm device node {} within {} ms", + dev_path, + MAX_WAIT_MS + )) +} + +/// Create a dm-verity device using devicemapper, offloading blocking ioctls to a dedicated thread. +pub async fn create_dmverity_device( + verity_info: &DmVerityInfo, + source_device_path: &Path, +) -> Result { + let verity_info = verity_info.clone(); + let source_path = source_device_path.to_path_buf(); + + let verity_name_string = build_dmverity_device_name(&source_path, &verity_info); + let verity_name_for_wait = verity_name_string.clone(); + + // Offload all blocking ioctl operations to a dedicated thread. + // Always use no-udev DmOptions inside spawn_blocking to avoid DM_UDEV_WAIT + // blocking on udevd event processing. When udev is running, we wait for the + // device node asynchronously after the ioctl completes (via wait_for_dm_dev_node). + let dev_path_or_need_udev = tokio::task::spawn_blocking(move || -> Result { + let dm = DM::new()?; + let verity_name = DmName::new(&verity_name_string)?; + let id = DevId::Name(verity_name); + + let opts = no_udev_dm_options(); + let ro_opts = dm_opts_readonly(); + + // Step 0: Remove stale device if it already exists + let remove_opts = dm_opts_deferred_remove(); + if dm.device_remove(&id, remove_opts).is_ok() { + // Stale device removed; continue with creation. + } + + // Step 1: Create device as read-only + dm.device_create(verity_name, None, ro_opts)?; + + // Calculate hash start block. + let hash_start_block: u64 = if verity_info.no_superblock { + verity_info.offset / verity_info.hashsize + } else { + let superblock_blocks = 512_u64.div_ceil(verity_info.hashsize); + (verity_info.offset / verity_info.hashsize) + superblock_blocks + }; + + let salt = verity_info.salt.as_deref().unwrap_or("-"); + let source_display = source_path.display().to_string(); + let verity_params = format!( + "{} {} {} {} {} {} {} {} {} {}", + verity_info.hash_type, + source_display, + source_display, + verity_info.blocksize, + verity_info.hashsize, + verity_info.blocknum, + hash_start_block, + verity_info.hashtype, + verity_info.hash, + salt + ); + + let verity_table = vec![( + 0, + verity_info.blocknum * verity_info.blocksize / 512, + "verity".into(), + verity_params.clone(), + )]; + + info!( + slog_scope::logger(), + "dm-verity table parameters"; + "device" => &source_display, + "data_blocks" => verity_info.blocknum, + "data_block_size" => verity_info.blocksize, + "hash_block_size" => verity_info.hashsize, + "hash_start_block" => hash_start_block, + "hash_algorithm" => &verity_info.hashtype, + "hash_type" => verity_info.hash_type, + "no_superblock" => verity_info.no_superblock, + "salt" => salt, + "table_params" => &verity_params, + ); + + // Step 2: Load table and resume (activate) + dm.table_load(&id, verity_table.as_slice(), ro_opts)?; + dm.device_suspend(&id, opts)?; + + // Step 3: Ensure the device node exists under /dev/mapper/. + let result = if has_udev() { + DmSetupResult::NeedUdevWait + } else { + info!( + slog_scope::logger(), + "udev is not running; creating dm-verity device node manually"; + "device-name" => &verity_name_string, + ); + let device_info = dm.device_info(&id)?; + let path = create_dm_dev_node(&verity_name_string, device_info.device())?; + DmSetupResult::Ready(path) + }; + + Ok(result) + }) + .await + .context("spawn_blocking for dm-verity ioctl panicked")??; + + // If udev is running, wait asynchronously for the device node (non-blocking poll). + let dev_path = match dev_path_or_need_udev { + DmSetupResult::Ready(path) => path, + DmSetupResult::NeedUdevWait => { + info!( + slog_scope::logger(), + "Waiting for udev to create dm-verity device node"; + "device-name" => &verity_name_for_wait, + ); + wait_for_dm_dev_node(&verity_name_for_wait).await? + } + }; + + Ok(dev_path) +} diff --git a/src/libs/kata-types/src/lib.rs b/src/libs/kata-types/src/lib.rs index 9049caf54d..4481013590 100644 --- a/src/libs/kata-types/src/lib.rs +++ b/src/libs/kata-types/src/lib.rs @@ -57,6 +57,10 @@ pub mod machine_type; /// GPT (GUID Partition Table) disk layout and metadata generation. pub mod gpt_disk; +/// dm-verity related constants and data types. +#[cfg(feature = "devicemapper")] +pub mod dmverity; + use std::path::Path; use crate::rootless::{is_rootless, rootless_dir};