From e900eae388e45befce00546c084ebe245d47cfe4 Mon Sep 17 00:00:00 2001 From: Alex Lyn Date: Tue, 26 May 2026 19:33:07 +0800 Subject: [PATCH] kata-agent: Add no-udev DmOptions builders and mknod device node helpers The kata guest VM runs without udev, so device-mapper nodes under /dev/mapper are never created automatically. Add the foundational helpers that subsequent dm-verity integration will rely on: It focus on the following key points: (1) DmOptions builders that disable all udev synchronization flags, with read-only and deferred-remove variants. (2) mknod-based device node creation/removal under /dev/mapper, since devtmpfs nodes are not auto-created without udev. Also add the devicemapper crate dependency (default-features = false). But note that the commit depends on device mapper with no-udev support with the PR:https://github.com/stratis-storage/devicemapper-rs/pull/1036 Signed-off-by: Alex Lyn --- Cargo.lock | 153 ++++++++++++++++++++- Cargo.toml | 2 + src/agent/Cargo.toml | 1 + src/agent/src/storage/multi_layer_erofs.rs | 77 ++++++++++- 4 files changed, 229 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88ecc0673e..ba0f14aabf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -694,6 +694,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.11.1", + "cexpr", + "clang-sys", + "itertools 0.10.5", + "log", + "prettyplease 0.2.37", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.117", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -938,6 +958,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -1036,6 +1065,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.6.1" @@ -1787,6 +1827,34 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f18f717c5c7c2e3483feb64cccebd077245ad6d19007c2db0fd341d38595353c" +[[package]] +name = "devicemapper" +version = "0.34.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "607791a4633fca6e032a66614f4fe96a721dd8641ebe98438283e53d361503cd" +dependencies = [ + "bitflags 2.11.1", + "cfg-if 1.0.4", + "devicemapper-sys", + "env_logger 0.11.10", + "log", + "nix 0.31.3", + "rand 0.10.1", + "retry", + "semver", + "serde", +] + +[[package]] +name = "devicemapper-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06421aaad10b53bd5d1fe004c26efddfaaeaa4438ff52b84a0f660b3c87d63e6" +dependencies = [ + "bindgen", + "pkg-config", +] + [[package]] name = "difflib" version = "0.4.0" @@ -1972,6 +2040,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "env_filter" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" +dependencies = [ + "log", + "regex", +] + [[package]] name = "env_home" version = "0.1.0" @@ -1991,6 +2069,19 @@ dependencies = [ "termcolor", ] +[[package]] +name = "env_logger" +version = "0.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + [[package]] name = "epoll" version = "4.3.1" @@ -2418,7 +2509,7 @@ dependencies = [ "clap", "containerd-client", "docker_credential", - "env_logger", + "env_logger 0.10.2", "flate2", "fs2", "json-patch 4.2.0", @@ -3268,6 +3359,30 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jiff" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-static" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "jni" version = "0.22.4" @@ -3478,6 +3593,7 @@ dependencies = [ "const_format", "container-device-interface", "derivative", + "devicemapper", "futures", "ipnetwork", "kata-agent-policy", @@ -3632,7 +3748,7 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", - "env_logger", + "env_logger 0.10.2", "k8s-openapi", "kube", "libc", @@ -3656,7 +3772,7 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", - "env_logger", + "env_logger 0.10.2", "k8s-openapi", "kube", "log", @@ -3885,6 +4001,16 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if 1.0.4", + "windows-link", +] + [[package]] name = "libredox" version = "0.1.16" @@ -5402,6 +5528,21 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -6366,6 +6507,12 @@ dependencies = [ "walkdir", ] +[[package]] +name = "retry" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cab9bd343c737660e523ee69f788018f3db686d537d2fd0f99c9f747c1bda4f" + [[package]] name = "ring" version = "0.17.14" diff --git a/Cargo.toml b/Cargo.toml index 7407919ec6..6d96c07191 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -211,6 +211,7 @@ ttrpc = "0.8.4" url = "2.5.4" which = "4.3.0" gpt = "4.1.0" +devicemapper = { version = "0.34", default-features = false } # Per-package release profile overrides for kata-deploy. The kata-deploy # binary runs once at pod start and then idles waiting for SIGTERM, so we @@ -221,3 +222,4 @@ gpt = "4.1.0" [profile.release.package."kata-deploy"] opt-level = "z" codegen-units = 1 + diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 35cfa417e6..0596701a7b 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -60,6 +60,7 @@ tracing-subscriber.workspace = true # TODO: bump tracing-opentelemetry to sync with version in workspace tracing-opentelemetry = "0.17.0" opentelemetry.workspace = true +devicemapper.workspace = true # Configuration serde.workspace = true diff --git a/src/agent/src/storage/multi_layer_erofs.rs b/src/agent/src/storage/multi_layer_erofs.rs index 633473e779..cd40675638 100644 --- a/src/agent/src/storage/multi_layer_erofs.rs +++ b/src/agent/src/storage/multi_layer_erofs.rs @@ -11,8 +11,10 @@ //! - Storage with X-kata.overlay-lower: erofs layers (lowerdir) //! - Creates overlay to combine them //! - Supports X-kata.mkdir.path options to create directories in upper layer before overlay mount -//! - Supports GPT-partitioned disks where each layer is a separate partition +//! - Supports GPT-partitioned disks with dm-verity integrity verification for each partition +#[allow(unused_imports)] +use nix::sys::stat::{self, Mode, SFlag}; use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; @@ -36,6 +38,10 @@ use safe_path::scoped_join; use slog::Logger; use tokio::sync::Mutex; +// no-udev device-mapper helpers +#[allow(unused_imports)] +use devicemapper::{DmFlags, DmOptions, DmUdevFlags}; + /// EROFS Type const EROFS_TYPE: &str = "erofs"; /// ext4 Type (upper virtio disk based rw layer) @@ -54,6 +60,75 @@ const OPT_GPT_PARTITIONED: &str = "X-kata.gpt-partitioned=true"; const OPT_MKDIR_PATH: &str = "X-kata.mkdir.path="; const OPT_PARTITION_NUMBER: &str = "X-kata.partition-number="; +/// Build DmOptions that fully disable udev synchronization. +#[allow(dead_code)] +fn no_udev_dm_options() -> DmOptions { + DmOptions::default().set_udev_flags( + DmUdevFlags::DM_UDEV_DISABLE_LIBRARY_FALLBACK + | DmUdevFlags::DM_UDEV_DISABLE_SUBSYSTEM_RULES_FLAG + | DmUdevFlags::DM_UDEV_DISABLE_DISK_RULES_FLAG + | DmUdevFlags::DM_UDEV_DISABLE_OTHER_RULES_FLAG + | DmUdevFlags::DM_UDEV_DISABLE_DM_RULES_FLAG, + ) +} + +/// Build DmOptions for read-only device removal in a no-udev environment. +#[allow(dead_code)] +fn dm_opts_readonly() -> DmOptions { + no_udev_dm_options().set_flags(DmFlags::DM_READONLY) +} + +/// Build DmOptions for deferred device removal in a no-udev environment. +#[allow(dead_code)] +fn dm_opts_deferred_remove() -> DmOptions { + no_udev_dm_options().set_flags(DmFlags::DM_DEFERRED_REMOVE) +} + +/// Create a block device node for a dm-verity device using mknod(2). +#[allow(dead_code)] +fn create_dm_dev_node(name: &str, dev: devicemapper::Device) -> Result { + // Ensure /dev/mapper exists. + let mapper_dir = Path::new("/dev/mapper"); + if !mapper_dir.exists() { + std::fs::create_dir_all(mapper_dir) + .with_context(|| format!("failed to create directory {}", mapper_dir.display()))?; + } + + let dev_path = format!("/dev/mapper/{}", name); + // Remove stale node from a previous failed run, if any + if Path::new(&dev_path).exists() { + std::fs::remove_file(&dev_path) + .with_context(|| format!("failed to remove stale device node {}", dev_path))?; + } + + // Use Device -> dev_t conversion from the crate instead of raw makedev. + let dev_t: nix::libc::dev_t = dev.into(); + stat::mknod( + dev_path.as_str(), + SFlag::S_IFBLK, + Mode::from_bits_truncate(0o600), + dev_t, + ) + .with_context(|| format!("failed to mknod block device {}", dev_path))?; + + Ok(dev_path) +} + +/// Remove a device node that was created by create_dm_dev_node. +#[allow(dead_code)] +fn remove_dm_dev_node(dev_path: &str) { + if dev_path.starts_with("/dev/mapper/") && Path::new(dev_path).exists() { + if let Err(e) = std::fs::remove_file(dev_path) { + slog::warn!( + slog_scope::logger(), + "failed to remove dm device node"; + "path" => dev_path, + "error" => %e, + ); + } + } +} + #[derive(Debug)] pub struct MultiLayerErofsHandler {}