Merge pull request #13347 from manuelh-dev/mahuber/erofs-deployment-validation

docs/kata-deploy: improvements to erofs snapshotter documentation and deployment validation

Cargo.lock

@@ -3718,12 +3718,14 @@ dependencies = [
  "anyhow",
  "clap",
  "env_logger 0.10.2",
+ "humantime",
  "k8s-openapi",
  "kube",
  "libc",
  "log",
  "regex",
  "rstest",
+ "semver",
  "serde_json",
  "serde_yaml 0.9.34+deprecated",
  "serial_test 0.10.0",

Cargo.toml

@@ -195,6 +195,7 @@ scan_fmt = "0.2.6"
 scopeguard = "1.0.0"
 serde = { version = "1.0.145", features = ["derive"] }
 serde_json = "1.0.91"
+semver = "1.0"
 serial_test = "0.10.0"
 sha2 = "0.11.0"
 slog = "2.5.2"

docs/how-to/how-to-use-erofs-snapshotter-with-kata.md

@@ -15,7 +15,7 @@ This section provides a quick overview of the steps to get started with EROFS sn
 
 ### Quick Steps
 
-1. **Install erofs-utils**: Install erofs-utils (version >= 1.8) on your host system
+1. **Install erofs-utils**: Install erofs-utils (version >= 1.8.2) on your host system
 2. **Configure containerd**: Enable EROFS snapshotter and differ in containerd configuration
 3. **Configure Kata Containers**: Set up runtime-rs with appropriate hypervisor settings
 4. **Run a container**: Use `ctr` or Kubernetes to run containers with EROFS snapshotter
@@ -25,17 +25,29 @@ This section provides a quick overview of the steps to get started with EROFS sn
 | Component | Version Requirement |
 |-----------|-------------------|
 | Linux kernel | >= 5.4 (with `erofs` module, higher recommended) |
-| erofs-utils | >= 1.8 (fsmerge `mkfs_options` require >= 1.8.2) |
+| erofs-utils | >= 1.8.2 |
 | containerd | >= 2.2 (with EROFS snapshotter and differ support, higher recommended) |
 | Kata Containers | Latest `main` branch with runtime-rs |
 | QEMU | >= 5.0 (VMDK flat-extent support and >= 9.0  higher recommended) |
 
+> **Note**: When enabling the EROFS snapshotter on a node that is already in
+> use, existing images in the containerd image store may not yet have EROFS
+> snapshots. Since those images are already present, kubelet may skip pulling
+> them again, so containerd can end up converting their layers into EROFS
+> snapshots during `CreateContainer` rather than during the image pull phase.
+> Configure kubelet's `runtimeRequestTimeout` to cover the slowest expected
+> EROFS preparation path for the target node and workload. Large images can
+> otherwise exceed kubelet's CRI request deadline before the sandbox starts.
+
 ## Installation Guide
 
 This section provides detailed step-by-step instructions for installing and configuring EROFS snapshotter with Kata Containers.
 
 ### Step 1: Install erofs-utils
 
+The configuration below uses fsmerge `mkfs_options` that require erofs-utils
+>= 1.8.2.
+
 ```bash
 # Debian/Ubuntu
 $ sudo apt install erofs-utils
@@ -48,7 +60,7 @@ Verify the version:
 
 ```bash
 $ mkfs.erofs --version
-# Should show 1.8 or higher (1.8.2+ required for fsmerge mkfs_options)
+# Should show 1.8.2 or higher
 ```
 
 Load the kernel module:
@@ -57,11 +69,25 @@ Load the kernel module:
 $ sudo modprobe erofs
 ```
 
+When using EROFS dm-verity mode, also load the dm-verity device-mapper target:
+
+```bash
+$ sudo modprobe dm-mod
+$ sudo modprobe dm-verity
+```
+
+Make these modules load persistently before containerd starts, for example via
+your distribution's modules-load mechanism. If the required kernel support is
+missing when containerd starts, enabling the EROFS snapshotter can fail early
+instead of producing a useful workload-level error.
+
 ### Step 2: Configure containerd
 
 #### Enable the EROFS snapshotter and differ
 
-> **Note**: The following settings target containerd v2.3.0-beta.0 and erofs-utils v1.8.10. Compatibility for other versions is not guaranteed, as configuration options evolve. Always cross-reference with the official documentation for your current version.
+> **Note**: The following settings target containerd v2.3.0. The
+> configured EROFS fsmerge `mkfs_options` require erofs-utils >= 1.8.2. Always
+> cross-reference with the official documentation for your current versions.
 
 Edit your containerd configuration (typically `/etc/containerd/config.toml`):
 
@@ -91,8 +117,46 @@ version = 3
   [plugins.'io.containerd.snapshotter.v1.erofs']
     default_size = '<SIZE>' # SIZE=6G or 10G or other size
     max_unmerged_layers = 0
+    enable_fsverity = true
 ```
 
+`enable_fsverity = true` enables host-side fs-verity handling for EROFS layer
+blobs when the kernel and backing filesystem support it. This is separate from
+the dm-verity mode described below: fs-verity protects layer blob files on the
+host, while dm-verity protects the block devices mounted by the Kata guest.
+
+Containerd treats fs-verity support as best-effort and may skip it if the host
+kernel or backing filesystem does not support it. If fs-verity protection is
+required, make sure the filesystem backing containerd's EROFS snapshotter state
+supports fs-verity. For example, on ext4 this requires the `verity` filesystem
+feature on the relevant device:
+
+```bash
+$ sudo tune2fs -O verity <device-backing-containerd-state>
+```
+
+For dm-verity, enable metadata generation in the differ and force strict
+dm-verity use in the snapshotter:
+
+```toml
+  [plugins.'io.containerd.differ.v1.erofs']
+    enable_dmverity = true
+
+  [plugins.'io.containerd.snapshotter.v1.erofs']
+    dmverity_mode = 'on'
+```
+
+Avoid switching a persistent node back and forth between dm-verity and
+non-dm-verity EROFS modes without cleaning or rebuilding the EROFS snapshotter
+state. Keep dm-verity consistently enabled or consistently disabled for a given
+snapshotter state, because existing layers may have been prepared without
+dm-verity metadata.
+
+When dm-verity is enabled, avoid `dmverity_mode = 'auto'`. Auto mode does not
+solve stale layer state, because layers prepared without dm-verity metadata can
+still be used without dm-verity protection. Strict `dmverity_mode = 'on'` makes
+such stale layers fail instead of silently falling back to an unprotected path.
+
 #### Verify the EROFS plugins are loaded
 
 Check if EROFS module is loaded

tools/packaging/kata-deploy/binary/Cargo.toml

@@ -14,6 +14,7 @@ path = "src/main.rs"
 anyhow.workspace = true
 clap.workspace = true
 env_logger = "0.10"
+humantime = "2.1.0"
 k8s-openapi = { version = "0.26", default-features = false, features = [
     "v1_33",
 ] }
@@ -32,6 +33,7 @@ kube = { version = "2.0", default-features = false, features = [
 libc.workspace = true
 log.workspace = true
 regex.workspace = true
+semver.workspace = true
 serde_json.workspace = true
 serde_yaml = "0.9"
 tar = "0.4.46"

tools/packaging/kata-deploy/binary/src/k8s/client.rs

@@ -7,7 +7,8 @@ use crate::config::Config;
 use anyhow::{Context, Result};
 use k8s_openapi::api::core::v1::Node;
 use kube::{
-    api::{Api, DeleteParams, DynamicObject, Patch, PatchParams},
+    api::{Api, DeleteParams, DynamicObject, GetParams, Patch, PatchParams},
+    core::Request,
     discovery::ApiResource,
     Client,
 };
@@ -71,6 +72,25 @@ impl K8sClient {
             .and_then(|labels| labels.get(key).cloned()))
     }
 
+    pub async fn get_kubelet_runtime_request_timeout(&self) -> Result<Option<String>> {
+        let request = Request::new(format!("/api/v1/nodes/{}/proxy", self.node_name))
+            .get("configz", &GetParams::default())?;
+
+        let configz: serde_json::Value = self.client.request(request).await.with_context(|| {
+            format!(
+                "Failed to query kubelet configz for node {}",
+                self.node_name
+            )
+        })?;
+
+        Ok(configz
+            .get("kubeletconfig")
+            .or_else(|| configz.get("kubeletConfig"))
+            .and_then(|kubelet_config| kubelet_config.get("runtimeRequestTimeout"))
+            .and_then(|value| value.as_str())
+            .map(|value| value.to_string()))
+    }
+
     pub async fn label_node(
         &self,
         label_key: &str,
@@ -612,6 +632,11 @@ pub async fn get_node_label(config: &Config, key: &str) -> Result<Option<String>
     client.get_node_label(key).await
 }
 
+pub async fn get_kubelet_runtime_request_timeout(config: &Config) -> Result<Option<String>> {
+    let client = K8sClient::new(&config.node_name).await?;
+    client.get_kubelet_runtime_request_timeout().await
+}
+
 pub async fn get_node_ready_status(config: &Config) -> Result<String> {
     let client = K8sClient::new(&config.node_name).await?;
     let node = client.get_node().await?;

tools/packaging/kata-deploy/binary/src/main.rs

@@ -13,6 +13,7 @@ mod utils;
 use anyhow::{Context, Result};
 use clap::Parser;
 use log::{error, info};
+use semver::Version;
 
 /// Env var name used to thread the detected container runtime through the
 /// post-install re-exec. Avoids re-querying the apiserver after we've already
@@ -99,6 +100,8 @@ enum Action {
 /// Node label applied to mark a node as kata-capable. Shared across the
 /// install/cleanup label stages so the key stays consistent.
 const KATA_RUNTIME_LABEL: &str = "katacontainers.io/kata-runtime";
+const SUGGESTED_KUBELET_RUNTIME_REQUEST_TIMEOUT_SECS: u64 = 10 * 60;
+const MIN_EROFS_UTILS_VERSION: &str = "1.8.2";
 
 // Cap the tokio runtime to a small fixed number of worker threads. The default
 // multi-thread runtime allocates `num_cpus()` workers (each with a ~2 MiB
@@ -429,8 +432,7 @@ async fn install_stage_host_check(config: &config::Config, runtime: &str) -> Res
                 for s in &non_empty_snapshotters {
                     match s.as_str() {
                         "erofs" => {
-                            runtime::containerd::containerd_erofs_snapshotter_version_check(config)
-                                .await?;
+                            validate_erofs_prerequisites(config).await?;
                         }
                         "nydus" => {}
                         _ => {
@@ -444,10 +446,292 @@ async fn install_stage_host_check(config: &config::Config, runtime: &str) -> Res
         }
     }
 
+    if config_uses_guest_pull(config) {
+        validate_kubelet_runtime_request_timeout(config, "guest pull").await?;
+    }
+
     info!("install (host-check): node prerequisites satisfied");
     Ok(())
 }
 
+async fn validate_erofs_prerequisites(config: &config::Config) -> Result<()> {
+    info!("Validating EROFS snapshotter prerequisites");
+
+    runtime::containerd::containerd_erofs_snapshotter_version_check(config).await?;
+
+    validate_host_kernel_feature_available(
+        HostKernelFeature::Erofs,
+        "Load or enable EROFS filesystem support before installing Kata and \
+         make it persistent across reboots.",
+    )?;
+
+    if config.erofs_dmverity {
+        validate_host_kernel_feature_available(
+            HostKernelFeature::DeviceMapper,
+            "Load or enable device-mapper support before installing Kata and \
+             make it persistent across reboots.",
+        )?;
+        validate_host_kernel_feature_available(
+            HostKernelFeature::DmVerity,
+            "Load or enable the dm-verity target before installing Kata and \
+             make it persistent across reboots.",
+        )?;
+    }
+
+    validate_mkfs_erofs_version()?;
+
+    // kata-deploy currently configures the EROFS snapshotter with
+    // enable_fsverity=true, but this host check does not know the final
+    // containerd configuration after user drop-ins, and it does not validate
+    // the backing filesystem's fs-verity feature. Keep this check warning-only.
+    warn_if_erofs_fsverity_may_be_unavailable();
+
+    validate_kubelet_runtime_request_timeout(config, "EROFS layer conversion").await?;
+
+    Ok(())
+}
+
+#[derive(Clone, Copy)]
+enum HostKernelFeature {
+    Erofs,
+    DeviceMapper,
+    DmVerity,
+    FsVerity,
+}
+
+impl HostKernelFeature {
+    fn name(self) -> &'static str {
+        match self {
+            Self::Erofs => "erofs",
+            Self::DeviceMapper => "device-mapper",
+            Self::DmVerity => "dm-verity",
+            Self::FsVerity => "fs-verity",
+        }
+    }
+
+    fn module_name(self) -> &'static str {
+        match self {
+            Self::Erofs => "erofs",
+            Self::DeviceMapper => "dm_mod",
+            Self::DmVerity => "dm_verity",
+            Self::FsVerity => "fsverity",
+        }
+    }
+
+    fn config_symbol(self) -> &'static str {
+        match self {
+            Self::Erofs => "CONFIG_EROFS_FS",
+            Self::DeviceMapper => "CONFIG_BLK_DEV_DM",
+            Self::DmVerity => "CONFIG_DM_VERITY",
+            Self::FsVerity => "CONFIG_FS_VERITY",
+        }
+    }
+}
+
+fn validate_host_kernel_feature_available(
+    feature: HostKernelFeature,
+    remediation: &str,
+) -> Result<()> {
+    if host_module_visible(feature.module_name())
+        || host_proc_config_has_builtin_feature(feature.config_symbol())
+        || host_boot_config_has_builtin_feature(feature.config_symbol())
+    {
+        return Ok(());
+    }
+
+    anyhow::bail!(
+        "Required host kernel feature `{}` is not available. {remediation}",
+        feature.name()
+    )
+}
+
+fn host_module_visible(module_name: &str) -> bool {
+    let sys_module_path = format!("/sys/module/{module_name}");
+    if utils::host_exec(&["test", "-d", &sys_module_path]).is_ok() {
+        return true;
+    }
+
+    let proc_modules_pattern = format!("^{module_name} ");
+    utils::host_exec(&["grep", "-q", &proc_modules_pattern, "/proc/modules"]).is_ok()
+}
+
+fn host_proc_config_has_builtin_feature(config_symbol: &str) -> bool {
+    let config_value = format!("{config_symbol}=y");
+
+    if utils::host_exec(&["test", "-r", "/proc/config.gz"]).is_err() {
+        return false;
+    }
+
+    let Ok(output) = utils::host_exec(&["gzip", "-dc", "/proc/config.gz"]) else {
+        return false;
+    };
+
+    output.lines().any(|line| line == config_value)
+}
+
+fn host_boot_config_has_builtin_feature(config_symbol: &str) -> bool {
+    let config_pattern = format!("^{config_symbol}=y");
+
+    let output = utils::host_exec(&["uname", "-r"]);
+    let Ok(kernel_release) = output else {
+        return false;
+    };
+
+    let kernel_config_path = format!("/boot/config-{}", kernel_release.trim());
+    utils::host_exec(&["grep", "-Eq", &config_pattern, &kernel_config_path]).is_ok()
+}
+
+fn validate_mkfs_erofs_version() -> Result<()> {
+    let output = utils::host_exec(&["mkfs.erofs", "--version"]).with_context(|| {
+        "Required host command `mkfs.erofs` is not available. Install \
+         erofs-utils >= 1.8.2 before enabling the EROFS snapshotter."
+    })?;
+
+    let version = parse_erofs_utils_version(&output).with_context(|| {
+        format!("Could not parse erofs-utils version from `mkfs.erofs --version`: {output}")
+    })?;
+    let minimum_version = Version::parse(MIN_EROFS_UTILS_VERSION)?;
+
+    if version < minimum_version {
+        anyhow::bail!(
+            "Host erofs-utils version {} is too old. kata-deploy configures \
+             EROFS fsmerge mkfs_options that require erofs-utils >= {}.",
+            version,
+            MIN_EROFS_UTILS_VERSION
+        );
+    }
+
+    info!(
+        "host erofs-utils version {} satisfies minimum {}",
+        version, MIN_EROFS_UTILS_VERSION
+    );
+
+    Ok(())
+}
+
+fn parse_erofs_utils_version(output: &str) -> Result<Version> {
+    let version_re = regex::Regex::new(r"([0-9]+)\.([0-9]+)(?:\.([0-9]+))?")?;
+    let captures = version_re
+        .captures(output)
+        .ok_or_else(|| anyhow::anyhow!("erofs-utils version not found"))?;
+
+    let major = captures[1].parse::<u64>()?;
+    let minor = captures[2].parse::<u64>()?;
+    let patch = captures
+        .get(3)
+        .map(|patch| patch.as_str().parse::<u64>())
+        .transpose()?
+        .unwrap_or(0);
+
+    Version::parse(&format!("{major}.{minor}.{patch}")).map_err(Into::into)
+}
+
+fn warn_if_erofs_fsverity_may_be_unavailable() {
+    if let Err(err) = validate_host_kernel_feature_available(
+        HostKernelFeature::FsVerity,
+        "Install, load, or enable fs-verity support if the final EROFS \
+         snapshotter configuration keeps enable_fsverity=true.",
+    ) {
+        log::warn!(
+            "kata-deploy's default EROFS snapshotter configuration sets \
+             enable_fsverity=true, but host fs-verity support was not detected \
+             ({err}). This is warning-only because the final containerd \
+             configuration may be changed by user drop-ins, and kata-deploy \
+             does not yet validate the backing filesystem's fs-verity feature."
+        );
+    } else {
+        log::warn!(
+            "kata-deploy's default EROFS snapshotter configuration sets \
+             enable_fsverity=true and host fs-verity support was detected, but \
+             kata-deploy does not yet validate the backing filesystem's \
+             fs-verity feature."
+        );
+    }
+}
+
+async fn validate_kubelet_runtime_request_timeout(
+    config: &config::Config,
+    operation: &str,
+) -> Result<()> {
+    let runtime_request_timeout = match k8s::get_kubelet_runtime_request_timeout(config).await {
+        Ok(Some(value)) => value,
+        Ok(None) => {
+            warn_runtime_request_timeout(
+                operation,
+                "kubelet /configz did not include runtimeRequestTimeout",
+            );
+            return Ok(());
+        }
+        Err(err) => {
+            warn_runtime_request_timeout(
+                operation,
+                &format!("could not query kubelet runtimeRequestTimeout from /configz: {err}"),
+            );
+            return Ok(());
+        }
+    };
+
+    let timeout_secs = match humantime::parse_duration(&runtime_request_timeout) {
+        Ok(timeout) => timeout.as_secs(),
+        Err(err) => {
+            warn_runtime_request_timeout(
+                operation,
+                &format!(
+                    "could not parse kubelet runtimeRequestTimeout value \
+                     `{runtime_request_timeout}` from /configz: {err}"
+                ),
+            );
+            return Ok(());
+        }
+    };
+
+    if timeout_secs < SUGGESTED_KUBELET_RUNTIME_REQUEST_TIMEOUT_SECS {
+        warn_runtime_request_timeout(
+            operation,
+            &format!(
+                "kubelet runtimeRequestTimeout from /configz is \
+                 `{runtime_request_timeout}` ({timeout_secs}s)"
+            ),
+        );
+    }
+
+    info!(
+        "kubelet runtimeRequestTimeout from /configz is {runtime_request_timeout} ({timeout_secs}s)"
+    );
+    Ok(())
+}
+
+fn warn_runtime_request_timeout(operation: &str, detail: &str) {
+    log::warn!(
+        "{detail}. {operation} may run during CreateContainer; consider \
+         configuring kubelet runtimeRequestTimeout to at least {}s on nodes \
+         that run large images.",
+        SUGGESTED_KUBELET_RUNTIME_REQUEST_TIMEOUT_SECS
+    );
+}
+
+fn config_uses_guest_pull(config: &config::Config) -> bool {
+    !config.experimental_force_guest_pull_for_arch.is_empty()
+        || mapping_contains_value(config.pull_type_mapping_for_arch.as_deref(), "guest-pull")
+        || config
+            .custom_runtimes
+            .iter()
+            .any(|runtime| runtime.crio_pull_type.as_deref() == Some("guest-pull"))
+}
+
+fn mapping_contains_value(mapping: Option<&str>, expected_value: &str) -> bool {
+    mapping.is_some_and(|mapping| {
+        mapping.split(',').any(|entry| {
+            let value = entry
+                .split_once(':')
+                .map(|(_, value)| value)
+                .unwrap_or(entry)
+                .trim();
+            value == expected_value
+        })
+    })
+}
+
 /// Install stage 1 (artifacts): place kata artifacts/config on the host and set
 /// up any configured snapshotters. This does not touch CRI configuration, but it
 /// still needs privileged host access: writing under the host install dir and
@@ -927,6 +1211,22 @@ mod tests {
         );
     }
 
+    #[rstest]
+    #[case("mkfs.erofs (erofs-utils) 1.9\navailable compressors: lz4\n", "1.9.0")]
+    #[case("mkfs.erofs (erofs-utils) 1.8.2\n", "1.8.2")]
+    #[case("erofs-utils 1.8\n", "1.8.0")]
+    fn test_parse_erofs_utils_version(#[case] output: &str, #[case] expected: &str) {
+        assert_eq!(
+            parse_erofs_utils_version(output).unwrap(),
+            Version::parse(expected).unwrap()
+        );
+    }
+
+    #[test]
+    fn test_parse_erofs_utils_version_rejects_invalid_output() {
+        assert!(parse_erofs_utils_version("mkfs.erofs unknown").is_err());
+    }
+
     /// All non-internal staged actions remain visible in `--help` so operators
     /// can discover and run individual stages.
     #[rstest]

tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml

@@ -26,6 +26,11 @@ rules:
 - apiGroups: [""]
   resources: ["nodes"]
   verbs: ["get", "patch"]
+# Read kubelet /configz via the apiserver node proxy during host-checks, for
+# example to warn about runtimeRequestTimeout values that may be too low.
+- apiGroups: [""]
+  resources: ["nodes/proxy"]
+  verbs: ["get"]
 - apiGroups: ["node.k8s.io"]
   resources: ["runtimeclasses"]
   verbs: ["get", "list", "patch"]