kata-deploy: validate erofs prerequisites

Extend the kata-deploy host-check stage with EROFS-specific
prerequisite validation.

Check that containerd supports the EROFS snapshotter, required
kernel features are available either as loaded modules or built-in
kernel configuration, and the host erofs-utils version satisfies the
minimum required by kata-deploy's configured mkfs options. Also warn
when kubelet runtimeRequestTimeout appears too low for EROFS layer
conversion, which can run during CreateContainer.

Read kubelet runtimeRequestTimeout from kubelet /configz through the
apiserver node proxy, and add the corresponding RBAC permission. Keep
fs-verity validation warning-only because user drop-ins may override
kata-deploy's default enable_fsverity setting and kata-deploy does
not yet validate the backing filesystem's fs-verity feature.

Signed-off-by: Manuel Huber <manuelh@nvidia.com>
Assisted-by: OpenAI <support@openai.com>

Cargo.lock

@@ -3718,12 +3718,14 @@ dependencies = [
  "anyhow",
  "clap",
  "env_logger 0.10.2",
+ "humantime",
  "k8s-openapi",
  "kube",
  "libc",
  "log",
  "regex",
  "rstest",
+ "semver",
  "serde_json",
  "serde_yaml 0.9.34+deprecated",
  "serial_test 0.10.0",

Cargo.toml

@@ -195,6 +195,7 @@ scan_fmt = "0.2.6"
 scopeguard = "1.0.0"
 serde = { version = "1.0.145", features = ["derive"] }
 serde_json = "1.0.91"
+semver = "1.0"
 serial_test = "0.10.0"
 sha2 = "0.11.0"
 slog = "2.5.2"

tools/packaging/kata-deploy/binary/Cargo.toml

@@ -14,6 +14,7 @@ path = "src/main.rs"
 anyhow.workspace = true
 clap.workspace = true
 env_logger = "0.10"
+humantime = "2.1.0"
 k8s-openapi = { version = "0.26", default-features = false, features = [
     "v1_33",
 ] }
@@ -32,6 +33,7 @@ kube = { version = "2.0", default-features = false, features = [
 libc.workspace = true
 log.workspace = true
 regex.workspace = true
+semver.workspace = true
 serde_json.workspace = true
 serde_yaml = "0.9"
 tar = "0.4.46"

tools/packaging/kata-deploy/binary/src/k8s/client.rs

@@ -7,7 +7,8 @@ use crate::config::Config;
 use anyhow::{Context, Result};
 use k8s_openapi::api::core::v1::Node;
 use kube::{
-    api::{Api, DeleteParams, DynamicObject, Patch, PatchParams},
+    api::{Api, DeleteParams, DynamicObject, GetParams, Patch, PatchParams},
+    core::Request,
     discovery::ApiResource,
     Client,
 };
@@ -71,6 +72,25 @@ impl K8sClient {
             .and_then(|labels| labels.get(key).cloned()))
     }
 
+    pub async fn get_kubelet_runtime_request_timeout(&self) -> Result<Option<String>> {
+        let request = Request::new(format!("/api/v1/nodes/{}/proxy", self.node_name))
+            .get("configz", &GetParams::default())?;
+
+        let configz: serde_json::Value = self.client.request(request).await.with_context(|| {
+            format!(
+                "Failed to query kubelet configz for node {}",
+                self.node_name
+            )
+        })?;
+
+        Ok(configz
+            .get("kubeletconfig")
+            .or_else(|| configz.get("kubeletConfig"))
+            .and_then(|kubelet_config| kubelet_config.get("runtimeRequestTimeout"))
+            .and_then(|value| value.as_str())
+            .map(|value| value.to_string()))
+    }
+
     pub async fn label_node(
         &self,
         label_key: &str,
@@ -612,6 +632,11 @@ pub async fn get_node_label(config: &Config, key: &str) -> Result<Option<String>
     client.get_node_label(key).await
 }
 
+pub async fn get_kubelet_runtime_request_timeout(config: &Config) -> Result<Option<String>> {
+    let client = K8sClient::new(&config.node_name).await?;
+    client.get_kubelet_runtime_request_timeout().await
+}
+
 pub async fn get_node_ready_status(config: &Config) -> Result<String> {
     let client = K8sClient::new(&config.node_name).await?;
     let node = client.get_node().await?;

tools/packaging/kata-deploy/binary/src/main.rs

@@ -13,6 +13,7 @@ mod utils;
 use anyhow::{Context, Result};
 use clap::Parser;
 use log::{error, info};
+use semver::Version;
 
 /// Env var name used to thread the detected container runtime through the
 /// post-install re-exec. Avoids re-querying the apiserver after we've already
@@ -99,6 +100,8 @@ enum Action {
 /// Node label applied to mark a node as kata-capable. Shared across the
 /// install/cleanup label stages so the key stays consistent.
 const KATA_RUNTIME_LABEL: &str = "katacontainers.io/kata-runtime";
+const SUGGESTED_KUBELET_RUNTIME_REQUEST_TIMEOUT_SECS: u64 = 10 * 60;
+const MIN_EROFS_UTILS_VERSION: &str = "1.8.2";
 
 // Cap the tokio runtime to a small fixed number of worker threads. The default
 // multi-thread runtime allocates `num_cpus()` workers (each with a ~2 MiB
@@ -429,8 +432,7 @@ async fn install_stage_host_check(config: &config::Config, runtime: &str) -> Res
                 for s in &non_empty_snapshotters {
                     match s.as_str() {
                         "erofs" => {
-                            runtime::containerd::containerd_erofs_snapshotter_version_check(config)
-                                .await?;
+                            validate_erofs_prerequisites(config).await?;
                         }
                         "nydus" => {}
                         _ => {
@@ -448,6 +450,262 @@ async fn install_stage_host_check(config: &config::Config, runtime: &str) -> Res
     Ok(())
 }
 
+async fn validate_erofs_prerequisites(config: &config::Config) -> Result<()> {
+    info!("Validating EROFS snapshotter prerequisites");
+
+    runtime::containerd::containerd_erofs_snapshotter_version_check(config).await?;
+
+    validate_host_kernel_feature_available(
+        HostKernelFeature::Erofs,
+        "Load or enable EROFS filesystem support before installing Kata and \
+         make it persistent across reboots.",
+    )?;
+
+    if config.erofs_dmverity {
+        validate_host_kernel_feature_available(
+            HostKernelFeature::DeviceMapper,
+            "Load or enable device-mapper support before installing Kata and \
+             make it persistent across reboots.",
+        )?;
+        validate_host_kernel_feature_available(
+            HostKernelFeature::DmVerity,
+            "Load or enable the dm-verity target before installing Kata and \
+             make it persistent across reboots.",
+        )?;
+    }
+
+    validate_mkfs_erofs_version()?;
+
+    // kata-deploy currently configures the EROFS snapshotter with
+    // enable_fsverity=true, but this host check does not know the final
+    // containerd configuration after user drop-ins, and it does not validate
+    // the backing filesystem's fs-verity feature. Keep this check warning-only.
+    warn_if_erofs_fsverity_may_be_unavailable();
+
+    validate_kubelet_runtime_request_timeout(config, "EROFS layer conversion").await?;
+
+    Ok(())
+}
+
+#[derive(Clone, Copy)]
+enum HostKernelFeature {
+    Erofs,
+    DeviceMapper,
+    DmVerity,
+    FsVerity,
+}
+
+impl HostKernelFeature {
+    fn name(self) -> &'static str {
+        match self {
+            Self::Erofs => "erofs",
+            Self::DeviceMapper => "device-mapper",
+            Self::DmVerity => "dm-verity",
+            Self::FsVerity => "fs-verity",
+        }
+    }
+
+    fn module_name(self) -> &'static str {
+        match self {
+            Self::Erofs => "erofs",
+            Self::DeviceMapper => "dm_mod",
+            Self::DmVerity => "dm_verity",
+            Self::FsVerity => "fsverity",
+        }
+    }
+
+    fn config_symbol(self) -> &'static str {
+        match self {
+            Self::Erofs => "CONFIG_EROFS_FS",
+            Self::DeviceMapper => "CONFIG_BLK_DEV_DM",
+            Self::DmVerity => "CONFIG_DM_VERITY",
+            Self::FsVerity => "CONFIG_FS_VERITY",
+        }
+    }
+}
+
+fn validate_host_kernel_feature_available(
+    feature: HostKernelFeature,
+    remediation: &str,
+) -> Result<()> {
+    if host_module_visible(feature.module_name())
+        || host_proc_config_has_builtin_feature(feature.config_symbol())
+        || host_boot_config_has_builtin_feature(feature.config_symbol())
+    {
+        return Ok(());
+    }
+
+    anyhow::bail!(
+        "Required host kernel feature `{}` is not available. {remediation}",
+        feature.name()
+    )
+}
+
+fn host_module_visible(module_name: &str) -> bool {
+    let sys_module_path = format!("/sys/module/{module_name}");
+    if utils::host_exec(&["test", "-d", &sys_module_path]).is_ok() {
+        return true;
+    }
+
+    let proc_modules_pattern = format!("^{module_name} ");
+    utils::host_exec(&["grep", "-q", &proc_modules_pattern, "/proc/modules"]).is_ok()
+}
+
+fn host_proc_config_has_builtin_feature(config_symbol: &str) -> bool {
+    let config_value = format!("{config_symbol}=y");
+
+    if utils::host_exec(&["test", "-r", "/proc/config.gz"]).is_err() {
+        return false;
+    }
+
+    let Ok(output) = utils::host_exec(&["gzip", "-dc", "/proc/config.gz"]) else {
+        return false;
+    };
+
+    output.lines().any(|line| line == config_value)
+}
+
+fn host_boot_config_has_builtin_feature(config_symbol: &str) -> bool {
+    let config_pattern = format!("^{config_symbol}=y");
+
+    let output = utils::host_exec(&["uname", "-r"]);
+    let Ok(kernel_release) = output else {
+        return false;
+    };
+
+    let kernel_config_path = format!("/boot/config-{}", kernel_release.trim());
+    utils::host_exec(&["grep", "-Eq", &config_pattern, &kernel_config_path]).is_ok()
+}
+
+fn validate_mkfs_erofs_version() -> Result<()> {
+    let output = utils::host_exec(&["mkfs.erofs", "--version"]).with_context(|| {
+        "Required host command `mkfs.erofs` is not available. Install \
+         erofs-utils >= 1.8.2 before enabling the EROFS snapshotter."
+    })?;
+
+    let version = parse_erofs_utils_version(&output).with_context(|| {
+        format!("Could not parse erofs-utils version from `mkfs.erofs --version`: {output}")
+    })?;
+    let minimum_version = Version::parse(MIN_EROFS_UTILS_VERSION)?;
+
+    if version < minimum_version {
+        anyhow::bail!(
+            "Host erofs-utils version {} is too old. kata-deploy configures \
+             EROFS fsmerge mkfs_options that require erofs-utils >= {}.",
+            version,
+            MIN_EROFS_UTILS_VERSION
+        );
+    }
+
+    info!(
+        "host erofs-utils version {} satisfies minimum {}",
+        version, MIN_EROFS_UTILS_VERSION
+    );
+
+    Ok(())
+}
+
+fn parse_erofs_utils_version(output: &str) -> Result<Version> {
+    let version_re = regex::Regex::new(r"([0-9]+)\.([0-9]+)(?:\.([0-9]+))?")?;
+    let captures = version_re
+        .captures(output)
+        .ok_or_else(|| anyhow::anyhow!("erofs-utils version not found"))?;
+
+    let major = captures[1].parse::<u64>()?;
+    let minor = captures[2].parse::<u64>()?;
+    let patch = captures
+        .get(3)
+        .map(|patch| patch.as_str().parse::<u64>())
+        .transpose()?
+        .unwrap_or(0);
+
+    Version::parse(&format!("{major}.{minor}.{patch}")).map_err(Into::into)
+}
+
+fn warn_if_erofs_fsverity_may_be_unavailable() {
+    if let Err(err) = validate_host_kernel_feature_available(
+        HostKernelFeature::FsVerity,
+        "Install, load, or enable fs-verity support if the final EROFS \
+         snapshotter configuration keeps enable_fsverity=true.",
+    ) {
+        log::warn!(
+            "kata-deploy's default EROFS snapshotter configuration sets \
+             enable_fsverity=true, but host fs-verity support was not detected \
+             ({err}). This is warning-only because the final containerd \
+             configuration may be changed by user drop-ins, and kata-deploy \
+             does not yet validate the backing filesystem's fs-verity feature."
+        );
+    } else {
+        log::warn!(
+            "kata-deploy's default EROFS snapshotter configuration sets \
+             enable_fsverity=true and host fs-verity support was detected, but \
+             kata-deploy does not yet validate the backing filesystem's \
+             fs-verity feature."
+        );
+    }
+}
+
+async fn validate_kubelet_runtime_request_timeout(
+    config: &config::Config,
+    operation: &str,
+) -> Result<()> {
+    let runtime_request_timeout = match k8s::get_kubelet_runtime_request_timeout(config).await {
+        Ok(Some(value)) => value,
+        Ok(None) => {
+            warn_runtime_request_timeout(
+                operation,
+                "kubelet /configz did not include runtimeRequestTimeout",
+            );
+            return Ok(());
+        }
+        Err(err) => {
+            warn_runtime_request_timeout(
+                operation,
+                &format!("could not query kubelet runtimeRequestTimeout from /configz: {err}"),
+            );
+            return Ok(());
+        }
+    };
+
+    let timeout_secs = match humantime::parse_duration(&runtime_request_timeout) {
+        Ok(timeout) => timeout.as_secs(),
+        Err(err) => {
+            warn_runtime_request_timeout(
+                operation,
+                &format!(
+                    "could not parse kubelet runtimeRequestTimeout value \
+                     `{runtime_request_timeout}` from /configz: {err}"
+                ),
+            );
+            return Ok(());
+        }
+    };
+
+    if timeout_secs < SUGGESTED_KUBELET_RUNTIME_REQUEST_TIMEOUT_SECS {
+        warn_runtime_request_timeout(
+            operation,
+            &format!(
+                "kubelet runtimeRequestTimeout from /configz is \
+                 `{runtime_request_timeout}` ({timeout_secs}s)"
+            ),
+        );
+    }
+
+    info!(
+        "kubelet runtimeRequestTimeout from /configz is {runtime_request_timeout} ({timeout_secs}s)"
+    );
+    Ok(())
+}
+
+fn warn_runtime_request_timeout(operation: &str, detail: &str) {
+    log::warn!(
+        "{detail}. {operation} may run during CreateContainer; consider \
+         configuring kubelet runtimeRequestTimeout to at least {}s on nodes \
+         that run large images.",
+        SUGGESTED_KUBELET_RUNTIME_REQUEST_TIMEOUT_SECS
+    );
+}
+
 /// Install stage 1 (artifacts): place kata artifacts/config on the host and set
 /// up any configured snapshotters. This does not touch CRI configuration, but it
 /// still needs privileged host access: writing under the host install dir and
@@ -927,6 +1185,22 @@ mod tests {
         );
     }
 
+    #[rstest]
+    #[case("mkfs.erofs (erofs-utils) 1.9\navailable compressors: lz4\n", "1.9.0")]
+    #[case("mkfs.erofs (erofs-utils) 1.8.2\n", "1.8.2")]
+    #[case("erofs-utils 1.8\n", "1.8.0")]
+    fn test_parse_erofs_utils_version(#[case] output: &str, #[case] expected: &str) {
+        assert_eq!(
+            parse_erofs_utils_version(output).unwrap(),
+            Version::parse(expected).unwrap()
+        );
+    }
+
+    #[test]
+    fn test_parse_erofs_utils_version_rejects_invalid_output() {
+        assert!(parse_erofs_utils_version("mkfs.erofs unknown").is_err());
+    }
+
     /// All non-internal staged actions remain visible in `--help` so operators
     /// can discover and run individual stages.
     #[rstest]

tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml

@@ -26,6 +26,11 @@ rules:
 - apiGroups: [""]
   resources: ["nodes"]
   verbs: ["get", "patch"]
+# Read kubelet /configz via the apiserver node proxy during host-checks, for
+# example to warn about runtimeRequestTimeout values that may be too low.
+- apiGroups: [""]
+  resources: ["nodes/proxy"]
+  verbs: ["get"]
 - apiGroups: ["node.k8s.io"]
   resources: ["runtimeclasses"]
   verbs: ["get", "list", "patch"]