diff --git a/Cargo.lock b/Cargo.lock index 8ccb51dcd5..81b7f91b5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3718,12 +3718,14 @@ dependencies = [ "anyhow", "clap", "env_logger 0.10.2", + "humantime", "k8s-openapi", "kube", "libc", "log", "regex", "rstest", + "semver", "serde_json", "serde_yaml 0.9.34+deprecated", "serial_test 0.10.0", diff --git a/Cargo.toml b/Cargo.toml index e4744642ac..6b424d2e34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -195,6 +195,7 @@ scan_fmt = "0.2.6" scopeguard = "1.0.0" serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0.91" +semver = "1.0" serial_test = "0.10.0" sha2 = "0.11.0" slog = "2.5.2" diff --git a/docs/how-to/how-to-use-erofs-snapshotter-with-kata.md b/docs/how-to/how-to-use-erofs-snapshotter-with-kata.md index 2460f5a780..c287f581b8 100644 --- a/docs/how-to/how-to-use-erofs-snapshotter-with-kata.md +++ b/docs/how-to/how-to-use-erofs-snapshotter-with-kata.md @@ -15,7 +15,7 @@ This section provides a quick overview of the steps to get started with EROFS sn ### Quick Steps -1. **Install erofs-utils**: Install erofs-utils (version >= 1.8) on your host system +1. **Install erofs-utils**: Install erofs-utils (version >= 1.8.2) on your host system 2. **Configure containerd**: Enable EROFS snapshotter and differ in containerd configuration 3. **Configure Kata Containers**: Set up runtime-rs with appropriate hypervisor settings 4. **Run a container**: Use `ctr` or Kubernetes to run containers with EROFS snapshotter @@ -25,17 +25,29 @@ This section provides a quick overview of the steps to get started with EROFS sn | Component | Version Requirement | |-----------|-------------------| | Linux kernel | >= 5.4 (with `erofs` module, higher recommended) | -| erofs-utils | >= 1.8 (fsmerge `mkfs_options` require >= 1.8.2) | +| erofs-utils | >= 1.8.2 | | containerd | >= 2.2 (with EROFS snapshotter and differ support, higher recommended) | | Kata Containers | Latest `main` branch with runtime-rs | | QEMU | >= 5.0 (VMDK flat-extent support and >= 9.0 higher recommended) | +> **Note**: When enabling the EROFS snapshotter on a node that is already in +> use, existing images in the containerd image store may not yet have EROFS +> snapshots. Since those images are already present, kubelet may skip pulling +> them again, so containerd can end up converting their layers into EROFS +> snapshots during `CreateContainer` rather than during the image pull phase. +> Configure kubelet's `runtimeRequestTimeout` to cover the slowest expected +> EROFS preparation path for the target node and workload. Large images can +> otherwise exceed kubelet's CRI request deadline before the sandbox starts. + ## Installation Guide This section provides detailed step-by-step instructions for installing and configuring EROFS snapshotter with Kata Containers. ### Step 1: Install erofs-utils +The configuration below uses fsmerge `mkfs_options` that require erofs-utils +>= 1.8.2. + ```bash # Debian/Ubuntu $ sudo apt install erofs-utils @@ -48,7 +60,7 @@ Verify the version: ```bash $ mkfs.erofs --version -# Should show 1.8 or higher (1.8.2+ required for fsmerge mkfs_options) +# Should show 1.8.2 or higher ``` Load the kernel module: @@ -57,11 +69,25 @@ Load the kernel module: $ sudo modprobe erofs ``` +When using EROFS dm-verity mode, also load the dm-verity device-mapper target: + +```bash +$ sudo modprobe dm-mod +$ sudo modprobe dm-verity +``` + +Make these modules load persistently before containerd starts, for example via +your distribution's modules-load mechanism. If the required kernel support is +missing when containerd starts, enabling the EROFS snapshotter can fail early +instead of producing a useful workload-level error. + ### Step 2: Configure containerd #### Enable the EROFS snapshotter and differ -> **Note**: The following settings target containerd v2.3.0-beta.0 and erofs-utils v1.8.10. Compatibility for other versions is not guaranteed, as configuration options evolve. Always cross-reference with the official documentation for your current version. +> **Note**: The following settings target containerd v2.3.0. The +> configured EROFS fsmerge `mkfs_options` require erofs-utils >= 1.8.2. Always +> cross-reference with the official documentation for your current versions. Edit your containerd configuration (typically `/etc/containerd/config.toml`): @@ -91,8 +117,46 @@ version = 3 [plugins.'io.containerd.snapshotter.v1.erofs'] default_size = '' # SIZE=6G or 10G or other size max_unmerged_layers = 0 + enable_fsverity = true ``` +`enable_fsverity = true` enables host-side fs-verity handling for EROFS layer +blobs when the kernel and backing filesystem support it. This is separate from +the dm-verity mode described below: fs-verity protects layer blob files on the +host, while dm-verity protects the block devices mounted by the Kata guest. + +Containerd treats fs-verity support as best-effort and may skip it if the host +kernel or backing filesystem does not support it. If fs-verity protection is +required, make sure the filesystem backing containerd's EROFS snapshotter state +supports fs-verity. For example, on ext4 this requires the `verity` filesystem +feature on the relevant device: + +```bash +$ sudo tune2fs -O verity +``` + +For dm-verity, enable metadata generation in the differ and force strict +dm-verity use in the snapshotter: + +```toml + [plugins.'io.containerd.differ.v1.erofs'] + enable_dmverity = true + + [plugins.'io.containerd.snapshotter.v1.erofs'] + dmverity_mode = 'on' +``` + +Avoid switching a persistent node back and forth between dm-verity and +non-dm-verity EROFS modes without cleaning or rebuilding the EROFS snapshotter +state. Keep dm-verity consistently enabled or consistently disabled for a given +snapshotter state, because existing layers may have been prepared without +dm-verity metadata. + +When dm-verity is enabled, avoid `dmverity_mode = 'auto'`. Auto mode does not +solve stale layer state, because layers prepared without dm-verity metadata can +still be used without dm-verity protection. Strict `dmverity_mode = 'on'` makes +such stale layers fail instead of silently falling back to an unprotected path. + #### Verify the EROFS plugins are loaded Check if EROFS module is loaded diff --git a/tools/packaging/kata-deploy/binary/Cargo.toml b/tools/packaging/kata-deploy/binary/Cargo.toml index e8753922ce..a67eda5836 100644 --- a/tools/packaging/kata-deploy/binary/Cargo.toml +++ b/tools/packaging/kata-deploy/binary/Cargo.toml @@ -14,6 +14,7 @@ path = "src/main.rs" anyhow.workspace = true clap.workspace = true env_logger = "0.10" +humantime = "2.1.0" k8s-openapi = { version = "0.26", default-features = false, features = [ "v1_33", ] } @@ -32,6 +33,7 @@ kube = { version = "2.0", default-features = false, features = [ libc.workspace = true log.workspace = true regex.workspace = true +semver.workspace = true serde_json.workspace = true serde_yaml = "0.9" tar = "0.4.46" diff --git a/tools/packaging/kata-deploy/binary/src/k8s/client.rs b/tools/packaging/kata-deploy/binary/src/k8s/client.rs index e624b32d2e..b5c4504b81 100644 --- a/tools/packaging/kata-deploy/binary/src/k8s/client.rs +++ b/tools/packaging/kata-deploy/binary/src/k8s/client.rs @@ -7,7 +7,8 @@ use crate::config::Config; use anyhow::{Context, Result}; use k8s_openapi::api::core::v1::Node; use kube::{ - api::{Api, DeleteParams, DynamicObject, Patch, PatchParams}, + api::{Api, DeleteParams, DynamicObject, GetParams, Patch, PatchParams}, + core::Request, discovery::ApiResource, Client, }; @@ -71,6 +72,25 @@ impl K8sClient { .and_then(|labels| labels.get(key).cloned())) } + pub async fn get_kubelet_runtime_request_timeout(&self) -> Result> { + let request = Request::new(format!("/api/v1/nodes/{}/proxy", self.node_name)) + .get("configz", &GetParams::default())?; + + let configz: serde_json::Value = self.client.request(request).await.with_context(|| { + format!( + "Failed to query kubelet configz for node {}", + self.node_name + ) + })?; + + Ok(configz + .get("kubeletconfig") + .or_else(|| configz.get("kubeletConfig")) + .and_then(|kubelet_config| kubelet_config.get("runtimeRequestTimeout")) + .and_then(|value| value.as_str()) + .map(|value| value.to_string())) + } + pub async fn label_node( &self, label_key: &str, @@ -612,6 +632,11 @@ pub async fn get_node_label(config: &Config, key: &str) -> Result client.get_node_label(key).await } +pub async fn get_kubelet_runtime_request_timeout(config: &Config) -> Result> { + let client = K8sClient::new(&config.node_name).await?; + client.get_kubelet_runtime_request_timeout().await +} + pub async fn get_node_ready_status(config: &Config) -> Result { let client = K8sClient::new(&config.node_name).await?; let node = client.get_node().await?; diff --git a/tools/packaging/kata-deploy/binary/src/main.rs b/tools/packaging/kata-deploy/binary/src/main.rs index baee18489b..00ba0c8db6 100644 --- a/tools/packaging/kata-deploy/binary/src/main.rs +++ b/tools/packaging/kata-deploy/binary/src/main.rs @@ -13,6 +13,7 @@ mod utils; use anyhow::{Context, Result}; use clap::Parser; use log::{error, info}; +use semver::Version; /// Env var name used to thread the detected container runtime through the /// post-install re-exec. Avoids re-querying the apiserver after we've already @@ -99,6 +100,8 @@ enum Action { /// Node label applied to mark a node as kata-capable. Shared across the /// install/cleanup label stages so the key stays consistent. const KATA_RUNTIME_LABEL: &str = "katacontainers.io/kata-runtime"; +const SUGGESTED_KUBELET_RUNTIME_REQUEST_TIMEOUT_SECS: u64 = 10 * 60; +const MIN_EROFS_UTILS_VERSION: &str = "1.8.2"; // Cap the tokio runtime to a small fixed number of worker threads. The default // multi-thread runtime allocates `num_cpus()` workers (each with a ~2 MiB @@ -429,8 +432,7 @@ async fn install_stage_host_check(config: &config::Config, runtime: &str) -> Res for s in &non_empty_snapshotters { match s.as_str() { "erofs" => { - runtime::containerd::containerd_erofs_snapshotter_version_check(config) - .await?; + validate_erofs_prerequisites(config).await?; } "nydus" => {} _ => { @@ -444,10 +446,292 @@ async fn install_stage_host_check(config: &config::Config, runtime: &str) -> Res } } + if config_uses_guest_pull(config) { + validate_kubelet_runtime_request_timeout(config, "guest pull").await?; + } + info!("install (host-check): node prerequisites satisfied"); Ok(()) } +async fn validate_erofs_prerequisites(config: &config::Config) -> Result<()> { + info!("Validating EROFS snapshotter prerequisites"); + + runtime::containerd::containerd_erofs_snapshotter_version_check(config).await?; + + validate_host_kernel_feature_available( + HostKernelFeature::Erofs, + "Load or enable EROFS filesystem support before installing Kata and \ + make it persistent across reboots.", + )?; + + if config.erofs_dmverity { + validate_host_kernel_feature_available( + HostKernelFeature::DeviceMapper, + "Load or enable device-mapper support before installing Kata and \ + make it persistent across reboots.", + )?; + validate_host_kernel_feature_available( + HostKernelFeature::DmVerity, + "Load or enable the dm-verity target before installing Kata and \ + make it persistent across reboots.", + )?; + } + + validate_mkfs_erofs_version()?; + + // kata-deploy currently configures the EROFS snapshotter with + // enable_fsverity=true, but this host check does not know the final + // containerd configuration after user drop-ins, and it does not validate + // the backing filesystem's fs-verity feature. Keep this check warning-only. + warn_if_erofs_fsverity_may_be_unavailable(); + + validate_kubelet_runtime_request_timeout(config, "EROFS layer conversion").await?; + + Ok(()) +} + +#[derive(Clone, Copy)] +enum HostKernelFeature { + Erofs, + DeviceMapper, + DmVerity, + FsVerity, +} + +impl HostKernelFeature { + fn name(self) -> &'static str { + match self { + Self::Erofs => "erofs", + Self::DeviceMapper => "device-mapper", + Self::DmVerity => "dm-verity", + Self::FsVerity => "fs-verity", + } + } + + fn module_name(self) -> &'static str { + match self { + Self::Erofs => "erofs", + Self::DeviceMapper => "dm_mod", + Self::DmVerity => "dm_verity", + Self::FsVerity => "fsverity", + } + } + + fn config_symbol(self) -> &'static str { + match self { + Self::Erofs => "CONFIG_EROFS_FS", + Self::DeviceMapper => "CONFIG_BLK_DEV_DM", + Self::DmVerity => "CONFIG_DM_VERITY", + Self::FsVerity => "CONFIG_FS_VERITY", + } + } +} + +fn validate_host_kernel_feature_available( + feature: HostKernelFeature, + remediation: &str, +) -> Result<()> { + if host_module_visible(feature.module_name()) + || host_proc_config_has_builtin_feature(feature.config_symbol()) + || host_boot_config_has_builtin_feature(feature.config_symbol()) + { + return Ok(()); + } + + anyhow::bail!( + "Required host kernel feature `{}` is not available. {remediation}", + feature.name() + ) +} + +fn host_module_visible(module_name: &str) -> bool { + let sys_module_path = format!("/sys/module/{module_name}"); + if utils::host_exec(&["test", "-d", &sys_module_path]).is_ok() { + return true; + } + + let proc_modules_pattern = format!("^{module_name} "); + utils::host_exec(&["grep", "-q", &proc_modules_pattern, "/proc/modules"]).is_ok() +} + +fn host_proc_config_has_builtin_feature(config_symbol: &str) -> bool { + let config_value = format!("{config_symbol}=y"); + + if utils::host_exec(&["test", "-r", "/proc/config.gz"]).is_err() { + return false; + } + + let Ok(output) = utils::host_exec(&["gzip", "-dc", "/proc/config.gz"]) else { + return false; + }; + + output.lines().any(|line| line == config_value) +} + +fn host_boot_config_has_builtin_feature(config_symbol: &str) -> bool { + let config_pattern = format!("^{config_symbol}=y"); + + let output = utils::host_exec(&["uname", "-r"]); + let Ok(kernel_release) = output else { + return false; + }; + + let kernel_config_path = format!("/boot/config-{}", kernel_release.trim()); + utils::host_exec(&["grep", "-Eq", &config_pattern, &kernel_config_path]).is_ok() +} + +fn validate_mkfs_erofs_version() -> Result<()> { + let output = utils::host_exec(&["mkfs.erofs", "--version"]).with_context(|| { + "Required host command `mkfs.erofs` is not available. Install \ + erofs-utils >= 1.8.2 before enabling the EROFS snapshotter." + })?; + + let version = parse_erofs_utils_version(&output).with_context(|| { + format!("Could not parse erofs-utils version from `mkfs.erofs --version`: {output}") + })?; + let minimum_version = Version::parse(MIN_EROFS_UTILS_VERSION)?; + + if version < minimum_version { + anyhow::bail!( + "Host erofs-utils version {} is too old. kata-deploy configures \ + EROFS fsmerge mkfs_options that require erofs-utils >= {}.", + version, + MIN_EROFS_UTILS_VERSION + ); + } + + info!( + "host erofs-utils version {} satisfies minimum {}", + version, MIN_EROFS_UTILS_VERSION + ); + + Ok(()) +} + +fn parse_erofs_utils_version(output: &str) -> Result { + let version_re = regex::Regex::new(r"([0-9]+)\.([0-9]+)(?:\.([0-9]+))?")?; + let captures = version_re + .captures(output) + .ok_or_else(|| anyhow::anyhow!("erofs-utils version not found"))?; + + let major = captures[1].parse::()?; + let minor = captures[2].parse::()?; + let patch = captures + .get(3) + .map(|patch| patch.as_str().parse::()) + .transpose()? + .unwrap_or(0); + + Version::parse(&format!("{major}.{minor}.{patch}")).map_err(Into::into) +} + +fn warn_if_erofs_fsverity_may_be_unavailable() { + if let Err(err) = validate_host_kernel_feature_available( + HostKernelFeature::FsVerity, + "Install, load, or enable fs-verity support if the final EROFS \ + snapshotter configuration keeps enable_fsverity=true.", + ) { + log::warn!( + "kata-deploy's default EROFS snapshotter configuration sets \ + enable_fsverity=true, but host fs-verity support was not detected \ + ({err}). This is warning-only because the final containerd \ + configuration may be changed by user drop-ins, and kata-deploy \ + does not yet validate the backing filesystem's fs-verity feature." + ); + } else { + log::warn!( + "kata-deploy's default EROFS snapshotter configuration sets \ + enable_fsverity=true and host fs-verity support was detected, but \ + kata-deploy does not yet validate the backing filesystem's \ + fs-verity feature." + ); + } +} + +async fn validate_kubelet_runtime_request_timeout( + config: &config::Config, + operation: &str, +) -> Result<()> { + let runtime_request_timeout = match k8s::get_kubelet_runtime_request_timeout(config).await { + Ok(Some(value)) => value, + Ok(None) => { + warn_runtime_request_timeout( + operation, + "kubelet /configz did not include runtimeRequestTimeout", + ); + return Ok(()); + } + Err(err) => { + warn_runtime_request_timeout( + operation, + &format!("could not query kubelet runtimeRequestTimeout from /configz: {err}"), + ); + return Ok(()); + } + }; + + let timeout_secs = match humantime::parse_duration(&runtime_request_timeout) { + Ok(timeout) => timeout.as_secs(), + Err(err) => { + warn_runtime_request_timeout( + operation, + &format!( + "could not parse kubelet runtimeRequestTimeout value \ + `{runtime_request_timeout}` from /configz: {err}" + ), + ); + return Ok(()); + } + }; + + if timeout_secs < SUGGESTED_KUBELET_RUNTIME_REQUEST_TIMEOUT_SECS { + warn_runtime_request_timeout( + operation, + &format!( + "kubelet runtimeRequestTimeout from /configz is \ + `{runtime_request_timeout}` ({timeout_secs}s)" + ), + ); + } + + info!( + "kubelet runtimeRequestTimeout from /configz is {runtime_request_timeout} ({timeout_secs}s)" + ); + Ok(()) +} + +fn warn_runtime_request_timeout(operation: &str, detail: &str) { + log::warn!( + "{detail}. {operation} may run during CreateContainer; consider \ + configuring kubelet runtimeRequestTimeout to at least {}s on nodes \ + that run large images.", + SUGGESTED_KUBELET_RUNTIME_REQUEST_TIMEOUT_SECS + ); +} + +fn config_uses_guest_pull(config: &config::Config) -> bool { + !config.experimental_force_guest_pull_for_arch.is_empty() + || mapping_contains_value(config.pull_type_mapping_for_arch.as_deref(), "guest-pull") + || config + .custom_runtimes + .iter() + .any(|runtime| runtime.crio_pull_type.as_deref() == Some("guest-pull")) +} + +fn mapping_contains_value(mapping: Option<&str>, expected_value: &str) -> bool { + mapping.is_some_and(|mapping| { + mapping.split(',').any(|entry| { + let value = entry + .split_once(':') + .map(|(_, value)| value) + .unwrap_or(entry) + .trim(); + value == expected_value + }) + }) +} + /// Install stage 1 (artifacts): place kata artifacts/config on the host and set /// up any configured snapshotters. This does not touch CRI configuration, but it /// still needs privileged host access: writing under the host install dir and @@ -927,6 +1211,22 @@ mod tests { ); } + #[rstest] + #[case("mkfs.erofs (erofs-utils) 1.9\navailable compressors: lz4\n", "1.9.0")] + #[case("mkfs.erofs (erofs-utils) 1.8.2\n", "1.8.2")] + #[case("erofs-utils 1.8\n", "1.8.0")] + fn test_parse_erofs_utils_version(#[case] output: &str, #[case] expected: &str) { + assert_eq!( + parse_erofs_utils_version(output).unwrap(), + Version::parse(expected).unwrap() + ); + } + + #[test] + fn test_parse_erofs_utils_version_rejects_invalid_output() { + assert!(parse_erofs_utils_version("mkfs.erofs unknown").is_err()); + } + /// All non-internal staged actions remain visible in `--help` so operators /// can discover and run individual stages. #[rstest] diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml index 70453bf6f2..36e31172e3 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml @@ -26,6 +26,11 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "patch"] +# Read kubelet /configz via the apiserver node proxy during host-checks, for +# example to warn about runtimeRequestTimeout values that may be too low. +- apiGroups: [""] + resources: ["nodes/proxy"] + verbs: ["get"] - apiGroups: ["node.k8s.io"] resources: ["runtimeclasses"] verbs: ["get", "list", "patch"]