diff --git a/Cargo.lock b/Cargo.lock index 8ccb51dcd5..81b7f91b5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3718,12 +3718,14 @@ dependencies = [ "anyhow", "clap", "env_logger 0.10.2", + "humantime", "k8s-openapi", "kube", "libc", "log", "regex", "rstest", + "semver", "serde_json", "serde_yaml 0.9.34+deprecated", "serial_test 0.10.0", diff --git a/Cargo.toml b/Cargo.toml index e4744642ac..6b424d2e34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -195,6 +195,7 @@ scan_fmt = "0.2.6" scopeguard = "1.0.0" serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0.91" +semver = "1.0" serial_test = "0.10.0" sha2 = "0.11.0" slog = "2.5.2" diff --git a/tools/packaging/kata-deploy/binary/Cargo.toml b/tools/packaging/kata-deploy/binary/Cargo.toml index e8753922ce..a67eda5836 100644 --- a/tools/packaging/kata-deploy/binary/Cargo.toml +++ b/tools/packaging/kata-deploy/binary/Cargo.toml @@ -14,6 +14,7 @@ path = "src/main.rs" anyhow.workspace = true clap.workspace = true env_logger = "0.10" +humantime = "2.1.0" k8s-openapi = { version = "0.26", default-features = false, features = [ "v1_33", ] } @@ -32,6 +33,7 @@ kube = { version = "2.0", default-features = false, features = [ libc.workspace = true log.workspace = true regex.workspace = true +semver.workspace = true serde_json.workspace = true serde_yaml = "0.9" tar = "0.4.46" diff --git a/tools/packaging/kata-deploy/binary/src/k8s/client.rs b/tools/packaging/kata-deploy/binary/src/k8s/client.rs index e624b32d2e..b5c4504b81 100644 --- a/tools/packaging/kata-deploy/binary/src/k8s/client.rs +++ b/tools/packaging/kata-deploy/binary/src/k8s/client.rs @@ -7,7 +7,8 @@ use crate::config::Config; use anyhow::{Context, Result}; use k8s_openapi::api::core::v1::Node; use kube::{ - api::{Api, DeleteParams, DynamicObject, Patch, PatchParams}, + api::{Api, DeleteParams, DynamicObject, GetParams, Patch, PatchParams}, + core::Request, discovery::ApiResource, Client, }; @@ -71,6 +72,25 @@ impl K8sClient { .and_then(|labels| labels.get(key).cloned())) } + pub async fn get_kubelet_runtime_request_timeout(&self) -> Result> { + let request = Request::new(format!("/api/v1/nodes/{}/proxy", self.node_name)) + .get("configz", &GetParams::default())?; + + let configz: serde_json::Value = self.client.request(request).await.with_context(|| { + format!( + "Failed to query kubelet configz for node {}", + self.node_name + ) + })?; + + Ok(configz + .get("kubeletconfig") + .or_else(|| configz.get("kubeletConfig")) + .and_then(|kubelet_config| kubelet_config.get("runtimeRequestTimeout")) + .and_then(|value| value.as_str()) + .map(|value| value.to_string())) + } + pub async fn label_node( &self, label_key: &str, @@ -612,6 +632,11 @@ pub async fn get_node_label(config: &Config, key: &str) -> Result client.get_node_label(key).await } +pub async fn get_kubelet_runtime_request_timeout(config: &Config) -> Result> { + let client = K8sClient::new(&config.node_name).await?; + client.get_kubelet_runtime_request_timeout().await +} + pub async fn get_node_ready_status(config: &Config) -> Result { let client = K8sClient::new(&config.node_name).await?; let node = client.get_node().await?; diff --git a/tools/packaging/kata-deploy/binary/src/main.rs b/tools/packaging/kata-deploy/binary/src/main.rs index baee18489b..c784ec133c 100644 --- a/tools/packaging/kata-deploy/binary/src/main.rs +++ b/tools/packaging/kata-deploy/binary/src/main.rs @@ -13,6 +13,7 @@ mod utils; use anyhow::{Context, Result}; use clap::Parser; use log::{error, info}; +use semver::Version; /// Env var name used to thread the detected container runtime through the /// post-install re-exec. Avoids re-querying the apiserver after we've already @@ -99,6 +100,8 @@ enum Action { /// Node label applied to mark a node as kata-capable. Shared across the /// install/cleanup label stages so the key stays consistent. const KATA_RUNTIME_LABEL: &str = "katacontainers.io/kata-runtime"; +const SUGGESTED_KUBELET_RUNTIME_REQUEST_TIMEOUT_SECS: u64 = 10 * 60; +const MIN_EROFS_UTILS_VERSION: &str = "1.8.2"; // Cap the tokio runtime to a small fixed number of worker threads. The default // multi-thread runtime allocates `num_cpus()` workers (each with a ~2 MiB @@ -429,8 +432,7 @@ async fn install_stage_host_check(config: &config::Config, runtime: &str) -> Res for s in &non_empty_snapshotters { match s.as_str() { "erofs" => { - runtime::containerd::containerd_erofs_snapshotter_version_check(config) - .await?; + validate_erofs_prerequisites(config).await?; } "nydus" => {} _ => { @@ -448,6 +450,262 @@ async fn install_stage_host_check(config: &config::Config, runtime: &str) -> Res Ok(()) } +async fn validate_erofs_prerequisites(config: &config::Config) -> Result<()> { + info!("Validating EROFS snapshotter prerequisites"); + + runtime::containerd::containerd_erofs_snapshotter_version_check(config).await?; + + validate_host_kernel_feature_available( + HostKernelFeature::Erofs, + "Load or enable EROFS filesystem support before installing Kata and \ + make it persistent across reboots.", + )?; + + if config.erofs_dmverity { + validate_host_kernel_feature_available( + HostKernelFeature::DeviceMapper, + "Load or enable device-mapper support before installing Kata and \ + make it persistent across reboots.", + )?; + validate_host_kernel_feature_available( + HostKernelFeature::DmVerity, + "Load or enable the dm-verity target before installing Kata and \ + make it persistent across reboots.", + )?; + } + + validate_mkfs_erofs_version()?; + + // kata-deploy currently configures the EROFS snapshotter with + // enable_fsverity=true, but this host check does not know the final + // containerd configuration after user drop-ins, and it does not validate + // the backing filesystem's fs-verity feature. Keep this check warning-only. + warn_if_erofs_fsverity_may_be_unavailable(); + + validate_kubelet_runtime_request_timeout(config, "EROFS layer conversion").await?; + + Ok(()) +} + +#[derive(Clone, Copy)] +enum HostKernelFeature { + Erofs, + DeviceMapper, + DmVerity, + FsVerity, +} + +impl HostKernelFeature { + fn name(self) -> &'static str { + match self { + Self::Erofs => "erofs", + Self::DeviceMapper => "device-mapper", + Self::DmVerity => "dm-verity", + Self::FsVerity => "fs-verity", + } + } + + fn module_name(self) -> &'static str { + match self { + Self::Erofs => "erofs", + Self::DeviceMapper => "dm_mod", + Self::DmVerity => "dm_verity", + Self::FsVerity => "fsverity", + } + } + + fn config_symbol(self) -> &'static str { + match self { + Self::Erofs => "CONFIG_EROFS_FS", + Self::DeviceMapper => "CONFIG_BLK_DEV_DM", + Self::DmVerity => "CONFIG_DM_VERITY", + Self::FsVerity => "CONFIG_FS_VERITY", + } + } +} + +fn validate_host_kernel_feature_available( + feature: HostKernelFeature, + remediation: &str, +) -> Result<()> { + if host_module_visible(feature.module_name()) + || host_proc_config_has_builtin_feature(feature.config_symbol()) + || host_boot_config_has_builtin_feature(feature.config_symbol()) + { + return Ok(()); + } + + anyhow::bail!( + "Required host kernel feature `{}` is not available. {remediation}", + feature.name() + ) +} + +fn host_module_visible(module_name: &str) -> bool { + let sys_module_path = format!("/sys/module/{module_name}"); + if utils::host_exec(&["test", "-d", &sys_module_path]).is_ok() { + return true; + } + + let proc_modules_pattern = format!("^{module_name} "); + utils::host_exec(&["grep", "-q", &proc_modules_pattern, "/proc/modules"]).is_ok() +} + +fn host_proc_config_has_builtin_feature(config_symbol: &str) -> bool { + let config_value = format!("{config_symbol}=y"); + + if utils::host_exec(&["test", "-r", "/proc/config.gz"]).is_err() { + return false; + } + + let Ok(output) = utils::host_exec(&["gzip", "-dc", "/proc/config.gz"]) else { + return false; + }; + + output.lines().any(|line| line == config_value) +} + +fn host_boot_config_has_builtin_feature(config_symbol: &str) -> bool { + let config_pattern = format!("^{config_symbol}=y"); + + let output = utils::host_exec(&["uname", "-r"]); + let Ok(kernel_release) = output else { + return false; + }; + + let kernel_config_path = format!("/boot/config-{}", kernel_release.trim()); + utils::host_exec(&["grep", "-Eq", &config_pattern, &kernel_config_path]).is_ok() +} + +fn validate_mkfs_erofs_version() -> Result<()> { + let output = utils::host_exec(&["mkfs.erofs", "--version"]).with_context(|| { + "Required host command `mkfs.erofs` is not available. Install \ + erofs-utils >= 1.8.2 before enabling the EROFS snapshotter." + })?; + + let version = parse_erofs_utils_version(&output).with_context(|| { + format!("Could not parse erofs-utils version from `mkfs.erofs --version`: {output}") + })?; + let minimum_version = Version::parse(MIN_EROFS_UTILS_VERSION)?; + + if version < minimum_version { + anyhow::bail!( + "Host erofs-utils version {} is too old. kata-deploy configures \ + EROFS fsmerge mkfs_options that require erofs-utils >= {}.", + version, + MIN_EROFS_UTILS_VERSION + ); + } + + info!( + "host erofs-utils version {} satisfies minimum {}", + version, MIN_EROFS_UTILS_VERSION + ); + + Ok(()) +} + +fn parse_erofs_utils_version(output: &str) -> Result { + let version_re = regex::Regex::new(r"([0-9]+)\.([0-9]+)(?:\.([0-9]+))?")?; + let captures = version_re + .captures(output) + .ok_or_else(|| anyhow::anyhow!("erofs-utils version not found"))?; + + let major = captures[1].parse::()?; + let minor = captures[2].parse::()?; + let patch = captures + .get(3) + .map(|patch| patch.as_str().parse::()) + .transpose()? + .unwrap_or(0); + + Version::parse(&format!("{major}.{minor}.{patch}")).map_err(Into::into) +} + +fn warn_if_erofs_fsverity_may_be_unavailable() { + if let Err(err) = validate_host_kernel_feature_available( + HostKernelFeature::FsVerity, + "Install, load, or enable fs-verity support if the final EROFS \ + snapshotter configuration keeps enable_fsverity=true.", + ) { + log::warn!( + "kata-deploy's default EROFS snapshotter configuration sets \ + enable_fsverity=true, but host fs-verity support was not detected \ + ({err}). This is warning-only because the final containerd \ + configuration may be changed by user drop-ins, and kata-deploy \ + does not yet validate the backing filesystem's fs-verity feature." + ); + } else { + log::warn!( + "kata-deploy's default EROFS snapshotter configuration sets \ + enable_fsverity=true and host fs-verity support was detected, but \ + kata-deploy does not yet validate the backing filesystem's \ + fs-verity feature." + ); + } +} + +async fn validate_kubelet_runtime_request_timeout( + config: &config::Config, + operation: &str, +) -> Result<()> { + let runtime_request_timeout = match k8s::get_kubelet_runtime_request_timeout(config).await { + Ok(Some(value)) => value, + Ok(None) => { + warn_runtime_request_timeout( + operation, + "kubelet /configz did not include runtimeRequestTimeout", + ); + return Ok(()); + } + Err(err) => { + warn_runtime_request_timeout( + operation, + &format!("could not query kubelet runtimeRequestTimeout from /configz: {err}"), + ); + return Ok(()); + } + }; + + let timeout_secs = match humantime::parse_duration(&runtime_request_timeout) { + Ok(timeout) => timeout.as_secs(), + Err(err) => { + warn_runtime_request_timeout( + operation, + &format!( + "could not parse kubelet runtimeRequestTimeout value \ + `{runtime_request_timeout}` from /configz: {err}" + ), + ); + return Ok(()); + } + }; + + if timeout_secs < SUGGESTED_KUBELET_RUNTIME_REQUEST_TIMEOUT_SECS { + warn_runtime_request_timeout( + operation, + &format!( + "kubelet runtimeRequestTimeout from /configz is \ + `{runtime_request_timeout}` ({timeout_secs}s)" + ), + ); + } + + info!( + "kubelet runtimeRequestTimeout from /configz is {runtime_request_timeout} ({timeout_secs}s)" + ); + Ok(()) +} + +fn warn_runtime_request_timeout(operation: &str, detail: &str) { + log::warn!( + "{detail}. {operation} may run during CreateContainer; consider \ + configuring kubelet runtimeRequestTimeout to at least {}s on nodes \ + that run large images.", + SUGGESTED_KUBELET_RUNTIME_REQUEST_TIMEOUT_SECS + ); +} + /// Install stage 1 (artifacts): place kata artifacts/config on the host and set /// up any configured snapshotters. This does not touch CRI configuration, but it /// still needs privileged host access: writing under the host install dir and @@ -927,6 +1185,22 @@ mod tests { ); } + #[rstest] + #[case("mkfs.erofs (erofs-utils) 1.9\navailable compressors: lz4\n", "1.9.0")] + #[case("mkfs.erofs (erofs-utils) 1.8.2\n", "1.8.2")] + #[case("erofs-utils 1.8\n", "1.8.0")] + fn test_parse_erofs_utils_version(#[case] output: &str, #[case] expected: &str) { + assert_eq!( + parse_erofs_utils_version(output).unwrap(), + Version::parse(expected).unwrap() + ); + } + + #[test] + fn test_parse_erofs_utils_version_rejects_invalid_output() { + assert!(parse_erofs_utils_version("mkfs.erofs unknown").is_err()); + } + /// All non-internal staged actions remain visible in `--help` so operators /// can discover and run individual stages. #[rstest] diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml index 70453bf6f2..36e31172e3 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml @@ -26,6 +26,11 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "patch"] +# Read kubelet /configz via the apiserver node proxy during host-checks, for +# example to warn about runtimeRequestTimeout values that may be too low. +- apiGroups: [""] + resources: ["nodes/proxy"] + verbs: ["get"] - apiGroups: ["node.k8s.io"] resources: ["runtimeclasses"] verbs: ["get", "list", "patch"]