From 65e4fa0dc2735c5790e23a7e56e93dbc12e5ef47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 22 Jul 2026 19:42:19 +0200 Subject: [PATCH] kata-deploy: add a devkit debug flag with per-shim RuntimeClasses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an opt-in "devkit" path to kata-deploy that wires the devkit debug guest extension into a dedicated per-shim runtime, so operators can debug a sandbox with a rich shell without touching their normal RuntimeClasses. Helm gains a `devkit` value (only effective together with `debug`, since the debug console must be enabled to reach it). When set it emits DEVKIT=true and, for every enabled shim, an extra `kata--devkit` RuntimeClass reusing the shim's overhead and node selectors. The RuntimeClass template is refactored to compute name/handler from an optional `variant`, keeping existing behavior. In the deploy binary, DEVKIT synthesizes one kata--devkit custom runtime per enabled shim. Modeling it as a custom runtime reuses the whole custom runtime install/register/cleanup machinery; the only devkit-specific piece is a 25-devkit.toml drop-in that cold-plugs the extension image (guest_extension_images name="devkit", with verity_params read from root_hash_devkit-extension.txt when present) and points the agent debug console at /run/kata-extensions/devkit/bin/devkit-sh. The generic rootfs-image-devkit-extension tarball is extracted based on the devkit flag rather than per-shim shim-components.json membership. Signed-off-by: Fabiano Fidêncio Assisted-by: Cursor --- .../binary/src/artifacts/install.rs | 462 ++++++++++++++++++ .../kata-deploy/binary/src/config.rs | 167 ++++++- .../kata-deploy/templates/_helpers.tpl | 6 + .../kata-deploy/templates/runtimeclasses.yaml | 56 ++- .../helm-chart/kata-deploy/values.yaml | 9 + 5 files changed, 684 insertions(+), 16 deletions(-) diff --git a/tools/packaging/kata-deploy/binary/src/artifacts/install.rs b/tools/packaging/kata-deploy/binary/src/artifacts/install.rs index 5ded144968..da91cddb23 100644 --- a/tools/packaging/kata-deploy/binary/src/artifacts/install.rs +++ b/tools/packaging/kata-deploy/binary/src/artifacts/install.rs @@ -14,6 +14,7 @@ use std::collections::HashSet; use std::fs; use std::os::unix::fs::PermissionsExt; use std::path::Path; +use toml_edit::{value, ArrayOfTables, DocumentMut, Item, Table}; #[cfg(test)] use walkdir::WalkDir; @@ -123,6 +124,10 @@ pub async fn install_artifacts(config: &Config, container_runtime: &str) -> Resu install_custom_runtime_configs(config, container_runtime)?; } + // Drop any devkit artifacts a previous deploy left behind but this one no + // longer wants (e.g. devkit toggled off, or a shim removed). + reconcile_devkit_artifacts(config)?; + let expand_runtime_classes_for_nfd = nfd::setup_nfd_rules(config).await?; if expand_runtime_classes_for_nfd { @@ -306,6 +311,44 @@ fn install_custom_runtime_configs(config: &Config, container_runtime: &str) -> R container_runtime, )?; + // Everything devkit needs (the extension image + the debug console shell) + // goes into the drop-in (25-devkit.toml), leaving the copied base config + // untouched for users to inspect. + // + // The drop-in merge REPLACES the guest_extension_images array rather than + // appending, so it must re-emit the base's existing extensions in order and + // append devkit last: this preserves them (stable device enumeration) and + // makes devkit the last image attached, perturbing the guest we want to + // debug as little as possible. + if runtime.devkit { + let image_path = format!( + "{}/share/kata-containers/kata-containers-devkit-extension.img", + config.dest_dir + ); + let verity_params = devkit_verity_params(&config.dest_dir); + let hypervisor_name = get_hypervisor_name(&runtime.base_config)?; + let existing = if Path::new(&dest_config).exists() { + read_guest_extension_images(Path::new(&dest_config), hypervisor_name)? + } else { + log::warn!( + "devkit: base config {} is missing; the devkit drop-in will only \ + carry the devkit extension image", + dest_config + ); + Vec::new() + }; + write_drop_in_file( + &config_d_dir, + "25-devkit.toml", + &generate_devkit_drop_in( + hypervisor_name, + &existing, + &image_path, + verity_params.as_deref(), + ), + )?; + } + // Copy user-provided drop-in file if provided (at 50-overrides.toml). // If it was removed from values in a later upgrade/migration, remove stale file. let drop_in_dest = format!("{}/50-overrides.toml", config_d_dir); @@ -373,6 +416,97 @@ fn remove_custom_runtime_configs(config: &Config) -> Result<()> { Ok(()) } +/// Remove on-node devkit artifacts the current configuration no longer wants. +/// +/// Devkit is materialized as kata--devkit custom runtimes only while +/// DEVKIT=true. When a redeploy disables devkit (or drops a shim) those runtimes +/// vanish from `config.custom_runtimes`, so the normal custom-runtime cleanup +/// never touches their leftovers. Installs only ever add files, so reconcile +/// explicitly: drop stale kata--devkit handler directories and, once no +/// devkit runtime remains, the now-unreferenced extension image and root-hash. +/// +/// The RuntimeClasses are Helm-managed and the containerd handlers regenerated +/// every install, so only these binary-installed files need reconciling here. +fn reconcile_devkit_artifacts(config: &Config) -> Result<()> { + // Handlers the current config owns (devkit-synthesized or user-provided). Any + // kata--devkit directory not in this set is a stale devkit leftover; a + // handler that happens to match a user's own custom runtime is protected. + let known: HashSet<&str> = config + .custom_runtimes + .iter() + .map(|r| r.handler.as_str()) + .collect(); + + let custom_runtimes_dir = format!( + "/host/{}/share/defaults/kata-containers/custom-runtimes", + config.dest_dir + ); + let share_dir = format!("{}/share/kata-containers", config.host_install_dir); + + reconcile_devkit_artifacts_in( + &known, + Path::new(&custom_runtimes_dir), + Path::new(&share_dir), + config.devkit_enabled, + ) +} + +/// Pure core of [`reconcile_devkit_artifacts`], separated so it can be unit +/// tested against real directories without a full [`Config`]. +/// +/// `known_handlers` are preserved; every other kata--devkit directory under +/// `custom_runtimes_dir` is removed. When `devkit_enabled` is false the +/// unreferenced extension image and root-hash under `share_dir` are removed too. +fn reconcile_devkit_artifacts_in( + known_handlers: &HashSet<&str>, + custom_runtimes_dir: &Path, + share_dir: &Path, + devkit_enabled: bool, +) -> Result<()> { + for shim in ALL_SHIMS { + let handler = format!("kata-{shim}-devkit"); + if known_handlers.contains(handler.as_str()) { + continue; + } + let handler_dir = custom_runtimes_dir.join(&handler); + if handler_dir.exists() { + info!( + "devkit: removing stale runtime directory {}", + handler_dir.display() + ); + if let Err(e) = fs::remove_dir_all(&handler_dir) { + log::warn!("devkit: failed to remove {}: {}", handler_dir.display(), e); + } + } + } + + if custom_runtimes_dir.exists() { + if let Ok(entries) = fs::read_dir(custom_runtimes_dir) { + if entries.count() == 0 { + let _ = fs::remove_dir(custom_runtimes_dir); + } + } + } + + // With no devkit runtime left, the extracted extension image is unreferenced. + if !devkit_enabled { + for name in [ + "kata-containers-devkit-extension.img", + "root_hash_devkit-extension.txt", + ] { + let path = share_dir.join(name); + if path.exists() { + info!("devkit: removing unreferenced {}", path.display()); + if let Err(e) = fs::remove_file(&path) { + log::warn!("devkit: failed to remove {}: {}", path.display(), e); + } + } + } + } + + Ok(()) +} + /// Copy an extracted artifact tree from `src` into `dst`. /// /// Used only by unit tests; production code now uses `extract_component_tarballs`. @@ -502,6 +636,13 @@ fn collect_required_tarballs(config: &Config) -> Result> { } } } + + // Generic, not tied to any shim in shim-components.json, so it is pulled in + // by the devkit flag rather than per-shim membership. + if config.devkit_enabled { + required.insert("rootfs-image-devkit-extension".to_string()); + } + Ok(required) } @@ -1302,6 +1443,154 @@ enable_debug = true Ok(content) } +/// Guest path of the devkit debug console shell. Must live under +/// /run/kata-extensions/, the prefix the agent enforces for debug_console_shell. +const DEVKIT_DEBUG_CONSOLE_SHELL: &str = "/run/kata-extensions/devkit/bin/devkit-sh"; + +/// Read the devkit extension's dm-verity params from its root-hash file. Absent +/// (e.g. s390x, built without a measured rootfs) means `None` and the image is +/// mounted unverified. +fn devkit_verity_params(dest_dir: &str) -> Option { + let root_hash_file = format!( + "/host{}/share/kata-containers/root_hash_devkit-extension.txt", + dest_dir + ); + match fs::read_to_string(&root_hash_file) { + Ok(content) => content + .lines() + .map(str::trim) + .find(|l| !l.is_empty()) + .map(|l| l.to_string()), + Err(_) => { + log::warn!( + "devkit: {} not found; mounting the devkit extension without dm-verity", + root_hash_file + ); + None + } + } +} + +/// A `guest_extension_images` entry read from a runtime's base config. +struct GuestExtension { + name: String, + path: String, + verity_params: Option, +} + +/// Read the base config's existing `guest_extension_images`, in order, so the +/// drop-in can re-emit them ahead of devkit (the merge replaces the array, so +/// anything omitted is lost). Any pre-existing `devkit` entry is skipped, since +/// the drop-in always appends its own. +fn read_guest_extension_images( + config_file: &Path, + hypervisor_name: &str, +) -> Result> { + let content = fs::read_to_string(config_file) + .with_context(|| format!("Failed to read config for devkit extension: {config_file:?}"))?; + let doc = content + .parse::() + .with_context(|| format!("Failed to parse config as TOML: {config_file:?}"))?; + + let images = doc + .get("hypervisor") + .and_then(|h| h.get(hypervisor_name)) + .and_then(|hv| hv.get("guest_extension_images")) + .and_then(|i| i.as_array_of_tables()); + + let mut out = Vec::new(); + if let Some(images) = images { + for t in images.iter() { + let name = t.get("name").and_then(|i| i.as_str()); + let path = t.get("path").and_then(|i| i.as_str()); + if let (Some(name), Some(path)) = (name, path) { + if name == "devkit" { + continue; + } + out.push(GuestExtension { + name: name.to_string(), + path: path.to_string(), + verity_params: t + .get("verity_params") + .and_then(|i| i.as_str()) + .map(str::to_string), + }); + } + } + } + Ok(out) +} + +fn push_guest_extension( + images: &mut ArrayOfTables, + name: &str, + path: &str, + verity_params: Option<&str>, +) { + let mut entry = Table::new(); + entry["name"] = value(name); + entry["path"] = value(path); + if let Some(vp) = verity_params { + entry["verity_params"] = value(vp); + } + images.push(entry); +} + +/// Generate the devkit drop-in (25-devkit.toml): the agent debug-console scalars +/// plus the full guest_extension_images array (base extensions in order, devkit +/// last). The base config is left untouched so the shipped configuration.toml +/// stays clean for users. +fn generate_devkit_drop_in( + hypervisor_name: &str, + existing: &[GuestExtension], + image_path: &str, + verity_params: Option<&str>, +) -> String { + let mut doc = DocumentMut::new(); + + // [agent.kata] scalars. The intermediate `agent` table is implicit so only + // the `[agent.kata]` header is emitted, not an empty `[agent]`. + let mut agent_kata = Table::new(); + agent_kata["debug_console_enabled"] = value(true); + agent_kata["debug_console_shell"] = value(DEVKIT_DEBUG_CONSOLE_SHELL); + let mut agent = Table::new(); + agent.set_implicit(true); + agent.insert("kata", Item::Table(agent_kata)); + doc.insert("agent", Item::Table(agent)); + + // [[hypervisor..guest_extension_images]]: base extensions first, devkit + // last. Built as explicit (non-inline) tables so the array of tables renders + // as headers; the intermediate `hypervisor` table is implicit. + let mut images = ArrayOfTables::new(); + for ext in existing { + push_guest_extension( + &mut images, + &ext.name, + &ext.path, + ext.verity_params.as_deref(), + ); + } + push_guest_extension(&mut images, "devkit", image_path, verity_params); + let mut hv = Table::new(); + hv.insert("guest_extension_images", Item::ArrayOfTables(images)); + let mut hypervisor = Table::new(); + hypervisor.set_implicit(true); + hypervisor.insert(hypervisor_name, Item::Table(hv)); + doc.insert("hypervisor", Item::Table(hypervisor)); + + let mut content = String::new(); + content.push_str("# Devkit debug extension\n"); + content.push_str("# Generated by kata-deploy\n"); + content.push_str("#\n"); + content.push_str("# Points the agent debug console at the shell shipped by the devkit\n"); + content.push_str("# extension, and cold-plugs the extension image as the last\n"); + content.push_str("# guest_extension_images entry (any extensions the base config already\n"); + content + .push_str("# ships are re-listed here first, since drop-in merging replaces arrays).\n\n"); + content.push_str(&doc.to_string()); + content +} + /// Get proxy value for a specific shim from config. /// Handles both per-shim format ("qemu-tdx=http://proxy:8080;qemu-snp=http://proxy2:8080") /// and global format ("http://proxy:8080"). @@ -1517,6 +1806,179 @@ mod tests { ); } + const DEVKIT_IMG: &str = "/opt/kata/share/kata-containers/kata-containers-devkit-extension.img"; + + /// Parse a generated drop-in and return the ordered `name` values of its + /// guest_extension_images array under the given hypervisor. + fn drop_in_extension_names(content: &str, hypervisor_name: &str) -> Vec { + let doc = content.parse::().unwrap(); + doc["hypervisor"][hypervisor_name]["guest_extension_images"] + .as_array_of_tables() + .unwrap() + .iter() + .map(|t| t["name"].as_str().unwrap().to_string()) + .collect() + } + + #[test] + fn test_generate_devkit_drop_in_scalars() { + // The drop-in always carries the agent debug-console scalars. + let content = generate_devkit_drop_in("qemu", &[], DEVKIT_IMG, None); + + assert!(content.contains("[agent.kata]")); + assert!(content.contains("debug_console_enabled = true")); + assert!( + content.contains("debug_console_shell = \"/run/kata-extensions/devkit/bin/devkit-sh\"") + ); + } + + #[test] + fn test_generate_devkit_drop_in_only_devkit_when_base_empty() { + // A base config with no extensions yields an array with devkit only. + let content = generate_devkit_drop_in("qemu", &[], DEVKIT_IMG, None); + + assert_eq!(drop_in_extension_names(&content, "qemu"), vec!["devkit"]); + assert!(content.contains(&format!("path = \"{DEVKIT_IMG}\""))); + // No root hash provided -> no verity params emitted (raw mount). + assert!(!content.contains("verity_params")); + } + + #[test] + fn test_generate_devkit_drop_in_reemits_existing_then_devkit() { + // A base config that already ships an extension (e.g. coco) must have it + // re-listed first, with devkit strictly after it, so the array-replacing + // drop-in merge does not drop it. + let existing = vec![GuestExtension { + name: "coco".to_string(), + path: "/opt/kata/share/kata-containers/coco.img".to_string(), + verity_params: Some("root_hash=abc".to_string()), + }]; + let content = generate_devkit_drop_in("qemu", &existing, DEVKIT_IMG, Some("root_hash=def")); + + assert_eq!( + drop_in_extension_names(&content, "qemu"), + vec!["coco", "devkit"] + ); + // The pre-existing coco entry (path + verity) is re-emitted verbatim. + assert!(content.contains("name = \"coco\"")); + assert!(content.contains("path = \"/opt/kata/share/kata-containers/coco.img\"")); + assert!(content.contains("root_hash=abc")); + // devkit is present with its own image path and verity params. + assert!(content.contains(&format!("path = \"{DEVKIT_IMG}\""))); + assert!(content.contains("root_hash=def")); + } + + #[test] + fn test_read_guest_extension_images_skips_devkit_and_keeps_order() { + // Base config carries two extensions plus a stray devkit; the reader keeps + // the real ones in order and drops any pre-existing devkit entry. + let tmp = tempfile::NamedTempFile::new().unwrap(); + std::fs::write( + tmp.path(), + "[hypervisor.qemu]\npath = \"/usr/bin/qemu\"\n\n\ + [[hypervisor.qemu.guest_extension_images]]\n\ + name = \"coco\"\npath = \"/opt/kata/share/kata-containers/coco.img\"\n\ + verity_params = \"root_hash=abc\"\n\n\ + [[hypervisor.qemu.guest_extension_images]]\n\ + name = \"gpu\"\npath = \"/opt/kata/share/kata-containers/gpu.img\"\n\n\ + [[hypervisor.qemu.guest_extension_images]]\n\ + name = \"devkit\"\npath = \"/stale/devkit.img\"\n", + ) + .unwrap(); + + let existing = read_guest_extension_images(tmp.path(), "qemu").unwrap(); + let names: Vec<_> = existing.iter().map(|e| e.name.as_str()).collect(); + assert_eq!(names, vec!["coco", "gpu"]); + assert_eq!(existing[0].verity_params.as_deref(), Some("root_hash=abc")); + assert_eq!(existing[1].verity_params, None); + } + + #[test] + fn test_read_guest_extension_images_none_when_absent() { + // A base config with no extension array yields an empty list. + let tmp = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(tmp.path(), "[hypervisor.qemu]\npath = \"/usr/bin/qemu\"\n").unwrap(); + + assert!(read_guest_extension_images(tmp.path(), "qemu") + .unwrap() + .is_empty()); + } + + /// Lay down a fake devkit handler directory and the extension image + root + /// hash, mimicking what a devkit-enabled install leaves on the node. + fn seed_devkit_layout(custom_runtimes_dir: &Path, share_dir: &Path, handlers: &[&str]) { + for handler in handlers { + let d = custom_runtimes_dir.join(handler).join("config.d"); + fs::create_dir_all(&d).unwrap(); + fs::write(d.join("25-devkit.toml"), "[agent.kata]\n").unwrap(); + } + fs::create_dir_all(share_dir).unwrap(); + fs::write( + share_dir.join("kata-containers-devkit-extension.img"), + "img", + ) + .unwrap(); + fs::write( + share_dir.join("root_hash_devkit-extension.txt"), + "root_hash=x", + ) + .unwrap(); + } + + #[test] + fn test_reconcile_devkit_disabled_removes_everything() { + let tmp = tempfile::tempdir().unwrap(); + let custom = tmp.path().join("custom-runtimes"); + let share = tmp.path().join("share/kata-containers"); + seed_devkit_layout(&custom, &share, &["kata-qemu-devkit", "kata-fc-devkit"]); + + // devkit disabled -> no known devkit handlers, image should go too. + reconcile_devkit_artifacts_in(&HashSet::new(), &custom, &share, false).unwrap(); + + assert!(!custom.join("kata-qemu-devkit").exists()); + assert!(!custom.join("kata-fc-devkit").exists()); + // Emptied custom-runtimes dir is dropped. + assert!(!custom.exists()); + assert!(!share.join("kata-containers-devkit-extension.img").exists()); + assert!(!share.join("root_hash_devkit-extension.txt").exists()); + } + + #[test] + fn test_reconcile_devkit_keeps_active_and_protects_user_runtimes() { + let tmp = tempfile::tempdir().unwrap(); + let custom = tmp.path().join("custom-runtimes"); + let share = tmp.path().join("share/kata-containers"); + // kata-qemu-devkit is still active; kata-fc-devkit is stale. + seed_devkit_layout(&custom, &share, &["kata-qemu-devkit", "kata-fc-devkit"]); + // A user custom runtime that must never be touched. + let user = custom.join("kata-mine"); + fs::create_dir_all(&user).unwrap(); + fs::write(user.join("marker"), "keep").unwrap(); + + let known: HashSet<&str> = ["kata-qemu-devkit", "kata-mine"].into_iter().collect(); + // devkit still enabled -> image is preserved. + reconcile_devkit_artifacts_in(&known, &custom, &share, true).unwrap(); + + assert!(custom.join("kata-qemu-devkit").exists(), "active kept"); + assert!(!custom.join("kata-fc-devkit").exists(), "stale removed"); + assert!(user.join("marker").exists(), "user runtime untouched"); + assert!( + share.join("kata-containers-devkit-extension.img").exists(), + "image kept while devkit enabled" + ); + } + + #[test] + fn test_reconcile_devkit_is_noop_without_leftovers() { + let tmp = tempfile::tempdir().unwrap(); + let custom = tmp.path().join("custom-runtimes"); + let share = tmp.path().join("share/kata-containers"); + // Nothing seeded: reconcile must succeed and create nothing. + reconcile_devkit_artifacts_in(&HashSet::new(), &custom, &share, false).unwrap(); + assert!(!custom.exists()); + assert!(!share.exists()); + } + #[test] fn test_copy_artifacts_overwrites_existing_files() { use std::fs; diff --git a/tools/packaging/kata-deploy/binary/src/config.rs b/tools/packaging/kata-deploy/binary/src/config.rs index 5c145554b3..404e072f59 100644 --- a/tools/packaging/kata-deploy/binary/src/config.rs +++ b/tools/packaging/kata-deploy/binary/src/config.rs @@ -161,6 +161,9 @@ pub struct CustomRuntime { pub containerd_snapshotter: Option, /// CRI-O pull type (e.g., "guest-pull") pub crio_pull_type: Option, + /// Internally-synthesized devkit runtime (kata--devkit): gets an extra + /// drop-in wiring the extension image and debug console shell. + pub devkit: bool, } #[derive(Debug, Clone)] @@ -199,6 +202,11 @@ pub struct Config { pub daemonset_name: String, pub custom_runtimes_enabled: bool, pub custom_runtimes: Vec, + /// Install the devkit extension and a kata--devkit custom runtime per + /// enabled shim. From the DEVKIT env var, honored only when debug is also on: + /// the devkit drop-in enables the agent debug console, which must never come + /// up without debug. + pub devkit_enabled: bool, /// EROFS snapshotter rw-layer backing mode ("disk" or "memory"). pub erofs_snapshotter_mode: Option, /// Enable dm-verity integrity for EROFS lower layers. @@ -238,7 +246,7 @@ impl Config { // Parse shims - only use arch-specific variable // Use architecture-specific default shims list (only shims supported for this arch) let default_shims = get_default_shims_for_arch(&arch); - let shims_for_arch = get_arch_var("SHIMS", default_shims, &arch) + let shims_for_arch: Vec = get_arch_var("SHIMS", default_shims, &arch) .split_whitespace() .map(|s| s.to_string()) .collect(); @@ -346,14 +354,54 @@ impl Config { .collect(); // Parse custom runtimes from ConfigMap - let custom_runtimes_enabled = + let custom_runtimes_from_configmap = env::var("CUSTOM_RUNTIMES_ENABLED").unwrap_or_else(|_| "false".to_string()) == "true"; - let custom_runtimes = if custom_runtimes_enabled { + let mut custom_runtimes = if custom_runtimes_from_configmap { parse_custom_runtimes()? } else { Vec::new() }; + // Model devkit as one custom runtime per shim (kata--devkit) to + // reuse the whole custom-runtime install/register/cleanup machinery; the + // only devkit-specific bit is an extra drop-in (in install.rs). + // + // Gate it on debug: the devkit drop-in turns on the agent debug console, + // so honoring DEVKIT without DEBUG would silently enable it, breaking the + // "only effective with debug" contract. + let devkit_requested = env::var("DEVKIT").unwrap_or_else(|_| "false".to_string()) == "true"; + if devkit_requested && !debug { + log::warn!("DEVKIT=true ignored: it requires DEBUG=true to take effect"); + } + let devkit_enabled = devkit_requested && debug; + if devkit_enabled { + for shim in &shims_for_arch { + // Inherit the base shim's image-pulling config: the devkit + // RuntimeClass shares the base config (same shared_fs), so it must + // pull the same way. Otherwise containerd/CRI-O default to + // overlayfs/node-pull, which breaks shims running shared_fs = none + // (e.g. nvidia runtime-rs) and makes every devkit pod terminate. + let containerd_snapshotter = snapshotter_handler_mapping_for_arch + .as_ref() + .and_then(|mapping| lookup_mapping_value(mapping, shim)); + let crio_pull_type = pull_type_mapping_for_arch + .as_ref() + .and_then(|mapping| lookup_mapping_value(mapping, shim)); + + custom_runtimes.push(CustomRuntime { + handler: format!("kata-{shim}-devkit"), + base_config: shim.to_string(), + drop_in_file: None, + containerd_snapshotter, + crio_pull_type, + devkit: true, + }); + } + } + + // Enable the custom-runtime code paths if either source produced entries. + let custom_runtimes_enabled = !custom_runtimes.is_empty(); + let erofs_snapshotter_mode = env::var("EROFS_SNAPSHOTTER_MODE") .ok() .map(|s| s.trim().to_string()) @@ -401,6 +449,7 @@ impl Config { daemonset_name, custom_runtimes_enabled, custom_runtimes, + devkit_enabled, erofs_snapshotter_mode, erofs_dmverity, startup_taints, @@ -795,6 +844,21 @@ fn get_arch() -> Result { .to_string()) } +/// Look up `shim`'s value in a comma-separated "shim1:value1,shim2:value2" +/// mapping (SNAPSHOTTER_HANDLER_MAPPING, PULL_TYPE_MAPPING). None if absent or +/// empty. +fn lookup_mapping_value(mapping: &str, shim: &str) -> Option { + mapping.split(',').find_map(|entry| { + let parts: Vec<&str> = entry.split(':').collect(); + if parts.len() == 2 && parts[0].trim() == shim { + let value = parts[1].trim(); + (!value.is_empty()).then(|| value.to_string()) + } else { + None + } + }) +} + /// Parse custom runtimes from the mounted ConfigMap at /custom-configs/ /// Reads the custom-runtimes.list file which contains entries in the format: /// handler:baseConfig:containerd_snapshotter:crio_pulltype @@ -872,6 +936,7 @@ fn parse_custom_runtimes() -> Result> { drop_in_file, containerd_snapshotter, crio_pull_type, + devkit: false, }); } @@ -987,6 +1052,8 @@ mod tests { "EXPERIMENTAL_FORCE_GUEST_PULL_PPC64LE", "CONTAINERD_CONFIG_FILE_NAME", "STARTUP_TAINTS", + "CUSTOM_RUNTIMES_ENABLED", + "DEVKIT", ]; for var in &vars { std::env::remove_var(var); @@ -1135,6 +1202,100 @@ mod tests { cleanup_env_vars(); } + #[serial] + #[test] + fn test_devkit_disabled_by_default() { + setup_minimal_env(); + + let config = Config::from_env().unwrap(); + + assert!(!config.devkit_enabled); + assert!(config.custom_runtimes.iter().all(|r| !r.devkit)); + cleanup_env_vars(); + } + + #[serial] + #[test] + fn test_devkit_requires_debug() { + setup_minimal_env(); + // DEBUG defaults to false in setup_minimal_env. + std::env::set_var("DEVKIT", "true"); + + let config = Config::from_env().unwrap(); + + assert!(!config.devkit_enabled); + assert!(config.custom_runtimes.iter().all(|r| !r.devkit)); + cleanup_env_vars(); + } + + #[serial] + #[test] + fn test_devkit_synthesizes_per_shim_runtime() { + setup_minimal_env(); + std::env::set_var("DEBUG", "true"); + std::env::set_var("DEVKIT", "true"); + + let config = Config::from_env().unwrap(); + + assert!(config.devkit_enabled); + assert!(config.custom_runtimes_enabled); + + // setup_minimal_env configures a single "qemu" shim for this arch. + let devkit: Vec<_> = config.custom_runtimes.iter().filter(|r| r.devkit).collect(); + assert_eq!(devkit.len(), 1); + assert_eq!(devkit[0].handler, "kata-qemu-devkit"); + assert_eq!(devkit[0].base_config, "qemu"); + assert!(devkit[0].drop_in_file.is_none()); + // With no snapshotter/pull-type mapping, the devkit runtime inherits + // nothing and containerd/CRI-O use their defaults for the base shim. + assert!(devkit[0].containerd_snapshotter.is_none()); + assert!(devkit[0].crio_pull_type.is_none()); + + cleanup_env_vars(); + } + + #[serial] + #[test] + fn test_devkit_inherits_base_shim_snapshotter_and_pull_type() { + setup_minimal_env(); + std::env::set_var("DEBUG", "true"); + std::env::set_var("DEVKIT", "true"); + // The base "qemu" shim is mapped to the erofs snapshotter / guest-pull; + // its devkit variant must inherit both so pods on the devkit + // RuntimeClass pull images the same way as the base shim (crucial for + // shims running with shared_fs = none). + set_arch_var("SNAPSHOTTER_HANDLER_MAPPING", "qemu:erofs"); + set_arch_var("PULL_TYPE_MAPPING", "qemu:guest-pull"); + + let config = Config::from_env().unwrap(); + + let devkit: Vec<_> = config.custom_runtimes.iter().filter(|r| r.devkit).collect(); + assert_eq!(devkit.len(), 1); + assert_eq!(devkit[0].handler, "kata-qemu-devkit"); + assert_eq!(devkit[0].containerd_snapshotter.as_deref(), Some("erofs")); + assert_eq!(devkit[0].crio_pull_type.as_deref(), Some("guest-pull")); + + cleanup_env_vars(); + } + + #[test] + fn test_lookup_mapping_value() { + let mapping = "qemu:erofs, fc:nydus,clh:"; + assert_eq!( + lookup_mapping_value(mapping, "qemu").as_deref(), + Some("erofs") + ); + // Entries may carry surrounding whitespace. + assert_eq!( + lookup_mapping_value(mapping, "fc").as_deref(), + Some("nydus") + ); + // Empty value is treated as absent. + assert!(lookup_mapping_value(mapping, "clh").is_none()); + // Unknown shim. + assert!(lookup_mapping_value(mapping, "stratovirt").is_none()); + } + #[serial] #[test] fn test_multi_install_suffix_with_value() { diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl index 6a9266a76e..490a1cdd58 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl @@ -696,6 +696,12 @@ e.g. `{{- include "kata-deploy.commonEnv" . | nindent 8 }}`. - name: CUSTOM_RUNTIMES_ENABLED value: "true" {{- end }} +{{- /* Devkit debug extension: only effective together with debug (the debug + console must be enabled for it to be reachable). */ -}} +{{- if and .Values.debug .Values.devkit }} +- name: DEVKIT + value: "true" +{{- end }} {{- with .Values.startupTaints }} - name: STARTUP_TAINTS value: {{ join "," . | quote }} diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/runtimeclasses.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/runtimeclasses.yaml index 787ec56759..eb63e7870f 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/runtimeclasses.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/runtimeclasses.yaml @@ -41,20 +41,36 @@ {{- /* Render a single RuntimeClass. Params (dict): root (.), shim, config, shimConfig, - nameOverride (optional; if set, use as metadata.name for default RC), useShimNodeSelectors. + nameOverride (optional; if set, use as metadata.name for default RC), + variant (optional; e.g. "devkit" -> name/handler kata--devkit), + useShimNodeSelectors. + + Name/handler precedence: variant > multiInstallSuffix > nameOverride > default. + A variant RuntimeClass mirrors the handler kata-deploy registers for the same + variant (e.g. the kata--devkit custom runtime), so it ignores the + multiInstallSuffix/nameOverride paths. */ -}} {{- define "kata-deploy.runtimeclass" -}} +{{- $rcName := "" -}} +{{- $rcHandler := "" -}} +{{- if .variant -}} +{{- $rcName = printf "kata-%s-%s" .shim .variant -}} +{{- $rcHandler = $rcName -}} +{{- else if .root.Values.env.multiInstallSuffix -}} +{{- $rcName = printf "kata-%s-%s" .shim .root.Values.env.multiInstallSuffix -}} +{{- $rcHandler = $rcName -}} +{{- else if .nameOverride -}} +{{- $rcName = .nameOverride -}} +{{- $rcHandler = printf "kata-%s" .shim -}} +{{- else -}} +{{- $rcName = printf "kata-%s" .shim -}} +{{- $rcHandler = $rcName -}} +{{- end -}} --- kind: RuntimeClass apiVersion: node.k8s.io/v1 metadata: -{{- if .root.Values.env.multiInstallSuffix }} - name: kata-{{ .shim }}-{{ .root.Values.env.multiInstallSuffix }} -{{- else if .nameOverride }} - name: {{ .nameOverride }} -{{- else }} - name: kata-{{ .shim }} -{{- end }} + name: {{ $rcName }} labels: app.kubernetes.io/managed-by: kata-deploy {{- if .root.Values.env.multiInstallSuffix }} @@ -64,11 +80,7 @@ metadata: {{- end }} annotations: {{- include "kata-deploy.runtimeclassAnnotations" .root | nindent 4 }} -{{- if .root.Values.env.multiInstallSuffix }} -handler: kata-{{ .shim }}-{{ .root.Values.env.multiInstallSuffix }} -{{- else }} -handler: kata-{{ .shim }} -{{- end }} +handler: {{ $rcHandler }} {{- /* Overhead section - controlled by global or per-shim overheadEnabled flag (default: true) */ -}} {{- $shimOverheadEnabled := true -}} {{- if hasKey .root.Values.runtimeClasses "overheadEnabled" -}} @@ -155,4 +167,22 @@ scheduling: {{- end }} {{- end }} {{- end }} + +{{- /* Devkit debug RuntimeClasses: an extra kata--devkit per enabled shim, + matching the handler kata-deploy registers when DEVKIT=true. Gated by + debug (reachable only with the debug console) and devkit; reuses each + shim's overhead/nodeSelectors. */ -}} +{{- if and .Values.debug .Values.devkit }} +{{- range $shim := $enabledShims }} +{{- $config := index $runtimeClassConfigs $shim }} +{{- $shimConfig := index $.Values.shims $shim }} +{{- if $config }} +{{- $effectiveConfig := deepCopy $config }} +{{- if and $shimConfig.runtimeClass $shimConfig.runtimeClass.overhead }} +{{- $effectiveConfig = mergeOverwrite $effectiveConfig $shimConfig.runtimeClass.overhead }} +{{- end }} +{{ include "kata-deploy.runtimeclass" (dict "root" $ "shim" $shim "config" $effectiveConfig "shimConfig" $shimConfig "variant" "devkit" "nameOverride" "" "useShimNodeSelectors" $useShimNodeSelectors) }} +{{- end }} +{{- end }} +{{- end }} {{- end }} diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml index 760e4058ab..418d7b759c 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml @@ -304,6 +304,15 @@ resources: debug: false +# Deploy the optional "devkit" debug guest extension and, per enabled shim, an +# extra `kata--devkit` RuntimeClass wired to it. Pods on that class boot +# with the extension cold-plugged, so the agent debug console drops into a rich +# shell (Ubuntu + apt + debug tools) overlaid on the read-only guest. +# +# Debugging aid: only effective with `debug: true`, ignored otherwise. Leave it +# off in production and in confidential (CoCo/TEE) deployments. +devkit: false + # Optional CLI log level override for kata-deploy. # Supported values: error, warn, info, debug, trace # When empty, debug:true implies --log-level debug; otherwise kata-deploy falls