kata-deploy: add a devkit debug flag with per-shim RuntimeClasses

Add an opt-in "devkit" path to kata-deploy that wires the devkit debug guest
extension into a dedicated per-shim runtime, so operators can debug a sandbox
with a rich shell without touching their normal RuntimeClasses.

Helm gains a `devkit` value (only effective together with `debug`, since the
debug console must be enabled to reach it). When set it emits DEVKIT=true and,
for every enabled shim, an extra `kata-<shim>-devkit` RuntimeClass reusing the
shim's overhead and node selectors. The RuntimeClass template is refactored to
compute name/handler from an optional `variant`, keeping existing behavior.

In the deploy binary, DEVKIT synthesizes one kata-<shim>-devkit custom runtime
per enabled shim. Modeling it as a custom runtime reuses the whole custom
runtime install/register/cleanup machinery; the only devkit-specific piece is a
25-devkit.toml drop-in that cold-plugs the extension image
(guest_extension_images name="devkit", with verity_params read from
root_hash_devkit-extension.txt when present) and points the agent debug console
at /run/kata-extensions/devkit/bin/devkit-sh. The generic
rootfs-image-devkit-extension tarball is extracted based on the devkit flag
rather than per-shim shim-components.json membership.

Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>
Assisted-by: Cursor <cursoragent@cursor.com>

tools/packaging/kata-deploy/binary/src/artifacts/install.rs

@@ -14,6 +14,7 @@ use std::collections::HashSet;
 use std::fs;
 use std::os::unix::fs::PermissionsExt;
 use std::path::Path;
+use toml_edit::{value, ArrayOfTables, DocumentMut, Item, Table};
 #[cfg(test)]
 use walkdir::WalkDir;
 
@@ -123,6 +124,10 @@ pub async fn install_artifacts(config: &Config, container_runtime: &str) -> Resu
         install_custom_runtime_configs(config, container_runtime)?;
     }
 
+    // Drop any devkit artifacts a previous deploy left behind but this one no
+    // longer wants (e.g. devkit toggled off, or a shim removed).
+    reconcile_devkit_artifacts(config)?;
+
     let expand_runtime_classes_for_nfd = nfd::setup_nfd_rules(config).await?;
 
     if expand_runtime_classes_for_nfd {
@@ -306,6 +311,44 @@ fn install_custom_runtime_configs(config: &Config, container_runtime: &str) -> R
             container_runtime,
         )?;
 
+        // Everything devkit needs (the extension image + the debug console shell)
+        // goes into the drop-in (25-devkit.toml), leaving the copied base config
+        // untouched for users to inspect.
+        //
+        // The drop-in merge REPLACES the guest_extension_images array rather than
+        // appending, so it must re-emit the base's existing extensions in order and
+        // append devkit last: this preserves them (stable device enumeration) and
+        // makes devkit the last image attached, perturbing the guest we want to
+        // debug as little as possible.
+        if runtime.devkit {
+            let image_path = format!(
+                "{}/share/kata-containers/kata-containers-devkit-extension.img",
+                config.dest_dir
+            );
+            let verity_params = devkit_verity_params(&config.dest_dir);
+            let hypervisor_name = get_hypervisor_name(&runtime.base_config)?;
+            let existing = if Path::new(&dest_config).exists() {
+                read_guest_extension_images(Path::new(&dest_config), hypervisor_name)?
+            } else {
+                log::warn!(
+                    "devkit: base config {} is missing; the devkit drop-in will only \
+                     carry the devkit extension image",
+                    dest_config
+                );
+                Vec::new()
+            };
+            write_drop_in_file(
+                &config_d_dir,
+                "25-devkit.toml",
+                &generate_devkit_drop_in(
+                    hypervisor_name,
+                    &existing,
+                    &image_path,
+                    verity_params.as_deref(),
+                ),
+            )?;
+        }
+
         // Copy user-provided drop-in file if provided (at 50-overrides.toml).
         // If it was removed from values in a later upgrade/migration, remove stale file.
         let drop_in_dest = format!("{}/50-overrides.toml", config_d_dir);
@@ -373,6 +416,97 @@ fn remove_custom_runtime_configs(config: &Config) -> Result<()> {
     Ok(())
 }
 
+/// Remove on-node devkit artifacts the current configuration no longer wants.
+///
+/// Devkit is materialized as kata-<shim>-devkit custom runtimes only while
+/// DEVKIT=true. When a redeploy disables devkit (or drops a shim) those runtimes
+/// vanish from `config.custom_runtimes`, so the normal custom-runtime cleanup
+/// never touches their leftovers. Installs only ever add files, so reconcile
+/// explicitly: drop stale kata-<shim>-devkit handler directories and, once no
+/// devkit runtime remains, the now-unreferenced extension image and root-hash.
+///
+/// The RuntimeClasses are Helm-managed and the containerd handlers regenerated
+/// every install, so only these binary-installed files need reconciling here.
+fn reconcile_devkit_artifacts(config: &Config) -> Result<()> {
+    // Handlers the current config owns (devkit-synthesized or user-provided). Any
+    // kata-<shim>-devkit directory not in this set is a stale devkit leftover; a
+    // handler that happens to match a user's own custom runtime is protected.
+    let known: HashSet<&str> = config
+        .custom_runtimes
+        .iter()
+        .map(|r| r.handler.as_str())
+        .collect();
+
+    let custom_runtimes_dir = format!(
+        "/host/{}/share/defaults/kata-containers/custom-runtimes",
+        config.dest_dir
+    );
+    let share_dir = format!("{}/share/kata-containers", config.host_install_dir);
+
+    reconcile_devkit_artifacts_in(
+        &known,
+        Path::new(&custom_runtimes_dir),
+        Path::new(&share_dir),
+        config.devkit_enabled,
+    )
+}
+
+/// Pure core of [`reconcile_devkit_artifacts`], separated so it can be unit
+/// tested against real directories without a full [`Config`].
+///
+/// `known_handlers` are preserved; every other kata-<shim>-devkit directory under
+/// `custom_runtimes_dir` is removed. When `devkit_enabled` is false the
+/// unreferenced extension image and root-hash under `share_dir` are removed too.
+fn reconcile_devkit_artifacts_in(
+    known_handlers: &HashSet<&str>,
+    custom_runtimes_dir: &Path,
+    share_dir: &Path,
+    devkit_enabled: bool,
+) -> Result<()> {
+    for shim in ALL_SHIMS {
+        let handler = format!("kata-{shim}-devkit");
+        if known_handlers.contains(handler.as_str()) {
+            continue;
+        }
+        let handler_dir = custom_runtimes_dir.join(&handler);
+        if handler_dir.exists() {
+            info!(
+                "devkit: removing stale runtime directory {}",
+                handler_dir.display()
+            );
+            if let Err(e) = fs::remove_dir_all(&handler_dir) {
+                log::warn!("devkit: failed to remove {}: {}", handler_dir.display(), e);
+            }
+        }
+    }
+
+    if custom_runtimes_dir.exists() {
+        if let Ok(entries) = fs::read_dir(custom_runtimes_dir) {
+            if entries.count() == 0 {
+                let _ = fs::remove_dir(custom_runtimes_dir);
+            }
+        }
+    }
+
+    // With no devkit runtime left, the extracted extension image is unreferenced.
+    if !devkit_enabled {
+        for name in [
+            "kata-containers-devkit-extension.img",
+            "root_hash_devkit-extension.txt",
+        ] {
+            let path = share_dir.join(name);
+            if path.exists() {
+                info!("devkit: removing unreferenced {}", path.display());
+                if let Err(e) = fs::remove_file(&path) {
+                    log::warn!("devkit: failed to remove {}: {}", path.display(), e);
+                }
+            }
+        }
+    }
+
+    Ok(())
+}
+
 /// Copy an extracted artifact tree from `src` into `dst`.
 ///
 /// Used only by unit tests; production code now uses `extract_component_tarballs`.
@@ -502,6 +636,13 @@ fn collect_required_tarballs(config: &Config) -> Result<HashSet<String>> {
             }
         }
     }
+
+    // Generic, not tied to any shim in shim-components.json, so it is pulled in
+    // by the devkit flag rather than per-shim membership.
+    if config.devkit_enabled {
+        required.insert("rootfs-image-devkit-extension".to_string());
+    }
+
     Ok(required)
 }
 
@@ -1302,6 +1443,154 @@ enable_debug = true
     Ok(content)
 }
 
+/// Guest path of the devkit debug console shell. Must live under
+/// /run/kata-extensions/, the prefix the agent enforces for debug_console_shell.
+const DEVKIT_DEBUG_CONSOLE_SHELL: &str = "/run/kata-extensions/devkit/bin/devkit-sh";
+
+/// Read the devkit extension's dm-verity params from its root-hash file. Absent
+/// (e.g. s390x, built without a measured rootfs) means `None` and the image is
+/// mounted unverified.
+fn devkit_verity_params(dest_dir: &str) -> Option<String> {
+    let root_hash_file = format!(
+        "/host{}/share/kata-containers/root_hash_devkit-extension.txt",
+        dest_dir
+    );
+    match fs::read_to_string(&root_hash_file) {
+        Ok(content) => content
+            .lines()
+            .map(str::trim)
+            .find(|l| !l.is_empty())
+            .map(|l| l.to_string()),
+        Err(_) => {
+            log::warn!(
+                "devkit: {} not found; mounting the devkit extension without dm-verity",
+                root_hash_file
+            );
+            None
+        }
+    }
+}
+
+/// A `guest_extension_images` entry read from a runtime's base config.
+struct GuestExtension {
+    name: String,
+    path: String,
+    verity_params: Option<String>,
+}
+
+/// Read the base config's existing `guest_extension_images`, in order, so the
+/// drop-in can re-emit them ahead of devkit (the merge replaces the array, so
+/// anything omitted is lost). Any pre-existing `devkit` entry is skipped, since
+/// the drop-in always appends its own.
+fn read_guest_extension_images(
+    config_file: &Path,
+    hypervisor_name: &str,
+) -> Result<Vec<GuestExtension>> {
+    let content = fs::read_to_string(config_file)
+        .with_context(|| format!("Failed to read config for devkit extension: {config_file:?}"))?;
+    let doc = content
+        .parse::<DocumentMut>()
+        .with_context(|| format!("Failed to parse config as TOML: {config_file:?}"))?;
+
+    let images = doc
+        .get("hypervisor")
+        .and_then(|h| h.get(hypervisor_name))
+        .and_then(|hv| hv.get("guest_extension_images"))
+        .and_then(|i| i.as_array_of_tables());
+
+    let mut out = Vec::new();
+    if let Some(images) = images {
+        for t in images.iter() {
+            let name = t.get("name").and_then(|i| i.as_str());
+            let path = t.get("path").and_then(|i| i.as_str());
+            if let (Some(name), Some(path)) = (name, path) {
+                if name == "devkit" {
+                    continue;
+                }
+                out.push(GuestExtension {
+                    name: name.to_string(),
+                    path: path.to_string(),
+                    verity_params: t
+                        .get("verity_params")
+                        .and_then(|i| i.as_str())
+                        .map(str::to_string),
+                });
+            }
+        }
+    }
+    Ok(out)
+}
+
+fn push_guest_extension(
+    images: &mut ArrayOfTables,
+    name: &str,
+    path: &str,
+    verity_params: Option<&str>,
+) {
+    let mut entry = Table::new();
+    entry["name"] = value(name);
+    entry["path"] = value(path);
+    if let Some(vp) = verity_params {
+        entry["verity_params"] = value(vp);
+    }
+    images.push(entry);
+}
+
+/// Generate the devkit drop-in (25-devkit.toml): the agent debug-console scalars
+/// plus the full guest_extension_images array (base extensions in order, devkit
+/// last). The base config is left untouched so the shipped configuration.toml
+/// stays clean for users.
+fn generate_devkit_drop_in(
+    hypervisor_name: &str,
+    existing: &[GuestExtension],
+    image_path: &str,
+    verity_params: Option<&str>,
+) -> String {
+    let mut doc = DocumentMut::new();
+
+    // [agent.kata] scalars. The intermediate `agent` table is implicit so only
+    // the `[agent.kata]` header is emitted, not an empty `[agent]`.
+    let mut agent_kata = Table::new();
+    agent_kata["debug_console_enabled"] = value(true);
+    agent_kata["debug_console_shell"] = value(DEVKIT_DEBUG_CONSOLE_SHELL);
+    let mut agent = Table::new();
+    agent.set_implicit(true);
+    agent.insert("kata", Item::Table(agent_kata));
+    doc.insert("agent", Item::Table(agent));
+
+    // [[hypervisor.<hv>.guest_extension_images]]: base extensions first, devkit
+    // last. Built as explicit (non-inline) tables so the array of tables renders
+    // as headers; the intermediate `hypervisor` table is implicit.
+    let mut images = ArrayOfTables::new();
+    for ext in existing {
+        push_guest_extension(
+            &mut images,
+            &ext.name,
+            &ext.path,
+            ext.verity_params.as_deref(),
+        );
+    }
+    push_guest_extension(&mut images, "devkit", image_path, verity_params);
+    let mut hv = Table::new();
+    hv.insert("guest_extension_images", Item::ArrayOfTables(images));
+    let mut hypervisor = Table::new();
+    hypervisor.set_implicit(true);
+    hypervisor.insert(hypervisor_name, Item::Table(hv));
+    doc.insert("hypervisor", Item::Table(hypervisor));
+
+    let mut content = String::new();
+    content.push_str("# Devkit debug extension\n");
+    content.push_str("# Generated by kata-deploy\n");
+    content.push_str("#\n");
+    content.push_str("# Points the agent debug console at the shell shipped by the devkit\n");
+    content.push_str("# extension, and cold-plugs the extension image as the last\n");
+    content.push_str("# guest_extension_images entry (any extensions the base config already\n");
+    content
+        .push_str("# ships are re-listed here first, since drop-in merging replaces arrays).\n\n");
+    content.push_str(&doc.to_string());
+    content
+}
+
 /// Get proxy value for a specific shim from config.
 /// Handles both per-shim format ("qemu-tdx=http://proxy:8080;qemu-snp=http://proxy2:8080")
 /// and global format ("http://proxy:8080").
@@ -1517,6 +1806,179 @@ mod tests {
         );
     }
 
+    const DEVKIT_IMG: &str = "/opt/kata/share/kata-containers/kata-containers-devkit-extension.img";
+
+    /// Parse a generated drop-in and return the ordered `name` values of its
+    /// guest_extension_images array under the given hypervisor.
+    fn drop_in_extension_names(content: &str, hypervisor_name: &str) -> Vec<String> {
+        let doc = content.parse::<DocumentMut>().unwrap();
+        doc["hypervisor"][hypervisor_name]["guest_extension_images"]
+            .as_array_of_tables()
+            .unwrap()
+            .iter()
+            .map(|t| t["name"].as_str().unwrap().to_string())
+            .collect()
+    }
+
+    #[test]
+    fn test_generate_devkit_drop_in_scalars() {
+        // The drop-in always carries the agent debug-console scalars.
+        let content = generate_devkit_drop_in("qemu", &[], DEVKIT_IMG, None);
+
+        assert!(content.contains("[agent.kata]"));
+        assert!(content.contains("debug_console_enabled = true"));
+        assert!(
+            content.contains("debug_console_shell = \"/run/kata-extensions/devkit/bin/devkit-sh\"")
+        );
+    }
+
+    #[test]
+    fn test_generate_devkit_drop_in_only_devkit_when_base_empty() {
+        // A base config with no extensions yields an array with devkit only.
+        let content = generate_devkit_drop_in("qemu", &[], DEVKIT_IMG, None);
+
+        assert_eq!(drop_in_extension_names(&content, "qemu"), vec!["devkit"]);
+        assert!(content.contains(&format!("path = \"{DEVKIT_IMG}\"")));
+        // No root hash provided -> no verity params emitted (raw mount).
+        assert!(!content.contains("verity_params"));
+    }
+
+    #[test]
+    fn test_generate_devkit_drop_in_reemits_existing_then_devkit() {
+        // A base config that already ships an extension (e.g. coco) must have it
+        // re-listed first, with devkit strictly after it, so the array-replacing
+        // drop-in merge does not drop it.
+        let existing = vec![GuestExtension {
+            name: "coco".to_string(),
+            path: "/opt/kata/share/kata-containers/coco.img".to_string(),
+            verity_params: Some("root_hash=abc".to_string()),
+        }];
+        let content = generate_devkit_drop_in("qemu", &existing, DEVKIT_IMG, Some("root_hash=def"));
+
+        assert_eq!(
+            drop_in_extension_names(&content, "qemu"),
+            vec!["coco", "devkit"]
+        );
+        // The pre-existing coco entry (path + verity) is re-emitted verbatim.
+        assert!(content.contains("name = \"coco\""));
+        assert!(content.contains("path = \"/opt/kata/share/kata-containers/coco.img\""));
+        assert!(content.contains("root_hash=abc"));
+        // devkit is present with its own image path and verity params.
+        assert!(content.contains(&format!("path = \"{DEVKIT_IMG}\"")));
+        assert!(content.contains("root_hash=def"));
+    }
+
+    #[test]
+    fn test_read_guest_extension_images_skips_devkit_and_keeps_order() {
+        // Base config carries two extensions plus a stray devkit; the reader keeps
+        // the real ones in order and drops any pre-existing devkit entry.
+        let tmp = tempfile::NamedTempFile::new().unwrap();
+        std::fs::write(
+            tmp.path(),
+            "[hypervisor.qemu]\npath = \"/usr/bin/qemu\"\n\n\
+             [[hypervisor.qemu.guest_extension_images]]\n\
+             name = \"coco\"\npath = \"/opt/kata/share/kata-containers/coco.img\"\n\
+             verity_params = \"root_hash=abc\"\n\n\
+             [[hypervisor.qemu.guest_extension_images]]\n\
+             name = \"gpu\"\npath = \"/opt/kata/share/kata-containers/gpu.img\"\n\n\
+             [[hypervisor.qemu.guest_extension_images]]\n\
+             name = \"devkit\"\npath = \"/stale/devkit.img\"\n",
+        )
+        .unwrap();
+
+        let existing = read_guest_extension_images(tmp.path(), "qemu").unwrap();
+        let names: Vec<_> = existing.iter().map(|e| e.name.as_str()).collect();
+        assert_eq!(names, vec!["coco", "gpu"]);
+        assert_eq!(existing[0].verity_params.as_deref(), Some("root_hash=abc"));
+        assert_eq!(existing[1].verity_params, None);
+    }
+
+    #[test]
+    fn test_read_guest_extension_images_none_when_absent() {
+        // A base config with no extension array yields an empty list.
+        let tmp = tempfile::NamedTempFile::new().unwrap();
+        std::fs::write(tmp.path(), "[hypervisor.qemu]\npath = \"/usr/bin/qemu\"\n").unwrap();
+
+        assert!(read_guest_extension_images(tmp.path(), "qemu")
+            .unwrap()
+            .is_empty());
+    }
+
+    /// Lay down a fake devkit handler directory and the extension image + root
+    /// hash, mimicking what a devkit-enabled install leaves on the node.
+    fn seed_devkit_layout(custom_runtimes_dir: &Path, share_dir: &Path, handlers: &[&str]) {
+        for handler in handlers {
+            let d = custom_runtimes_dir.join(handler).join("config.d");
+            fs::create_dir_all(&d).unwrap();
+            fs::write(d.join("25-devkit.toml"), "[agent.kata]\n").unwrap();
+        }
+        fs::create_dir_all(share_dir).unwrap();
+        fs::write(
+            share_dir.join("kata-containers-devkit-extension.img"),
+            "img",
+        )
+        .unwrap();
+        fs::write(
+            share_dir.join("root_hash_devkit-extension.txt"),
+            "root_hash=x",
+        )
+        .unwrap();
+    }
+
+    #[test]
+    fn test_reconcile_devkit_disabled_removes_everything() {
+        let tmp = tempfile::tempdir().unwrap();
+        let custom = tmp.path().join("custom-runtimes");
+        let share = tmp.path().join("share/kata-containers");
+        seed_devkit_layout(&custom, &share, &["kata-qemu-devkit", "kata-fc-devkit"]);
+
+        // devkit disabled -> no known devkit handlers, image should go too.
+        reconcile_devkit_artifacts_in(&HashSet::new(), &custom, &share, false).unwrap();
+
+        assert!(!custom.join("kata-qemu-devkit").exists());
+        assert!(!custom.join("kata-fc-devkit").exists());
+        // Emptied custom-runtimes dir is dropped.
+        assert!(!custom.exists());
+        assert!(!share.join("kata-containers-devkit-extension.img").exists());
+        assert!(!share.join("root_hash_devkit-extension.txt").exists());
+    }
+
+    #[test]
+    fn test_reconcile_devkit_keeps_active_and_protects_user_runtimes() {
+        let tmp = tempfile::tempdir().unwrap();
+        let custom = tmp.path().join("custom-runtimes");
+        let share = tmp.path().join("share/kata-containers");
+        // kata-qemu-devkit is still active; kata-fc-devkit is stale.
+        seed_devkit_layout(&custom, &share, &["kata-qemu-devkit", "kata-fc-devkit"]);
+        // A user custom runtime that must never be touched.
+        let user = custom.join("kata-mine");
+        fs::create_dir_all(&user).unwrap();
+        fs::write(user.join("marker"), "keep").unwrap();
+
+        let known: HashSet<&str> = ["kata-qemu-devkit", "kata-mine"].into_iter().collect();
+        // devkit still enabled -> image is preserved.
+        reconcile_devkit_artifacts_in(&known, &custom, &share, true).unwrap();
+
+        assert!(custom.join("kata-qemu-devkit").exists(), "active kept");
+        assert!(!custom.join("kata-fc-devkit").exists(), "stale removed");
+        assert!(user.join("marker").exists(), "user runtime untouched");
+        assert!(
+            share.join("kata-containers-devkit-extension.img").exists(),
+            "image kept while devkit enabled"
+        );
+    }
+
+    #[test]
+    fn test_reconcile_devkit_is_noop_without_leftovers() {
+        let tmp = tempfile::tempdir().unwrap();
+        let custom = tmp.path().join("custom-runtimes");
+        let share = tmp.path().join("share/kata-containers");
+        // Nothing seeded: reconcile must succeed and create nothing.
+        reconcile_devkit_artifacts_in(&HashSet::new(), &custom, &share, false).unwrap();
+        assert!(!custom.exists());
+        assert!(!share.exists());
+    }
+
     #[test]
     fn test_copy_artifacts_overwrites_existing_files() {
         use std::fs;

tools/packaging/kata-deploy/binary/src/config.rs

@@ -161,6 +161,9 @@ pub struct CustomRuntime {
     pub containerd_snapshotter: Option<String>,
     /// CRI-O pull type (e.g., "guest-pull")
     pub crio_pull_type: Option<String>,
+    /// Internally-synthesized devkit runtime (kata-<shim>-devkit): gets an extra
+    /// drop-in wiring the extension image and debug console shell.
+    pub devkit: bool,
 }
 
 #[derive(Debug, Clone)]
@@ -199,6 +202,11 @@ pub struct Config {
     pub daemonset_name: String,
     pub custom_runtimes_enabled: bool,
     pub custom_runtimes: Vec<CustomRuntime>,
+    /// Install the devkit extension and a kata-<shim>-devkit custom runtime per
+    /// enabled shim. From the DEVKIT env var, honored only when debug is also on:
+    /// the devkit drop-in enables the agent debug console, which must never come
+    /// up without debug.
+    pub devkit_enabled: bool,
     /// EROFS snapshotter rw-layer backing mode ("disk" or "memory").
     pub erofs_snapshotter_mode: Option<String>,
     /// Enable dm-verity integrity for EROFS lower layers.
@@ -238,7 +246,7 @@ impl Config {
         // Parse shims - only use arch-specific variable
         // Use architecture-specific default shims list (only shims supported for this arch)
         let default_shims = get_default_shims_for_arch(&arch);
-        let shims_for_arch = get_arch_var("SHIMS", default_shims, &arch)
+        let shims_for_arch: Vec<String> = get_arch_var("SHIMS", default_shims, &arch)
             .split_whitespace()
             .map(|s| s.to_string())
             .collect();
@@ -346,14 +354,54 @@ impl Config {
                 .collect();
 
         // Parse custom runtimes from ConfigMap
-        let custom_runtimes_enabled =
+        let custom_runtimes_from_configmap =
             env::var("CUSTOM_RUNTIMES_ENABLED").unwrap_or_else(|_| "false".to_string()) == "true";
-        let custom_runtimes = if custom_runtimes_enabled {
+        let mut custom_runtimes = if custom_runtimes_from_configmap {
             parse_custom_runtimes()?
         } else {
             Vec::new()
         };
 
+        // Model devkit as one custom runtime per shim (kata-<shim>-devkit) to
+        // reuse the whole custom-runtime install/register/cleanup machinery; the
+        // only devkit-specific bit is an extra drop-in (in install.rs).
+        //
+        // Gate it on debug: the devkit drop-in turns on the agent debug console,
+        // so honoring DEVKIT without DEBUG would silently enable it, breaking the
+        // "only effective with debug" contract.
+        let devkit_requested = env::var("DEVKIT").unwrap_or_else(|_| "false".to_string()) == "true";
+        if devkit_requested && !debug {
+            log::warn!("DEVKIT=true ignored: it requires DEBUG=true to take effect");
+        }
+        let devkit_enabled = devkit_requested && debug;
+        if devkit_enabled {
+            for shim in &shims_for_arch {
+                // Inherit the base shim's image-pulling config: the devkit
+                // RuntimeClass shares the base config (same shared_fs), so it must
+                // pull the same way. Otherwise containerd/CRI-O default to
+                // overlayfs/node-pull, which breaks shims running shared_fs = none
+                // (e.g. nvidia runtime-rs) and makes every devkit pod terminate.
+                let containerd_snapshotter = snapshotter_handler_mapping_for_arch
+                    .as_ref()
+                    .and_then(|mapping| lookup_mapping_value(mapping, shim));
+                let crio_pull_type = pull_type_mapping_for_arch
+                    .as_ref()
+                    .and_then(|mapping| lookup_mapping_value(mapping, shim));
+
+                custom_runtimes.push(CustomRuntime {
+                    handler: format!("kata-{shim}-devkit"),
+                    base_config: shim.to_string(),
+                    drop_in_file: None,
+                    containerd_snapshotter,
+                    crio_pull_type,
+                    devkit: true,
+                });
+            }
+        }
+
+        // Enable the custom-runtime code paths if either source produced entries.
+        let custom_runtimes_enabled = !custom_runtimes.is_empty();
+
         let erofs_snapshotter_mode = env::var("EROFS_SNAPSHOTTER_MODE")
             .ok()
             .map(|s| s.trim().to_string())
@@ -401,6 +449,7 @@ impl Config {
             daemonset_name,
             custom_runtimes_enabled,
             custom_runtimes,
+            devkit_enabled,
             erofs_snapshotter_mode,
             erofs_dmverity,
             startup_taints,
@@ -795,6 +844,21 @@ fn get_arch() -> Result<String> {
     .to_string())
 }
 
+/// Look up `shim`'s value in a comma-separated "shim1:value1,shim2:value2"
+/// mapping (SNAPSHOTTER_HANDLER_MAPPING, PULL_TYPE_MAPPING). None if absent or
+/// empty.
+fn lookup_mapping_value(mapping: &str, shim: &str) -> Option<String> {
+    mapping.split(',').find_map(|entry| {
+        let parts: Vec<&str> = entry.split(':').collect();
+        if parts.len() == 2 && parts[0].trim() == shim {
+            let value = parts[1].trim();
+            (!value.is_empty()).then(|| value.to_string())
+        } else {
+            None
+        }
+    })
+}
+
 /// Parse custom runtimes from the mounted ConfigMap at /custom-configs/
 /// Reads the custom-runtimes.list file which contains entries in the format:
 /// handler:baseConfig:containerd_snapshotter:crio_pulltype
@@ -872,6 +936,7 @@ fn parse_custom_runtimes() -> Result<Vec<CustomRuntime>> {
             drop_in_file,
             containerd_snapshotter,
             crio_pull_type,
+            devkit: false,
         });
     }
 
@@ -987,6 +1052,8 @@ mod tests {
             "EXPERIMENTAL_FORCE_GUEST_PULL_PPC64LE",
             "CONTAINERD_CONFIG_FILE_NAME",
             "STARTUP_TAINTS",
+            "CUSTOM_RUNTIMES_ENABLED",
+            "DEVKIT",
         ];
         for var in &vars {
             std::env::remove_var(var);
@@ -1135,6 +1202,100 @@ mod tests {
         cleanup_env_vars();
     }
 
+    #[serial]
+    #[test]
+    fn test_devkit_disabled_by_default() {
+        setup_minimal_env();
+
+        let config = Config::from_env().unwrap();
+
+        assert!(!config.devkit_enabled);
+        assert!(config.custom_runtimes.iter().all(|r| !r.devkit));
+        cleanup_env_vars();
+    }
+
+    #[serial]
+    #[test]
+    fn test_devkit_requires_debug() {
+        setup_minimal_env();
+        // DEBUG defaults to false in setup_minimal_env.
+        std::env::set_var("DEVKIT", "true");
+
+        let config = Config::from_env().unwrap();
+
+        assert!(!config.devkit_enabled);
+        assert!(config.custom_runtimes.iter().all(|r| !r.devkit));
+        cleanup_env_vars();
+    }
+
+    #[serial]
+    #[test]
+    fn test_devkit_synthesizes_per_shim_runtime() {
+        setup_minimal_env();
+        std::env::set_var("DEBUG", "true");
+        std::env::set_var("DEVKIT", "true");
+
+        let config = Config::from_env().unwrap();
+
+        assert!(config.devkit_enabled);
+        assert!(config.custom_runtimes_enabled);
+
+        // setup_minimal_env configures a single "qemu" shim for this arch.
+        let devkit: Vec<_> = config.custom_runtimes.iter().filter(|r| r.devkit).collect();
+        assert_eq!(devkit.len(), 1);
+        assert_eq!(devkit[0].handler, "kata-qemu-devkit");
+        assert_eq!(devkit[0].base_config, "qemu");
+        assert!(devkit[0].drop_in_file.is_none());
+        // With no snapshotter/pull-type mapping, the devkit runtime inherits
+        // nothing and containerd/CRI-O use their defaults for the base shim.
+        assert!(devkit[0].containerd_snapshotter.is_none());
+        assert!(devkit[0].crio_pull_type.is_none());
+
+        cleanup_env_vars();
+    }
+
+    #[serial]
+    #[test]
+    fn test_devkit_inherits_base_shim_snapshotter_and_pull_type() {
+        setup_minimal_env();
+        std::env::set_var("DEBUG", "true");
+        std::env::set_var("DEVKIT", "true");
+        // The base "qemu" shim is mapped to the erofs snapshotter / guest-pull;
+        // its devkit variant must inherit both so pods on the devkit
+        // RuntimeClass pull images the same way as the base shim (crucial for
+        // shims running with shared_fs = none).
+        set_arch_var("SNAPSHOTTER_HANDLER_MAPPING", "qemu:erofs");
+        set_arch_var("PULL_TYPE_MAPPING", "qemu:guest-pull");
+
+        let config = Config::from_env().unwrap();
+
+        let devkit: Vec<_> = config.custom_runtimes.iter().filter(|r| r.devkit).collect();
+        assert_eq!(devkit.len(), 1);
+        assert_eq!(devkit[0].handler, "kata-qemu-devkit");
+        assert_eq!(devkit[0].containerd_snapshotter.as_deref(), Some("erofs"));
+        assert_eq!(devkit[0].crio_pull_type.as_deref(), Some("guest-pull"));
+
+        cleanup_env_vars();
+    }
+
+    #[test]
+    fn test_lookup_mapping_value() {
+        let mapping = "qemu:erofs, fc:nydus,clh:";
+        assert_eq!(
+            lookup_mapping_value(mapping, "qemu").as_deref(),
+            Some("erofs")
+        );
+        // Entries may carry surrounding whitespace.
+        assert_eq!(
+            lookup_mapping_value(mapping, "fc").as_deref(),
+            Some("nydus")
+        );
+        // Empty value is treated as absent.
+        assert!(lookup_mapping_value(mapping, "clh").is_none());
+        // Unknown shim.
+        assert!(lookup_mapping_value(mapping, "stratovirt").is_none());
+    }
+
     #[serial]
     #[test]
     fn test_multi_install_suffix_with_value() {

tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl

@@ -696,6 +696,12 @@ e.g. `{{- include "kata-deploy.commonEnv" . | nindent 8 }}`.
 - name: CUSTOM_RUNTIMES_ENABLED
   value: "true"
 {{- end }}
+{{- /* Devkit debug extension: only effective together with debug (the debug
+       console must be enabled for it to be reachable). */ -}}
+{{- if and .Values.debug .Values.devkit }}
+- name: DEVKIT
+  value: "true"
+{{- end }}
 {{- with .Values.startupTaints }}
 - name: STARTUP_TAINTS
   value: {{ join "," . | quote }}

tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/runtimeclasses.yaml

@@ -41,20 +41,36 @@
 
 {{- /*
   Render a single RuntimeClass. Params (dict): root (.), shim, config, shimConfig,
-  nameOverride (optional; if set, use as metadata.name for default RC), useShimNodeSelectors.
+  nameOverride (optional; if set, use as metadata.name for default RC),
+  variant (optional; e.g. "devkit" -> name/handler kata-<shim>-devkit),
+  useShimNodeSelectors.
+
+  Name/handler precedence: variant > multiInstallSuffix > nameOverride > default.
+  A variant RuntimeClass mirrors the handler kata-deploy registers for the same
+  variant (e.g. the kata-<shim>-devkit custom runtime), so it ignores the
+  multiInstallSuffix/nameOverride paths.
 */ -}}
 {{- define "kata-deploy.runtimeclass" -}}
+{{- $rcName := "" -}}
+{{- $rcHandler := "" -}}
+{{- if .variant -}}
+{{- $rcName = printf "kata-%s-%s" .shim .variant -}}
+{{- $rcHandler = $rcName -}}
+{{- else if .root.Values.env.multiInstallSuffix -}}
+{{- $rcName = printf "kata-%s-%s" .shim .root.Values.env.multiInstallSuffix -}}
+{{- $rcHandler = $rcName -}}
+{{- else if .nameOverride -}}
+{{- $rcName = .nameOverride -}}
+{{- $rcHandler = printf "kata-%s" .shim -}}
+{{- else -}}
+{{- $rcName = printf "kata-%s" .shim -}}
+{{- $rcHandler = $rcName -}}
+{{- end -}}
 ---
 kind: RuntimeClass
 apiVersion: node.k8s.io/v1
 metadata:
-{{- if .root.Values.env.multiInstallSuffix }}
-  name: kata-{{ .shim }}-{{ .root.Values.env.multiInstallSuffix }}
-{{- else if .nameOverride }}
-  name: {{ .nameOverride }}
-{{- else }}
-  name: kata-{{ .shim }}
-{{- end }}
+  name: {{ $rcName }}
   labels:
     app.kubernetes.io/managed-by: kata-deploy
 {{- if .root.Values.env.multiInstallSuffix }}
@@ -64,11 +80,7 @@ metadata:
 {{- end }}
   annotations:
 {{- include "kata-deploy.runtimeclassAnnotations" .root | nindent 4 }}
-{{- if .root.Values.env.multiInstallSuffix }}
-handler: kata-{{ .shim }}-{{ .root.Values.env.multiInstallSuffix }}
-{{- else }}
-handler: kata-{{ .shim }}
-{{- end }}
+handler: {{ $rcHandler }}
 {{- /* Overhead section - controlled by global or per-shim overheadEnabled flag (default: true) */ -}}
 {{- $shimOverheadEnabled := true -}}
 {{- if hasKey .root.Values.runtimeClasses "overheadEnabled" -}}
@@ -155,4 +167,22 @@ scheduling:
 {{- end }}
 {{- end }}
 {{- end }}
+
+{{- /* Devkit debug RuntimeClasses: an extra kata-<shim>-devkit per enabled shim,
+       matching the handler kata-deploy registers when DEVKIT=true. Gated by
+       debug (reachable only with the debug console) and devkit; reuses each
+       shim's overhead/nodeSelectors. */ -}}
+{{- if and .Values.debug .Values.devkit }}
+{{- range $shim := $enabledShims }}
+{{- $config := index $runtimeClassConfigs $shim }}
+{{- $shimConfig := index $.Values.shims $shim }}
+{{- if $config }}
+{{- $effectiveConfig := deepCopy $config }}
+{{- if and $shimConfig.runtimeClass $shimConfig.runtimeClass.overhead }}
+{{- $effectiveConfig = mergeOverwrite $effectiveConfig $shimConfig.runtimeClass.overhead }}
+{{- end }}
+{{ include "kata-deploy.runtimeclass" (dict "root" $ "shim" $shim "config" $effectiveConfig "shimConfig" $shimConfig "variant" "devkit" "nameOverride" "" "useShimNodeSelectors" $useShimNodeSelectors) }}
+{{- end }}
+{{- end }}
+{{- end }}
 {{- end }}

tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml

@@ -304,6 +304,15 @@ resources:
 
 debug: false
 
+# Deploy the optional "devkit" debug guest extension and, per enabled shim, an
+# extra `kata-<shim>-devkit` RuntimeClass wired to it. Pods on that class boot
+# with the extension cold-plugged, so the agent debug console drops into a rich
+# shell (Ubuntu + apt + debug tools) overlaid on the read-only guest.
+#
+# Debugging aid: only effective with `debug: true`, ignored otherwise. Leave it
+# off in production and in confidential (CoCo/TEE) deployments.
+devkit: false
+
 # Optional CLI log level override for kata-deploy.
 # Supported values: error, warn, info, debug, trace
 # When empty, debug:true implies --log-level debug; otherwise kata-deploy falls