kata-deploy: Add infrastructure for per-shim drop-in configuration

Instead of modifying original config files directly, set up a per-shim directory structure that uses symlinks to the original configs and config.d/ directories for drop-in overrides. This enables cleaner configuration management where the original files remain untouched and all kata-deploy customizations are in separate drop-in files that can be easily inspected and removed. Directory structure: {config_path}/runtimes/{shim}/ {config_path}/runtimes/{shim}/configuration-{shim}.toml -> symlink {config_path}/runtimes/{shim}/config.d/ Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>
2026-02-21 22:34:29 +00:00 · 2026-02-04 14:09:02 +01:00
parent c5ad3f9b26
commit cd76d61a3d
4 changed files with 267 additions and 106 deletions
--- a/tools/packaging/kata-deploy/binary/Cargo.lock
+++ b/tools/packaging/kata-deploy/binary/Cargo.lock
@@ -498,6 +498,7 @@ checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
 dependencies = [
 "futures-channel",
 "futures-core",
+ "futures-executor",
 "futures-io",
 "futures-sink",
 "futures-task",
@@ -520,6 +521,17 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"

+[[package]]
+name = "futures-executor"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
 [[package]]
 name = "futures-io"
 version = "0.3.31"
@@ -549,6 +561,12 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"

+[[package]]
+name = "futures-timer"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"
+
 [[package]]
 name = "futures-util"
 version = "0.3.31"
@@ -600,6 +618,12 @@ dependencies = [
 "wasip2",
 ]

+[[package]]
+name = "glob"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
+
 [[package]]
 name = "gloo-timers"
 version = "0.3.0"
@@ -887,6 +911,7 @@ dependencies = [
 "libc",
 "log",
 "regex",
+ "rstest",
 "serde",
 "serde_json",
 "serde_yaml",
@@ -1294,6 +1319,12 @@ version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"

+[[package]]
+name = "relative-path"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2"
+
 [[package]]
 name = "ring"
 version = "0.17.14"
@@ -1308,6 +1339,44 @@ dependencies = [
 "windows-sys 0.52.0",
 ]

+[[package]]
+name = "rstest"
+version = "0.18.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97eeab2f3c0a199bc4be135c36c924b6590b88c377d416494288c14f2db30199"
+dependencies = [
+ "futures",
+ "futures-timer",
+ "rstest_macros",
+ "rustc_version",
+]
+
+[[package]]
+name = "rstest_macros"
+version = "0.18.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605"
+dependencies = [
+ "cfg-if",
+ "glob",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "relative-path",
+ "rustc_version",
+ "syn",
+ "unicode-ident",
+]
+
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustix"
 version = "1.1.2"
@@ -1461,6 +1530,12 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "semver"
+version = "1.0.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
+
 [[package]]
 name = "serde"
 version = "1.0.228"
--- a/tools/packaging/kata-deploy/binary/Cargo.toml
+++ b/tools/packaging/kata-deploy/binary/Cargo.toml
@@ -50,3 +50,4 @@ tokio = { version = "1.38", features = ["rt-multi-thread", "macros", "time"] }

 [dev-dependencies]
 tempfile = "3.8"
+rstest = "0.18"
--- a/tools/packaging/kata-deploy/binary/src/artifacts/install.rs
+++ b/tools/packaging/kata-deploy/binary/src/artifacts/install.rs
@@ -123,7 +123,14 @@ pub async fn install_artifacts(config: &Config) -> Result<()> {
 pub async fn remove_artifacts(config: &Config) -> Result<()> {
    info!("deleting kata artifacts");

-    // Remove custom runtime configs first (before removing main install dir)
+    // Remove runtime directories for each shim (drop-in configs, symlinks)
+    for shim in &config.shims_for_arch {
+        if let Err(e) = remove_runtime_directory(config, shim) {
+            log::warn!("Failed to remove runtime directory for {}: {}", shim, e);
+        }
+    }
+
+    // Remove custom runtime configs (before removing main install dir)
    if config.custom_runtimes_enabled && !config.custom_runtimes.is_empty() {
        remove_custom_runtime_configs(config)?;
    }
@@ -152,24 +159,36 @@ fn install_custom_runtime_configs(config: &Config) -> Result<()> {
        fs::create_dir_all(&config_d_dir)
            .with_context(|| format!("Failed to create config.d directory: {}", config_d_dir))?;

-        // Copy base config (already modified by kata-deploy with debug, proxy, annotations)
-        let base_src = format!(
-            "/host/{}/share/defaults/kata-containers/configuration-{}.toml",
-            config.dest_dir, runtime.base_config
+        // Copy base config to the handler directory
+        // Custom runtime drop-ins will overlay on top of this
+        let base_config_filename = format!("configuration-{}.toml", runtime.base_config);
+        let original_config = format!(
+            "/host/{}/share/defaults/kata-containers/{}",
+            config.dest_dir, base_config_filename
        );
-        let base_dest = format!("{}/configuration-{}.toml", handler_dir, runtime.base_config);
+        let dest_config = format!("{}/{}", handler_dir, base_config_filename);

-        info!(
-            "Copying base config for {}: {} -> {}",
-            runtime.handler, base_src, base_dest
-        );
+        if Path::new(&original_config).exists() {
+            // Remove existing destination (might be a symlink from older versions)
+            let dest_path = Path::new(&dest_config);
+            if dest_path.exists() || dest_path.is_symlink() {
+                fs::remove_file(&dest_config).with_context(|| {
+                    format!("Failed to remove existing config: {}", dest_config)
+                })?;
+            }

-        fs::copy(&base_src, &base_dest).with_context(|| {
-            format!(
-                "Failed to copy base config from {} to {}",
-                base_src, base_dest
-            )
-        })?;
+            fs::copy(&original_config, &dest_config).with_context(|| {
+                format!(
+                    "Failed to copy config: {} -> {}",
+                    original_config, dest_config
+                )
+            })?;
+
+            info!(
+                "Copied config for custom runtime {}: {} -> {}",
+                runtime.handler, original_config, dest_config
+            );
+        }

        // Copy drop-in file if provided
        if let Some(ref drop_in_src) = runtime.drop_in_file {
@@ -288,21 +307,102 @@ fn set_executable_permissions(dir: &str) -> Result<()> {
    Ok(())
 }

-async fn configure_shim_config(config: &Config, shim: &str) -> Result<()> {
-    let config_path = format!(
-        "/host/{}",
+/// Set up the runtime directory structure for a shim.
+/// Creates: {config_path}/runtimes/{shim}/
+///          {config_path}/runtimes/{shim}/config.d/
+///          {config_path}/runtimes/{shim}/configuration-{shim}.toml (copy of original)
+///
+/// Note: We copy the config file instead of symlinking because kata-containers'
+/// ResolvePath uses filepath.EvalSymlinks, which would resolve to the original
+/// location and look for config.d there instead of in our per-shim directory.
+fn setup_runtime_directory(config: &Config, shim: &str) -> Result<()> {
+    let original_config_dir = format!(
+        "/host{}",
+        utils::get_kata_containers_original_config_path(shim, &config.dest_dir)
+    );
+    let runtime_config_dir = format!(
+        "/host{}",
+        utils::get_kata_containers_config_path(shim, &config.dest_dir)
+    );
+    let config_d_dir = format!("{}/config.d", runtime_config_dir);
+
+    // Create the runtime directory and config.d subdirectory
+    fs::create_dir_all(&config_d_dir)
+        .with_context(|| format!("Failed to create config.d directory: {}", config_d_dir))?;
+
+    // Copy the original config file to the runtime directory
+    let original_config_file = format!("{}/configuration-{}.toml", original_config_dir, shim);
+    let dest_config_file = format!("{}/configuration-{}.toml", runtime_config_dir, shim);
+
+    // Only copy if original exists
+    if Path::new(&original_config_file).exists() {
+        // Remove existing destination (might be a symlink from older versions)
+        // fs::copy follows symlinks and would write to the wrong location
+        let dest_path = Path::new(&dest_config_file);
+        if dest_path.exists() || dest_path.is_symlink() {
+            fs::remove_file(&dest_config_file)
+                .with_context(|| format!("Failed to remove existing config: {}", dest_config_file))?;
+        }
+
+        // Copy the base config file
+        fs::copy(&original_config_file, &dest_config_file)
+            .with_context(|| format!("Failed to copy config: {} -> {}", original_config_file, dest_config_file))?;
+        
+        log::debug!(
+            "Copied config for {}: {} -> {}",
+            shim,
+            original_config_file,
+            dest_config_file
+        );
+    }
+
+    Ok(())
+}
+
+/// Remove the runtime directory for a shim during cleanup
+fn remove_runtime_directory(config: &Config, shim: &str) -> Result<()> {
+    let runtime_config_dir = format!(
+        "/host{}",
        utils::get_kata_containers_config_path(shim, &config.dest_dir)
    );

-    let kata_config_file = Path::new(&config_path).join(format!("configuration-{shim}.toml"));
+    if Path::new(&runtime_config_dir).exists() {
+        fs::remove_dir_all(&runtime_config_dir)
+            .with_context(|| format!("Failed to remove runtime directory: {}", runtime_config_dir))?;
+        log::debug!("Removed runtime directory: {}", runtime_config_dir);
+    }

-    // The configuration file should exist after copy_artifacts() copied the kata artifacts.
-    // If it doesn't exist, it means either the copy failed or this shim doesn't have a config
-    // file in the artifacts (which would be a packaging error).
+    // Try to clean up parent 'runtimes' directory if empty
+    let runtimes_dir = Path::new(&runtime_config_dir).parent();
+    if let Some(runtimes_path) = runtimes_dir {
+        if runtimes_path.exists() {
+            if let Ok(entries) = fs::read_dir(runtimes_path) {
+                if entries.count() == 0 {
+                    let _ = fs::remove_dir(runtimes_path);
+                }
+            }
+        }
+    }
+
+    Ok(())
+}
+
+async fn configure_shim_config(config: &Config, shim: &str) -> Result<()> {
+    // Set up the runtime directory structure with symlink to original config
+    setup_runtime_directory(config, shim)?;
+
+    let runtime_config_dir = format!(
+        "/host{}",
+        utils::get_kata_containers_config_path(shim, &config.dest_dir)
+    );
+
+    let kata_config_file = Path::new(&runtime_config_dir).join(format!("configuration-{shim}.toml"));
+
+    // The configuration file (symlink) should exist after setup_runtime_directory()
    if !kata_config_file.exists() {
        return Err(anyhow::anyhow!(
            "Configuration file not found: {kata_config_file:?}. This file should have been \
-             copied from the kata-artifacts. Check that the shim '{}' has a valid configuration \
+             symlinked from the original config. Check that the shim '{}' has a valid configuration \
             file in the artifacts.",
            shim
        ));
--- a/tools/packaging/kata-deploy/binary/src/utils/system.rs
+++ b/tools/packaging/kata-deploy/binary/src/utils/system.rs
@@ -48,9 +48,18 @@ pub fn host_systemctl(args: &[&str]) -> Result<()> {
    Ok(())
 }

-/// Get kata containers config path based on shim type
-pub fn get_kata_containers_config_path(shim: &str, dest_dir: &str) -> String {
-    let golang_config_path = format!("{dest_dir}/share/defaults/kata-containers");
+/// Get kata containers config path based on shim type.
+/// This returns the path where the shim's configuration will be read from.
+/// For standard runtimes using drop-in configuration, this is the per-shim directory.
+pub fn get_kata_containers_config_path(shim: &str, base_dir: &str) -> String {
+    let base_path = get_kata_containers_original_config_path(shim, base_dir);
+    format!("{base_path}/runtimes/{shim}")
+}
+
+/// Get the original kata containers config path (where configs are installed).
+/// This is where the original, unmodified configuration files live.
+pub fn get_kata_containers_original_config_path(shim: &str, base_dir: &str) -> String {
+    let golang_config_path = format!("{base_dir}/share/defaults/kata-containers");
    let rust_config_path = format!("{golang_config_path}/runtime-rs");

    if is_rust_shim(shim) {
@@ -61,35 +70,24 @@ pub fn get_kata_containers_config_path(shim: &str, dest_dir: &str) -> String {
 }

 /// Get kata containers runtime path based on shim type
-pub fn get_kata_containers_runtime_path(shim: &str, dest_dir: &str) -> String {
+pub fn get_kata_containers_runtime_path(shim: &str, base_dir: &str) -> String {
    if is_rust_shim(shim) {
-        format!("{dest_dir}/runtime-rs/bin/containerd-shim-kata-v2")
+        format!("{base_dir}/runtime-rs/bin/containerd-shim-kata-v2")
    } else {
-        format!("{dest_dir}/bin/containerd-shim-kata-v2")
+        format!("{base_dir}/bin/containerd-shim-kata-v2")
    }
 }

 #[cfg(test)]
 mod tests {
    use super::*;
-
-    /// Helper to test config paths for multiple shims expecting the same result
-    fn assert_config_paths(shims: &[&str], dest_dir: &str, expected: &str) {
-        for shim in shims {
-            assert_eq!(
-                get_kata_containers_config_path(shim, dest_dir),
-                expected,
-                "Config path mismatch for shim '{}'",
-                shim
-            );
-        }
-    }
+    use rstest::rstest;

    /// Helper to test runtime paths for multiple shims expecting the same result
-    fn assert_runtime_paths(shims: &[&str], dest_dir: &str, expected: &str) {
+    fn assert_runtime_paths(shims: &[&str], base_dir: &str, expected: &str) {
        for shim in shims {
            assert_eq!(
-                get_kata_containers_runtime_path(shim, dest_dir),
+                get_kata_containers_runtime_path(shim, base_dir),
                expected,
                "Runtime path mismatch for shim '{}'",
                shim
@@ -97,37 +95,38 @@ mod tests {
        }
    }

-    #[test]
-    fn test_get_kata_containers_config_path_golang() {
-        let go_shims = ["qemu", "qemu-tdx", "qemu-snp", "fc"];
-        assert_config_paths(
-            &go_shims,
-            "/opt/kata",
-            "/opt/kata/share/defaults/kata-containers",
-        );
+    // Tests for get_kata_containers_original_config_path (where original configs live)
+    #[rstest]
+    #[case("qemu", "/opt/kata", "/opt/kata/share/defaults/kata-containers")]
+    #[case("qemu-tdx", "/opt/kata", "/opt/kata/share/defaults/kata-containers")]
+    #[case("fc", "/opt/kata", "/opt/kata/share/defaults/kata-containers")]
+    #[case("clh", "/opt/kata", "/opt/kata/share/defaults/kata-containers")]
+    #[case("cloud-hypervisor", "/opt/kata", "/opt/kata/share/defaults/kata-containers/runtime-rs")]
+    #[case("qemu-runtime-rs", "/opt/kata", "/opt/kata/share/defaults/kata-containers/runtime-rs")]
+    #[case("qemu", "/custom/path", "/custom/path/share/defaults/kata-containers")]
+    #[case("cloud-hypervisor", "/custom/path", "/custom/path/share/defaults/kata-containers/runtime-rs")]
+    fn test_get_kata_containers_original_config_path(
+        #[case] shim: &str,
+        #[case] base_dir: &str,
+        #[case] expected: &str,
+    ) {
+        assert_eq!(get_kata_containers_original_config_path(shim, base_dir), expected);
    }

-    #[test]
-    fn test_get_kata_containers_config_path_rust() {
-        assert_config_paths(
-            RUST_SHIMS,
-            "/opt/kata",
-            "/opt/kata/share/defaults/kata-containers/runtime-rs",
-        );
-    }
-
-    #[test]
-    fn test_get_kata_containers_config_path_custom_dest() {
-        assert_config_paths(
-            &["qemu"],
-            "/usr/local/kata",
-            "/usr/local/kata/share/defaults/kata-containers",
-        );
-        assert_config_paths(
-            &["cloud-hypervisor"],
-            "/usr/local/kata",
-            "/usr/local/kata/share/defaults/kata-containers/runtime-rs",
-        );
+    // Tests for get_kata_containers_config_path (per-shim runtime directories)
+    #[rstest]
+    #[case("qemu", "/opt/kata", "/opt/kata/share/defaults/kata-containers/runtimes/qemu")]
+    #[case("qemu-tdx", "/opt/kata", "/opt/kata/share/defaults/kata-containers/runtimes/qemu-tdx")]
+    #[case("fc", "/opt/kata", "/opt/kata/share/defaults/kata-containers/runtimes/fc")]
+    #[case("cloud-hypervisor", "/opt/kata", "/opt/kata/share/defaults/kata-containers/runtime-rs/runtimes/cloud-hypervisor")]
+    #[case("qemu-runtime-rs", "/opt/kata", "/opt/kata/share/defaults/kata-containers/runtime-rs/runtimes/qemu-runtime-rs")]
+    #[case("qemu", "/custom/path", "/custom/path/share/defaults/kata-containers/runtimes/qemu")]
+    fn test_get_kata_containers_config_path(
+        #[case] shim: &str,
+        #[case] base_dir: &str,
+        #[case] expected: &str,
+    ) {
+        assert_eq!(get_kata_containers_config_path(shim, base_dir), expected);
    }

    #[test]
@@ -197,27 +196,6 @@ mod tests {
        );
    }

-    #[test]
-    fn test_config_paths_share_defaults() {
-        // Test Go runtime config paths use /opt/kata/share/defaults/kata-containers
-        let go_shims = ["qemu", "qemu-tdx", "fc", "clh"];
-        assert_config_paths(
-            &go_shims,
-            "/opt/kata",
-            "/opt/kata/share/defaults/kata-containers",
-        );
-    }
-
-    #[test]
-    fn test_config_paths_runtime_rs() {
-        // Test Rust runtime config paths use /opt/kata/share/defaults/kata-containers/runtime-rs
-        assert_config_paths(
-            RUST_SHIMS,
-            "/opt/kata",
-            "/opt/kata/share/defaults/kata-containers/runtime-rs",
-        );
-    }
-
    #[test]
    fn test_full_deployment_paths_go_runtime() {
        // Test complete deployment structure for Go runtime
@@ -225,17 +203,19 @@ mod tests {
        let shim = "qemu-tdx";

        let config_path = get_kata_containers_config_path(shim, dest_dir);
+        let original_path = get_kata_containers_original_config_path(shim, dest_dir);
        let runtime_path = get_kata_containers_runtime_path(shim, dest_dir);

-        // Expected paths for Go runtime
-        assert_eq!(config_path, "/opt/kata/share/defaults/kata-containers");
+        // Expected paths for Go runtime with per-shim directory
+        assert_eq!(config_path, "/opt/kata/share/defaults/kata-containers/runtimes/qemu-tdx");
+        assert_eq!(original_path, "/opt/kata/share/defaults/kata-containers");
        assert_eq!(runtime_path, "/opt/kata/bin/containerd-shim-kata-v2");

-        // Config file would be at
+        // Config file would be at (symlink to original)
        let config_file = format!("{}/configuration-{}.toml", config_path, shim);
        assert_eq!(
            config_file,
-            "/opt/kata/share/defaults/kata-containers/configuration-qemu-tdx.toml"
+            "/opt/kata/share/defaults/kata-containers/runtimes/qemu-tdx/configuration-qemu-tdx.toml"
        );
    }

@@ -246,11 +226,16 @@ mod tests {
        let shim = "cloud-hypervisor";

        let config_path = get_kata_containers_config_path(shim, dest_dir);
+        let original_path = get_kata_containers_original_config_path(shim, dest_dir);
        let runtime_path = get_kata_containers_runtime_path(shim, dest_dir);

-        // Expected paths for Rust runtime
+        // Expected paths for Rust runtime with per-shim directory
        assert_eq!(
            config_path,
+            "/opt/kata/share/defaults/kata-containers/runtime-rs/runtimes/cloud-hypervisor"
+        );
+        assert_eq!(
+            original_path,
            "/opt/kata/share/defaults/kata-containers/runtime-rs"
        );
        assert_eq!(
@@ -258,17 +243,17 @@ mod tests {
            "/opt/kata/runtime-rs/bin/containerd-shim-kata-v2"
        );

-        // Config file would be at
+        // Config file would be at (symlink to original)
        let config_file = format!("{}/configuration-{}.toml", config_path, shim);
        assert_eq!(
            config_file,
-            "/opt/kata/share/defaults/kata-containers/runtime-rs/configuration-cloud-hypervisor.toml"
+            "/opt/kata/share/defaults/kata-containers/runtime-rs/runtimes/cloud-hypervisor/configuration-cloud-hypervisor.toml"
        );
    }

    #[test]
    fn test_mixed_deployment_both_runtimes() {
-        // Test that both Go and Rust runtimes can coexist
+        // Test that both Go and Rust runtimes can coexist with separate directories
        let dest_dir = "/opt/kata";

        // Go runtime
@@ -283,12 +268,12 @@ mod tests {
        assert_ne!(qemu_config, clh_config);
        assert_ne!(qemu_binary, clh_binary);

-        // Verify Go runtime paths
-        assert!(qemu_config.ends_with("kata-containers"));
+        // Verify Go runtime paths include per-shim directory
+        assert!(qemu_config.contains("/runtimes/qemu"));
        assert!(qemu_binary.ends_with("/bin/containerd-shim-kata-v2"));

-        // Verify Rust runtime paths
-        assert!(clh_config.ends_with("runtime-rs"));
+        // Verify Rust runtime paths include per-shim directory
+        assert!(clh_config.contains("/runtimes/cloud-hypervisor"));
        assert!(clh_binary.ends_with("/runtime-rs/bin/containerd-shim-kata-v2"));
    }
 }