From cd76d61a3d15936b342c97e4eed33cb8a784de37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 4 Feb 2026 14:09:02 +0100 Subject: [PATCH] kata-deploy: Add infrastructure for per-shim drop-in configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of modifying original config files directly, set up a per-shim directory structure that uses symlinks to the original configs and config.d/ directories for drop-in overrides. This enables cleaner configuration management where the original files remain untouched and all kata-deploy customizations are in separate drop-in files that can be easily inspected and removed. Directory structure: {config_path}/runtimes/{shim}/ {config_path}/runtimes/{shim}/configuration-{shim}.toml -> symlink {config_path}/runtimes/{shim}/config.d/ Signed-off-by: Fabiano Fidêncio --- tools/packaging/kata-deploy/binary/Cargo.lock | 75 +++++++++ tools/packaging/kata-deploy/binary/Cargo.toml | 1 + .../binary/src/artifacts/install.rs | 148 ++++++++++++++--- .../kata-deploy/binary/src/utils/system.rs | 149 ++++++++---------- 4 files changed, 267 insertions(+), 106 deletions(-) diff --git a/tools/packaging/kata-deploy/binary/Cargo.lock b/tools/packaging/kata-deploy/binary/Cargo.lock index 1f66671e83..7983eafa2e 100644 --- a/tools/packaging/kata-deploy/binary/Cargo.lock +++ b/tools/packaging/kata-deploy/binary/Cargo.lock @@ -498,6 +498,7 @@ checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -520,6 +521,17 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.31" @@ -549,6 +561,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + [[package]] name = "futures-util" version = "0.3.31" @@ -600,6 +618,12 @@ dependencies = [ "wasip2", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "gloo-timers" version = "0.3.0" @@ -887,6 +911,7 @@ dependencies = [ "libc", "log", "regex", + "rstest", "serde", "serde_json", "serde_yaml", @@ -1294,6 +1319,12 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "ring" version = "0.17.14" @@ -1308,6 +1339,44 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rstest" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97eeab2f3c0a199bc4be135c36c924b6590b88c377d416494288c14f2db30199" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605" +dependencies = [ + "cfg-if", + "glob", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn", + "unicode-ident", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.1.2" @@ -1461,6 +1530,12 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + [[package]] name = "serde" version = "1.0.228" diff --git a/tools/packaging/kata-deploy/binary/Cargo.toml b/tools/packaging/kata-deploy/binary/Cargo.toml index d0489b9138..fb24aabe7b 100644 --- a/tools/packaging/kata-deploy/binary/Cargo.toml +++ b/tools/packaging/kata-deploy/binary/Cargo.toml @@ -50,3 +50,4 @@ tokio = { version = "1.38", features = ["rt-multi-thread", "macros", "time"] } [dev-dependencies] tempfile = "3.8" +rstest = "0.18" diff --git a/tools/packaging/kata-deploy/binary/src/artifacts/install.rs b/tools/packaging/kata-deploy/binary/src/artifacts/install.rs index 4a9bf09a71..7aa82546fa 100644 --- a/tools/packaging/kata-deploy/binary/src/artifacts/install.rs +++ b/tools/packaging/kata-deploy/binary/src/artifacts/install.rs @@ -123,7 +123,14 @@ pub async fn install_artifacts(config: &Config) -> Result<()> { pub async fn remove_artifacts(config: &Config) -> Result<()> { info!("deleting kata artifacts"); - // Remove custom runtime configs first (before removing main install dir) + // Remove runtime directories for each shim (drop-in configs, symlinks) + for shim in &config.shims_for_arch { + if let Err(e) = remove_runtime_directory(config, shim) { + log::warn!("Failed to remove runtime directory for {}: {}", shim, e); + } + } + + // Remove custom runtime configs (before removing main install dir) if config.custom_runtimes_enabled && !config.custom_runtimes.is_empty() { remove_custom_runtime_configs(config)?; } @@ -152,24 +159,36 @@ fn install_custom_runtime_configs(config: &Config) -> Result<()> { fs::create_dir_all(&config_d_dir) .with_context(|| format!("Failed to create config.d directory: {}", config_d_dir))?; - // Copy base config (already modified by kata-deploy with debug, proxy, annotations) - let base_src = format!( - "/host/{}/share/defaults/kata-containers/configuration-{}.toml", - config.dest_dir, runtime.base_config + // Copy base config to the handler directory + // Custom runtime drop-ins will overlay on top of this + let base_config_filename = format!("configuration-{}.toml", runtime.base_config); + let original_config = format!( + "/host/{}/share/defaults/kata-containers/{}", + config.dest_dir, base_config_filename ); - let base_dest = format!("{}/configuration-{}.toml", handler_dir, runtime.base_config); + let dest_config = format!("{}/{}", handler_dir, base_config_filename); - info!( - "Copying base config for {}: {} -> {}", - runtime.handler, base_src, base_dest - ); + if Path::new(&original_config).exists() { + // Remove existing destination (might be a symlink from older versions) + let dest_path = Path::new(&dest_config); + if dest_path.exists() || dest_path.is_symlink() { + fs::remove_file(&dest_config).with_context(|| { + format!("Failed to remove existing config: {}", dest_config) + })?; + } - fs::copy(&base_src, &base_dest).with_context(|| { - format!( - "Failed to copy base config from {} to {}", - base_src, base_dest - ) - })?; + fs::copy(&original_config, &dest_config).with_context(|| { + format!( + "Failed to copy config: {} -> {}", + original_config, dest_config + ) + })?; + + info!( + "Copied config for custom runtime {}: {} -> {}", + runtime.handler, original_config, dest_config + ); + } // Copy drop-in file if provided if let Some(ref drop_in_src) = runtime.drop_in_file { @@ -288,21 +307,102 @@ fn set_executable_permissions(dir: &str) -> Result<()> { Ok(()) } -async fn configure_shim_config(config: &Config, shim: &str) -> Result<()> { - let config_path = format!( - "/host/{}", +/// Set up the runtime directory structure for a shim. +/// Creates: {config_path}/runtimes/{shim}/ +/// {config_path}/runtimes/{shim}/config.d/ +/// {config_path}/runtimes/{shim}/configuration-{shim}.toml (copy of original) +/// +/// Note: We copy the config file instead of symlinking because kata-containers' +/// ResolvePath uses filepath.EvalSymlinks, which would resolve to the original +/// location and look for config.d there instead of in our per-shim directory. +fn setup_runtime_directory(config: &Config, shim: &str) -> Result<()> { + let original_config_dir = format!( + "/host{}", + utils::get_kata_containers_original_config_path(shim, &config.dest_dir) + ); + let runtime_config_dir = format!( + "/host{}", + utils::get_kata_containers_config_path(shim, &config.dest_dir) + ); + let config_d_dir = format!("{}/config.d", runtime_config_dir); + + // Create the runtime directory and config.d subdirectory + fs::create_dir_all(&config_d_dir) + .with_context(|| format!("Failed to create config.d directory: {}", config_d_dir))?; + + // Copy the original config file to the runtime directory + let original_config_file = format!("{}/configuration-{}.toml", original_config_dir, shim); + let dest_config_file = format!("{}/configuration-{}.toml", runtime_config_dir, shim); + + // Only copy if original exists + if Path::new(&original_config_file).exists() { + // Remove existing destination (might be a symlink from older versions) + // fs::copy follows symlinks and would write to the wrong location + let dest_path = Path::new(&dest_config_file); + if dest_path.exists() || dest_path.is_symlink() { + fs::remove_file(&dest_config_file) + .with_context(|| format!("Failed to remove existing config: {}", dest_config_file))?; + } + + // Copy the base config file + fs::copy(&original_config_file, &dest_config_file) + .with_context(|| format!("Failed to copy config: {} -> {}", original_config_file, dest_config_file))?; + + log::debug!( + "Copied config for {}: {} -> {}", + shim, + original_config_file, + dest_config_file + ); + } + + Ok(()) +} + +/// Remove the runtime directory for a shim during cleanup +fn remove_runtime_directory(config: &Config, shim: &str) -> Result<()> { + let runtime_config_dir = format!( + "/host{}", utils::get_kata_containers_config_path(shim, &config.dest_dir) ); - let kata_config_file = Path::new(&config_path).join(format!("configuration-{shim}.toml")); + if Path::new(&runtime_config_dir).exists() { + fs::remove_dir_all(&runtime_config_dir) + .with_context(|| format!("Failed to remove runtime directory: {}", runtime_config_dir))?; + log::debug!("Removed runtime directory: {}", runtime_config_dir); + } - // The configuration file should exist after copy_artifacts() copied the kata artifacts. - // If it doesn't exist, it means either the copy failed or this shim doesn't have a config - // file in the artifacts (which would be a packaging error). + // Try to clean up parent 'runtimes' directory if empty + let runtimes_dir = Path::new(&runtime_config_dir).parent(); + if let Some(runtimes_path) = runtimes_dir { + if runtimes_path.exists() { + if let Ok(entries) = fs::read_dir(runtimes_path) { + if entries.count() == 0 { + let _ = fs::remove_dir(runtimes_path); + } + } + } + } + + Ok(()) +} + +async fn configure_shim_config(config: &Config, shim: &str) -> Result<()> { + // Set up the runtime directory structure with symlink to original config + setup_runtime_directory(config, shim)?; + + let runtime_config_dir = format!( + "/host{}", + utils::get_kata_containers_config_path(shim, &config.dest_dir) + ); + + let kata_config_file = Path::new(&runtime_config_dir).join(format!("configuration-{shim}.toml")); + + // The configuration file (symlink) should exist after setup_runtime_directory() if !kata_config_file.exists() { return Err(anyhow::anyhow!( "Configuration file not found: {kata_config_file:?}. This file should have been \ - copied from the kata-artifacts. Check that the shim '{}' has a valid configuration \ + symlinked from the original config. Check that the shim '{}' has a valid configuration \ file in the artifacts.", shim )); diff --git a/tools/packaging/kata-deploy/binary/src/utils/system.rs b/tools/packaging/kata-deploy/binary/src/utils/system.rs index 0b5af3e928..e25a741df4 100644 --- a/tools/packaging/kata-deploy/binary/src/utils/system.rs +++ b/tools/packaging/kata-deploy/binary/src/utils/system.rs @@ -48,9 +48,18 @@ pub fn host_systemctl(args: &[&str]) -> Result<()> { Ok(()) } -/// Get kata containers config path based on shim type -pub fn get_kata_containers_config_path(shim: &str, dest_dir: &str) -> String { - let golang_config_path = format!("{dest_dir}/share/defaults/kata-containers"); +/// Get kata containers config path based on shim type. +/// This returns the path where the shim's configuration will be read from. +/// For standard runtimes using drop-in configuration, this is the per-shim directory. +pub fn get_kata_containers_config_path(shim: &str, base_dir: &str) -> String { + let base_path = get_kata_containers_original_config_path(shim, base_dir); + format!("{base_path}/runtimes/{shim}") +} + +/// Get the original kata containers config path (where configs are installed). +/// This is where the original, unmodified configuration files live. +pub fn get_kata_containers_original_config_path(shim: &str, base_dir: &str) -> String { + let golang_config_path = format!("{base_dir}/share/defaults/kata-containers"); let rust_config_path = format!("{golang_config_path}/runtime-rs"); if is_rust_shim(shim) { @@ -61,35 +70,24 @@ pub fn get_kata_containers_config_path(shim: &str, dest_dir: &str) -> String { } /// Get kata containers runtime path based on shim type -pub fn get_kata_containers_runtime_path(shim: &str, dest_dir: &str) -> String { +pub fn get_kata_containers_runtime_path(shim: &str, base_dir: &str) -> String { if is_rust_shim(shim) { - format!("{dest_dir}/runtime-rs/bin/containerd-shim-kata-v2") + format!("{base_dir}/runtime-rs/bin/containerd-shim-kata-v2") } else { - format!("{dest_dir}/bin/containerd-shim-kata-v2") + format!("{base_dir}/bin/containerd-shim-kata-v2") } } #[cfg(test)] mod tests { use super::*; - - /// Helper to test config paths for multiple shims expecting the same result - fn assert_config_paths(shims: &[&str], dest_dir: &str, expected: &str) { - for shim in shims { - assert_eq!( - get_kata_containers_config_path(shim, dest_dir), - expected, - "Config path mismatch for shim '{}'", - shim - ); - } - } + use rstest::rstest; /// Helper to test runtime paths for multiple shims expecting the same result - fn assert_runtime_paths(shims: &[&str], dest_dir: &str, expected: &str) { + fn assert_runtime_paths(shims: &[&str], base_dir: &str, expected: &str) { for shim in shims { assert_eq!( - get_kata_containers_runtime_path(shim, dest_dir), + get_kata_containers_runtime_path(shim, base_dir), expected, "Runtime path mismatch for shim '{}'", shim @@ -97,37 +95,38 @@ mod tests { } } - #[test] - fn test_get_kata_containers_config_path_golang() { - let go_shims = ["qemu", "qemu-tdx", "qemu-snp", "fc"]; - assert_config_paths( - &go_shims, - "/opt/kata", - "/opt/kata/share/defaults/kata-containers", - ); + // Tests for get_kata_containers_original_config_path (where original configs live) + #[rstest] + #[case("qemu", "/opt/kata", "/opt/kata/share/defaults/kata-containers")] + #[case("qemu-tdx", "/opt/kata", "/opt/kata/share/defaults/kata-containers")] + #[case("fc", "/opt/kata", "/opt/kata/share/defaults/kata-containers")] + #[case("clh", "/opt/kata", "/opt/kata/share/defaults/kata-containers")] + #[case("cloud-hypervisor", "/opt/kata", "/opt/kata/share/defaults/kata-containers/runtime-rs")] + #[case("qemu-runtime-rs", "/opt/kata", "/opt/kata/share/defaults/kata-containers/runtime-rs")] + #[case("qemu", "/custom/path", "/custom/path/share/defaults/kata-containers")] + #[case("cloud-hypervisor", "/custom/path", "/custom/path/share/defaults/kata-containers/runtime-rs")] + fn test_get_kata_containers_original_config_path( + #[case] shim: &str, + #[case] base_dir: &str, + #[case] expected: &str, + ) { + assert_eq!(get_kata_containers_original_config_path(shim, base_dir), expected); } - #[test] - fn test_get_kata_containers_config_path_rust() { - assert_config_paths( - RUST_SHIMS, - "/opt/kata", - "/opt/kata/share/defaults/kata-containers/runtime-rs", - ); - } - - #[test] - fn test_get_kata_containers_config_path_custom_dest() { - assert_config_paths( - &["qemu"], - "/usr/local/kata", - "/usr/local/kata/share/defaults/kata-containers", - ); - assert_config_paths( - &["cloud-hypervisor"], - "/usr/local/kata", - "/usr/local/kata/share/defaults/kata-containers/runtime-rs", - ); + // Tests for get_kata_containers_config_path (per-shim runtime directories) + #[rstest] + #[case("qemu", "/opt/kata", "/opt/kata/share/defaults/kata-containers/runtimes/qemu")] + #[case("qemu-tdx", "/opt/kata", "/opt/kata/share/defaults/kata-containers/runtimes/qemu-tdx")] + #[case("fc", "/opt/kata", "/opt/kata/share/defaults/kata-containers/runtimes/fc")] + #[case("cloud-hypervisor", "/opt/kata", "/opt/kata/share/defaults/kata-containers/runtime-rs/runtimes/cloud-hypervisor")] + #[case("qemu-runtime-rs", "/opt/kata", "/opt/kata/share/defaults/kata-containers/runtime-rs/runtimes/qemu-runtime-rs")] + #[case("qemu", "/custom/path", "/custom/path/share/defaults/kata-containers/runtimes/qemu")] + fn test_get_kata_containers_config_path( + #[case] shim: &str, + #[case] base_dir: &str, + #[case] expected: &str, + ) { + assert_eq!(get_kata_containers_config_path(shim, base_dir), expected); } #[test] @@ -197,27 +196,6 @@ mod tests { ); } - #[test] - fn test_config_paths_share_defaults() { - // Test Go runtime config paths use /opt/kata/share/defaults/kata-containers - let go_shims = ["qemu", "qemu-tdx", "fc", "clh"]; - assert_config_paths( - &go_shims, - "/opt/kata", - "/opt/kata/share/defaults/kata-containers", - ); - } - - #[test] - fn test_config_paths_runtime_rs() { - // Test Rust runtime config paths use /opt/kata/share/defaults/kata-containers/runtime-rs - assert_config_paths( - RUST_SHIMS, - "/opt/kata", - "/opt/kata/share/defaults/kata-containers/runtime-rs", - ); - } - #[test] fn test_full_deployment_paths_go_runtime() { // Test complete deployment structure for Go runtime @@ -225,17 +203,19 @@ mod tests { let shim = "qemu-tdx"; let config_path = get_kata_containers_config_path(shim, dest_dir); + let original_path = get_kata_containers_original_config_path(shim, dest_dir); let runtime_path = get_kata_containers_runtime_path(shim, dest_dir); - // Expected paths for Go runtime - assert_eq!(config_path, "/opt/kata/share/defaults/kata-containers"); + // Expected paths for Go runtime with per-shim directory + assert_eq!(config_path, "/opt/kata/share/defaults/kata-containers/runtimes/qemu-tdx"); + assert_eq!(original_path, "/opt/kata/share/defaults/kata-containers"); assert_eq!(runtime_path, "/opt/kata/bin/containerd-shim-kata-v2"); - // Config file would be at + // Config file would be at (symlink to original) let config_file = format!("{}/configuration-{}.toml", config_path, shim); assert_eq!( config_file, - "/opt/kata/share/defaults/kata-containers/configuration-qemu-tdx.toml" + "/opt/kata/share/defaults/kata-containers/runtimes/qemu-tdx/configuration-qemu-tdx.toml" ); } @@ -246,11 +226,16 @@ mod tests { let shim = "cloud-hypervisor"; let config_path = get_kata_containers_config_path(shim, dest_dir); + let original_path = get_kata_containers_original_config_path(shim, dest_dir); let runtime_path = get_kata_containers_runtime_path(shim, dest_dir); - // Expected paths for Rust runtime + // Expected paths for Rust runtime with per-shim directory assert_eq!( config_path, + "/opt/kata/share/defaults/kata-containers/runtime-rs/runtimes/cloud-hypervisor" + ); + assert_eq!( + original_path, "/opt/kata/share/defaults/kata-containers/runtime-rs" ); assert_eq!( @@ -258,17 +243,17 @@ mod tests { "/opt/kata/runtime-rs/bin/containerd-shim-kata-v2" ); - // Config file would be at + // Config file would be at (symlink to original) let config_file = format!("{}/configuration-{}.toml", config_path, shim); assert_eq!( config_file, - "/opt/kata/share/defaults/kata-containers/runtime-rs/configuration-cloud-hypervisor.toml" + "/opt/kata/share/defaults/kata-containers/runtime-rs/runtimes/cloud-hypervisor/configuration-cloud-hypervisor.toml" ); } #[test] fn test_mixed_deployment_both_runtimes() { - // Test that both Go and Rust runtimes can coexist + // Test that both Go and Rust runtimes can coexist with separate directories let dest_dir = "/opt/kata"; // Go runtime @@ -283,12 +268,12 @@ mod tests { assert_ne!(qemu_config, clh_config); assert_ne!(qemu_binary, clh_binary); - // Verify Go runtime paths - assert!(qemu_config.ends_with("kata-containers")); + // Verify Go runtime paths include per-shim directory + assert!(qemu_config.contains("/runtimes/qemu")); assert!(qemu_binary.ends_with("/bin/containerd-shim-kata-v2")); - // Verify Rust runtime paths - assert!(clh_config.ends_with("runtime-rs")); + // Verify Rust runtime paths include per-shim directory + assert!(clh_config.contains("/runtimes/cloud-hypervisor")); assert!(clh_binary.ends_with("/runtime-rs/bin/containerd-shim-kata-v2")); } }