From 65a5f272f85b660eb0d604f3c9c9a0580906e654 Mon Sep 17 00:00:00 2001 From: Cameron Baird Date: Tue, 9 Jun 2026 18:55:28 +0000 Subject: [PATCH] ci: Introduce tests for VM template factory Add k8s-vm-templating-test.bats which exercises pod create with the factory initialized on the target node. Signed-off-by: Cameron Baird --- .../run-k8s-tests-on-free-runner.yaml | 5 + src/runtime/virtcontainers/clh.go | 13 +- src/runtime/virtcontainers/clh_test.go | 6 +- .../virtcontainers/factory/factory_linux.go | 1 - tests/gha-run-k8s-common.sh | 21 +++ .../kubernetes/k8s-vm-templating.bats | 114 +++++++++++++++ .../kubernetes/run_kubernetes_tests.sh | 1 + .../binary/src/artifacts/snapshotters.rs | 92 ++++++++---- .../kata-deploy/binary/src/config.rs | 26 ++++ .../kata-deploy/binary/src/utils/toml.rs | 137 ++++++++++++++++++ .../kata-deploy/templates/_helpers.tpl | 12 ++ .../helm-chart/kata-deploy/values.yaml | 12 ++ 12 files changed, 402 insertions(+), 38 deletions(-) create mode 100644 tests/integration/kubernetes/k8s-vm-templating.bats diff --git a/.github/workflows/run-k8s-tests-on-free-runner.yaml b/.github/workflows/run-k8s-tests-on-free-runner.yaml index d8d9b4c636..87a86ac8b0 100644 --- a/.github/workflows/run-k8s-tests-on-free-runner.yaml +++ b/.github/workflows/run-k8s-tests-on-free-runner.yaml @@ -41,10 +41,12 @@ jobs: matrix: environment: [ { vmm: clh, containerd_version: latest }, + { vmm: clh, containerd_version: latest, snapshotter: erofs, erofs_mode: disk, erofs_merge_mode: unmerged }, { vmm: clh, containerd_version: minimum }, { vmm: dragonball, containerd_version: latest }, { vmm: dragonball, containerd_version: minimum }, { vmm: qemu, containerd_version: latest }, + { vmm: qemu, containerd_version: latest, snapshotter: erofs, erofs_mode: disk, erofs_merge_mode: unmerged }, { vmm: qemu, containerd_version: minimum }, { vmm: qemu-runtime-rs, containerd_version: latest }, { vmm: qemu-runtime-rs, containerd_version: minimum }, @@ -68,6 +70,9 @@ jobs: K8S_TEST_HOST_TYPE: baremetal-no-attestation CONTAINER_ENGINE: containerd CONTAINER_ENGINE_VERSION: ${{ matrix.environment.containerd_version }} + SNAPSHOTTER: ${{ matrix.environment.snapshotter }} + EROFS_SNAPSHOTTER_MODE: ${{ matrix.environment.erofs_mode }} + EROFS_MERGE_MODE: ${{ matrix.environment.erofs_merge_mode }} GH_TOKEN: ${{ github.token }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 28d5ef63d5..171dd2bf22 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -822,7 +822,7 @@ func (clh *cloudHypervisor) shouldRestoreFromTemplate() bool { return true } -// copyFile copies a file from src to dst +// copyFile copies a file from src to dst, preserving the source file's permissions. func (clh *cloudHypervisor) copyFile(src, dst string) error { srcFile, err := os.Open(src) if err != nil { @@ -830,7 +830,12 @@ func (clh *cloudHypervisor) copyFile(src, dst string) error { } defer srcFile.Close() - dstFile, err := os.Create(dst) + srcInfo, err := srcFile.Stat() + if err != nil { + return err + } + + dstFile, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, srcInfo.Mode()) if err != nil { return err } @@ -880,7 +885,7 @@ func (clh *cloudHypervisor) updateVsockSocketPath(configPath, vmID string) error return err } - return os.WriteFile(configPath, updatedConfig, 0644) + return os.WriteFile(configPath, updatedConfig, 0600) } // setupInitdata prepares and attaches the initdata disk if present. @@ -1578,7 +1583,7 @@ func (clh *cloudHypervisor) SaveVM() error { return err } - if err := os.WriteFile(snapshotConfigPath, modifiedConfig, 0644); err != nil { + if err := os.WriteFile(snapshotConfigPath, modifiedConfig, 0600); err != nil { clh.Logger().WithError(err).Error("Failed to write modified snapshot config") return err } diff --git a/src/runtime/virtcontainers/clh_test.go b/src/runtime/virtcontainers/clh_test.go index 93cf3b1a70..d8773a8dfd 100644 --- a/src/runtime/virtcontainers/clh_test.go +++ b/src/runtime/virtcontainers/clh_test.go @@ -457,7 +457,8 @@ func TestCloudHypervisorCleanupVM(t *testing.T) { assert.NoError(err, "persist.GetDriver() unexpected error") dir := filepath.Join(store.RunVMStoragePath(), clh.id) - os.MkdirAll(dir, os.ModePerm) + err = os.MkdirAll(dir, os.ModePerm) + assert.NoError(err, "failed to create dir %s", dir) err = clh.cleanupVM(false) assert.NoError(err, "persist.GetDriver() unexpected error") @@ -566,7 +567,8 @@ func TestClhRestoreVM(t *testing.T) { assert.Contains(err.Error(), filepath.Join(clhConfig.VMStorePath, "state.json")) // Now create the VM snapshot files and call restoreVM again. - os.MkdirAll(clhConfig.VMStorePath, os.ModePerm) + err = os.MkdirAll(clhConfig.VMStorePath, os.ModePerm) + assert.NoError(err, "failed to create dir %s", clhConfig.VMStorePath) stateFile := filepath.Join(clhConfig.VMStorePath, "state.json") configFile := filepath.Join(clhConfig.VMStorePath, "config.json") err = os.WriteFile(stateFile, []byte("{}"), 0o600) diff --git a/src/runtime/virtcontainers/factory/factory_linux.go b/src/runtime/virtcontainers/factory/factory_linux.go index c010916943..e2d28696ae 100644 --- a/src/runtime/virtcontainers/factory/factory_linux.go +++ b/src/runtime/virtcontainers/factory/factory_linux.go @@ -82,7 +82,6 @@ func resetHypervisorConfig(config *vc.VMConfig) { config.HypervisorConfig.RunStorePath = "" config.HypervisorConfig.SandboxName = "" config.HypervisorConfig.SandboxNamespace = "" - config.HypervisorConfig.DefaultMaxVCPUs = 0 } // It's important that baseConfig and newConfig are passed by value! diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index c23f014573..5560e6497e 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -43,6 +43,7 @@ TEST_CLUSTER_NAMESPACE="${TEST_CLUSTER_NAMESPACE:-}" CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-containerd}" SNAPSHOTTER="${SNAPSHOTTER:-}" EROFS_SNAPSHOTTER_MODE="${EROFS_SNAPSHOTTER_MODE:-}" +EROFS_MERGE_MODE="${EROFS_MERGE_MODE:-}" # Wait for the Kubernetes API to recover after kata-deploy uninstall, then # retry the uninstall to purge any stale helm release state. On k3s/rke2, @@ -851,6 +852,26 @@ function helm_helper() { yq -i '.containerd.userDropIn = strenv(HELM_CONTAINERD_USER_DROP_IN)' "${values_yaml}" fi + # EROFS merge mode ("merged" default, or "unmerged"). This is orthogonal + # to EROFS_SNAPSHOTTER_MODE (which controls default_size): it controls + # whether containerd merges layers into a single fsmeta.erofs (merged, + # runtime-rs only) or keeps per-layer layer.erofs (unmerged, required by + # the Go runtime). + if [[ -n "${EROFS_MERGE_MODE}" ]]; then + if [[ "${SNAPSHOTTER}" != "erofs" ]]; then + die "EROFS_MERGE_MODE is only supported with SNAPSHOTTER=erofs" + fi + + case "${EROFS_MERGE_MODE}" in + merged|unmerged) ;; + *) + die "Unsupported EROFS_MERGE_MODE: ${EROFS_MERGE_MODE}" + ;; + esac + + yq -i ".snapshotter.erofsMergeMode = \"${EROFS_MERGE_MODE}\"" "${values_yaml}" + fi + if [[ -z "${HELM_SHIMS}" ]]; then die "A list of shims is expected but none was provided" fi diff --git a/tests/integration/kubernetes/k8s-vm-templating.bats b/tests/integration/kubernetes/k8s-vm-templating.bats new file mode 100644 index 0000000000..c3fb0e01fa --- /dev/null +++ b/tests/integration/kubernetes/k8s-vm-templating.bats @@ -0,0 +1,114 @@ +#!/usr/bin/env bats +# +# Copyright (c) 2024 Kata Containers +# +# SPDX-License-Identifier: Apache-2.0 +# +# Tests for Kata VM templating (factory) functionality in Kubernetes integration mode + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/../../common.bash" +load "${BATS_TEST_DIRNAME}/confidential_common.sh" +load "${BATS_TEST_DIRNAME}/tests_common.sh" + +# Returns 0 if the current environment supports VM templating, non-zero +# otherwise. VM templating is only supported on non-confidential clh/qemu +# hypervisors, and because it uses shared_fs="none" it also requires a +# block-device-based snapshotter (blockfile or erofs). +vm_templating_supported() { + [[ "${KATA_HYPERVISOR}" == "clh" || "${KATA_HYPERVISOR}" == "qemu" ]] || return 1 + is_confidential_runtime_class && return 1 + [[ "${SNAPSHOTTER:-}" =~ ^(blockfile|erofs)$ ]] || return 1 + return 0 +} + +setup() { + if ! vm_templating_supported; then + skip "VM templating requires a non-confidential clh/qemu hypervisor and a blockfile/erofs snapshotter (KATA_HYPERVISOR=${KATA_HYPERVISOR}, SNAPSHOTTER=${SNAPSHOTTER:-unset})" + fi + + setup_common || die "setup_common failed" + + # Build a Kata runtime config drop-in that enables VM templating and + # disables shared_fs (incompatible with templating). + # QEMU VM templating requires an initrd, CLH does not. + local rootfs_override="" + if [[ "${KATA_HYPERVISOR}" == "qemu" ]]; then + rootfs_override=$'image = ""\ninitrd = "/opt/kata/share/kata-containers/kata-containers-initrd.img"' + fi + + local runtime_config_dropin_file="${BATS_TEST_TMPDIR}/99-k8s-vm-templating.toml" + cat > "${runtime_config_dropin_file}" </, whereas a factory-spawned VM + # stores its state under a generated UUID and /run/vc/vm/ is a + # symlink pointing at it (see assignSandbox() in + # src/runtime/virtcontainers/vm.go). Inspect PID 1's mount namespace, where + # the shim creates these entries alongside the template tmpfs. + exec_host "$node" \ + "nsenter --mount=/proc/1/ns/mnt find /run/vc/vm -maxdepth 1 -mindepth 1 -type l ! -name template | grep -q . && echo symlink" \ + | grep -q symlink +} + +teardown() { + vm_templating_supported || return 0 + + rm -f "${pod_config:-}" + + # Destroy the VM template and remove the config drop-in on the target node. + # factory destroy must run in PID 1's mount namespace to unmount the template + # tmpfs that factory init created there (see the @test for details). + exec_host "$node" "nsenter --mount=/proc/1/ns/mnt /opt/kata/bin/kata-runtime --config ${kata_config_path} factory destroy" \ + || echo "Warning: Failed to destroy VM template on node $node" + + remove_kata_runtime_config_dropin_file "$node" "${dropin_path:-}" \ + || echo "Warning: Failed to remove Kata runtime config drop-in on node $node" + + teardown_common "${node:-}" "${node_start_time:-}" +} diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 6e4f64f45b..115c6fb949 100755 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -104,6 +104,7 @@ else "k8s-security-context.bats" \ "k8s-shared-volume.bats" \ "k8s-volume.bats" \ + "k8s-vm-templating.bats" \ "k8s-nginx-connectivity.bats" \ ) diff --git a/tools/packaging/kata-deploy/binary/src/artifacts/snapshotters.rs b/tools/packaging/kata-deploy/binary/src/artifacts/snapshotters.rs index bafec197ab..a2b7599d3e 100644 --- a/tools/packaging/kata-deploy/binary/src/artifacts/snapshotters.rs +++ b/tools/packaging/kata-deploy/binary/src/artifacts/snapshotters.rs @@ -15,36 +15,50 @@ use std::path::Path; pub async fn configure_erofs_snapshotter(config: &Config, configuration_file: &Path) -> Result<()> { info!("Configuring erofs-snapshotter"); + // "unmerged" mode keeps each image layer as its own per-layer `layer.erofs` + // (containerd's default, non-fsmerged layout), which is the only layout the + // Go runtime can consume. In the default "merged" mode we force containerd + // to merge layers into a single `fsmeta.erofs`, which is runtime-rs only. + let unmerged = config.erofs_merge_mode.as_deref() == Some("unmerged"); + // The Go runtime does not support fsmerged EROFS (fsmeta.erofs). // If the snapshotter handler mapping explicitly pairs a Go shim with - // erofs, that is a hard misconfiguration — bail out so the operator - // fixes the mapping instead of hitting cryptic runtime errors later. - if let Some(mapping) = config.snapshotter_handler_mapping_for_arch.as_ref() { - let mut go_shims_on_erofs = Vec::new(); - for entry in mapping.split(',') { - let parts: Vec<&str> = entry.split(':').collect(); - if parts.len() == 2 && parts[1] == "erofs" && !utils::is_rust_shim(parts[0]) { - go_shims_on_erofs.push(parts[0].to_string()); + // erofs in the (default) merged mode, that is a hard misconfiguration — + // bail out so the operator fixes the mapping instead of hitting cryptic + // runtime errors later. In "unmerged" mode the Go runtime is supported, so + // skip this guard. + if !unmerged { + if let Some(mapping) = config.snapshotter_handler_mapping_for_arch.as_ref() { + let mut go_shims_on_erofs = Vec::new(); + for entry in mapping.split(',') { + let parts: Vec<&str> = entry.split(':').collect(); + if parts.len() == 2 && parts[1] == "erofs" && !utils::is_rust_shim(parts[0]) { + go_shims_on_erofs.push(parts[0].to_string()); + } } - } - if !go_shims_on_erofs.is_empty() { - warn!("##########################################################################"); - warn!("# #"); - warn!("# Go runtime shim(s) mapped to the erofs snapshotter: #"); - for s in &go_shims_on_erofs { - warn!("# - {:<64} #", s); + if !go_shims_on_erofs.is_empty() { + warn!("##########################################################################"); + warn!("# #"); + warn!("# Go runtime shim(s) mapped to the erofs snapshotter: #"); + for s in &go_shims_on_erofs { + warn!("# - {:<64} #", s); + } + warn!("# #"); + warn!( + "# The Go runtime does NOT support fsmerged EROFS (fsmeta.erofs). #" + ); + warn!("# Only runtime-rs shims are supported with merged erofs. Set #"); + warn!("# EROFS_MERGE_MODE=unmerged to use the Go runtime with erofs. #"); + warn!("# #"); + warn!("##########################################################################"); + return Err(anyhow::anyhow!( + "erofs snapshotter: Go runtime shim(s) [{}] cannot be mapped to merged erofs. \ + The Go runtime does not support fsmerged EROFS. \ + Set EROFS_MERGE_MODE=unmerged, remove these shims from \ + SNAPSHOTTER_HANDLER_MAPPING, or switch them to runtime-rs.", + go_shims_on_erofs.join(", ") + )); } - warn!("# #"); - warn!("# The Go runtime does NOT support fsmerged EROFS (fsmeta.erofs). #"); - warn!("# Only runtime-rs shims are supported with the erofs snapshotter. #"); - warn!("# #"); - warn!("##########################################################################"); - return Err(anyhow::anyhow!( - "erofs snapshotter: Go runtime shim(s) [{}] cannot be mapped to erofs. \ - The Go runtime does not support fsmerged EROFS. \ - Remove these shims from SNAPSHOTTER_HANDLER_MAPPING or switch them to runtime-rs.", - go_shims_on_erofs.join(", ") - )); } } @@ -88,11 +102,27 @@ pub async fn configure_erofs_snapshotter(config: &Config, configuration_file: &P ".plugins.\"io.containerd.snapshotter.v1.erofs\".default_size", "\"10G\"", )?; - toml_utils::set_toml_value( - configuration_file, - ".plugins.\"io.containerd.snapshotter.v1.erofs\".max_unmerged_layers", - "0", - )?; + // In the default "merged" mode, force containerd to merge all layers into a + // single fsmeta.erofs (max_unmerged_layers = 0). In "unmerged" mode we delete + // any previously-written value so each layer stays a separate layer.erofs, + // which the Go runtime requires. + // + // Because kata-deploy edits the containerd config in place, switching from + // merged to unmerged must actively remove the old `max_unmerged_layers = 0` + // left behind by a previous install. Otherwise the stale `0` would keep + // forcing the merged layout and break Go-runtime compatibility. + if !unmerged { + toml_utils::set_toml_value( + configuration_file, + ".plugins.\"io.containerd.snapshotter.v1.erofs\".max_unmerged_layers", + "0", + )?; + } else { + toml_utils::delete_toml_value( + configuration_file, + ".plugins.\"io.containerd.snapshotter.v1.erofs\".max_unmerged_layers", + )?; + } Ok(()) } diff --git a/tools/packaging/kata-deploy/binary/src/config.rs b/tools/packaging/kata-deploy/binary/src/config.rs index c3c7f68e8c..4700940740 100644 --- a/tools/packaging/kata-deploy/binary/src/config.rs +++ b/tools/packaging/kata-deploy/binary/src/config.rs @@ -178,6 +178,14 @@ pub struct Config { pub multi_install_suffix: Option, pub helm_post_delete_hook: bool, pub experimental_setup_snapshotter: Option>, + /// EROFS snapshotter merge mode: "merged" (default) or "unmerged". + /// + /// In "unmerged" mode kata-deploy does not force containerd's erofs + /// snapshotter to merge layers (it leaves `max_unmerged_layers` at the + /// containerd default), so each image layer is exposed as its own + /// per-layer `layer.erofs`. This is the only layout the Go runtime can + /// consume; the merged (`fsmeta.erofs`) layout is runtime-rs only. + pub erofs_merge_mode: Option, pub experimental_force_guest_pull_for_arch: Vec, pub dest_dir: String, pub host_install_dir: String, @@ -307,6 +315,11 @@ impl Config { .filter(|s| !s.is_empty()) .map(|s| s.split(',').map(|s| s.trim().to_string()).collect()); + let erofs_merge_mode = env::var("EROFS_MERGE_MODE") + .ok() + .map(|s| s.trim().to_lowercase()) + .filter(|s| !s.is_empty()); + // Only use arch-specific variable for experimental force guest pull let experimental_force_guest_pull_for_arch = get_arch_var("EXPERIMENTAL_FORCE_GUEST_PULL", "", &arch) @@ -338,6 +351,7 @@ impl Config { multi_install_suffix, helm_post_delete_hook, experimental_setup_snapshotter, + erofs_merge_mode, experimental_force_guest_pull_for_arch, dest_dir, host_install_dir, @@ -508,6 +522,17 @@ impl Config { _ => {} } + // Validate EROFS_MERGE_MODE + // Only "merged" (default) and "unmerged" are accepted. + if let Some(mode) = self.erofs_merge_mode.as_ref() { + if mode != "merged" && mode != "unmerged" { + return Err(anyhow::anyhow!( + "EROFS_MERGE_MODE must be either 'merged' or 'unmerged', got '{}'", + mode + )); + } + } + // Validate EXPERIMENTAL_FORCE_GUEST_PULL_FOR_ARCH // This is a list of shim names for shim in &self.experimental_force_guest_pull_for_arch { @@ -551,6 +576,7 @@ impl Config { "* EXPERIMENTAL_SETUP_SNAPSHOTTER: {:?}", self.experimental_setup_snapshotter ); + info!("* EROFS_MERGE_MODE: {:?}", self.erofs_merge_mode); info!( "* EXPERIMENTAL_FORCE_GUEST_PULL: {}", self.experimental_force_guest_pull_for_arch.join(",") diff --git a/tools/packaging/kata-deploy/binary/src/utils/toml.rs b/tools/packaging/kata-deploy/binary/src/utils/toml.rs index 080f678875..58477ffa46 100644 --- a/tools/packaging/kata-deploy/binary/src/utils/toml.rs +++ b/tools/packaging/kata-deploy/binary/src/utils/toml.rs @@ -121,6 +121,47 @@ pub fn set_toml_value(file_path: &Path, path: &str, value: &str) -> Result<()> { Ok(()) } +/// Delete a TOML value (or table) at a given path. +/// +/// Navigates to the parent table and removes the final key. This is a no-op if +/// any path component (including the final key) does not exist, so callers can +/// unconditionally remove a value that may or may not be present. +pub fn delete_toml_value(file_path: &Path, path: &str) -> Result<()> { + let content = std::fs::read_to_string(file_path) + .with_context(|| format!("Failed to read TOML file: {file_path:?}"))?; + + let (header, toml_content) = split_non_toml_header(&content); + let mut doc = toml_content + .parse::() + .context("Failed to parse TOML")?; + + let parts = parse_toml_path(path)?; + + let mut current_table = doc.as_table_mut(); + for (i, part) in parts.iter().enumerate() { + let is_last = i == parts.len() - 1; + + if is_last { + // Remove the value; absent key is fine (no-op). + current_table.remove(part.as_str()); + } else { + // Navigate into the intermediate table. If it does not exist, there + // is nothing to delete. + match current_table + .get_mut(part.as_str()) + .and_then(|item| item.as_table_mut()) + { + Some(table) => current_table = table, + None => return Ok(()), + } + } + } + + write_toml_with_header(file_path, header, &doc)?; + + Ok(()) +} + /// Get a TOML value at a given path pub fn get_toml_value(file_path: &Path, path: &str) -> Result { let content = std::fs::read_to_string(file_path) @@ -1714,4 +1755,100 @@ imports = ["/etc/containerd/conf.d/*.toml", "/opt/kata/containerd/config.d/kata- .unwrap(); assert_eq!(runtime_type, "io.containerd.kata-qemu.v2"); } + + #[test] + fn test_delete_toml_value() { + let temp_file = NamedTempFile::new().unwrap(); + let temp_path = temp_file.path(); + std::fs::write( + temp_path, + "[plugins.\"io.containerd.snapshotter.v1.erofs\"]\nmax_unmerged_layers = 0\nenable_fsverity = true\n", + ) + .unwrap(); + + // Sanity check: value is present before deletion. + let before = get_toml_value( + temp_path, + ".plugins.\"io.containerd.snapshotter.v1.erofs\".max_unmerged_layers", + ) + .unwrap(); + assert_eq!(before, "0"); + + delete_toml_value( + temp_path, + ".plugins.\"io.containerd.snapshotter.v1.erofs\".max_unmerged_layers", + ) + .unwrap(); + + // The deleted key is gone, but sibling keys remain. + let result = get_toml_value( + temp_path, + ".plugins.\"io.containerd.snapshotter.v1.erofs\".max_unmerged_layers", + ); + assert!(result.is_err(), "deleted key should no longer be found"); + + let sibling = get_toml_value( + temp_path, + ".plugins.\"io.containerd.snapshotter.v1.erofs\".enable_fsverity", + ) + .unwrap(); + assert_eq!(sibling, "true", "sibling keys must be preserved"); + } + + #[test] + fn test_delete_toml_value_missing_key_is_noop() { + let temp_file = NamedTempFile::new().unwrap(); + let temp_path = temp_file.path(); + let initial = "[plugins.\"io.containerd.snapshotter.v1.erofs\"]\nenable_fsverity = true\n"; + std::fs::write(temp_path, initial).unwrap(); + + // Deleting a key that does not exist must succeed and leave the file usable. + delete_toml_value( + temp_path, + ".plugins.\"io.containerd.snapshotter.v1.erofs\".max_unmerged_layers", + ) + .unwrap(); + + // Deleting through a non-existent intermediate table is also a no-op. + delete_toml_value(temp_path, ".plugins.\"nonexistent.plugin\".some_key").unwrap(); + let sibling = get_toml_value( + temp_path, + ".plugins.\"io.containerd.snapshotter.v1.erofs\".enable_fsverity", + ) + .unwrap(); + assert_eq!(sibling, "true"); + } + + #[test] + fn test_delete_toml_value_preserves_k3s_header() { + let temp_file = NamedTempFile::new().unwrap(); + let temp_path = temp_file.path(); + std::fs::write( + temp_path, + "{{ template \"base\" . }}\n[plugins.\"io.containerd.snapshotter.v1.erofs\"]\nmax_unmerged_layers = 0\n", + ) + .unwrap(); + + delete_toml_value( + temp_path, + ".plugins.\"io.containerd.snapshotter.v1.erofs\".max_unmerged_layers", + ) + .unwrap(); + + let content = std::fs::read_to_string(temp_path).unwrap(); + assert!( + content.starts_with("{{ template \"base\" . }}\n"), + "non-TOML header must be preserved" + ); + assert!( + !content.contains("max_unmerged_layers"), + "value must be removed" + ); + } + + #[test] + fn test_delete_toml_value_nonexistent_file() { + let result = delete_toml_value(Path::new("/nonexistent/file.toml"), "some.path"); + assert!(result.is_err()); + } } diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl index 457cb00ab6..24f309ae3c 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl @@ -413,6 +413,13 @@ Get snapshotter setup list from structured config {{- join "," .Values.snapshotter.setup -}} {{- end -}} +{{/* +Get EROFS merge mode from structured config ("merged" or "unmerged") +*/}} +{{- define "kata-deploy.getErofsMergeMode" -}} +{{- .Values.snapshotter.erofsMergeMode | default "" -}} +{{- end -}} + {{/* Get debug value from structured config */}} @@ -569,6 +576,11 @@ e.g. `{{- include "kata-deploy.commonEnv" . | nindent 8 }}`. - name: EXPERIMENTAL_SETUP_SNAPSHOTTER value: {{ $snapshotterSetup | quote }} {{- end }} +{{- $erofsMergeMode := include "kata-deploy.getErofsMergeMode" . | trim -}} +{{- if $erofsMergeMode }} +- name: EROFS_MERGE_MODE + value: {{ $erofsMergeMode | quote }} +{{- end }} {{- $forceGuestPullAmd64 := include "kata-deploy.getForceGuestPullForArch" (dict "root" . "arch" "amd64") | trim -}} {{- if $forceGuestPullAmd64 }} - name: EXPERIMENTAL_FORCE_GUEST_PULL_X86_64 diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml index 120bfd5027..7832ddb995 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml @@ -271,6 +271,18 @@ health: snapshotter: setup: ["nydus"] # ["nydus", "erofs"] or [] + # EROFS merge mode: "merged" (default) or "unmerged". + # + # "merged" forces containerd's erofs snapshotter to merge all image layers + # into a single fsmeta.erofs (max_unmerged_layers = 0). This layout is only + # supported by runtime-rs shims. + # + # "unmerged" leaves max_unmerged_layers at the containerd default so each + # image layer is exposed as its own per-layer layer.erofs. This is the only + # layout the Go runtime can consume, so set this when mapping a Go shim to the + # erofs snapshotter. When empty, kata-deploy uses its built-in default + # (merged). + erofsMergeMode: "" # Shim configuration # By default (disableAll: false), all shims with enabled: ~ (null) are enabled.