From 9ffdb1219d2665fd9363af0ec13ea66c5db4bfc2 Mon Sep 17 00:00:00 2001 From: Manuel Huber Date: Fri, 12 Jun 2026 21:14:58 +0000 Subject: [PATCH] tests: add runtime config drop-in helpers Add common Kubernetes test helpers for locating the active per-shim Kata runtime config directory and copying/removing TOML fragments under config.d. Update the NVIDIA NUMA test to install its temporary numa_mapping override through those helpers. This gives follow-up tests a shared pattern for temporary runtime config overrides. Signed-off-by: Manuel Huber Assisted-by: OpenAI Codex --- .../kubernetes/k8s-nvidia-numa.bats | 57 ++++------- tests/integration/kubernetes/tests_common.sh | 95 +++++++++++++++++++ 2 files changed, 112 insertions(+), 40 deletions(-) diff --git a/tests/integration/kubernetes/k8s-nvidia-numa.bats b/tests/integration/kubernetes/k8s-nvidia-numa.bats index dd695e6811..909de33d11 100644 --- a/tests/integration/kubernetes/k8s-nvidia-numa.bats +++ b/tests/integration/kubernetes/k8s-nvidia-numa.bats @@ -276,40 +276,17 @@ gpu_numa_skip_reason() { # and merge them into the loaded config on every sandbox start. These # helpers drop in a single override fragment so the main config file is # never edited — teardown just deletes the fragment. -# -# WARNING: must run on the k8s node (sudo required) and patch/restore must -# be paired — a leaked drop-in would silently affect every subsequent pod -# on the same node. - -# kata_runtime_config_dir echoes the per-shim runtime config directory -# (the one that holds configuration-.toml and config.d/). Handles -# both the Go layout (.../runtimes/) and the runtime-rs layout -# (.../runtime-rs/runtimes/) by probing the filesystem rather than -# parsing the shim name (some Rust shims like `dragonball` lack the -# `-runtime-rs` suffix). -kata_runtime_config_dir() { - local base="/opt/kata/share/defaults/kata-containers" - local rs_dir="${base}/runtime-rs/runtimes/${KATA_HYPERVISOR}" - local go_dir="${base}/runtimes/${KATA_HYPERVISOR}" - if [[ -d "${rs_dir}" ]]; then - echo "${rs_dir}" - elif [[ -d "${go_dir}" ]]; then - echo "${go_dir}" - else - die "no Kata runtime config dir for ${KATA_HYPERVISOR} (looked in ${rs_dir} and ${go_dir})" - fi -} # kata_hypervisor_section echoes the [hypervisor.X] header from the active # config so the drop-in fragment targets the right table. Discovering it # at runtime keeps us hypervisor-agnostic (qemu / clh / firecracker / ...). kata_hypervisor_section() { - local dir - dir=$(kata_runtime_config_dir) - local cfg="${dir}/configuration-${KATA_HYPERVISOR}.toml" - [[ -f "${cfg}" ]] || die "Kata config not found at ${cfg}" + local cfg + cfg=$(get_kata_runtime_config_file "${node}") || \ + die "no Kata runtime config file for ${KATA_HYPERVISOR}" + local section - section=$(sudo grep -oE '^\[hypervisor\.[a-z0-9_-]+\]' "${cfg}" | head -1) + section=$(exec_host "${node}" "grep -oE '^\\[hypervisor\\.[a-z0-9_-]+\\]' '${cfg}' | head -1") [[ -n "${section}" ]] || die "no [hypervisor.X] section in ${cfg}" echo "${section}" } @@ -321,28 +298,28 @@ kata_hypervisor_section() { # it. No restart needed — the next sandbox start picks it up. patch_kata_numa_mapping() { local value="${1}" - local dir section - dir=$(kata_runtime_config_dir) + local local_dropin section section=$(kata_hypervisor_section) - KATA_NUMA_DROPIN_PATH="${dir}/config.d/99-numa-test.toml" - export KATA_NUMA_DROPIN_PATH - - sudo mkdir -p "${dir}/config.d" - sudo tee "${KATA_NUMA_DROPIN_PATH}" >/dev/null < "${local_dropin}" <.toml and config.d/. +# Probe the filesystem instead of parsing the shim name, since some runtime-rs +# shims like dragonball do not use the -runtime-rs suffix. +get_kata_runtime_config_dir() { + local node_name="$1" + local base="/opt/kata/share/defaults/kata-containers" + local rs_dir="${base}/runtime-rs/runtimes/${KATA_HYPERVISOR}" + local go_dir="${base}/runtimes/${KATA_HYPERVISOR}" + local legacy_dir="${base}" + + if exec_host "${node_name}" "test -d '${rs_dir}'" >/dev/null 2>&1; then + echo "${rs_dir}" + elif exec_host "${node_name}" "test -d '${go_dir}'" >/dev/null 2>&1; then + echo "${go_dir}" + elif exec_host "${node_name}" "test -f '${legacy_dir}/configuration-${KATA_HYPERVISOR}.toml'" >/dev/null 2>&1; then + echo "${legacy_dir}" + else + return 1 + fi +} + +get_kata_runtime_config_file() { + local node_name="$1" + local config_dir + + config_dir="$(get_kata_runtime_config_dir "${node_name}")" || return 1 + echo "${config_dir}/configuration-${KATA_HYPERVISOR}.toml" +} + +get_kata_runtime_config_dropin_dir() { + local node_name="$1" + local config_dir + + config_dir="$(get_kata_runtime_config_dir "${node_name}")" || return 1 + echo "${config_dir}/config.d" +} + +# Copy a local TOML fragment under the active Kata runtime config.d directory +# on a k8s node. Echoes the full drop-in path. +# +# Callers must pair this with remove_kata_runtime_config_dropin_file during +# teardown. A leaked drop-in would silently affect every subsequent pod on the +# same node. +set_kata_runtime_config_dropin_file() { + local node_name="$1" + local local_dropin="$2" + local dropin_file + local dropin_dir + local dropin_path + local quoted_dropin_dir + + [[ -f "${local_dropin}" ]] || die "Kata runtime config drop-in file does not exist: ${local_dropin}" + dropin_file="$(basename "${local_dropin}")" + + case "${dropin_file}" in + ""|*/*|*[^A-Za-z0-9._-]*) + die "Invalid Kata runtime config drop-in file name: ${dropin_file}" + ;; + esac + case "${dropin_file}" in + *.toml) ;; + *) die "Kata runtime config drop-in file must end in .toml: ${dropin_file}" ;; + esac + + dropin_dir="$(get_kata_runtime_config_dropin_dir "${node_name}")" || return 1 + dropin_path="${dropin_dir}/${dropin_file}" + printf -v quoted_dropin_dir "%q" "${dropin_dir}" + exec_host "${node_name}" "mkdir -p ${quoted_dropin_dir}" || return 1 + copy_file_to_host "${local_dropin}" "${node_name}" "${dropin_path}" || return 1 + echo "${dropin_path}" +} + +# Remove a TOML fragment created under the active Kata runtime config.d +# directory. Empty paths are accepted as a no-op for teardown convenience. +remove_kata_runtime_config_dropin_file() { + local node_name="$1" + local dropin_path="${2:-}" + local dropin_dir + local quoted_dropin_path + + [[ -n "${dropin_path}" ]] || return 0 + + dropin_dir="$(get_kata_runtime_config_dropin_dir "${node_name}")" || return 1 + case "${dropin_path}" in + "${dropin_dir}"/*.toml) ;; + *) die "Refusing to remove path outside Kata runtime config.d: ${dropin_path}" ;; + esac + + printf -v quoted_dropin_path "%q" "${dropin_path}" + exec_host "${node_name}" "rm -f ${quoted_dropin_path}" + echo "# Removed drop-in ${dropin_path}" +} + is_runtime_rs() { [[ "${KATA_HYPERVISOR}" == *-runtime-rs ]] }