From fc662daf0f70e65571a1b06ae1bb15d19c7ed243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 22 Jul 2026 19:47:31 +0200 Subject: [PATCH] tests/k8s: exercise the devkit debug console MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add k8s-devkit-debug-console.bats, which launches a pod with the kata--devkit RuntimeClass, resolves its sandbox id from the node, and reaches the agent debug console via `kata-ctl exec`. The console is an interactive PTY, so it is driven with `script` (util-linux) fed through a FIFO, and it asserts we land in the devkit shell (a working apk, plus an Alpine guest id) rather than the base guest shell. The test targets the two non-confidential runtime-rs classes it is validated on (qemu-runtime-rs, qemu-nvidia-cpu-runtime-rs) and skips elsewhere or when the kata--devkit RuntimeClass is absent (devkit not deployed). Register it in the small-host suite, add a HELM_DEVKIT knob to the Helm test helper, and turn it on for those two hypervisors in the k8s CI (which already runs with debug). Signed-off-by: Fabiano Fidêncio Assisted-by: Cursor --- tests/gha-run-k8s-common.sh | 11 ++ tests/integration/kubernetes/gha-run.sh | 13 ++ .../kubernetes/k8s-devkit-debug-console.bats | 133 ++++++++++++++++++ .../kubernetes/run_kubernetes_tests.sh | 1 + 4 files changed, 158 insertions(+) create mode 100644 tests/integration/kubernetes/k8s-devkit-debug-console.bats diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 9b9baac76c..695f2ea4b1 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -25,6 +25,7 @@ HELM_ALLOWED_HYPERVISOR_ANNOTATIONS="${HELM_ALLOWED_HYPERVISOR_ANNOTATIONS:-}" HELM_CREATE_RUNTIME_CLASSES="${HELM_CREATE_RUNTIME_CLASSES:-}" HELM_CREATE_DEFAULT_RUNTIME_CLASS="${HELM_CREATE_DEFAULT_RUNTIME_CLASS:-}" HELM_DEBUG="${HELM_DEBUG:-}" +HELM_DEVKIT="${HELM_DEVKIT:-}" HELM_DEFAULT_SHIM="${HELM_DEFAULT_SHIM:-}" HELM_IMAGE_REFERENCE="${HELM_IMAGE_REFERENCE:-}" HELM_IMAGE_TAG="${HELM_IMAGE_TAG:-}" @@ -766,6 +767,16 @@ function helm_helper() { fi fi + # Deploy the devkit debug extension + per-shim kata--devkit + # RuntimeClasses (only effective together with debug). + if [[ -n "${HELM_DEVKIT}" ]]; then + if [[ "${HELM_DEVKIT}" == "true" ]]; then + yq -i ".devkit = true" "${values_yaml}" + else + yq -i ".devkit = false" "${values_yaml}" + fi + fi + # Configure shims using new structured format if [[ -n "${HELM_SHIMS}" ]]; then # HELM_SHIMS is a space-separated list of shim names diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 607c0511ce..c384d772f1 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -237,6 +237,19 @@ function deploy_kata() { export HELM_IMAGE_REFERENCE="${DOCKER_REGISTRY}/${DOCKER_REPO}" export HELM_IMAGE_TAG="${DOCKER_TAG}" export HELM_DEBUG="true" + # Deploy the devkit debug extension for the (non-confidential) runtime-rs + # class that k8s-devkit-debug-console.bats exercises. Restricted to + # x86_64/aarch64: the devkit image and kata-ctl (its debug-console client) + # are not shipped on s390x/ppc64le, so devkit is neither built nor testable + # there. + export HELM_DEVKIT="false" + case "${TARGET_ARCH}" in + x86_64 | aarch64) + case "${KATA_HYPERVISOR}" in + qemu-nvidia-cpu-runtime-rs) export HELM_DEVKIT="true" ;; + esac + ;; + esac export HELM_SHIMS="${KATA_HYPERVISOR}" export HELM_DEFAULT_SHIM="${KATA_HYPERVISOR}" export HELM_CREATE_DEFAULT_RUNTIME_CLASS="true" diff --git a/tests/integration/kubernetes/k8s-devkit-debug-console.bats b/tests/integration/kubernetes/k8s-devkit-debug-console.bats new file mode 100644 index 0000000000..29d162e9a8 --- /dev/null +++ b/tests/integration/kubernetes/k8s-devkit-debug-console.bats @@ -0,0 +1,133 @@ +#!/usr/bin/env bats +# +# Copyright (c) Kata Containers Community +# +# SPDX-License-Identifier: Apache-2.0 +# +# Exercises the devkit debug guest extension through the agent debug console +# (kata-ctl exec ) on the NVIDIA CPU runtime-rs class: with the +# kata--devkit RuntimeClass, the console drops into the rich Ubuntu-based +# devkit shell overlaid on the read-only guest rootfs. + +load "${BATS_TEST_DIRNAME}/../../common.bash" +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/tests_common.sh" + +# The devkit debug console is a non-confidential debugging aid. For now it is +# validated only on the (non-confidential) NVIDIA CPU runtime-rs class; other +# hypervisors don't ship it in CI. +devkit_supported() { + case "${KATA_HYPERVISOR}" in + qemu-nvidia-cpu-runtime-rs) return 0 ;; + *) return 1 ;; + esac +} + +devkit_runtimeclass() { + echo "kata-${KATA_HYPERVISOR}-devkit" +} + +# A probe the *guest* shell must evaluate: $((6*7)) only becomes "SHELL_OK=42" +# when a real shell runs it. The literal command text, even if echoed back by +# the PTY, never contains the expanded value - so "SHELL_OK=42" in the output is +# unambiguous proof that a debug console shell actually executed. The /real_root +# symlink is created only by devkit-init when the overlay is set up, so it tells +# the devkit chroot apart from the (also Ubuntu-based) NVIDIA base rootfs. +DEVKIT_PROBE='echo "SHELL_OK=$((6*7))"; . /etc/os-release 2>/dev/null; echo "GUEST_ID=${ID}"; test -L /real_root && echo "DEVKIT_OVERLAY=yes" || true' + +check_and_skip() { + if ! devkit_supported; then + skip "devkit debug console not exercised for hypervisor: ${KATA_HYPERVISOR}" + fi + + # The debug console client used here is kata-ctl, which ships only on x86_64 + # and aarch64. s390x and ppc64le cannot build it statically (it needs glibc), + # so it is not installed there and this test cannot run. + case "$(uname -m)" in + x86_64 | aarch64) ;; + *) skip "kata-ctl not shipped for $(uname -m); devkit debug console not exercised" ;; + esac + + # The kata--devkit RuntimeClass only exists when kata-deploy was + # installed with both debug and devkit enabled. Skip (rather than fail) + # where the extension was not deployed. + if ! kubectl get runtimeclass "$(devkit_runtimeclass)" >/dev/null 2>&1; then + skip "RuntimeClass $(devkit_runtimeclass) not found; devkit not deployed" + fi +} + +# Resolves the sandbox id into the global sandbox_id. +launch_pod() { + local runtimeclass="$1" + + pod_config="$(new_pod_config "${nginx_image}" "${runtimeclass}")" + set_node "${pod_config}" "${node}" + yq -i ".metadata.name = \"${pod_name}\"" "${pod_config}" + + echo "Pod ${pod_config} (runtimeClass=${runtimeclass}):" + cat "${pod_config}" + + kubectl create -f "${pod_config}" + kubectl wait --for=condition=Ready --timeout="${timeout}" "pod/${pod_name}" + + sandbox_id="$(get_node_kata_sandbox_id "${node}")" + [[ -n "${sandbox_id}" ]] || die "Failed to resolve kata sandbox id on node ${node}" + echo "sandbox id: ${sandbox_id}" +} + +# Drive the interactive agent debug console for sandbox_id with DEVKIT_PROBE and +# echo the combined output. +# +# The console is an interactive PTY, so a bare pipe races the guest login shell +# startup and loses the input. Drive it with a real terminal via `script` +# (util-linux), feeding commands through a FIFO whose writer stays open long +# enough for the shell to be ready before input arrives and to flush output +# before we send `exit`. +run_debug_console() { + local sandbox_id="$1" + local remote=" +fifo=\$(mktemp -u); mkfifo \"\${fifo}\" +( sleep 2; printf '%s\\n' '${DEVKIT_PROBE}'; sleep 3; printf 'exit\\n'; sleep 1 ) > \"\${fifo}\" & +timeout 120 script -qec \"nsenter --mount=/proc/1/ns/mnt /opt/kata/bin/kata-ctl exec ${sandbox_id}\" /dev/null < \"\${fifo}\" 2>&1 +rm -f \"\${fifo}\" +" + exec_host "${node}" "${remote}" || true +} + +setup() { + check_and_skip + + setup_common || die "setup_common failed" + + ensure_yq + nginx_registry=$(get_from_kata_deps ".docker_images.nginx.registry") + nginx_digest=$(get_from_kata_deps ".docker_images.nginx.digest") + nginx_image="${nginx_registry}@${nginx_digest}" + + pod_name="devkit-debug-console" +} + +@test "Debug console drops into the devkit shell" { + launch_pod "$(devkit_runtimeclass)" + + local output + output="$(run_debug_console "${sandbox_id}")" + echo "debug console output:" + echo "${output}" + + echo "${output}" | grep -q 'SHELL_OK=42' \ + || die "devkit debug console did not provide a working shell" + echo "${output}" | grep -q 'GUEST_ID=ubuntu' \ + || die "devkit debug console did not report an Ubuntu guest" + echo "${output}" | grep -q 'DEVKIT_OVERLAY=yes' \ + || die "devkit debug console shell lacks /real_root; not the devkit overlay" +} + +teardown() { + check_and_skip + + kubectl describe "pod/${pod_name}" || true + kubectl delete pod "${pod_name}" --ignore-not-found || true + + teardown_common "${node:-}" "${node_start_time:-}" +} diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index e84ba426ce..47dd783c59 100755 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -73,6 +73,7 @@ else "k8s-credentials-secrets.bats" \ "k8s-cron-job.bats" \ "k8s-custom-dns.bats" \ + "k8s-devkit-debug-console.bats" \ "k8s-empty-dirs.bats" \ "k8s-env.bats" \ "k8s-erofs-dmverity.bats" \