From dab981b0bcd6312b6e3eef0077ac00c7dbcb6054 Mon Sep 17 00:00:00 2001 From: Dan Mihai Date: Thu, 13 Mar 2025 23:13:54 +0000 Subject: [PATCH] tests: k8s: retry "kubectl exec" on empty output Retry "kubectl exec" a few times if it unexpectedly produced an empty output string. This is an attempt to work around test failures similar to: https://github.com/kata-containers/kata-containers/actions/runs/13840930994/job/38730153687?pr=10983 not ok 1 Environment variables (from function `grep_pod_exec_output' in file tests_common.sh, line 394, in test file k8s-env.bats, line 36) `grep_pod_exec_output "${pod_name}" "HOST_IP=\([0-9]\+\(\.\|$\)\)\{4\}" "${exec_command[@]}"' failed That test obtained correct ouput from "sh -c printenv" one time, but the second execution of the same command returned an empty output string. Signed-off-by: Dan Mihai --- tests/integration/kubernetes/tests_common.sh | 21 +++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index 911a34a3e..bd65cbdee 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -378,6 +378,9 @@ teardown_common() { # Invoke "kubectl exec", log its output, and check that a grep pattern is present in the output. # +# Retry "kubectl exec" several times in case it unexpectedly returns an empty output string, +# in an attempt to work around issues similar to https://github.com/kubernetes/kubernetes/issues/124571. +# # Parameters: # $1 - pod name # $2 - the grep pattern @@ -388,8 +391,20 @@ grep_pod_exec_output() { shift local -r grep_arg="$1" shift + local grep_out="" + local cmd_out="" - local -r pod_env=$(kubectl exec "${pod_name}" -- "$@") - info "pod_env: ${pod_env}" - echo "${pod_env}" | grep "${grep_arg}" + for _ in {1..10}; do + info "Executing in pod ${pod_name}: $*" + cmd_out=$(kubectl exec "${pod_name}" -- "$@") + if [[ -n "${cmd_out}" ]]; then + info "command output: ${cmd_out}" + grep_out=$(echo "${cmd_out}" | grep "${grep_arg}") + info "grep output: ${grep_out}" + break + fi + warn "Empty output from kubectl exec" + sleep 1 + done + [[ -n "${grep_out}" ]] }