From dab981b0bcd6312b6e3eef0077ac00c7dbcb6054 Mon Sep 17 00:00:00 2001
From: Dan Mihai <dmihai@microsoft.com>
Date: Thu, 13 Mar 2025 23:13:54 +0000
Subject: [PATCH] tests: k8s: retry "kubectl exec" on empty output
Retry "kubectl exec" a few times if it unexpectedly produced an empty
output string.
This is an attempt to work around test failures similar to:
https://github.com/kata-containers/kata-containers/actions/runs/13840930994/job/38730153687?pr=10983
not ok 1 Environment variables
(from function `grep_pod_exec_output' in file tests_common.sh, line 394,
in test file k8s-env.bats, line 36)
`grep_pod_exec_output "${pod_name}" "HOST_IP=\([0-9]\+\(\.\|$\)\)\{4\}" "${exec_command[@]}"' failed
That test obtained correct ouput from "sh -c printenv" one time, but the
second execution of the same command returned an empty output string.
Signed-off-by: Dan Mihai <dmihai@microsoft.com>
---
tests/integration/kubernetes/tests_common.sh | 21 +++++++++++++++++---
1 file changed, 18 insertions(+), 3 deletions(-)
diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh
index 911a34a3e2..bd65cbdee8 100644
--- a/tests/integration/kubernetes/tests_common.sh
+++ b/tests/integration/kubernetes/tests_common.sh
@@ -378,6 +378,9 @@ teardown_common() {
# Invoke "kubectl exec", log its output, and check that a grep pattern is present in the output.
#
+# Retry "kubectl exec" several times in case it unexpectedly returns an empty output string,
+# in an attempt to work around issues similar to https://github.com/kubernetes/kubernetes/issues/124571.
+#
# Parameters:
# $1 - pod name
# $2 - the grep pattern
@@ -388,8 +391,20 @@ grep_pod_exec_output() {
shift
local -r grep_arg="$1"
shift
+ local grep_out=""
+ local cmd_out=""
- local -r pod_env=$(kubectl exec "${pod_name}" -- "$@")
- info "pod_env: ${pod_env}"
- echo "${pod_env}" | grep "${grep_arg}"
+ for _ in {1..10}; do
+ info "Executing in pod ${pod_name}: $*"
+ cmd_out=$(kubectl exec "${pod_name}" -- "$@")
+ if [[ -n "${cmd_out}" ]]; then
+ info "command output: ${cmd_out}"
+ grep_out=$(echo "${cmd_out}" | grep "${grep_arg}")
+ info "grep output: ${grep_out}"
+ break
+ fi
+ warn "Empty output from kubectl exec"
+ sleep 1
+ done
+ [[ -n "${grep_out}" ]]
}