Merge pull request #10735 from BbolroC/kubectl-create-retry-trusted-storage

tests: Introduce retry_kubectl_apply() for trusted storage
This commit is contained in:
Zvonko Kaiser
2025-01-14 21:59:45 -05:00
committed by GitHub
2 changed files with 46 additions and 5 deletions

View File

@@ -101,7 +101,7 @@ setup() {
cat $storage_config
# Create persistent volume and persistent volume claim
kubectl create -f $storage_config
retry_kubectl_apply $storage_config
pod_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${pod_config_template}").XXX")
IMAGE="$image_pulled_time_less_than_default_time" NODE_NAME="$node" envsubst < "$pod_config_template" > "$pod_config"
@@ -146,7 +146,7 @@ setup() {
cat $storage_config
# Create persistent volume and persistent volume claim
kubectl create -f $storage_config
retry_kubectl_apply $storage_config
pod_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${pod_config_template}").XXX")
IMAGE="$large_image" NODE_NAME="$node" envsubst < "$pod_config_template" > "$pod_config"
@@ -191,7 +191,7 @@ setup() {
cat $storage_config
# Create persistent volume and persistent volume claim
kubectl create -f $storage_config
retry_kubectl_apply $storage_config
pod_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${pod_config_template}").XXX")
IMAGE="$large_image" NODE_NAME="$node" envsubst < "$pod_config_template" > "$pod_config"

View File

@@ -33,6 +33,47 @@ k8s_wait_pod_be_ready() {
kubectl wait --timeout="${wait_time}s" --for=condition=ready "pods/$pod_name"
}
# Create a pod with a given number of retries if an output includes a timeout.
#
# Parameters:
# $1 - the pod configuration file.
#
retry_kubectl_apply() {
local file_path=$1
local retries=5
local delay=5
local attempt=1
local func_name="${FUNCNAME[0]}"
while true; do
output=$(kubectl apply -f "$file_path" 2>&1) || true
echo ""
echo "$func_name: Attempt $attempt/$retries"
echo "$output"
# Check for timeout and retry if needed
if echo "$output" | grep -iq "timed out"; then
if [ $attempt -ge $retries ]; then
echo "$func_name: Max ${retries} retries reached. Failed due to timeout."
return 1
fi
echo "$func_name: Timeout encountered, retrying in $delay seconds..."
sleep $delay
attempt=$((attempt + 1))
continue
fi
# Check for any other kind of error
if echo "$output" | grep -iq "error"; then
echo "$func_name: Error detected in kubectl output. Aborting."
return 1
fi
echo "$func_name: Resource created successfully."
return 0
done
}
# Create a pod and wait it be ready, otherwise fail.
#
# Parameters:
@@ -49,7 +90,7 @@ k8s_create_pod() {
return 1
fi
kubectl apply -f "${config_file}"
retry_kubectl_apply "${config_file}"
if ! pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}'); then
echo "Failed to create the pod"
return 1
@@ -143,7 +184,7 @@ assert_pod_fail() {
echo "In assert_pod_fail: $container_config"
echo "Attempt to create the container but it should fail"
kubectl apply -f "${container_config}"
retry_kubectl_apply "${container_config}"
if ! pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}'); then
echo "Failed to create the pod"
return 1