diff --git a/tests/integration/kubernetes/k8s-guest-pull-image.bats b/tests/integration/kubernetes/k8s-guest-pull-image.bats index bae5499109..dff148785e 100644 --- a/tests/integration/kubernetes/k8s-guest-pull-image.bats +++ b/tests/integration/kubernetes/k8s-guest-pull-image.bats @@ -101,7 +101,7 @@ setup() { cat $storage_config # Create persistent volume and persistent volume claim - kubectl create -f $storage_config + retry_kubectl_apply $storage_config pod_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${pod_config_template}").XXX") IMAGE="$image_pulled_time_less_than_default_time" NODE_NAME="$node" envsubst < "$pod_config_template" > "$pod_config" @@ -146,7 +146,7 @@ setup() { cat $storage_config # Create persistent volume and persistent volume claim - kubectl create -f $storage_config + retry_kubectl_apply $storage_config pod_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${pod_config_template}").XXX") IMAGE="$large_image" NODE_NAME="$node" envsubst < "$pod_config_template" > "$pod_config" @@ -191,7 +191,7 @@ setup() { cat $storage_config # Create persistent volume and persistent volume claim - kubectl create -f $storage_config + retry_kubectl_apply $storage_config pod_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${pod_config_template}").XXX") IMAGE="$large_image" NODE_NAME="$node" envsubst < "$pod_config_template" > "$pod_config" diff --git a/tests/integration/kubernetes/lib.sh b/tests/integration/kubernetes/lib.sh index 9c2297f0f5..bfc4c3c3d9 100644 --- a/tests/integration/kubernetes/lib.sh +++ b/tests/integration/kubernetes/lib.sh @@ -33,6 +33,47 @@ k8s_wait_pod_be_ready() { kubectl wait --timeout="${wait_time}s" --for=condition=ready "pods/$pod_name" } +# Create a pod with a given number of retries if an output includes a timeout. +# +# Parameters: +# $1 - the pod configuration file. +# +retry_kubectl_apply() { + local file_path=$1 + local retries=5 + local delay=5 + local attempt=1 + local func_name="${FUNCNAME[0]}" + + while true; do + output=$(kubectl apply -f "$file_path" 2>&1) || true + echo "" + echo "$func_name: Attempt $attempt/$retries" + echo "$output" + + # Check for timeout and retry if needed + if echo "$output" | grep -iq "timed out"; then + if [ $attempt -ge $retries ]; then + echo "$func_name: Max ${retries} retries reached. Failed due to timeout." + return 1 + fi + echo "$func_name: Timeout encountered, retrying in $delay seconds..." + sleep $delay + attempt=$((attempt + 1)) + continue + fi + + # Check for any other kind of error + if echo "$output" | grep -iq "error"; then + echo "$func_name: Error detected in kubectl output. Aborting." + return 1 + fi + + echo "$func_name: Resource created successfully." + return 0 + done +} + # Create a pod and wait it be ready, otherwise fail. # # Parameters: @@ -49,7 +90,7 @@ k8s_create_pod() { return 1 fi - kubectl apply -f "${config_file}" + retry_kubectl_apply "${config_file}" if ! pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}'); then echo "Failed to create the pod" return 1 @@ -143,7 +184,7 @@ assert_pod_fail() { echo "In assert_pod_fail: $container_config" echo "Attempt to create the container but it should fail" - kubectl apply -f "${container_config}" + retry_kubectl_apply "${container_config}" if ! pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}'); then echo "Failed to create the pod" return 1