tests: enhance CI bats for stabilities

Debug it

Signed-off-by: Alex Lyn <alex.lyn@antgroup.com>

tests/integration/kubernetes/k8s-attach-handlers.bats

@@ -30,10 +30,12 @@ setup() {
 
 @test "Running with postStart and preStop handlers" {
 	# Create the pod with postStart and preStop handlers
-	kubectl create -f "${yaml_file}"
+	# kubectl create -f "${yaml_file}"
 
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check postStart message
 	check_postStart=$(kubectl exec $pod_name -- "${exec_command[@]}")

tests/integration/kubernetes/k8s-block-volume.bats

@@ -82,8 +82,9 @@ setup() {
 	auto_generate_policy "${policy_settings_dir}" "${tmp_pod_yaml}"
 
 	# Start the workload.
-	kubectl create -f "$tmp_pod_yaml"
-	kubectl wait --for condition=ready --timeout=$timeout "pod/${pod_name}"
+	# kubectl create -f "$tmp_pod_yaml"
+	# kubectl wait --for condition=ready --timeout=$timeout "pod/${pod_name}"
+	k8s_create_pod_ready "${pod_name}" "$tmp_pod_yaml"
 
 	# Verify persistent volume claim is bound
 	kubectl get "pvc/${volume_claim}" | grep "Bound"

tests/integration/kubernetes/k8s-caps.bats

@@ -40,9 +40,10 @@ setup() {
 
 @test "Check capabilities of pod" {
         # Create pod
-        kubectl create -f "${yaml_file}"
+        # kubectl create -f "${yaml_file}"
         # Check pod creation
-        kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+        # kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+        k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
         # Verify expected capabilities for the running container. Add retry to ensure
         # that the container had time to execute:

tests/integration/kubernetes/k8s-configmap.bats

@@ -34,10 +34,10 @@ setup() {
 	kubectl get configmaps $config_name -o yaml | grep -q "data-"
 
 	# Create a pod that consumes the ConfigMap
-	kubectl create -f "${pod_yaml_file}"
-
+	# kubectl create -f "${pod_yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	k8s_create_pod_ready "${pod_name}" "${pod_yaml_file}"
 
 	# Check env
 	grep_pod_exec_output "${pod_name}" "KUBE_CONFIG_1=value-1" "${exec_command[@]}"

tests/integration/kubernetes/k8s-copy-file.bats

@@ -41,10 +41,11 @@ setup() {
 	auto_generate_policy "${policy_settings_dir}" "${pod_config}"
 
 	# Create pod
-	kubectl create -f "${pod_config}"
-
+	# kubectl create -f "${pod_config}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	#kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${pod_config}"
 
 	# Create a file
 	echo "$content" > "$file_name"
@@ -78,10 +79,11 @@ setup() {
 	auto_generate_policy "${policy_settings_dir}" "${pod_config}"
 
 	# Create pod
-	kubectl create -f "${pod_config}"
-
+	# kubectl create -f "${pod_config}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${pod_config}"
 
 	kubectl logs "$pod_name" || true
 	kubectl describe pod "$pod_name" || true

tests/integration/kubernetes/k8s-exec.bats

@@ -38,10 +38,11 @@ setup() {
 
 @test "Kubectl exec" {
 	# Create the pod
-	kubectl create -f "${test_yaml_file}"
-
+	# kubectl create -f "${test_yaml_file}"
 	# Get pod specification
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${test_yaml_file}"
 
 	# Run commands in Pod
 	## Cases for -it options

tests/integration/kubernetes/k8s-file-volume.bats

@@ -47,10 +47,11 @@ setup() {
 
 @test "Test readonly volume for pods" {
 	# Create pod
-	kubectl create -f "${test_yaml}"
-
+	# kubectl create -f "${test_yaml}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${test_yaml}"
 
 	# Validate file volume body inside the pod
 	file_in_container=$(kubectl exec $pod_name -- "${command[@]}")

tests/integration/kubernetes/k8s-footloose.bats

@@ -37,10 +37,11 @@ setup() {
 	kubectl create -f "$configmap_yaml"
 
 	# Create pod
-	kubectl create -f "${pod_yaml}"
-
+	# kubectl create -f "${pod_yaml}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${pod_yaml}"
 
 	# Get pod ip
 	pod_ip=$(kubectl get pod $pod_name --template={{.status.podIP}})

tests/integration/kubernetes/k8s-guest-pull-image-authenticated.bats

@@ -88,7 +88,11 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
+
     echo "Kata pod test-e2e from authenticated image is running"
 }
 
@@ -132,7 +136,11 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
+
     echo "Kata pod test-e2e from authenticated image is running"
 }
 

tests/integration/kubernetes/k8s-guest-pull-image-encrypted.bats

@@ -65,7 +65,11 @@ function setup_kbs_decryption_key() {
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
+
     echo "Kata pod test-e2e from encrypted image is running"
 }
 

tests/integration/kubernetes/k8s-guest-pull-image-signature.bats

@@ -98,7 +98,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 
@@ -124,7 +127,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 
@@ -150,7 +156,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 
@@ -167,7 +176,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 
@@ -201,7 +213,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 
@@ -234,7 +249,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 

tests/integration/kubernetes/k8s-hostpath-volume.bats

@@ -29,8 +29,9 @@ setup() {
 }
 
 @test "/dev hostPath volume bind mounts the guest device and skips virtio-fs" {
-	kubectl apply -f "${yaml_file}"
-	kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}"
+	# kubectl apply -f "${yaml_file}"
+	# kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}"
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check the mount info.
 

tests/integration/kubernetes/k8s-kill-all-process-in-container.bats

@@ -26,10 +26,11 @@ setup() {
 
 @test "Kill all processes in container" {
 	# Create the pod
-	kubectl create -f "${yaml_file}"
-
+	# kubectl create -f "${yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check PID from first container
 	first_pid_container=$(kubectl exec $pod_name -c $first_container_name \

tests/integration/kubernetes/k8s-liveness-probes.bats

@@ -27,10 +27,11 @@ setup() {
 	auto_generate_policy "${pod_config_dir}" "${yaml_file}"
 
 	# Create pod
-	kubectl create -f "${yaml_file}"
-
+	# kubectl create -f "${yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check liveness probe returns a success code
 	kubectl describe pod "$pod_name" | grep -E "Liveness|#success=1"
@@ -52,10 +53,11 @@ setup() {
 	auto_generate_policy "${pod_config_dir}" "${yaml_file}"
 
 	# Create pod
-	kubectl create -f "${yaml_file}"
-
+	# kubectl create -f "${yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check liveness probe returns a success code
 	kubectl describe pod "$pod_name" | grep -E "Liveness|#success=1"
@@ -78,10 +80,11 @@ setup() {
 	auto_generate_policy "${pod_config_dir}" "${yaml_file}"
 
 	# Create pod
-	kubectl create -f "${yaml_file}"
-
+	# kubectl create -f "${yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check liveness probe returns a success code
 	kubectl describe pod "$pod_name" | grep -E "Liveness|#success=1"

tests/integration/kubernetes/k8s-nginx-connectivity.bats

@@ -25,9 +25,11 @@ setup() {
 
 @test "Verify nginx connectivity between pods" {
 
-	kubectl create -f "${yaml_file}"
-	kubectl wait --for=condition=Available --timeout=$timeout deployment/${deployment}
-	kubectl expose deployment/${deployment}
+	# kubectl create -f "${yaml_file}"
+	# kubectl wait --for=condition=Available --timeout=$timeout deployment/${deployment}
+	# Retries
+	k8s_create_deployment_ready "${yaml_file}" ${deployment}
+	kubectl expose deployment/${deployment} ${deployment}
 
 	busybox_pod="test-nginx"
 	kubectl run $busybox_pod --restart=Never -it --image="$busybox_image" \

tests/integration/kubernetes/k8s-oom.bats

@@ -18,10 +18,11 @@ setup() {
 
 @test "Test OOM events for pods" {
 	# Create pod
-	kubectl create -f "${yaml_file}"
-
+	# kubectl create -f "${yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	#kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check if OOMKilled
     container_name=$(kubectl get pod "$pod_name" -o jsonpath='{.status.containerStatuses[0].name}')

tests/integration/kubernetes/k8s-optional-empty-configmap.bats

@@ -34,10 +34,11 @@ setup() {
 	kubectl create configmap "$config_name"
 
 	# Create a pod that consumes the "empty-config" and "optional-missing-config" ConfigMaps as volumes
-	kubectl create -f "${pod_yaml}"
-
+	# kubectl create -f "${pod_yaml}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retry for ready pod
+	k8s_create_pod_ready "$pod_name" "${pod_yaml}"
 
 	# Check configmap folders exist
 	kubectl exec $pod_name -- "${exec_empty_command[@]}"

tests/integration/kubernetes/k8s-policy-job.bats

@@ -51,22 +51,66 @@ setup() {
 
 # Common function for all test cases that expect CreateContainer to be blocked by policy.
 test_job_policy_error() {
-    # Initiate job creation
-    kubectl apply -f "${incorrect_yaml}"
+    local max_attempts=5
+    local attempt_num
+    local sleep_between_attempts=5
 
-    # Wait for the job to be created
-    cmd="kubectl describe job ${job_name} | grep SuccessfulCreate"
-    info "Waiting for: ${cmd}"
-    waitForProcess "${wait_time}" "${sleep_time}" "${cmd}" || return 1
+    for attempt_num in $(seq 1 "${max_attempts}"); do
+        info "Starting attempt #${attempt_num}"
 
-    # List the pods that belong to the job
-    pod_names=$(kubectl get pods "--selector=job-name=${job_name}" --output=jsonpath='{.items[*].metadata.name}')
-    info "pod_names: ${pod_names}"
+        # Cleanup possible previous resources
+        kubectl delete -f "${incorrect_yaml}" --ignore-not-found=true --now --timeout=120s
 
-    # CreateContainerRequest must have been denied by the policy.
-    for pod_name in ${pod_names[@]}; do
-        wait_for_blocked_request "CreateContainerRequest" "${pod_name}" || return 1
+        # 1. Apply Job
+        kubectl apply -f "${incorrect_yaml}"
+        if [ $? -ne 0 ]; then
+            warn "Failed to apply Job. Retrying..."
+            continue
+        fi
+
+        # 2. Wait for Job creation event
+        cmd="kubectl describe job ${job_name} | grep SuccessfulCreate"
+        info "Waiting for: ${cmd}"
+
+        run waitForProcess "${wait_time}" "${sleep_time}" "${cmd}"
+        if [ "$status" -ne 0 ]; then
+            warn "waitForProcess FAILED on attempt #${attempt_num}"
+            continue
+        fi
+
+        # 3. Get pod list
+        pod_names=$(kubectl get pods "--selector=job-name=${job_name}" --output=jsonpath='{.items[*].metadata.name}')
+        info "pod_names: ${pod_names}"
+
+        if [ -z "${pod_names}" ]; then
+            warn "No pods found for job. Retrying..."
+            continue
+        fi
+
+        # 4. Check each pod for blocked CreateContainerRequest
+        for pod_name in ${pod_names[@]}; do
+            info "Checking pod: ${pod_name}"
+
+            run wait_for_blocked_request "CreateContainerRequest" "${pod_name}"
+            if [ "$status" -eq 0 ]; then
+                info "wait_for_blocked_request succeeded for pod ${pod_name} on attempt #${attempt_num}"
+                return 0
+            else
+                warn "wait_for_blocked_request FAILED for pod ${pod_name} on attempt #${attempt_num}"
+                # We break pod loop, but the attempt will continue
+                break
+            fi
+        done
+
+        # Retry if not last attempt
+        if [ "${attempt_num}" -lt "${max_attempts}" ]; then
+            info "Retrying in ${sleep_between_attempts} seconds..."
+            sleep "${sleep_between_attempts}"
+        fi
     done
+
+    error "Test failed after ${max_attempts} attempts."
+    return 1
 }
 
 @test "Policy failure: unexpected environment variable" {
@@ -76,6 +120,8 @@ test_job_policy_error() {
         "${incorrect_yaml}"
 
     test_job_policy_error
+    test_result=$?
+    [ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected command line argument" {
@@ -85,6 +131,8 @@ test_job_policy_error() {
         "${incorrect_yaml}"
 
     test_job_policy_error
+    test_result=$?
+    [ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected emptyDir volume" {
@@ -98,6 +146,8 @@ test_job_policy_error() {
         "${incorrect_yaml}"
 
     test_job_policy_error
+    test_result=$?
+    [ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected projected volume" {
@@ -122,6 +172,8 @@ test_job_policy_error() {
     ' "${incorrect_yaml}"
 
     test_job_policy_error
+    test_result=$?
+    [ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected readOnlyRootFilesystem" {
@@ -131,6 +183,8 @@ test_job_policy_error() {
         "${incorrect_yaml}"
 
     test_job_policy_error
+    test_result=$?
+    [ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected UID = 222" {
@@ -140,6 +194,8 @@ test_job_policy_error() {
         "${incorrect_yaml}"
 
     test_job_policy_error
+    test_result=$?
+    [ "${test_result}" -eq 0 ]
 }
 
 teardown() {

tests/integration/kubernetes/k8s-policy-pod.bats

@@ -130,9 +130,47 @@ create_and_wait_for_pod_ready() {
 
 # Common function for several test cases from this bats script.
 test_pod_policy_error() {
-	kubectl create -f "${correct_configmap_yaml}"
-	kubectl create -f "${incorrect_pod_yaml}"
-	wait_for_blocked_request "CreateContainerRequest" "${pod_name}"
+	local max_attempts=5
+	local attempt_num
+	local sleep_between_attempts=5
+
+	for attempt_num in $(seq 1 "${max_attempts}"); do
+		info "Starting attempt #${attempt_num}"
+		kubectl delete -f "${incorrect_pod_yaml}" --ignore-not-found=true --now --timeout=120s
+		kubectl delete -f "${correct_configmap_yaml}" --ignore-not-found=true
+
+		# Create ConfigMap
+		kubectl create -f "${correct_configmap_yaml}"
+		if [ $? -ne 0 ]; then
+			warn "Failed to create ConfigMap. Retrying..."
+			continue
+		fi
+
+		# Create the incorrect pod (expected to be blocked)
+		kubectl create -f "${incorrect_pod_yaml}"
+		if [ $? -ne 0 ]; then
+			warn "Failed to create Pod. Retrying..."
+			continue
+		fi
+
+		# Wait for CreateContainerRequest to be blocked
+		run wait_for_blocked_request "CreateContainerRequest" "${pod_name}"
+		if [ "$status" -eq 0 ]; then
+			info "wait_for_blocked_request succeeded on attempt #${attempt_num}"
+			return 0
+		else
+			warn "wait_for_blocked_request FAILED on attempt #${attempt_num}"
+		fi
+
+		# Retry if not the last attempt
+		if [ "${attempt_num}" -lt "${max_attempts}" ]; then
+			info "Retrying in ${sleep_between_attempts} seconds..."
+			sleep "${sleep_between_attempts}"
+		fi
+	done
+
+	error "Test failed after ${max_attempts} attempts."
+	return 1
 }
 
 @test "Policy failure: unexpected container image" {
@@ -143,6 +181,8 @@ test_pod_policy_error() {
 		"${incorrect_pod_yaml}"
 
 	test_pod_policy_error
+	test_result=$?
+	[ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected privileged security context" {
@@ -152,6 +192,8 @@ test_pod_policy_error() {
 		"${incorrect_pod_yaml}"
 
 	test_pod_policy_error
+	test_result=$?
+	[ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected terminationMessagePath" {
@@ -161,6 +203,8 @@ test_pod_policy_error() {
 		"${incorrect_pod_yaml}"
 
 	test_pod_policy_error
+	test_result=$?
+	[ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected hostPath volume mount" {
@@ -174,6 +218,8 @@ test_pod_policy_error() {
 		"${incorrect_pod_yaml}"
 
 	test_pod_policy_error
+	test_result=$?
+	[ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected config map" {
@@ -265,6 +311,8 @@ test_pod_policy_error() {
 		"${incorrect_pod_yaml}"
 
 	test_pod_policy_error
+	test_result=$?
+	[ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected UID = 1234" {
@@ -276,6 +324,8 @@ test_pod_policy_error() {
 		"${incorrect_pod_yaml}"
 
 	test_pod_policy_error
+	test_result=$?
+	[ "${test_result}" -eq 0 ]
 }
 
 teardown() {

tests/integration/kubernetes/k8s-scale-nginx.bats

@@ -24,8 +24,11 @@ setup() {
 }
 
 @test "Scale nginx deployment" {
-	kubectl create -f "${test_yaml}"
-	kubectl wait --for=condition=Available --timeout=$timeout deployment/${deployment}
+	# kubectl create -f "${test_yaml}"
+	# kubectl wait --for=condition=Available --timeout=$timeout deployment/${deployment}
+	# Retries
+	k8s_create_deployment_ready "${test_yaml}" ${deployment}
+
 	kubectl expose deployment/${deployment}
 	kubectl scale deployment/${deployment} --replicas=${replicas}
 	cmd="kubectl get deployment/${deployment} -o yaml | grep 'availableReplicas: ${replicas}'"

tests/integration/kubernetes/tests_common.sh

@@ -420,6 +420,80 @@ wait_for_blocked_request() {
 	waitForProcess "${wait_time}" "${sleep_time}" "${command}" >/dev/null 2>/dev/null
 }
 
+# k8s create a ready pod
+k8s_create_pod_ready() {
+	local pod_name="$1"
+	local pod_yaml="$2"
+	local wait_time="${3:-300}"
+	local max_attempts="${4:-5}"
+
+	local attempt_num
+
+	for attempt_num in $(seq 1 "${max_attempts}"); do
+		# First,forcefully deleting resources
+		kubectl delete -f "${pod_yaml}" --ignore-not-found=true --now --timeout=$timeout
+
+		kubectl create -f "${pod_yaml}"
+		if [ $? -ne 0 ]; then
+			# Failed to create Pod.Aborting test.
+			continue
+		fi
+
+		# Check pod creation
+		run kubectl wait --for=condition=Ready --timeout="${wait_time}s" pod "${pod_name}"
+		if [ "$status" -eq 0 ]; then
+			# Test Succeeded on attempt #${attempt_num}
+			info "Waiting ${wait_time} seconds for pod ${pod_name} Ready."
+			return 0
+		fi
+
+		# Retry
+		if [ "${attempt_num}" -lt "${max_attempts}" ]; then
+			info "Waiting for 5 seconds before next attempt..."
+			sleep 5
+		fi
+	done
+
+	info "Test Failed after ${max_attempts} attempts for pod ${pod_name}."
+	return 1
+}
+
+k8s_create_deployment_ready() {
+	local deployment_yaml="$1"
+	local deployment="$2"
+
+	local wait_time=300
+	local max_attempts=5
+	local attempt_num
+
+	for attempt_num in $(seq 1 "${max_attempts}"); do
+		# First,forcefully deleting resources
+		kubectl delete -f "${deployment_yaml}" --ignore-not-found=true --now --timeout=$timeout
+
+		kubectl create -f "${deployment_yaml}"
+		if [ $? -ne 0 ]; then
+			# Failed to create Pod.Aborting test.
+			continue
+		fi
+
+		# Check deployment ready
+		run kubectl wait --for=condition=Available --timeout="${wait_time}s" deployment/${deployment}
+		if [ "$status" -eq 0 ]; then
+			# Test Succeeded on attempt #${attempt_num}
+			return 0
+		fi
+
+		# Retry
+		if [ "${attempt_num}" -lt "${max_attempts}" ]; then
+			info "Waiting for 5 seconds before next attempt..."
+			sleep 5
+		fi
+	done
+
+	#Test Failed after ${max_attempts} attempts.
+	return 1
+}
+
 # Execute in a pod a command that is allowed by policy.
 pod_exec_allowed_command() {
 	local -r pod_name="$1"