tests: ensure pod/deployment ready before tests

Debug it

Signed-off-by: Alex Lyn <alex.lyn@antgroup.com>

tests/integration/kubernetes/k8s-oom.bats

@@ -22,7 +22,7 @@ setup() {
 	# Check pod creation
 	#kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
 	# Retries
-	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
+	k8s_create_pod_ready "${pod_name}" "${test_yaml}"
 
 	# Check if OOMKilled
     container_name=$(kubectl get pod "$pod_name" -o jsonpath='{.status.containerStatuses[0].name}')

tests/integration/kubernetes/k8s-optional-empty-configmap.bats

@@ -26,6 +26,42 @@ setup() {
 	auto_generate_policy "${policy_settings_dir}" "${pod_yaml}"
 }
 
+k8s_create_pod_ready() {
+	local pod_name="$1"
+	local pod_yaml="$2"
+
+	local wait_time=300
+	local max_attempts=5
+	local attempt_num
+
+	for attempt_num in $(seq 1 "${max_attempts}"); do
+		# First,forcefully deleting resources
+		kubectl delete -f "${pod_yaml}" --ignore-not-found=true --now --timeout=$wait_time
+
+		kubectl create -f "${pod_yaml}"
+		if [ $? -ne 0 ]; then
+			# Failed to create Pod.Aborting test.
+			continue
+		fi
+
+		# Check pod creation
+		kubectl wait --for=condition=Ready --timeout=$wait_time pod "$pod_name"
+		if [ "$status" -eq 0 ]; then
+			# Test Succeeded on attempt #${attempt_num}
+			return 0
+		fi
+
+		# Retry
+		if [ "${attempt_num}" -lt "${max_attempts}" ]; then
+			info "Waiting for 5 seconds before next attempt..."
+			sleep 5
+		fi
+	done
+
+	#Test Failed after ${max_attempts} attempts.
+	return 1
+}
+
 @test "Optional and Empty ConfigMap Volume for a pod" {
 	config_name="empty-config"
 	pod_name="optional-empty-config-test-pod"

tests/integration/kubernetes/k8s-policy-job.bats

@@ -51,66 +51,22 @@ setup() {
 
 # Common function for all test cases that expect CreateContainer to be blocked by policy.
 test_job_policy_error() {
-    local max_attempts=5
+    # Initiate job creation
-    local attempt_num
+    kubectl apply -f "${incorrect_yaml}"
-    local sleep_between_attempts=5
 
-    for attempt_num in $(seq 1 "${max_attempts}"); do
+    # Wait for the job to be created
-        info "Starting attempt #${attempt_num}"
+    cmd="kubectl describe job ${job_name} | grep SuccessfulCreate"
+    info "Waiting for: ${cmd}"
+    waitForProcess "${wait_time}" "${sleep_time}" "${cmd}" || return 1
 
-        # Cleanup possible previous resources
+    # List the pods that belong to the job
-        kubectl delete -f "${incorrect_yaml}" --ignore-not-found=true --now --timeout=120s
+    pod_names=$(kubectl get pods "--selector=job-name=${job_name}" --output=jsonpath='{.items[*].metadata.name}')
+    info "pod_names: ${pod_names}"
 
-        # 1. Apply Job
+    # CreateContainerRequest must have been denied by the policy.
-        kubectl apply -f "${incorrect_yaml}"
+    for pod_name in ${pod_names[@]}; do
-        if [ $? -ne 0 ]; then
+        wait_for_blocked_request "CreateContainerRequest" "${pod_name}" || return 1
-            warn "Failed to apply Job. Retrying..."
-            continue
-        fi
-
-        # 2. Wait for Job creation event
-        cmd="kubectl describe job ${job_name} | grep SuccessfulCreate"
-        info "Waiting for: ${cmd}"
-
-        run waitForProcess "${wait_time}" "${sleep_time}" "${cmd}"
-        if [ "$status" -ne 0 ]; then
-            warn "waitForProcess FAILED on attempt #${attempt_num}"
-            continue
-        fi
-
-        # 3. Get pod list
-        pod_names=$(kubectl get pods "--selector=job-name=${job_name}" --output=jsonpath='{.items[*].metadata.name}')
-        info "pod_names: ${pod_names}"
-
-        if [ -z "${pod_names}" ]; then
-            warn "No pods found for job. Retrying..."
-            continue
-        fi
-
-        # 4. Check each pod for blocked CreateContainerRequest
-        for pod_name in ${pod_names[@]}; do
-            info "Checking pod: ${pod_name}"
-
-            run wait_for_blocked_request "CreateContainerRequest" "${pod_name}"
-            if [ "$status" -eq 0 ]; then
-                info "wait_for_blocked_request succeeded for pod ${pod_name} on attempt #${attempt_num}"
-                return 0
-            else
-                warn "wait_for_blocked_request FAILED for pod ${pod_name} on attempt #${attempt_num}"
-                # We break pod loop, but the attempt will continue
-                break
-            fi
-        done
-
-        # Retry if not last attempt
-        if [ "${attempt_num}" -lt "${max_attempts}" ]; then
-            info "Retrying in ${sleep_between_attempts} seconds..."
-            sleep "${sleep_between_attempts}"
-        fi
     done
-
-    error "Test failed after ${max_attempts} attempts."
-    return 1
 }
 
 @test "Policy failure: unexpected environment variable" {
@@ -120,8 +76,6 @@ test_job_policy_error() {
         "${incorrect_yaml}"
 
     test_job_policy_error
-    test_result=$?
-    [ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected command line argument" {
@@ -131,8 +85,6 @@ test_job_policy_error() {
         "${incorrect_yaml}"
 
     test_job_policy_error
-    test_result=$?
-    [ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected emptyDir volume" {
@@ -146,8 +98,6 @@ test_job_policy_error() {
         "${incorrect_yaml}"
 
     test_job_policy_error
-    test_result=$?
-    [ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected projected volume" {
@@ -172,8 +122,6 @@ test_job_policy_error() {
     ' "${incorrect_yaml}"
 
     test_job_policy_error
-    test_result=$?
-    [ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected readOnlyRootFilesystem" {
@@ -183,8 +131,6 @@ test_job_policy_error() {
         "${incorrect_yaml}"
 
     test_job_policy_error
-    test_result=$?
-    [ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected UID = 222" {
@@ -194,8 +140,6 @@ test_job_policy_error() {
         "${incorrect_yaml}"
 
     test_job_policy_error
-    test_result=$?
-    [ "${test_result}" -eq 0 ]
 }
 
 teardown() {

tests/integration/kubernetes/k8s-policy-pod.bats

@@ -130,47 +130,9 @@ create_and_wait_for_pod_ready() {
 
 # Common function for several test cases from this bats script.
 test_pod_policy_error() {
-	local max_attempts=5
+	kubectl create -f "${correct_configmap_yaml}"
-	local attempt_num
+	kubectl create -f "${incorrect_pod_yaml}"
-	local sleep_between_attempts=5
+	wait_for_blocked_request "CreateContainerRequest" "${pod_name}"
-
-	for attempt_num in $(seq 1 "${max_attempts}"); do
-		info "Starting attempt #${attempt_num}"
-		kubectl delete -f "${incorrect_pod_yaml}" --ignore-not-found=true --now --timeout=120s
-		kubectl delete -f "${correct_configmap_yaml}" --ignore-not-found=true
-
-		# Create ConfigMap
-		kubectl create -f "${correct_configmap_yaml}"
-		if [ $? -ne 0 ]; then
-			warn "Failed to create ConfigMap. Retrying..."
-			continue
-		fi
-
-		# Create the incorrect pod (expected to be blocked)
-		kubectl create -f "${incorrect_pod_yaml}"
-		if [ $? -ne 0 ]; then
-			warn "Failed to create Pod. Retrying..."
-			continue
-		fi
-
-		# Wait for CreateContainerRequest to be blocked
-		run wait_for_blocked_request "CreateContainerRequest" "${pod_name}"
-		if [ "$status" -eq 0 ]; then
-			info "wait_for_blocked_request succeeded on attempt #${attempt_num}"
-			return 0
-		else
-			warn "wait_for_blocked_request FAILED on attempt #${attempt_num}"
-		fi
-
-		# Retry if not the last attempt
-		if [ "${attempt_num}" -lt "${max_attempts}" ]; then
-			info "Retrying in ${sleep_between_attempts} seconds..."
-			sleep "${sleep_between_attempts}"
-		fi
-	done
-
-	error "Test failed after ${max_attempts} attempts."
-	return 1
 }
 
 @test "Policy failure: unexpected container image" {
@@ -181,8 +143,6 @@ test_pod_policy_error() {
 		"${incorrect_pod_yaml}"
 
 	test_pod_policy_error
-	test_result=$?
-	[ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected privileged security context" {
@@ -192,8 +152,6 @@ test_pod_policy_error() {
 		"${incorrect_pod_yaml}"
 
 	test_pod_policy_error
-	test_result=$?
-	[ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected terminationMessagePath" {
@@ -203,8 +161,6 @@ test_pod_policy_error() {
 		"${incorrect_pod_yaml}"
 
 	test_pod_policy_error
-	test_result=$?
-	[ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected hostPath volume mount" {
@@ -218,8 +174,6 @@ test_pod_policy_error() {
 		"${incorrect_pod_yaml}"
 
 	test_pod_policy_error
-	test_result=$?
-	[ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected config map" {
@@ -311,8 +265,6 @@ test_pod_policy_error() {
 		"${incorrect_pod_yaml}"
 
 	test_pod_policy_error
-	test_result=$?
-	[ "${test_result}" -eq 0 ]
 }
 
 @test "Policy failure: unexpected UID = 1234" {
@@ -324,8 +276,6 @@ test_pod_policy_error() {
 		"${incorrect_pod_yaml}"
 
 	test_pod_policy_error
-	test_result=$?
-	[ "${test_result}" -eq 0 ]
 }
 
 teardown() {

tests/integration/kubernetes/tests_common.sh

@@ -431,7 +431,7 @@ k8s_create_pod_ready() {
 
 	for attempt_num in $(seq 1 "${max_attempts}"); do
 		# First,forcefully deleting resources
-		kubectl delete -f "${pod_yaml}" --ignore-not-found=true --now --timeout=$timeout
+		kubectl delete -f "${pod_yaml}" --ignore-not-found=true --now --timeout=$wait_time
 
 		kubectl create -f "${pod_yaml}"
 		if [ $? -ne 0 ]; then
@@ -440,7 +440,7 @@ k8s_create_pod_ready() {
 		fi
 
 		# Check pod creation
-		run kubectl wait --for=condition=Ready --timeout="${wait_time}s" pod "${pod_name}"
+		kubectl wait --for=condition=Ready --timeout=$wait_time pod "$pod_name"
 		if [ "$status" -eq 0 ]; then
 			# Test Succeeded on attempt #${attempt_num}
 			info "Waiting ${wait_time} seconds for pod ${pod_name} Ready."
@@ -468,7 +468,7 @@ k8s_create_deployment_ready() {
 
 	for attempt_num in $(seq 1 "${max_attempts}"); do
 		# First,forcefully deleting resources
-		kubectl delete -f "${deployment_yaml}" --ignore-not-found=true --now --timeout=$timeout
+		kubectl delete -f "${deployment_yaml}" --ignore-not-found=true --now --timeout=$wait_time
 
 		kubectl create -f "${deployment_yaml}"
 		if [ $? -ne 0 ]; then
@@ -477,7 +477,7 @@ k8s_create_deployment_ready() {
 		fi
 
 		# Check deployment ready
-		run kubectl wait --for=condition=Available --timeout="${wait_time}s" deployment/${deployment}
+		kubectl wait --for=condition=Available --timeout=$timeout deployment/${deployment}
 		if [ "$status" -eq 0 ]; then
 			# Test Succeeded on attempt #${attempt_num}
 			return 0