tests: ensure pod/deployment ready before tests

Debug it

Signed-off-by: Alex Lyn <alex.lyn@antgroup.com>

tests/integration/kubernetes/k8s-attach-handlers.bats

@@ -30,10 +30,12 @@ setup() {
 
 @test "Running with postStart and preStop handlers" {
 	# Create the pod with postStart and preStop handlers
-	kubectl create -f "${yaml_file}"
+	# kubectl create -f "${yaml_file}"
 
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check postStart message
 	check_postStart=$(kubectl exec $pod_name -- "${exec_command[@]}")

tests/integration/kubernetes/k8s-block-volume.bats

@@ -82,8 +82,9 @@ setup() {
 	auto_generate_policy "${policy_settings_dir}" "${tmp_pod_yaml}"
 
 	# Start the workload.
-	kubectl create -f "$tmp_pod_yaml"
-	kubectl wait --for condition=ready --timeout=$timeout "pod/${pod_name}"
+	# kubectl create -f "$tmp_pod_yaml"
+	# kubectl wait --for condition=ready --timeout=$timeout "pod/${pod_name}"
+	k8s_create_pod_ready "${pod_name}" "$tmp_pod_yaml"
 
 	# Verify persistent volume claim is bound
 	kubectl get "pvc/${volume_claim}" | grep "Bound"

tests/integration/kubernetes/k8s-caps.bats

@@ -40,9 +40,10 @@ setup() {
 
 @test "Check capabilities of pod" {
         # Create pod
-        kubectl create -f "${yaml_file}"
+        # kubectl create -f "${yaml_file}"
         # Check pod creation
-        kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+        # kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+        k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
         # Verify expected capabilities for the running container. Add retry to ensure
         # that the container had time to execute:

tests/integration/kubernetes/k8s-configmap.bats

@@ -34,10 +34,10 @@ setup() {
 	kubectl get configmaps $config_name -o yaml | grep -q "data-"
 
 	# Create a pod that consumes the ConfigMap
-	kubectl create -f "${pod_yaml_file}"
-
+	# kubectl create -f "${pod_yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	k8s_create_pod_ready "${pod_name}" "${pod_yaml_file}"
 
 	# Check env
 	grep_pod_exec_output "${pod_name}" "KUBE_CONFIG_1=value-1" "${exec_command[@]}"

tests/integration/kubernetes/k8s-copy-file.bats

@@ -41,10 +41,11 @@ setup() {
 	auto_generate_policy "${policy_settings_dir}" "${pod_config}"
 
 	# Create pod
-	kubectl create -f "${pod_config}"
-
+	# kubectl create -f "${pod_config}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	#kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${pod_config}"
 
 	# Create a file
 	echo "$content" > "$file_name"
@@ -78,10 +79,11 @@ setup() {
 	auto_generate_policy "${policy_settings_dir}" "${pod_config}"
 
 	# Create pod
-	kubectl create -f "${pod_config}"
-
+	# kubectl create -f "${pod_config}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${pod_config}"
 
 	kubectl logs "$pod_name" || true
 	kubectl describe pod "$pod_name" || true

tests/integration/kubernetes/k8s-exec.bats

@@ -38,10 +38,11 @@ setup() {
 
 @test "Kubectl exec" {
 	# Create the pod
-	kubectl create -f "${test_yaml_file}"
-
+	# kubectl create -f "${test_yaml_file}"
 	# Get pod specification
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${test_yaml_file}"
 
 	# Run commands in Pod
 	## Cases for -it options

tests/integration/kubernetes/k8s-file-volume.bats

@@ -47,10 +47,11 @@ setup() {
 
 @test "Test readonly volume for pods" {
 	# Create pod
-	kubectl create -f "${test_yaml}"
-
+	# kubectl create -f "${test_yaml}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${test_yaml}"
 
 	# Validate file volume body inside the pod
 	file_in_container=$(kubectl exec $pod_name -- "${command[@]}")

tests/integration/kubernetes/k8s-footloose.bats

@@ -37,10 +37,11 @@ setup() {
 	kubectl create -f "$configmap_yaml"
 
 	# Create pod
-	kubectl create -f "${pod_yaml}"
-
+	# kubectl create -f "${pod_yaml}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${pod_yaml}"
 
 	# Get pod ip
 	pod_ip=$(kubectl get pod $pod_name --template={{.status.podIP}})

tests/integration/kubernetes/k8s-guest-pull-image-authenticated.bats

@@ -88,7 +88,11 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
+
     echo "Kata pod test-e2e from authenticated image is running"
 }
 
@@ -132,7 +136,11 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
+
     echo "Kata pod test-e2e from authenticated image is running"
 }
 

tests/integration/kubernetes/k8s-guest-pull-image-encrypted.bats

@@ -65,7 +65,11 @@ function setup_kbs_decryption_key() {
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
+
     echo "Kata pod test-e2e from encrypted image is running"
 }
 

tests/integration/kubernetes/k8s-guest-pull-image-signature.bats

@@ -98,7 +98,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 
@@ -124,7 +127,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 
@@ -150,7 +156,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 
@@ -167,7 +176,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 
@@ -201,7 +213,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 
@@ -234,7 +249,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 

tests/integration/kubernetes/k8s-hostpath-volume.bats

@@ -29,8 +29,9 @@ setup() {
 }
 
 @test "/dev hostPath volume bind mounts the guest device and skips virtio-fs" {
-	kubectl apply -f "${yaml_file}"
-	kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}"
+	# kubectl apply -f "${yaml_file}"
+	# kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}"
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check the mount info.
 

tests/integration/kubernetes/k8s-kill-all-process-in-container.bats

@@ -26,10 +26,11 @@ setup() {
 
 @test "Kill all processes in container" {
 	# Create the pod
-	kubectl create -f "${yaml_file}"
-
+	# kubectl create -f "${yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check PID from first container
 	first_pid_container=$(kubectl exec $pod_name -c $first_container_name \

tests/integration/kubernetes/k8s-liveness-probes.bats

@@ -27,10 +27,11 @@ setup() {
 	auto_generate_policy "${pod_config_dir}" "${yaml_file}"
 
 	# Create pod
-	kubectl create -f "${yaml_file}"
-
+	# kubectl create -f "${yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check liveness probe returns a success code
 	kubectl describe pod "$pod_name" | grep -E "Liveness|#success=1"
@@ -52,10 +53,11 @@ setup() {
 	auto_generate_policy "${pod_config_dir}" "${yaml_file}"
 
 	# Create pod
-	kubectl create -f "${yaml_file}"
-
+	# kubectl create -f "${yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check liveness probe returns a success code
 	kubectl describe pod "$pod_name" | grep -E "Liveness|#success=1"
@@ -78,10 +80,11 @@ setup() {
 	auto_generate_policy "${pod_config_dir}" "${yaml_file}"
 
 	# Create pod
-	kubectl create -f "${yaml_file}"
-
+	# kubectl create -f "${yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check liveness probe returns a success code
 	kubectl describe pod "$pod_name" | grep -E "Liveness|#success=1"

tests/integration/kubernetes/k8s-nginx-connectivity.bats

@@ -25,9 +25,11 @@ setup() {
 
 @test "Verify nginx connectivity between pods" {
 
-	kubectl create -f "${yaml_file}"
-	kubectl wait --for=condition=Available --timeout=$timeout deployment/${deployment}
-	kubectl expose deployment/${deployment}
+	# kubectl create -f "${yaml_file}"
+	# kubectl wait --for=condition=Available --timeout=$timeout deployment/${deployment}
+	# Retries
+	k8s_create_deployment_ready "${yaml_file}" ${deployment}
+	kubectl expose deployment/${deployment} ${deployment}
 
 	busybox_pod="test-nginx"
 	kubectl run $busybox_pod --restart=Never -it --image="$busybox_image" \

tests/integration/kubernetes/k8s-oom.bats

@@ -18,10 +18,11 @@ setup() {
 
 @test "Test OOM events for pods" {
 	# Create pod
-	kubectl create -f "${yaml_file}"
-
+	# kubectl create -f "${yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	#kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${test_yaml}"
 
 	# Check if OOMKilled
     container_name=$(kubectl get pod "$pod_name" -o jsonpath='{.status.containerStatuses[0].name}')

tests/integration/kubernetes/k8s-optional-empty-configmap.bats

@@ -26,6 +26,42 @@ setup() {
 	auto_generate_policy "${policy_settings_dir}" "${pod_yaml}"
 }
 
+k8s_create_pod_ready() {
+	local pod_name="$1"
+	local pod_yaml="$2"
+
+	local wait_time=300
+	local max_attempts=5
+	local attempt_num
+
+	for attempt_num in $(seq 1 "${max_attempts}"); do
+		# First,forcefully deleting resources
+		kubectl delete -f "${pod_yaml}" --ignore-not-found=true --now --timeout=$wait_time
+
+		kubectl create -f "${pod_yaml}"
+		if [ $? -ne 0 ]; then
+			# Failed to create Pod.Aborting test.
+			continue
+		fi
+
+		# Check pod creation
+		kubectl wait --for=condition=Ready --timeout=$wait_time pod "$pod_name"
+		if [ "$status" -eq 0 ]; then
+			# Test Succeeded on attempt #${attempt_num}
+			return 0
+		fi
+
+		# Retry
+		if [ "${attempt_num}" -lt "${max_attempts}" ]; then
+			info "Waiting for 5 seconds before next attempt..."
+			sleep 5
+		fi
+	done
+
+	#Test Failed after ${max_attempts} attempts.
+	return 1
+}
+
 @test "Optional and Empty ConfigMap Volume for a pod" {
 	config_name="empty-config"
 	pod_name="optional-empty-config-test-pod"
@@ -34,10 +70,11 @@ setup() {
 	kubectl create configmap "$config_name"
 
 	# Create a pod that consumes the "empty-config" and "optional-missing-config" ConfigMaps as volumes
-	kubectl create -f "${pod_yaml}"
-
+	# kubectl create -f "${pod_yaml}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retry for ready pod
+	k8s_create_pod_ready "$pod_name" "${pod_yaml}"
 
 	# Check configmap folders exist
 	kubectl exec $pod_name -- "${exec_empty_command[@]}"

tests/integration/kubernetes/k8s-scale-nginx.bats

@@ -24,8 +24,11 @@ setup() {
 }
 
 @test "Scale nginx deployment" {
-	kubectl create -f "${test_yaml}"
-	kubectl wait --for=condition=Available --timeout=$timeout deployment/${deployment}
+	# kubectl create -f "${test_yaml}"
+	# kubectl wait --for=condition=Available --timeout=$timeout deployment/${deployment}
+	# Retries
+	k8s_create_deployment_ready "${test_yaml}" ${deployment}
+
 	kubectl expose deployment/${deployment}
 	kubectl scale deployment/${deployment} --replicas=${replicas}
 	cmd="kubectl get deployment/${deployment} -o yaml | grep 'availableReplicas: ${replicas}'"

tests/integration/kubernetes/tests_common.sh

@@ -420,6 +420,80 @@ wait_for_blocked_request() {
 	waitForProcess "${wait_time}" "${sleep_time}" "${command}" >/dev/null 2>/dev/null
 }
 
+# k8s create a ready pod
+k8s_create_pod_ready() {
+	local pod_name="$1"
+	local pod_yaml="$2"
+	local wait_time="${3:-300}"
+	local max_attempts="${4:-5}"
+
+	local attempt_num
+
+	for attempt_num in $(seq 1 "${max_attempts}"); do
+		# First,forcefully deleting resources
+		kubectl delete -f "${pod_yaml}" --ignore-not-found=true --now --timeout=$wait_time
+
+		kubectl create -f "${pod_yaml}"
+		if [ $? -ne 0 ]; then
+			# Failed to create Pod.Aborting test.
+			continue
+		fi
+
+		# Check pod creation
+		kubectl wait --for=condition=Ready --timeout=$wait_time pod "$pod_name"
+		if [ "$status" -eq 0 ]; then
+			# Test Succeeded on attempt #${attempt_num}
+			info "Waiting ${wait_time} seconds for pod ${pod_name} Ready."
+			return 0
+		fi
+
+		# Retry
+		if [ "${attempt_num}" -lt "${max_attempts}" ]; then
+			info "Waiting for 5 seconds before next attempt..."
+			sleep 5
+		fi
+	done
+
+	info "Test Failed after ${max_attempts} attempts for pod ${pod_name}."
+	return 1
+}
+
+k8s_create_deployment_ready() {
+	local deployment_yaml="$1"
+	local deployment="$2"
+
+	local wait_time=300
+	local max_attempts=5
+	local attempt_num
+
+	for attempt_num in $(seq 1 "${max_attempts}"); do
+		# First,forcefully deleting resources
+		kubectl delete -f "${deployment_yaml}" --ignore-not-found=true --now --timeout=$wait_time
+
+		kubectl create -f "${deployment_yaml}"
+		if [ $? -ne 0 ]; then
+			# Failed to create Pod.Aborting test.
+			continue
+		fi
+
+		# Check deployment ready
+		kubectl wait --for=condition=Available --timeout=$timeout deployment/${deployment}
+		if [ "$status" -eq 0 ]; then
+			# Test Succeeded on attempt #${attempt_num}
+			return 0
+		fi
+
+		# Retry
+		if [ "${attempt_num}" -lt "${max_attempts}" ]; then
+			info "Waiting for 5 seconds before next attempt..."
+			sleep 5
+		fi
+	done
+
+	#Test Failed after ${max_attempts} attempts.
+	return 1
+}
+
 # Execute in a pod a command that is allowed by policy.
 pod_exec_allowed_command() {
 	local -r pod_name="$1"