tests: ensure pod/deployment ready before tests

Debug it

Signed-off-by: Alex Lyn <alex.lyn@antgroup.com>

docs/README.md

@@ -83,3 +83,7 @@ Documents that help to understand and contribute to Kata Containers.
 If you have a suggestion for how we can improve the
 [website](https://katacontainers.io), please raise an issue (or a PR) on
 [the repository that holds the source for the website](https://github.com/OpenStackweb/kata-netlify-refresh).
+
+### Toolchain Guidance
+
+* [Toolchain Guidance](./Toochain-Guidance.md)

docs/Toochain-Guidance.md

@@ -0,0 +1,39 @@
+# Toolchains
+
+As a community we want to strike a balance between having up-to-date toolchains, to receive the
+latest security fixes and to be able to benefit from new features and packages, whilst not being
+too bleeding edge and disrupting downstream and other consumers. As a result we have the following
+guidelines (note, not hard rules) for our go and rust toolchains that we are attempting to try out:
+
+## Go toolchain
+
+Go is released [every six months](https://go.dev/wiki/Go-Release-Cycle) with support for the
+[last two major release versions](https://go.dev/doc/devel/release#policy). We always want to
+ensure that we are on a supported version so we receive security fixes. To try and make
+things easier for some of our users, we aim to be using the older of the two supported major
+versions, unless there is a compelling reason to adopt the newer version.
+
+In practice this means that we bump our major version of the go toolchain every six months to
+version (1.x-1) in response to a new version (1.x) coming out, which makes our current version
+(1.x-2) no longer supported. We will bump the minor version whenever required to satisfy
+dependency updates, or security fixes.
+
+Our go toolchain version is recorded in [`versions.yaml`](../versions.yaml) under
+`.languages.golang.version` and should match with the version in our `go.mod` files.
+
+## Rust toolchain
+
+Rust has a [six week](https://doc.rust-lang.org/book/appendix-05-editions.html#:~:text=The%20Rust%20language%20and%20compiler,these%20tiny%20changes%20add%20up.)
+release cycle and they only support the latest stable release, so if we wanted to remain on a
+supported release we would only ever build with the latest stable and bump every 6 weeks.
+However feedback from our community has indicated that this is a challenge as downstream consumers
+often want to get rust from their distro, or downstream fork and these struggle to keep up with
+the six week release schedule. As a result the community has agreed to try out a policy of
+"stable-2", where we aim to build with a rust version that is two versions behind the latest stable
+version.
+
+In practice this should mean that we bump our rust toolchain every six weeks, to version
+1.x-2 when 1.x is released as stable and we should be picking up the latest point release
+of that version, if there were any.
+
+The rust-toolchain that we are using is recorded in [`rust-toolchain.toml`](../rust-toolchain.toml).

tests/integration/kubernetes/k8s-attach-handlers.bats

@@ -9,35 +9,33 @@ load "${BATS_TEST_DIRNAME}/../../common.bash"
 load "${BATS_TEST_DIRNAME}/tests_common.sh"
 
 setup() {
-	nginx_version="${docker_images_nginx_version}"
-	nginx_image="nginx:$nginx_version"
-
 	pod_name="handlers"
 
 	get_pod_config_dir
 	yaml_file="${pod_config_dir}/test-lifecycle-events.yaml"
 
 	# Create yaml
-	sed -e "s/\${nginx_version}/${nginx_image}/" \
-		"${pod_config_dir}/lifecycle-events.yaml" > "${yaml_file}"
+	set_nginx_image "${pod_config_dir}/lifecycle-events.yaml" "${yaml_file}"
 
 	# Add policy to yaml
 	policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
-	
+
 	display_message="cat /usr/share/message"
 	exec_command=(sh -c "${display_message}")
 	add_exec_to_policy_settings "${policy_settings_dir}" "${exec_command[@]}"
-	
+
 	add_requests_to_policy_settings "${policy_settings_dir}" "ReadStreamRequest"
 	auto_generate_policy "${policy_settings_dir}" "${yaml_file}"
 }
 
 @test "Running with postStart and preStop handlers" {
 	# Create the pod with postStart and preStop handlers
-	kubectl create -f "${yaml_file}"
+	# kubectl create -f "${yaml_file}"
 
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check postStart message
 	check_postStart=$(kubectl exec $pod_name -- "${exec_command[@]}")

tests/integration/kubernetes/k8s-block-volume.bats

@@ -82,8 +82,9 @@ setup() {
 	auto_generate_policy "${policy_settings_dir}" "${tmp_pod_yaml}"
 
 	# Start the workload.
-	kubectl create -f "$tmp_pod_yaml"
-	kubectl wait --for condition=ready --timeout=$timeout "pod/${pod_name}"
+	# kubectl create -f "$tmp_pod_yaml"
+	# kubectl wait --for condition=ready --timeout=$timeout "pod/${pod_name}"
+	k8s_create_pod_ready "${pod_name}" "$tmp_pod_yaml"
 
 	# Verify persistent volume claim is bound
 	kubectl get "pvc/${volume_claim}" | grep "Bound"

tests/integration/kubernetes/k8s-caps.bats

@@ -40,9 +40,10 @@ setup() {
 
 @test "Check capabilities of pod" {
         # Create pod
-        kubectl create -f "${yaml_file}"
+        # kubectl create -f "${yaml_file}"
         # Check pod creation
-        kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+        # kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+        k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
         # Verify expected capabilities for the running container. Add retry to ensure
         # that the container had time to execute:

tests/integration/kubernetes/k8s-configmap.bats

@@ -34,10 +34,10 @@ setup() {
 	kubectl get configmaps $config_name -o yaml | grep -q "data-"
 
 	# Create a pod that consumes the ConfigMap
-	kubectl create -f "${pod_yaml_file}"
-
+	# kubectl create -f "${pod_yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	k8s_create_pod_ready "${pod_name}" "${pod_yaml_file}"
 
 	# Check env
 	grep_pod_exec_output "${pod_name}" "KUBE_CONFIG_1=value-1" "${exec_command[@]}"

tests/integration/kubernetes/k8s-copy-file.bats

@@ -41,10 +41,11 @@ setup() {
 	auto_generate_policy "${policy_settings_dir}" "${pod_config}"
 
 	# Create pod
-	kubectl create -f "${pod_config}"
-
+	# kubectl create -f "${pod_config}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	#kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${pod_config}"
 
 	# Create a file
 	echo "$content" > "$file_name"
@@ -78,10 +79,11 @@ setup() {
 	auto_generate_policy "${policy_settings_dir}" "${pod_config}"
 
 	# Create pod
-	kubectl create -f "${pod_config}"
-
+	# kubectl create -f "${pod_config}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${pod_config}"
 
 	kubectl logs "$pod_name" || true
 	kubectl describe pod "$pod_name" || true

tests/integration/kubernetes/k8s-exec.bats

@@ -38,10 +38,11 @@ setup() {
 
 @test "Kubectl exec" {
 	# Create the pod
-	kubectl create -f "${test_yaml_file}"
-
+	# kubectl create -f "${test_yaml_file}"
 	# Get pod specification
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${test_yaml_file}"
 
 	# Run commands in Pod
 	## Cases for -it options

tests/integration/kubernetes/k8s-file-volume.bats

@@ -47,10 +47,11 @@ setup() {
 
 @test "Test readonly volume for pods" {
 	# Create pod
-	kubectl create -f "${test_yaml}"
-
+	# kubectl create -f "${test_yaml}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${test_yaml}"
 
 	# Validate file volume body inside the pod
 	file_in_container=$(kubectl exec $pod_name -- "${command[@]}")

tests/integration/kubernetes/k8s-footloose.bats

@@ -37,10 +37,11 @@ setup() {
 	kubectl create -f "$configmap_yaml"
 
 	# Create pod
-	kubectl create -f "${pod_yaml}"
-
+	# kubectl create -f "${pod_yaml}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${pod_yaml}"
 
 	# Get pod ip
 	pod_ip=$(kubectl get pod $pod_name --template={{.status.podIP}})

tests/integration/kubernetes/k8s-guest-pull-image-authenticated.bats

@@ -88,7 +88,11 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
+
     echo "Kata pod test-e2e from authenticated image is running"
 }
 
@@ -132,7 +136,11 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
+
     echo "Kata pod test-e2e from authenticated image is running"
 }
 

tests/integration/kubernetes/k8s-guest-pull-image-encrypted.bats

@@ -65,7 +65,11 @@ function setup_kbs_decryption_key() {
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
+
     echo "Kata pod test-e2e from encrypted image is running"
 }
 

tests/integration/kubernetes/k8s-guest-pull-image-signature.bats

@@ -98,7 +98,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 
@@ -124,7 +127,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 
@@ -150,7 +156,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 
@@ -167,7 +176,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 
@@ -201,7 +213,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 
@@ -234,7 +249,10 @@ EOF
     # For debug sake
     echo "Pod ${kata_pod}: $(cat ${kata_pod})"
 
-    k8s_create_pod "${kata_pod}"
+    # Default wait timeout is 120
+    local wait_time=120
+    [[ "${KATA_HYPERVISOR}" == qemu-coco-dev-runtime-rs ]] && wait_time=300
+    k8s_create_pod "${kata_pod}" "$wait_time"
     echo "Kata pod test-e2e from image security policy is running"
 }
 

tests/integration/kubernetes/k8s-hostpath-volume.bats

@@ -29,8 +29,9 @@ setup() {
 }
 
 @test "/dev hostPath volume bind mounts the guest device and skips virtio-fs" {
-	kubectl apply -f "${yaml_file}"
-	kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}"
+	# kubectl apply -f "${yaml_file}"
+	# kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}"
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check the mount info.
 

tests/integration/kubernetes/k8s-kill-all-process-in-container.bats

@@ -26,10 +26,11 @@ setup() {
 
 @test "Kill all processes in container" {
 	# Create the pod
-	kubectl create -f "${yaml_file}"
-
+	# kubectl create -f "${yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check PID from first container
 	first_pid_container=$(kubectl exec $pod_name -c $first_container_name \

tests/integration/kubernetes/k8s-liveness-probes.bats

@@ -27,10 +27,11 @@ setup() {
 	auto_generate_policy "${pod_config_dir}" "${yaml_file}"
 
 	# Create pod
-	kubectl create -f "${yaml_file}"
-
+	# kubectl create -f "${yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check liveness probe returns a success code
 	kubectl describe pod "$pod_name" | grep -E "Liveness|#success=1"
@@ -52,10 +53,11 @@ setup() {
 	auto_generate_policy "${pod_config_dir}" "${yaml_file}"
 
 	# Create pod
-	kubectl create -f "${yaml_file}"
-
+	# kubectl create -f "${yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check liveness probe returns a success code
 	kubectl describe pod "$pod_name" | grep -E "Liveness|#success=1"
@@ -78,10 +80,11 @@ setup() {
 	auto_generate_policy "${pod_config_dir}" "${yaml_file}"
 
 	# Create pod
-	kubectl create -f "${yaml_file}"
-
+	# kubectl create -f "${yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${yaml_file}"
 
 	# Check liveness probe returns a success code
 	kubectl describe pod "$pod_name" | grep -E "Liveness|#success=1"

tests/integration/kubernetes/k8s-nginx-connectivity.bats

@@ -11,8 +11,6 @@ load "${BATS_TEST_DIRNAME}/tests_common.sh"
 setup() {
 	[ "${CONTAINER_RUNTIME}" == "crio" ] && skip "test not working see: https://github.com/kata-containers/kata-containers/issues/10414"
 
-	nginx_version="${docker_images_nginx_version}"
-	nginx_image="nginx:$nginx_version"
 	busybox_image="quay.io/prometheus/busybox:latest"
 	deployment="nginx-deployment"
 
@@ -20,18 +18,18 @@ setup() {
 
 	# Create test .yaml
 	yaml_file="${pod_config_dir}/test-${deployment}.yaml"
+	set_nginx_image "${pod_config_dir}/${deployment}.yaml" "${yaml_file}"
 
-	sed -e "s/\${nginx_version}/${nginx_image}/" \
-		"${pod_config_dir}/${deployment}.yaml" > "${yaml_file}"
-	
 	auto_generate_policy "${pod_config_dir}" "${yaml_file}"
 }
 
 @test "Verify nginx connectivity between pods" {
 
-	kubectl create -f "${yaml_file}"
-	kubectl wait --for=condition=Available --timeout=$timeout deployment/${deployment}
-	kubectl expose deployment/${deployment}
+	# kubectl create -f "${yaml_file}"
+	# kubectl wait --for=condition=Available --timeout=$timeout deployment/${deployment}
+	# Retries
+	k8s_create_deployment_ready "${yaml_file}" ${deployment}
+	kubectl expose deployment/${deployment} ${deployment}
 
 	busybox_pod="test-nginx"
 	kubectl run $busybox_pod --restart=Never -it --image="$busybox_image" \

tests/integration/kubernetes/k8s-nvidia-cuda.bats

@@ -46,10 +46,10 @@ setup() {
     kubectl apply -f "${pod_yaml}"
 
     # Wait for pod to complete successfully (with retry)
-    kubectl_retry 10 30 wait --for=jsonpath='{.status.phase}'=Succeeded --timeout="${POD_WAIT_TIMEOUT}" pod "${POD_NAME_CUDA}"
+    kubectl wait --for=jsonpath='{.status.phase}'=Succeeded --timeout="${POD_WAIT_TIMEOUT}" pod "${POD_NAME_CUDA}"
 
     # Get and verify the output contains expected CUDA success message
-    kubectl_retry 10 30 logs "${POD_NAME_CUDA}"
+    kubectl logs "${POD_NAME_CUDA}"
     output=$(kubectl logs "${POD_NAME_CUDA}")
     echo "# CUDA Vector Add Output: ${output}" >&3
 

tests/integration/kubernetes/k8s-nvidia-nim.bats

@@ -82,10 +82,10 @@ create_inference_pod() {
     add_allow_all_policy_to_yaml "${POD_INSTRUCT_YAML}"
 
     kubectl apply -f "${POD_INSTRUCT_YAML}"
-    kubectl_retry 10 30 wait --for=condition=Ready --timeout="${POD_READY_TIMEOUT_INSTRUCT}" pod "${POD_NAME_INSTRUCT}"
+    kubectl wait --for=condition=Ready --timeout="${POD_READY_TIMEOUT_INSTRUCT}" pod "${POD_NAME_INSTRUCT}"
 
     # shellcheck disable=SC2030  # Variable is shared via file between BATS tests
-    kubectl_retry 10 30 get pod "${POD_NAME_INSTRUCT}" -o jsonpath='{.status.podIP}'
+    kubectl get pod "${POD_NAME_INSTRUCT}" -o jsonpath='{.status.podIP}'
     POD_IP_INSTRUCT=$(kubectl get pod "${POD_NAME_INSTRUCT}" -o jsonpath='{.status.podIP}')
     [[ -n "${POD_IP_INSTRUCT}" ]]
 
@@ -98,10 +98,10 @@ create_embedqa_pod() {
     add_allow_all_policy_to_yaml "${POD_EMBEDQA_YAML}"
 
     kubectl apply -f "${POD_EMBEDQA_YAML}"
-    kubectl_retry 10 30 wait --for=condition=Ready --timeout="${POD_READY_TIMEOUT_EMBEDQA}" pod "${POD_NAME_EMBEDQA}"
+    kubectl wait --for=condition=Ready --timeout="${POD_READY_TIMEOUT_EMBEDQA}" pod "${POD_NAME_EMBEDQA}"
 
     # shellcheck disable=SC2030  # Variable is shared via file between BATS tests
-    kubectl_retry 10 30 get pod "${POD_NAME_EMBEDQA}" -o jsonpath='{.status.podIP}'
+    kubectl get pod "${POD_NAME_EMBEDQA}" -o jsonpath='{.status.podIP}'
     POD_IP_EMBEDQA=$(kubectl get pod "${POD_NAME_EMBEDQA}" -o jsonpath='{.status.podIP}')
 
     [[ -n "${POD_IP_EMBEDQA}" ]]

tests/integration/kubernetes/k8s-oom.bats

@@ -18,10 +18,11 @@ setup() {
 
 @test "Test OOM events for pods" {
 	# Create pod
-	kubectl create -f "${yaml_file}"
-
+	# kubectl create -f "${yaml_file}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	#kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retries
+	k8s_create_pod_ready "${pod_name}" "${test_yaml}"
 
 	# Check if OOMKilled
     container_name=$(kubectl get pod "$pod_name" -o jsonpath='{.status.containerStatuses[0].name}')

tests/integration/kubernetes/k8s-optional-empty-configmap.bats

@@ -26,6 +26,42 @@ setup() {
 	auto_generate_policy "${policy_settings_dir}" "${pod_yaml}"
 }
 
+k8s_create_pod_ready() {
+	local pod_name="$1"
+	local pod_yaml="$2"
+
+	local wait_time=300
+	local max_attempts=5
+	local attempt_num
+
+	for attempt_num in $(seq 1 "${max_attempts}"); do
+		# First,forcefully deleting resources
+		kubectl delete -f "${pod_yaml}" --ignore-not-found=true --now --timeout=$wait_time
+
+		kubectl create -f "${pod_yaml}"
+		if [ $? -ne 0 ]; then
+			# Failed to create Pod.Aborting test.
+			continue
+		fi
+
+		# Check pod creation
+		kubectl wait --for=condition=Ready --timeout=$wait_time pod "$pod_name"
+		if [ "$status" -eq 0 ]; then
+			# Test Succeeded on attempt #${attempt_num}
+			return 0
+		fi
+
+		# Retry
+		if [ "${attempt_num}" -lt "${max_attempts}" ]; then
+			info "Waiting for 5 seconds before next attempt..."
+			sleep 5
+		fi
+	done
+
+	#Test Failed after ${max_attempts} attempts.
+	return 1
+}
+
 @test "Optional and Empty ConfigMap Volume for a pod" {
 	config_name="empty-config"
 	pod_name="optional-empty-config-test-pod"
@@ -34,10 +70,11 @@ setup() {
 	kubectl create configmap "$config_name"
 
 	# Create a pod that consumes the "empty-config" and "optional-missing-config" ConfigMaps as volumes
-	kubectl create -f "${pod_yaml}"
-
+	# kubectl create -f "${pod_yaml}"
 	# Check pod creation
-	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+	# Retry for ready pod
+	k8s_create_pod_ready "$pod_name" "${pod_yaml}"
 
 	# Check configmap folders exist
 	kubectl exec $pod_name -- "${exec_empty_command[@]}"

tests/integration/kubernetes/k8s-policy-rc.bats

@@ -22,11 +22,7 @@ setup() {
     # Save some time by executing genpolicy a single time.
     if [ "${BATS_TEST_NUMBER}" == "1" ]; then
         # Create the correct yaml file
-        nginx_version="${docker_images_nginx_version}"
-        nginx_image="nginx:$nginx_version"
-
-        sed -e "s/\${nginx_version}/${nginx_image}/" \
-            "${pod_config_dir}/k8s-policy-rc.yaml" > "${correct_yaml}"
+        set_nginx_image "${pod_config_dir}/k8s-policy-rc.yaml" "${correct_yaml}"
 
         # Add policy to the correct yaml file
         policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"

tests/integration/kubernetes/k8s-replication.bats

@@ -9,15 +9,11 @@ load "${BATS_TEST_DIRNAME}/../../common.bash"
 load "${BATS_TEST_DIRNAME}/tests_common.sh"
 
 setup() {
-	nginx_version="${docker_images_nginx_version}"
-	nginx_image="nginx:$nginx_version"
-
 	get_pod_config_dir
 
 	# Create yaml
 	test_yaml="${pod_config_dir}/test-replication-controller.yaml"
-	sed -e "s/\${nginx_version}/${nginx_image}/" \
-		"${pod_config_dir}/replication-controller.yaml" > "${test_yaml}"
+	set_nginx_image "${pod_config_dir}/replication-controller.yaml" "${test_yaml}"
 
 	# Add policy to the yaml file
 	policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"

tests/integration/kubernetes/k8s-scale-nginx.bats

@@ -9,16 +9,13 @@ load "${BATS_TEST_DIRNAME}/../../common.bash"
 load "${BATS_TEST_DIRNAME}/tests_common.sh"
 
 setup() {
-	nginx_version="${docker_images_nginx_version}"
-	nginx_image="nginx:$nginx_version"
 	replicas="3"
 	deployment="nginx-deployment"
 	get_pod_config_dir
 
 	# Create the yaml file
 	test_yaml="${pod_config_dir}/test-${deployment}.yaml"
-	sed -e "s/\${nginx_version}/${nginx_image}/" \
-		"${pod_config_dir}/${deployment}.yaml" > "${test_yaml}"
+	set_nginx_image "${pod_config_dir}/${deployment}.yaml" "${test_yaml}"
 
 	# Add policy to the yaml file
 	policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
@@ -27,8 +24,11 @@ setup() {
 }
 
 @test "Scale nginx deployment" {
-	kubectl create -f "${test_yaml}"
-	kubectl wait --for=condition=Available --timeout=$timeout deployment/${deployment}
+	# kubectl create -f "${test_yaml}"
+	# kubectl wait --for=condition=Available --timeout=$timeout deployment/${deployment}
+	# Retries
+	k8s_create_deployment_ready "${test_yaml}" ${deployment}
+
 	kubectl expose deployment/${deployment}
 	kubectl scale deployment/${deployment} --replicas=${replicas}
 	cmd="kubectl get deployment/${deployment} -o yaml | grep 'availableReplicas: ${replicas}'"

tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-rc.yaml

@@ -21,7 +21,7 @@ spec:
       runtimeClassName: kata
       containers:
       - name: nginxtest
-        image: quay.io/fidencio/${nginx_version}
+        image: ${NGINX_IMAGE}
         ports:
         - containerPort: 80
         volumeMounts:
@@ -43,4 +43,4 @@ spec:
           path: /tmp/results
       - name: hostpath-vol-read-only
         hostPath:
-          path: /tmp/results-read-only
+          path: /tmp/results-read-only

tests/integration/kubernetes/runtimeclass_workloads/lifecycle-events.yaml

@@ -13,7 +13,8 @@ spec:
   runtimeClassName: kata
   containers:
   - name: handlers-container
-    image: quay.io/fidencio/${nginx_version}
+    image: ${NGINX_IMAGE}
+    imagePullPolicy: Always
     lifecycle:
       postStart:
         exec:

tests/integration/kubernetes/runtimeclass_workloads/nginx-deployment.yaml

@@ -21,6 +21,7 @@ spec:
       runtimeClassName: kata
       containers:
       - name: nginx
-        image: quay.io/fidencio/${nginx_version}
+        image: ${NGINX_IMAGE}
+        imagePullPolicy: Always
         ports:
         - containerPort: 80

tests/integration/kubernetes/runtimeclass_workloads/replication-controller.yaml

@@ -21,6 +21,6 @@ spec:
       runtimeClassName: kata
       containers:
       - name: nginxtest
-        image: quay.io/fidencio/${nginx_version}
+        image: ${NGINX_IMAGE}
         ports:
         - containerPort: 80

tests/integration/kubernetes/tests_common.sh

@@ -11,7 +11,6 @@
 # This contains variables and functions common to all e2e tests.
 
 # Variables used by the kubernetes tests
-export docker_images_nginx_version="1.15-alpine"
 export container_images_agnhost_name="registry.k8s.io/e2e-test-images/agnhost"
 export container_images_agnhost_version="2.21"
 
@@ -421,6 +420,80 @@ wait_for_blocked_request() {
 	waitForProcess "${wait_time}" "${sleep_time}" "${command}" >/dev/null 2>/dev/null
 }
 
+# k8s create a ready pod
+k8s_create_pod_ready() {
+	local pod_name="$1"
+	local pod_yaml="$2"
+	local wait_time="${3:-300}"
+	local max_attempts="${4:-5}"
+
+	local attempt_num
+
+	for attempt_num in $(seq 1 "${max_attempts}"); do
+		# First,forcefully deleting resources
+		kubectl delete -f "${pod_yaml}" --ignore-not-found=true --now --timeout=$wait_time
+
+		kubectl create -f "${pod_yaml}"
+		if [ $? -ne 0 ]; then
+			# Failed to create Pod.Aborting test.
+			continue
+		fi
+
+		# Check pod creation
+		kubectl wait --for=condition=Ready --timeout=$wait_time pod "$pod_name"
+		if [ "$status" -eq 0 ]; then
+			# Test Succeeded on attempt #${attempt_num}
+			info "Waiting ${wait_time} seconds for pod ${pod_name} Ready."
+			return 0
+		fi
+
+		# Retry
+		if [ "${attempt_num}" -lt "${max_attempts}" ]; then
+			info "Waiting for 5 seconds before next attempt..."
+			sleep 5
+		fi
+	done
+
+	info "Test Failed after ${max_attempts} attempts for pod ${pod_name}."
+	return 1
+}
+
+k8s_create_deployment_ready() {
+	local deployment_yaml="$1"
+	local deployment="$2"
+
+	local wait_time=300
+	local max_attempts=5
+	local attempt_num
+
+	for attempt_num in $(seq 1 "${max_attempts}"); do
+		# First,forcefully deleting resources
+		kubectl delete -f "${deployment_yaml}" --ignore-not-found=true --now --timeout=$wait_time
+
+		kubectl create -f "${deployment_yaml}"
+		if [ $? -ne 0 ]; then
+			# Failed to create Pod.Aborting test.
+			continue
+		fi
+
+		# Check deployment ready
+		kubectl wait --for=condition=Available --timeout=$timeout deployment/${deployment}
+		if [ "$status" -eq 0 ]; then
+			# Test Succeeded on attempt #${attempt_num}
+			return 0
+		fi
+
+		# Retry
+		if [ "${attempt_num}" -lt "${max_attempts}" ]; then
+			info "Waiting for 5 seconds before next attempt..."
+			sleep 5
+		fi
+	done
+
+	#Test Failed after ${max_attempts} attempts.
+	return 1
+}
+
 # Execute in a pod a command that is allowed by policy.
 pod_exec_allowed_command() {
 	local -r pod_name="$1"
@@ -557,3 +630,15 @@ container_exec_with_retries() {
 
 	echo "${cmd_out}"
 }
+
+set_nginx_image() {
+	input_yaml=$1
+	output_yaml=$2
+
+	ensure_yq
+	nginx_registry=$(get_from_kata_deps ".docker_images.nginx.registry")
+	nginx_digest=$(get_from_kata_deps ".docker_images.nginx.digest")
+	nginx_image="${nginx_registry}@${nginx_digest}"
+
+	NGINX_IMAGE="${nginx_image}" envsubst < "${input_yaml}" > "${output_yaml}"
+}

tests/stability/soak_parallel_rm.sh

@@ -173,8 +173,9 @@ function init() {
 	fi
 
 	versions_file="${cidir}/../../versions.yaml"
-	nginx_version=$("${GOPATH}/bin/yq" ".docker_images.nginx.version" "$versions_file")
-	nginx_image="docker.io/library/nginx:$nginx_version"
+	nginx_registry=$("${GOPATH}/bin/yq" ".docker_images.nginx.registry" "${versions_file}")
+	nginx_digest=$("${GOPATH}/bin/yq" ".docker_images.nginx.digest" "${versions_file}")
+	nginx_image="${nginx_registry}@${nginx_digest}"
 
 	# Pull nginx image
 	sudo "${CTR_EXE}" image pull "${nginx_image}"

tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh

@@ -1187,6 +1187,13 @@ install_tools_helper() {
 	[ ${tool} = "trace-forwarder" ] && tool_binary="kata-trace-forwarder"
 	binary=$(find ${repo_root_dir}/src/tools/${tool}/ -type f -name ${tool_binary})
 
+	binary_count=$(echo "${binary}" | grep -c '^' || echo "0")
+	if [[ "${binary_count}" -eq 0 ]]; then
+		die "No binary found for ${tool} (expected: ${tool_binary})."
+	elif [[ "${binary_count}" -gt 1 ]]; then
+		die "Multiple binaries found for ${tool} (expected single ${tool_binary}). Found:"$'\n'"${binary}"
+	fi
+
 	if [[ "${tool}" == "genpolicy" ]]; then
 		defaults_path="${destdir}/opt/kata/share/defaults/kata-containers"
 		mkdir -p "${defaults_path}"

versions.yaml

@@ -475,9 +475,10 @@ plugins:
     version: "b7f6d3e0679796e907ecca88cfab0e32e326850d"
 
 docker_images:
-  description: "Docker hub images used for testing"
+  description: "Docker images used for testing"
 
   nginx:
     description: "Proxy server for HTTP, HTTPS, SMTP, POP3 and IMAP protocols"
-    url: "https://hub.docker.com/_/nginx/"
-    version: "1.15-alpine"
+    registry: "quay.io/kata-containers/nginx"
+    # yamllint disable-line rule:line-length
+    digest: "sha256:a905609e0f9adc2607f06da2f76893c6da07caa396c41f2806fee162064cfb4b" # 1.15-alpine