github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-08-08 03:24:15 +00:00
Code Issues Projects Releases Wiki Activity

ci.ocp: Wait for all pods to finish the work

Browse Source 
previously we only waited for a random pod to finish the selinux
relabel, which could be error-prone. Let's wait for all of the podst to
contain the expected message.

Increase the timeout to 120s as some pods might take a little bit longer
to finish.

Signed-off-by: Lukáš Doktor <ldoktor@redhat.com>
This commit is contained in:
Lukáš Doktor 2024-03-06 11:01:45 +01:00
parent f7febd07a0
commit 76c452d4e0
No known key found for this signature in database
GPG Key ID: 26B362E47FCF22C1
1 changed files with 43 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
ci/openshift-ci/cluster/install_kata.sh
View File

@ -147,6 +147,48 @@ debug_pod() {
oc logs "$pod" oc logs "$pod"
} }
# Wait for all pods of the app label to contain expected message
#
# Params:
# $1 - app labela
# $2 - expected pods count (>=1)
# $3 - message to be present in the logs
# $4 - timeout (60)
# $5 - namespace (the current one)
wait_for_app_pods_message() {
local app="$1"
local pod_count="$2"
local message="$3"
local timeout="$4"
local namespace="$5"
[ -z "$pod_count" ] && pod_count=1
[ -z "$timeout" ] && timeout=60
[ -n "$namespace" ] && namespace=" -n $namespace "
local pod
local pods
local i
SECONDS=0
while :; do
pods=($(oc get pods -l app="$app" --no-headers=true $namespace | awk '{print $1}'))
[ "${#pods}" -ge "$pod_count" ] && break
if [ "$SECONDS" -gt "$timeout" ]; then
echo "Unable to find ${pod_count} pods for '-l app=\"$app\"' in ${SECONDS}s (${pods[@]})"
return -1
fi
done
for pod in "${pods[@]}"; do
while :; do
oc logs $namespace "$pod" | grep "$message" -q && echo "Found $message in $pod's log ($SECONDS)" && break;
if [ "$SECONDS" -gt "$timeout" ]; then
echo -n "Message '$message' not present in '${pod}' pod of the '-l app=\"$app\"' "
echo "pods after ${SECONDS}s (${pods[@]})"
return -1
fi
sleep 1;
done
done
}
oc config set-context --current --namespace=default oc config set-context --current --namespace=default
worker_nodes=$(oc get nodes | awk '{if ($3 == "worker") { print $1 } }') worker_nodes=$(oc get nodes | awk '{if ($3 == "worker") { print $1 } }')
@ -185,7 +227,4 @@ fi
# FIXME: Remove when https://github.com/kata-containers/kata-containers/pull/8417 is resolved # FIXME: Remove when https://github.com/kata-containers/kata-containers/pull/8417 is resolved
# Selinux context is currently not handled by kata-deploy # Selinux context is currently not handled by kata-deploy
oc apply -f ${deployments_dir}/relabel_selinux.yaml oc apply -f ${deployments_dir}/relabel_selinux.yaml
( for I in $(seq 30); do wait_for_app_pods_message restorecon "$num_nodes" "NSENTER_FINISHED_WITH:" 120 "kube-system" || { echo "Selinux relabel failed, check the logs"; exit -1; }
sleep 10
oc logs -n kube-system ds/relabel-selinux-daemonset | grep "NSENTER_FINISHED_WITH:" && exit
done ) || { echo "Selinux relabel failed, check the logs"; exit -1; }