Merge pull request #11130 from wainersm/tests-better-report

tests/k8s: better tests reporting for CI

.github/workflows/run-kata-coco-tests.yaml

@@ -303,6 +303,10 @@ jobs:
         timeout-minutes: 80
         run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
+      - name: Report tests
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh report-tests
+
       - name: Delete AKS cluster
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh delete-cluster

tests/integration/kubernetes/.gitignore

@@ -0,0 +1 @@
+reports/

tests/integration/kubernetes/gha-run.sh

@@ -294,6 +294,49 @@ function run_tests() {
 	popd
 }
 
+# Print a report about tests executed.
+#
+# Crawl over the output files found on each "reports/yyyy-mm-dd-hh:mm:ss"
+# directory.
+#
+function report_tests() {
+	local reports_dir="${kubernetes_dir}/reports"
+	local ok
+	local not_ok
+	local status
+
+	if [[ ! -d "${reports_dir}" ]]; then
+		info "no reports directory found: ${reports_dir}"
+		return
+	fi
+
+	for report_dir in "${reports_dir}"/*; do
+		mapfile -t ok < <(find "${report_dir}" -name "ok-*.out")
+		mapfile -t not_ok < <(find "${report_dir}" -name "not_ok-*.out")
+
+		cat <<-EOF
+		SUMMARY ($(basename "${report_dir}")):
+		 Pass:  ${#ok[*]}
+		 Fail:  ${#not_ok[*]}
+		EOF
+
+		echo -e "\nSTATUSES:"
+		for out in "${not_ok[@]}" "${ok[@]}"; do
+			status=$(basename "${out}" | cut -d '-' -f1)
+			bats=$(basename "${out}" | cut -d '-' -f2- | sed 's/.out$//')
+			echo " ${status} ${bats}"
+		done
+
+		echo -e "\nOUTPUTS:"
+		for out in "${not_ok[@]}" "${ok[@]}"; do
+			bats=$(basename "${out}" | cut -d '-' -f2- | sed 's/.out$//')
+			echo "::group::${bats}"
+			cat "${out}"
+			echo "::endgroup::"
+		done
+	done
+}
+
 function collect_artifacts() {
 	if [[ -z "${start_time:-}" ]]; then
 		warn "tests start time is not defined. Cannot gather journal information"
@@ -547,6 +590,7 @@ function main() {
 		deploy-kata-garm) deploy_kata "garm" ;;
 		deploy-kata-zvsi) deploy_kata "zvsi" ;;
 		deploy-snapshotter) deploy_snapshotter ;;
+		report-tests) report_tests ;;
 		run-tests) run_tests ;;
 		run-tests-kcli) run_tests "kcli" ;;
 		collect-artifacts) collect_artifacts ;;

tests/integration/kubernetes/run_kubernetes_tests.sh

@@ -6,6 +6,7 @@
 #
 
 set -e
+set -o pipefail
 
 kubernetes_dir=$(dirname "$(readlink -f "$0")")
 source "${kubernetes_dir}/../../common.bash"
@@ -131,7 +132,10 @@ fi
 
 ensure_yq
 
-info "Running tests with bats version: $(bats --version)"
+report_dir="${kubernetes_dir}/reports/$(date +'%F-%T')"
+mkdir -p "${report_dir}"
+
+info "Running tests with bats version: $(bats --version). Save outputs to ${report_dir}"
 
 tests_fail=()
 for K8S_TEST_ENTRY in "${K8S_TEST_UNION[@]}"
@@ -139,9 +143,14 @@ do
 	K8S_TEST_ENTRY=$(echo "$K8S_TEST_ENTRY" | tr -d '[:space:][:cntrl:]')
 	info "$(kubectl get pods --all-namespaces 2>&1)"
 	info "Executing ${K8S_TEST_ENTRY}"
-	if ! bats --show-output-of-passing-tests "${K8S_TEST_ENTRY}"; then
+	# Output file will be prefixed with "ok" or "not_ok" based on the result
+	out_file="${report_dir}/${K8S_TEST_ENTRY}.out"
+	if ! bats --show-output-of-passing-tests "${K8S_TEST_ENTRY}" | tee "${out_file}"; then
 		tests_fail+=("${K8S_TEST_ENTRY}")
+		mv "${out_file}" "$(dirname "${out_file}")/not_ok-$(basename "${out_file}")"
 		[ "${K8S_TEST_FAIL_FAST}" = "yes" ] && break
+	else
+		mv "${out_file}" "$(dirname "${out_file}")/ok-$(basename "${out_file}")"
 	fi
 done