diff --git a/.github/workflows/run-kata-coco-tests.yaml b/.github/workflows/run-kata-coco-tests.yaml index 3f129a3eaa..c4a173eb87 100644 --- a/.github/workflows/run-kata-coco-tests.yaml +++ b/.github/workflows/run-kata-coco-tests.yaml @@ -303,6 +303,10 @@ jobs: timeout-minutes: 80 run: bash tests/integration/kubernetes/gha-run.sh run-tests + - name: Report tests + if: always() + run: bash tests/integration/kubernetes/gha-run.sh report-tests + - name: Delete AKS cluster if: always() run: bash tests/integration/kubernetes/gha-run.sh delete-cluster diff --git a/tests/integration/kubernetes/.gitignore b/tests/integration/kubernetes/.gitignore new file mode 100644 index 0000000000..a9a1bd38ab --- /dev/null +++ b/tests/integration/kubernetes/.gitignore @@ -0,0 +1 @@ +reports/ diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index cbdff162e8..7149545ee5 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -294,6 +294,49 @@ function run_tests() { popd } +# Print a report about tests executed. +# +# Crawl over the output files found on each "reports/yyyy-mm-dd-hh:mm:ss" +# directory. +# +function report_tests() { + local reports_dir="${kubernetes_dir}/reports" + local ok + local not_ok + local status + + if [[ ! -d "${reports_dir}" ]]; then + info "no reports directory found: ${reports_dir}" + return + fi + + for report_dir in "${reports_dir}"/*; do + mapfile -t ok < <(find "${report_dir}" -name "ok-*.out") + mapfile -t not_ok < <(find "${report_dir}" -name "not_ok-*.out") + + cat <<-EOF + SUMMARY ($(basename "${report_dir}")): + Pass: ${#ok[*]} + Fail: ${#not_ok[*]} + EOF + + echo -e "\nSTATUSES:" + for out in "${not_ok[@]}" "${ok[@]}"; do + status=$(basename "${out}" | cut -d '-' -f1) + bats=$(basename "${out}" | cut -d '-' -f2- | sed 's/.out$//') + echo " ${status} ${bats}" + done + + echo -e "\nOUTPUTS:" + for out in "${not_ok[@]}" "${ok[@]}"; do + bats=$(basename "${out}" | cut -d '-' -f2- | sed 's/.out$//') + echo "::group::${bats}" + cat "${out}" + echo "::endgroup::" + done + done +} + function collect_artifacts() { if [[ -z "${start_time:-}" ]]; then warn "tests start time is not defined. Cannot gather journal information" @@ -547,6 +590,7 @@ function main() { deploy-kata-garm) deploy_kata "garm" ;; deploy-kata-zvsi) deploy_kata "zvsi" ;; deploy-snapshotter) deploy_snapshotter ;; + report-tests) report_tests ;; run-tests) run_tests ;; run-tests-kcli) run_tests "kcli" ;; collect-artifacts) collect_artifacts ;; diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 9baeb8194b..162bd4808a 100755 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -6,6 +6,7 @@ # set -e +set -o pipefail kubernetes_dir=$(dirname "$(readlink -f "$0")") source "${kubernetes_dir}/../../common.bash" @@ -131,7 +132,10 @@ fi ensure_yq -info "Running tests with bats version: $(bats --version)" +report_dir="${kubernetes_dir}/reports/$(date +'%F-%T')" +mkdir -p "${report_dir}" + +info "Running tests with bats version: $(bats --version). Save outputs to ${report_dir}" tests_fail=() for K8S_TEST_ENTRY in "${K8S_TEST_UNION[@]}" @@ -139,9 +143,14 @@ do K8S_TEST_ENTRY=$(echo "$K8S_TEST_ENTRY" | tr -d '[:space:][:cntrl:]') info "$(kubectl get pods --all-namespaces 2>&1)" info "Executing ${K8S_TEST_ENTRY}" - if ! bats --show-output-of-passing-tests "${K8S_TEST_ENTRY}"; then + # Output file will be prefixed with "ok" or "not_ok" based on the result + out_file="${report_dir}/${K8S_TEST_ENTRY}.out" + if ! bats --show-output-of-passing-tests "${K8S_TEST_ENTRY}" | tee "${out_file}"; then tests_fail+=("${K8S_TEST_ENTRY}") + mv "${out_file}" "$(dirname "${out_file}")/not_ok-$(basename "${out_file}")" [ "${K8S_TEST_FAIL_FAST}" = "yes" ] && break + else + mv "${out_file}" "$(dirname "${out_file}")/ok-$(basename "${out_file}")" fi done