From 5e3b8a019a1b586a4d86b1097d3535f1bc29500d Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 14 Feb 2025 19:51:04 -0300 Subject: [PATCH 1/3] tests/k8s: split and save bats outputs in files Currently run_kubernetes_tests.sh sends all the bats outputs to stdout which can be very difficult to browse to find a problem, mainly on CI. With this change, each bats execution have its output sent to 'reports/yyy-mm-dd-hh:mm:ss/-.log' where is either 'ok' (tests passed) or 'not_ok' (some tests failed). Signed-off-by: Wainer dos Santos Moschetta --- tests/integration/kubernetes/.gitignore | 1 + .../integration/kubernetes/run_kubernetes_tests.sh | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 tests/integration/kubernetes/.gitignore diff --git a/tests/integration/kubernetes/.gitignore b/tests/integration/kubernetes/.gitignore new file mode 100644 index 0000000000..a9a1bd38ab --- /dev/null +++ b/tests/integration/kubernetes/.gitignore @@ -0,0 +1 @@ +reports/ diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 9baeb8194b..162bd4808a 100755 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -6,6 +6,7 @@ # set -e +set -o pipefail kubernetes_dir=$(dirname "$(readlink -f "$0")") source "${kubernetes_dir}/../../common.bash" @@ -131,7 +132,10 @@ fi ensure_yq -info "Running tests with bats version: $(bats --version)" +report_dir="${kubernetes_dir}/reports/$(date +'%F-%T')" +mkdir -p "${report_dir}" + +info "Running tests with bats version: $(bats --version). Save outputs to ${report_dir}" tests_fail=() for K8S_TEST_ENTRY in "${K8S_TEST_UNION[@]}" @@ -139,9 +143,14 @@ do K8S_TEST_ENTRY=$(echo "$K8S_TEST_ENTRY" | tr -d '[:space:][:cntrl:]') info "$(kubectl get pods --all-namespaces 2>&1)" info "Executing ${K8S_TEST_ENTRY}" - if ! bats --show-output-of-passing-tests "${K8S_TEST_ENTRY}"; then + # Output file will be prefixed with "ok" or "not_ok" based on the result + out_file="${report_dir}/${K8S_TEST_ENTRY}.out" + if ! bats --show-output-of-passing-tests "${K8S_TEST_ENTRY}" | tee "${out_file}"; then tests_fail+=("${K8S_TEST_ENTRY}") + mv "${out_file}" "$(dirname "${out_file}")/not_ok-$(basename "${out_file}")" [ "${K8S_TEST_FAIL_FAST}" = "yes" ] && break + else + mv "${out_file}" "$(dirname "${out_file}")/ok-$(basename "${out_file}")" fi done From 8c4637d62981b23820926f12badfa72a2952efaf Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Mon, 17 Feb 2025 15:37:48 -0300 Subject: [PATCH 2/3] tests/k8s: print tests report Added 'report-tests' command to gha-run.sh to print to stdout a report of the tests executed. For example: ``` SUMMARY (2025-02-17-14:43:53): Pass: 0 Fail: 1 STATUSES: not_ok foo.bats OUTPUTS: ::group::foo.bats 1..3 not ok 1 test 1 not ok 2 test 2 ok 3 test 3 1..2 not ok 1 test 1 not ok 2 test 2 ::endgroup:: ``` Signed-off-by: Wainer dos Santos Moschetta --- tests/integration/kubernetes/gha-run.sh | 44 +++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index cbdff162e8..7149545ee5 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -294,6 +294,49 @@ function run_tests() { popd } +# Print a report about tests executed. +# +# Crawl over the output files found on each "reports/yyyy-mm-dd-hh:mm:ss" +# directory. +# +function report_tests() { + local reports_dir="${kubernetes_dir}/reports" + local ok + local not_ok + local status + + if [[ ! -d "${reports_dir}" ]]; then + info "no reports directory found: ${reports_dir}" + return + fi + + for report_dir in "${reports_dir}"/*; do + mapfile -t ok < <(find "${report_dir}" -name "ok-*.out") + mapfile -t not_ok < <(find "${report_dir}" -name "not_ok-*.out") + + cat <<-EOF + SUMMARY ($(basename "${report_dir}")): + Pass: ${#ok[*]} + Fail: ${#not_ok[*]} + EOF + + echo -e "\nSTATUSES:" + for out in "${not_ok[@]}" "${ok[@]}"; do + status=$(basename "${out}" | cut -d '-' -f1) + bats=$(basename "${out}" | cut -d '-' -f2- | sed 's/.out$//') + echo " ${status} ${bats}" + done + + echo -e "\nOUTPUTS:" + for out in "${not_ok[@]}" "${ok[@]}"; do + bats=$(basename "${out}" | cut -d '-' -f2- | sed 's/.out$//') + echo "::group::${bats}" + cat "${out}" + echo "::endgroup::" + done + done +} + function collect_artifacts() { if [[ -z "${start_time:-}" ]]; then warn "tests start time is not defined. Cannot gather journal information" @@ -547,6 +590,7 @@ function main() { deploy-kata-garm) deploy_kata "garm" ;; deploy-kata-zvsi) deploy_kata "zvsi" ;; deploy-snapshotter) deploy_snapshotter ;; + report-tests) report_tests ;; run-tests) run_tests ;; run-tests-kcli) run_tests "kcli" ;; collect-artifacts) collect_artifacts ;; From 80a816db9de2be8ae661f1c2019bea2f717a3d34 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Mon, 17 Feb 2025 16:40:25 -0300 Subject: [PATCH 3/3] workflows/run-k8s-tests-coco-nontee: add step to report tests Run `gha-run.sh report-tests` to generate the report of the tests. Signed-off-by: Wainer dos Santos Moschetta --- .github/workflows/run-kata-coco-tests.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/run-kata-coco-tests.yaml b/.github/workflows/run-kata-coco-tests.yaml index 3f129a3eaa..c4a173eb87 100644 --- a/.github/workflows/run-kata-coco-tests.yaml +++ b/.github/workflows/run-kata-coco-tests.yaml @@ -303,6 +303,10 @@ jobs: timeout-minutes: 80 run: bash tests/integration/kubernetes/gha-run.sh run-tests + - name: Report tests + if: always() + run: bash tests/integration/kubernetes/gha-run.sh report-tests + - name: Delete AKS cluster if: always() run: bash tests/integration/kubernetes/gha-run.sh delete-cluster