workflows: Improve metrics jobs

- As the metrics tests are largely independent
then allow subsequent tests to run even if previous
ones failed. The results might not be perfect if
clean-up is required, but we can work on that later.
- Move the test results check out of the latency
test that seems arbitrary and into it's own job step
- Add timeouts to steps that might fail/hang if there
are containerd/K8s issues

Signed-off-by: stevenhorsman <steven@uk.ibm.com>

.github/workflows/run-metrics.yaml

@@ -66,29 +66,48 @@ jobs:
         run: bash tests/metrics/gha-run.sh enabling-hypervisor
 
       - name: run launch times test
+        timeout-minutes: 15
+        continue-on-error: true
         run: bash tests/metrics/gha-run.sh run-test-launchtimes
 
       - name: run memory foot print test
+        timeout-minutes: 15
+        continue-on-error: true
         run:  bash tests/metrics/gha-run.sh run-test-memory-usage
 
       - name: run memory usage inside container test
+        timeout-minutes: 15
+        continue-on-error: true
         run:  bash tests/metrics/gha-run.sh run-test-memory-usage-inside-container
 
       - name: run blogbench test
+        timeout-minutes: 15
+        continue-on-error: true
         run:  bash tests/metrics/gha-run.sh run-test-blogbench
 
       - name: run tensorflow test
+        timeout-minutes: 15
+        continue-on-error: true
         run:  bash tests/metrics/gha-run.sh run-test-tensorflow
 
       - name: run fio test
+        timeout-minutes: 15
+        continue-on-error: true
         run:  bash tests/metrics/gha-run.sh run-test-fio
 
       - name: run iperf test
+        timeout-minutes: 15
+        continue-on-error: true
         run:  bash tests/metrics/gha-run.sh run-test-iperf
 
       - name: run latency test
+        timeout-minutes: 15
+        continue-on-error: true
         run:  bash tests/metrics/gha-run.sh run-test-latency
 
+      - name: check metrics
+        run:  bash tests/metrics/gha-run.sh check-metrics
+
       - name: make metrics tarball ${{ matrix.vmm }}
         run: bash tests/metrics/gha-run.sh make-tarball-results
 
@@ -101,5 +120,6 @@ jobs:
           if-no-files-found: error
 
       - name: Delete kata-deploy
+        timeout-minutes: 10
         if: always()
         run: bash tests/integration/kubernetes/gha-run.sh cleanup-kubeadm

tests/metrics/gha-run.sh

@@ -114,8 +114,6 @@ function run_test_latency() {
 	info "Running Latency test using ${KATA_HYPERVISOR} hypervisor"
 
 	bash tests/metrics/network/latency_kubernetes/latency-network.sh
-
-	check_metrics
 }
 
 function main() {
@@ -132,6 +130,7 @@ function main() {
 		run-test-fio) run_test_fio ;;
 		run-test-iperf) run_test_iperf ;;
 		run-test-latency) run_test_latency ;;
+		check-metrics) check_metrics;;
 		*) >&2 die "Invalid argument: ${action}" ;;
 	esac
 }