From 61fe20cf9a07772054f652ace6b47ecbe71668a4 Mon Sep 17 00:00:00 2001 From: Liu Wenyuan Date: Thu, 23 Nov 2023 19:28:52 +0800 Subject: [PATCH] gha: Fix some of gha metrics failure for StratoVirt Update the Speed & Density metric tests baseline for StratoVirt and re-enable them, and skip other metric tests temporarily. Fixes: #8656 Signed-off-by: Liu Wenyuan --- .github/workflows/run-metrics.yaml | 2 +- tests/common.bash | 6 +- ...kmetrics-json-stratovirt-kata-metric8.toml | 131 +----------------- tests/metrics/gha-run.sh | 15 ++ 4 files changed, 28 insertions(+), 126 deletions(-) diff --git a/.github/workflows/run-metrics.yaml b/.github/workflows/run-metrics.yaml index 53deeb7864..7628a41369 100644 --- a/.github/workflows/run-metrics.yaml +++ b/.github/workflows/run-metrics.yaml @@ -48,7 +48,7 @@ jobs: # all the tests due to a single flaky instance. fail-fast: false matrix: - vmm: ['clh', 'qemu'] + vmm: ['clh', 'qemu', 'stratovirt'] max-parallel: 1 runs-on: metrics env: diff --git a/tests/common.bash b/tests/common.bash index 1604ff7172..b0434f31d6 100644 --- a/tests/common.bash +++ b/tests/common.bash @@ -125,7 +125,11 @@ function extract_kata_env() { HYPERVISOR_PATH=$(sudo ${cmd} env --json | jq -r ${hypervisor_path}) # TODO: there is no ${cmd} of rust version currently if [ "${KATA_HYPERVISOR}" != "dragonball" ]; then - HYPERVISOR_VERSION=$(sudo -E ${HYPERVISOR_PATH} --version | head -n1) + if [ "${KATA_HYPERVISOR}" = "stratovirt" ]; then + HYPERVISOR_VERSION=$(sudo -E ${HYPERVISOR_PATH} -version | head -n1) + else + HYPERVISOR_VERSION=$(sudo -E ${HYPERVISOR_PATH} --version | head -n1) + fi fi VIRTIOFSD_PATH=$(sudo ${cmd} env --json | jq -r ${virtio_fs_daemon_path}) diff --git a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-stratovirt-kata-metric8.toml b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-stratovirt-kata-metric8.toml index b5ad92ca19..88cfc4a276 100644 --- a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-stratovirt-kata-metric8.toml +++ b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-stratovirt-kata-metric8.toml @@ -14,9 +14,9 @@ description = "measure container lifecycle timings" # within (inclusive) checkvar = ".\"boot-times\".Results | .[] | .\"to-workload\".Result" checktype = "mean" -midval = 0.62 -minpercent = 40.0 -maxpercent = 40.0 +midval = 0.48 +minpercent = 30.0 +maxpercent = 30.0 [[metric]] name = "memory-footprint" @@ -27,9 +27,9 @@ description = "measure memory usage" # within (inclusive) checkvar = ".\"memory-footprint\".Results | .[] | .average.Result" checktype = "mean" -midval = 129842.10 -minpercent = 30.0 -maxpercent = 30.0 +midval = 132550.90 +minpercent = 20.0 +maxpercent = 20.0 [[metric]] name = "memory-footprint-inside-container" @@ -40,123 +40,6 @@ description = "measure memory inside the container" # within (inclusive) checkvar = ".\"memory-footprint-inside-container\".Results | .[] | .memtotal.Result" checktype = "mean" -midval = 2040568.0 +midval = 4101768.0 minpercent = 30.0 maxpercent = 30.0 - -[[metric]] -name = "blogbench" -type = "json" -description = "measure container average of blogbench write" -# Min and Max values to set a 'range' that -# the median of the CSV Results data must fall -# within (inclusive) -checkvar = ".\"blogbench\".Results | .[] | .write.Result" -checktype = "mean" -midval = 603.0 -minpercent = 30.0 -maxpercent = 30.0 - -[[metric]] -name = "blogbench" -type = "json" -description = "measure container average of blogbench read" -# Min and Max values to set a 'range' that -# the median of the CSV Results data must fall -# within (inclusive) -checkvar = ".\"blogbench\".Results | .[] | .read.Result" -checktype = "mean" -midval = 37669.0 -minpercent = 30.0 -maxpercent = 30.0 - -[[metric]] -name = "tensorflow_nhwc" -type = "json" -description = "tensorflow resnet model" -# Min and Max values to set a 'range' that -# the median of the CSV Results data must fall -# within (inclusive) -checkvar = ".\"tensorflow_nhwc\".Results | .[] | .resnet.Result" -checktype = "mean" -midval = 2025.0 -minpercent = 30.0 -maxpercent = 30.0 - -[[metric]] -name = "tensorflow_nhwc" -type = "json" -description = "tensorflow alexnet model" -# Min and Max values to set a 'range' that -# the median of the CSV Results data must fall -# within (inclusive) -checkvar = ".\"tensorflow_nhwc\".Results | .[] | .alexnet.Result" -checktype = "mean" -midval = 75.0 -minpercent = 30.0 -maxpercent = 30.0 - -[[metric]] -name = "latency" -type = "json" -description = "measure container latency" -# Min and Max values to set a 'range' that -# the median of the CSV Results data must fall -# within (inclusive) -checkvar = ".\"latency\".Results | .[] | .latency.Result" -checktype = "mean" -midval = 0.78 -minpercent = 30.0 -maxpercent = 30.0 - -[[metric]] -name = "network-iperf3" -type = "json" -description = "measure container cpu utilization using iperf3" -# Min and Max values to set a 'range' that -# the median of the CSV Results data must fall -# within (inclusive) -checkvar = ".\"network-iperf3\".Results | .[] | .cpu.Result" -checktype = "mean" -midval = 60.10 -minpercent = 30.0 -maxpercent = 30.0 - -[[metric]] -name = "network-iperf3" -type = "json" -description = "measure container bandwidth using iperf3" -# Min and Max values to set a 'range' that -# the median of the CSV Results data must fall -# within (inclusive) -checkvar = ".\"network-iperf3\".Results | .[] | .bandwidth.Result" -checktype = "mean" -midval = 19959440840.94 -minpercent = 30.0 -maxpercent = 30.0 - -[[metric]] -name = "network-iperf3" -type = "json" -description = "measure container parallel bandwidth using iperf3" -# Min and Max values to set a 'range' that -# the median of the CSV Results data must fall -# within (inclusive) -checkvar = ".\"network-iperf3\".Results | .[] | .parallel.Result" -checktype = "mean" -midval = 25487333685.04 -minpercent = 30.0 -maxpercent = 30.0 - -[[metric]] -name = "network-iperf3" -type = "json" -description = "iperf" -# Min and Max values to set a 'range' that -# the median of the CSV Results data must fall -# within (inclusive) -checkvar = ".\"network-iperf3\".Results | .[] | .jitter.Result" -checktype = "mean" -midval = 0.038 -minpercent = 40.0 -maxpercent = 40.0 diff --git a/tests/metrics/gha-run.sh b/tests/metrics/gha-run.sh index 28180d270f..4edf79f028 100755 --- a/tests/metrics/gha-run.sh +++ b/tests/metrics/gha-run.sh @@ -72,30 +72,45 @@ function run_test_memory_usage_inside_container() { } function run_test_blogbench() { + if [ "${KATA_HYPERVISOR}" = "stratovirt" ]; then + exit 0 + fi info "Running Blogbench test using ${KATA_HYPERVISOR} hypervisor" bash tests/metrics/storage/blogbench.sh } function run_test_tensorflow() { + if [ "${KATA_HYPERVISOR}" = "stratovirt" ]; then + exit 0 + fi info "Running TensorFlow test using ${KATA_HYPERVISOR} hypervisor" bash tests/metrics/machine_learning/tensorflow_nhwc.sh 1 20 } function run_test_fio() { + if [ "${KATA_HYPERVISOR}" = "stratovirt" ]; then + exit 0 + fi info "Running FIO test using ${KATA_HYPERVISOR} hypervisor" bash tests/metrics/storage/fio_test.sh } function run_test_iperf() { + if [ "${KATA_HYPERVISOR}" = "stratovirt" ]; then + exit 0 + fi info "Running Iperf test using ${KATA_HYPERVISOR} hypervisor" bash tests/metrics/network/iperf3_kubernetes/k8s-network-metrics-iperf3.sh -a } function run_test_latency() { + if [ "${KATA_HYPERVISOR}" = "stratovirt" ]; then + exit 0 + fi info "Running Latency test using ${KATA_HYPERVISOR} hypervisor" bash tests/metrics/network/latency_kubernetes/latency-network.sh