From 6c68924230c88f9a42d353143a9bcbcf7da7af49 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 6 Jul 2023 17:07:42 +0000 Subject: [PATCH 1/4] metrics: Enable memory inside container metrics This PR will enable the memory inside container metrics for the Kata CI. Fixes #7254 Signed-off-by: Gabriela Cervantes --- .../density/memory_usage_inside_container.sh | 37 ++++++++++--------- tests/metrics/gha-run.sh | 2 - 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/tests/metrics/density/memory_usage_inside_container.sh b/tests/metrics/density/memory_usage_inside_container.sh index 071ded175e..5da73425d5 100755 --- a/tests/metrics/density/memory_usage_inside_container.sh +++ b/tests/metrics/density/memory_usage_inside_container.sh @@ -9,6 +9,7 @@ # is measured by using /proc/meminfo. set -e +set -x # General env SCRIPT_PATH=$(dirname "$(readlink -f "$0")") @@ -20,7 +21,7 @@ IMAGE='quay.io/prometheus/busybox:latest' CMD="sleep 10; cat /proc/meminfo" # We specify here in 'k', as that then matches the results we get from the meminfo, # which makes later direct comparison easier. -MEMSIZE=${MEMSIZE:-$((2048*1024))} +MEMSIZE="${MEMSIZE:-$((2048*1024))}" # this variable determines the number of attempts when a test # result is considered not valid (a zero value or a negative value) @@ -47,20 +48,20 @@ parse_results() { local memfree_acu="${3:-0}" local memavailable_acu="${4:-0}" - local memtotal=$(echo "$raw_results" | awk '/MemTotal/ {print $2}') - units_memtotal=$(echo "$raw_results" | awk '/MemTotal/ {print $3}') + local memtotal=$(echo "${raw_results}" | awk '/MemTotal/ {print $2}') + units_memtotal=$(echo "${raw_results}" | awk '/MemTotal/ {print $3}') - local memfree=$(echo "$raw_results" | awk '/MemFree/ {print $2}') - units_memfree=$(echo "$raw_results" | awk '/MemFree/ {print $3}') + local memfree=$(echo "${raw_results}" | awk '/MemFree/ {print $2}') + units_memfree=$(echo "${raw_results}" | awk '/MemFree/ {print $3}') - local memavailable=$(echo "$raw_results" | awk '/MemAvailable/ {print $2}') - units_memavailable=$(echo "$raw_results" | awk '/MemAvailable/ {print $3}') + local memavailable=$(echo "${raw_results}" | awk '/MemAvailable/ {print $2}') + units_memavailable=$(echo "${raw_results}" | awk '/MemAvailable/ {print $3}') # check results: if any result is zero or negative, it is considered as invalid, and the test will be repeated. - if (( $(echo "$memtotal <= 0" | bc -l) )) || (( $(echo "$memfree <= 0" | bc -l) )) || (( $(echo "$memavailable <= 0" | bc -l) )); then + if (( $(echo "${memtotal} <= 0" | bc -l) )) || (( $(echo "${memfree} <= 0" | bc -l) )) || (( $(echo "${memavailable} <= 0" | bc -l) )); then MAX_FAILED_ATTEMPTS=$((MAX_FAILED_ATTEMPTS-1)) valid_result=0 - info "Skipping invalid result: memtotal: $memtotal memfree: $memfree memavailable: $memavailable" + info "Skipping invalid result: memtotal: ${memtotal} memfree: ${memfree} memavailable: ${memavailable}" return 0 fi @@ -68,14 +69,14 @@ parse_results() { memfreeAvg=$((memfree+memfree_acu)) memavailableAvg=$((memavailable+memavailable_acu)) valid_result=1 - info "Iteration# $count_iters memtotal: $memtotal memfree: $memfree memavailable: $memavailable" + info "Iteration# ${count_iters} memtotal: ${memtotal} memfree: ${memfree} memavailable: ${memavailable}" } store_results_json() { metrics_json_start_array - memtotalAvg=$(echo "scale=2; $memtotalAvg / $count_iters" | bc) - memfreeAvg=$(echo "scale=2; $memfreeAvg / $count_iters" | bc) - memavailableAvg=$(echo "scale=2; $memavailableAvg / $count_iters" | bc) + memtotalAvg=$(echo "scale=2; ${memtotalAvg} / ${count_iters}" | bc) + memfreeAvg=$(echo "scale=2; ${memfreeAvg} / ${count_iters}" | bc) + memavailableAvg=$(echo "scale=2; ${memavailableAvg} / ${count_iters}" | bc) local json="$(cat << EOF { @@ -109,7 +110,7 @@ EOF function main() { # switch to select output format local num_iterations=${1:-1} - info "Iterations: $num_iterations" + info "Iterations: ${num_iterations}" # Check tools/commands dependencies cmds=("awk" "ctr") @@ -117,13 +118,13 @@ function main() { check_cmds "${cmds[@]}" check_images "${IMAGE}" metrics_json_init - while [ $count_iters -lt $num_iterations ]; do - local output=$(sudo -E "${CTR_EXE}" run --memory-limit $((MEMSIZE*1024)) --rm --runtime=$CTR_RUNTIME $IMAGE busybox sh -c "$CMD" 2>&1) + while [ "${count_iters}" -lt "${num_iterations}" ]; do + local output=$(sudo -E "${CTR_EXE}" run --memory-limit $((MEMSIZE*1024)) --rm --runtime="${CTR_RUNTIME}" "${IMAGE}" busybox sh -c "${CMD}" 2>&1) parse_results "${output}" "${memtotalAvg}" "${memfreeAvg}" "${memavailableAvg}" # quit if number of attempts exceeds the allowed value. - [ ${MAX_FAILED_ATTEMPTS} -eq 0 ] && die "Max number of attempts exceeded." - [ ${valid_result} -eq 1 ] && count_iters=$((count_iters+1)) + [ "${MAX_FAILED_ATTEMPTS}" -eq 0 ] && die "Max number of attempts exceeded." + [ "${valid_result}" -eq 1 ] && count_iters=$((count_iters+1)) done store_results_json clean_env_ctr diff --git a/tests/metrics/gha-run.sh b/tests/metrics/gha-run.sh index 5f8bfbf08a..055fe0b9fc 100755 --- a/tests/metrics/gha-run.sh +++ b/tests/metrics/gha-run.sh @@ -127,8 +127,6 @@ function run_test_memory_usage() { function run_test_memory_usage_inside_container() { info "Running memory-usage inside the container test using ${KATA_HYPERVISOR} hypervisor" - # ToDo: remove the exit once the metrics workflow is stable - exit 0 create_symbolic_links bash tests/metrics/density/memory_usage_inside_container.sh 5 } From 6ca34f949ef1f6ebceadc8181ff334773758ecdc Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 6 Jul 2023 20:41:08 +0000 Subject: [PATCH 2/4] checkmetrics: Add memory inside container value for clh Add memory inside container value for clh. Signed-off-by: Gabriela Cervantes --- .../checkmetrics-json-clh-kata-metric8.toml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-clh-kata-metric8.toml b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-clh-kata-metric8.toml index 562b2c83b1..ca1c4355ea 100644 --- a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-clh-kata-metric8.toml +++ b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-clh-kata-metric8.toml @@ -32,3 +32,16 @@ checktype = "mean" midval = 2518364.00 minpercent = 20.0 maxpercent = 20.0 + +[[metric]] +name = "memory-footprint-inside-container" +type = "json" +description = "measure memory inside the container" +# Min and Max values to set a 'range' that +# the median of the CSV Results data must fall +# within (inclusive) +checkvar = ".\"memory-footprint-inside-container\".Results | .[] | .memtotal.Result" +checktype = "mean" +midval = 4135244.0 +minpercent = 20.0 +maxpercent = 20.0 From 2be342023bfa3075eb45ac072a8bbcfee7833133 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 6 Jul 2023 20:42:48 +0000 Subject: [PATCH 3/4] checkmetrics: Add memory usage inside container value for qemu This PR adds the memory usage inside container value for qemu. Signed-off-by: Gabriela Cervantes --- .../checkmetrics-json-qemu-kata-metric8.toml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml index c6bc851471..93eadf4f0f 100644 --- a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml +++ b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml @@ -32,3 +32,16 @@ checktype = "mean" midval = 2435844.00 minpercent = 20.0 maxpercent = 20.0 + +[[metric]] +name = "memory-footprint-inside-container" +type = "json" +description = "measure memory inside the container" +# Min and Max values to set a 'range' that +# the median of the CSV Results data must fall +# within (inclusive) +checkvar = ".\"memory-footprint-inside-container\".Results | .[] | .memtotal.Result" +checktype = "mean" +midval = 4135112.0 +minpercent = 20.0 +maxpercent = 20.0 From 310e069f731bb7673d38fe3835c0fdc8e50b608f Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Fri, 7 Jul 2023 19:26:47 +0000 Subject: [PATCH 4/4] checkmetrics: Enable checkmetrics for memory inside test This PR enables the checkmetrics to include the memory inside container test. Signed-off-by: Gabriela Cervantes --- .../ci_worker/checkmetrics-json-qemu-kata-metric8.toml | 6 +++--- tests/metrics/density/memory_usage_inside_container.sh | 1 - tests/metrics/gha-run.sh | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml index 93eadf4f0f..e599ee2657 100644 --- a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml +++ b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml @@ -42,6 +42,6 @@ description = "measure memory inside the container" # within (inclusive) checkvar = ".\"memory-footprint-inside-container\".Results | .[] | .memtotal.Result" checktype = "mean" -midval = 4135112.0 -minpercent = 20.0 -maxpercent = 20.0 +midval = 3677280.0 +minpercent = 25.0 +maxpercent = 25.0 diff --git a/tests/metrics/density/memory_usage_inside_container.sh b/tests/metrics/density/memory_usage_inside_container.sh index 5da73425d5..68d859b4fe 100755 --- a/tests/metrics/density/memory_usage_inside_container.sh +++ b/tests/metrics/density/memory_usage_inside_container.sh @@ -9,7 +9,6 @@ # is measured by using /proc/meminfo. set -e -set -x # General env SCRIPT_PATH=$(dirname "$(readlink -f "$0")") diff --git a/tests/metrics/gha-run.sh b/tests/metrics/gha-run.sh index 055fe0b9fc..665acd7c9a 100755 --- a/tests/metrics/gha-run.sh +++ b/tests/metrics/gha-run.sh @@ -120,8 +120,6 @@ function run_test_memory_usage() { create_symbolic_links bash tests/metrics/density/memory_usage.sh 20 5 - - check_metrics } function run_test_memory_usage_inside_container() { @@ -129,6 +127,8 @@ function run_test_memory_usage_inside_container() { create_symbolic_links bash tests/metrics/density/memory_usage_inside_container.sh 5 + + check_metrics } function run_test_blogbench() {