From cc8f0a24e493c84e68c87d45d37a35b02c39b2c1 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Wed, 28 Jun 2023 16:39:10 +0000 Subject: [PATCH 1/6] metrics: Add checkmetrics to gha-run.sh for metrics CI This PR adds checkmetrics installation for gha-run.sh in order to compare results limits as part of the metrics CI. Fixes #7198 Signed-off-by: Gabriela Cervantes --- tests/metrics/gha-run.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/metrics/gha-run.sh b/tests/metrics/gha-run.sh index 867161673..d7108eaf7 100755 --- a/tests/metrics/gha-run.sh +++ b/tests/metrics/gha-run.sh @@ -13,6 +13,10 @@ kata_tarball_dir=${2:-kata-artifacts} metrics_dir="$(dirname "$(readlink -f "$0")")" source "${metrics_dir}/../common.bash" +declare -r results_dir="${metrics_dir}/results" +declare -r checkmetrics_dir="${metrics_dir}/cmd/checkmetrics" +declare -r checkmetrics_config_dir="${checkmetrics_dir}/ci_worker" + function create_symbolic_links() { local link_configuration_file="/opt/kata/share/defaults/kata-containers/configuration.toml" local source_configuration_file="/opt/kata/share/defaults/kata-containers/configuration-${KATA_HYPERVISOR}.toml" @@ -83,6 +87,22 @@ function check_containerd_config_for_kata() { fi } +function check_metrics() { + # Ensure we have the latest checkemtrics + pushd "${checkmetrics_dir}" + make + sudo make install + popd + + local cm_base_file="${checkmetrics_config_dir}/checkmetrics-json-${hypervisor}-$(uname -n).toml" + checkmetrics --debug --percentage --basefile "${cm_base_file}" --metricsdir "${results_dir}" + cm_result=$? + if [ "${cm_result}" != 0 ]; then + info "run-metrics-ci: checkmetrics FAILED (${cm_result})" + exit "${cm_result}" + fi +} + function run_test_launchtimes() { info "Running Launch Time test using ${KATA_HYPERVISOR} hypervisor" @@ -119,6 +139,8 @@ function main() { run-test-memory-usage-inside-container) run_test_memory_usage_inside_container ;; *) >&2 die "Invalid argument" ;; esac + + check_metrics } main "$@" From 917576e6fb5e67c2325c9158593eed57558a4a6e Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Wed, 28 Jun 2023 16:46:01 +0000 Subject: [PATCH 2/6] metrics: Add double quotes in all variables This PR adds double quotes in all variables to have uniformity across all the gha-run.sh script. Signed-off-by: Gabriela Cervantes --- tests/metrics/gha-run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/metrics/gha-run.sh b/tests/metrics/gha-run.sh index d7108eaf7..fca5c3ff6 100755 --- a/tests/metrics/gha-run.sh +++ b/tests/metrics/gha-run.sh @@ -9,7 +9,7 @@ set -o errexit set -o nounset set -o pipefail -kata_tarball_dir=${2:-kata-artifacts} +kata_tarball_dir="${2:-kata-artifacts}" metrics_dir="$(dirname "$(readlink -f "$0")")" source "${metrics_dir}/../common.bash" @@ -58,7 +58,7 @@ function install_kata() { # Removing previous kata installation sudo rm -rf "${katadir}" - pushd ${kata_tarball_dir} + pushd "${kata_tarball_dir}" sudo tar -xvf "${kata_tarball}" -C "${destdir}" popd @@ -77,9 +77,9 @@ function check_containerd_config_for_kata() { declare -r line2="runtime_type = \"io.containerd.kata.v2\"" declare -r num_lines_containerd=2 declare -r containerd_path="/etc/containerd/config.toml" - local count_matches=$(grep -ic "$line1\|$line2" ${containerd_path}) + local count_matches=$(grep -ic "$line1\|$line2" "${containerd_path}") - if [ $count_matches = $num_lines_containerd ]; then + if [ "${count_matches}" = "${num_lines_containerd}" ]; then info "containerd ok" else info "overwriting containerd configuration w/ a valid one" From e94aaed3c7d009e2fd8ea4b64cf558c2730e430c Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Wed, 28 Jun 2023 17:13:20 +0000 Subject: [PATCH 3/6] ci_worker: Add checkmetrics ci worker for cloud hypervisor This PR adds the checkmetrics ci worker file for cloud hypervisor in order to check the boot times limit. Signed-off-by: Gabriela Cervantes --- ...cs-json-cloud-hypervisor-kata-metric8.toml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-cloud-hypervisor-kata-metric8.toml diff --git a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-cloud-hypervisor-kata-metric8.toml b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-cloud-hypervisor-kata-metric8.toml new file mode 100644 index 000000000..af5c8c634 --- /dev/null +++ b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-cloud-hypervisor-kata-metric8.toml @@ -0,0 +1,21 @@ +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# This file contains baseline expectations +# for checked results by checkmetrics tool. +# +# values set specifically for packet.com c1.small worker. + +[[metric]] +name = "boot-times" +type = "json" +description = "measure container lifecycle timings" +# Min and Max values to set a 'range' that +# the median of the CSV Results data must fall +# within (inclusive) +checkvar = ".\"boot-times\".Results | .[] | .\"to-workload\".Result" +checktype = "mean" +midval = 0.54 +minpercent = 10.0 +maxpercent = 10.0 From b481ef18833aed9ccfae019ae4e65b957d1dec77 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Wed, 28 Jun 2023 19:03:37 +0000 Subject: [PATCH 4/6] makefile: Add -buildvcs=false flag to go build This PR adds the -buildvcs=false flag to the go build of checkmetrics. Signed-off-by: Gabriela Cervantes --- ...ric8.toml => checkmetrics-json-clh-kata-metric8.toml} | 6 +++--- tests/metrics/gha-run.sh | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) rename tests/metrics/cmd/checkmetrics/ci_worker/{checkmetrics-json-cloud-hypervisor-kata-metric8.toml => checkmetrics-json-clh-kata-metric8.toml} (91%) diff --git a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-cloud-hypervisor-kata-metric8.toml b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-clh-kata-metric8.toml similarity index 91% rename from tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-cloud-hypervisor-kata-metric8.toml rename to tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-clh-kata-metric8.toml index af5c8c634..7966b69eb 100644 --- a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-cloud-hypervisor-kata-metric8.toml +++ b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-clh-kata-metric8.toml @@ -16,6 +16,6 @@ description = "measure container lifecycle timings" # within (inclusive) checkvar = ".\"boot-times\".Results | .[] | .\"to-workload\".Result" checktype = "mean" -midval = 0.54 -minpercent = 10.0 -maxpercent = 10.0 +midval = 0.42 +minpercent = 15.0 +maxpercent = 15.0 diff --git a/tests/metrics/gha-run.sh b/tests/metrics/gha-run.sh index fca5c3ff6..05b426fbc 100755 --- a/tests/metrics/gha-run.sh +++ b/tests/metrics/gha-run.sh @@ -8,6 +8,7 @@ set -o errexit set -o nounset set -o pipefail +set -x kata_tarball_dir="${2:-kata-artifacts}" metrics_dir="$(dirname "$(readlink -f "$0")")" @@ -94,7 +95,7 @@ function check_metrics() { sudo make install popd - local cm_base_file="${checkmetrics_config_dir}/checkmetrics-json-${hypervisor}-$(uname -n).toml" + local cm_base_file="${checkmetrics_config_dir}/checkmetrics-json-clh-kata-metric8.toml" checkmetrics --debug --percentage --basefile "${cm_base_file}" --metricsdir "${results_dir}" cm_result=$? if [ "${cm_result}" != 0 ]; then @@ -110,6 +111,10 @@ function run_test_launchtimes() { exit 0 create_symbolic_links bash tests/metrics/time/launch_times.sh -i public.ecr.aws/ubuntu/ubuntu:latest -n 20 + + if [ "${hypervisor}" = "clh" ]; then + check_metrics + fi } function run_test_memory_usage() { @@ -139,8 +144,6 @@ function main() { run-test-memory-usage-inside-container) run_test_memory_usage_inside_container ;; *) >&2 die "Invalid argument" ;; esac - - check_metrics } main "$@" From 0502354b42bfd3ca22877d93814a596c01024849 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Fri, 30 Jun 2023 20:26:33 +0000 Subject: [PATCH 5/6] checkmetrics: Add checkmetrics json for qemu This PR adds checkmetrics json file for qemu metrics. Signed-off-by: Gabriela Cervantes --- .../checkmetrics-json-qemu-kata-metric8.toml | 21 +++++++++++++++++++ tests/metrics/gha-run.sh | 9 ++++---- 2 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml diff --git a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml new file mode 100644 index 000000000..c2b3379a4 --- /dev/null +++ b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml @@ -0,0 +1,21 @@ +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# This file contains baseline expectations +# for checked results by checkmetrics tool. +# +# values set specifically for packet.com c1.small worker. + +[[metric]] +name = "boot-times" +type = "json" +description = "measure container lifecycle timings" +# Min and Max values to set a 'range' that +# the median of the CSV Results data must fall +# within (inclusive) +checkvar = ".\"boot-times\".Results | .[] | .\"to-workload\".Result" +checktype = "mean" +midval = 0.61 +minpercent = 15.0 +maxpercent = 15.0 diff --git a/tests/metrics/gha-run.sh b/tests/metrics/gha-run.sh index 05b426fbc..e30a8a7d0 100755 --- a/tests/metrics/gha-run.sh +++ b/tests/metrics/gha-run.sh @@ -22,7 +22,7 @@ function create_symbolic_links() { local link_configuration_file="/opt/kata/share/defaults/kata-containers/configuration.toml" local source_configuration_file="/opt/kata/share/defaults/kata-containers/configuration-${KATA_HYPERVISOR}.toml" - if [ ${KATA_HYPERVISOR} != 'qemu' ] && [ ${KATA_HYPERVISOR} != 'clh' ]; then + if [ "${KATA_HYPERVISOR}" != 'qemu' ] && [ "${KATA_HYPERVISOR}" != 'clh' ]; then die "Failed to set the configuration.toml: '${KATA_HYPERVISOR}' is not recognized as a valid hypervisor name." fi @@ -89,13 +89,14 @@ function check_containerd_config_for_kata() { } function check_metrics() { + KATA_HYPERVISOR="${1}" # Ensure we have the latest checkemtrics pushd "${checkmetrics_dir}" make sudo make install popd - local cm_base_file="${checkmetrics_config_dir}/checkmetrics-json-clh-kata-metric8.toml" + local cm_base_file="${checkmetrics_config_dir}/checkmetrics-json-${KATA_HYPERVISOR}-kata-metric8.toml" checkmetrics --debug --percentage --basefile "${cm_base_file}" --metricsdir "${results_dir}" cm_result=$? if [ "${cm_result}" != 0 ]; then @@ -112,9 +113,7 @@ function run_test_launchtimes() { create_symbolic_links bash tests/metrics/time/launch_times.sh -i public.ecr.aws/ubuntu/ubuntu:latest -n 20 - if [ "${hypervisor}" = "clh" ]; then - check_metrics - fi + check_metrics "${KATA_HYPERVISOR}" } function run_test_memory_usage() { From d8b8f7e94d679bc76e64272ba2a50c6a0454d4c0 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Mon, 3 Jul 2023 16:48:41 +0000 Subject: [PATCH 6/6] metrics: Enable launch tests time metrics This PR enables the launch tests metrics for kata CI. Signed-off-by: Gabriela Cervantes --- .../checkmetrics-json-qemu-kata-metric8.toml | 2 +- tests/metrics/gha-run.sh | 23 ++++++++----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml index c2b3379a4..c243ed2c5 100644 --- a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml +++ b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml @@ -5,7 +5,7 @@ # This file contains baseline expectations # for checked results by checkmetrics tool. # -# values set specifically for packet.com c1.small worker. +# values set specifically for Equinix m3.small.x86. [[metric]] name = "boot-times" diff --git a/tests/metrics/gha-run.sh b/tests/metrics/gha-run.sh index e30a8a7d0..a684c4f60 100755 --- a/tests/metrics/gha-run.sh +++ b/tests/metrics/gha-run.sh @@ -8,7 +8,6 @@ set -o errexit set -o nounset set -o pipefail -set -x kata_tarball_dir="${2:-kata-artifacts}" metrics_dir="$(dirname "$(readlink -f "$0")")" @@ -49,8 +48,6 @@ EOF } function install_kata() { - # ToDo: remove the exit once the metrics workflow is stable - exit 0 local kata_tarball="kata-static.tar.xz" declare -r katadir="/opt/kata" declare -r destdir="/" @@ -70,6 +67,15 @@ function install_kata() { check_containerd_config_for_kata restart_containerd_service + install_checkmetrics +} + +function install_checkmetrics() { + # Ensure we have the latest checkmetrics + pushd "${checkmetrics_dir}" + make + sudo make install + popd } function check_containerd_config_for_kata() { @@ -90,26 +96,17 @@ function check_containerd_config_for_kata() { function check_metrics() { KATA_HYPERVISOR="${1}" - # Ensure we have the latest checkemtrics - pushd "${checkmetrics_dir}" - make - sudo make install - popd - local cm_base_file="${checkmetrics_config_dir}/checkmetrics-json-${KATA_HYPERVISOR}-kata-metric8.toml" checkmetrics --debug --percentage --basefile "${cm_base_file}" --metricsdir "${results_dir}" cm_result=$? if [ "${cm_result}" != 0 ]; then - info "run-metrics-ci: checkmetrics FAILED (${cm_result})" - exit "${cm_result}" + die "run-metrics-ci: checkmetrics FAILED (${cm_result})" fi } function run_test_launchtimes() { info "Running Launch Time test using ${KATA_HYPERVISOR} hypervisor" - # ToDo: remove the exit once the metrics workflow is stable - exit 0 create_symbolic_links bash tests/metrics/time/launch_times.sh -i public.ecr.aws/ubuntu/ubuntu:latest -n 20