diff --git a/tests/metrics/lib/common.bash b/tests/metrics/lib/common.bash
index e7d253136d..b16976df53 100755
--- a/tests/metrics/lib/common.bash
+++ b/tests/metrics/lib/common.bash
@@ -41,6 +41,10 @@ public.ecr.aws/lts
mirror.gcr.io/library
quay.io/libpod"
+# Default Kata Configuration Directory
+readonly DEFAULT_KATA_CONFIG_DIR="/opt/kata/share/defaults/kata-containers"
+readonly DEFAULT_KATA_CONFIG_FNAME="configuration.toml"
+
# This function checks existence of commands.
# They can be received standalone or as an array, e.g.
#
@@ -86,10 +90,14 @@ function generate_build_dockerfile()
local map_key="$3"
local text_to_replace="$4"
local regs=(${registries["${map_key}"]})
+
for r in ${regs[@]}; do
sed 's|'${text_to_replace}'|'${r}'|g' \
"${dockerfile}.in" > "${dockerfile}"
- if sudo "${DOCKER_EXE}" build --build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" --label "$image" --tag "${image}" -f "$dockerfile" "$dockerfile_dir"; then
+ if sudo -E "${DOCKER_EXE}" build \
+ --build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" \
+ --build-arg HTTP_PROXY="${http_proxy}" --build-arg HTTPS_PROXY="${https_proxy}" \
+ --label "$image" --tag "${image}" -f "$dockerfile" "$dockerfile_dir"; then
return 0
fi
done
@@ -107,7 +115,10 @@ function build_dockerfile_image()
if [ -f "$dockerfile_path" ]; then
info "docker building $image"
- if ! sudo "${DOCKER_EXE}" build --build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" --label "$image" --tag "${image}" -f "$dockerfile_path" "$dockerfile_dir"; then
+ if ! sudo -E "${DOCKER_EXE}" build \
+ --build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" \
+ --build-arg HTTP_PROXY="${http_proxy}" --build-arg HTTPS_PROXY="${https_proxy}" \
+ --label "$image" --tag "${image}" -f "$dockerfile_path" "$dockerfile_dir"; then
die "Failed to docker build image $image"
fi
return 0
@@ -425,3 +436,75 @@ function check_containers_are_running() {
return 1
fi
}
+
+# This function receives an identifier used to name a performance Kata configuration file.
+# The current kata configuration is copied to a new kata config file with the difference
+# that the parameters 'default_vcpus' and 'default_memory' will be updated with the values
+# of the max vcpus and the available memory in the system.
+# Finally it makes Kata point to the new configuration file.
+set_kata_configuration_performance() {
+ WORKLOAD_CONFIG_FILE="${1}"
+
+ [ -z "${WORKLOAD_CONFIG_FILE}" ] && die "Unable to set a performance Kata configuration because the passed identifier is empty."
+
+ local NUM_CPUS="$(nproc --all)"
+ local MEM_AVAIL_KB="$(grep MemAvailable /proc/meminfo | awk '{print $2}')"
+ local MEM_AVAIL_MB=$(echo "scale=0; $MEM_AVAIL_KB / 1024" | bc)
+
+ info "Updating Kata configuration to increase memory and cpu resources assigned to the workload."
+
+ # Copy the current kata configuration file to the workload config file,
+ # and increase memory size and num of vcpus assigned.
+
+ pushd "${DEFAULT_KATA_CONFIG_DIR}" > /dev/null
+
+ if [ ! -f "${DEFAULT_KATA_CONFIG_FNAME}" ]; then
+ die "Kata config file not found."
+ fi
+
+ info "Changing the kata configuration to assign '${NUM_CPUS} vcpus' and ${MEM_AVAIL_MB} MB of memory to the performance workload."
+
+ cp "${DEFAULT_KATA_CONFIG_FNAME}" "${WORKLOAD_CONFIG_FILE}"
+ ln -sf "${WORKLOAD_CONFIG_FILE}" "${DEFAULT_KATA_CONFIG_FNAME}"
+
+ sed -i "s/default_memory =[^=&]*/default_memory = $MEM_AVAIL_MB/g" "${WORKLOAD_CONFIG_FILE}"
+ sed -i "s/default_vcpus =[^=&]*/default_vcpus = $NUM_CPUS/g" "${WORKLOAD_CONFIG_FILE}"
+ popd > /dev/null
+}
+
+function clean_cache() {
+ sudo sync; echo 1 > /proc/sys/vm/drop_caches
+}
+
+# This function receives as single parameter, the name of a valid Kata configuration file
+# which will be established as the default Kata configuration to start new Kata containers.
+function set_kata_config_file() {
+ NEW_KATA_CONFIG=${1}
+
+ [ -z "${NEW_KATA_CONFIG}" ] && die "Failed to set a new Kata configuration because the configuration file was not not provided."
+ [ ! -d "${DEFAULT_KATA_CONFIG_DIR}" ] && die "Kata configuration directory was not found: ${DEFAULT_KATA_CONFIG_DIR}."
+
+ pushd "${DEFAULT_KATA_CONFIG_DIR}" > /dev/null
+
+ [ ! -f "${NEW_KATA_CONFIG}" ] && die "The Kata configuration file provided: ${NEW_KATA_CONFIG} was not found."
+
+ info "Aplying a new Kata configuration using the file: ${NEW_KATA_CONFIG}."
+
+ ln -sf "${NEW_KATA_CONFIG}" "${DEFAULT_KATA_CONFIG_FNAME}"
+
+ popd > /dev/null
+}
+
+function get_current_kata_config_file() {
+ declare -n current_config_file=$1
+
+ pushd "${DEFAULT_KATA_CONFIG_DIR}" > /dev/null
+ KATA_CONFIG_FNAME="$(readlink -f ${DEFAULT_KATA_CONFIG_FNAME})"
+ popd > /dev/null
+
+ current_config_file="${KATA_CONFIG_FNAME}"
+}
+
+function check_if_root() {
+ [ "$EUID" -ne 0 ] && die "Please run as root or use sudo."
+}
diff --git a/tests/metrics/machine_learning/onednn-dockerfile/Dockerfile b/tests/metrics/machine_learning/onednn-dockerfile/Dockerfile
new file mode 100644
index 0000000000..24ebb8cc9c
--- /dev/null
+++ b/tests/metrics/machine_learning/onednn-dockerfile/Dockerfile
@@ -0,0 +1,44 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Set up an Ubuntu image with 'phoronix-test-suite' installed
+
+FROM ubuntu:22.04
+
+LABEL DOCKERFILE_VERSION="1.0"
+
+ENV PHORONIX_TAR_URL="https://phoronix-test-suite.com/releases/"
+ENV PHORONIX_VER="phoronix-test-suite-10.8.4.tar.gz"
+ENV PHORONIX_CFG="/usr/share/phoronix-test-suite/pts-core/static/user-config-defaults.xml"
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN true
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends git curl build-essential autoconf && \
+ apt-get install -y --no-install-recommends php libapache2-mod-php php-bz2 php-gd php-sqlite3 php-xml && \
+ apt-get install -y --no-install-recommends cmake cmake-data mesa-utils vulkan-tools unzip apt-file && \
+ curl -OkL ${PHORONIX_TAR_URL}/${PHORONIX_VER} && \
+ tar zxf ${PHORONIX_VER} && cd phoronix-test-suite && ./install-sh && \
+ apt-get clean && rm -rf /var/lib/apt/lists && \
+ rm -f ${PHORONIX_VER} && \
+ if [ -n "$http_proxy" ] ; then \
+ sed -i "s||$(getent hosts "$(echo $http_proxy | sed 's~http[s]*://~~g' | sed -e 's/:[0-9]*//g')" | awk '{ print $1 }')|g" ${PHORONIX_CFG} ; \
+ sed -i "s||$(echo $http_proxy | sed 's/^.*://g')|g" ${PHORONIX_CFG} ; \
+ fi && \
+ sed -i "s|TRUE|FALSE|g" ${PHORONIX_CFG} && \
+ sed -i "s|TRUE|FALSE|g" ${PHORONIX_CFG} && \
+ sed -i "s|TRUE|FALSE|g" ${PHORONIX_CFG} && \
+ sed -i "s|TRUE|FALSE|g" ${PHORONIX_CFG} && \
+ sed -i "s|FALSE|TRUE|g" ${PHORONIX_CFG} && \
+ sed -i "s|20|2|g" ${PHORONIX_CFG} && \
+ sed -i "s|FALSE|TRUE|g" ${PHORONIX_CFG} && \
+ phoronix-test-suite download-test-files onednn && \
+ phoronix-test-suite install-dependencies onednn && \
+ phoronix-test-suite install onednn && \
+ sed -i "s/FALSE/TRUE/g" ${PHORONIX_CFG} && \
+ sed -i "s/20/3/g" ${PHORONIX_CFG}
+
+CMD ["/bin/bash"]
diff --git a/tests/metrics/machine_learning/onednn.sh b/tests/metrics/machine_learning/onednn.sh
new file mode 100755
index 0000000000..399807707b
--- /dev/null
+++ b/tests/metrics/machine_learning/onednn.sh
@@ -0,0 +1,117 @@
+#!/bin/bash
+#
+# Copyright (c) 2024 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Description of the test:
+# This test runs the 'onednn benchmark'
+# https://github.com/v8/web-tooling-benchmark
+
+set -o pipefail
+
+# General env
+SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
+source "${SCRIPT_PATH}/../lib/common.bash"
+
+# TEST_NAME is required to collect results and name the workload container.
+TEST_NAME="onednn-bench"
+
+WORKLOAD="phoronix-test-suite batch-run onednn"
+PAYLOAD_ARGS="${PAYLOAD_ARGS:-tail -f /dev/null}"
+
+IMAGE="docker.io/library/pts-onednn:latest"
+DOCKERFILE="${SCRIPT_PATH}/onednn-dockerfile/Dockerfile"
+
+TMP_DIR=$(mktemp --tmpdir -d onednn.XXXXXXXXXX)
+KATA_PERF_CONFIG="${TMP_DIR}/onednn_config.toml"
+TEST_RESULTS_FNAME="${TMP_DIR}/onednn-results.json"
+
+# Variable used to store the initial configuration file name.
+# This file is again pointed to by kata once the script finishes.
+KATA_INITIAL_CONFIG_FNAME=""
+
+function restore_kata_config() {
+ rm -rf "${TMP_DIR}"
+ set_kata_config_file "${KATA_INITIAL_CONFIG_FNAME}"
+}
+trap restore_kata_config EXIT
+
+# Show help about this script
+function help(){
+cat << EOF
+Usage: $0
+ Description:
+ Runs onednn benchmark using the maximum number of cpus and memory available.
+EOF
+}
+
+function save_config() {
+ metrics_json_start_array
+
+ pushd "${DEFAULT_KATA_CONFIG_DIR}"
+ local MEM_AVAIL_MB="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep -i "default_memory =" | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
+ local NUM_CPUS="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep -i "default_vcpus =" | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
+ local SHARED_FS="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep shared_fs | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
+ popd
+
+ local json="$(cat << EOF
+ {
+ "image": "${IMAGE}",
+ "units": "ms",
+ "mode": "Lower Is Better",
+ "shared-fs": "${SHARED_FS}",
+ "num-vcpus": "${NUM_CPUS}",
+ "memory-available-mb": "${MEM_AVAIL_MB}"
+ }
+EOF
+)"
+ metrics_json_add_array_element "${json}"
+ metrics_json_end_array "Config"
+}
+
+function main() {
+ local i=0
+ local cmds=("docker")
+ local RES_DIR="/var/lib/phoronix-test-suite/test-results"
+
+ # Check tools/commands dependencies
+ init_env
+ check_cmds "${cmds[@]}"
+ check_ctr_images "$IMAGE" "$DOCKERFILE"
+
+ clean_cache
+
+ # Configure Kata to use the maximum number of available CPUs
+ # and to use the available free memory.
+ get_current_kata_config_file KATA_INITIAL_CONFIG_FNAME
+ set_kata_configuration_performance "${KATA_PERF_CONFIG}"
+
+ # Launch container.
+ sudo -E "${CTR_EXE}" run -d --runtime "${CTR_RUNTIME}" "${IMAGE}" "${TEST_NAME}" sh -c "${PAYLOAD_ARGS}"
+
+ # Run the test.
+ sudo -E "${CTR_EXE}" t exec -t --exec-id "$(random_name)" "${TEST_NAME}" sh -c "${WORKLOAD}"
+
+ results_fname=$(sudo -E "${CTR_EXE}" t exec --exec-id $(random_name) ${TEST_NAME} sh -c "ls ${RES_DIR}")
+ SAVE_RESULTS_CMD="phoronix-test-suite result-file-to-json ${results_fname}"
+
+ # Save results.
+ sudo -E "${CTR_EXE}" t exec --exec-id "$(random_name)" "${TEST_NAME}" sh -c "${SAVE_RESULTS_CMD}"
+
+ # Extract results.
+ sudo -E "${CTR_EXE}" t exec --exec-id "${RANDOM}" "${TEST_NAME}" sh -c "cat /root/${results_fname}.json" > "${TEST_RESULTS_FNAME}"
+
+ cat <<< $(jq 'del(.systems[].data)' "${TEST_RESULTS_FNAME}") > "${TEST_RESULTS_FNAME}"
+ local results="$(cat ${TEST_RESULTS_FNAME})"
+
+ metrics_json_init
+ save_config
+ metrics_json_start_array
+ metrics_json_add_array_element "${results}"
+ metrics_json_end_array "Results"
+ metrics_json_save
+ clean_env_ctr
+}
+
+main "$@"
diff --git a/tests/metrics/machine_learning/openvino-dockerfile/Dockerfile b/tests/metrics/machine_learning/openvino-dockerfile/Dockerfile
new file mode 100644
index 0000000000..0349cdc419
--- /dev/null
+++ b/tests/metrics/machine_learning/openvino-dockerfile/Dockerfile
@@ -0,0 +1,46 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Set up an Ubuntu image with 'phoronix-test-suite' installed
+
+FROM ubuntu:22.04
+
+LABEL DOCKERFILE_VERSION="1.0"
+
+ENV PHORONIX_VER="10.8.4"
+ENV PHORONIX_URL="https://phoronix-test-suite.com/releases/"
+ENV PHORONIX_TAR_FILE="phoronix-test-suite-${PHORONIX_VER}.tar.gz"
+ENV PHORONIX_SRC="${PHORONIX_URL}/${PHORONIX_TAR_FILE}"
+ENV PHORONIX_CFG_ALT="/etc/phoronix-test-suite.xml"
+ENV PHORONIX_CFG="/usr/share/phoronix-test-suite/pts-core/static/user-config-defaults.xml"
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN true
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends git curl build-essential autoconf && \
+ apt-get install -y --no-install-recommends php libapache2-mod-php php-bz2 php-gd php-sqlite3 php-xml && \
+ apt-get install -y --no-install-recommends cmake cmake-data mesa-utils vulkan-tools unzip apt-file && \
+ curl -OkL ${PHORONIX_SRC} && \
+ tar zxf ${PHORONIX_TAR_FILE} && cd phoronix-test-suite && ./install-sh && \
+ apt-get clean && rm -rf /var/lib/apt/lists && \
+ rm -f ${PHORONIX_TAR_FILE} && \
+ if [ -n "$http_proxy" ] ; then \
+ sed -i "s||$(getent hosts "$(echo $http_proxy | sed 's~http[s]*://~~g' | sed -e 's/:[0-9]*//g')" | awk '{ print $1 }')|g" ${PHORONIX_CFG} ; \
+ sed -i "s||$(echo $http_proxy | sed 's/^.*://g')|g" ${PHORONIX_CFG} ; \
+ fi && \
+ sed -i "s|TRUE|FALSE|g" ${PHORONIX_CFG} && \
+ sed -i "s|TRUE|FALSE|g" ${PHORONIX_CFG} && \
+ sed -i "s|TRUE|FALSE|g" ${PHORONIX_CFG} && \
+ sed -i "s|TRUE|FALSE|g" ${PHORONIX_CFG} && \
+ sed -i "s|FALSE|TRUE|g" ${PHORONIX_CFG} && \
+ sed -i "s|20|2|g" ${PHORONIX_CFG} && \
+ sed -i "s|FALSE|TRUE|g" ${PHORONIX_CFG} && \
+ phoronix-test-suite download-test-files openvino && \
+ phoronix-test-suite install openvino && \
+ sed -i 's/FALSE/TRUE/g' ${PHORONIX_CFG} && \
+ sed -i 's/20/3/g' ${PHORONIX_CFG}
+
+CMD ["/bin/bash"]
diff --git a/tests/metrics/machine_learning/openvino.sh b/tests/metrics/machine_learning/openvino.sh
new file mode 100755
index 0000000000..358d3a3e8a
--- /dev/null
+++ b/tests/metrics/machine_learning/openvino.sh
@@ -0,0 +1,117 @@
+#!/bin/bash
+#
+# Copyright (c) 2024 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Description of the test:
+# This test runs the 'openvino benchmark'
+# https://github.com/v8/web-tooling-benchmark
+
+set -o pipefail
+
+# General env
+SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
+source "${SCRIPT_PATH}/../lib/common.bash"
+
+# TEST_NAME is required to collect results and name the workload container.
+TEST_NAME="openvino-bench"
+
+WORKLOAD="phoronix-test-suite batch-run openvino"
+PAYLOAD_ARGS="${PAYLOAD_ARGS:-tail -f /dev/null}"
+
+IMAGE="docker.io/library/pts-openvino:latest"
+DOCKERFILE="${SCRIPT_PATH}/openvino-dockerfile/Dockerfile"
+
+TMP_DIR=$(mktemp --tmpdir -d openvino.XXXXXXXXXX)
+KATA_PERF_CONFIG="${TMP_DIR}/openvino_config.toml"
+TEST_RESULTS_FNAME="${TMP_DIR}/openvino-results.json"
+
+# Variable used to store the initial configuration file name.
+# This file is again pointed to by kata once the script finishes.
+KATA_INITIAL_CONFIG_FNAME=""
+
+function restore_kata_config() {
+ rm -rf "${TMP_DIR}"
+ set_kata_config_file "${KATA_INITIAL_CONFIG_FNAME}"
+}
+trap restore_kata_config EXIT
+
+# Show help about this script
+function help(){
+cat << EOF
+Usage: $0
+ Description:
+ Runs onednn benchmark.
+EOF
+}
+
+function save_config() {
+ metrics_json_start_array
+
+ pushd "${DEFAULT_KATA_CONFIG_DIR}"
+ local MEM_AVAIL_MB="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep -i "default_memory =" | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
+ local NUM_CPUS="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep -i "default_vcpus =" | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
+ local SHARED_FS="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep shared_fs | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
+ popd
+
+ local json="$(cat << EOF
+ {
+ "image": "${IMAGE}",
+ "units": "ms",
+ "mode": "Lower Is Better",
+ "shared-fs": "${SHARED_FS}",
+ "num-vcpus": "${NUM_CPUS}",
+ "memory-available-mb": "${MEM_AVAIL_MB}"
+ }
+EOF
+)"
+ metrics_json_add_array_element "${json}"
+ metrics_json_end_array "Config"
+}
+
+function main() {
+ local i=0
+ local cmds=("docker")
+ local RES_DIR="/var/lib/phoronix-test-suite/test-results"
+
+ # Check tools/commands dependencies
+ init_env
+ check_cmds "${cmds[@]}"
+ check_ctr_images "$IMAGE" "$DOCKERFILE"
+
+ clean_cache
+
+ # Configure Kata to use the maximum number of available CPUs
+ # and to use the available free memory.
+ get_current_kata_config_file KATA_INITIAL_CONFIG_FNAME
+ set_kata_configuration_performance "${KATA_PERF_CONFIG}"
+
+ # Launch container.
+ sudo -E "${CTR_EXE}" run -d --runtime "${CTR_RUNTIME}" "${IMAGE}" "${TEST_NAME}" sh -c "${PAYLOAD_ARGS}"
+
+ # Run the test.
+ sudo -E "${CTR_EXE}" t exec -t --exec-id "$(random_name)" "${TEST_NAME}" sh -c "${WORKLOAD}"
+
+ results_fname=$(sudo -E "${CTR_EXE}" t exec --exec-id $(random_name) ${TEST_NAME} sh -c "ls ${RES_DIR}")
+ SAVE_RESULTS_CMD="phoronix-test-suite result-file-to-json ${results_fname}"
+
+ # Save results.
+ sudo -E "${CTR_EXE}" t exec --exec-id "$(random_name)" "${TEST_NAME}" sh -c "${SAVE_RESULTS_CMD}"
+
+ # Extract results.
+ sudo -E "${CTR_EXE}" t exec --exec-id "${RANDOM}" "${TEST_NAME}" sh -c "cat /root/${results_fname}.json" > "${TEST_RESULTS_FNAME}"
+
+ cat <<< $(jq 'del(.systems[].data)' "${TEST_RESULTS_FNAME}") > "${TEST_RESULTS_FNAME}"
+ local results="$(cat "${TEST_RESULTS_FNAME}")"
+
+ metrics_json_init
+ save_config
+ metrics_json_start_array
+ metrics_json_add_array_element "${results}"
+ metrics_json_end_array "Results"
+ metrics_json_save
+ clean_env_ctr
+}
+
+main "$@"