diff --git a/tests/metrics/lib/common.bash b/tests/metrics/lib/common.bash index e7d253136d..b16976df53 100755 --- a/tests/metrics/lib/common.bash +++ b/tests/metrics/lib/common.bash @@ -41,6 +41,10 @@ public.ecr.aws/lts mirror.gcr.io/library quay.io/libpod" +# Default Kata Configuration Directory +readonly DEFAULT_KATA_CONFIG_DIR="/opt/kata/share/defaults/kata-containers" +readonly DEFAULT_KATA_CONFIG_FNAME="configuration.toml" + # This function checks existence of commands. # They can be received standalone or as an array, e.g. # @@ -86,10 +90,14 @@ function generate_build_dockerfile() local map_key="$3" local text_to_replace="$4" local regs=(${registries["${map_key}"]}) + for r in ${regs[@]}; do sed 's|'${text_to_replace}'|'${r}'|g' \ "${dockerfile}.in" > "${dockerfile}" - if sudo "${DOCKER_EXE}" build --build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" --label "$image" --tag "${image}" -f "$dockerfile" "$dockerfile_dir"; then + if sudo -E "${DOCKER_EXE}" build \ + --build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" \ + --build-arg HTTP_PROXY="${http_proxy}" --build-arg HTTPS_PROXY="${https_proxy}" \ + --label "$image" --tag "${image}" -f "$dockerfile" "$dockerfile_dir"; then return 0 fi done @@ -107,7 +115,10 @@ function build_dockerfile_image() if [ -f "$dockerfile_path" ]; then info "docker building $image" - if ! sudo "${DOCKER_EXE}" build --build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" --label "$image" --tag "${image}" -f "$dockerfile_path" "$dockerfile_dir"; then + if ! sudo -E "${DOCKER_EXE}" build \ + --build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" \ + --build-arg HTTP_PROXY="${http_proxy}" --build-arg HTTPS_PROXY="${https_proxy}" \ + --label "$image" --tag "${image}" -f "$dockerfile_path" "$dockerfile_dir"; then die "Failed to docker build image $image" fi return 0 @@ -425,3 +436,75 @@ function check_containers_are_running() { return 1 fi } + +# This function receives an identifier used to name a performance Kata configuration file. +# The current kata configuration is copied to a new kata config file with the difference +# that the parameters 'default_vcpus' and 'default_memory' will be updated with the values +# of the max vcpus and the available memory in the system. +# Finally it makes Kata point to the new configuration file. +set_kata_configuration_performance() { + WORKLOAD_CONFIG_FILE="${1}" + + [ -z "${WORKLOAD_CONFIG_FILE}" ] && die "Unable to set a performance Kata configuration because the passed identifier is empty." + + local NUM_CPUS="$(nproc --all)" + local MEM_AVAIL_KB="$(grep MemAvailable /proc/meminfo | awk '{print $2}')" + local MEM_AVAIL_MB=$(echo "scale=0; $MEM_AVAIL_KB / 1024" | bc) + + info "Updating Kata configuration to increase memory and cpu resources assigned to the workload." + + # Copy the current kata configuration file to the workload config file, + # and increase memory size and num of vcpus assigned. + + pushd "${DEFAULT_KATA_CONFIG_DIR}" > /dev/null + + if [ ! -f "${DEFAULT_KATA_CONFIG_FNAME}" ]; then + die "Kata config file not found." + fi + + info "Changing the kata configuration to assign '${NUM_CPUS} vcpus' and ${MEM_AVAIL_MB} MB of memory to the performance workload." + + cp "${DEFAULT_KATA_CONFIG_FNAME}" "${WORKLOAD_CONFIG_FILE}" + ln -sf "${WORKLOAD_CONFIG_FILE}" "${DEFAULT_KATA_CONFIG_FNAME}" + + sed -i "s/default_memory =[^=&]*/default_memory = $MEM_AVAIL_MB/g" "${WORKLOAD_CONFIG_FILE}" + sed -i "s/default_vcpus =[^=&]*/default_vcpus = $NUM_CPUS/g" "${WORKLOAD_CONFIG_FILE}" + popd > /dev/null +} + +function clean_cache() { + sudo sync; echo 1 > /proc/sys/vm/drop_caches +} + +# This function receives as single parameter, the name of a valid Kata configuration file +# which will be established as the default Kata configuration to start new Kata containers. +function set_kata_config_file() { + NEW_KATA_CONFIG=${1} + + [ -z "${NEW_KATA_CONFIG}" ] && die "Failed to set a new Kata configuration because the configuration file was not not provided." + [ ! -d "${DEFAULT_KATA_CONFIG_DIR}" ] && die "Kata configuration directory was not found: ${DEFAULT_KATA_CONFIG_DIR}." + + pushd "${DEFAULT_KATA_CONFIG_DIR}" > /dev/null + + [ ! -f "${NEW_KATA_CONFIG}" ] && die "The Kata configuration file provided: ${NEW_KATA_CONFIG} was not found." + + info "Aplying a new Kata configuration using the file: ${NEW_KATA_CONFIG}." + + ln -sf "${NEW_KATA_CONFIG}" "${DEFAULT_KATA_CONFIG_FNAME}" + + popd > /dev/null +} + +function get_current_kata_config_file() { + declare -n current_config_file=$1 + + pushd "${DEFAULT_KATA_CONFIG_DIR}" > /dev/null + KATA_CONFIG_FNAME="$(readlink -f ${DEFAULT_KATA_CONFIG_FNAME})" + popd > /dev/null + + current_config_file="${KATA_CONFIG_FNAME}" +} + +function check_if_root() { + [ "$EUID" -ne 0 ] && die "Please run as root or use sudo." +} diff --git a/tests/metrics/machine_learning/onednn-dockerfile/Dockerfile b/tests/metrics/machine_learning/onednn-dockerfile/Dockerfile new file mode 100644 index 0000000000..24ebb8cc9c --- /dev/null +++ b/tests/metrics/machine_learning/onednn-dockerfile/Dockerfile @@ -0,0 +1,44 @@ +# Copyright (c) 2024 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +# Set up an Ubuntu image with 'phoronix-test-suite' installed + +FROM ubuntu:22.04 + +LABEL DOCKERFILE_VERSION="1.0" + +ENV PHORONIX_TAR_URL="https://phoronix-test-suite.com/releases/" +ENV PHORONIX_VER="phoronix-test-suite-10.8.4.tar.gz" +ENV PHORONIX_CFG="/usr/share/phoronix-test-suite/pts-core/static/user-config-defaults.xml" + +ENV DEBIAN_FRONTEND noninteractive +ENV DEBCONF_NONINTERACTIVE_SEEN true + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN apt-get update && \ + apt-get install -y --no-install-recommends git curl build-essential autoconf && \ + apt-get install -y --no-install-recommends php libapache2-mod-php php-bz2 php-gd php-sqlite3 php-xml && \ + apt-get install -y --no-install-recommends cmake cmake-data mesa-utils vulkan-tools unzip apt-file && \ + curl -OkL ${PHORONIX_TAR_URL}/${PHORONIX_VER} && \ + tar zxf ${PHORONIX_VER} && cd phoronix-test-suite && ./install-sh && \ + apt-get clean && rm -rf /var/lib/apt/lists && \ + rm -f ${PHORONIX_VER} && \ + if [ -n "$http_proxy" ] ; then \ + sed -i "s||$(getent hosts "$(echo $http_proxy | sed 's~http[s]*://~~g' | sed -e 's/:[0-9]*//g')" | awk '{ print $1 }')|g" ${PHORONIX_CFG} ; \ + sed -i "s||$(echo $http_proxy | sed 's/^.*://g')|g" ${PHORONIX_CFG} ; \ + fi && \ + sed -i "s|TRUE|FALSE|g" ${PHORONIX_CFG} && \ + sed -i "s|TRUE|FALSE|g" ${PHORONIX_CFG} && \ + sed -i "s|TRUE|FALSE|g" ${PHORONIX_CFG} && \ + sed -i "s|TRUE|FALSE|g" ${PHORONIX_CFG} && \ + sed -i "s|FALSE|TRUE|g" ${PHORONIX_CFG} && \ + sed -i "s|20|2|g" ${PHORONIX_CFG} && \ + sed -i "s|FALSE|TRUE|g" ${PHORONIX_CFG} && \ + phoronix-test-suite download-test-files onednn && \ + phoronix-test-suite install-dependencies onednn && \ + phoronix-test-suite install onednn && \ + sed -i "s/FALSE/TRUE/g" ${PHORONIX_CFG} && \ + sed -i "s/20/3/g" ${PHORONIX_CFG} + +CMD ["/bin/bash"] diff --git a/tests/metrics/machine_learning/onednn.sh b/tests/metrics/machine_learning/onednn.sh new file mode 100755 index 0000000000..399807707b --- /dev/null +++ b/tests/metrics/machine_learning/onednn.sh @@ -0,0 +1,117 @@ +#!/bin/bash +# +# Copyright (c) 2024 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +# Description of the test: +# This test runs the 'onednn benchmark' +# https://github.com/v8/web-tooling-benchmark + +set -o pipefail + +# General env +SCRIPT_PATH=$(dirname "$(readlink -f "$0")") +source "${SCRIPT_PATH}/../lib/common.bash" + +# TEST_NAME is required to collect results and name the workload container. +TEST_NAME="onednn-bench" + +WORKLOAD="phoronix-test-suite batch-run onednn" +PAYLOAD_ARGS="${PAYLOAD_ARGS:-tail -f /dev/null}" + +IMAGE="docker.io/library/pts-onednn:latest" +DOCKERFILE="${SCRIPT_PATH}/onednn-dockerfile/Dockerfile" + +TMP_DIR=$(mktemp --tmpdir -d onednn.XXXXXXXXXX) +KATA_PERF_CONFIG="${TMP_DIR}/onednn_config.toml" +TEST_RESULTS_FNAME="${TMP_DIR}/onednn-results.json" + +# Variable used to store the initial configuration file name. +# This file is again pointed to by kata once the script finishes. +KATA_INITIAL_CONFIG_FNAME="" + +function restore_kata_config() { + rm -rf "${TMP_DIR}" + set_kata_config_file "${KATA_INITIAL_CONFIG_FNAME}" +} +trap restore_kata_config EXIT + +# Show help about this script +function help(){ +cat << EOF +Usage: $0 + Description: + Runs onednn benchmark using the maximum number of cpus and memory available. +EOF +} + +function save_config() { + metrics_json_start_array + + pushd "${DEFAULT_KATA_CONFIG_DIR}" + local MEM_AVAIL_MB="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep -i "default_memory =" | cut -d "=" -f2 | tr -d ' ' | tr -d '"')" + local NUM_CPUS="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep -i "default_vcpus =" | cut -d "=" -f2 | tr -d ' ' | tr -d '"')" + local SHARED_FS="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep shared_fs | cut -d "=" -f2 | tr -d ' ' | tr -d '"')" + popd + + local json="$(cat << EOF + { + "image": "${IMAGE}", + "units": "ms", + "mode": "Lower Is Better", + "shared-fs": "${SHARED_FS}", + "num-vcpus": "${NUM_CPUS}", + "memory-available-mb": "${MEM_AVAIL_MB}" + } +EOF +)" + metrics_json_add_array_element "${json}" + metrics_json_end_array "Config" +} + +function main() { + local i=0 + local cmds=("docker") + local RES_DIR="/var/lib/phoronix-test-suite/test-results" + + # Check tools/commands dependencies + init_env + check_cmds "${cmds[@]}" + check_ctr_images "$IMAGE" "$DOCKERFILE" + + clean_cache + + # Configure Kata to use the maximum number of available CPUs + # and to use the available free memory. + get_current_kata_config_file KATA_INITIAL_CONFIG_FNAME + set_kata_configuration_performance "${KATA_PERF_CONFIG}" + + # Launch container. + sudo -E "${CTR_EXE}" run -d --runtime "${CTR_RUNTIME}" "${IMAGE}" "${TEST_NAME}" sh -c "${PAYLOAD_ARGS}" + + # Run the test. + sudo -E "${CTR_EXE}" t exec -t --exec-id "$(random_name)" "${TEST_NAME}" sh -c "${WORKLOAD}" + + results_fname=$(sudo -E "${CTR_EXE}" t exec --exec-id $(random_name) ${TEST_NAME} sh -c "ls ${RES_DIR}") + SAVE_RESULTS_CMD="phoronix-test-suite result-file-to-json ${results_fname}" + + # Save results. + sudo -E "${CTR_EXE}" t exec --exec-id "$(random_name)" "${TEST_NAME}" sh -c "${SAVE_RESULTS_CMD}" + + # Extract results. + sudo -E "${CTR_EXE}" t exec --exec-id "${RANDOM}" "${TEST_NAME}" sh -c "cat /root/${results_fname}.json" > "${TEST_RESULTS_FNAME}" + + cat <<< $(jq 'del(.systems[].data)' "${TEST_RESULTS_FNAME}") > "${TEST_RESULTS_FNAME}" + local results="$(cat ${TEST_RESULTS_FNAME})" + + metrics_json_init + save_config + metrics_json_start_array + metrics_json_add_array_element "${results}" + metrics_json_end_array "Results" + metrics_json_save + clean_env_ctr +} + +main "$@" diff --git a/tests/metrics/machine_learning/openvino-dockerfile/Dockerfile b/tests/metrics/machine_learning/openvino-dockerfile/Dockerfile new file mode 100644 index 0000000000..0349cdc419 --- /dev/null +++ b/tests/metrics/machine_learning/openvino-dockerfile/Dockerfile @@ -0,0 +1,46 @@ +# Copyright (c) 2024 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +# Set up an Ubuntu image with 'phoronix-test-suite' installed + +FROM ubuntu:22.04 + +LABEL DOCKERFILE_VERSION="1.0" + +ENV PHORONIX_VER="10.8.4" +ENV PHORONIX_URL="https://phoronix-test-suite.com/releases/" +ENV PHORONIX_TAR_FILE="phoronix-test-suite-${PHORONIX_VER}.tar.gz" +ENV PHORONIX_SRC="${PHORONIX_URL}/${PHORONIX_TAR_FILE}" +ENV PHORONIX_CFG_ALT="/etc/phoronix-test-suite.xml" +ENV PHORONIX_CFG="/usr/share/phoronix-test-suite/pts-core/static/user-config-defaults.xml" + +ENV DEBIAN_FRONTEND noninteractive +ENV DEBCONF_NONINTERACTIVE_SEEN true + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN apt-get update && \ + apt-get install -y --no-install-recommends git curl build-essential autoconf && \ + apt-get install -y --no-install-recommends php libapache2-mod-php php-bz2 php-gd php-sqlite3 php-xml && \ + apt-get install -y --no-install-recommends cmake cmake-data mesa-utils vulkan-tools unzip apt-file && \ + curl -OkL ${PHORONIX_SRC} && \ + tar zxf ${PHORONIX_TAR_FILE} && cd phoronix-test-suite && ./install-sh && \ + apt-get clean && rm -rf /var/lib/apt/lists && \ + rm -f ${PHORONIX_TAR_FILE} && \ + if [ -n "$http_proxy" ] ; then \ + sed -i "s||$(getent hosts "$(echo $http_proxy | sed 's~http[s]*://~~g' | sed -e 's/:[0-9]*//g')" | awk '{ print $1 }')|g" ${PHORONIX_CFG} ; \ + sed -i "s||$(echo $http_proxy | sed 's/^.*://g')|g" ${PHORONIX_CFG} ; \ + fi && \ + sed -i "s|TRUE|FALSE|g" ${PHORONIX_CFG} && \ + sed -i "s|TRUE|FALSE|g" ${PHORONIX_CFG} && \ + sed -i "s|TRUE|FALSE|g" ${PHORONIX_CFG} && \ + sed -i "s|TRUE|FALSE|g" ${PHORONIX_CFG} && \ + sed -i "s|FALSE|TRUE|g" ${PHORONIX_CFG} && \ + sed -i "s|20|2|g" ${PHORONIX_CFG} && \ + sed -i "s|FALSE|TRUE|g" ${PHORONIX_CFG} && \ + phoronix-test-suite download-test-files openvino && \ + phoronix-test-suite install openvino && \ + sed -i 's/FALSE/TRUE/g' ${PHORONIX_CFG} && \ + sed -i 's/20/3/g' ${PHORONIX_CFG} + +CMD ["/bin/bash"] diff --git a/tests/metrics/machine_learning/openvino.sh b/tests/metrics/machine_learning/openvino.sh new file mode 100755 index 0000000000..358d3a3e8a --- /dev/null +++ b/tests/metrics/machine_learning/openvino.sh @@ -0,0 +1,117 @@ +#!/bin/bash +# +# Copyright (c) 2024 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +# Description of the test: +# This test runs the 'openvino benchmark' +# https://github.com/v8/web-tooling-benchmark + +set -o pipefail + +# General env +SCRIPT_PATH=$(dirname "$(readlink -f "$0")") +source "${SCRIPT_PATH}/../lib/common.bash" + +# TEST_NAME is required to collect results and name the workload container. +TEST_NAME="openvino-bench" + +WORKLOAD="phoronix-test-suite batch-run openvino" +PAYLOAD_ARGS="${PAYLOAD_ARGS:-tail -f /dev/null}" + +IMAGE="docker.io/library/pts-openvino:latest" +DOCKERFILE="${SCRIPT_PATH}/openvino-dockerfile/Dockerfile" + +TMP_DIR=$(mktemp --tmpdir -d openvino.XXXXXXXXXX) +KATA_PERF_CONFIG="${TMP_DIR}/openvino_config.toml" +TEST_RESULTS_FNAME="${TMP_DIR}/openvino-results.json" + +# Variable used to store the initial configuration file name. +# This file is again pointed to by kata once the script finishes. +KATA_INITIAL_CONFIG_FNAME="" + +function restore_kata_config() { + rm -rf "${TMP_DIR}" + set_kata_config_file "${KATA_INITIAL_CONFIG_FNAME}" +} +trap restore_kata_config EXIT + +# Show help about this script +function help(){ +cat << EOF +Usage: $0 + Description: + Runs onednn benchmark. +EOF +} + +function save_config() { + metrics_json_start_array + + pushd "${DEFAULT_KATA_CONFIG_DIR}" + local MEM_AVAIL_MB="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep -i "default_memory =" | cut -d "=" -f2 | tr -d ' ' | tr -d '"')" + local NUM_CPUS="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep -i "default_vcpus =" | cut -d "=" -f2 | tr -d ' ' | tr -d '"')" + local SHARED_FS="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep shared_fs | cut -d "=" -f2 | tr -d ' ' | tr -d '"')" + popd + + local json="$(cat << EOF + { + "image": "${IMAGE}", + "units": "ms", + "mode": "Lower Is Better", + "shared-fs": "${SHARED_FS}", + "num-vcpus": "${NUM_CPUS}", + "memory-available-mb": "${MEM_AVAIL_MB}" + } +EOF +)" + metrics_json_add_array_element "${json}" + metrics_json_end_array "Config" +} + +function main() { + local i=0 + local cmds=("docker") + local RES_DIR="/var/lib/phoronix-test-suite/test-results" + + # Check tools/commands dependencies + init_env + check_cmds "${cmds[@]}" + check_ctr_images "$IMAGE" "$DOCKERFILE" + + clean_cache + + # Configure Kata to use the maximum number of available CPUs + # and to use the available free memory. + get_current_kata_config_file KATA_INITIAL_CONFIG_FNAME + set_kata_configuration_performance "${KATA_PERF_CONFIG}" + + # Launch container. + sudo -E "${CTR_EXE}" run -d --runtime "${CTR_RUNTIME}" "${IMAGE}" "${TEST_NAME}" sh -c "${PAYLOAD_ARGS}" + + # Run the test. + sudo -E "${CTR_EXE}" t exec -t --exec-id "$(random_name)" "${TEST_NAME}" sh -c "${WORKLOAD}" + + results_fname=$(sudo -E "${CTR_EXE}" t exec --exec-id $(random_name) ${TEST_NAME} sh -c "ls ${RES_DIR}") + SAVE_RESULTS_CMD="phoronix-test-suite result-file-to-json ${results_fname}" + + # Save results. + sudo -E "${CTR_EXE}" t exec --exec-id "$(random_name)" "${TEST_NAME}" sh -c "${SAVE_RESULTS_CMD}" + + # Extract results. + sudo -E "${CTR_EXE}" t exec --exec-id "${RANDOM}" "${TEST_NAME}" sh -c "cat /root/${results_fname}.json" > "${TEST_RESULTS_FNAME}" + + cat <<< $(jq 'del(.systems[].data)' "${TEST_RESULTS_FNAME}") > "${TEST_RESULTS_FNAME}" + local results="$(cat "${TEST_RESULTS_FNAME}")" + + metrics_json_init + save_config + metrics_json_start_array + metrics_json_add_array_element "${results}" + metrics_json_end_array "Results" + metrics_json_save + clean_env_ctr +} + +main "$@"