Merge pull request #9391 from dborquez/add-onednn-openvino-ml-benchs

add onednn and openvino ml-benchmarks
2025-08-30 23:37:45 +00:00 · 2024-04-09 19:03:00 -06:00
parent ea31df8bff facf3c9364
commit 9e1368dbc5
5 changed files with 409 additions and 2 deletions
--- a/tests/metrics/lib/common.bash
+++ b/tests/metrics/lib/common.bash
@@ -41,6 +41,10 @@ public.ecr.aws/lts
 mirror.gcr.io/library
 quay.io/libpod"

+# Default Kata Configuration Directory
+readonly DEFAULT_KATA_CONFIG_DIR="/opt/kata/share/defaults/kata-containers"
+readonly DEFAULT_KATA_CONFIG_FNAME="configuration.toml"
+
 # This function checks existence of commands.
 # They can be received standalone or as an array, e.g.
 #
@@ -86,10 +90,14 @@ function generate_build_dockerfile()
 	local map_key="$3"
 	local text_to_replace="$4"
 	local regs=(${registries["${map_key}"]})
+
 	for r in ${regs[@]}; do
 		sed 's|'${text_to_replace}'|'${r}'|g' \
 			"${dockerfile}.in" > "${dockerfile}"
-		if sudo "${DOCKER_EXE}" build --build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" --label "$image" --tag "${image}" -f "$dockerfile" "$dockerfile_dir"; then
+		if sudo -E "${DOCKER_EXE}" build \
+			--build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" \
+			--build-arg HTTP_PROXY="${http_proxy}" --build-arg HTTPS_PROXY="${https_proxy}" \
+			--label "$image" --tag "${image}" -f "$dockerfile" "$dockerfile_dir"; then
 			return 0
 		fi
 	done
@@ -107,7 +115,10 @@ function build_dockerfile_image()

 	if [ -f "$dockerfile_path" ]; then
 		info "docker building $image"
-		if ! sudo "${DOCKER_EXE}" build --build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" --label "$image" --tag "${image}" -f "$dockerfile_path" "$dockerfile_dir"; then
+		if ! sudo -E "${DOCKER_EXE}" build \
+			--build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" \
+			--build-arg HTTP_PROXY="${http_proxy}" --build-arg HTTPS_PROXY="${https_proxy}" \
+			--label "$image" --tag "${image}" -f "$dockerfile_path" "$dockerfile_dir"; then
 			die "Failed to docker build image $image"
 		fi
 		return 0
@@ -425,3 +436,75 @@ function check_containers_are_running() {
 		return 1
 	fi
 }
+
+# This function receives an identifier used to name a performance Kata configuration file.
+# The current kata configuration is copied to a new kata config file with the difference
+# that the parameters 'default_vcpus' and 'default_memory' will be updated with the values
+# of the max vcpus and the available memory in the system.
+# Finally it makes Kata point to the new configuration file.
+set_kata_configuration_performance() {
+	WORKLOAD_CONFIG_FILE="${1}"
+
+	[ -z "${WORKLOAD_CONFIG_FILE}" ] && die "Unable to set a performance Kata configuration because the passed identifier is empty."
+
+        local NUM_CPUS="$(nproc --all)"
+        local MEM_AVAIL_KB="$(grep MemAvailable /proc/meminfo | awk '{print $2}')"
+        local MEM_AVAIL_MB=$(echo "scale=0; $MEM_AVAIL_KB / 1024" | bc)
+
+        info "Updating Kata configuration to increase memory and cpu resources assigned to the workload."
+
+        # Copy the current kata configuration file to the workload config file,
+        # and increase memory size and num of vcpus assigned.
+
+        pushd "${DEFAULT_KATA_CONFIG_DIR}" > /dev/null
+
+                if [ ! -f "${DEFAULT_KATA_CONFIG_FNAME}" ]; then
+                        die "Kata config file not found."
+                fi
+
+                info "Changing the kata configuration to assign '${NUM_CPUS} vcpus' and ${MEM_AVAIL_MB} MB of memory to the performance workload."
+
+                cp "${DEFAULT_KATA_CONFIG_FNAME}" "${WORKLOAD_CONFIG_FILE}"
+                ln -sf "${WORKLOAD_CONFIG_FILE}" "${DEFAULT_KATA_CONFIG_FNAME}"
+
+                sed -i "s/default_memory =[^=&]*/default_memory = $MEM_AVAIL_MB/g" "${WORKLOAD_CONFIG_FILE}"
+                sed -i "s/default_vcpus =[^=&]*/default_vcpus = $NUM_CPUS/g" "${WORKLOAD_CONFIG_FILE}"
+        popd > /dev/null
+}
+
+function clean_cache() {
+        sudo sync; echo 1 > /proc/sys/vm/drop_caches
+}
+
+# This function receives as single parameter, the name of a valid Kata configuration file
+# which will be established as the default Kata configuration to start new Kata containers.
+function set_kata_config_file() {
+	NEW_KATA_CONFIG=${1}
+
+	[ -z "${NEW_KATA_CONFIG}" ] && die "Failed to set a new Kata configuration because the configuration file was not not provided."
+	[ ! -d "${DEFAULT_KATA_CONFIG_DIR}" ] && die "Kata configuration directory was not found: ${DEFAULT_KATA_CONFIG_DIR}."
+
+	pushd "${DEFAULT_KATA_CONFIG_DIR}" > /dev/null
+
+	[ ! -f "${NEW_KATA_CONFIG}" ] && die "The Kata configuration file provided: ${NEW_KATA_CONFIG} was not found."
+
+	info "Aplying a new Kata configuration using the file: ${NEW_KATA_CONFIG}."
+
+        ln -sf "${NEW_KATA_CONFIG}" "${DEFAULT_KATA_CONFIG_FNAME}"
+
+        popd > /dev/null
+}
+
+function get_current_kata_config_file() {
+	declare -n current_config_file=$1
+
+	pushd "${DEFAULT_KATA_CONFIG_DIR}" > /dev/null
+	KATA_CONFIG_FNAME="$(readlink -f ${DEFAULT_KATA_CONFIG_FNAME})"
+	popd > /dev/null
+
+	current_config_file="${KATA_CONFIG_FNAME}"
+}
+
+function check_if_root() {
+	[ "$EUID" -ne 0 ] && die "Please run as root or use sudo."
+}
--- a/tests/metrics/machine_learning/onednn-dockerfile/Dockerfile
+++ b/tests/metrics/machine_learning/onednn-dockerfile/Dockerfile
@@ -0,0 +1,44 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Set up an Ubuntu image with 'phoronix-test-suite' installed
+
+FROM ubuntu:22.04
+
+LABEL DOCKERFILE_VERSION="1.0"
+
+ENV PHORONIX_TAR_URL="https://phoronix-test-suite.com/releases/"
+ENV PHORONIX_VER="phoronix-test-suite-10.8.4.tar.gz"
+ENV PHORONIX_CFG="/usr/share/phoronix-test-suite/pts-core/static/user-config-defaults.xml"
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN true
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN apt-get update && \
+	apt-get install -y --no-install-recommends git curl build-essential autoconf && \
+	apt-get install -y --no-install-recommends php libapache2-mod-php php-bz2 php-gd php-sqlite3 php-xml && \
+	apt-get install -y --no-install-recommends cmake cmake-data mesa-utils vulkan-tools unzip apt-file && \
+        curl -OkL ${PHORONIX_TAR_URL}/${PHORONIX_VER} && \
+	tar zxf ${PHORONIX_VER} && cd phoronix-test-suite && ./install-sh && \
+	apt-get clean && rm -rf /var/lib/apt/lists && \
+	rm -f ${PHORONIX_VER} && \
+	if [ -n "$http_proxy" ] ; then \
+	sed -i "s|<ProxyAddress></ProxyAddress>|<ProxyAddress>$(getent hosts "$(echo $http_proxy | sed 's~http[s]*://~~g' | sed -e 's/:[0-9]*//g')" | awk '{ print $1 }')</ProxyAddress>|g" ${PHORONIX_CFG} ; \
+	sed -i "s|<ProxyPort></ProxyPort>|<ProxyPort>$(echo $http_proxy | sed 's/^.*://g')</ProxyPort>|g" ${PHORONIX_CFG} ; \
+	fi && \
+	sed -i "s|<UploadResults>TRUE</UploadResults>|<UploadResults>FALSE</UploadResults>|g" ${PHORONIX_CFG} && \
+	sed -i "s|<PromptForTestIdentifier>TRUE</PromptForTestIdentifier>|<PromptForTestIdentifier>FALSE</PromptForTestIdentifier>|g" ${PHORONIX_CFG} && \
+	sed -i "s|<PromptSaveName>TRUE</PromptSaveName>|<PromptSaveName>FALSE</PromptSaveName>|g" ${PHORONIX_CFG} && \
+	sed -i "s|<PromptForTestDescription>TRUE</PromptForTestDescription>|<PromptForTestDescription>FALSE</PromptForTestDescription>|g" ${PHORONIX_CFG} && \
+	sed -i "s|<Configured>FALSE</Configured>|<Configured>TRUE</Configured>|g" ${PHORONIX_CFG} && \
+	sed -i "s|<Timeout>20</Timeout>|<Timeout>2</Timeout>|g" ${PHORONIX_CFG} && \
+	sed -i "s|<SaveResults>FALSE</SaveResults>|<SaveResults>TRUE</SaveResults>|g" ${PHORONIX_CFG} && \
+	phoronix-test-suite download-test-files onednn && \
+	phoronix-test-suite install-dependencies onednn && \
+	phoronix-test-suite install onednn && \
+	sed -i "s/<NoNetworkCommunication>FALSE/<NoNetworkCommunication>TRUE/g" ${PHORONIX_CFG} && \
+	sed -i "s/<Timeout>20/<Timeout>3/g" ${PHORONIX_CFG}
+
+CMD ["/bin/bash"]
--- a/tests/metrics/machine_learning/onednn.sh
+++ b/tests/metrics/machine_learning/onednn.sh
@@ -0,0 +1,117 @@
+#!/bin/bash
+#
+# Copyright (c) 2024 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Description of the test:
+# This test runs the 'onednn benchmark'
+# https://github.com/v8/web-tooling-benchmark
+
+set -o pipefail
+
+# General env
+SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
+source "${SCRIPT_PATH}/../lib/common.bash"
+
+# TEST_NAME is required to collect results and name the workload container.
+TEST_NAME="onednn-bench"
+
+WORKLOAD="phoronix-test-suite batch-run onednn"
+PAYLOAD_ARGS="${PAYLOAD_ARGS:-tail -f /dev/null}"
+
+IMAGE="docker.io/library/pts-onednn:latest"
+DOCKERFILE="${SCRIPT_PATH}/onednn-dockerfile/Dockerfile"
+
+TMP_DIR=$(mktemp --tmpdir -d onednn.XXXXXXXXXX)
+KATA_PERF_CONFIG="${TMP_DIR}/onednn_config.toml"
+TEST_RESULTS_FNAME="${TMP_DIR}/onednn-results.json"
+
+# Variable used to store the initial configuration file name.
+# This file is again pointed to by kata once the script finishes.
+KATA_INITIAL_CONFIG_FNAME=""
+
+function restore_kata_config() {
+	rm -rf "${TMP_DIR}"
+	set_kata_config_file "${KATA_INITIAL_CONFIG_FNAME}"
+}
+trap restore_kata_config EXIT
+
+# Show help about this script
+function help(){
+cat << EOF
+Usage: $0
+   Description:
+       Runs onednn benchmark using the maximum number of cpus and memory available.
+EOF
+}
+
+function save_config() {
+	metrics_json_start_array
+
+	pushd "${DEFAULT_KATA_CONFIG_DIR}"
+        local MEM_AVAIL_MB="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep -i "default_memory =" | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
+	local NUM_CPUS="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep -i "default_vcpus =" | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
+	local SHARED_FS="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep shared_fs | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
+	popd
+
+	local json="$(cat << EOF
+	{
+		"image": "${IMAGE}",
+		"units": "ms",
+		"mode": "Lower Is Better",
+		"shared-fs": "${SHARED_FS}",
+		"num-vcpus": "${NUM_CPUS}",
+		"memory-available-mb": "${MEM_AVAIL_MB}"
+	}
+EOF
+)"
+	metrics_json_add_array_element "${json}"
+	metrics_json_end_array "Config"
+}
+
+function main() {
+	local i=0
+	local cmds=("docker")
+	local RES_DIR="/var/lib/phoronix-test-suite/test-results"
+
+	# Check tools/commands dependencies
+ 	init_env
+	check_cmds "${cmds[@]}"
+ 	check_ctr_images "$IMAGE" "$DOCKERFILE"
+
+	clean_cache
+
+	# Configure Kata to use the maximum number of available CPUs
+	# and to use the available free memory.
+	get_current_kata_config_file KATA_INITIAL_CONFIG_FNAME
+	set_kata_configuration_performance "${KATA_PERF_CONFIG}"
+
+	# Launch container.
+	sudo -E "${CTR_EXE}" run -d --runtime "${CTR_RUNTIME}" "${IMAGE}" "${TEST_NAME}" sh -c "${PAYLOAD_ARGS}"
+
+	# Run the test.
+	sudo -E "${CTR_EXE}" t exec -t --exec-id "$(random_name)" "${TEST_NAME}" sh -c "${WORKLOAD}"
+
+	results_fname=$(sudo -E "${CTR_EXE}" t exec --exec-id $(random_name) ${TEST_NAME} sh -c "ls ${RES_DIR}")
+	SAVE_RESULTS_CMD="phoronix-test-suite result-file-to-json ${results_fname}"
+
+	# Save results.
+	sudo -E "${CTR_EXE}" t exec --exec-id "$(random_name)" "${TEST_NAME}" sh -c "${SAVE_RESULTS_CMD}"
+
+	# Extract results.
+	sudo -E "${CTR_EXE}" t exec --exec-id "${RANDOM}" "${TEST_NAME}" sh -c "cat /root/${results_fname}.json" > "${TEST_RESULTS_FNAME}"
+
+	cat <<< $(jq 'del(.systems[].data)' "${TEST_RESULTS_FNAME}") > "${TEST_RESULTS_FNAME}"
+	local results="$(cat ${TEST_RESULTS_FNAME})"
+
+	metrics_json_init
+	save_config
+	metrics_json_start_array
+	metrics_json_add_array_element "${results}"
+	metrics_json_end_array "Results"
+	metrics_json_save
+	clean_env_ctr
+}
+
+main "$@"
--- a/tests/metrics/machine_learning/openvino-dockerfile/Dockerfile
+++ b/tests/metrics/machine_learning/openvino-dockerfile/Dockerfile
@@ -0,0 +1,46 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Set up an Ubuntu image with 'phoronix-test-suite' installed
+
+FROM ubuntu:22.04
+
+LABEL DOCKERFILE_VERSION="1.0"
+
+ENV PHORONIX_VER="10.8.4"
+ENV PHORONIX_URL="https://phoronix-test-suite.com/releases/"
+ENV PHORONIX_TAR_FILE="phoronix-test-suite-${PHORONIX_VER}.tar.gz"
+ENV PHORONIX_SRC="${PHORONIX_URL}/${PHORONIX_TAR_FILE}"
+ENV PHORONIX_CFG_ALT="/etc/phoronix-test-suite.xml"
+ENV PHORONIX_CFG="/usr/share/phoronix-test-suite/pts-core/static/user-config-defaults.xml"
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN true
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN apt-get update && \
+	apt-get install -y --no-install-recommends git curl build-essential autoconf && \
+	apt-get install -y --no-install-recommends php libapache2-mod-php php-bz2 php-gd php-sqlite3 php-xml && \
+	apt-get install -y --no-install-recommends cmake cmake-data mesa-utils vulkan-tools unzip apt-file && \
+        curl -OkL ${PHORONIX_SRC} && \
+	tar zxf ${PHORONIX_TAR_FILE} && cd phoronix-test-suite && ./install-sh && \
+	apt-get clean && rm -rf /var/lib/apt/lists && \
+	rm -f ${PHORONIX_TAR_FILE} && \
+	if [ -n "$http_proxy" ] ; then \
+	sed -i "s|<ProxyAddress></ProxyAddress>|<ProxyAddress>$(getent hosts "$(echo $http_proxy | sed 's~http[s]*://~~g' | sed -e 's/:[0-9]*//g')" | awk '{ print $1 }')</ProxyAddress>|g" ${PHORONIX_CFG} ; \
+	sed -i "s|<ProxyPort></ProxyPort>|<ProxyPort>$(echo $http_proxy | sed 's/^.*://g')</ProxyPort>|g" ${PHORONIX_CFG} ; \
+	fi && \
+	sed -i "s|<UploadResults>TRUE</UploadResults>|<UploadResults>FALSE</UploadResults>|g" ${PHORONIX_CFG} && \
+	sed -i "s|<PromptForTestIdentifier>TRUE</PromptForTestIdentifier>|<PromptForTestIdentifier>FALSE</PromptForTestIdentifier>|g" ${PHORONIX_CFG} && \
+	sed -i "s|<PromptSaveName>TRUE</PromptSaveName>|<PromptSaveName>FALSE</PromptSaveName>|g" ${PHORONIX_CFG} && \
+	sed -i "s|<PromptForTestDescription>TRUE</PromptForTestDescription>|<PromptForTestDescription>FALSE</PromptForTestDescription>|g" ${PHORONIX_CFG} && \
+	sed -i "s|<Configured>FALSE</Configured>|<Configured>TRUE</Configured>|g" ${PHORONIX_CFG} && \
+	sed -i "s|<Timeout>20</Timeout>|<Timeout>2</Timeout>|g" ${PHORONIX_CFG} && \
+	sed -i "s|<SaveResults>FALSE</SaveResults>|<SaveResults>TRUE</SaveResults>|g" ${PHORONIX_CFG} && \
+	phoronix-test-suite download-test-files openvino && \
+	phoronix-test-suite install openvino && \
+	sed -i 's/<NoNetworkCommunication>FALSE/<NoNetworkCommunication>TRUE/g' ${PHORONIX_CFG} && \
+	sed -i 's/<Timeout>20/<Timeout>3/g' ${PHORONIX_CFG}
+
+CMD ["/bin/bash"]
--- a/tests/metrics/machine_learning/openvino.sh
+++ b/tests/metrics/machine_learning/openvino.sh
@@ -0,0 +1,117 @@
+#!/bin/bash
+#
+# Copyright (c) 2024 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Description of the test:
+# This test runs the 'openvino benchmark'
+# https://github.com/v8/web-tooling-benchmark
+
+set -o pipefail
+
+# General env
+SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
+source "${SCRIPT_PATH}/../lib/common.bash"
+
+# TEST_NAME is required to collect results and name the workload container.
+TEST_NAME="openvino-bench"
+
+WORKLOAD="phoronix-test-suite batch-run openvino"
+PAYLOAD_ARGS="${PAYLOAD_ARGS:-tail -f /dev/null}"
+
+IMAGE="docker.io/library/pts-openvino:latest"
+DOCKERFILE="${SCRIPT_PATH}/openvino-dockerfile/Dockerfile"
+
+TMP_DIR=$(mktemp --tmpdir -d openvino.XXXXXXXXXX)
+KATA_PERF_CONFIG="${TMP_DIR}/openvino_config.toml"
+TEST_RESULTS_FNAME="${TMP_DIR}/openvino-results.json"
+
+# Variable used to store the initial configuration file name.
+# This file is again pointed to by kata once the script finishes.
+KATA_INITIAL_CONFIG_FNAME=""
+
+function restore_kata_config() {
+	rm -rf "${TMP_DIR}"
+	set_kata_config_file "${KATA_INITIAL_CONFIG_FNAME}"
+}
+trap restore_kata_config EXIT
+
+# Show help about this script
+function help(){
+cat << EOF
+Usage: $0
+   Description:
+       Runs onednn benchmark.
+EOF
+}
+
+function save_config() {
+	metrics_json_start_array
+
+	pushd "${DEFAULT_KATA_CONFIG_DIR}"
+	local MEM_AVAIL_MB="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep -i "default_memory =" | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
+	local NUM_CPUS="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep -i "default_vcpus =" | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
+	local SHARED_FS="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep shared_fs | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
+	popd
+
+	local json="$(cat << EOF
+	{
+		"image": "${IMAGE}",
+		"units": "ms",
+		"mode": "Lower Is Better",
+		"shared-fs": "${SHARED_FS}",
+		"num-vcpus": "${NUM_CPUS}",
+		"memory-available-mb": "${MEM_AVAIL_MB}"
+	}
+EOF
+)"
+	metrics_json_add_array_element "${json}"
+	metrics_json_end_array "Config"
+}
+
+function main() {
+	local i=0
+	local cmds=("docker")
+	local RES_DIR="/var/lib/phoronix-test-suite/test-results"
+
+	# Check tools/commands dependencies
+ 	init_env
+	check_cmds "${cmds[@]}"
+ 	check_ctr_images "$IMAGE" "$DOCKERFILE"
+
+	clean_cache
+
+	# Configure Kata to use the maximum number of available CPUs
+	# and to use the available free memory.
+	get_current_kata_config_file KATA_INITIAL_CONFIG_FNAME
+	set_kata_configuration_performance "${KATA_PERF_CONFIG}"
+
+	# Launch container.
+	sudo -E "${CTR_EXE}" run -d --runtime "${CTR_RUNTIME}" "${IMAGE}" "${TEST_NAME}" sh -c "${PAYLOAD_ARGS}"
+
+	# Run the test.
+	sudo -E "${CTR_EXE}" t exec -t --exec-id "$(random_name)" "${TEST_NAME}" sh -c "${WORKLOAD}"
+
+	results_fname=$(sudo -E "${CTR_EXE}" t exec --exec-id $(random_name) ${TEST_NAME} sh -c "ls ${RES_DIR}")
+	SAVE_RESULTS_CMD="phoronix-test-suite result-file-to-json ${results_fname}"
+
+	# Save results.
+	sudo -E "${CTR_EXE}" t exec --exec-id "$(random_name)" "${TEST_NAME}" sh -c "${SAVE_RESULTS_CMD}"
+
+	# Extract results.
+	sudo -E "${CTR_EXE}" t exec --exec-id "${RANDOM}" "${TEST_NAME}" sh -c "cat /root/${results_fname}.json" > "${TEST_RESULTS_FNAME}"
+
+	cat <<< $(jq 'del(.systems[].data)' "${TEST_RESULTS_FNAME}") > "${TEST_RESULTS_FNAME}"
+	local results="$(cat "${TEST_RESULTS_FNAME}")"
+
+	metrics_json_init
+	save_config
+	metrics_json_start_array
+	metrics_json_add_array_element "${results}"
+	metrics_json_end_array "Results"
+	metrics_json_save
+	clean_env_ctr
+}
+
+main "$@"