Merge pull request #9391 from dborquez/add-onednn-openvino-ml-benchs

add onednn and openvino ml-benchmarks
This commit is contained in:
David Esparza
2024-04-09 19:03:00 -06:00
committed by GitHub
5 changed files with 409 additions and 2 deletions

View File

@@ -41,6 +41,10 @@ public.ecr.aws/lts
mirror.gcr.io/library
quay.io/libpod"
# Default Kata Configuration Directory
readonly DEFAULT_KATA_CONFIG_DIR="/opt/kata/share/defaults/kata-containers"
readonly DEFAULT_KATA_CONFIG_FNAME="configuration.toml"
# This function checks existence of commands.
# They can be received standalone or as an array, e.g.
#
@@ -86,10 +90,14 @@ function generate_build_dockerfile()
local map_key="$3"
local text_to_replace="$4"
local regs=(${registries["${map_key}"]})
for r in ${regs[@]}; do
sed 's|'${text_to_replace}'|'${r}'|g' \
"${dockerfile}.in" > "${dockerfile}"
if sudo "${DOCKER_EXE}" build --build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" --label "$image" --tag "${image}" -f "$dockerfile" "$dockerfile_dir"; then
if sudo -E "${DOCKER_EXE}" build \
--build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" \
--build-arg HTTP_PROXY="${http_proxy}" --build-arg HTTPS_PROXY="${https_proxy}" \
--label "$image" --tag "${image}" -f "$dockerfile" "$dockerfile_dir"; then
return 0
fi
done
@@ -107,7 +115,10 @@ function build_dockerfile_image()
if [ -f "$dockerfile_path" ]; then
info "docker building $image"
if ! sudo "${DOCKER_EXE}" build --build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" --label "$image" --tag "${image}" -f "$dockerfile_path" "$dockerfile_dir"; then
if ! sudo -E "${DOCKER_EXE}" build \
--build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" \
--build-arg HTTP_PROXY="${http_proxy}" --build-arg HTTPS_PROXY="${https_proxy}" \
--label "$image" --tag "${image}" -f "$dockerfile_path" "$dockerfile_dir"; then
die "Failed to docker build image $image"
fi
return 0
@@ -425,3 +436,75 @@ function check_containers_are_running() {
return 1
fi
}
# This function receives an identifier used to name a performance Kata configuration file.
# The current kata configuration is copied to a new kata config file with the difference
# that the parameters 'default_vcpus' and 'default_memory' will be updated with the values
# of the max vcpus and the available memory in the system.
# Finally it makes Kata point to the new configuration file.
set_kata_configuration_performance() {
WORKLOAD_CONFIG_FILE="${1}"
[ -z "${WORKLOAD_CONFIG_FILE}" ] && die "Unable to set a performance Kata configuration because the passed identifier is empty."
local NUM_CPUS="$(nproc --all)"
local MEM_AVAIL_KB="$(grep MemAvailable /proc/meminfo | awk '{print $2}')"
local MEM_AVAIL_MB=$(echo "scale=0; $MEM_AVAIL_KB / 1024" | bc)
info "Updating Kata configuration to increase memory and cpu resources assigned to the workload."
# Copy the current kata configuration file to the workload config file,
# and increase memory size and num of vcpus assigned.
pushd "${DEFAULT_KATA_CONFIG_DIR}" > /dev/null
if [ ! -f "${DEFAULT_KATA_CONFIG_FNAME}" ]; then
die "Kata config file not found."
fi
info "Changing the kata configuration to assign '${NUM_CPUS} vcpus' and ${MEM_AVAIL_MB} MB of memory to the performance workload."
cp "${DEFAULT_KATA_CONFIG_FNAME}" "${WORKLOAD_CONFIG_FILE}"
ln -sf "${WORKLOAD_CONFIG_FILE}" "${DEFAULT_KATA_CONFIG_FNAME}"
sed -i "s/default_memory =[^=&]*/default_memory = $MEM_AVAIL_MB/g" "${WORKLOAD_CONFIG_FILE}"
sed -i "s/default_vcpus =[^=&]*/default_vcpus = $NUM_CPUS/g" "${WORKLOAD_CONFIG_FILE}"
popd > /dev/null
}
function clean_cache() {
sudo sync; echo 1 > /proc/sys/vm/drop_caches
}
# This function receives as single parameter, the name of a valid Kata configuration file
# which will be established as the default Kata configuration to start new Kata containers.
function set_kata_config_file() {
NEW_KATA_CONFIG=${1}
[ -z "${NEW_KATA_CONFIG}" ] && die "Failed to set a new Kata configuration because the configuration file was not not provided."
[ ! -d "${DEFAULT_KATA_CONFIG_DIR}" ] && die "Kata configuration directory was not found: ${DEFAULT_KATA_CONFIG_DIR}."
pushd "${DEFAULT_KATA_CONFIG_DIR}" > /dev/null
[ ! -f "${NEW_KATA_CONFIG}" ] && die "The Kata configuration file provided: ${NEW_KATA_CONFIG} was not found."
info "Aplying a new Kata configuration using the file: ${NEW_KATA_CONFIG}."
ln -sf "${NEW_KATA_CONFIG}" "${DEFAULT_KATA_CONFIG_FNAME}"
popd > /dev/null
}
function get_current_kata_config_file() {
declare -n current_config_file=$1
pushd "${DEFAULT_KATA_CONFIG_DIR}" > /dev/null
KATA_CONFIG_FNAME="$(readlink -f ${DEFAULT_KATA_CONFIG_FNAME})"
popd > /dev/null
current_config_file="${KATA_CONFIG_FNAME}"
}
function check_if_root() {
[ "$EUID" -ne 0 ] && die "Please run as root or use sudo."
}

View File

@@ -0,0 +1,44 @@
# Copyright (c) 2024 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
# Set up an Ubuntu image with 'phoronix-test-suite' installed
FROM ubuntu:22.04
LABEL DOCKERFILE_VERSION="1.0"
ENV PHORONIX_TAR_URL="https://phoronix-test-suite.com/releases/"
ENV PHORONIX_VER="phoronix-test-suite-10.8.4.tar.gz"
ENV PHORONIX_CFG="/usr/share/phoronix-test-suite/pts-core/static/user-config-defaults.xml"
ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN true
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN apt-get update && \
apt-get install -y --no-install-recommends git curl build-essential autoconf && \
apt-get install -y --no-install-recommends php libapache2-mod-php php-bz2 php-gd php-sqlite3 php-xml && \
apt-get install -y --no-install-recommends cmake cmake-data mesa-utils vulkan-tools unzip apt-file && \
curl -OkL ${PHORONIX_TAR_URL}/${PHORONIX_VER} && \
tar zxf ${PHORONIX_VER} && cd phoronix-test-suite && ./install-sh && \
apt-get clean && rm -rf /var/lib/apt/lists && \
rm -f ${PHORONIX_VER} && \
if [ -n "$http_proxy" ] ; then \
sed -i "s|<ProxyAddress></ProxyAddress>|<ProxyAddress>$(getent hosts "$(echo $http_proxy | sed 's~http[s]*://~~g' | sed -e 's/:[0-9]*//g')" | awk '{ print $1 }')</ProxyAddress>|g" ${PHORONIX_CFG} ; \
sed -i "s|<ProxyPort></ProxyPort>|<ProxyPort>$(echo $http_proxy | sed 's/^.*://g')</ProxyPort>|g" ${PHORONIX_CFG} ; \
fi && \
sed -i "s|<UploadResults>TRUE</UploadResults>|<UploadResults>FALSE</UploadResults>|g" ${PHORONIX_CFG} && \
sed -i "s|<PromptForTestIdentifier>TRUE</PromptForTestIdentifier>|<PromptForTestIdentifier>FALSE</PromptForTestIdentifier>|g" ${PHORONIX_CFG} && \
sed -i "s|<PromptSaveName>TRUE</PromptSaveName>|<PromptSaveName>FALSE</PromptSaveName>|g" ${PHORONIX_CFG} && \
sed -i "s|<PromptForTestDescription>TRUE</PromptForTestDescription>|<PromptForTestDescription>FALSE</PromptForTestDescription>|g" ${PHORONIX_CFG} && \
sed -i "s|<Configured>FALSE</Configured>|<Configured>TRUE</Configured>|g" ${PHORONIX_CFG} && \
sed -i "s|<Timeout>20</Timeout>|<Timeout>2</Timeout>|g" ${PHORONIX_CFG} && \
sed -i "s|<SaveResults>FALSE</SaveResults>|<SaveResults>TRUE</SaveResults>|g" ${PHORONIX_CFG} && \
phoronix-test-suite download-test-files onednn && \
phoronix-test-suite install-dependencies onednn && \
phoronix-test-suite install onednn && \
sed -i "s/<NoNetworkCommunication>FALSE/<NoNetworkCommunication>TRUE/g" ${PHORONIX_CFG} && \
sed -i "s/<Timeout>20/<Timeout>3/g" ${PHORONIX_CFG}
CMD ["/bin/bash"]

View File

@@ -0,0 +1,117 @@
#!/bin/bash
#
# Copyright (c) 2024 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
# Description of the test:
# This test runs the 'onednn benchmark'
# https://github.com/v8/web-tooling-benchmark
set -o pipefail
# General env
SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
source "${SCRIPT_PATH}/../lib/common.bash"
# TEST_NAME is required to collect results and name the workload container.
TEST_NAME="onednn-bench"
WORKLOAD="phoronix-test-suite batch-run onednn"
PAYLOAD_ARGS="${PAYLOAD_ARGS:-tail -f /dev/null}"
IMAGE="docker.io/library/pts-onednn:latest"
DOCKERFILE="${SCRIPT_PATH}/onednn-dockerfile/Dockerfile"
TMP_DIR=$(mktemp --tmpdir -d onednn.XXXXXXXXXX)
KATA_PERF_CONFIG="${TMP_DIR}/onednn_config.toml"
TEST_RESULTS_FNAME="${TMP_DIR}/onednn-results.json"
# Variable used to store the initial configuration file name.
# This file is again pointed to by kata once the script finishes.
KATA_INITIAL_CONFIG_FNAME=""
function restore_kata_config() {
rm -rf "${TMP_DIR}"
set_kata_config_file "${KATA_INITIAL_CONFIG_FNAME}"
}
trap restore_kata_config EXIT
# Show help about this script
function help(){
cat << EOF
Usage: $0
Description:
Runs onednn benchmark using the maximum number of cpus and memory available.
EOF
}
function save_config() {
metrics_json_start_array
pushd "${DEFAULT_KATA_CONFIG_DIR}"
local MEM_AVAIL_MB="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep -i "default_memory =" | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
local NUM_CPUS="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep -i "default_vcpus =" | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
local SHARED_FS="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep shared_fs | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
popd
local json="$(cat << EOF
{
"image": "${IMAGE}",
"units": "ms",
"mode": "Lower Is Better",
"shared-fs": "${SHARED_FS}",
"num-vcpus": "${NUM_CPUS}",
"memory-available-mb": "${MEM_AVAIL_MB}"
}
EOF
)"
metrics_json_add_array_element "${json}"
metrics_json_end_array "Config"
}
function main() {
local i=0
local cmds=("docker")
local RES_DIR="/var/lib/phoronix-test-suite/test-results"
# Check tools/commands dependencies
init_env
check_cmds "${cmds[@]}"
check_ctr_images "$IMAGE" "$DOCKERFILE"
clean_cache
# Configure Kata to use the maximum number of available CPUs
# and to use the available free memory.
get_current_kata_config_file KATA_INITIAL_CONFIG_FNAME
set_kata_configuration_performance "${KATA_PERF_CONFIG}"
# Launch container.
sudo -E "${CTR_EXE}" run -d --runtime "${CTR_RUNTIME}" "${IMAGE}" "${TEST_NAME}" sh -c "${PAYLOAD_ARGS}"
# Run the test.
sudo -E "${CTR_EXE}" t exec -t --exec-id "$(random_name)" "${TEST_NAME}" sh -c "${WORKLOAD}"
results_fname=$(sudo -E "${CTR_EXE}" t exec --exec-id $(random_name) ${TEST_NAME} sh -c "ls ${RES_DIR}")
SAVE_RESULTS_CMD="phoronix-test-suite result-file-to-json ${results_fname}"
# Save results.
sudo -E "${CTR_EXE}" t exec --exec-id "$(random_name)" "${TEST_NAME}" sh -c "${SAVE_RESULTS_CMD}"
# Extract results.
sudo -E "${CTR_EXE}" t exec --exec-id "${RANDOM}" "${TEST_NAME}" sh -c "cat /root/${results_fname}.json" > "${TEST_RESULTS_FNAME}"
cat <<< $(jq 'del(.systems[].data)' "${TEST_RESULTS_FNAME}") > "${TEST_RESULTS_FNAME}"
local results="$(cat ${TEST_RESULTS_FNAME})"
metrics_json_init
save_config
metrics_json_start_array
metrics_json_add_array_element "${results}"
metrics_json_end_array "Results"
metrics_json_save
clean_env_ctr
}
main "$@"

View File

@@ -0,0 +1,46 @@
# Copyright (c) 2024 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
# Set up an Ubuntu image with 'phoronix-test-suite' installed
FROM ubuntu:22.04
LABEL DOCKERFILE_VERSION="1.0"
ENV PHORONIX_VER="10.8.4"
ENV PHORONIX_URL="https://phoronix-test-suite.com/releases/"
ENV PHORONIX_TAR_FILE="phoronix-test-suite-${PHORONIX_VER}.tar.gz"
ENV PHORONIX_SRC="${PHORONIX_URL}/${PHORONIX_TAR_FILE}"
ENV PHORONIX_CFG_ALT="/etc/phoronix-test-suite.xml"
ENV PHORONIX_CFG="/usr/share/phoronix-test-suite/pts-core/static/user-config-defaults.xml"
ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN true
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN apt-get update && \
apt-get install -y --no-install-recommends git curl build-essential autoconf && \
apt-get install -y --no-install-recommends php libapache2-mod-php php-bz2 php-gd php-sqlite3 php-xml && \
apt-get install -y --no-install-recommends cmake cmake-data mesa-utils vulkan-tools unzip apt-file && \
curl -OkL ${PHORONIX_SRC} && \
tar zxf ${PHORONIX_TAR_FILE} && cd phoronix-test-suite && ./install-sh && \
apt-get clean && rm -rf /var/lib/apt/lists && \
rm -f ${PHORONIX_TAR_FILE} && \
if [ -n "$http_proxy" ] ; then \
sed -i "s|<ProxyAddress></ProxyAddress>|<ProxyAddress>$(getent hosts "$(echo $http_proxy | sed 's~http[s]*://~~g' | sed -e 's/:[0-9]*//g')" | awk '{ print $1 }')</ProxyAddress>|g" ${PHORONIX_CFG} ; \
sed -i "s|<ProxyPort></ProxyPort>|<ProxyPort>$(echo $http_proxy | sed 's/^.*://g')</ProxyPort>|g" ${PHORONIX_CFG} ; \
fi && \
sed -i "s|<UploadResults>TRUE</UploadResults>|<UploadResults>FALSE</UploadResults>|g" ${PHORONIX_CFG} && \
sed -i "s|<PromptForTestIdentifier>TRUE</PromptForTestIdentifier>|<PromptForTestIdentifier>FALSE</PromptForTestIdentifier>|g" ${PHORONIX_CFG} && \
sed -i "s|<PromptSaveName>TRUE</PromptSaveName>|<PromptSaveName>FALSE</PromptSaveName>|g" ${PHORONIX_CFG} && \
sed -i "s|<PromptForTestDescription>TRUE</PromptForTestDescription>|<PromptForTestDescription>FALSE</PromptForTestDescription>|g" ${PHORONIX_CFG} && \
sed -i "s|<Configured>FALSE</Configured>|<Configured>TRUE</Configured>|g" ${PHORONIX_CFG} && \
sed -i "s|<Timeout>20</Timeout>|<Timeout>2</Timeout>|g" ${PHORONIX_CFG} && \
sed -i "s|<SaveResults>FALSE</SaveResults>|<SaveResults>TRUE</SaveResults>|g" ${PHORONIX_CFG} && \
phoronix-test-suite download-test-files openvino && \
phoronix-test-suite install openvino && \
sed -i 's/<NoNetworkCommunication>FALSE/<NoNetworkCommunication>TRUE/g' ${PHORONIX_CFG} && \
sed -i 's/<Timeout>20/<Timeout>3/g' ${PHORONIX_CFG}
CMD ["/bin/bash"]

View File

@@ -0,0 +1,117 @@
#!/bin/bash
#
# Copyright (c) 2024 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
# Description of the test:
# This test runs the 'openvino benchmark'
# https://github.com/v8/web-tooling-benchmark
set -o pipefail
# General env
SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
source "${SCRIPT_PATH}/../lib/common.bash"
# TEST_NAME is required to collect results and name the workload container.
TEST_NAME="openvino-bench"
WORKLOAD="phoronix-test-suite batch-run openvino"
PAYLOAD_ARGS="${PAYLOAD_ARGS:-tail -f /dev/null}"
IMAGE="docker.io/library/pts-openvino:latest"
DOCKERFILE="${SCRIPT_PATH}/openvino-dockerfile/Dockerfile"
TMP_DIR=$(mktemp --tmpdir -d openvino.XXXXXXXXXX)
KATA_PERF_CONFIG="${TMP_DIR}/openvino_config.toml"
TEST_RESULTS_FNAME="${TMP_DIR}/openvino-results.json"
# Variable used to store the initial configuration file name.
# This file is again pointed to by kata once the script finishes.
KATA_INITIAL_CONFIG_FNAME=""
function restore_kata_config() {
rm -rf "${TMP_DIR}"
set_kata_config_file "${KATA_INITIAL_CONFIG_FNAME}"
}
trap restore_kata_config EXIT
# Show help about this script
function help(){
cat << EOF
Usage: $0
Description:
Runs onednn benchmark.
EOF
}
function save_config() {
metrics_json_start_array
pushd "${DEFAULT_KATA_CONFIG_DIR}"
local MEM_AVAIL_MB="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep -i "default_memory =" | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
local NUM_CPUS="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep -i "default_vcpus =" | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
local SHARED_FS="$(cat ${DEFAULT_KATA_CONFIG_FNAME} | grep shared_fs | cut -d "=" -f2 | tr -d ' ' | tr -d '"')"
popd
local json="$(cat << EOF
{
"image": "${IMAGE}",
"units": "ms",
"mode": "Lower Is Better",
"shared-fs": "${SHARED_FS}",
"num-vcpus": "${NUM_CPUS}",
"memory-available-mb": "${MEM_AVAIL_MB}"
}
EOF
)"
metrics_json_add_array_element "${json}"
metrics_json_end_array "Config"
}
function main() {
local i=0
local cmds=("docker")
local RES_DIR="/var/lib/phoronix-test-suite/test-results"
# Check tools/commands dependencies
init_env
check_cmds "${cmds[@]}"
check_ctr_images "$IMAGE" "$DOCKERFILE"
clean_cache
# Configure Kata to use the maximum number of available CPUs
# and to use the available free memory.
get_current_kata_config_file KATA_INITIAL_CONFIG_FNAME
set_kata_configuration_performance "${KATA_PERF_CONFIG}"
# Launch container.
sudo -E "${CTR_EXE}" run -d --runtime "${CTR_RUNTIME}" "${IMAGE}" "${TEST_NAME}" sh -c "${PAYLOAD_ARGS}"
# Run the test.
sudo -E "${CTR_EXE}" t exec -t --exec-id "$(random_name)" "${TEST_NAME}" sh -c "${WORKLOAD}"
results_fname=$(sudo -E "${CTR_EXE}" t exec --exec-id $(random_name) ${TEST_NAME} sh -c "ls ${RES_DIR}")
SAVE_RESULTS_CMD="phoronix-test-suite result-file-to-json ${results_fname}"
# Save results.
sudo -E "${CTR_EXE}" t exec --exec-id "$(random_name)" "${TEST_NAME}" sh -c "${SAVE_RESULTS_CMD}"
# Extract results.
sudo -E "${CTR_EXE}" t exec --exec-id "${RANDOM}" "${TEST_NAME}" sh -c "cat /root/${results_fname}.json" > "${TEST_RESULTS_FNAME}"
cat <<< $(jq 'del(.systems[].data)' "${TEST_RESULTS_FNAME}") > "${TEST_RESULTS_FNAME}"
local results="$(cat "${TEST_RESULTS_FNAME}")"
metrics_json_init
save_config
metrics_json_start_array
metrics_json_add_array_element "${results}"
metrics_json_end_array "Results"
metrics_json_save
clean_env_ctr
}
main "$@"