From 788d2a254eeb635ba63d25e632226f73f044aa39 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 13 Jul 2023 21:18:25 +0000 Subject: [PATCH 1/3] tests: Add tensorflow mobilenet performance test This PR adds tensorflow mobilenet performance test for kata metrics. Fixes #7334 Signed-off-by: Gabriela Cervantes --- .../tensorflow_mobilenet_benchmark.sh | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100755 tests/metrics/machine_learning/tensorflow_mobilenet_benchmark.sh diff --git a/tests/metrics/machine_learning/tensorflow_mobilenet_benchmark.sh b/tests/metrics/machine_learning/tensorflow_mobilenet_benchmark.sh new file mode 100755 index 000000000..72950b5b4 --- /dev/null +++ b/tests/metrics/machine_learning/tensorflow_mobilenet_benchmark.sh @@ -0,0 +1,190 @@ +#!/bin/bash +# +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o pipefial + +# General env +SCRIPT_PATH=$(dirname "$(readlink -f "$0")") +source "${SCRIPT_PATH}/../lib/common.bash" + +IMAGE="docker.io/library/amextdx:latest" +DOCKERFILE="${SCRIPT_PATH}/amex_tensorflow_dockerfile/Dockerfile" +tensorflow_file=$(mktemp tensorflowresults.XXXXXXXXXX) +NUM_CONTAINERS="$1" +TIMEOUT="$2" +TEST_NAME="tensorflow-intelai" +PAYLOAD_ARGS="tail -f /dev/null" +TESTDIR="${TESTDIR:-/testdir}" +# Options to control the start of the workload using a trigger-file +dst_dir="/host" +src_dir=$(mktemp --tmpdir -d tensorflowai.XXXXXXXXXX) +MOUNT_OPTIONS="type=bind,src=$src_dir,dst=$dst_dir,options=rbind:ro" +start_script="mobilnet_start.sh" +# CMD points to the script that starts the workload +CMD="$dst_dir/$start_script" +guest_trigger_file="$dst_dir/$trigger_file" +host_trigger_file="$src_dir/$trigger_file" +timeout=600 +INITIAL_NUM_PIDS=1 +CMD_FILE="cat results | grep 'Average Throughput' | wc -l" +CMD_RESULTS="cat results | grep 'Average Throughput' | cut -d':' -f2 | cut -d' ' -f2 | tr '\n' ','" + +function remove_tmp_file() { + rm -rf "${tensorflow_file}" +} + +trap remove_tmp_file EXIT + +function help() { +cat << EOF +Usage: $0 + Description: + This script launches n number of containers + to run the tf cnn benchmarks using a Tensorflow + container. + Options: + : Number of containers to run. + : Timeout to launch the containers. +EOF +} + +function create_start_script() { + local script="${src_dir}/${start_script}" + rm -rf "${script}" + +cat <>"${script}" +#!/bin/bash +python3.8 models/benchmarks/launch_benchmark.py --benchmark-only --framework tensorflow --model-name mobilenet_v1 --mode inference --precision bfloat16 --batch-size 100 --in-graph /mobilenet_v1_1.0_224_frozen.pb --num-intra-threads 16 --num-inter-threads 1 --verbose --\ input_height=224 input_width=224 warmup_steps=20 steps=20 \ input_layer=input output_layer=MobilenetV1/Predictions/Reshape_1 > results +EOF + chmod +x "${script}" +} + +function mobilenet_test() { + local CMD_EXPORT_VAR="export KMP_AFFINITY=granularity=fine,verbose,compact && export OMP_NUM_THREADS=16" + + info "Export environment variables" + for i in "${containers[@]}"; do + sudo -E "${CTR_EXE}" t exec -d --exec-id "$(random_name)" "${i}" sh -c "${CMD_EXPORT_VAR}" + done + + info "Running Mobilenet Tensorflow test" + local pids=() + local j=0 + for i in "${containers[@]}"; do + $(sudo -E "${CTR_EXE}" t exec --exec-id "$(random_name)" "${i}" sh -c "${CMD}")& + pids["${j}"]=$! + ((j++)) + done + + # wait for all pids + for pid in ${pids[*]}; do + wait "${pid}" + done + + touch "${host_trigger_file}" + info "All containers are running the workload..." + + for i in "${containers[@]}"; do + check_file=$(sudo -E "${CTR_EXE}" t exec -d --exec-id "$(random_name)" "${i}" sh -c "${CMD_FILE}") + retries="30" + for j in $(seq 1 "${retries}"); do + [ "${check_file}" -eq "1" ] && break + sleep 1 + done + done + + for i in "${containers[@]}"; do + sudo -E "${CTR_EXE}" t exec --exec-id "$(random_name)" "${i}" sh -c "${CMD_RESULTS}" >> "${tensorflow_file}" + done + + local mobilnet_results=$(cat "${tensorflow_file}" | sed 's/.$//') + local average_mobilnet=$(echo "${mobilnet_results}" | sed 's/.$//' | sed "s/,/+/g;s/.*/(&)\/$NUM_CONTAINERS/g" | bc -l) + local json="$(cat << EOF + { + "Mobilnet": { + "Result": "${mobilnet_results}", + "Average": "${average_mobilnet}", + "Units": "images/s" + } + } +EOF +)" + metrics_json_add_array_element "$json" + metrics_json_end_array "Results" +} + +function check_containers_are_up() { + local containers_launched=0 + for i in $(seq "${TIMEOUT}") ; do + info "Verify that the containers are running" + containers_launched="$(sudo ${CTR_EXE} t list | grep -c "RUNNING")" + [ "${containers_launched}" -eq "${NUM_CONTAINERS}" ] && break + sleep 1 + [ "${i}" == "${TIMEOUT}" ] && return 1 + done +} + +function main() { + # Verify enough arguments + if [ $# != 2 ]; then + echo >&2 "error: Not enough arguments [$@]" + help + exit 1 + fi + + local i=0 + local containers=() + local not_started_count="${NUM_CONTAINERS}" + + # Check tools/commands dependencies + cmds=("awk" "docker" "bc") + check_cmds "${cmds[@]}" + check_ctr_images "${IMAGE}" "${DOCKERFILE}" + + init_env + create_start_script + + info "Creating ${NUM_CONTAINERS} containers" + + for ((i=1; i<= "${NUM_CONTAINERS}"; i++)); do + containers+=($(random_name)) + sudo -E "${CTR_EXE}" run -d --runtime "${CTR_RUNTIME}" --mount="${MOUNT_OPTIONS}" "${IMAGE}" "${containers[-1]}" sh -c "${PAYLOAD_ARGS}" + ((not_started_count--)) + info "${not_started_count} remaining containers" + done + + metrics_json_init + metrics_json_start_array + + # Check that the requested number of containers are running + check_containers_are_up + + # Check that the requested number of containers are running + local timeout_launch="10" + check_containers_are_up & pid=$! + (sleep "${timeout_launch}" && kill -HUP "${pid}") 2>/dev/null & pid_tout=$! + + if wait "${pid}" 2>/dev/null; then + pkill -HUP -P "${pid_tout}" + wait "${pid_tout}" + else + warn "Time out exceeded" + return 1 + fi + + # Get the initial number of pids in a single container before the workload starts + INITIAL_NUM_PIDS=$(sudo -E "${CTR_EXE}" t metrics "${containers[-1]}" | grep pids.current | grep pids.current | xargs | cut -d ' ' -f 2) + ((INITIAL_NUM_PIDS++)) + + mobilenet_test + + metrics_json_save + + sudo rm -rf "${src_dir}" + + clean_env_ctr +} +main "$@" From a864d0e349464b7961f1c77dff55d9bdb13aec0d Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 13 Jul 2023 21:23:44 +0000 Subject: [PATCH 2/3] tests: Add tensorflow mobilenet dockerfile This PR adds the tensorflow mobilenet dockerfile. Signed-off-by: Gabriela Cervantes --- .../tensorflow_mobilenet_benchmark.sh | 4 ++-- .../Dockerfile | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 tests/metrics/machine_learning/tensorflow_mobilenet_dockerfile/Dockerfile diff --git a/tests/metrics/machine_learning/tensorflow_mobilenet_benchmark.sh b/tests/metrics/machine_learning/tensorflow_mobilenet_benchmark.sh index 72950b5b4..6bc7598d8 100755 --- a/tests/metrics/machine_learning/tensorflow_mobilenet_benchmark.sh +++ b/tests/metrics/machine_learning/tensorflow_mobilenet_benchmark.sh @@ -10,8 +10,8 @@ set -o pipefial SCRIPT_PATH=$(dirname "$(readlink -f "$0")") source "${SCRIPT_PATH}/../lib/common.bash" -IMAGE="docker.io/library/amextdx:latest" -DOCKERFILE="${SCRIPT_PATH}/amex_tensorflow_dockerfile/Dockerfile" +IMAGE="docker.io/library/tensorflowmobilenet:latest" +DOCKERFILE="${SCRIPT_PATH}/tensorflow_mobilenet_dockerfile/Dockerfile" tensorflow_file=$(mktemp tensorflowresults.XXXXXXXXXX) NUM_CONTAINERS="$1" TIMEOUT="$2" diff --git a/tests/metrics/machine_learning/tensorflow_mobilenet_dockerfile/Dockerfile b/tests/metrics/machine_learning/tensorflow_mobilenet_dockerfile/Dockerfile new file mode 100644 index 000000000..a7672bd7f --- /dev/null +++ b/tests/metrics/machine_learning/tensorflow_mobilenet_dockerfile/Dockerfile @@ -0,0 +1,21 @@ +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +# Usage: FROM [image name] +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive + +# Version of the Dockerfile +LABEL DOCKERFILE_VERSION="1.0" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends wget nano curl build-essential git && \ + apt-get install -y python3.8 python3-pip && \ + pip install intel-tensorflow-avx512==2.8.0 && \ + pip install protobuf==3.20.* && \ + wget https://storage.googleapis.com/intel-optimized-tensorflow/models/v1_8/mobilenet_v1_1.0_224_frozen.pb && \ + git clone https://github.com/IntelAI/models.git + +CMD ["/bin/bash"] From 620b945975c9d6086cfe8787a9ada2d8faa4433b Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 13 Jul 2023 21:31:10 +0000 Subject: [PATCH 3/3] metrics: Add Tensorflow Mobilenet documentation This PR adds the Tensorflow mobilinet documentation for the machine learning README. Signed-off-by: Gabriela Cervantes --- tests/metrics/machine_learning/README.md | 17 +++++++++++++++++ .../tensorflow_mobilenet_benchmark.sh | 14 +++++++------- .../tensorflow_mobilenet_dockerfile/Dockerfile | 8 ++++---- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/tests/metrics/machine_learning/README.md b/tests/metrics/machine_learning/README.md index 8c62b1566..f797a38c3 100644 --- a/tests/metrics/machine_learning/README.md +++ b/tests/metrics/machine_learning/README.md @@ -27,3 +27,20 @@ Individual tests can be run by hand, for example: $ cd metrics/machine_learning $ ./tensorflow.sh 40 100 ``` +# Kata Containers Tensorflow `MobileNet` Metrics + +`MobileNets` are small, low-latency, low-power models parameterized to meet the resource +constraints of a variety of use cases. They can be built upon for classification, detection, +embeddings and segmentation similar to how other popular large scale models, such as Inception, are used. +`MobileNets` can be run efficiently on mobile devices with `Tensorflow` Lite. + +Kata Containers provides a test for running `MobileNet V1` inference using Intel-Optimized `Tensorflow`. + +## Running the `Tensorflow` `MobileNet` test +Individual test can be run by hand, for example: + +``` +$ cd metrics/machine_learning +$ ./tensorflow_mobilenet_benchmark.sh 25 60 +``` + diff --git a/tests/metrics/machine_learning/tensorflow_mobilenet_benchmark.sh b/tests/metrics/machine_learning/tensorflow_mobilenet_benchmark.sh index 6bc7598d8..63b42c1e7 100755 --- a/tests/metrics/machine_learning/tensorflow_mobilenet_benchmark.sh +++ b/tests/metrics/machine_learning/tensorflow_mobilenet_benchmark.sh @@ -4,7 +4,7 @@ # # SPDX-License-Identifier: Apache-2.0 -set -o pipefial +set -o pipefail # General env SCRIPT_PATH=$(dirname "$(readlink -f "$0")") @@ -22,7 +22,7 @@ TESTDIR="${TESTDIR:-/testdir}" dst_dir="/host" src_dir=$(mktemp --tmpdir -d tensorflowai.XXXXXXXXXX) MOUNT_OPTIONS="type=bind,src=$src_dir,dst=$dst_dir,options=rbind:ro" -start_script="mobilnet_start.sh" +start_script="mobilenet_start.sh" # CMD points to the script that starts the workload CMD="$dst_dir/$start_script" guest_trigger_file="$dst_dir/$trigger_file" @@ -100,13 +100,13 @@ function mobilenet_test() { sudo -E "${CTR_EXE}" t exec --exec-id "$(random_name)" "${i}" sh -c "${CMD_RESULTS}" >> "${tensorflow_file}" done - local mobilnet_results=$(cat "${tensorflow_file}" | sed 's/.$//') - local average_mobilnet=$(echo "${mobilnet_results}" | sed 's/.$//' | sed "s/,/+/g;s/.*/(&)\/$NUM_CONTAINERS/g" | bc -l) + local mobilenet_results=$(cat "${tensorflow_file}" | sed 's/.$//') + local average_mobilenet=$(echo "${mobilenet_results}" | sed 's/.$//' | sed "s/,/+/g;s/.*/(&)\/$NUM_CONTAINERS/g" | bc -l) local json="$(cat << EOF { - "Mobilnet": { - "Result": "${mobilnet_results}", - "Average": "${average_mobilnet}", + "Mobilenet": { + "Result": "${mobilenet_results}", + "Average": "${average_mobilenet}", "Units": "images/s" } } diff --git a/tests/metrics/machine_learning/tensorflow_mobilenet_dockerfile/Dockerfile b/tests/metrics/machine_learning/tensorflow_mobilenet_dockerfile/Dockerfile index a7672bd7f..fd1bd31ec 100644 --- a/tests/metrics/machine_learning/tensorflow_mobilenet_dockerfile/Dockerfile +++ b/tests/metrics/machine_learning/tensorflow_mobilenet_dockerfile/Dockerfile @@ -11,11 +11,11 @@ ENV DEBIAN_FRONTEND=noninteractive LABEL DOCKERFILE_VERSION="1.0" RUN apt-get update && \ - apt-get install -y --no-install-recommends wget nano curl build-essential git && \ + apt-get install -y wget nano curl build-essential git && \ apt-get install -y python3.8 python3-pip && \ - pip install intel-tensorflow-avx512==2.8.0 && \ - pip install protobuf==3.20.* && \ - wget https://storage.googleapis.com/intel-optimized-tensorflow/models/v1_8/mobilenet_v1_1.0_224_frozen.pb && \ + pip install --no-cache-dir intel-tensorflow-avx512==2.8.0 && \ + pip install --no-cache-dir protobuf==3.20.* && \ + wget -q https://storage.googleapis.com/intel-optimized-tensorflow/models/v1_8/mobilenet_v1_1.0_224_frozen.pb && \ git clone https://github.com/IntelAI/models.git CMD ["/bin/bash"]