From 6d971ba8df26a8b537ae5d38bbead8d9c1d1fd21 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Tue, 15 Aug 2023 17:30:22 +0000 Subject: [PATCH 1/4] metrics: Add Tensorflow ResNet50 int8 benchmark This PR adds the Tensorflow ResNet50 int8 script for kata metrics. Fixes #7652 Signed-off-by: Gabriela Cervantes --- .../tensorflow_resnet50_int8.sh | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100755 tests/metrics/machine_learning/tensorflow_resnet50_int8.sh diff --git a/tests/metrics/machine_learning/tensorflow_resnet50_int8.sh b/tests/metrics/machine_learning/tensorflow_resnet50_int8.sh new file mode 100755 index 0000000000..177a62c980 --- /dev/null +++ b/tests/metrics/machine_learning/tensorflow_resnet50_int8.sh @@ -0,0 +1,155 @@ +#!/bin/bash +# +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o pipefial + +# General env +SCRIPT_PATH=$(dirname "$(readlink -f "$0")") +source "${SCRIPT_PATH}/../lib/common.bash" + +IMAGE="docker.io/library/resnet50int8:latest" +DOCKERFILE="${SCRIPT_PATH}/resnet50_int8/Dockerfile" +tensorflow_file=$(mktemp tensorflowresults.XXXXXXXXXX) +NUM_CONTAINERS="$1" +TIMEOUT="$2" +TEST_NAME="tensorflow-resnet50int8" +PAYLOAD_ARGS="tail -f /dev/null" +TESTDIR="${TESTDIR:-/testdir}" +# Options to control the start of the workload using a trigger-file +dst_dir="/host" +src_dir=$(mktemp --tmpdir -d tensorflowresnet50int8.XXXXXXXXXX) +MOUNT_OPTIONS="type=bind,src=$src_dir,dst=$dst_dir,options=rbind:ro" +start_script="resnet50_start.sh" +# CMD points to the script that starts the workload +# export DNNL_MAX_CPU_ISA=AVX512_CORE_AMX +CMD="export KMP_AFFINITY=granularity=fine,verbose,compact && export OMP_NUM_THREADS=16 && $dst_dir/$start_script" +guest_trigger_file="$dst_dir/$trigger_file" +host_trigger_file="$src_dir/$trigger_file" +INITIAL_NUM_PIDS=1 +CMD_FILE="cat results | grep 'Throughput' | wc -l" +CMD_RESULTS="cat results | grep 'Throughput' | cut -d':' -f2 | cut -d' ' -f2 | tr '\n' ','" + +function remove_tmp_file() { + rm -rf "${tensorflow_file}" +} + +trap remove_tmp_file EXIT + +function help() { +cat << EOF +Usage: $0 + Description: + This script launches n number of containers + to run the ResNet50 int8 using a Tensorflow + container. + Options: + : Number of containers to run. + : Timeout to launch the containers. +EOF +} + +function create_start_script() { + local script="${src_dir}/${start_script}" + rm -rf "${script}" + +cat <>"${script}" +#!/bin/bash +python3.8 models/benchmarks/launch_benchmark.py --benchmark-only --framework tensorflow --model-name resnet50 --precision int8 --mode inference --in-graph /resnet50_int8_pretrained_model.pb --batch-size 116 --num-intra-threads 16 >> results +EOF + chmod +x "${script}" +} + +function resnet50_int8_test() { + info "Running ResNet50 Int8 Tensorflow test" + local pids=() + local j=0 + for i in "${containers[@]}"; do + $(sudo -E "${CTR_EXE}" t exec --exec-id "$(random_name)" "${i}" sh -c "${CMD}")& + pids["${j}"]=$! + ((j++)) + done + + # wait for all pids + for pid in ${pids[*]}; do + wait "${pid}" + done + + touch "${host_trigger_file}" + info "All containers are running the workload..." + collect_results "${CMD_FILE}" + + for i in "${containers[@]}"; do + sudo -E "${CTR_EXE}" t exec --exec-id "$(random_name)" "${i}" sh -c "${CMD_RESULTS}" >> "${tensorflow_file}" + done + + local resnet50_int8_results=$(cat "${tensorflow_file}" | sed 's/.$//') + local average_resnet50_int8=$(echo "${resnet50_int8_results}" | sed 's/.$//'| sed "s/,/+/g;s/.*/(&)\/2/g" | bc -l) + + local json="$(cat << EOF + { + "ResNet50Int8": { + "Result": "${resnet50_int8_results}", + "Average": "${average_resnet50_int8}", + "Units": "images/s" + } + } +EOF +)" + metrics_json_add_array_element "$json" + metrics_json_end_array "Results" +} + +function main() { + # Verify enough arguments + if [ $# != 2 ]; then + echo >&2 "error: Not enough arguments [$@]" + help + exit 1 + fi + + local i=0 + local containers=() + local not_started_count="${NUM_CONTAINERS}" + + # Check tools/commands dependencies + cmds=("awk" "docker" "bc") + check_cmds "${cmds[@]}" + check_ctr_images "${IMAGE}" "${DOCKERFILE}" + + init_env + create_start_script + + info "Creating ${NUM_CONTAINERS} containers" + + for ((i=1; i<= "${NUM_CONTAINERS}"; i++)); do + containers+=($(random_name)) + sudo -E "${CTR_EXE}" run -d --runtime "${CTR_RUNTIME}" --mount="${MOUNT_OPTIONS}" "${IMAGE}" "${containers[-1]}" sh -c "${PAYLOAD_ARGS}" + ((not_started_count--)) + info "$not_started_count remaining containers" + done + + metrics_json_init + metrics_json_start_array + + # Check that the requested number of containers are running + check_containers_are_up "${NUM_CONTAINERS}" + + # Check that the requested number of containers are running + check_containers_are_running "${NUM_CONTAINERS}" + + # Get the initial number of pids in a single container before the workload starts + INITIAL_NUM_PIDS=$(sudo -E "${CTR_EXE}" t metrics "${containers[-1]}" | grep pids.current | grep pids.current | xargs | cut -d ' ' -f 2) + ((INITIAL_NUM_PIDS++)) + + resnet50_int8_test + + metrics_json_save + + sudo rm -rf "${src_dir}" + + clean_env_ctr +} +main "$@" From 24baededc041dc306155ce29344ef3bc68c988f0 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Tue, 15 Aug 2023 17:36:48 +0000 Subject: [PATCH 2/4] metrics: Add Dockerfile for ResNet50 int8 This PR adds the dockerfile for ResNet50 int8 benchmark. Signed-off-by: Gabriela Cervantes --- .../resnet50_int8_dockerfile/Dockerfile | 21 +++++++++++++++++++ .../tensorflow_resnet50_int8.sh | 4 ++-- 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 tests/metrics/machine_learning/resnet50_int8_dockerfile/Dockerfile diff --git a/tests/metrics/machine_learning/resnet50_int8_dockerfile/Dockerfile b/tests/metrics/machine_learning/resnet50_int8_dockerfile/Dockerfile new file mode 100644 index 0000000000..9782d72e4b --- /dev/null +++ b/tests/metrics/machine_learning/resnet50_int8_dockerfile/Dockerfile @@ -0,0 +1,21 @@ +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +# Usage: FROM [image name] +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive + +# Version of the Dockerfile +LABEL DOCKERFILE_VERSION="1.0" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends wget nano curl build-essential git && \ + apt-get install -y python3.8 python3-pip && \ + pip install intel-tensorflow-avx512==2.8.0 && \ + pip install protobuf==3.20.* && \ + wget https://storage.googleapis.com/intel-optimized-tensorflow/models/v1_8/resnet50_int8_pretrained_model.pb && \ + git clone https://github.com/IntelAI/models.git + +CMD ["/bin/bash"] diff --git a/tests/metrics/machine_learning/tensorflow_resnet50_int8.sh b/tests/metrics/machine_learning/tensorflow_resnet50_int8.sh index 177a62c980..19c3e1192e 100755 --- a/tests/metrics/machine_learning/tensorflow_resnet50_int8.sh +++ b/tests/metrics/machine_learning/tensorflow_resnet50_int8.sh @@ -11,7 +11,7 @@ SCRIPT_PATH=$(dirname "$(readlink -f "$0")") source "${SCRIPT_PATH}/../lib/common.bash" IMAGE="docker.io/library/resnet50int8:latest" -DOCKERFILE="${SCRIPT_PATH}/resnet50_int8/Dockerfile" +DOCKERFILE="${SCRIPT_PATH}/resnet50_int8_dockerfile/Dockerfile" tensorflow_file=$(mktemp tensorflowresults.XXXXXXXXXX) NUM_CONTAINERS="$1" TIMEOUT="$2" @@ -22,7 +22,7 @@ TESTDIR="${TESTDIR:-/testdir}" dst_dir="/host" src_dir=$(mktemp --tmpdir -d tensorflowresnet50int8.XXXXXXXXXX) MOUNT_OPTIONS="type=bind,src=$src_dir,dst=$dst_dir,options=rbind:ro" -start_script="resnet50_start.sh" +start_script="resnet50int8_start.sh" # CMD points to the script that starts the workload # export DNNL_MAX_CPU_ISA=AVX512_CORE_AMX CMD="export KMP_AFFINITY=granularity=fine,verbose,compact && export OMP_NUM_THREADS=16 && $dst_dir/$start_script" From 1a1b20776066bd96801332952e11785d0471cff3 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Tue, 15 Aug 2023 17:46:44 +0000 Subject: [PATCH 3/4] docs: Add Tensorflow Resnet50 documentation This PR adds the Tensorflow Resnet50 documentation. Signed-off-by: Gabriela Cervantes --- tests/metrics/machine_learning/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/metrics/machine_learning/README.md b/tests/metrics/machine_learning/README.md index f55adba207..1872e9150d 100644 --- a/tests/metrics/machine_learning/README.md +++ b/tests/metrics/machine_learning/README.md @@ -47,3 +47,16 @@ $ cd metrics/machine_learning $ ./tensorflow_mobilenet_benchmark.sh 25 60 ``` +# Kata Containers TensorFlow `ResNet50` Metrics + +`ResNet50` is an image classification model pre-trained on the `ImageNet` dataset. +Kata Containers provides a test for running `ResNet50` inference using Intel optimized +`TensorFlow`. + +## Running the `TensorFlow` `ResNet50` test +Individual test can be run by hand, for example: + +``` +$ cd metrics/machine_learning +$ ./tensorflow_resnet50_int8.sh 25 60 +``` From bade6a5c3b7b0f5acfece10787675882bb2c7ada Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Tue, 15 Aug 2023 17:48:27 +0000 Subject: [PATCH 4/4] docs: Fix TensorFlow word across the document This PR fixes the TensorFlow word across the document to have uniformity across all the document. Signed-off-by: Gabriela Cervantes --- tests/metrics/machine_learning/README.md | 10 +++++----- .../resnet50_int8_dockerfile/Dockerfile | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/metrics/machine_learning/README.md b/tests/metrics/machine_learning/README.md index 1872e9150d..41cc139dcf 100644 --- a/tests/metrics/machine_learning/README.md +++ b/tests/metrics/machine_learning/README.md @@ -1,4 +1,4 @@ -# Kata Containers Tensorflow Metrics +# Kata Containers TensorFlow Metrics Kata Containers provides a series of performance tests using the TensorFlow reference benchmarks (tf_cnn_benchmarks). @@ -30,16 +30,16 @@ Individual tests can be run by hand, for example: $ cd metrics/machine_learning $ ./tensorflow.sh 40 100 ``` -# Kata Containers Tensorflow `MobileNet` Metrics +# Kata Containers TensorFlow `MobileNet` Metrics `MobileNets` are small, low-latency, low-power models parameterized to meet the resource constraints of a variety of use cases. They can be built upon for classification, detection, embeddings and segmentation similar to how other popular large scale models, such as Inception, are used. `MobileNets` can be run efficiently on mobile devices with `Tensorflow` Lite. -Kata Containers provides a test for running `MobileNet V1` inference using Intel-Optimized `Tensorflow`. +Kata Containers provides a test for running `MobileNet V1` inference using Intel-Optimized `TensorFlow`. -## Running the `Tensorflow` `MobileNet` test +## Running the `TensorFlow` `MobileNet` test Individual test can be run by hand, for example: ``` @@ -50,7 +50,7 @@ $ ./tensorflow_mobilenet_benchmark.sh 25 60 # Kata Containers TensorFlow `ResNet50` Metrics `ResNet50` is an image classification model pre-trained on the `ImageNet` dataset. -Kata Containers provides a test for running `ResNet50` inference using Intel optimized +Kata Containers provides a test for running `ResNet50` inference using Intel-Optimized `TensorFlow`. ## Running the `TensorFlow` `ResNet50` test diff --git a/tests/metrics/machine_learning/resnet50_int8_dockerfile/Dockerfile b/tests/metrics/machine_learning/resnet50_int8_dockerfile/Dockerfile index 9782d72e4b..a033475ee5 100644 --- a/tests/metrics/machine_learning/resnet50_int8_dockerfile/Dockerfile +++ b/tests/metrics/machine_learning/resnet50_int8_dockerfile/Dockerfile @@ -13,9 +13,9 @@ LABEL DOCKERFILE_VERSION="1.0" RUN apt-get update && \ apt-get install -y --no-install-recommends wget nano curl build-essential git && \ apt-get install -y python3.8 python3-pip && \ - pip install intel-tensorflow-avx512==2.8.0 && \ - pip install protobuf==3.20.* && \ - wget https://storage.googleapis.com/intel-optimized-tensorflow/models/v1_8/resnet50_int8_pretrained_model.pb && \ + pip install --no-cache-dir intel-tensorflow-avx512==2.8.0 && \ + pip install --no-cache-dir protobuf==3.20.* && \ + wget -q https://storage.googleapis.com/intel-optimized-tensorflow/models/v1_8/resnet50_int8_pretrained_model.pb && \ git clone https://github.com/IntelAI/models.git CMD ["/bin/bash"]