From 73c57b9a19e53b199c74bee0e564fa44760a0c78 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 27 Jul 2023 16:39:35 +0000 Subject: [PATCH 1/5] metrics: Add FIO report files for kata metrics This PR adds FIO report files for kata metrics. Fixes #7472 Signed-off-by: Gabriela Cervantes --- .../scripts/dax-compare-test/report/fio.ipynb | 51 +++++++++ .../scripts/dax-compare-test/report/fio.py | 102 ++++++++++++++++++ .../report/gen-html-fio-report.sh | 48 +++++++++ .../report/run-docker-jupyter-server.sh | 39 +++++++ 4 files changed, 240 insertions(+) create mode 100644 tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/fio.ipynb create mode 100644 tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/fio.py create mode 100644 tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/gen-html-fio-report.sh create mode 100644 tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/run-docker-jupyter-server.sh diff --git a/tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/fio.ipynb b/tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/fio.ipynb new file mode 100644 index 0000000000..d2e8da2023 --- /dev/null +++ b/tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/fio.ipynb @@ -0,0 +1,51 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "tWacOPbMYPtc" + }, + "source": [ + "# FIO comparision" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jXtTs6yldl_y" + }, + "outputs": [], + "source": [ + "import fio\n", + "fio.generate_report()" + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "fio.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/fio.py b/tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/fio.py new file mode 100644 index 0000000000..313f633130 --- /dev/null +++ b/tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/fio.py @@ -0,0 +1,102 @@ +# Copyright (c) 2021-2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +import pandas as pd +import os +import re +import io +import glob +from IPython.display import display, Markdown +import matplotlib.pyplot as plt + +#Compare the tests results group by fio job. +#Input: +# df: dataset from `import_data()` +# metric: string of metrics provided in `df` +def compare_tests_group_by_fio_job(df, metric): + test_names, metric_df = group_metrics_group_by_testname(df, metric) + show_df(metric_df) + plot_df(metric_df,test_names) + +# Given a metric return results per test group by fio job. +# input: +# df: dataset from `import_data()` +# metric: string with the name of the metric to filter. +# output: +# dataset with fomat: +# 'workload' , 'name[0]' , ... , 'name[n]' +# +def group_metrics_group_by_testname(df, metric): + #name of each tests from results + names = set() + # Rows of new data set + rows = [] + # map: + # keys: name of fio job + # value: dict[k]:v where k: name of a test, v: value of test for metric` + workload = {} + + for k, row in df.iterrows(): + # name of a fio job + w = row['WORKLOAD'] + # name of tests + tname = row['NAME'] + names.add(tname) + # given a fio job name get dict of values + # if not previous values init empty dict + dict_values = workload.get(w, {}) + # For a given metric, add it into as value of dict_values[testname]=val + #e.g + # dict_values["test-name"] = row["IOPS"] + dict_values[tname] = row[metric] + workload[w] = dict_values + + names = list(names) + cols = ['WORKLOAD'] + list(names) + rdf = pd.DataFrame(workload,columns = cols) + + for k in workload: + d = workload[k] + + if not d[names[0]] == 0: + d["WORKLOAD"] = k; + rdf = rdf.append(d,ignore_index=True) + rdf = rdf.dropna() + return names, rdf + +def plot_df(df, names,sort_key=""): + if sort_key != "": + df.sort_values(sort_key, ascending=False) + df.plot(kind='bar',x="WORKLOAD",y=names, figsize=(30, 10)) + plt.show() + + +def import_data(): + frames = [] + for f in glob.glob('./results/*/results.csv'): + print("reading:" + f) + df = pd.read_csv(f) + frames.append(df) + return pd.concat(frames) + +def show_df(df): + pd.set_option('display.max_rows', df.shape[0]+1) + print(df) + +def print_md(s): + display(Markdown(s)) + +#notebook entrypoint +def generate_report(): + #Load the all test results in a single dataset + df_results = import_data() + print_md("Show all data from results") + show_df(df_results) + print_md("### Compare the tests results group by fio job. The metric used to compare is write bandwidth") + compare_tests_group_by_fio_job(df_results, 'bw_w') + print_md("### Compare the tests results group by fio job. The metric used to compare is read bandwidth") + compare_tests_group_by_fio_job(df_results, 'bw_r') + print_md("### Compare the tests results group by fio job. The metric used to compare is write IOPS(Input/Output Operations Per Second)") + compare_tests_group_by_fio_job(df_results, 'IOPS_w') + print_md("### Compare the tests results group by fio job. The metric used to compare is read IOPS(Input/Output Operations Per Second)") + compare_tests_group_by_fio_job(df_results, 'IOPS_r') diff --git a/tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/gen-html-fio-report.sh b/tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/gen-html-fio-report.sh new file mode 100644 index 0000000000..8061c059d0 --- /dev/null +++ b/tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/gen-html-fio-report.sh @@ -0,0 +1,48 @@ +#!/bin/bash +#Copyright (c) 2021-2023 Intel Corporation +# +#SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +script_dir=$(dirname "$(readlink -f "$0")") + +results_dir=${1:-} + +usage(){ + echo "$0 " +} + +if [ "${results_dir}" == "" ];then + echo "missing results directory" + usage + exit 1 +fi + +if [ ! -d "${results_dir}" ];then + echo "${results_dir} is not a directory" + usage + exit 1 +fi + +results_dir=$(realpath "${results_dir}") + +generate_report(){ + sudo chown "${USER}:${USER}" -R ${results_dir} + sudo docker run --rm -e JUPYTER_ENABLE_LAB=yes \ + -v "${script_dir}:/home/jovyan" \ + -v "${results_dir}:/home/jovyan/results" \ + --user $(id -u):$(id -g) \ + jupyter/scipy-notebook:399cbb986c6b \ + bash -e -c ' + cd results; + jupyter nbconvert --execute /home/jovyan/fio.ipynb --to html; + cp /home/jovyan/fio.html /home/jovyan/results; + ' +} + +generate_report diff --git a/tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/run-docker-jupyter-server.sh b/tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/run-docker-jupyter-server.sh new file mode 100644 index 0000000000..f386da4e22 --- /dev/null +++ b/tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/run-docker-jupyter-server.sh @@ -0,0 +1,39 @@ +#!/bin/bash +#Copyright (c) 2021-2023 Intel Corporation +# +#SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +script_dir=$(dirname "$(readlink -f "$0")") +NOTEBOOK_PORT="8888" + +results_dir=${1:-} + +usage(){ + echo "$0 " +} + +if [ "${results_dir}" == "" ];then + echo "missing results directory" + usage + exit 1 +fi + +if [ ! -d "${results_dir}" ];then + echo "${results_dir} is not a directory" + usage + exit 1 +fi + +results_dir=$(realpath "${results_dir}") + +sudo -E docker run --rm -p "${NOTEBOOK_PORT}:${NOTEBOOK_PORT}" -e JUPYTER_ENABLE_LAB=yes \ + -v "${script_dir}:/home/jovyan" \ + -v "${results_dir}:/home/jovyan/results" \ + jupyter/scipy-notebook:399cbb986c6b \ + start.sh jupyter lab --LabApp.token='' From b0bea47c53e88c1534f1b2f2ab6fea7fe9f3a337 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 27 Jul 2023 16:42:11 +0000 Subject: [PATCH 2/5] metrics: Add makefile to report generator This PR adds the makefile to report generator for the FIO test. Signed-off-by: Gabriela Cervantes --- tests/metrics/storage/fio-k8s/scripts/Makefile | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 tests/metrics/storage/fio-k8s/scripts/Makefile diff --git a/tests/metrics/storage/fio-k8s/scripts/Makefile b/tests/metrics/storage/fio-k8s/scripts/Makefile new file mode 100644 index 0000000000..a33a5015c0 --- /dev/null +++ b/tests/metrics/storage/fio-k8s/scripts/Makefile @@ -0,0 +1,10 @@ +# +# Copyright (c) 2021-2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +MKFILE_DIR := $(dir $(MKFILE_PATH)) +run: + $(MKFILE_DIR)/compare-virtiofsd-dax.sh + "$(MKFILE_DIR)/report/gen-html-fio-report.sh" "./results" From 5e937fa62252134f33cc388bc3f135fde5bc4693 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 27 Jul 2023 16:47:17 +0000 Subject: [PATCH 3/5] metrics: Update general FIO tests This PR updates general FIO tests by adding the recent date of a change. Signed-off-by: Gabriela Cervantes --- tests/metrics/storage/fio-k8s/cmd/fiotest/Makefile | 2 +- tests/metrics/storage/fio-k8s/cmd/fiotest/main.go | 2 +- tests/metrics/storage/fio-k8s/pkg/env/Makefile | 2 +- tests/metrics/storage/fio-k8s/pkg/env/env.go | 2 +- tests/metrics/storage/fio-k8s/pkg/exec/Exec.go | 2 +- tests/metrics/storage/fio-k8s/pkg/k8s/Makefile | 2 +- tests/metrics/storage/fio-k8s/pkg/k8s/exec.go | 2 +- tests/metrics/storage/fio-k8s/pkg/k8s/k8s.go | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/metrics/storage/fio-k8s/cmd/fiotest/Makefile b/tests/metrics/storage/fio-k8s/cmd/fiotest/Makefile index 69888b39d8..4c8a27c69e 100644 --- a/tests/metrics/storage/fio-k8s/cmd/fiotest/Makefile +++ b/tests/metrics/storage/fio-k8s/cmd/fiotest/Makefile @@ -1,5 +1,5 @@ # -# Copyright (c) 2021-2022 Intel Corporation +# Copyright (c) 2021-2023 Intel Corporation # # SPDX-License-Identifier: Apache-2.0 # diff --git a/tests/metrics/storage/fio-k8s/cmd/fiotest/main.go b/tests/metrics/storage/fio-k8s/cmd/fiotest/main.go index 641836c556..de7e7b2ff0 100644 --- a/tests/metrics/storage/fio-k8s/cmd/fiotest/main.go +++ b/tests/metrics/storage/fio-k8s/cmd/fiotest/main.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Intel Corporation +// Copyright (c) 2021-2023 Intel Corporation // // SPDX-License-Identifier: Apache-2.0 package main diff --git a/tests/metrics/storage/fio-k8s/pkg/env/Makefile b/tests/metrics/storage/fio-k8s/pkg/env/Makefile index 5f0a330df5..a1c0a7779c 100644 --- a/tests/metrics/storage/fio-k8s/pkg/env/Makefile +++ b/tests/metrics/storage/fio-k8s/pkg/env/Makefile @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Intel Corporation +# Copyright (c) 2021-2023 Intel Corporation # # SPDX-License-Identifier: Apache-2.0 # diff --git a/tests/metrics/storage/fio-k8s/pkg/env/env.go b/tests/metrics/storage/fio-k8s/pkg/env/env.go index efb0e9adad..034127f962 100644 --- a/tests/metrics/storage/fio-k8s/pkg/env/env.go +++ b/tests/metrics/storage/fio-k8s/pkg/env/env.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Intel Corporation +// Copyright (c) 2021-2023 Intel Corporation // // SPDX-License-Identifier: Apache-2.0 package env diff --git a/tests/metrics/storage/fio-k8s/pkg/exec/Exec.go b/tests/metrics/storage/fio-k8s/pkg/exec/Exec.go index 7d8a724b88..6e409f427a 100644 --- a/tests/metrics/storage/fio-k8s/pkg/exec/Exec.go +++ b/tests/metrics/storage/fio-k8s/pkg/exec/Exec.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Intel Corporation +// Copyright (c) 2021-2023 Intel Corporation // // SPDX-License-Identifier: Apache-2.0 package exec diff --git a/tests/metrics/storage/fio-k8s/pkg/k8s/Makefile b/tests/metrics/storage/fio-k8s/pkg/k8s/Makefile index fb26740dc0..f5bee6d16f 100644 --- a/tests/metrics/storage/fio-k8s/pkg/k8s/Makefile +++ b/tests/metrics/storage/fio-k8s/pkg/k8s/Makefile @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Intel Corporation +# Copyright (c) 2021-2023 Intel Corporation # # SPDX-License-Identifier: Apache-2.0 # diff --git a/tests/metrics/storage/fio-k8s/pkg/k8s/exec.go b/tests/metrics/storage/fio-k8s/pkg/k8s/exec.go index 8ed5aaa2c2..9be25f6184 100644 --- a/tests/metrics/storage/fio-k8s/pkg/k8s/exec.go +++ b/tests/metrics/storage/fio-k8s/pkg/k8s/exec.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Intel Corporation +// Copyright (c) 2021-2023 Intel Corporation // // SPDX-License-Identifier: Apache-2.0 package k8s diff --git a/tests/metrics/storage/fio-k8s/pkg/k8s/k8s.go b/tests/metrics/storage/fio-k8s/pkg/k8s/k8s.go index 8e218bd3dd..2fef788cc1 100644 --- a/tests/metrics/storage/fio-k8s/pkg/k8s/k8s.go +++ b/tests/metrics/storage/fio-k8s/pkg/k8s/k8s.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Intel Corporation +// Copyright (c) 2021-2023 Intel Corporation // // SPDX-License-Identifier: Apache-2.0 package k8s From a5d4e3388011e9314f18cf33fac15bc2e139e9d8 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 27 Jul 2023 16:53:50 +0000 Subject: [PATCH 4/5] metrics: Add compare virtiofsd dax script This PR adds the compare virtiofsd dax script for kata metrics. Signed-off-by: Gabriela Cervantes --- .../fio-k8s/scripts/compare-virtiofsd-dax.sh | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 tests/metrics/storage/fio-k8s/scripts/compare-virtiofsd-dax.sh diff --git a/tests/metrics/storage/fio-k8s/scripts/compare-virtiofsd-dax.sh b/tests/metrics/storage/fio-k8s/scripts/compare-virtiofsd-dax.sh new file mode 100644 index 0000000000..bd89c615bb --- /dev/null +++ b/tests/metrics/storage/fio-k8s/scripts/compare-virtiofsd-dax.sh @@ -0,0 +1,151 @@ +#!/bin/bash +#Copyright (c) 2021-2023 Intel Corporation +# +#SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +script_dir=$(dirname "$(readlink -f "$0")") + +runtime_path="/usr/local/bin/kata-runtime" +kata_config_path="/usr/share/defaults/kata-containers/configuration.toml" + +results_dir="$(realpath ./)/results" + +KATA_RUNTIME="${KATA_RUNTIME_CLASS:-kata}" +BAREMETAL_RUNTIME="runc" +RUNTIME_CLASS="" + +FIO_SIZE="${FIO_SIZE:-500M}" +FIO_BLOCKSIZE="${FIO_BLOCKSIZE:-4K}" +VIRTIOFS_DAX_SIZE=${VIRTIOFS_DAX_SIZE:-600M} + +# set the base case for virtiofsd +set_base_virtiofs_config() { + # Running kata-qemu-virtiofs + # Defaults for virtiofs + sudo crudini --set --existing "$kata_config_path" hypervisor.qemu virtio_fs_cache '"auto"' + sudo crudini --set --existing "$kata_config_path" hypervisor.qemu virtio_fs_cache_size ${VIRTIOFS_DAX_SIZE} +} + +## helper function: get name of current bash function +fn_name() { + echo "${FUNCNAME[1]}" +} + +# directory where results are stored +get_results_dir() { + local test_name + local test_result_dir + test_name="${1}" + test_result_dir="${results_dir}/${test_name}" + mkdir -p "${test_result_dir}" + echo "${test_result_dir}" +} + +# Collect kata env +# save kata config toml +# save output from kata-env +kata_env() { + local suffix=${1} + local config_path + local kata_env_bk + local kata_config_bk + kata_env_bk="$(get_results_dir "${suffix}")/kata-env.toml" + kata_config_bk="$(get_results_dir "${suffix}")/kata-config.toml" + + ${runtime_path} kata-env >"${kata_env_bk}" + config_path="$(${runtime_path} kata-env --json | jq .Runtime.Config.Path -r)" + cp "${config_path}" "${kata_config_bk}" +} + +# Collect the command used by virtiofsd +collect_qemu_virtiofs_cmd() { + local rdir + local test_name + test_name="${1}" + + rdir=$(get_results_dir "${test_name}") + # TODO +} + +# Run metrics runner +run_workload() { + local test_name + local test_result_file + local test_result_dir + + test_name="${1}" + + test_result_dir="$(get_results_dir "${test_name}")" + test_result_file="${test_result_dir}/test-out.txt" + + echo "Running for kata config: ${test_name}" + collect_qemu_virtiofs_cmd "$test_name" + + fio_runner_dir="${script_dir}/../../cmd/fiotest/" + fio_jobs="${script_dir}/../../configs/test-config/" + make -C "${fio_runner_dir}" build + pwd + set -x + "${fio_runner_dir}fio-k8s" \ + --debug \ + --fio.size "${FIO_SIZE}" \ + --fio.block-size "${FIO_BLOCKSIZE}" \ + --container-runtime "${RUNTIME_CLASS}" \ + --test-name "${test_name}" \ + --output-dir "$(dirname ${test_result_dir})" \ + "${fio_jobs}" | + tee \ + "${test_result_file}" + set +x +} + +pool_0_cache_auto_dax() { + local suffix="$(fn_name)" + + set_base_virtiofs_config + sudo crudini --set --existing "$kata_config_path" hypervisor.qemu virtio_fs_extra_args '["--thread-pool-size=0","-o","no_posix_lock","-o","xattr"]' + sudo crudini --set --existing "$kata_config_path" hypervisor.qemu virtio_fs_cache '"auto"' + sudo crudini --set --existing "$kata_config_path" hypervisor.qemu virtio_fs_cache_size 1024 + kata_env "${suffix}" + RUNTIME_CLASS="${KATA_RUNTIME}" + run_workload "${suffix}" +} + +pool_0_cache_auto_no_dax() { + local suffix="$(fn_name)" + + set_base_virtiofs_config + sudo crudini --set --existing "$kata_config_path" hypervisor.qemu virtio_fs_extra_args '["--thread-pool-size=0","-o","no_posix_lock","-o","xattr"]' + sudo crudini --set --existing "$kata_config_path" hypervisor.qemu virtio_fs_cache '"auto"' + sudo crudini --set --existing "$kata_config_path" hypervisor.qemu virtio_fs_cache_size 0 + + kata_env "${suffix}" + + RUNTIME_CLASS="${KATA_RUNTIME}" + run_workload "${suffix}" + echo "done" +} + +k8s_baremetal() { + local suffix="$(fn_name)" + + RUNTIME_CLASS="${BAREMETAL_RUNTIME}" + run_workload "${suffix}" +} + +main() { + + mkdir -p "${results_dir}" + + k8s_baremetal + pool_0_cache_auto_dax + pool_0_cache_auto_no_dax +} + +main $* From ff227906174c85777dfcc676cdfc5d2d6a76bd00 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 27 Jul 2023 17:14:03 +0000 Subject: [PATCH 5/5] metrics: Update runtime and configuration paths This PR updates the runtime and configuration paths for kata containers. Signed-off-by: Gabriela Cervantes --- .../metrics/storage/fio-k8s/scripts/compare-virtiofsd-dax.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 tests/metrics/storage/fio-k8s/scripts/compare-virtiofsd-dax.sh diff --git a/tests/metrics/storage/fio-k8s/scripts/compare-virtiofsd-dax.sh b/tests/metrics/storage/fio-k8s/scripts/compare-virtiofsd-dax.sh old mode 100644 new mode 100755 index bd89c615bb..248e377762 --- a/tests/metrics/storage/fio-k8s/scripts/compare-virtiofsd-dax.sh +++ b/tests/metrics/storage/fio-k8s/scripts/compare-virtiofsd-dax.sh @@ -11,8 +11,8 @@ set -o errtrace script_dir=$(dirname "$(readlink -f "$0")") -runtime_path="/usr/local/bin/kata-runtime" -kata_config_path="/usr/share/defaults/kata-containers/configuration.toml" +runtime_path="/opt/kata/bin/kata-runtime" +kata_config_path="/opt/kata/share/defaults/kata-containers/configuration.toml" results_dir="$(realpath ./)/results"