Merge pull request #7473 from GabyCT/topic/addfioreport

metrics: Add FIO report files for kata metrics

tests/metrics/storage/fio-k8s/cmd/fiotest/Makefile

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2021-2022 Intel Corporation
+# Copyright (c) 2021-2023 Intel Corporation
 #
 # SPDX-License-Identifier: Apache-2.0
 #

tests/metrics/storage/fio-k8s/cmd/fiotest/main.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2021 Intel Corporation
+// Copyright (c) 2021-2023 Intel Corporation
 //
 // SPDX-License-Identifier: Apache-2.0
 package main

tests/metrics/storage/fio-k8s/pkg/env/Makefile

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2021 Intel Corporation
+# Copyright (c) 2021-2023 Intel Corporation
 #
 # SPDX-License-Identifier: Apache-2.0
 #

tests/metrics/storage/fio-k8s/pkg/env/env.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2021 Intel Corporation
+// Copyright (c) 2021-2023 Intel Corporation
 //
 // SPDX-License-Identifier: Apache-2.0
 package env

tests/metrics/storage/fio-k8s/pkg/exec/Exec.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2021 Intel Corporation
+// Copyright (c) 2021-2023 Intel Corporation
 //
 // SPDX-License-Identifier: Apache-2.0
 package exec

tests/metrics/storage/fio-k8s/pkg/k8s/Makefile

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2021 Intel Corporation
+# Copyright (c) 2021-2023 Intel Corporation
 #
 # SPDX-License-Identifier: Apache-2.0
 #

tests/metrics/storage/fio-k8s/pkg/k8s/exec.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2021 Intel Corporation
+// Copyright (c) 2021-2023 Intel Corporation
 //
 // SPDX-License-Identifier: Apache-2.0
 package k8s

tests/metrics/storage/fio-k8s/pkg/k8s/k8s.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2021 Intel Corporation
+// Copyright (c) 2021-2023 Intel Corporation
 //
 // SPDX-License-Identifier: Apache-2.0
 package k8s

tests/metrics/storage/fio-k8s/scripts/Makefile

@@ -0,0 +1,10 @@
+#
+# Copyright (c) 2021-2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
+MKFILE_DIR := $(dir $(MKFILE_PATH))
+run:
+	$(MKFILE_DIR)/compare-virtiofsd-dax.sh
+	"$(MKFILE_DIR)/report/gen-html-fio-report.sh" "./results"

tests/metrics/storage/fio-k8s/scripts/compare-virtiofsd-dax.sh

@@ -0,0 +1,151 @@
+#!/bin/bash
+#Copyright (c) 2021-2023 Intel Corporation
+#
+#SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+script_dir=$(dirname "$(readlink -f "$0")")
+
+runtime_path="/opt/kata/bin/kata-runtime"
+kata_config_path="/opt/kata/share/defaults/kata-containers/configuration.toml"
+
+results_dir="$(realpath ./)/results"
+
+KATA_RUNTIME="${KATA_RUNTIME_CLASS:-kata}"
+BAREMETAL_RUNTIME="runc"
+RUNTIME_CLASS=""
+
+FIO_SIZE="${FIO_SIZE:-500M}"
+FIO_BLOCKSIZE="${FIO_BLOCKSIZE:-4K}"
+VIRTIOFS_DAX_SIZE=${VIRTIOFS_DAX_SIZE:-600M}
+
+# set the base case for virtiofsd
+set_base_virtiofs_config() {
+	# Running kata-qemu-virtiofs
+	# Defaults for virtiofs
+	sudo crudini --set --existing "$kata_config_path" hypervisor.qemu virtio_fs_cache '"auto"'
+	sudo crudini --set --existing "$kata_config_path" hypervisor.qemu virtio_fs_cache_size ${VIRTIOFS_DAX_SIZE}
+}
+
+## helper function: get name of current bash function
+fn_name() {
+	echo "${FUNCNAME[1]}"
+}
+
+# directory where results are stored
+get_results_dir() {
+	local test_name
+	local test_result_dir
+	test_name="${1}"
+	test_result_dir="${results_dir}/${test_name}"
+	mkdir -p "${test_result_dir}"
+	echo "${test_result_dir}"
+}
+
+# Collect kata env
+# save kata config toml
+# save output from kata-env
+kata_env() {
+	local suffix=${1}
+	local config_path
+	local kata_env_bk
+	local kata_config_bk
+	kata_env_bk="$(get_results_dir "${suffix}")/kata-env.toml"
+	kata_config_bk="$(get_results_dir "${suffix}")/kata-config.toml"
+
+	${runtime_path} kata-env >"${kata_env_bk}"
+	config_path="$(${runtime_path} kata-env --json | jq .Runtime.Config.Path -r)"
+	cp "${config_path}" "${kata_config_bk}"
+}
+
+# Collect the command used by virtiofsd
+collect_qemu_virtiofs_cmd() {
+	local rdir
+	local test_name
+	test_name="${1}"
+
+	rdir=$(get_results_dir "${test_name}")
+	# TODO
+}
+
+# Run metrics runner
+run_workload() {
+	local test_name
+	local test_result_file
+	local test_result_dir
+
+	test_name="${1}"
+
+	test_result_dir="$(get_results_dir "${test_name}")"
+	test_result_file="${test_result_dir}/test-out.txt"
+
+	echo "Running for kata config: ${test_name}"
+	collect_qemu_virtiofs_cmd "$test_name"
+
+	fio_runner_dir="${script_dir}/../../cmd/fiotest/"
+	fio_jobs="${script_dir}/../../configs/test-config/"
+	make -C "${fio_runner_dir}" build
+	pwd
+	set -x
+	"${fio_runner_dir}fio-k8s" \
+		--debug \
+		--fio.size "${FIO_SIZE}" \
+		--fio.block-size "${FIO_BLOCKSIZE}" \
+		--container-runtime "${RUNTIME_CLASS}" \
+		--test-name "${test_name}" \
+		--output-dir "$(dirname ${test_result_dir})" \
+		"${fio_jobs}" |
+		tee \
+			"${test_result_file}"
+	set +x
+}
+
+pool_0_cache_auto_dax() {
+	local suffix="$(fn_name)"
+
+	set_base_virtiofs_config
+	sudo crudini --set --existing "$kata_config_path" hypervisor.qemu virtio_fs_extra_args '["--thread-pool-size=0","-o","no_posix_lock","-o","xattr"]'
+	sudo crudini --set --existing "$kata_config_path" hypervisor.qemu virtio_fs_cache '"auto"'
+	sudo crudini --set --existing "$kata_config_path" hypervisor.qemu virtio_fs_cache_size 1024
+	kata_env "${suffix}"
+	RUNTIME_CLASS="${KATA_RUNTIME}"
+	run_workload "${suffix}"
+}
+
+pool_0_cache_auto_no_dax() {
+	local suffix="$(fn_name)"
+
+	set_base_virtiofs_config
+	sudo crudini --set --existing "$kata_config_path" hypervisor.qemu virtio_fs_extra_args '["--thread-pool-size=0","-o","no_posix_lock","-o","xattr"]'
+	sudo crudini --set --existing "$kata_config_path" hypervisor.qemu virtio_fs_cache '"auto"'
+	sudo crudini --set --existing "$kata_config_path" hypervisor.qemu virtio_fs_cache_size 0
+
+	kata_env "${suffix}"
+
+	RUNTIME_CLASS="${KATA_RUNTIME}"
+	run_workload "${suffix}"
+	echo "done"
+}
+
+k8s_baremetal() {
+	local suffix="$(fn_name)"
+
+	RUNTIME_CLASS="${BAREMETAL_RUNTIME}"
+	run_workload "${suffix}"
+}
+
+main() {
+
+	mkdir -p "${results_dir}"
+
+	k8s_baremetal
+	pool_0_cache_auto_dax
+	pool_0_cache_auto_no_dax
+}
+
+main $*

tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/fio.ipynb

@@ -0,0 +1,51 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "tWacOPbMYPtc"
+   },
+   "source": [
+    "# FIO comparision"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "jXtTs6yldl_y"
+   },
+   "outputs": [],
+   "source": [
+    "import fio\n",
+    "fio.generate_report()"
+   ]
+  }
+ ],
+ "metadata": {
+  "colab": {
+   "collapsed_sections": [],
+   "name": "fio.ipynb",
+   "provenance": []
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/fio.py

@@ -0,0 +1,102 @@
+# Copyright (c) 2021-2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+import pandas as pd
+import os
+import re
+import io
+import glob
+from IPython.display import display, Markdown
+import matplotlib.pyplot as plt
+
+#Compare the tests results group by fio job.
+#Input:
+# df: dataset from `import_data()`
+# metric: string of metrics provided in `df`
+def compare_tests_group_by_fio_job(df, metric):
+    test_names, metric_df = group_metrics_group_by_testname(df, metric)
+    show_df(metric_df)
+    plot_df(metric_df,test_names)
+
+# Given a metric return results per test group by fio job.
+# input:
+#    df: dataset from `import_data()`
+#    metric: string with the name of the metric to filter.
+# output:
+#    dataset with fomat:
+#      'workload' , 'name[0]' , ... , 'name[n]'
+#
+def group_metrics_group_by_testname(df, metric):
+  #name of each tests from results
+  names = set()
+  # Rows of new data set
+  rows = []
+  # map:
+  # keys: name of fio job
+  # value: dict[k]:v where k: name of a test, v: value of test for  metric`
+  workload = {}
+
+  for k, row in df.iterrows():
+    # name of a fio job
+    w = row['WORKLOAD']
+    # name of tests
+    tname = row['NAME']
+    names.add(tname)
+    # given a fio job name get dict of values
+    # if not previous values init empty dict
+    dict_values = workload.get(w, {})
+    # For a given metric, add it into as value of dict_values[testname]=val
+    #e.g
+    # dict_values["test-name"] = row["IOPS"]
+    dict_values[tname] = row[metric]
+    workload[w] = dict_values
+
+  names = list(names)
+  cols = ['WORKLOAD'] + list(names)
+  rdf = pd.DataFrame(workload,columns = cols)
+
+  for k in workload:
+    d = workload[k]
+
+    if not d[names[0]] == 0:
+      d["WORKLOAD"] = k;
+      rdf = rdf.append(d,ignore_index=True)
+  rdf = rdf.dropna()
+  return names, rdf
+
+def plot_df(df, names,sort_key=""):
+  if sort_key != "":
+    df.sort_values(sort_key, ascending=False)
+  df.plot(kind='bar',x="WORKLOAD",y=names,  figsize=(30, 10))
+  plt.show()
+
+
+def import_data():
+    frames = []
+    for f in glob.glob('./results/*/results.csv'):
+        print("reading:" + f)
+        df = pd.read_csv(f)
+        frames.append(df)
+    return pd.concat(frames)
+
+def show_df(df):
+    pd.set_option('display.max_rows', df.shape[0]+1)
+    print(df)
+
+def print_md(s):
+     display(Markdown(s))
+
+#notebook entrypoint
+def generate_report():
+    #Load the all test results in a single dataset
+    df_results = import_data()
+    print_md("Show all data from results")
+    show_df(df_results)
+    print_md("### Compare the tests results group by fio job. The metric used to compare is write bandwidth")
+    compare_tests_group_by_fio_job(df_results, 'bw_w')
+    print_md("### Compare the tests results group by fio job. The metric used to compare is read bandwidth")
+    compare_tests_group_by_fio_job(df_results, 'bw_r')
+    print_md("### Compare the tests results group by fio job. The metric used to compare is write IOPS(Input/Output Operations Per Second)")
+    compare_tests_group_by_fio_job(df_results, 'IOPS_w')
+    print_md("### Compare the tests results group by fio job. The metric used to compare is read IOPS(Input/Output Operations Per Second)")
+    compare_tests_group_by_fio_job(df_results, 'IOPS_r')

tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/gen-html-fio-report.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+#Copyright (c) 2021-2023 Intel Corporation
+#
+#SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+script_dir=$(dirname "$(readlink -f "$0")")
+
+results_dir=${1:-}
+
+usage(){
+	echo "$0 <results_dir>"
+}
+
+if [ "${results_dir}" == "" ];then
+	echo "missing results directory"
+	usage
+	exit 1
+fi
+
+if [ ! -d "${results_dir}" ];then
+	echo "${results_dir} is not a directory"
+	usage
+	exit 1
+fi
+
+results_dir=$(realpath "${results_dir}")
+
+generate_report(){
+	sudo chown "${USER}:${USER}" -R ${results_dir}
+	sudo docker run --rm -e JUPYTER_ENABLE_LAB=yes \
+		-v "${script_dir}:/home/jovyan" \
+		-v "${results_dir}:/home/jovyan/results" \
+		--user $(id -u):$(id -g) \
+		jupyter/scipy-notebook:399cbb986c6b \
+		bash  -e -c '
+			cd results;
+			jupyter nbconvert --execute /home/jovyan/fio.ipynb --to html;
+			cp /home/jovyan/fio.html /home/jovyan/results;
+			'
+}
+
+generate_report

tests/metrics/storage/fio-k8s/scripts/dax-compare-test/report/run-docker-jupyter-server.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#Copyright (c) 2021-2023 Intel Corporation
+#
+#SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+script_dir=$(dirname "$(readlink -f "$0")")
+NOTEBOOK_PORT="8888"
+
+results_dir=${1:-}
+
+usage(){
+	echo "$0 <results_dir>"
+}
+
+if [ "${results_dir}" == "" ];then
+	echo "missing results directory"
+	usage
+	exit 1
+fi
+
+if [ ! -d "${results_dir}" ];then
+	echo "${results_dir} is not a directory"
+	usage
+	exit 1
+fi
+
+results_dir=$(realpath "${results_dir}")
+
+sudo -E docker run --rm -p "${NOTEBOOK_PORT}:${NOTEBOOK_PORT}" -e JUPYTER_ENABLE_LAB=yes \
+	-v "${script_dir}:/home/jovyan" \
+	-v "${results_dir}:/home/jovyan/results" \
+	jupyter/scipy-notebook:399cbb986c6b \
+	start.sh jupyter lab --LabApp.token=''