Merge pull request #7503 from GabyCT/topic/ghafio

metrics: Add FIO test to gha for kata metrics CI

.github/workflows/run-metrics.yaml

@@ -49,6 +49,9 @@ jobs:
       - name: run tensorflow test
         run:  bash tests/metrics/gha-run.sh run-test-tensorflow
 
+      - name: run fio test
+        run:  bash tests/metrics/gha-run.sh run-test-fio
+
       - name: make metrics tarball ${{ matrix.vmm }}
         run: bash tests/metrics/gha-run.sh make-tarball-results
 

tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-clh-kata-metric8.toml

@@ -17,7 +17,7 @@ description = "measure container lifecycle timings"
 checkvar = ".\"boot-times\".Results | .[] | .\"to-workload\".Result"
 checktype = "mean"
 midval = 0.69
-minpercent = 30.0
+minpercent = 40.0
 maxpercent = 30.0
 
 [[metric]]

tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml

@@ -17,7 +17,7 @@ description = "measure container lifecycle timings"
 checkvar = ".\"boot-times\".Results | .[] | .\"to-workload\".Result"
 checktype = "mean"
 midval = 0.71
-minpercent = 30.0
+minpercent = 40.0
 maxpercent = 30.0
 
 [[metric]]

tests/metrics/gha-run.sh

@@ -85,6 +85,14 @@ function run_test_tensorflow() {
 	check_metrics
 }
 
+function run_test_fio() {
+	info "Running FIO test using ${KATA_HYPERVISOR} hypervisor"
+        # ToDo: remove the exit once the metrics workflow is stable
+        exit 0
+
+	bash storage/fio-k8s/fio-test-ci.sh
+}
+
 function main() {
 	action="${1:-}"
 	case "${action}" in
@@ -95,6 +103,7 @@ function main() {
 		run-test-memory-usage-inside-container) run_test_memory_usage_inside_container ;;
 		run-test-blogbench) run_test_blogbench ;;
 		run-test-tensorflow) run_test_tensorflow ;;
+		run-test-fio) run_test_fio ;;
 		*) >&2 die "Invalid argument" ;;
 	esac
 }

tests/metrics/storage/README.md

@@ -1,11 +1,27 @@
 # Kata Containers storage I/O tests
+
 The metrics tests in this directory are designed to be used to assess storage IO.
+
 ## `Blogbench` test
+
 The `blogbench` script is based on the `blogbench` program which is designed to emulate a busy blog server with a number of concurrent 
 threads performing a mixture of reads, writes and rewrites.
+
 ### Running the `blogbench` test
+
 The `blogbench` test can be run by hand, for example:
 ```
 $ cd metrics
 $ bash storage/blogbench.sh
 ```
+## `fio` test
+
+The `fio` test utilises the [fio tool](https://github.com/axboe/fio), configured
+to perform measurements upon a single test file.
+
+The test configuration used by the script can be modified by setting a number of
+environment variables to change or over-ride the test defaults.
+
+## DAX `virtio-fs` `fio` Kubernetes tests
+
+[Test](fio-k8s/README.md) to compare the use of DAX option in `virtio-fs`.

tests/metrics/storage/fio-k8s/README.md

@@ -0,0 +1,30 @@
+# FIO test in Kubernetes
+
+This is an automation to run `fio` with Kubernetes.
+
+## Requirements:
+
+- Kubernetes cluster running.
+- Kata configured as `runtimeclass`. 
+
+## Test structure:
+
+- [fio-test]: Program wrapper to launch `fio` in a K8s pod. 
+- [pkg]: Library code that could be used for more `fio` automation. 
+- [configs]: Configuration files used by [fio-test]. 
+- [DAX-compare-test]: Script to run [fio-test] to generate `fio` data for Kata with/without `virtio-fs DAX` and K8s bare-metal runtime(`runc`). 
+- [report] Jupyter Notebook to create reports for data generated by [DAX-compare-test].
+
+## Top-level Makefile targets
+
+- `build`: Build `fio` metrics.
+- `test`: quick test, used to verify changes in [fio-test]. 
+- `run`: Run `fio` metrics and generate reports. 
+- `test-report-interactive`: Run python notebook in `localhost:8888`, useful to edit the report. 
+- `test-report`: Generate report from data generated by `make test`. 
+
+[fio-test]:cmd/fiotest 
+[configs]:configs 
+[pkg]:pkg 
+[report]:scripts/dax-compare-test/report
+[DAX-compare-test]:scripts/dax-compare-test/README.md

tests/metrics/storage/fio-k8s/fio-test-ci.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+#
+# Copyright (c) 2022-2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+# General env
+SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
+source "${SCRIPT_PATH}/../../lib/common.bash"
+FIO_PATH="${GOPATH}/src/github.com/kata-containers/kata-containers/tests/metrics/storage/fio-k8s"
+TEST_NAME="${TEST_NAME:-fio}"
+
+function main() {
+	cmds=("bc" "jq")
+	check_cmds "${cmds[@]}"
+	check_processes
+	init_env
+
+	export KUBECONFIG="$HOME/.kube/config"
+
+	pushd "${FIO_PATH}"
+		echo "INFO: Running K8S FIO test"
+		make test-ci
+	popd
+
+	test_result_file="${FIO_PATH}/cmd/fiotest/test-results/kata/randrw-sync.job/output.json"
+
+	metrics_json_init
+	local read_io=$(cat $test_result_file | grep io_bytes | head -1 | sed 's/[[:blank:]]//g' | cut -f2 -d ':' | cut -f1 -d ',')
+	local read_bw=$(cat $test_result_file | grep bw_bytes | head -1 | sed 's/[[:blank:]]//g' | cut -f2 -d ':' | cut -f1 -d ',')
+ 	local read_90_percentile=$(cat $test_result_file | grep 90.000000 | head -1 | sed 's/[[:blank:]]//g' | cut -f2 -d ':' | cut -f1 -d ',')
+	local read_95_percentile=$(cat $test_result_file | grep 95.000000 | head -1 | sed 's/[[:blank:]]//g' | cut -f2 -d ':' | cut -f1 -d ',')
+	local write_io=$(cat $test_result_file | grep io_bytes | head -2 | tail -1 | sed 's/[[:blank:]]//g' | cut -f2 -d ':' | cut -f1 -d ',')
+	local write_bw=$(cat $test_result_file | grep bw_bytes | head -2 | tail -1 | sed 's/[[:blank:]]//g' | cut -f2 -d ':' | cut -f1 -d ',')
+	local write_90_percentile=$(cat $test_result_file | grep 90.000000 | head -2 | tail -1 | sed 's/[[:blank:]]//g' | cut -f2 -d ':' | cut -f1 -d ',')
+	local write_95_percentile=$(cat $test_result_file | grep 95.000000 | head -2 | tail -1 | sed 's/[[:blank:]]//g' | cut -f2 -d ':' | cut -f1 -d ',')
+
+	metrics_json_start_array
+	local json="$(cat << EOF
+	{
+		"readio": {
+			"Result" : $read_io,
+			"Units" : "bytes"
+		},
+		"readbw": {
+			"Result" : $read_bw,
+			"Units" : "bytes/sec"
+		},
+		"read90percentile": {
+			"Result" : $read_90_percentile,
+			"Units" : "ns"
+		},
+		"read95percentile": {
+			"Result" : $read_95_percentile,
+			"Units" : "ns"
+		},
+		"writeio": {
+			"Result" : $write_io,
+			"Units" : "bytes"
+		},
+		"writebw": {
+			"Result" : $write_bw,
+			"Units" : "bytes/sec"
+		},
+		"write90percentile": {
+			"Result" : $write_90_percentile,
+			"Units" : "ns"
+		},
+		"write95percentile": {
+			"Result" : $write_95_percentile,
+			"Units" : "ns"
+		}
+	}
+EOF
+)"
+	metrics_json_add_array_element "$json"
+	metrics_json_end_array "Results"
+	metrics_json_save
+
+	check_processes
+}
+
+main "$@"

tests/metrics/storage/fio-k8s/scripts/dax-compare-test/README.md

@@ -0,0 +1,47 @@
+# FIO in Kubernetes
+
+This test runs `fio` jobs to measure how Kata Containers work using virtio-fs DAX. The test works using Kubernetes.  
+The test has to run in a single node cluster, it is needed as the test modifies Kata configuration file. 
+
+The `virtio-fs` options that this test will use are: 
+
+* `cache mode` Only `auto`, this is the most compatible mode for most of the Kata use cases. Today this is default in Kata. 
+* `thread pool size` Restrict the number of worker threads per request queue, zero means no thread pool. 
+* `DAX` 
+``` 
+File contents can be mapped into a memory window on the host, allowing the guest to directly access data from the host page cache. This has several advantages: The guest page cache is bypassed, reducing the memory footprint.  No communication is necessary 
+to access file contents, improving I/O performance. Shared file access is coherent between virtual machines on the same host even with mmap. 
+``` 
+
+This test by default iterates over different `virtio-fs` configurations.
+
+| test name                 | DAX | thread pool size | cache mode |
+|---------------------------|-----|------------------|------------|
+| pool_0_cache_auto_no_DAX  | no  | 0                | auto       |
+| pool_0_cache_auto_DAX     | yes | 0                | auto       |
+
+The `fio` options used are: 
+
+`ioengine`: How the IO requests are issued to the kernel. 
+* `libaio`: Supports async IO for both direct and buffered IO. 
+* `mmap`: File is memory mapped with mmap(2) and data copied to/from using memcpy(3). 
+
+`rw type`: Type of I/O pattern. 
+* `randread`: Random reads. 
+* `randrw`: Random mixed reads and writes. 
+* `randwrite`: Random writes. 
+* `read`: Sequential reads. 
+* `write`: Sequential writes.
+
+Additional notes: Some jobs contain a `multi` prefix. This means that the same job runs more than once at the same time using its own file.
+
+### Static `fio` values:
+
+Some `fio` values are not modified over all the jobs. 
+
+* `runtime`: Tell `fio` to terminate processing after the specified period of time(seconds).
+* `loops`: Run the specified number of iterations of this job. Used to repeat the same workload a given number of times.
+* `iodepth`: Number of I/O units to keep in flight against the file. Note that increasing `iodepth` beyond 1 will not affect synchronous `ioengine`.
+* `size`: The total size of file I/O for each thread of this job.
+* `direct`: If value is true, use non-buffered I/O. This is usually O_`DIRECT`.
+* `blocksize`: The block size in bytes used for I/O units.