Merge pull request #7442 from GabyCT/topic/addgofilesfio

metrics: Add FIO benchmark for metrics tests

tests/metrics/storage/fio-k8s/.gitignore

@@ -0,0 +1 @@
+./cmd/fiotest/fio-k8s

tests/metrics/storage/fio-k8s/Makefile

@@ -0,0 +1,28 @@
+#
+# Copyright (c) 2021-2022 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
+MKFILE_DIR := $(dir $(MKFILE_PATH))
+
+build:
+	make -C $(MKFILE_DIR)/cmd/fiotest/ gomod
+	make -C $(MKFILE_DIR)/cmd/fiotest/ build
+
+test-report:
+	$(MKFILE_DIR)/scripts/dax-compare-test/report/gen-html-fio-report.sh $(MKFILE_DIR)/cmd/fiotest/test-results/
+
+test-report-interactive:
+	$(MKFILE_DIR)/scripts/dax-compare-test/report/run-docker-jupyter-server.sh $(MKFILE_DIR)/cmd/fiotest/test-results/
+
+test: build
+	make -C $(MKFILE_DIR)/cmd/fiotest/ run
+	make test-report
+
+run: build
+	make -C $(MKFILE_DIR)/scripts/dax-compare-test/ run
+
+test-ci: build
+	make -C $(MKFILE_DIR)/cmd/fiotest/ runci

tests/metrics/storage/fio-k8s/cmd/fiotest/Makefile

@@ -0,0 +1,24 @@
+#
+# Copyright (c) 2021-2022 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
+MKFILE_DIR := $(dir $(MKFILE_PATH))
+
+build:
+	GO111MODULE=on go build
+
+run: build
+	$(MKFILE_DIR)/fio-k8s --debug --fio.size 10M --output-dir test-results --test-name kata $(MKFILE_DIR)/../../configs/example-config/
+	$(MKFILE_DIR)/fio-k8s --debug --fio.size 10M --output-dir test-results --test-name runc --container-runtime runc $(MKFILE_DIR)/../../configs/example-config/
+
+gomod:
+	 go mod edit -replace=github.com/kata-containers/kata-containers/tests/metrics/k8s=../../pkg/k8s
+	 go mod edit -replace=github.com/kata-containers/kata-containers/tests/metrics/exec=../../pkg/exec
+	 go mod edit -replace=github.com/kata-containers/kata-containers/tests/metrics/env=../../pkg/env
+	 go mod tidy
+
+runci: build
+	$(MKFILE_DIR)/fio-k8s --debug --fio.size 10M --output-dir test-results --test-name kata $(MKFILE_DIR)/../../configs/example-config/

tests/metrics/storage/fio-k8s/cmd/fiotest/go.mod

@@ -0,0 +1,24 @@
+module github.com/kata-containers/kata-containers/tests/metrics/storage/fio-k8s
+
+go 1.19
+
+replace github.com/kata-containers/kata-containers/tests/metrics/exec => ../../pkg/exec
+
+replace github.com/kata-containers/kata-containers/tests/metrics/k8s => ../../pkg/k8s
+
+replace github.com/kata-containers/kata-containers/tests/metrics/env => ../../pkg/env
+
+require (
+	github.com/kata-containers/kata-containers/tests/metrics/env v0.0.0-00010101000000-000000000000
+	github.com/kata-containers/kata-containers/tests/metrics/exec v0.0.0-00010101000000-000000000000
+	github.com/kata-containers/kata-containers/tests/metrics/k8s v0.0.0-00010101000000-000000000000
+	github.com/pkg/errors v0.9.1
+	github.com/sirupsen/logrus v1.9.3
+	github.com/urfave/cli v1.22.14
+)
+
+require (
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+)

tests/metrics/storage/fio-k8s/cmd/fiotest/go.sum

@@ -0,0 +1,31 @@
+github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk=
+github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

tests/metrics/storage/fio-k8s/cmd/fiotest/main.go

@@ -0,0 +1,373 @@
+// Copyright (c) 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+package main
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	env "github.com/kata-containers/kata-containers/tests/metrics/env"
+	exec "github.com/kata-containers/kata-containers/tests/metrics/exec"
+	"github.com/kata-containers/kata-containers/tests/metrics/k8s"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var log = logrus.New()
+
+var (
+	optContainerRuntime = "container-runtime"
+	optDebug            = "debug"
+	optOutputDir        = "output-dir"
+	optTestName         = "test-name"
+	// fio options
+	optFioBlockSize = "fio.block-size"
+	optFioDirect    = "fio.direct"
+	optFioIoDepth   = "fio.iodepth"
+	optFioSize      = "fio.size"
+	optFioNumJobs   = "fio.numjobs"
+)
+
+type RwFioOp struct {
+	BandwidthKb int     `json:"bw"`
+	IOPS        float64 `json:"iops"`
+}
+
+type fioResult struct {
+	GlobalOptions struct {
+		IOEngine string `json:"ioengine"`
+		RW       string `json:"rw"`
+	} `json:"global options"`
+	Jobs []struct {
+		JobName string  `json:"jobname"`
+		Read    RwFioOp `json:"read"`
+		Write   RwFioOp `json:"write"`
+	} `json:"jobs"`
+}
+
+// Run fio in k8s metrics test in K8s
+func (c fioTestConfig) run() (result fioResult, err error) {
+	log.Infof("Running fio config: %s", c.jobFile)
+
+	pod := k8s.Pod{YamlPath: c.k8sYaml}
+
+	log.Infof("Delete pod if already created")
+	err = pod.Delete()
+	if err != nil {
+		return result, err
+	}
+
+	log.Infof("Create pod: %s", pod.YamlPath)
+	err = pod.Run()
+	if err != nil {
+		return result, err
+	}
+
+	defer func() {
+		log.Info("Deleting pod")
+		delErr := pod.Delete()
+		if delErr != nil {
+			log.Error(delErr)
+			if err != nil {
+				err = errors.Wrapf(err, "Could not delete pod after: %s", delErr)
+			}
+		}
+	}()
+
+	destDir := "/home/fio-jobs"
+	_, err = pod.Exec("mkdir " + destDir)
+	if err != nil {
+		return result, err
+	}
+
+	dstJobFile := path.Join(destDir, "jobFile")
+	err = pod.CopyFromHost(c.jobFile, dstJobFile)
+	if err != nil {
+		return result, err
+	}
+
+	_, err = pod.Exec("apt update")
+	if err != nil {
+		return result, err
+	}
+	_, err = pod.Exec("apt install -y fio")
+	if err != nil {
+		return result, err
+	}
+
+	err = env.DropCaches()
+	if err != nil {
+		return result, err
+	}
+
+	var directStr string
+	if c.direct {
+		directStr = "1"
+	} else {
+		directStr = "0"
+	}
+
+	cmdFio := "fio"
+	cmdFio += " --append-terse "
+	cmdFio += " --blocksize=" + c.blocksize
+	cmdFio += " --direct=" + directStr
+	cmdFio += " --directory=" + c.directory
+	cmdFio += " --iodepth=" + c.iodepth
+	cmdFio += " --numjobs=" + c.numjobs
+	cmdFio += " --runtime=" + c.runtime
+	cmdFio += " --size=" + c.size
+	cmdFio += " --output-format=json"
+	cmdFio += " " + dstJobFile
+
+	log.Infof("Exec fio")
+	output, err := pod.Exec(cmdFio, k8s.ExecOptShowStdOut())
+	if err != nil {
+		return result, err
+	}
+	err = json.Unmarshal([]byte(output), &result)
+	if err != nil {
+		return result, errors.Wrapf(err, "failed to unmarshall output : %s", output)
+	}
+
+	log.Infof("ioengine:%s", result.GlobalOptions.IOEngine)
+	log.Infof("rw:%s", result.GlobalOptions.RW)
+	if len(result.Jobs) == 0 {
+		return result, errors.New("No jobs found after parsing fio results")
+	}
+
+	testDir := path.Join(c.outputDir, filepath.Base(c.jobFile))
+	err = os.MkdirAll(testDir, 0775)
+	if err != nil {
+		return result, errors.Wrapf(err, "failed to create test directory for :%s", c.jobFile)
+	}
+	outputFile := path.Join(testDir, "output.json")
+	log.Infof("Store results output in : %s", outputFile)
+
+	err = os.WriteFile(outputFile, []byte(output), 0644)
+	if err != nil {
+		return result, err
+	}
+
+	return result, nil
+}
+
+type fioTestConfig struct {
+	//test options
+	k8sYaml          string
+	containerRuntime string
+	outputDir        string
+
+	//fio options
+	blocksize string
+	directory string
+	iodepth   string
+	numjobs   string
+	jobFile   string
+	loops     string
+	runtime   string
+	size      string
+
+	direct bool
+}
+
+func runFioJobs(testDirPath string, cfg fioTestConfig) (results []fioResult, err error) {
+	fioJobsDir, err := filepath.Abs(path.Join(testDirPath, "fio-jobs"))
+	if err != nil {
+		return results, err
+	}
+
+	files, err := os.ReadDir(fioJobsDir)
+	if err != nil {
+		log.Fatal(err)
+		return results, err
+	}
+
+	if cfg.containerRuntime == "" {
+		return results, errors.New("containerRuntime is empty")
+	}
+
+	podYAMLName := cfg.containerRuntime + ".yaml"
+	cfg.k8sYaml = path.Join(testDirPath, podYAMLName)
+
+	if len(files) == 0 {
+		return results, errors.New("No fio configs found")
+	}
+
+	for _, file := range files {
+		cfg.jobFile = path.Join(fioJobsDir, file.Name())
+		r, err := cfg.run()
+		if err != nil {
+			return results, err
+		}
+		results = append(results, r)
+
+		log.Infof("workload:%s", r.Jobs[0].JobName)
+		log.Infof("bw_r:%d", r.Jobs[0].Read.BandwidthKb)
+		log.Infof("IOPS_r:%f", r.Jobs[0].Read.IOPS)
+		log.Infof("bw_w:%d", r.Jobs[0].Write.BandwidthKb)
+		log.Infof("IOPS_w:%f", r.Jobs[0].Write.IOPS)
+
+		waitTime := 5
+		log.Debugf("Sleep %d seconds(if not wait sometimes create another pod timesout)", waitTime)
+		time.Sleep(time.Duration(waitTime) * time.Second)
+	}
+	return results, err
+
+}
+
+func generateResultsView(testName string, results []fioResult, outputDir string) error {
+	outputFile := path.Join(outputDir, "results.csv")
+	f, err := os.Create(outputFile)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	log.Infof("Creating results output in %s", outputFile)
+
+	w := csv.NewWriter(f)
+
+	headers := []string{"NAME", "WORKLOAD", "bw_r", "bw_w", "IOPS_r", "IOPS_w"}
+	err = w.Write(headers)
+	if err != nil {
+		return err
+	}
+
+	for _, r := range results {
+		if len(r.Jobs) == 0 {
+			return errors.Errorf("fio result has no jobs: %v", r)
+		}
+		row := []string{testName}
+		row = append(row, r.Jobs[0].JobName)
+		row = append(row, fmt.Sprintf("%d", r.Jobs[0].Read.BandwidthKb))
+		row = append(row, fmt.Sprintf("%d", r.Jobs[0].Write.BandwidthKb))
+		row = append(row, fmt.Sprintf("%f", r.Jobs[0].Read.IOPS))
+		row = append(row, fmt.Sprintf("%f", r.Jobs[0].Write.IOPS))
+		if err := w.Write(row); err != nil {
+			return err
+		}
+	}
+
+	w.Flush()
+
+	return w.Error()
+}
+
+func main() {
+
+	app := &cli.App{
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:  optDebug,
+				Usage: "Logs in debug level",
+			},
+			&cli.StringFlag{
+				Name:  optTestName,
+				Value: "kata-fio-test",
+				Usage: "Change the fio test name for reports",
+			},
+			&cli.StringFlag{
+				Name:  optOutputDir,
+				Value: ".",
+				Usage: "Use a file to store results",
+			},
+			&cli.StringFlag{
+				Name:  optContainerRuntime,
+				Value: "kata",
+				Usage: "Choose the runtime to use",
+			},
+			//fio options
+			&cli.StringFlag{
+				Name:  optFioSize,
+				Value: "200M",
+				Usage: "File size to use for tests",
+			},
+			&cli.StringFlag{
+				Name:  optFioBlockSize,
+				Value: "4K",
+				Usage: "Block size for fio tests",
+			},
+			&cli.BoolFlag{
+				Name:  optFioDirect,
+				Usage: "Use direct io",
+			},
+			&cli.StringFlag{
+				Name:  optFioIoDepth,
+				Value: "16",
+				Usage: "Number of I/O units to keep in flight against the file",
+			},
+			&cli.StringFlag{
+				Name:  optFioNumJobs,
+				Value: "1",
+				Usage: "Number of clones (processes/threads performing the same workload) of this job",
+			},
+		},
+		Action: func(c *cli.Context) error {
+			jobsDir := c.Args().First()
+
+			if jobsDir == "" {
+				cli.SubcommandHelpTemplate = strings.Replace(cli.SubcommandHelpTemplate, "[arguments...]", "<test-config-dir>", -1)
+				cli.ShowCommandHelp(c, "")
+				return errors.New("Missing <test-config-dir>")
+			}
+
+			if c.Bool(optDebug) {
+				log.SetLevel(logrus.DebugLevel)
+				k8s.Debug = true
+				env.Debug = true
+			}
+
+			exec.SetLogger(log)
+			k8s.SetLogger(log)
+			env.SetLogger(log)
+
+			testName := c.String(optTestName)
+
+			outputDir, err := filepath.Abs(path.Join(c.String(optOutputDir), testName))
+			if err != nil {
+				return err
+			}
+
+			cfg := fioTestConfig{
+				blocksize:        c.String(optFioBlockSize),
+				direct:           c.Bool(optFioDirect),
+				directory:        ".",
+				iodepth:          c.String(optFioIoDepth),
+				loops:            "3",
+				numjobs:          c.String(optFioNumJobs),
+				runtime:          "20",
+				size:             c.String(optFioSize),
+				containerRuntime: c.String(optContainerRuntime),
+				outputDir:        outputDir,
+			}
+
+			log.Infof("Results will be created in %s", cfg.outputDir)
+
+			err = os.MkdirAll(cfg.outputDir, 0775)
+			if err != nil {
+				return err
+			}
+
+			results, err := runFioJobs(jobsDir, cfg)
+			if err != nil {
+				return err
+			}
+
+			return generateResultsView(c.String(optTestName), results, outputDir)
+		},
+	}
+
+	err := app.Run(os.Args)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+}

tests/metrics/storage/fio-k8s/configs/example-config/fio-jobs/randrw-async.job

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2022 Intel Corporation
+[global]
+name=io_uring
+filename=fio-file
+rw=randrw
+rwmixread=75
+ioengine=io_uring
+
+[randrw-io_uring]

tests/metrics/storage/fio-k8s/configs/example-config/fio-jobs/randrw-libaio.job

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2021 Intel Corporation
+[global]
+name=randrw-libaio
+filename=fio-file
+rw=randrw
+rwmixread=75
+ioengine=libaio
+
+[randrw-libaio]

tests/metrics/storage/fio-k8s/configs/example-config/fio-jobs/randrw-sync.job

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2022 Intel Corporation
+[global]
+name=sync
+filename=fio-file
+rw=randrw
+rwmixread=75
+ioengine=sync
+
+[randrw-sync]

tests/metrics/storage/fio-k8s/configs/example-config/kata.yaml

@@ -0,0 +1,16 @@
+## Copyright (c) 2021 Intel Corporation
+#
+## SPDX-License-Identifier: Apache-2.0
+#
+apiVersion: v1
+kind: Pod
+metadata:
+  name: iometrics
+spec:
+  runtimeClassName: kata
+  containers:
+  - name: iometrics
+    image: ubuntu:latest
+    # Just spin & wait forever
+    command: [ "/bin/bash", "-c", "--" ]
+    args: [ "sleep infinity" ]

tests/metrics/storage/fio-k8s/configs/example-config/runc.yaml

@@ -0,0 +1,15 @@
+## Copyright (c) 2021 Intel Corporation
+#
+## SPDX-License-Identifier: Apache-2.0
+#
+apiVersion: v1
+kind: Pod
+metadata:
+  name: iometrics
+spec:
+  containers:
+  - name: iometrics
+    image: ubuntu:latest
+    # Just spin & wait forever
+    command: [ "/bin/bash", "-c", "--" ]
+    args: [ "sleep infinity" ]

tests/metrics/storage/fio-k8s/configs/test-config/fio-jobs/randread-libaio.job

@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2021 Intel Corporation
+[global]
+name=randread-libaio
+filename=fio-file
+rw=randread
+ioengine=libaio
+
+[randread-libaio]

tests/metrics/storage/fio-k8s/configs/test-config/fio-jobs/randread-mmpap.job

@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2021 Intel Corporation
+[global]
+name=randread-mmap
+rw=randread
+ioengine=mmap
+
+[randread-mmap]
+filename=fio-file

tests/metrics/storage/fio-k8s/configs/test-config/fio-jobs/randrw-libaio.job

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2021 Intel Corporation
+[global]
+name=randrw-libaio
+filename=fio-file
+rw=randrw
+rwmixread=75
+ioengine=libaio
+
+[randrw-libaio]

tests/metrics/storage/fio-k8s/configs/test-config/fio-jobs/randrw-mmap.job

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2021 Intel Corporation
+[global]
+name=randrw-mmap
+rw=randrw
+rwmixread=75
+ioengine=mmap
+
+[randrw-mmap]
+filename=fio-file

tests/metrics/storage/fio-k8s/configs/test-config/fio-jobs/randwrite-libaio.job

@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2021 Intel Corporation
+[global]
+name=randwrite-libaio
+filename=fio-file
+rw=randwrite
+ioengine=libaio
+
+[randwrite-libaio]

tests/metrics/storage/fio-k8s/configs/test-config/fio-jobs/randwrite-mmap.job

@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2021 Intel Corporation
+[global]
+name=randwrite-mmap
+rw=randwrite
+ioengine=mmap
+
+[randwrite-mmap]
+filename=fio-file

tests/metrics/storage/fio-k8s/configs/test-config/fio-jobs/seqread-libaio.job

@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2021 Intel Corporation
+[global]
+name=seqread-libaio
+filename=fio-file
+rw=read
+ioengine=libaio
+
+[seqread-libaio]

tests/metrics/storage/fio-k8s/configs/test-config/fio-jobs/seqread-mmap.job

@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2021 Intel Corporation
+[global]
+name=seqread-mmap
+rw=read
+ioengine=mmap
+
+[seqread-mmap]
+filename=fio-file

tests/metrics/storage/fio-k8s/configs/test-config/fio-jobs/seqread-psync.job

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2021 Intel Corporation
+[global]
+name=seqread-psync
+filename=fio-file
+rw=read
+
+[seqread-psync]

tests/metrics/storage/fio-k8s/configs/test-config/fio-jobs/seqwrite-libaio.job

@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2021 Intel Corporation
+[global]
+name=seqwrite-libaio
+filename=fio-file
+rw=write
+ioengine=libaio
+
+[seqwrite-libaio]

tests/metrics/storage/fio-k8s/configs/test-config/fio-jobs/seqwrite-mmap.job

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2021 Intel Corporation
+[global]
+name=seqwrite-mmap
+filename=fio-file
+rw=write
+ioengine=mmap
+
+[seqwrite-mmap]
+filename=fio-file

tests/metrics/storage/fio-k8s/configs/test-config/kata.yaml

@@ -0,0 +1,16 @@
+## Copyright (c) 2021 Intel Corporation
+#
+## SPDX-License-Identifier: Apache-2.0
+#
+apiVersion: v1
+kind: Pod
+metadata:
+  name: iometrics
+spec:
+  runtimeClassName: kata
+  containers:
+  - name: iometrics
+    image: ubuntu:latest
+    # Just spin & wait forever
+    command: [ "/bin/bash", "-c", "--" ]
+    args: [ "sleep infinity" ]

tests/metrics/storage/fio-k8s/configs/test-config/runc.yaml

@@ -0,0 +1,15 @@
+## Copyright (c) 2021 Intel Corporation
+#
+## SPDX-License-Identifier: Apache-2.0
+#
+apiVersion: v1
+kind: Pod
+metadata:
+  name: iometrics
+spec:
+  containers:
+  - name: iometrics
+    image: ubuntu:latest
+    # Just spin & wait forever
+    command: [ "/bin/bash", "-c", "--" ]
+    args: [ "sleep infinity" ]

tests/metrics/storage/fio-k8s/pkg/env/Makefile

@@ -0,0 +1,9 @@
+#
+# Copyright (c) 2021 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+gomod:
+	 GO111MODULE=on go mod edit -replace=github.com/kata-containers/kata-containers/tests/metrics/exec=../exec
+	 GO111MODULE=on go mod tidy

tests/metrics/storage/fio-k8s/pkg/env/env.go

@@ -0,0 +1,38 @@
+// Copyright (c) 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+package env
+
+import (
+	exec "github.com/kata-containers/kata-containers/tests/metrics/exec"
+)
+
+// logger interface for pkg
+var log logger
+var Debug bool = false
+
+type logger interface {
+	Infof(string, ...interface{})
+	Debugf(string, ...interface{})
+	Errorf(string, ...interface{})
+}
+
+func SetLogger(l logger) {
+	log = l
+}
+
+var sysDropCachesPath = "/proc/sys/vm/drop_caches"
+
+func DropCaches() (err error) {
+	log.Infof("drop caches")
+	_, err = exec.ExecCmd("sync", Debug)
+	if err != nil {
+		return err
+	}
+
+	_, err = exec.ExecCmd("echo 3 | sudo tee "+sysDropCachesPath, Debug)
+	if err != nil {
+		return err
+	}
+	return nil
+}

tests/metrics/storage/fio-k8s/pkg/env/go.mod

@@ -0,0 +1,10 @@
+module github.com/kata-containers/kata-containers/tests/metrics/storage/fio-k8s/exec
+
+go 1.19
+
+require (
+	github.com/kata-containers/kata-containers/tests/metrics/exec v0.0.0-00010101000000-000000000000 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+)
+
+replace github.com/kata-containers/kata-containers/tests/metrics/exec => ../exec

tests/metrics/storage/fio-k8s/pkg/env/go.sum

@@ -0,0 +1,2 @@
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

tests/metrics/storage/fio-k8s/pkg/exec/Exec.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+package exec
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"os/exec"
+
+	"github.com/pkg/errors"
+)
+
+// logger interface for pkg
+var log logger
+
+type logger interface {
+	Infof(string, ...interface{})
+	Debugf(string, ...interface{})
+	Errorf(string, ...interface{})
+}
+
+func SetLogger(l logger) {
+	log = l
+}
+
+// Exec a command
+// err != nil if command fails to execute
+// output is a string with a combined stdout and stderr
+func ExecCmd(c string, showInStdout bool) (stdout string, err error) {
+	if c == "" {
+		return "", errors.New("command is empty")
+	}
+
+	log.Debugf("Exec: %s", c)
+	cmd := exec.Command("bash", "-o", "pipefail", "-c", c)
+	var stdBuffer bytes.Buffer
+	var writers []io.Writer
+	writers = append(writers, &stdBuffer)
+	if showInStdout {
+		writers = append(writers, os.Stdout)
+	}
+	mw := io.MultiWriter(writers...)
+
+	cmd.Stdout = mw
+	cmd.Stderr = mw
+
+	err = cmd.Run()
+	output := stdBuffer.String()
+
+	return stdBuffer.String(), errors.Wrap(err, output)
+}
+
+// Exec a command
+// Send output to Stdout and Stderr
+func ExecStdout(c string) error {
+	if c == "" {
+		return errors.New("command is empty")
+	}
+
+	log.Debugf("Exec: %s", c)
+	cmd := exec.Command("bash", "-o", "pipefail", "-c", c)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}

tests/metrics/storage/fio-k8s/pkg/exec/go.mod

@@ -0,0 +1,5 @@
+module github.com/kata-containers/kata-containers/tests/metrics/storage/fio-k8s/exec
+
+go 1.19
+
+require github.com/pkg/errors v0.9.1

tests/metrics/storage/fio-k8s/pkg/exec/go.sum

@@ -0,0 +1,2 @@
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

tests/metrics/storage/fio-k8s/pkg/k8s/Makefile

@@ -0,0 +1,8 @@
+#
+# Copyright (c) 2021 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+gomod:
+	 GO111MODULE=on go mod edit -replace=github.com/kata-containers/kata-containers/tests/metrics/exec=../exec
+	 GO111MODULE=on go mod tidy

tests/metrics/storage/fio-k8s/pkg/k8s/exec.go

@@ -0,0 +1,34 @@
+// Copyright (c) 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+package k8s
+
+import (
+	"fmt"
+
+	exec "github.com/kata-containers/kata-containers/tests/metrics/exec"
+)
+
+type execOpt struct {
+	showInStdOut bool
+}
+
+type ExecOption func(e *execOpt)
+
+func ExecOptShowStdOut() ExecOption {
+	return func(e *execOpt) {
+		e.showInStdOut = true
+	}
+
+}
+
+func (p *Pod) Exec(cmd string, opts ...ExecOption) (output string, err error) {
+	log.Debugf("Exec %q in %s", cmd, p.YamlPath)
+	o := &execOpt{showInStdOut: false}
+	for _, opt := range opts {
+		opt(o)
+
+	}
+	execCmd := fmt.Sprintf("kubectl exec -f  %s -- /bin/bash -c %q", p.YamlPath, cmd)
+	return exec.ExecCmd(execCmd, Debug || o.showInStdOut)
+}

tests/metrics/storage/fio-k8s/pkg/k8s/go.mod

@@ -0,0 +1,10 @@
+module github.com/kata-containers/kata-containers/tests/metrics/k8s
+
+go 1.19
+
+replace github.com/kata-containers/kata-containers/tests/metrics/exec => ../exec
+
+require (
+	github.com/kata-containers/kata-containers/tests/metrics/exec v0.0.0-00010101000000-000000000000 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+)

tests/metrics/storage/fio-k8s/pkg/k8s/go.sum

@@ -0,0 +1,2 @@
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

tests/metrics/storage/fio-k8s/pkg/k8s/k8s.go

@@ -0,0 +1,68 @@
+// Copyright (c) 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+package k8s
+
+import (
+	"fmt"
+
+	exec "github.com/kata-containers/kata-containers/tests/metrics/exec"
+	"github.com/pkg/errors"
+)
+
+// logger interface for pkg
+var log logger
+var Debug bool = false
+
+type logger interface {
+	Infof(string, ...interface{})
+	Debugf(string, ...interface{})
+	Errorf(string, ...interface{})
+}
+
+func SetLogger(l logger) {
+	log = l
+}
+
+type Pod struct {
+	YamlPath string
+}
+
+func (p *Pod) waitForReady() (err error) {
+	log.Debugf("Wait for pod %s", p.YamlPath)
+	_, err = exec.ExecCmd("kubectl wait --for=condition=ready -f "+p.YamlPath, Debug)
+	return err
+}
+
+func (p *Pod) Run() (err error) {
+
+	log.Debugf("Creating K8s Pod %s", p.YamlPath)
+	_, err = exec.ExecCmd("kubectl apply -f "+p.YamlPath, Debug)
+	if err != nil {
+		return errors.Wrapf(err, "Failed to run pod %s", p.YamlPath)
+	}
+
+	err = p.waitForReady()
+	if err != nil {
+		return errors.Wrapf(err, "Failed to wait for pod  %s", p.YamlPath)
+	}
+	return err
+}
+
+func (p *Pod) Delete() (err error) {
+	log.Debugf("Delete pod %s", p.YamlPath)
+	_, err = exec.ExecCmd("kubectl delete --ignore-not-found -f "+p.YamlPath, Debug)
+	return errors.Wrapf(err, "Failed to delete pod %s", p.YamlPath)
+}
+
+func (p *Pod) CopyFromHost(src, dst string) (err error) {
+	podName, err := exec.ExecCmd("kubectl get -f "+p.YamlPath+" -o jsonpath={.metadata.name}", Debug)
+	if err != nil {
+		return nil
+	}
+
+	log.Debugf("Copy from host %q->%q in pod %s", src, dst, p.YamlPath)
+	execCmd := fmt.Sprintf("kubectl cp %s %s:%s", src, podName, dst)
+	_, err = exec.ExecCmd(execCmd, Debug)
+	return err
+}