From 0f3e0b40f68553b13a9495cf21666520774ed29a Mon Sep 17 00:00:00 2001
From: Jan Chaloupka <jchaloup@redhat.com>
Date: Fri, 21 Feb 2020 17:19:32 +0100
Subject: [PATCH] scheduler_perf: describe how to run BenchmarkPerfScheduling
 manually

---
 test/integration/scheduler_perf/README.md     | 36 +++++++++++--
 .../scheduler_perf/test-performance.sh        | 52 -------------------
 2 files changed, 32 insertions(+), 56 deletions(-)
 delete mode 100755 test/integration/scheduler_perf/test-performance.sh

diff --git a/test/integration/scheduler_perf/README.md b/test/integration/scheduler_perf/README.md
index d3a1f8d789d..9ec46336b87 100644
--- a/test/integration/scheduler_perf/README.md
+++ b/test/integration/scheduler_perf/README.md
@@ -31,14 +31,42 @@ Currently the test suite has the following:
 
 How To Run
 ------
+
+## Density tests
+
 ```shell
 # In Kubernetes root path
-make generated_files
-
-cd test/integration/scheduler_perf
-./test-performance.sh
+make test-integration WHAT=./test/integration/scheduler_perf KUBE_TEST_VMODULE="''" KUBE_TEST_ARGS="-alsologtostderr=true -logtostderr=true -run=." KUBE_TIMEOUT="--timeout=60m" SHORT="--short=false"
 ```
 
+## Benchmark tests
+
+```shell
+# In Kubernetes root path
+make test-integration WHAT=./test/integration/scheduler_perf KUBE_TEST_VMODULE="''" KUBE_TEST_ARGS="-alsologtostderr=false -logtostderr=false -run=^$$ -benchtime=1ns -bench=BenchmarkPerfScheduling"
+```
+
+The benchmark suite runs all the tests specified under config/performance-config.yaml.
+
+Once the benchmark is finished, JSON file with metrics is available in the current directory (test/integration/scheduler_perf). Look for `BenchmarkPerfScheduling_YYYY-MM-DDTHH:MM:SSZ.json`.
+You can use `-data-items-dir` to generate the metrics file elsewhere.
+
+In case you want to run a specific test in the suite, you can specify the test through `-bench` flag:
+
+Also, bench time is explicitly set to 1ns (`-benchtime=1ns` flag) so each test is run only once.
+Otherwise, the golang benchmark framework will try to run a test more than once in case it ran for less than 1s.
+
+```shell
+# In Kubernetes root path
+make test-integration WHAT=./test/integration/scheduler_perf KUBE_TEST_VMODULE="''" KUBE_TEST_ARGS="-alsologtostderr=false -logtostderr=false -run=^$$ -benchtime=1ns -bench=BenchmarkPerfScheduling/SchedulingBasic/5000Nodes/5000InitPods/1000PodsToSchedule"
+```
+
+To produce a cpu profile:
+
+```shell
+# In Kubernetes root path
+make test-integration WHAT=./test/integration/scheduler_perf KUBE_TIMEOUT="-timeout=3600s" KUBE_TEST_VMODULE="''" KUBE_TEST_ARGS="-alsologtostderr=false -logtostderr=false -run=^$$ -benchtime=1ns -bench=BenchmarkPerfScheduling -cpuprofile ~/cpu-profile.out"
+```
 
 <!-- BEGIN MUNGE: GENERATED_ANALYTICS -->
 [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/test/component/scheduler/perf/README.md?pixel)]()
diff --git a/test/integration/scheduler_perf/test-performance.sh b/test/integration/scheduler_perf/test-performance.sh
deleted file mode 100755
index ad75fae5dfd..00000000000
--- a/test/integration/scheduler_perf/test-performance.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2014 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../../
-source "${KUBE_ROOT}/hack/lib/init.sh"
-
-kube::golang::setup_env
-
-DIR_BASENAME=$(dirname "${BASH_SOURCE[0]}")
-pushd "${DIR_BASENAME}"
-
-cleanup() {
-  popd 2> /dev/null
-  kube::etcd::cleanup
-  kube::log::status "performance test cleanup complete"
-}
-
-trap cleanup EXIT
-
-kube::etcd::start
-
-# We are using the benchmark suite to do profiling. Because it only runs a few pods and
-# theoretically it has less variance.
-if ${RUN_BENCHMARK:-false}; then
-  kube::log::status "performance test (benchmark) compiling"
-  go test -c -o "perf.test"
-
-  kube::log::status "performance test (benchmark) start"
-  "./perf.test" -test.bench=. -test.run=xxxx -test.cpuprofile=prof.out -test.short=false
-  kube::log::status "...benchmark tests finished"
-fi
-# Running density tests. It might take a long time.
-kube::log::status "performance test (density) start"
-go test -test.run=. -test.timeout=60m -test.short=false
-kube::log::status "...density tests finished"