From 550d4c00746721ba06b0756d4dde5857d3dd7ea8 Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Fri, 17 Mar 2023 15:06:11 +0100
Subject: [PATCH] scheduler_perf: support test case selection via labels

Entire test cases and workloads can have labels attached to them. The union of
these must match the label filter which works as in GitHub. The benchmark by
default runs the tests that are labeled "performance", which is the same as
before.
---
 test/integration/scheduler_perf/README.md     | 11 +++
 .../config/performance-config.yaml            | 33 +++++++++
 .../scheduler_perf/label_selector.go          | 50 +++++++++++++
 .../scheduler_perf/label_selector_test.go     | 71 +++++++++++++++++++
 .../scheduler_perf/scheduler_perf_test.go     | 10 +++
 5 files changed, 175 insertions(+)
 create mode 100644 test/integration/scheduler_perf/label_selector.go
 create mode 100644 test/integration/scheduler_perf/label_selector_test.go

diff --git a/test/integration/scheduler_perf/README.md b/test/integration/scheduler_perf/README.md
index a10eec0bf49..0087f50088c 100644
--- a/test/integration/scheduler_perf/README.md
+++ b/test/integration/scheduler_perf/README.md
@@ -36,6 +36,17 @@ make test-integration WHAT=./test/integration/scheduler_perf ETCD_LOGLEVEL=warn
 ```
 
 The benchmark suite runs all the tests specified under config/performance-config.yaml.
+By default, it runs all workloads that have the "performance" label. In the configuration,
+labels can be added to a test case and/or individual workloads. Each workload also has
+all labels of its test case. The `perf-scheduling-label-filter` command line flag can
+be used to select workloads. It works like GitHub label filtering: the flag accepts
+a comma-separated list of label names. Each label may have a `+` or `-` as prefix. Labels with
+`+` or no prefix must be set for a workload for it to be run. `-` means that the label must not
+be set. For example, this runs all performance benchmarks except those that are labeled
+as "fast":
+```shell
+make test-integration WHAT=./test/integration/scheduler_perf ETCD_LOGLEVEL=warn KUBE_TEST_VMODULE="''" KUBE_TEST_ARGS="-run=^$$ -benchtime=1ns -bench=BenchmarkPerfScheduling -perf-scheduling-label-filter=performance,-fast"
+```
 
 Once the benchmark is finished, JSON file with metrics is available in the current directory (test/integration/scheduler_perf). Look for `BenchmarkPerfScheduling_YYYY-MM-DDTHH:MM:SSZ.json`.
 You can use `-data-items-dir` to generate the metrics file elsewhere.
diff --git a/test/integration/scheduler_perf/config/performance-config.yaml b/test/integration/scheduler_perf/config/performance-config.yaml
index 84aa647c437..9fa9b31f023 100644
--- a/test/integration/scheduler_perf/config/performance-config.yaml
+++ b/test/integration/scheduler_perf/config/performance-config.yaml
@@ -1,4 +1,5 @@
 - name: SchedulingBasic
+  labels: [performance]
   defaultPodTemplatePath: config/pod-default.yaml
   workloadTemplate:
   - opcode: createNodes
@@ -10,6 +11,7 @@
     collectMetrics: true
   workloads:
   - name: 500Nodes
+    labels: [fast]
     params:
       initNodes: 500
       initPods: 500
@@ -21,6 +23,7 @@
       measurePods: 1000
 
 - name: SchedulingPodAntiAffinity
+  labels: [performance]
   defaultPodTemplatePath: config/pod-with-pod-anti-affinity.yaml
   workloadTemplate:
   - opcode: createNodes
@@ -39,6 +42,7 @@
     namespace: sched-1
   workloads:
   - name: 500Nodes
+    labels: [fast]
     params:
       initNodes: 500
       initPods: 100
@@ -50,6 +54,7 @@
       measurePods: 1000
 
 - name: SchedulingSecrets
+  labels: [performance]
   defaultPodTemplatePath: config/pod-with-secret-volume.yaml
   workloadTemplate:
   - opcode: createNodes
@@ -61,6 +66,7 @@
     collectMetrics: true
   workloads:
   - name: 500Nodes
+    labels: [fast]
     params:
       initNodes: 500
       initPods: 500
@@ -72,6 +78,7 @@
       measurePods: 1000
 
 - name: SchedulingInTreePVs
+  labels: [performance]
   workloadTemplate:
   - opcode: createNodes
     countParam: $initNodes
@@ -86,6 +93,7 @@
     collectMetrics: true
   workloads:
   - name: 500Nodes
+    labels: [fast]
     params:
       initNodes: 500
       initPods: 500
@@ -97,6 +105,7 @@
       measurePods: 1000
 
 - name: SchedulingMigratedInTreePVs
+  labels: [performance]
   workloadTemplate:
   - opcode: createNodes
     countParam: $initNodes
@@ -120,6 +129,7 @@
     collectMetrics: true
   workloads:
   - name: 500Nodes
+    labels: [fast]
     params:
       initNodes: 500
       initPods: 500
@@ -131,6 +141,7 @@
       measurePods: 1000
 
 - name: SchedulingCSIPVs
+  labels: [performance]
   workloadTemplate:
   - opcode: createNodes
     countParam: $initNodes
@@ -152,6 +163,7 @@
     collectMetrics: true
   workloads:
   - name: 500Nodes
+    labels: [fast]
     params:
       initNodes: 500
       initPods: 500
@@ -163,6 +175,7 @@
       measurePods: 1000
 
 - name: SchedulingPodAffinity
+  labels: [performance]
   defaultPodTemplatePath: config/pod-with-pod-affinity.yaml
   workloadTemplate:
   - opcode: createNodes
@@ -183,6 +196,7 @@
     collectMetrics: true
   workloads:
   - name: 500Nodes
+    labels: [fast]
     params:
       initNodes: 500
       initPods: 500
@@ -194,6 +208,7 @@
       measurePods: 1000
 
 - name: SchedulingPreferredPodAffinity
+  labels: [performance]
   defaultPodTemplatePath: config/pod-with-preferred-pod-affinity.yaml
   workloadTemplate:
   - opcode: createNodes
@@ -212,6 +227,7 @@
     collectMetrics: true
   workloads:
   - name: 500Nodes
+    labels: [fast]
     params:
       initNodes: 500
       initPods: 500
@@ -223,6 +239,7 @@
       measurePods: 1000
 
 - name: SchedulingPreferredPodAntiAffinity
+  labels: [performance]
   defaultPodTemplatePath: config/pod-with-preferred-pod-affinity.yaml
   workloadTemplate:
   - opcode: createNodes
@@ -241,6 +258,7 @@
     collectMetrics: true
   workloads:
   - name: 500Nodes
+    labels: [fast]
     params:
       initNodes: 500
       initPods: 500
@@ -252,6 +270,7 @@
       measurePods: 1000
 
 - name: SchedulingNodeAffinity
+  labels: [performance]
   defaultPodTemplatePath: config/pod-with-node-affinity.yaml
   workloadTemplate:
   - opcode: createNodes
@@ -267,6 +286,7 @@
     collectMetrics: true
   workloads:
   - name: 500Nodes
+    labels: [fast]
     params:
       initNodes: 500
       initPods: 500
@@ -278,6 +298,7 @@
       measurePods: 1000
 
 - name: TopologySpreading
+  labels: [performance]
   workloadTemplate:
   - opcode: createNodes
     countParam: $initNodes
@@ -294,6 +315,7 @@
     collectMetrics: true
   workloads:
   - name: 500Nodes
+    labels: [fast]
     params:
       initNodes: 500
       initPods: 1000
@@ -305,6 +327,7 @@
       measurePods: 2000
 
 - name: PreferredTopologySpreading
+  labels: [performance]
   workloadTemplate:
   - opcode: createNodes
     countParam: $initNodes
@@ -321,6 +344,7 @@
     collectMetrics: true
   workloads:
   - name: 500Nodes
+    labels: [fast]
     params:
       initNodes: 500
       initPods: 1000
@@ -332,6 +356,7 @@
       measurePods: 2000
 
 - name: MixedSchedulingBasePod
+  labels: [performance]
   defaultPodTemplatePath: config/pod-default.yaml
   workloadTemplate:
   - opcode: createNodes
@@ -367,6 +392,7 @@
     collectMetrics: true
   workloads:
   - name: 500Nodes
+    labels: [fast]
     params:
       initNodes: 500
       initPods: 200
@@ -378,6 +404,7 @@
       measurePods: 1000
 
 - name: PreemptionBasic
+  labels: [performance]
   workloadTemplate:
   - opcode: createNodes
     countParam: $initNodes
@@ -404,6 +431,7 @@
 #      measurePods: 5000
 
 - name: PreemptionPVs
+  labels: [performance]
   workloadTemplate:
   - opcode: createNodes
     countParam: $initNodes
@@ -418,6 +446,7 @@
     collectMetrics: true
   workloads:
   - name: 500Nodes
+    labels: [fast]
     params:
       initNodes: 500
       initPods: 2000
@@ -432,6 +461,7 @@
 #      measurePods: 5000
 
 - name: Unschedulable
+  labels: [performance]
   workloadTemplate:
   - opcode: createNodes
     countParam: $initNodes
@@ -445,6 +475,7 @@
     collectMetrics: true
   workloads:
   - name: 500Nodes/200InitPods
+    labels: [fast]
     params:
       initNodes: 500
       initPods: 200
@@ -478,6 +509,7 @@
     collectMetrics: true
   workloads:
   - name: 1000Nodes
+    labels: [fast]
     params:
       initNodes: 1000
       measurePods: 1000
@@ -638,6 +670,7 @@
     collectMetrics: true
   workloads:
   - name: 500Nodes
+    labels: [fast]
     params:
       taintNodes: 100
       normalNodes: 400
diff --git a/test/integration/scheduler_perf/label_selector.go b/test/integration/scheduler_perf/label_selector.go
new file mode 100644
index 00000000000..2d3a72bccdc
--- /dev/null
+++ b/test/integration/scheduler_perf/label_selector.go
@@ -0,0 +1,50 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package benchmark
+
+import "strings"
+
+// enabled checks a a label filter that works as in GitHub:
+// - empty string means enabled
+// - individual labels are comma-separated
+// - [+]<label> means the workload must have that label
+// - -<label> means the workload must not have that label
+func enabled(labelFilter string, labels ...string) bool {
+	for _, label := range strings.Split(labelFilter, ",") {
+		if label == "" {
+			continue
+		}
+		mustHaveLabel := label[0] != '-'
+		if label[0] == '-' || label[0] == '+' {
+			label = label[1:]
+		}
+		haveLabel := containsStr(labels, label)
+		if haveLabel != mustHaveLabel {
+			return false
+		}
+	}
+	return true
+}
+
+func containsStr(hay []string, needle string) bool {
+	for _, item := range hay {
+		if item == needle {
+			return true
+		}
+	}
+	return false
+}
diff --git a/test/integration/scheduler_perf/label_selector_test.go b/test/integration/scheduler_perf/label_selector_test.go
new file mode 100644
index 00000000000..901e72dd600
--- /dev/null
+++ b/test/integration/scheduler_perf/label_selector_test.go
@@ -0,0 +1,71 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package benchmark
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestLabelFilter(t *testing.T) {
+	empty := ""
+	performance := "performance"
+	fastPerformance := "performance,fast"
+	notFastPerformance := "+performance,-fast"
+	notFast := "-fast"
+
+	testcases := map[string]map[string]bool{
+		empty: {
+			empty:           true,
+			performance:     true,
+			fastPerformance: true,
+		},
+		performance: {
+			empty:           false,
+			performance:     true,
+			fastPerformance: true,
+		},
+		fastPerformance: {
+			empty:           false,
+			performance:     false,
+			fastPerformance: true,
+		},
+		notFast: {
+			empty:           true,
+			performance:     true,
+			fastPerformance: false,
+		},
+		notFastPerformance: {
+			empty:           false,
+			performance:     true,
+			fastPerformance: false,
+		},
+	}
+
+	for labelFilter, labelResults := range testcases {
+		t.Run(labelFilter, func(t *testing.T) {
+			for labels, expected := range labelResults {
+				t.Run(labels, func(t *testing.T) {
+					actual := enabled(labelFilter, strings.Split(labels, ",")...)
+					if actual != expected {
+						t.Errorf("expected enabled to be %v, got %v", expected, actual)
+					}
+				})
+			}
+		})
+	}
+}
diff --git a/test/integration/scheduler_perf/scheduler_perf_test.go b/test/integration/scheduler_perf/scheduler_perf_test.go
index 9177fc6bb89..dd5076cc240 100644
--- a/test/integration/scheduler_perf/scheduler_perf_test.go
+++ b/test/integration/scheduler_perf/scheduler_perf_test.go
@@ -121,6 +121,8 @@ type testCase struct {
 	// This path can be overridden in createPodsOp by setting PodTemplatePath .
 	// Optional
 	DefaultPodTemplatePath *string
+	// Labels can be used to enable or disable workloads inside this test case.
+	Labels []string
 }
 
 func (tc *testCase) collectsMetrics() bool {
@@ -151,6 +153,8 @@ type workload struct {
 	Name string
 	// Values of parameters used in the workloadTemplate.
 	Params params
+	// Labels can be used to enable or disable a workload.
+	Labels []string
 }
 
 type params struct {
@@ -608,6 +612,8 @@ func initTestOutput(tb testing.TB) io.Writer {
 	return output
 }
 
+var perfSchedulingLabelFilter = flag.String("perf-scheduling-label-filter", "performance", "comma-separated list of labels which a testcase must have (no prefix or +) or must not have (-)")
+
 func BenchmarkPerfScheduling(b *testing.B) {
 	testCases, err := getTestCases(configFile)
 	if err != nil {
@@ -632,6 +638,10 @@ func BenchmarkPerfScheduling(b *testing.B) {
 		b.Run(tc.Name, func(b *testing.B) {
 			for _, w := range tc.Workloads {
 				b.Run(w.Name, func(b *testing.B) {
+					if !enabled(*perfSchedulingLabelFilter, append(tc.Labels, w.Labels...)...) {
+						b.Skipf("disabled by label filter %q", *perfSchedulingLabelFilter)
+					}
+
 					// Ensure that there are no leaked
 					// goroutines.  They could influence
 					// performance of the next benchmark.