mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-12 13:31:52 +00:00
scheduler_perf: support test case selection via labels
Entire test cases and workloads can have labels attached to them. The union of these must match the label filter which works as in GitHub. The benchmark by default runs the tests that are labeled "performance", which is the same as before.
This commit is contained in:
Patrick Ohly
parent
c9ff286668
commit
550d4c0074
@ -36,6 +36,17 @@ make test-integration WHAT=./test/integration/scheduler_perf ETCD_LOGLEVEL=warn
|
||||
```
|
||||
|
||||
The benchmark suite runs all the tests specified under config/performance-config.yaml.
|
||||
By default, it runs all workloads that have the "performance" label. In the configuration,
|
||||
labels can be added to a test case and/or individual workloads. Each workload also has
|
||||
all labels of its test case. The `perf-scheduling-label-filter` command line flag can
|
||||
be used to select workloads. It works like GitHub label filtering: the flag accepts
|
||||
a comma-separated list of label names. Each label may have a `+` or `-` as prefix. Labels with
|
||||
`+` or no prefix must be set for a workload for it to be run. `-` means that the label must not
|
||||
be set. For example, this runs all performance benchmarks except those that are labeled
|
||||
as "fast":
|
||||
```shell
|
||||
make test-integration WHAT=./test/integration/scheduler_perf ETCD_LOGLEVEL=warn KUBE_TEST_VMODULE="''" KUBE_TEST_ARGS="-run=^$$ -benchtime=1ns -bench=BenchmarkPerfScheduling -perf-scheduling-label-filter=performance,-fast"
|
||||
```
|
||||
|
||||
Once the benchmark is finished, JSON file with metrics is available in the current directory (test/integration/scheduler_perf). Look for `BenchmarkPerfScheduling_YYYY-MM-DDTHH:MM:SSZ.json`.
|
||||
You can use `-data-items-dir` to generate the metrics file elsewhere.
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
- name: SchedulingBasic
|
||||
labels: [performance]
|
||||
defaultPodTemplatePath: config/pod-default.yaml
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
@ -10,6 +11,7 @@
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: 500Nodes
|
||||
labels: [fast]
|
||||
params:
|
||||
initNodes: 500
|
||||
initPods: 500
|
||||
@ -21,6 +23,7 @@
|
||||
measurePods: 1000
|
||||
|
||||
- name: SchedulingPodAntiAffinity
|
||||
labels: [performance]
|
||||
defaultPodTemplatePath: config/pod-with-pod-anti-affinity.yaml
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
@ -39,6 +42,7 @@
|
||||
namespace: sched-1
|
||||
workloads:
|
||||
- name: 500Nodes
|
||||
labels: [fast]
|
||||
params:
|
||||
initNodes: 500
|
||||
initPods: 100
|
||||
@ -50,6 +54,7 @@
|
||||
measurePods: 1000
|
||||
|
||||
- name: SchedulingSecrets
|
||||
labels: [performance]
|
||||
defaultPodTemplatePath: config/pod-with-secret-volume.yaml
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
@ -61,6 +66,7 @@
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: 500Nodes
|
||||
labels: [fast]
|
||||
params:
|
||||
initNodes: 500
|
||||
initPods: 500
|
||||
@ -72,6 +78,7 @@
|
||||
measurePods: 1000
|
||||
|
||||
- name: SchedulingInTreePVs
|
||||
labels: [performance]
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
countParam: $initNodes
|
||||
@ -86,6 +93,7 @@
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: 500Nodes
|
||||
labels: [fast]
|
||||
params:
|
||||
initNodes: 500
|
||||
initPods: 500
|
||||
@ -97,6 +105,7 @@
|
||||
measurePods: 1000
|
||||
|
||||
- name: SchedulingMigratedInTreePVs
|
||||
labels: [performance]
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
countParam: $initNodes
|
||||
@ -120,6 +129,7 @@
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: 500Nodes
|
||||
labels: [fast]
|
||||
params:
|
||||
initNodes: 500
|
||||
initPods: 500
|
||||
@ -131,6 +141,7 @@
|
||||
measurePods: 1000
|
||||
|
||||
- name: SchedulingCSIPVs
|
||||
labels: [performance]
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
countParam: $initNodes
|
||||
@ -152,6 +163,7 @@
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: 500Nodes
|
||||
labels: [fast]
|
||||
params:
|
||||
initNodes: 500
|
||||
initPods: 500
|
||||
@ -163,6 +175,7 @@
|
||||
measurePods: 1000
|
||||
|
||||
- name: SchedulingPodAffinity
|
||||
labels: [performance]
|
||||
defaultPodTemplatePath: config/pod-with-pod-affinity.yaml
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
@ -183,6 +196,7 @@
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: 500Nodes
|
||||
labels: [fast]
|
||||
params:
|
||||
initNodes: 500
|
||||
initPods: 500
|
||||
@ -194,6 +208,7 @@
|
||||
measurePods: 1000
|
||||
|
||||
- name: SchedulingPreferredPodAffinity
|
||||
labels: [performance]
|
||||
defaultPodTemplatePath: config/pod-with-preferred-pod-affinity.yaml
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
@ -212,6 +227,7 @@
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: 500Nodes
|
||||
labels: [fast]
|
||||
params:
|
||||
initNodes: 500
|
||||
initPods: 500
|
||||
@ -223,6 +239,7 @@
|
||||
measurePods: 1000
|
||||
|
||||
- name: SchedulingPreferredPodAntiAffinity
|
||||
labels: [performance]
|
||||
defaultPodTemplatePath: config/pod-with-preferred-pod-affinity.yaml
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
@ -241,6 +258,7 @@
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: 500Nodes
|
||||
labels: [fast]
|
||||
params:
|
||||
initNodes: 500
|
||||
initPods: 500
|
||||
@ -252,6 +270,7 @@
|
||||
measurePods: 1000
|
||||
|
||||
- name: SchedulingNodeAffinity
|
||||
labels: [performance]
|
||||
defaultPodTemplatePath: config/pod-with-node-affinity.yaml
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
@ -267,6 +286,7 @@
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: 500Nodes
|
||||
labels: [fast]
|
||||
params:
|
||||
initNodes: 500
|
||||
initPods: 500
|
||||
@ -278,6 +298,7 @@
|
||||
measurePods: 1000
|
||||
|
||||
- name: TopologySpreading
|
||||
labels: [performance]
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
countParam: $initNodes
|
||||
@ -294,6 +315,7 @@
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: 500Nodes
|
||||
labels: [fast]
|
||||
params:
|
||||
initNodes: 500
|
||||
initPods: 1000
|
||||
@ -305,6 +327,7 @@
|
||||
measurePods: 2000
|
||||
|
||||
- name: PreferredTopologySpreading
|
||||
labels: [performance]
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
countParam: $initNodes
|
||||
@ -321,6 +344,7 @@
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: 500Nodes
|
||||
labels: [fast]
|
||||
params:
|
||||
initNodes: 500
|
||||
initPods: 1000
|
||||
@ -332,6 +356,7 @@
|
||||
measurePods: 2000
|
||||
|
||||
- name: MixedSchedulingBasePod
|
||||
labels: [performance]
|
||||
defaultPodTemplatePath: config/pod-default.yaml
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
@ -367,6 +392,7 @@
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: 500Nodes
|
||||
labels: [fast]
|
||||
params:
|
||||
initNodes: 500
|
||||
initPods: 200
|
||||
@ -378,6 +404,7 @@
|
||||
measurePods: 1000
|
||||
|
||||
- name: PreemptionBasic
|
||||
labels: [performance]
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
countParam: $initNodes
|
||||
@ -404,6 +431,7 @@
|
||||
# measurePods: 5000
|
||||
|
||||
- name: PreemptionPVs
|
||||
labels: [performance]
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
countParam: $initNodes
|
||||
@ -418,6 +446,7 @@
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: 500Nodes
|
||||
labels: [fast]
|
||||
params:
|
||||
initNodes: 500
|
||||
initPods: 2000
|
||||
@ -432,6 +461,7 @@
|
||||
# measurePods: 5000
|
||||
|
||||
- name: Unschedulable
|
||||
labels: [performance]
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
countParam: $initNodes
|
||||
@ -445,6 +475,7 @@
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: 500Nodes/200InitPods
|
||||
labels: [fast]
|
||||
params:
|
||||
initNodes: 500
|
||||
initPods: 200
|
||||
@ -478,6 +509,7 @@
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: 1000Nodes
|
||||
labels: [fast]
|
||||
params:
|
||||
initNodes: 1000
|
||||
measurePods: 1000
|
||||
@ -638,6 +670,7 @@
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: 500Nodes
|
||||
labels: [fast]
|
||||
params:
|
||||
taintNodes: 100
|
||||
normalNodes: 400
|
||||
|
||||
50
test/integration/scheduler_perf/label_selector.go
Normal file
50
test/integration/scheduler_perf/label_selector.go
Normal file
@ -0,0 +1,50 @@
|
||||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package benchmark
|
||||
|
||||
import "strings"
|
||||
|
||||
// enabled checks a a label filter that works as in GitHub:
|
||||
// - empty string means enabled
|
||||
// - individual labels are comma-separated
|
||||
// - [+]<label> means the workload must have that label
|
||||
// - -<label> means the workload must not have that label
|
||||
func enabled(labelFilter string, labels ...string) bool {
|
||||
for _, label := range strings.Split(labelFilter, ",") {
|
||||
if label == "" {
|
||||
continue
|
||||
}
|
||||
mustHaveLabel := label[0] != '-'
|
||||
if label[0] == '-' || label[0] == '+' {
|
||||
label = label[1:]
|
||||
}
|
||||
haveLabel := containsStr(labels, label)
|
||||
if haveLabel != mustHaveLabel {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func containsStr(hay []string, needle string) bool {
|
||||
for _, item := range hay {
|
||||
if item == needle {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
71
test/integration/scheduler_perf/label_selector_test.go
Normal file
71
test/integration/scheduler_perf/label_selector_test.go
Normal file
@ -0,0 +1,71 @@
|
||||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package benchmark
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLabelFilter(t *testing.T) {
|
||||
empty := ""
|
||||
performance := "performance"
|
||||
fastPerformance := "performance,fast"
|
||||
notFastPerformance := "+performance,-fast"
|
||||
notFast := "-fast"
|
||||
|
||||
testcases := map[string]map[string]bool{
|
||||
empty: {
|
||||
empty: true,
|
||||
performance: true,
|
||||
fastPerformance: true,
|
||||
},
|
||||
performance: {
|
||||
empty: false,
|
||||
performance: true,
|
||||
fastPerformance: true,
|
||||
},
|
||||
fastPerformance: {
|
||||
empty: false,
|
||||
performance: false,
|
||||
fastPerformance: true,
|
||||
},
|
||||
notFast: {
|
||||
empty: true,
|
||||
performance: true,
|
||||
fastPerformance: false,
|
||||
},
|
||||
notFastPerformance: {
|
||||
empty: false,
|
||||
performance: true,
|
||||
fastPerformance: false,
|
||||
},
|
||||
}
|
||||
|
||||
for labelFilter, labelResults := range testcases {
|
||||
t.Run(labelFilter, func(t *testing.T) {
|
||||
for labels, expected := range labelResults {
|
||||
t.Run(labels, func(t *testing.T) {
|
||||
actual := enabled(labelFilter, strings.Split(labels, ",")...)
|
||||
if actual != expected {
|
||||
t.Errorf("expected enabled to be %v, got %v", expected, actual)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -121,6 +121,8 @@ type testCase struct {
|
||||
// This path can be overridden in createPodsOp by setting PodTemplatePath .
|
||||
// Optional
|
||||
DefaultPodTemplatePath *string
|
||||
// Labels can be used to enable or disable workloads inside this test case.
|
||||
Labels []string
|
||||
}
|
||||
|
||||
func (tc *testCase) collectsMetrics() bool {
|
||||
@ -151,6 +153,8 @@ type workload struct {
|
||||
Name string
|
||||
// Values of parameters used in the workloadTemplate.
|
||||
Params params
|
||||
// Labels can be used to enable or disable a workload.
|
||||
Labels []string
|
||||
}
|
||||
|
||||
type params struct {
|
||||
@ -608,6 +612,8 @@ func initTestOutput(tb testing.TB) io.Writer {
|
||||
return output
|
||||
}
|
||||
|
||||
var perfSchedulingLabelFilter = flag.String("perf-scheduling-label-filter", "performance", "comma-separated list of labels which a testcase must have (no prefix or +) or must not have (-)")
|
||||
|
||||
func BenchmarkPerfScheduling(b *testing.B) {
|
||||
testCases, err := getTestCases(configFile)
|
||||
if err != nil {
|
||||
@ -632,6 +638,10 @@ func BenchmarkPerfScheduling(b *testing.B) {
|
||||
b.Run(tc.Name, func(b *testing.B) {
|
||||
for _, w := range tc.Workloads {
|
||||
b.Run(w.Name, func(b *testing.B) {
|
||||
if !enabled(*perfSchedulingLabelFilter, append(tc.Labels, w.Labels...)...) {
|
||||
b.Skipf("disabled by label filter %q", *perfSchedulingLabelFilter)
|
||||
}
|
||||
|
||||
// Ensure that there are no leaked
|
||||
// goroutines. They could influence
|
||||
// performance of the next benchmark.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user