From 0effe15543448c652ead342b0d068b9caedd2d0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Skocze=C5=84?= Date: Fri, 14 Mar 2025 13:26:53 +0000 Subject: [PATCH] Move scheduler_perf topology spreading tests to a separate package --- .../misc/performance-config.yaml | 248 ---------------- .../performance-config.yaml | 266 ++++++++++++++++++ .../topology_spreading_test.go | 43 +++ 3 files changed, 309 insertions(+), 248 deletions(-) create mode 100644 test/integration/scheduler_perf/topology_spreading/performance-config.yaml create mode 100644 test/integration/scheduler_perf/topology_spreading/topology_spreading_test.go diff --git a/test/integration/scheduler_perf/misc/performance-config.yaml b/test/integration/scheduler_perf/misc/performance-config.yaml index 026748ba63d..30bba3f2c5a 100644 --- a/test/integration/scheduler_perf/misc/performance-config.yaml +++ b/test/integration/scheduler_perf/misc/performance-config.yaml @@ -135,199 +135,6 @@ initNodes: 15000 measurePods: 30000 -- name: TopologySpreading - workloadTemplate: - - opcode: createNodes - countParam: $initNodes - nodeTemplatePath: ../templates/node-default.yaml - labelNodePrepareStrategy: - labelKey: "topology.kubernetes.io/zone" - labelValues: ["moon-1", "moon-2", "moon-3"] - - opcode: createPods - countParam: $initPods - podTemplatePath: ../templates/pod-default.yaml - - opcode: createPods - countParam: $measurePods - podTemplatePath: ../templates/pod-with-topology-spreading.yaml - collectMetrics: true - workloads: - - name: 5Nodes - featureGates: - SchedulerQueueingHints: false - labels: [integration-test, short] - params: - initNodes: 5 - initPods: 10 - measurePods: 10 - - name: 5Nodes_QueueingHintsEnabled - featureGates: - SchedulerQueueingHints: true - labels: [integration-test, short] - params: - initNodes: 5 - initPods: 10 - measurePods: 10 - - name: 500Nodes - labels: [performance, short] - params: - initNodes: 500 - initPods: 1000 - measurePods: 1000 - - name: 5000Nodes - labels: [performance, short] - params: - initNodes: 5000 - initPods: 5000 - measurePods: 2000 - - name: 5000Nodes_5000Pods - featureGates: - SchedulerQueueingHints: false - labels: [performance] - threshold: 85 - params: - initNodes: 5000 - initPods: 5000 - measurePods: 5000 - - name: 5000Nodes_5000Pods_QueueingHintsEnabled - featureGates: - SchedulerQueueingHints: true - labels: [performance] - threshold: 85 - params: - initNodes: 5000 - initPods: 5000 - measurePods: 5000 - -- name: PreferredTopologySpreading - workloadTemplate: - - opcode: createNodes - countParam: $initNodes - nodeTemplatePath: ../templates/node-default.yaml - labelNodePrepareStrategy: - labelKey: "topology.kubernetes.io/zone" - labelValues: ["moon-1", "moon-2", "moon-3"] - - opcode: createPods - countParam: $initPods - podTemplatePath: ../templates/pod-default.yaml - - opcode: createPods - countParam: $measurePods - podTemplatePath: ../templates/pod-with-preferred-topology-spreading.yaml - collectMetrics: true - workloads: - - name: 5Nodes - featureGates: - SchedulerQueueingHints: false - labels: [integration-test, short] - params: - initNodes: 5 - initPods: 10 - measurePods: 10 - - name: 5Nodes_QueueingHintsEnabled - featureGates: - SchedulerQueueingHints: true - labels: [integration-test, short] - params: - initNodes: 5 - initPods: 10 - measurePods: 10 - - name: 500Nodes - labels: [performance, short] - params: - initNodes: 500 - initPods: 1000 - measurePods: 1000 - - name: 5000Nodes - labels: [performance] - params: - initNodes: 5000 - initPods: 5000 - measurePods: 2000 - - name: 5000Nodes_5000Pods - featureGates: - SchedulerQueueingHints: false - labels: [performance] - threshold: 125 - params: - initNodes: 5000 - initPods: 5000 - measurePods: 5000 - - name: 5000Nodes_5000Pods_QueueingHintsEnabled - featureGates: - SchedulerQueueingHints: true - labels: [performance] - threshold: 125 - params: - initNodes: 5000 - initPods: 5000 - measurePods: 5000 - -# This test case simulates the scheduling of pods with service. -# It benchmarks default PodTopologySpread constraints that calculate the label selector from services. -- name: DefaultTopologySpreading - workloadTemplate: - - opcode: createNodes - countParam: $initNodes - nodeTemplatePath: ../templates/node-default.yaml - labelNodePrepareStrategy: - labelKey: "topology.kubernetes.io/zone" - labelValues: ["moon-1", "moon-2", "moon-3"] - - opcode: createAny - namespace: service-ns - templatePath: ../templates/service.yaml - - opcode: createPods - countParam: $initPods - podTemplatePath: ../templates/pod-default.yaml - - opcode: createPods - namespace: service-ns - countParam: $measurePods - podTemplatePath: ../templates/pod-with-label.yaml - collectMetrics: true - workloads: - - name: 5Nodes - featureGates: - SchedulerQueueingHints: false - labels: [integration-test, short] - params: - initNodes: 5 - initPods: 10 - measurePods: 10 - - name: 5Nodes_QueueingHintsEnabled - featureGates: - SchedulerQueueingHints: true - labels: [integration-test, short] - params: - initNodes: 5 - initPods: 10 - measurePods: 10 - - name: 500Nodes - labels: [performance, short] - params: - initNodes: 500 - initPods: 1000 - measurePods: 1000 - - name: 5000Nodes_10000Pods - labels: [performance] - params: - initNodes: 5000 - initPods: 5000 - measurePods: 10000 - - name: 5000Nodes_50000Pods - featureGates: - SchedulerQueueingHints: false - labels: [performance] - params: - initNodes: 5000 - initPods: 5000 - measurePods: 50000 - - name: 5000Nodes_50000Pods_QueueingHintsEnabled - featureGates: - SchedulerQueueingHints: true - labels: [performance] - params: - initNodes: 5000 - initPods: 5000 - measurePods: 50000 - - name: PreemptionBasic workloadTemplate: - opcode: createNodes @@ -596,61 +403,6 @@ initNodes: 5000 measurePods: 10000 -- name: SchedulingWithNodeInclusionPolicy - featureGates: - NodeInclusionPolicyInPodTopologySpread: true - defaultPodTemplatePath: ../templates/pod-with-node-inclusion-policy.yaml - workloadTemplate: - - opcode: createNodes - countParam: $normalNodes - - opcode: createNodes - nodeTemplatePath: ../templates/node-with-taint.yaml - countParam: $taintNodes - - opcode: createPods - countParam: $measurePods - collectMetrics: true - workloads: - - name: 5Nodes - featureGates: - SchedulerQueueingHints: false - labels: [integration-test, short] - params: - taintNodes: 1 - normalNodes: 4 - measurePods: 4 - - name: 5Nodes_QueueingHintsEnabled - featureGates: - SchedulerQueueingHints: true - labels: [integration-test, short] - params: - taintNodes: 1 - normalNodes: 4 - measurePods: 4 - - name: 500Nodes - labels: [performance, short] - params: - taintNodes: 100 - normalNodes: 400 - measurePods: 400 - - name: 5000Nodes - featureGates: - SchedulerQueueingHints: false - labels: [performance, short] - threshold: 68 - params: - taintNodes: 1000 - normalNodes: 4000 - measurePods: 4000 - - name: 5000Nodes_QueueingHintsEnabled - featureGates: - SchedulerQueueingHints: true - labels: [performance, short] - threshold: 68 - params: - taintNodes: 1000 - normalNodes: 4000 - measurePods: 4000 - # This test case simulates the scheduling when many pods are gated and others are gradually deleted. # https://github.com/kubernetes/kubernetes/issues/124384 - name: SchedulingWhileGated diff --git a/test/integration/scheduler_perf/topology_spreading/performance-config.yaml b/test/integration/scheduler_perf/topology_spreading/performance-config.yaml new file mode 100644 index 00000000000..3bee5b7d028 --- /dev/null +++ b/test/integration/scheduler_perf/topology_spreading/performance-config.yaml @@ -0,0 +1,266 @@ +# The following labels are used in this file. (listed in ascending order of the number of covered test cases) +# +# - integration-test: test cases to run as the integration test, usually to spot some issues in the scheduler implementation or scheduler-perf itself. +# - performance: test cases to run in the performance test. +# - short: supplemental label for the above two labels (must not used alone), which literally means short execution time test cases. +# +# Specifically, the CIs use labels like the following: +# - `ci-kubernetes-integration-master` (`integration-test`): Test cases are chosen based on a tradeoff between code coverage and overall runtime. +# It basically covers all test cases but with their smallest workload. +# - `pull-kubernetes-integration` (`integration-test`,`short`): Test cases are chosen so that they should take less than total 5 min to complete. +# - `ci-benchmark-scheduler-perf` (`performance`): Long enough test cases are chosen (ideally, longer than 10 seconds) +# to provide meaningful samples for the pod scheduling rate. +# +# Also, `performance`+`short` isn't used in the CIs, but it's used to test the performance test locally. +# (Sometimes, the test cases with `integration-test` are too small to spot issues.) +# +# Combining `performance` and `short` selects suitable workloads for a local +# before/after comparisons with benchstat. + +- name: TopologySpreading + workloadTemplate: + - opcode: createNodes + countParam: $initNodes + nodeTemplatePath: ../templates/node-default.yaml + labelNodePrepareStrategy: + labelKey: "topology.kubernetes.io/zone" + labelValues: ["moon-1", "moon-2", "moon-3"] + - opcode: createPods + countParam: $initPods + podTemplatePath: ../templates/pod-default.yaml + - opcode: createPods + countParam: $measurePods + podTemplatePath: ../templates/pod-with-topology-spreading.yaml + collectMetrics: true + workloads: + - name: 5Nodes + featureGates: + SchedulerQueueingHints: false + labels: [integration-test, short] + params: + initNodes: 5 + initPods: 10 + measurePods: 10 + - name: 5Nodes_QueueingHintsEnabled + featureGates: + SchedulerQueueingHints: true + labels: [integration-test, short] + params: + initNodes: 5 + initPods: 10 + measurePods: 10 + - name: 500Nodes + labels: [performance, short] + params: + initNodes: 500 + initPods: 1000 + measurePods: 1000 + - name: 5000Nodes + labels: [performance, short] + params: + initNodes: 5000 + initPods: 5000 + measurePods: 2000 + - name: 5000Nodes_5000Pods + featureGates: + SchedulerQueueingHints: false + labels: [performance] + threshold: 85 + params: + initNodes: 5000 + initPods: 5000 + measurePods: 5000 + - name: 5000Nodes_5000Pods_QueueingHintsEnabled + featureGates: + SchedulerQueueingHints: true + labels: [performance] + threshold: 85 + params: + initNodes: 5000 + initPods: 5000 + measurePods: 5000 + +- name: PreferredTopologySpreading + workloadTemplate: + - opcode: createNodes + countParam: $initNodes + nodeTemplatePath: ../templates/node-default.yaml + labelNodePrepareStrategy: + labelKey: "topology.kubernetes.io/zone" + labelValues: ["moon-1", "moon-2", "moon-3"] + - opcode: createPods + countParam: $initPods + podTemplatePath: ../templates/pod-default.yaml + - opcode: createPods + countParam: $measurePods + podTemplatePath: ../templates/pod-with-preferred-topology-spreading.yaml + collectMetrics: true + workloads: + - name: 5Nodes + featureGates: + SchedulerQueueingHints: false + labels: [integration-test, short] + params: + initNodes: 5 + initPods: 10 + measurePods: 10 + - name: 5Nodes_QueueingHintsEnabled + featureGates: + SchedulerQueueingHints: true + labels: [integration-test, short] + params: + initNodes: 5 + initPods: 10 + measurePods: 10 + - name: 500Nodes + labels: [performance, short] + params: + initNodes: 500 + initPods: 1000 + measurePods: 1000 + - name: 5000Nodes + labels: [performance] + params: + initNodes: 5000 + initPods: 5000 + measurePods: 2000 + - name: 5000Nodes_5000Pods + featureGates: + SchedulerQueueingHints: false + labels: [performance] + threshold: 125 + params: + initNodes: 5000 + initPods: 5000 + measurePods: 5000 + - name: 5000Nodes_5000Pods_QueueingHintsEnabled + featureGates: + SchedulerQueueingHints: true + labels: [performance] + threshold: 125 + params: + initNodes: 5000 + initPods: 5000 + measurePods: 5000 + +# This test case simulates the scheduling of pods with service. +# It benchmarks default PodTopologySpread constraints that calculate the label selector from services. +- name: DefaultTopologySpreading + workloadTemplate: + - opcode: createNodes + countParam: $initNodes + nodeTemplatePath: ../templates/node-default.yaml + labelNodePrepareStrategy: + labelKey: "topology.kubernetes.io/zone" + labelValues: ["moon-1", "moon-2", "moon-3"] + - opcode: createAny + namespace: service-ns + templatePath: ../templates/service.yaml + - opcode: createPods + countParam: $initPods + podTemplatePath: ../templates/pod-default.yaml + - opcode: createPods + namespace: service-ns + countParam: $measurePods + podTemplatePath: ../templates/pod-with-label.yaml + collectMetrics: true + workloads: + - name: 5Nodes + featureGates: + SchedulerQueueingHints: false + labels: [integration-test, short] + params: + initNodes: 5 + initPods: 10 + measurePods: 10 + - name: 5Nodes_QueueingHintsEnabled + featureGates: + SchedulerQueueingHints: true + labels: [integration-test, short] + params: + initNodes: 5 + initPods: 10 + measurePods: 10 + - name: 500Nodes + labels: [performance, short] + params: + initNodes: 500 + initPods: 1000 + measurePods: 1000 + - name: 5000Nodes_10000Pods + labels: [performance] + params: + initNodes: 5000 + initPods: 5000 + measurePods: 10000 + - name: 5000Nodes_50000Pods + featureGates: + SchedulerQueueingHints: false + labels: [performance] + params: + initNodes: 5000 + initPods: 5000 + measurePods: 50000 + - name: 5000Nodes_50000Pods_QueueingHintsEnabled + featureGates: + SchedulerQueueingHints: true + labels: [performance] + params: + initNodes: 5000 + initPods: 5000 + measurePods: 50000 + +- name: SchedulingWithNodeInclusionPolicy + featureGates: + NodeInclusionPolicyInPodTopologySpread: true + defaultPodTemplatePath: ../templates/pod-with-node-inclusion-policy.yaml + workloadTemplate: + - opcode: createNodes + countParam: $normalNodes + - opcode: createNodes + nodeTemplatePath: ../templates/node-with-taint.yaml + countParam: $taintNodes + - opcode: createPods + countParam: $measurePods + collectMetrics: true + workloads: + - name: 5Nodes + featureGates: + SchedulerQueueingHints: false + labels: [integration-test, short] + params: + taintNodes: 1 + normalNodes: 4 + measurePods: 4 + - name: 5Nodes_QueueingHintsEnabled + featureGates: + SchedulerQueueingHints: true + labels: [integration-test, short] + params: + taintNodes: 1 + normalNodes: 4 + measurePods: 4 + - name: 500Nodes + labels: [performance, short] + params: + taintNodes: 100 + normalNodes: 400 + measurePods: 400 + - name: 5000Nodes + featureGates: + SchedulerQueueingHints: false + labels: [performance, short] + threshold: 68 + params: + taintNodes: 1000 + normalNodes: 4000 + measurePods: 4000 + - name: 5000Nodes_QueueingHintsEnabled + featureGates: + SchedulerQueueingHints: true + labels: [performance, short] + threshold: 68 + params: + taintNodes: 1000 + normalNodes: 4000 + measurePods: 4000 diff --git a/test/integration/scheduler_perf/topology_spreading/topology_spreading_test.go b/test/integration/scheduler_perf/topology_spreading/topology_spreading_test.go new file mode 100644 index 00000000000..436b2640083 --- /dev/null +++ b/test/integration/scheduler_perf/topology_spreading/topology_spreading_test.go @@ -0,0 +1,43 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package topologyspreading + +import ( + "fmt" + "os" + "testing" + + _ "k8s.io/component-base/logs/json/register" + perf "k8s.io/kubernetes/test/integration/scheduler_perf" +) + +func TestMain(m *testing.M) { + if err := perf.InitTests(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + + m.Run() +} + +func TestSchedulerPerf(t *testing.T) { + perf.RunIntegrationPerfScheduling(t, "performance-config.yaml") +} + +func BenchmarkPerfScheduling(b *testing.B) { + perf.RunBenchmarkPerfScheduling(b, "performance-config.yaml", "topologyspreading", nil) +}