diff --git a/test/integration/scheduler_perf/scheduler_bench_test.go b/test/integration/scheduler_perf/scheduler_bench_test.go index 80cc98cfd0e..ef781443ee2 100644 --- a/test/integration/scheduler_perf/scheduler_bench_test.go +++ b/test/integration/scheduler_perf/scheduler_bench_test.go @@ -45,20 +45,23 @@ var ( // From PV controller annBindCompleted = "pv.kubernetes.io/bind-completed" - tests = []struct{ nodes, existingPods, minPods int }{ + defaultTests = []struct{ nodes, existingPods, minPods int }{ {nodes: 500, existingPods: 500, minPods: 1000}, {nodes: 5000, existingPods: 5000, minPods: 1000}, } + testNamespace = "sched-test" + setupNamespace = "sched-setup" ) // BenchmarkScheduling benchmarks the scheduling rate when the cluster has // various quantities of nodes and scheduled pods. func BenchmarkScheduling(b *testing.B) { testStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("rc1") - for _, test := range tests { + for _, test := range defaultTests { name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) b.Run(name, func(b *testing.B) { - benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b) + nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: defaultNodeStrategy}} + benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) }) } } @@ -67,15 +70,26 @@ func BenchmarkScheduling(b *testing.B) { // PodAntiAffinity rules when the cluster has various quantities of nodes and // scheduled pods. func BenchmarkSchedulingPodAntiAffinity(b *testing.B) { + // Since the pods has anti affinity to each other, the number of pods to schedule + // can't exceed the number of nodes (the topology used in the test) + tests := []struct{ nodes, existingPods, minPods int }{ + {nodes: 500, existingPods: 100, minPods: 400}, + {nodes: 5000, existingPods: 1000, minPods: 1000}, + } testBasePod := makeBasePodWithPodAntiAffinity( map[string]string{"name": "test", "color": "green"}, map[string]string{"color": "green"}) - // The test strategy creates pods with anti-affinity for each other. + // The test strategy creates pods with anti-affinity to each other, each pod ending up in a separate node. testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) for _, test := range tests { name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) b.Run(name, func(b *testing.B) { - benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b) + var nodeStrategies []testutils.CountToStrategy + for i := 0; i < test.nodes; i++ { + nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelHostname, fmt.Sprintf("node-%d", i)) + nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{Count: 1, Strategy: nodeStrategy}) + } + benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) }) } } @@ -88,10 +102,11 @@ func BenchmarkSchedulingSecrets(b *testing.B) { // The test strategy creates pods with a secret. testBasePod := makeBasePodWithSecret() testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) - for _, test := range tests { + for _, test := range defaultTests { name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) b.Run(name, func(b *testing.B) { - benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b) + nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: defaultNodeStrategy}} + benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) }) } } @@ -104,10 +119,11 @@ func BenchmarkSchedulingInTreePVs(b *testing.B) { baseClaim := makeBasePersistentVolumeClaim() basePod := makeBasePod() testStrategy := testutils.NewCreatePodWithPersistentVolumeStrategy(baseClaim, awsVolumeFactory, basePod) - for _, test := range tests { + for _, test := range defaultTests { name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) b.Run(name, func(b *testing.B) { - benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b) + nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: defaultNodeStrategy}} + benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) }) } } @@ -134,12 +150,13 @@ func BenchmarkSchedulingMigratedInTreePVs(b *testing.B) { }, } nodeStrategy := testutils.NewNodeAllocatableStrategy(allocatable, csiAllocatable, []string{csilibplugins.AWSEBSInTreePluginName}) - for _, test := range tests { + for _, test := range defaultTests { name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) b.Run(name, func(b *testing.B) { defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.CSIMigration, true)() defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.CSIMigrationAWS, true)() - benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b) + nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}} + benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) }) } } @@ -164,10 +181,11 @@ func BenchmarkSchedulingCSIPVs(b *testing.B) { }, } nodeStrategy := testutils.NewNodeAllocatableStrategy(allocatable, csiAllocatable, []string{}) - for _, test := range tests { + for _, test := range defaultTests { name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) b.Run(name, func(b *testing.B) { - benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b) + nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}} + benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) }) } } @@ -183,10 +201,11 @@ func BenchmarkSchedulingPodAffinity(b *testing.B) { // The test strategy creates pods with affinity for each other. testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelZoneFailureDomain, "zone1") - for _, test := range tests { + for _, test := range defaultTests { name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) b.Run(name, func(b *testing.B) { - benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b) + nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}} + benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) }) } } @@ -201,11 +220,15 @@ func BenchmarkSchedulingPreferredPodAffinity(b *testing.B) { ) // The test strategy creates pods with affinity for each other. testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) - nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelZoneFailureDomain, "zone1") - for _, test := range tests { + for _, test := range defaultTests { name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) b.Run(name, func(b *testing.B) { - benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b) + var nodeStrategies []testutils.CountToStrategy + for i := 0; i < test.nodes; i++ { + nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelHostname, fmt.Sprintf("node-%d", i)) + nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{Count: 1, Strategy: nodeStrategy}) + } + benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) }) } } @@ -218,13 +241,17 @@ func BenchmarkSchedulingPreferredPodAntiAffinity(b *testing.B) { map[string]string{"foo": ""}, map[string]string{"foo": ""}, ) - // The test strategy creates pods with affinity for each other. + // The test strategy creates pods with anti affinity to each other. testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) - nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelZoneFailureDomain, "zone1") - for _, test := range tests { + for _, test := range defaultTests { name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) b.Run(name, func(b *testing.B) { - benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b) + var nodeStrategies []testutils.CountToStrategy + for i := 0; i < test.nodes; i++ { + nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelHostname, fmt.Sprintf("node-%d", i)) + nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{Count: 1, Strategy: nodeStrategy}) + } + benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) }) } } @@ -237,10 +264,11 @@ func BenchmarkSchedulingNodeAffinity(b *testing.B) { // The test strategy creates pods with node-affinity for each other. testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelZoneFailureDomain, "zone1") - for _, test := range tests { + for _, test := range defaultTests { name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) b.Run(name, func(b *testing.B) { - benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b) + nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}} + benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b) }) } } @@ -263,6 +291,7 @@ func makeBasePodWithPodAntiAffinity(podLabels, affinityLabels map[string]string) MatchLabels: affinityLabels, }, TopologyKey: v1.LabelHostname, + Namespaces: []string{testNamespace, setupNamespace}, }, }, }, @@ -289,6 +318,7 @@ func makeBasePodWithPreferredPodAntiAffinity(podLabels, affinityLabels map[strin MatchLabels: affinityLabels, }, TopologyKey: v1.LabelHostname, + Namespaces: []string{testNamespace, setupNamespace}, }, Weight: 1, }, @@ -317,6 +347,7 @@ func makeBasePodWithPreferredPodAffinity(podLabels, affinityLabels map[string]st MatchLabels: affinityLabels, }, TopologyKey: v1.LabelHostname, + Namespaces: []string{testNamespace, setupNamespace}, }, Weight: 1, }, @@ -344,6 +375,7 @@ func makeBasePodWithPodAffinity(podLabels, affinityZoneLabels map[string]string) MatchLabels: affinityZoneLabels, }, TopologyKey: v1.LabelZoneFailureDomain, + Namespaces: []string{testNamespace, setupNamespace}, }, }, }, @@ -384,8 +416,8 @@ func makeBasePodWithNodeAffinity(key string, vals []string) *v1.Pod { // and specific number of pods already scheduled. // This will schedule numExistingPods pods before the benchmark starts, and at // least minPods pods during the benchmark. -func benchmarkScheduling(numNodes, numExistingPods, minPods int, - nodeStrategy testutils.PrepareNodeStrategy, +func benchmarkScheduling(numExistingPods, minPods int, + nodeStrategies []testutils.CountToStrategy, testPodStrategy testutils.TestPodCreateStrategy, b *testing.B) { if b.N < minPods { @@ -396,16 +428,15 @@ func benchmarkScheduling(numNodes, numExistingPods, minPods int, nodePreparer := framework.NewIntegrationTestNodePreparer( clientset, - []testutils.CountToStrategy{{Count: numNodes, Strategy: nodeStrategy}}, - "scheduler-perf-", - ) + nodeStrategies, + "scheduler-perf-") if err := nodePreparer.PrepareNodes(); err != nil { klog.Fatalf("%v", err) } defer nodePreparer.CleanupNodes() config := testutils.NewTestPodCreatorConfig() - config.AddStrategy("sched-setup", numExistingPods, testPodStrategy) + config.AddStrategy(setupNamespace, numExistingPods, testPodStrategy) podCreator := testutils.NewTestPodCreator(clientset, config) podCreator.CreatePods() @@ -439,7 +470,7 @@ func benchmarkScheduling(numNodes, numExistingPods, minPods int, // start benchmark b.ResetTimer() config = testutils.NewTestPodCreatorConfig() - config.AddStrategy("sched-test", b.N, testPodStrategy) + config.AddStrategy(testNamespace, b.N, testPodStrategy) podCreator = testutils.NewTestPodCreator(clientset, config) podCreator.CreatePods()