Merge pull request #57215 from misterikkit/topologyBenchmark

Automatic merge from submit-queue (batch tested with PRs 57215, 57489). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add scheduler benchmark tests for PodAntiAffinity rules

**What this PR does / why we need it**:
This adds a benchmark test that measures the scheduling latency of pods with PodAntiAffinity rules where topologyKey=kubernetes.io/hostname.

Additionally, this PR imposes a minimum `b.N` value to make the test results more meaningful.

The benchmark is needed to validate any performance improvements made for #54189.


Command used to invoke:
```sh
make test-integration WHAT=./test/integration/scheduler_perf KUBE_TEST_ARGS="-run=xxxx -bench=."
```

Sample output:
```
pkg: k8s.io/kubernetes/test/integration/scheduler_perf
BenchmarkScheduling/100Nodes/0Pods-40        100          36316838 ns/op
BenchmarkScheduling/100Nodes/1000Pods-40                     100          69460001 ns/op
BenchmarkScheduling/1000Nodes/0Pods-40                       100         193755560 ns/op
BenchmarkScheduling/1000Nodes/1000Pods-40                    100         343451472 ns/op
BenchmarkSchedulingAntiAffinity/500Nodes/250Pods-40                  250         127236420 ns/op
BenchmarkSchedulingAntiAffinity/500Nodes/5000Pods-40                 250         478661925 ns/op
PASS
ok      k8s.io/kubernetes/test/integration/scheduler_perf       354.008s
```

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:


**Release note**:

```release-note
NONE
```

test/integration/scheduler_perf/BUILD

@@ -42,6 +42,7 @@ go_test(
     library = ":go_default_library",
     tags = ["integration"],
     deps = [
+        "//pkg/kubelet/apis:go_default_library",
         "//plugin/pkg/scheduler:go_default_library",
         "//test/integration/framework:go_default_library",
         "//test/utils:go_default_library",

test/integration/scheduler_perf/scheduler_bench_test.go

@@ -21,7 +21,10 @@ import (
 	"testing"
 	"time"
 
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/kubernetes/pkg/kubelet/apis"
 	"k8s.io/kubernetes/test/integration/framework"
 	testutils "k8s.io/kubernetes/test/utils"
 
@@ -31,22 +34,81 @@ import (
 // BenchmarkScheduling benchmarks the scheduling rate when the cluster has
 // various quantities of nodes and scheduled pods.
 func BenchmarkScheduling(b *testing.B) {
-	tests := []struct{ nodes, pods int }{
-		{nodes: 100, pods: 0},
-		{nodes: 100, pods: 1000},
-		{nodes: 1000, pods: 0},
-		{nodes: 1000, pods: 1000},
+	tests := []struct{ nodes, existingPods, minPods int }{
+		{nodes: 100, existingPods: 0, minPods: 100},
+		{nodes: 100, existingPods: 1000, minPods: 100},
+		{nodes: 1000, existingPods: 0, minPods: 100},
+		{nodes: 1000, existingPods: 1000, minPods: 100},
 	}
+	setupStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("rc1")
+	testStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("rc2")
 	for _, test := range tests {
-		name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.pods)
-		b.Run(name, func(b *testing.B) { benchmarkScheduling(test.nodes, test.pods, b) })
+		name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
+		b.Run(name, func(b *testing.B) {
+			benchmarkScheduling(test.nodes, test.existingPods, test.minPods, setupStrategy, testStrategy, b)
+		})
 	}
 }
 
+// BenchmarkSchedulingAntiAffinity benchmarks the scheduling rate of pods with
+// PodAntiAffinity rules when the cluster has various quantities of nodes and
+// scheduled pods.
+func BenchmarkSchedulingAntiAffinity(b *testing.B) {
+	tests := []struct{ nodes, existingPods, minPods int }{
+		{nodes: 500, existingPods: 250, minPods: 250},
+		{nodes: 500, existingPods: 5000, minPods: 250},
+	}
+	// The setup strategy creates pods with no affinity rules.
+	setupStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("setup")
+	// The test strategy creates pods with anti-affinity for each other.
+	testBasePod := makeBasePodWithAntiAffinity(
+		map[string]string{"name": "test", "color": "green"},
+		map[string]string{"color": "green"})
+	testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
+	for _, test := range tests {
+		name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
+		b.Run(name, func(b *testing.B) {
+			benchmarkScheduling(test.nodes, test.existingPods, test.minPods, setupStrategy, testStrategy, b)
+		})
+	}
+
+}
+
+// makeBasePodWithAntiAffinity creates a Pod object to be used as a template.
+// The Pod has a PodAntiAffinity requirement against pods with the given labels.
+func makeBasePodWithAntiAffinity(podLabels, affinityLabels map[string]string) *v1.Pod {
+	basePod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "affinity-pod-",
+			Labels:       podLabels,
+		},
+		Spec: testutils.MakePodSpec(),
+	}
+	basePod.Spec.Affinity = &v1.Affinity{
+		PodAntiAffinity: &v1.PodAntiAffinity{
+			RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
+				{
+					LabelSelector: &metav1.LabelSelector{
+						MatchLabels: affinityLabels,
+					},
+					TopologyKey: apis.LabelHostname,
+				},
+			},
+		},
+	}
+	return basePod
+}
+
 // benchmarkScheduling benchmarks scheduling rate with specific number of nodes
-// and specific number of pods already scheduled. Since an operation takes relatively
-// long time, b.N should be small: 10 - 100.
-func benchmarkScheduling(numNodes, numScheduledPods int, b *testing.B) {
+// and specific number of pods already scheduled.
+// This will schedule numExistingPods pods before the benchmark starts, and at
+// least minPods pods during the benchmark.
+func benchmarkScheduling(numNodes, numExistingPods, minPods int,
+	setupPodStrategy, testPodStrategy testutils.TestPodCreateStrategy,
+	b *testing.B) {
+	if b.N < minPods {
+		b.N = minPods
+	}
 	schedulerConfigFactory, finalFunc := mustSetupScheduler()
 	defer finalFunc()
 	c := schedulerConfigFactory.GetClient()
@@ -62,7 +124,7 @@ func benchmarkScheduling(numNodes, numScheduledPods int, b *testing.B) {
 	defer nodePreparer.CleanupNodes()
 
 	config := testutils.NewTestPodCreatorConfig()
-	config.AddStrategy("sched-test", numScheduledPods, testutils.NewSimpleWithControllerCreatePodStrategy("rc1"))
+	config.AddStrategy("sched-test", numExistingPods, setupPodStrategy)
 	podCreator := testutils.NewTestPodCreator(c, config)
 	podCreator.CreatePods()
 
@@ -71,7 +133,7 @@ func benchmarkScheduling(numNodes, numScheduledPods int, b *testing.B) {
 		if err != nil {
 			glog.Fatalf("%v", err)
 		}
-		if len(scheduled) >= numScheduledPods {
+		if len(scheduled) >= numExistingPods {
 			break
 		}
 		time.Sleep(1 * time.Second)
@@ -79,7 +141,7 @@ func benchmarkScheduling(numNodes, numScheduledPods int, b *testing.B) {
 	// start benchmark
 	b.ResetTimer()
 	config = testutils.NewTestPodCreatorConfig()
-	config.AddStrategy("sched-test", b.N, testutils.NewSimpleWithControllerCreatePodStrategy("rc2"))
+	config.AddStrategy("sched-test", b.N, testPodStrategy)
 	podCreator = testutils.NewTestPodCreator(c, config)
 	podCreator.CreatePods()
 	for {
@@ -89,7 +151,7 @@ func benchmarkScheduling(numNodes, numScheduledPods int, b *testing.B) {
 		if err != nil {
 			glog.Fatalf("%v", err)
 		}
-		if len(scheduled) >= numScheduledPods+b.N {
+		if len(scheduled) >= numExistingPods+b.N {
 			break
 		}
 		// Note: This might introduce slight deviation in accuracy of benchmark results.