Add multiple init pods to perf test cases

Add test case with several init pods with affinity or antiaffinity.

Signed-off-by: Aldo Culquicondor <acondor@google.com>

test/integration/scheduler_perf/config/performance-config.yaml

@@ -1,60 +1,60 @@
 - template:
     desc: SchedulingBasic
     initPods:
-      podTemplatePath: config/pod-default.yaml
+    - podTemplatePath: config/pod-default.yaml
     podsToSchedule:
       podTemplatePath: config/pod-default.yaml
   params:
-    - numNodes: 500
-      numInitPods: 500
-      numPodsToSchedule: 1000
-    - numNodes: 5000
-      numInitPods: 5000
-      numPodsToSchedule: 1000
+  - numNodes: 500
+    numInitPods: [500]
+    numPodsToSchedule: 1000
+  - numNodes: 5000
+    numInitPods: [5000]
+    numPodsToSchedule: 1000
 - template:
     desc: SchedulingPodAntiAffinity
     nodes:
       uniqueNodeLabelStrategy:
         labelKey: kubernetes.io/hostname
     initPods:
-      podTemplatePath: config/pod-with-pod-anti-affinity.yaml
+    - podTemplatePath: config/pod-with-pod-anti-affinity.yaml
     podsToSchedule:
       podTemplatePath: config/pod-with-pod-anti-affinity.yaml
   params:
-    - numNodes: 500
-      numInitPods: 100
-      numPodsToSchedule: 400
-    - numNodes: 5000
-      numInitPods: 1000
-      numPodsToSchedule: 1000
+  - numNodes: 500
+    numInitPods: [100]
+    numPodsToSchedule: 400
+  - numNodes: 5000
+    numInitPods: [1000]
+    numPodsToSchedule: 1000
 - template:
     desc: SchedulingSecrets
     initPods:
-      podTemplatePath: config/pod-with-secret-volume.yaml
+    - podTemplatePath: config/pod-with-secret-volume.yaml
     podsToSchedule:
       podTemplatePath: config/pod-with-secret-volume.yaml
   params:
-    - numNodes: 500
-      numInitPods: 500
-      numPodsToSchedule: 1000
-    - numNodes: 5000
-      numInitPods: 5000
-      numPodsToSchedule: 1000
+  - numNodes: 500
+    numInitPods: [500]
+    numPodsToSchedule: 1000
+  - numNodes: 5000
+    numInitPods: [5000]
+    numPodsToSchedule: 1000
 - template:
     desc: SchedulingInTreePVs
     initPods:
-      persistentVolumeTemplatePath: config/pv-aws.yaml
+    - persistentVolumeTemplatePath: config/pv-aws.yaml
       persistentVolumeClaimTemplatePath: config/pvc.yaml
     podsToSchedule:
       persistentVolumeTemplatePath: config/pv-aws.yaml
       persistentVolumeClaimTemplatePath: config/pvc.yaml
   params:
-    - numNodes: 500
-      numInitPods: 500
-      numPodsToSchedule: 1000
-    - numNodes: 5000
-      numInitPods: 5000
-      numPodsToSchedule: 1000
+  - numNodes: 500
+    numInitPods: [500]
+    numPodsToSchedule: 1000
+  - numNodes: 5000
+    numInitPods: [5000]
+    numPodsToSchedule: 1000
 - template:
     desc: SchedulingMigratedInTreePVs
     nodes:
@@ -66,9 +66,9 @@
           ebs.csi.aws.com:
             count: 39
         migratedPlugins:
-          - "kubernetes.io/aws-ebs"
+        - "kubernetes.io/aws-ebs"
     initPods:
-      persistentVolumeTemplatePath: config/pv-aws.yaml
+    - persistentVolumeTemplatePath: config/pv-aws.yaml
       persistentVolumeClaimTemplatePath: config/pvc.yaml
     podsToSchedule:
       persistentVolumeTemplatePath: config/pv-aws.yaml
@@ -77,12 +77,12 @@
       CSIMigration: true
       CSIMigrationAWS: true
   params:
-    - numNodes: 500
-      numInitPods: 500
-      numPodsToSchedule: 1000
-    - numNodes: 5000
-      numInitPods: 5000
-      numPodsToSchedule: 1000
+  - numNodes: 500
+    numInitPods: [500]
+    numPodsToSchedule: 1000
+  - numNodes: 5000
+    numInitPods: [5000]
+    numPodsToSchedule: 1000
 - template:
     desc: SchedulingCSIPVs
     nodes:
@@ -94,18 +94,18 @@
           ebs.csi.aws.com:
             count: 39
     initPods:
-      persistentVolumeTemplatePath: config/pv-csi.yaml
+    - persistentVolumeTemplatePath: config/pv-csi.yaml
       persistentVolumeClaimTemplatePath: config/pvc.yaml
     podsToSchedule:
       persistentVolumeTemplatePath: config/pv-csi.yaml
       persistentVolumeClaimTemplatePath: config/pvc.yaml
   params:
-    - numNodes: 500
-      numInitPods: 500
-      numPodsToSchedule: 1000
-    - numNodes: 5000
-      numInitPods: 5000
-      numPodsToSchedule: 1000
+  - numNodes: 500
+    numInitPods: [500]
+    numPodsToSchedule: 1000
+  - numNodes: 5000
+    numInitPods: [5000]
+    numPodsToSchedule: 1000
 - template:
     desc: SchedulingPodAffinity
     nodes:
@@ -114,48 +114,48 @@
         labelKey: "failure-domain.beta.kubernetes.io/zone"
         labelValue: "zone1"
     initPods:
-      podTemplatePath: config/pod-with-pod-affinity.yaml
+    - podTemplatePath: config/pod-with-pod-affinity.yaml
     podsToSchedule:
       podTemplatePath: config/pod-with-pod-affinity.yaml
   params:
-    - numNodes: 500
-      numInitPods: 500
-      numPodsToSchedule: 1000
-    - numNodes: 5000
-      numInitPods: 5000
-      numPodsToSchedule: 1000
+  - numNodes: 500
+    numInitPods: [500]
+    numPodsToSchedule: 1000
+  - numNodes: 5000
+    numInitPods: [5000]
+    numPodsToSchedule: 1000
 - template:
     desc: SchedulingPreferredPodAffinity
     nodes:
       uniqueNodeLabelStrategy:
         labelKey: kubernetes.io/hostname
     initPods:
-      podTemplatePath: config/pod-with-preferred-pod-affinity.yaml
+    - podTemplatePath: config/pod-with-preferred-pod-affinity.yaml
     podsToSchedule:
       podTemplatePath: config/pod-with-preferred-pod-affinity.yaml
   params:
-    - numNodes: 500
-      numInitPods: 500
-      numPodsToSchedule: 1000
-    - numNodes: 5000
-      numInitPods: 5000
-      numPodsToSchedule: 1000
+  - numNodes: 500
+    numInitPods: [500]
+    numPodsToSchedule: 1000
+  - numNodes: 5000
+    numInitPods: [5000]
+    numPodsToSchedule: 1000
 - template:
     desc: SchedulingPreferredPodAntiAffinity
     nodes:
       uniqueNodeLabelStrategy:
         labelKey: kubernetes.io/hostname
     initPods:
-      podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml
+    - podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml
     podsToSchedule:
       podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml
   params:
-    - numNodes: 500
-      numInitPods: 500
-      numPodsToSchedule: 1000
-    - numNodes: 5000
-      numInitPods: 5000
-      numPodsToSchedule: 1000
+  - numNodes: 500
+    numInitPods: [500]
+    numPodsToSchedule: 1000
+  - numNodes: 5000
+    numInitPods: [5000]
+    numPodsToSchedule: 1000
 - template:
     desc: SchedulingNodeAffinity
     nodes:
@@ -164,13 +164,30 @@
         labelKey: "failure-domain.beta.kubernetes.io/zone"
         labelValue: "zone1"
     initPods:
-      podTemplatePath: config/pod-with-node-affinity.yaml
+    - podTemplatePath: config/pod-with-node-affinity.yaml
     podsToSchedule:
       podTemplatePath: config/pod-with-node-affinity.yaml
   params:
-    - numNodes: 500
-      numInitPods: 500
-      numPodsToSchedule: 1000
-    - numNodes: 5000
-      numInitPods: 5000
-      numPodsToSchedule: 1000
+  - numNodes: 500
+    numInitPods: [500]
+    numPodsToSchedule: 1000
+  - numNodes: 5000
+    numInitPods: [5000]
+    numPodsToSchedule: 1000
+- template:
+    desc: MixedSchedulingBasePod
+    initPods:
+    - podTemplatePath: config/pod-default.yaml
+    - podTemplatePath: config/pod-with-pod-affinity.yaml
+    - podTemplatePath: config/pod-with-pod-anti-affinity.yaml
+    - podTemplatePath: config/pod-with-preferred-pod-affinity.yaml
+    - podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml
+    podsToSchedule:
+      podTemplatePath: config/pod-default.yaml
+  params:
+  - numNodes: 500
+    numInitPods: [200, 200, 200, 200, 200]
+    numPodsToSchedule: 1000
+  - numNodes: 5000
+    numInitPods: [2000, 2000, 2000, 2000, 2000]
+    numPodsToSchedule: 1000

test/integration/scheduler_perf/config/pod-with-pod-affinity.yaml

@@ -3,14 +3,14 @@ kind: Pod
 metadata:
   generateName: affinity-pod-
   labels:
-    foo: ""
+    color: blue
 spec:
   affinity:
     podAffinity:
       requiredDuringSchedulingIgnoredDuringExecution:
       - labelSelector:
           matchLabels:
-            foo: ""
+            color: blue
         topologyKey: failure-domain.beta.kubernetes.io/zone
         namespaces: ["sched-test", "sched-setup"]
   containers:

test/integration/scheduler_perf/config/pod-with-pod-anti-affinity.yaml

@@ -4,7 +4,6 @@ metadata:
   generateName: anti-affinity-pod-
   labels:
     color: green
-    name: test
 spec:
   affinity:
     podAntiAffinity:

test/integration/scheduler_perf/config/pod-with-preferred-pod-affinity.yaml

@@ -3,7 +3,7 @@ kind: Pod
 metadata:
   generateName: preferred-affinity-pod-
   labels:
-    foo: ""
+    color: red
 spec:
   affinity:
     podAffinity:
@@ -11,7 +11,7 @@ spec:
         - podAffinityTerm:
             labelSelector:
               matchLabels:
-                foo: ""
+                color: red
             topologyKey: kubernetes.io/hostname
             namespaces: ["sched-test", "sched-setup"]
           weight: 1

test/integration/scheduler_perf/config/pod-with-preferred-pod-anti-affinity.yaml

@@ -3,7 +3,7 @@ kind: Pod
 metadata:
   generateName: preferred-anti-affinity-pod-
   labels:
-    foo: ""
+    color: yellow
 spec:
   affinity:
     podAntiAffinity:
@@ -11,7 +11,7 @@ spec:
         - podAffinityTerm:
             labelSelector:
               matchLabels:
-                foo: ""
+                color: yellow
             topologyKey: kubernetes.io/hostname
             namespaces: ["sched-test", "sched-setup"]
           weight: 1

test/integration/scheduler_perf/scheduler_perf_test.go

@@ -65,7 +65,7 @@ type testCase struct {
 	// configures nodes in the cluster
 	Nodes nodeCase
 	// configures pods in the cluster before running the tests
-	InitPods podCase
+	InitPods []podCase
 	// pods to be scheduled during the test.
 	PodsToSchedule podCase
 	// optional, feature gates to set before running the test
@@ -100,7 +100,7 @@ type simpleTestCases struct {
 
 type testParams struct {
 	NumNodes          int
-	NumInitPods       int
+	NumInitPods       []int
 	NumPodsToSchedule int
 }
 
@@ -117,7 +117,11 @@ func BenchmarkPerfScheduling(b *testing.B) {
 	}
 
 	for _, test := range tests {
-		name := fmt.Sprintf("%v/%vNodes/%vInitPods/%vPodsToSchedule", test.Desc, test.Nodes.Num, test.InitPods.Num, test.PodsToSchedule.Num)
+		initPods := 0
+		for _, p := range test.InitPods {
+			initPods += p.Num
+		}
+		name := fmt.Sprintf("%v/%vNodes/%vInitPods/%vPodsToSchedule", test.Desc, test.Nodes.Num, initPods, test.PodsToSchedule.Num)
 		b.Run(name, func(b *testing.B) {
 			for feature, flag := range test.FeatureGates {
 				defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, feature, flag)()
@@ -143,10 +147,14 @@ func perfScheduling(test testCase, b *testing.B) []DataItem {
 	}
 	defer nodePreparer.CleanupNodes()
 
-	if err := createPods(setupNamespace, test.InitPods, clientset); err != nil {
-		b.Fatal(err)
+	total := 0
+	for _, p := range test.InitPods {
+		if err := createPods(setupNamespace, p, clientset); err != nil {
+			b.Fatal(err)
+		}
+		total += p.Num
 	}
-	if err := waitNumPodsScheduled(b, test.InitPods.Num, podInformer); err != nil {
+	if err := waitNumPodsScheduled(b, total, podInformer); err != nil {
 		b.Fatal(err)
 	}
 
@@ -164,7 +172,7 @@ func perfScheduling(test testCase, b *testing.B) []DataItem {
 	if err := createPods(testNamespace, test.PodsToSchedule, clientset); err != nil {
 		b.Fatal(err)
 	}
-	if err := waitNumPodsScheduled(b, test.InitPods.Num+test.PodsToSchedule.Num, podInformer); err != nil {
+	if err := waitNumPodsScheduled(b, total+test.PodsToSchedule.Num, podInformer); err != nil {
 		b.Fatal(err)
 	}
 
@@ -278,7 +286,10 @@ func parseTestCases(path string) ([]testCase, error) {
 		testCase := s.Template
 		for _, p := range s.Params {
 			testCase.Nodes.Num = p.NumNodes
-			testCase.InitPods.Num = p.NumInitPods
+			testCase.InitPods = append([]podCase(nil), testCase.InitPods...)
+			for i, v := range p.NumInitPods {
+				testCase.InitPods[i].Num = v
+			}
 			testCase.PodsToSchedule.Num = p.NumPodsToSchedule
 			testCases = append(testCases, testCase)
 		}