From 5adc4c41e3c079557f0fcb9746326101fecfbbee Mon Sep 17 00:00:00 2001 From: Aldo Culquicondor Date: Mon, 23 Mar 2020 14:55:12 -0400 Subject: [PATCH] Add multiple init pods to perf test cases Add test case with several init pods with affinity or antiaffinity. Signed-off-by: Aldo Culquicondor --- .../config/performance-config.yaml | 159 ++++++++++-------- .../config/pod-with-pod-affinity.yaml | 4 +- .../config/pod-with-pod-anti-affinity.yaml | 1 - .../pod-with-preferred-pod-affinity.yaml | 4 +- .../pod-with-preferred-pod-anti-affinity.yaml | 4 +- .../scheduler_perf/scheduler_perf_test.go | 27 ++- 6 files changed, 113 insertions(+), 86 deletions(-) diff --git a/test/integration/scheduler_perf/config/performance-config.yaml b/test/integration/scheduler_perf/config/performance-config.yaml index c625d39e5fa..4ea03b9959a 100644 --- a/test/integration/scheduler_perf/config/performance-config.yaml +++ b/test/integration/scheduler_perf/config/performance-config.yaml @@ -1,60 +1,60 @@ - template: desc: SchedulingBasic initPods: - podTemplatePath: config/pod-default.yaml + - podTemplatePath: config/pod-default.yaml podsToSchedule: podTemplatePath: config/pod-default.yaml params: - - numNodes: 500 - numInitPods: 500 - numPodsToSchedule: 1000 - - numNodes: 5000 - numInitPods: 5000 - numPodsToSchedule: 1000 + - numNodes: 500 + numInitPods: [500] + numPodsToSchedule: 1000 + - numNodes: 5000 + numInitPods: [5000] + numPodsToSchedule: 1000 - template: desc: SchedulingPodAntiAffinity nodes: uniqueNodeLabelStrategy: labelKey: kubernetes.io/hostname initPods: - podTemplatePath: config/pod-with-pod-anti-affinity.yaml + - podTemplatePath: config/pod-with-pod-anti-affinity.yaml podsToSchedule: podTemplatePath: config/pod-with-pod-anti-affinity.yaml params: - - numNodes: 500 - numInitPods: 100 - numPodsToSchedule: 400 - - numNodes: 5000 - numInitPods: 1000 - numPodsToSchedule: 1000 + - numNodes: 500 + numInitPods: [100] + numPodsToSchedule: 400 + - numNodes: 5000 + numInitPods: [1000] + numPodsToSchedule: 1000 - template: desc: SchedulingSecrets initPods: - podTemplatePath: config/pod-with-secret-volume.yaml + - podTemplatePath: config/pod-with-secret-volume.yaml podsToSchedule: podTemplatePath: config/pod-with-secret-volume.yaml params: - - numNodes: 500 - numInitPods: 500 - numPodsToSchedule: 1000 - - numNodes: 5000 - numInitPods: 5000 - numPodsToSchedule: 1000 + - numNodes: 500 + numInitPods: [500] + numPodsToSchedule: 1000 + - numNodes: 5000 + numInitPods: [5000] + numPodsToSchedule: 1000 - template: desc: SchedulingInTreePVs initPods: - persistentVolumeTemplatePath: config/pv-aws.yaml + - persistentVolumeTemplatePath: config/pv-aws.yaml persistentVolumeClaimTemplatePath: config/pvc.yaml podsToSchedule: persistentVolumeTemplatePath: config/pv-aws.yaml persistentVolumeClaimTemplatePath: config/pvc.yaml params: - - numNodes: 500 - numInitPods: 500 - numPodsToSchedule: 1000 - - numNodes: 5000 - numInitPods: 5000 - numPodsToSchedule: 1000 + - numNodes: 500 + numInitPods: [500] + numPodsToSchedule: 1000 + - numNodes: 5000 + numInitPods: [5000] + numPodsToSchedule: 1000 - template: desc: SchedulingMigratedInTreePVs nodes: @@ -66,9 +66,9 @@ ebs.csi.aws.com: count: 39 migratedPlugins: - - "kubernetes.io/aws-ebs" + - "kubernetes.io/aws-ebs" initPods: - persistentVolumeTemplatePath: config/pv-aws.yaml + - persistentVolumeTemplatePath: config/pv-aws.yaml persistentVolumeClaimTemplatePath: config/pvc.yaml podsToSchedule: persistentVolumeTemplatePath: config/pv-aws.yaml @@ -77,12 +77,12 @@ CSIMigration: true CSIMigrationAWS: true params: - - numNodes: 500 - numInitPods: 500 - numPodsToSchedule: 1000 - - numNodes: 5000 - numInitPods: 5000 - numPodsToSchedule: 1000 + - numNodes: 500 + numInitPods: [500] + numPodsToSchedule: 1000 + - numNodes: 5000 + numInitPods: [5000] + numPodsToSchedule: 1000 - template: desc: SchedulingCSIPVs nodes: @@ -94,18 +94,18 @@ ebs.csi.aws.com: count: 39 initPods: - persistentVolumeTemplatePath: config/pv-csi.yaml + - persistentVolumeTemplatePath: config/pv-csi.yaml persistentVolumeClaimTemplatePath: config/pvc.yaml podsToSchedule: persistentVolumeTemplatePath: config/pv-csi.yaml persistentVolumeClaimTemplatePath: config/pvc.yaml params: - - numNodes: 500 - numInitPods: 500 - numPodsToSchedule: 1000 - - numNodes: 5000 - numInitPods: 5000 - numPodsToSchedule: 1000 + - numNodes: 500 + numInitPods: [500] + numPodsToSchedule: 1000 + - numNodes: 5000 + numInitPods: [5000] + numPodsToSchedule: 1000 - template: desc: SchedulingPodAffinity nodes: @@ -114,48 +114,48 @@ labelKey: "failure-domain.beta.kubernetes.io/zone" labelValue: "zone1" initPods: - podTemplatePath: config/pod-with-pod-affinity.yaml + - podTemplatePath: config/pod-with-pod-affinity.yaml podsToSchedule: podTemplatePath: config/pod-with-pod-affinity.yaml params: - - numNodes: 500 - numInitPods: 500 - numPodsToSchedule: 1000 - - numNodes: 5000 - numInitPods: 5000 - numPodsToSchedule: 1000 + - numNodes: 500 + numInitPods: [500] + numPodsToSchedule: 1000 + - numNodes: 5000 + numInitPods: [5000] + numPodsToSchedule: 1000 - template: desc: SchedulingPreferredPodAffinity nodes: uniqueNodeLabelStrategy: labelKey: kubernetes.io/hostname initPods: - podTemplatePath: config/pod-with-preferred-pod-affinity.yaml + - podTemplatePath: config/pod-with-preferred-pod-affinity.yaml podsToSchedule: podTemplatePath: config/pod-with-preferred-pod-affinity.yaml params: - - numNodes: 500 - numInitPods: 500 - numPodsToSchedule: 1000 - - numNodes: 5000 - numInitPods: 5000 - numPodsToSchedule: 1000 + - numNodes: 500 + numInitPods: [500] + numPodsToSchedule: 1000 + - numNodes: 5000 + numInitPods: [5000] + numPodsToSchedule: 1000 - template: desc: SchedulingPreferredPodAntiAffinity nodes: uniqueNodeLabelStrategy: labelKey: kubernetes.io/hostname initPods: - podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml + - podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml podsToSchedule: podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml params: - - numNodes: 500 - numInitPods: 500 - numPodsToSchedule: 1000 - - numNodes: 5000 - numInitPods: 5000 - numPodsToSchedule: 1000 + - numNodes: 500 + numInitPods: [500] + numPodsToSchedule: 1000 + - numNodes: 5000 + numInitPods: [5000] + numPodsToSchedule: 1000 - template: desc: SchedulingNodeAffinity nodes: @@ -164,13 +164,30 @@ labelKey: "failure-domain.beta.kubernetes.io/zone" labelValue: "zone1" initPods: - podTemplatePath: config/pod-with-node-affinity.yaml + - podTemplatePath: config/pod-with-node-affinity.yaml podsToSchedule: podTemplatePath: config/pod-with-node-affinity.yaml params: - - numNodes: 500 - numInitPods: 500 - numPodsToSchedule: 1000 - - numNodes: 5000 - numInitPods: 5000 - numPodsToSchedule: 1000 + - numNodes: 500 + numInitPods: [500] + numPodsToSchedule: 1000 + - numNodes: 5000 + numInitPods: [5000] + numPodsToSchedule: 1000 +- template: + desc: MixedSchedulingBasePod + initPods: + - podTemplatePath: config/pod-default.yaml + - podTemplatePath: config/pod-with-pod-affinity.yaml + - podTemplatePath: config/pod-with-pod-anti-affinity.yaml + - podTemplatePath: config/pod-with-preferred-pod-affinity.yaml + - podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml + podsToSchedule: + podTemplatePath: config/pod-default.yaml + params: + - numNodes: 500 + numInitPods: [200, 200, 200, 200, 200] + numPodsToSchedule: 1000 + - numNodes: 5000 + numInitPods: [2000, 2000, 2000, 2000, 2000] + numPodsToSchedule: 1000 diff --git a/test/integration/scheduler_perf/config/pod-with-pod-affinity.yaml b/test/integration/scheduler_perf/config/pod-with-pod-affinity.yaml index 393d8f49954..2b60af6822b 100644 --- a/test/integration/scheduler_perf/config/pod-with-pod-affinity.yaml +++ b/test/integration/scheduler_perf/config/pod-with-pod-affinity.yaml @@ -3,14 +3,14 @@ kind: Pod metadata: generateName: affinity-pod- labels: - foo: "" + color: blue spec: affinity: podAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: - foo: "" + color: blue topologyKey: failure-domain.beta.kubernetes.io/zone namespaces: ["sched-test", "sched-setup"] containers: diff --git a/test/integration/scheduler_perf/config/pod-with-pod-anti-affinity.yaml b/test/integration/scheduler_perf/config/pod-with-pod-anti-affinity.yaml index 8ce2007284d..a6291875ce2 100644 --- a/test/integration/scheduler_perf/config/pod-with-pod-anti-affinity.yaml +++ b/test/integration/scheduler_perf/config/pod-with-pod-anti-affinity.yaml @@ -4,7 +4,6 @@ metadata: generateName: anti-affinity-pod- labels: color: green - name: test spec: affinity: podAntiAffinity: diff --git a/test/integration/scheduler_perf/config/pod-with-preferred-pod-affinity.yaml b/test/integration/scheduler_perf/config/pod-with-preferred-pod-affinity.yaml index d6e927845f3..13e80d32ac2 100644 --- a/test/integration/scheduler_perf/config/pod-with-preferred-pod-affinity.yaml +++ b/test/integration/scheduler_perf/config/pod-with-preferred-pod-affinity.yaml @@ -3,7 +3,7 @@ kind: Pod metadata: generateName: preferred-affinity-pod- labels: - foo: "" + color: red spec: affinity: podAffinity: @@ -11,7 +11,7 @@ spec: - podAffinityTerm: labelSelector: matchLabels: - foo: "" + color: red topologyKey: kubernetes.io/hostname namespaces: ["sched-test", "sched-setup"] weight: 1 diff --git a/test/integration/scheduler_perf/config/pod-with-preferred-pod-anti-affinity.yaml b/test/integration/scheduler_perf/config/pod-with-preferred-pod-anti-affinity.yaml index 8ee0b694d9b..3df57a06f5e 100644 --- a/test/integration/scheduler_perf/config/pod-with-preferred-pod-anti-affinity.yaml +++ b/test/integration/scheduler_perf/config/pod-with-preferred-pod-anti-affinity.yaml @@ -3,7 +3,7 @@ kind: Pod metadata: generateName: preferred-anti-affinity-pod- labels: - foo: "" + color: yellow spec: affinity: podAntiAffinity: @@ -11,7 +11,7 @@ spec: - podAffinityTerm: labelSelector: matchLabels: - foo: "" + color: yellow topologyKey: kubernetes.io/hostname namespaces: ["sched-test", "sched-setup"] weight: 1 diff --git a/test/integration/scheduler_perf/scheduler_perf_test.go b/test/integration/scheduler_perf/scheduler_perf_test.go index 5bb5fe6a0ad..7383d5fe964 100644 --- a/test/integration/scheduler_perf/scheduler_perf_test.go +++ b/test/integration/scheduler_perf/scheduler_perf_test.go @@ -65,7 +65,7 @@ type testCase struct { // configures nodes in the cluster Nodes nodeCase // configures pods in the cluster before running the tests - InitPods podCase + InitPods []podCase // pods to be scheduled during the test. PodsToSchedule podCase // optional, feature gates to set before running the test @@ -100,7 +100,7 @@ type simpleTestCases struct { type testParams struct { NumNodes int - NumInitPods int + NumInitPods []int NumPodsToSchedule int } @@ -117,7 +117,11 @@ func BenchmarkPerfScheduling(b *testing.B) { } for _, test := range tests { - name := fmt.Sprintf("%v/%vNodes/%vInitPods/%vPodsToSchedule", test.Desc, test.Nodes.Num, test.InitPods.Num, test.PodsToSchedule.Num) + initPods := 0 + for _, p := range test.InitPods { + initPods += p.Num + } + name := fmt.Sprintf("%v/%vNodes/%vInitPods/%vPodsToSchedule", test.Desc, test.Nodes.Num, initPods, test.PodsToSchedule.Num) b.Run(name, func(b *testing.B) { for feature, flag := range test.FeatureGates { defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, feature, flag)() @@ -143,10 +147,14 @@ func perfScheduling(test testCase, b *testing.B) []DataItem { } defer nodePreparer.CleanupNodes() - if err := createPods(setupNamespace, test.InitPods, clientset); err != nil { - b.Fatal(err) + total := 0 + for _, p := range test.InitPods { + if err := createPods(setupNamespace, p, clientset); err != nil { + b.Fatal(err) + } + total += p.Num } - if err := waitNumPodsScheduled(b, test.InitPods.Num, podInformer); err != nil { + if err := waitNumPodsScheduled(b, total, podInformer); err != nil { b.Fatal(err) } @@ -164,7 +172,7 @@ func perfScheduling(test testCase, b *testing.B) []DataItem { if err := createPods(testNamespace, test.PodsToSchedule, clientset); err != nil { b.Fatal(err) } - if err := waitNumPodsScheduled(b, test.InitPods.Num+test.PodsToSchedule.Num, podInformer); err != nil { + if err := waitNumPodsScheduled(b, total+test.PodsToSchedule.Num, podInformer); err != nil { b.Fatal(err) } @@ -278,7 +286,10 @@ func parseTestCases(path string) ([]testCase, error) { testCase := s.Template for _, p := range s.Params { testCase.Nodes.Num = p.NumNodes - testCase.InitPods.Num = p.NumInitPods + testCase.InitPods = append([]podCase(nil), testCase.InitPods...) + for i, v := range p.NumInitPods { + testCase.InitPods[i].Num = v + } testCase.PodsToSchedule.Num = p.NumPodsToSchedule testCases = append(testCases, testCase) }