From 5259d09c38ee96bde61667b071d6778fd9ecf586 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 17 Oct 2018 22:49:32 -0700 Subject: [PATCH] add more scheduler benchmark testcases - add benchmark test for PodAffinity - add benchmark test for NodeAffinity - add 1000-nodes test for PodAntiAffinity/PodAffinity/NodeAffinity --- .../scheduler_perf/scheduler_bench_test.go | 130 ++++++++++++++++-- 1 file changed, 119 insertions(+), 11 deletions(-) diff --git a/test/integration/scheduler_perf/scheduler_bench_test.go b/test/integration/scheduler_perf/scheduler_bench_test.go index fb28da4ec60..30403d14080 100644 --- a/test/integration/scheduler_perf/scheduler_bench_test.go +++ b/test/integration/scheduler_perf/scheduler_bench_test.go @@ -31,6 +31,10 @@ import ( "github.com/golang/glog" ) +var ( + defaultNodeStrategy = &testutils.TrivialNodePrepareStrategy{} +) + // BenchmarkScheduling benchmarks the scheduling rate when the cluster has // various quantities of nodes and scheduled pods. func BenchmarkScheduling(b *testing.B) { @@ -45,41 +49,90 @@ func BenchmarkScheduling(b *testing.B) { for _, test := range tests { name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) b.Run(name, func(b *testing.B) { - benchmarkScheduling(test.nodes, test.existingPods, test.minPods, setupStrategy, testStrategy, b) + benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, setupStrategy, testStrategy, b) }) } } -// BenchmarkSchedulingAntiAffinity benchmarks the scheduling rate of pods with +// BenchmarkSchedulingPodAntiAffinity benchmarks the scheduling rate of pods with // PodAntiAffinity rules when the cluster has various quantities of nodes and // scheduled pods. -func BenchmarkSchedulingAntiAffinity(b *testing.B) { +func BenchmarkSchedulingPodAntiAffinity(b *testing.B) { tests := []struct{ nodes, existingPods, minPods int }{ {nodes: 500, existingPods: 250, minPods: 250}, {nodes: 500, existingPods: 5000, minPods: 250}, + {nodes: 1000, existingPods: 1000, minPods: 500}, } // The setup strategy creates pods with no affinity rules. setupStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("setup") - // The test strategy creates pods with anti-affinity for each other. - testBasePod := makeBasePodWithAntiAffinity( + testBasePod := makeBasePodWithPodAntiAffinity( map[string]string{"name": "test", "color": "green"}, map[string]string{"color": "green"}) + // The test strategy creates pods with anti-affinity for each other. testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) for _, test := range tests { name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) b.Run(name, func(b *testing.B) { - benchmarkScheduling(test.nodes, test.existingPods, test.minPods, setupStrategy, testStrategy, b) + benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, setupStrategy, testStrategy, b) }) } - } -// makeBasePodWithAntiAffinity creates a Pod object to be used as a template. +// BenchmarkSchedulingPodAffinity benchmarks the scheduling rate of pods with +// PodAffinity rules when the cluster has various quantities of nodes and +// scheduled pods. +func BenchmarkSchedulingPodAffinity(b *testing.B) { + tests := []struct{ nodes, existingPods, minPods int }{ + {nodes: 500, existingPods: 250, minPods: 250}, + {nodes: 500, existingPods: 5000, minPods: 250}, + {nodes: 1000, existingPods: 1000, minPods: 500}, + } + // The setup strategy creates pods with no affinity rules. + setupStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("setup") + testBasePod := makeBasePodWithPodAffinity( + map[string]string{"foo": ""}, + map[string]string{"foo": ""}, + ) + // The test strategy creates pods with affinity for each other. + testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) + nodeStrategy := testutils.NewLabelNodePrepareStrategy(apis.LabelZoneFailureDomain, "zone1") + for _, test := range tests { + name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) + b.Run(name, func(b *testing.B) { + benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, setupStrategy, testStrategy, b) + }) + } +} + +// BenchmarkSchedulingNodeAffinity benchmarks the scheduling rate of pods with +// NodeAffinity rules when the cluster has various quantities of nodes and +// scheduled pods. +func BenchmarkSchedulingNodeAffinity(b *testing.B) { + tests := []struct{ nodes, existingPods, minPods int }{ + {nodes: 500, existingPods: 250, minPods: 250}, + {nodes: 500, existingPods: 5000, minPods: 250}, + {nodes: 1000, existingPods: 1000, minPods: 500}, + } + // The setup strategy creates pods with no affinity rules. + setupStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("setup") + testBasePod := makeBasePodWithNodeAffinity(apis.LabelZoneFailureDomain, []string{"zone1", "zone2"}) + // The test strategy creates pods with node-affinity for each other. + testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) + nodeStrategy := testutils.NewLabelNodePrepareStrategy(apis.LabelZoneFailureDomain, "zone1") + for _, test := range tests { + name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) + b.Run(name, func(b *testing.B) { + benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, setupStrategy, testStrategy, b) + }) + } +} + +// makeBasePodWithPodAntiAffinity creates a Pod object to be used as a template. // The Pod has a PodAntiAffinity requirement against pods with the given labels. -func makeBasePodWithAntiAffinity(podLabels, affinityLabels map[string]string) *v1.Pod { +func makeBasePodWithPodAntiAffinity(podLabels, affinityLabels map[string]string) *v1.Pod { basePod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ - GenerateName: "affinity-pod-", + GenerateName: "anit-affinity-pod-", Labels: podLabels, }, Spec: testutils.MakePodSpec(), @@ -99,11 +152,66 @@ func makeBasePodWithAntiAffinity(podLabels, affinityLabels map[string]string) *v return basePod } +// makeBasePodWithPodAffinity creates a Pod object to be used as a template. +// The Pod has a PodAffinity requirement against pods with the given labels. +func makeBasePodWithPodAffinity(podLabels, affinityZoneLabels map[string]string) *v1.Pod { + basePod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "affinity-pod-", + Labels: podLabels, + }, + Spec: testutils.MakePodSpec(), + } + basePod.Spec.Affinity = &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affinityZoneLabels, + }, + TopologyKey: apis.LabelZoneFailureDomain, + }, + }, + }, + } + return basePod +} + +// makeBasePodWithNodeAffinity creates a Pod object to be used as a template. +// The Pod has a NodeAffinity requirement against nodes with the given expressions. +func makeBasePodWithNodeAffinity(key string, vals []string) *v1.Pod { + basePod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "node-affinity-", + }, + Spec: testutils.MakePodSpec(), + } + basePod.Spec.Affinity = &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: key, + Operator: v1.NodeSelectorOpIn, + Values: vals, + }, + }, + }, + }, + }, + }, + } + return basePod +} + // benchmarkScheduling benchmarks scheduling rate with specific number of nodes // and specific number of pods already scheduled. // This will schedule numExistingPods pods before the benchmark starts, and at // least minPods pods during the benchmark. func benchmarkScheduling(numNodes, numExistingPods, minPods int, + nodeStrategy testutils.PrepareNodeStrategy, setupPodStrategy, testPodStrategy testutils.TestPodCreateStrategy, b *testing.B) { if b.N < minPods { @@ -115,7 +223,7 @@ func benchmarkScheduling(numNodes, numExistingPods, minPods int, nodePreparer := framework.NewIntegrationTestNodePreparer( c, - []testutils.CountToStrategy{{Count: numNodes, Strategy: &testutils.TrivialNodePrepareStrategy{}}}, + []testutils.CountToStrategy{{Count: numNodes, Strategy: nodeStrategy}}, "scheduler-perf-", ) if err := nodePreparer.PrepareNodes(); err != nil {