From 03bdef404503b579458962026b3c88ef8b181cf4 Mon Sep 17 00:00:00 2001 From: Jonathan Basseri Date: Wed, 6 Dec 2017 18:19:23 -0800 Subject: [PATCH] Add scheduler benchmark tests for affinity rules. This adds new benchmark tests that measure scheduler latency of pods that use affinity rules. Specifically, this tests affinity rules with topologyKey="kubernetes.io/hostname". --- .../scheduler_perf/scheduler_bench_test.go | 89 +++++++++++++++---- 1 file changed, 74 insertions(+), 15 deletions(-) diff --git a/test/integration/scheduler_perf/scheduler_bench_test.go b/test/integration/scheduler_perf/scheduler_bench_test.go index 608e4888b51..fb28da4ec60 100644 --- a/test/integration/scheduler_perf/scheduler_bench_test.go +++ b/test/integration/scheduler_perf/scheduler_bench_test.go @@ -21,7 +21,10 @@ import ( "testing" "time" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/kubernetes/pkg/kubelet/apis" "k8s.io/kubernetes/test/integration/framework" testutils "k8s.io/kubernetes/test/utils" @@ -31,24 +34,80 @@ import ( // BenchmarkScheduling benchmarks the scheduling rate when the cluster has // various quantities of nodes and scheduled pods. func BenchmarkScheduling(b *testing.B) { - tests := []struct{ nodes, pods, minOps int }{ - {nodes: 100, pods: 0, minOps: 100}, - {nodes: 100, pods: 1000, minOps: 100}, - {nodes: 1000, pods: 0, minOps: 100}, - {nodes: 1000, pods: 1000, minOps: 100}, + tests := []struct{ nodes, existingPods, minPods int }{ + {nodes: 100, existingPods: 0, minPods: 100}, + {nodes: 100, existingPods: 1000, minPods: 100}, + {nodes: 1000, existingPods: 0, minPods: 100}, + {nodes: 1000, existingPods: 1000, minPods: 100}, } + setupStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("rc1") + testStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("rc2") for _, test := range tests { - name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.pods) - b.Run(name, func(b *testing.B) { benchmarkScheduling(test.nodes, test.pods, test.minOps, b) }) + name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) + b.Run(name, func(b *testing.B) { + benchmarkScheduling(test.nodes, test.existingPods, test.minPods, setupStrategy, testStrategy, b) + }) } } +// BenchmarkSchedulingAntiAffinity benchmarks the scheduling rate of pods with +// PodAntiAffinity rules when the cluster has various quantities of nodes and +// scheduled pods. +func BenchmarkSchedulingAntiAffinity(b *testing.B) { + tests := []struct{ nodes, existingPods, minPods int }{ + {nodes: 500, existingPods: 250, minPods: 250}, + {nodes: 500, existingPods: 5000, minPods: 250}, + } + // The setup strategy creates pods with no affinity rules. + setupStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("setup") + // The test strategy creates pods with anti-affinity for each other. + testBasePod := makeBasePodWithAntiAffinity( + map[string]string{"name": "test", "color": "green"}, + map[string]string{"color": "green"}) + testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) + for _, test := range tests { + name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) + b.Run(name, func(b *testing.B) { + benchmarkScheduling(test.nodes, test.existingPods, test.minPods, setupStrategy, testStrategy, b) + }) + } + +} + +// makeBasePodWithAntiAffinity creates a Pod object to be used as a template. +// The Pod has a PodAntiAffinity requirement against pods with the given labels. +func makeBasePodWithAntiAffinity(podLabels, affinityLabels map[string]string) *v1.Pod { + basePod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "affinity-pod-", + Labels: podLabels, + }, + Spec: testutils.MakePodSpec(), + } + basePod.Spec.Affinity = &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affinityLabels, + }, + TopologyKey: apis.LabelHostname, + }, + }, + }, + } + return basePod +} + // benchmarkScheduling benchmarks scheduling rate with specific number of nodes // and specific number of pods already scheduled. -// Since an operation typically takes more than 1 second, we put a minimum bound on b.N of minOps. -func benchmarkScheduling(numNodes, numScheduledPods, minOps int, b *testing.B) { - if b.N < minOps { - b.N = minOps +// This will schedule numExistingPods pods before the benchmark starts, and at +// least minPods pods during the benchmark. +func benchmarkScheduling(numNodes, numExistingPods, minPods int, + setupPodStrategy, testPodStrategy testutils.TestPodCreateStrategy, + b *testing.B) { + if b.N < minPods { + b.N = minPods } schedulerConfigFactory, finalFunc := mustSetupScheduler() defer finalFunc() @@ -65,7 +124,7 @@ func benchmarkScheduling(numNodes, numScheduledPods, minOps int, b *testing.B) { defer nodePreparer.CleanupNodes() config := testutils.NewTestPodCreatorConfig() - config.AddStrategy("sched-test", numScheduledPods, testutils.NewSimpleWithControllerCreatePodStrategy("rc1")) + config.AddStrategy("sched-test", numExistingPods, setupPodStrategy) podCreator := testutils.NewTestPodCreator(c, config) podCreator.CreatePods() @@ -74,7 +133,7 @@ func benchmarkScheduling(numNodes, numScheduledPods, minOps int, b *testing.B) { if err != nil { glog.Fatalf("%v", err) } - if len(scheduled) >= numScheduledPods { + if len(scheduled) >= numExistingPods { break } time.Sleep(1 * time.Second) @@ -82,7 +141,7 @@ func benchmarkScheduling(numNodes, numScheduledPods, minOps int, b *testing.B) { // start benchmark b.ResetTimer() config = testutils.NewTestPodCreatorConfig() - config.AddStrategy("sched-test", b.N, testutils.NewSimpleWithControllerCreatePodStrategy("rc2")) + config.AddStrategy("sched-test", b.N, testPodStrategy) podCreator = testutils.NewTestPodCreator(c, config) podCreator.CreatePods() for { @@ -92,7 +151,7 @@ func benchmarkScheduling(numNodes, numScheduledPods, minOps int, b *testing.B) { if err != nil { glog.Fatalf("%v", err) } - if len(scheduled) >= numScheduledPods+b.N { + if len(scheduled) >= numExistingPods+b.N { break } // Note: This might introduce slight deviation in accuracy of benchmark results.