github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-26 05:03:09 +00:00
Code Issues Projects Releases Wiki Activity

update scheduler benchmarks to be more representative

Browse Source
This commit is contained in:
Abdullah Gharaibeh 2019-12-02 11:18:57 -05:00
parent aa67744438
commit e9c400aabc
1 changed files with 16 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
test/integration/scheduler_perf/scheduler_bench_test.go
View File

@ -44,24 +44,21 @@ var (
testCSIDriver = plugins.AWSEBSDriverName testCSIDriver = plugins.AWSEBSDriverName
// From PV controller // From PV controller
annBindCompleted = "pv.kubernetes.io/bind-completed" annBindCompleted = "pv.kubernetes.io/bind-completed"
tests = []struct{ nodes, existingPods, minPods int }{
{nodes: 500, existingPods: 500, minPods: 1000},
{nodes: 5000, existingPods: 5000, minPods: 1000},
}
) )
// BenchmarkScheduling benchmarks the scheduling rate when the cluster has // BenchmarkScheduling benchmarks the scheduling rate when the cluster has
// various quantities of nodes and scheduled pods. // various quantities of nodes and scheduled pods.
func BenchmarkScheduling(b *testing.B) { func BenchmarkScheduling(b *testing.B) {
tests := []struct{ nodes, existingPods, minPods int }{ testStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("rc1")
{nodes: 100, existingPods: 0, minPods: 100},
{nodes: 100, existingPods: 1000, minPods: 100},
{nodes: 1000, existingPods: 0, minPods: 100},
{nodes: 1000, existingPods: 1000, minPods: 100},
{nodes: 5000, existingPods: 1000, minPods: 1000},
}
setupStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("rc1")
testStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("rc2")
for _, test := range tests { for _, test := range tests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) { b.Run(name, func(b *testing.B) {
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, setupStrategy, testStrategy, b) benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b)
}) })
} }
} }
@ -70,14 +67,6 @@ func BenchmarkScheduling(b *testing.B) {
// PodAntiAffinity rules when the cluster has various quantities of nodes and // PodAntiAffinity rules when the cluster has various quantities of nodes and
// scheduled pods. // scheduled pods.
func BenchmarkSchedulingPodAntiAffinity(b *testing.B) { func BenchmarkSchedulingPodAntiAffinity(b *testing.B) {
tests := []struct{ nodes, existingPods, minPods int }{
{nodes: 500, existingPods: 250, minPods: 250},
{nodes: 500, existingPods: 5000, minPods: 250},
{nodes: 1000, existingPods: 1000, minPods: 500},
{nodes: 5000, existingPods: 1000, minPods: 1000},
}
// The setup strategy creates pods with no affinity rules.
setupStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("setup")
testBasePod := makeBasePodWithPodAntiAffinity( testBasePod := makeBasePodWithPodAntiAffinity(
map[string]string{"name": "test", "color": "green"}, map[string]string{"name": "test", "color": "green"},
map[string]string{"color": "green"}) map[string]string{"color": "green"})
@ -86,7 +75,7 @@ func BenchmarkSchedulingPodAntiAffinity(b *testing.B) {
for _, test := range tests { for _, test := range tests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) { b.Run(name, func(b *testing.B) {
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, setupStrategy, testStrategy, b) benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b)
}) })
} }
} }
@ -96,21 +85,13 @@ func BenchmarkSchedulingPodAntiAffinity(b *testing.B) {
// It can be used to compare scheduler efficiency with the other benchmarks // It can be used to compare scheduler efficiency with the other benchmarks
// that use volume scheduling predicates. // that use volume scheduling predicates.
func BenchmarkSchedulingSecrets(b *testing.B) { func BenchmarkSchedulingSecrets(b *testing.B) {
tests := []struct{ nodes, existingPods, minPods int }{
{nodes: 500, existingPods: 250, minPods: 250},
{nodes: 500, existingPods: 5000, minPods: 250},
{nodes: 1000, existingPods: 1000, minPods: 500},
{nodes: 5000, existingPods: 1000, minPods: 1000},
}
// The setup strategy creates pods with no volumes.
setupStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("setup")
// The test strategy creates pods with a secret. // The test strategy creates pods with a secret.
testBasePod := makeBasePodWithSecret() testBasePod := makeBasePodWithSecret()
testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
for _, test := range tests { for _, test := range tests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) { b.Run(name, func(b *testing.B) {
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, setupStrategy, testStrategy, b) benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b)
}) })
} }
} }
@ -119,15 +100,6 @@ func BenchmarkSchedulingSecrets(b *testing.B) {
// in-tree volumes (used via PV/PVC). Nodes have default hardcoded attach limits // in-tree volumes (used via PV/PVC). Nodes have default hardcoded attach limits
// (39 for AWS EBS). // (39 for AWS EBS).
func BenchmarkSchedulingInTreePVs(b *testing.B) { func BenchmarkSchedulingInTreePVs(b *testing.B) {
tests := []struct{ nodes, existingPods, minPods int }{
{nodes: 500, existingPods: 250, minPods: 250},
{nodes: 500, existingPods: 5000, minPods: 250},
{nodes: 1000, existingPods: 1000, minPods: 500},
{nodes: 5000, existingPods: 1000, minPods: 1000},
}
// The setup strategy creates pods with no volumes.
setupStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("setup")
// The test strategy creates pods with AWS EBS volume used via PV. // The test strategy creates pods with AWS EBS volume used via PV.
baseClaim := makeBasePersistentVolumeClaim() baseClaim := makeBasePersistentVolumeClaim()
basePod := makeBasePod() basePod := makeBasePod()
@ -135,7 +107,7 @@ func BenchmarkSchedulingInTreePVs(b *testing.B) {
for _, test := range tests { for _, test := range tests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) { b.Run(name, func(b *testing.B) {
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, setupStrategy, testStrategy, b) benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b)
}) })
} }
} }
@ -144,15 +116,6 @@ func BenchmarkSchedulingInTreePVs(b *testing.B) {
// in-tree volumes (used via PV/PVC) that are migrated to CSI. CSINode instances exist // in-tree volumes (used via PV/PVC) that are migrated to CSI. CSINode instances exist
// for all nodes and have proper annotation that AWS is migrated. // for all nodes and have proper annotation that AWS is migrated.
func BenchmarkSchedulingMigratedInTreePVs(b *testing.B) { func BenchmarkSchedulingMigratedInTreePVs(b *testing.B) {
tests := []struct{ nodes, existingPods, minPods int }{
{nodes: 500, existingPods: 250, minPods: 250},
{nodes: 500, existingPods: 5000, minPods: 250},
{nodes: 1000, existingPods: 1000, minPods: 500},
{nodes: 5000, existingPods: 1000, minPods: 1000},
}
// The setup strategy creates pods with no volumes.
setupStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("setup")
// The test strategy creates pods with AWS EBS volume used via PV. // The test strategy creates pods with AWS EBS volume used via PV.
baseClaim := makeBasePersistentVolumeClaim() baseClaim := makeBasePersistentVolumeClaim()
basePod := makeBasePod() basePod := makeBasePod()
@ -176,23 +139,13 @@ func BenchmarkSchedulingMigratedInTreePVs(b *testing.B) {
b.Run(name, func(b *testing.B) { b.Run(name, func(b *testing.B) {
defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.CSIMigration, true)() defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.CSIMigration, true)()
defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.CSIMigrationAWS, true)() defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.CSIMigrationAWS, true)()
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, setupStrategy, testStrategy, b) benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
}) })
} }
} }
// node.status.allocatable. // node.status.allocatable.
func BenchmarkSchedulingCSIPVs(b *testing.B) { func BenchmarkSchedulingCSIPVs(b *testing.B) {
tests := []struct{ nodes, existingPods, minPods int }{
{nodes: 500, existingPods: 250, minPods: 250},
{nodes: 500, existingPods: 5000, minPods: 250},
{nodes: 1000, existingPods: 1000, minPods: 500},
{nodes: 5000, existingPods: 1000, minPods: 1000},
}
// The setup strategy creates pods with no volumes.
setupStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("setup")
// The test strategy creates pods with CSI volume via PV. // The test strategy creates pods with CSI volume via PV.
baseClaim := makeBasePersistentVolumeClaim() baseClaim := makeBasePersistentVolumeClaim()
basePod := makeBasePod() basePod := makeBasePod()
@ -214,7 +167,7 @@ func BenchmarkSchedulingCSIPVs(b *testing.B) {
for _, test := range tests { for _, test := range tests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) { b.Run(name, func(b *testing.B) {
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, setupStrategy, testStrategy, b) benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
}) })
} }
} }
@ -223,14 +176,6 @@ func BenchmarkSchedulingCSIPVs(b *testing.B) {
// PodAffinity rules when the cluster has various quantities of nodes and // PodAffinity rules when the cluster has various quantities of nodes and
// scheduled pods. // scheduled pods.
func BenchmarkSchedulingPodAffinity(b *testing.B) { func BenchmarkSchedulingPodAffinity(b *testing.B) {
tests := []struct{ nodes, existingPods, minPods int }{
{nodes: 500, existingPods: 250, minPods: 250},
{nodes: 500, existingPods: 5000, minPods: 250},
{nodes: 1000, existingPods: 1000, minPods: 500},
{nodes: 5000, existingPods: 1000, minPods: 1000},
}
// The setup strategy creates pods with no affinity rules.
setupStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("setup")
testBasePod := makeBasePodWithPodAffinity( testBasePod := makeBasePodWithPodAffinity(
map[string]string{"foo": ""}, map[string]string{"foo": ""},
map[string]string{"foo": ""}, map[string]string{"foo": ""},
@ -241,7 +186,7 @@ func BenchmarkSchedulingPodAffinity(b *testing.B) {
for _, test := range tests { for _, test := range tests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) { b.Run(name, func(b *testing.B) {
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, setupStrategy, testStrategy, b) benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
}) })
} }
} }
@ -250,14 +195,6 @@ func BenchmarkSchedulingPodAffinity(b *testing.B) {
// NodeAffinity rules when the cluster has various quantities of nodes and // NodeAffinity rules when the cluster has various quantities of nodes and
// scheduled pods. // scheduled pods.
func BenchmarkSchedulingNodeAffinity(b *testing.B) { func BenchmarkSchedulingNodeAffinity(b *testing.B) {
tests := []struct{ nodes, existingPods, minPods int }{
{nodes: 500, existingPods: 250, minPods: 250},
{nodes: 500, existingPods: 5000, minPods: 250},
{nodes: 1000, existingPods: 1000, minPods: 500},
{nodes: 5000, existingPods: 1000, minPods: 1000},
}
// The setup strategy creates pods with no affinity rules.
setupStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("setup")
testBasePod := makeBasePodWithNodeAffinity(v1.LabelZoneFailureDomain, []string{"zone1", "zone2"}) testBasePod := makeBasePodWithNodeAffinity(v1.LabelZoneFailureDomain, []string{"zone1", "zone2"})
// The test strategy creates pods with node-affinity for each other. // The test strategy creates pods with node-affinity for each other.
testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod) testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
@ -265,7 +202,7 @@ func BenchmarkSchedulingNodeAffinity(b *testing.B) {
for _, test := range tests { for _, test := range tests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods) name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) { b.Run(name, func(b *testing.B) {
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, setupStrategy, testStrategy, b) benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
}) })
} }
} }
@ -355,7 +292,7 @@ func makeBasePodWithNodeAffinity(key string, vals []string) *v1.Pod {
// least minPods pods during the benchmark. // least minPods pods during the benchmark.
func benchmarkScheduling(numNodes, numExistingPods, minPods int, func benchmarkScheduling(numNodes, numExistingPods, minPods int,
nodeStrategy testutils.PrepareNodeStrategy, nodeStrategy testutils.PrepareNodeStrategy,
setupPodStrategy, testPodStrategy testutils.TestPodCreateStrategy, testPodStrategy testutils.TestPodCreateStrategy,
b *testing.B) { b *testing.B) {
if b.N < minPods { if b.N < minPods {
b.N = minPods b.N = minPods
@ -374,7 +311,7 @@ func benchmarkScheduling(numNodes, numExistingPods, minPods int,
defer nodePreparer.CleanupNodes() defer nodePreparer.CleanupNodes()
config := testutils.NewTestPodCreatorConfig() config := testutils.NewTestPodCreatorConfig()
config.AddStrategy("sched-test", numExistingPods, setupPodStrategy) config.AddStrategy("sched-setup", numExistingPods, testPodStrategy)
podCreator := testutils.NewTestPodCreator(clientset, config) podCreator := testutils.NewTestPodCreator(clientset, config)
podCreator.CreatePods() podCreator.CreatePods()