github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 11:50:44 +00:00
Code Issues Projects Releases Wiki Activity

Fix inter-pod affinity scheduler benchmarks

Browse Source
This commit is contained in:
Abdullah Gharaibeh 2019-12-07 13:46:53 -05:00
parent 50437b4c5d
commit a051c591a1
1 changed files with 62 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
test/integration/scheduler_perf/scheduler_bench_test.go
View File

@ -45,20 +45,23 @@ var (
// From PV controller
annBindCompleted = "pv.kubernetes.io/bind-completed"
tests = []struct{ nodes, existingPods, minPods int }{
defaultTests = []struct{ nodes, existingPods, minPods int }{
{nodes: 500, existingPods: 500, minPods: 1000},
{nodes: 5000, existingPods: 5000, minPods: 1000},
}
testNamespace = "sched-test"
setupNamespace = "sched-setup"
)
// BenchmarkScheduling benchmarks the scheduling rate when the cluster has
// various quantities of nodes and scheduled pods.
func BenchmarkScheduling(b *testing.B) {
testStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("rc1")
for _, test := range tests {
for _, test := range defaultTests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) {
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b)
nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: defaultNodeStrategy}}
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
})
}
}
@ -67,15 +70,26 @@ func BenchmarkScheduling(b *testing.B) {
// PodAntiAffinity rules when the cluster has various quantities of nodes and
// scheduled pods.
func BenchmarkSchedulingPodAntiAffinity(b *testing.B) {
// Since the pods has anti affinity to each other, the number of pods to schedule
// can't exceed the number of nodes (the topology used in the test)
tests := []struct{ nodes, existingPods, minPods int }{
{nodes: 500, existingPods: 100, minPods: 400},
{nodes: 5000, existingPods: 1000, minPods: 1000},
}
testBasePod := makeBasePodWithPodAntiAffinity(
map[string]string{"name": "test", "color": "green"},
map[string]string{"color": "green"})
// The test strategy creates pods with anti-affinity for each other.
// The test strategy creates pods with anti-affinity to each other, each pod ending up in a separate node.
testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
for _, test := range tests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) {
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b)
var nodeStrategies []testutils.CountToStrategy
for i := 0; i < test.nodes; i++ {
nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelHostname, fmt.Sprintf("node-%d", i))
nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{Count: 1, Strategy: nodeStrategy})
}
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
})
}
}
@ -88,10 +102,11 @@ func BenchmarkSchedulingSecrets(b *testing.B) {
// The test strategy creates pods with a secret.
testBasePod := makeBasePodWithSecret()
testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
for _, test := range tests {
for _, test := range defaultTests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) {
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b)
nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: defaultNodeStrategy}}
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
})
}
}
@ -104,10 +119,11 @@ func BenchmarkSchedulingInTreePVs(b *testing.B) {
baseClaim := makeBasePersistentVolumeClaim()
basePod := makeBasePod()
testStrategy := testutils.NewCreatePodWithPersistentVolumeStrategy(baseClaim, awsVolumeFactory, basePod)
for _, test := range tests {
for _, test := range defaultTests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) {
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b)
nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: defaultNodeStrategy}}
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
})
}
}
@ -134,12 +150,13 @@ func BenchmarkSchedulingMigratedInTreePVs(b *testing.B) {
},
}
nodeStrategy := testutils.NewNodeAllocatableStrategy(allocatable, csiAllocatable, []string{csilibplugins.AWSEBSInTreePluginName})
for _, test := range tests {
for _, test := range defaultTests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) {
defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.CSIMigration, true)()
defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.CSIMigrationAWS, true)()
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}}
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
})
}
}
@ -164,10 +181,11 @@ func BenchmarkSchedulingCSIPVs(b *testing.B) {
},
}
nodeStrategy := testutils.NewNodeAllocatableStrategy(allocatable, csiAllocatable, []string{})
for _, test := range tests {
for _, test := range defaultTests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) {
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}}
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
})
}
}
@ -183,10 +201,11 @@ func BenchmarkSchedulingPodAffinity(b *testing.B) {
// The test strategy creates pods with affinity for each other.
testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelZoneFailureDomain, "zone1")
for _, test := range tests {
for _, test := range defaultTests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) {
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}}
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
})
}
}
@ -201,11 +220,15 @@ func BenchmarkSchedulingPreferredPodAffinity(b *testing.B) {
)
// The test strategy creates pods with affinity for each other.
testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelZoneFailureDomain, "zone1")
for _, test := range tests {
for _, test := range defaultTests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) {
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
var nodeStrategies []testutils.CountToStrategy
for i := 0; i < test.nodes; i++ {
nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelHostname, fmt.Sprintf("node-%d", i))
nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{Count: 1, Strategy: nodeStrategy})
}
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
})
}
}
@ -218,13 +241,17 @@ func BenchmarkSchedulingPreferredPodAntiAffinity(b *testing.B) {
map[string]string{"foo": ""},
map[string]string{"foo": ""},
)
// The test strategy creates pods with affinity for each other.
// The test strategy creates pods with anti affinity to each other.
testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelZoneFailureDomain, "zone1")
for _, test := range tests {
for _, test := range defaultTests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) {
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
var nodeStrategies []testutils.CountToStrategy
for i := 0; i < test.nodes; i++ {
nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelHostname, fmt.Sprintf("node-%d", i))
nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{Count: 1, Strategy: nodeStrategy})
}
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
})
}
}
@ -237,10 +264,11 @@ func BenchmarkSchedulingNodeAffinity(b *testing.B) {
// The test strategy creates pods with node-affinity for each other.
testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelZoneFailureDomain, "zone1")
for _, test := range tests {
for _, test := range defaultTests {
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
b.Run(name, func(b *testing.B) {
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}}
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
})
}
}
@ -263,6 +291,7 @@ func makeBasePodWithPodAntiAffinity(podLabels, affinityLabels map[string]string)
MatchLabels: affinityLabels,
},
TopologyKey: v1.LabelHostname,
Namespaces: []string{testNamespace, setupNamespace},
},
},
},
@ -289,6 +318,7 @@ func makeBasePodWithPreferredPodAntiAffinity(podLabels, affinityLabels map[strin
MatchLabels: affinityLabels,
},
TopologyKey: v1.LabelHostname,
Namespaces: []string{testNamespace, setupNamespace},
},
Weight: 1,
},
@ -317,6 +347,7 @@ func makeBasePodWithPreferredPodAffinity(podLabels, affinityLabels map[string]st
MatchLabels: affinityLabels,
},
TopologyKey: v1.LabelHostname,
Namespaces: []string{testNamespace, setupNamespace},
},
Weight: 1,
},
@ -344,6 +375,7 @@ func makeBasePodWithPodAffinity(podLabels, affinityZoneLabels map[string]string)
MatchLabels: affinityZoneLabels,
},
TopologyKey: v1.LabelZoneFailureDomain,
Namespaces: []string{testNamespace, setupNamespace},
},
},
},
@ -384,8 +416,8 @@ func makeBasePodWithNodeAffinity(key string, vals []string) *v1.Pod {
// and specific number of pods already scheduled.
// This will schedule numExistingPods pods before the benchmark starts, and at
// least minPods pods during the benchmark.
func benchmarkScheduling(numNodes, numExistingPods, minPods int,
nodeStrategy testutils.PrepareNodeStrategy,
func benchmarkScheduling(numExistingPods, minPods int,
nodeStrategies []testutils.CountToStrategy,
testPodStrategy testutils.TestPodCreateStrategy,
b *testing.B) {
if b.N < minPods {
@ -396,16 +428,15 @@ func benchmarkScheduling(numNodes, numExistingPods, minPods int,
nodePreparer := framework.NewIntegrationTestNodePreparer(
clientset,
[]testutils.CountToStrategy{{Count: numNodes, Strategy: nodeStrategy}},
"scheduler-perf-",
)
nodeStrategies,
"scheduler-perf-")
if err := nodePreparer.PrepareNodes(); err != nil {
klog.Fatalf("%v", err)
}
defer nodePreparer.CleanupNodes()
config := testutils.NewTestPodCreatorConfig()
config.AddStrategy("sched-setup", numExistingPods, testPodStrategy)
config.AddStrategy(setupNamespace, numExistingPods, testPodStrategy)
podCreator := testutils.NewTestPodCreator(clientset, config)
podCreator.CreatePods()
@ -439,7 +470,7 @@ func benchmarkScheduling(numNodes, numExistingPods, minPods int,
// start benchmark
b.ResetTimer()
config = testutils.NewTestPodCreatorConfig()
config.AddStrategy("sched-test", b.N, testPodStrategy)
config.AddStrategy(testNamespace, b.N, testPodStrategy)
podCreator = testutils.NewTestPodCreator(clientset, config)
podCreator.CreatePods()