github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-07-17 20:00:07 +00:00
Code Issues Projects Releases Wiki Activity

DRA scheduler_perf: benchmark baseline test cases with different allocators

Browse Source 
When adding new features to the "experimental" allocator we are not directly
breaking "normal" usage of DRA because that uses the "incubating"
allocator. But we still want to know about potential performance problems also
for those test cases. If there is one, we might not be able to implement an
experimental feature in a way that allows promoting it later.

This is achieved by running the benchmarks twice as sub-tests, with different
topic names. The topic names then can be used to select different thresholds,
using a new extension of the scheduler_pref JSON config format.
This commit is contained in:
Patrick Ohly
2025-11-03 15:22:32 +01:00
parent afb2b97425
commit d132afc87e
3 changed files with 83 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
test/integration/scheduler_perf/dra/dra_test.go
View File

@@ -69,10 +69,25 @@ func TestSchedulerPerf(t *testing.T) {
}
}
// These benchmarks have to contain the word BenchmarkPerfScheduling to be picked up
// by benchmark jobs.
//
// Sub-tests should have worked, too, but gotestsum was unhappy.
func BenchmarkPerfScheduling(b *testing.B) {
// Restrict benchmarking to the default allocator.
structured.EnableAllocators("incubating")
defer structured.EnableAllocators()
// "dra" is how this was called traditionally.
// It's kept to avoid changing perf-dash results.
perf.RunBenchmarkPerfScheduling(b, "performance-config.yaml", "dra", nil)
}
func BenchmarkPerfSchedulingExperimental(b *testing.B) {
// And now the experimental allocator.
structured.EnableAllocators("experimental")
defer structured.EnableAllocators()
perf.RunBenchmarkPerfScheduling(b, "performance-config.yaml", "dra_experimental", nil)
}

40
test/integration/scheduler_perf/dra/performance-config.yaml
View File

@@ -130,8 +130,9 @@
featureGates:
SchedulerQueueingHints: false
labels: [performance]
# https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSchedulingWithResourceClaimTemplate%2F5000pods_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable
threshold: 40 # typically above 51
threshold:
# https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSchedulingWithResourceClaimTemplate%2F5000pods_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable
dra: 40 # typically above 51
params:
nodesWithDRA: 500
nodesWithoutDRA: 0
@@ -142,8 +143,9 @@
featureGates:
SchedulerQueueingHints: true
labels: [performance]
# https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSchedulingWithResourceClaimTemplate%2F5000pods_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable
threshold: 56 # typically above 70
threshold:
# https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSchedulingWithResourceClaimTemplate%2F5000pods_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable
dra: 56 # typically above 70
params:
nodesWithDRA: 500
nodesWithoutDRA: 0
@@ -208,8 +210,9 @@
featureGates:
SchedulerQueueingHints: false
labels: [performance]
# https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fempty_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable
threshold: 64 # typically above 81
threshold:
# https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fempty_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable
dra: 64 # typically above 81
params:
nodesWithDRA: 500
nodesWithoutDRA: 0
@@ -220,8 +223,9 @@
featureGates:
SchedulerQueueingHints: true
labels: [performance]
# https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fempty_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable
threshold: 64 # typically above 81
threshold:
# https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fempty_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable
dra: 64 # typically above 81
params:
nodesWithDRA: 500
nodesWithoutDRA: 0
@@ -247,8 +251,9 @@
featureGates:
SchedulerQueueingHints: false
labels: [performance]
# https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fhalf_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable
threshold: 70 # typically over 78
threshold:
# https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fhalf_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable
dra: 70 # typically over 78
params:
nodesWithDRA: 500
nodesWithoutDRA: 0
@@ -259,8 +264,9 @@
featureGates:
SchedulerQueueingHints: true
labels: [performance]
# https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fhalf_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable
threshold: 60 # typically over 75
threshold:
# https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fhalf_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable
dra: 60 # typically over 75
params:
nodesWithDRA: 500
nodesWithoutDRA: 0
@@ -286,8 +292,9 @@
featureGates:
SchedulerQueueingHints: false
labels: [performance]
# https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Ffull_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable
threshold: 60 # typically over 75
threshold:
# https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Ffull_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable
dra: 60 # typically over 75
params:
nodesWithDRA: 500
nodesWithoutDRA: 0
@@ -298,8 +305,9 @@
featureGates:
SchedulerQueueingHints: true
labels: [performance]
# https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Ffull_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable
threshold: 60 # typically over 75
threshold:
# https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Ffull_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable
dra: 60 # typically over 75
params:
nodesWithDRA: 500
nodesWithoutDRA: 0

53
test/integration/scheduler_perf/scheduler_perf.go
View File

@@ -332,8 +332,14 @@ type workload struct {
// The comparison is performed for op with CollectMetrics set to true.
// If the measured value is below the threshold, the workload's test case will fail.
// If set to zero, the threshold check is disabled.
//
// May contain a single value or map of topic name to value.
// The single value is used if there is no entry in the map for the topic name.
// Topic names are passed to RunBenchmarkPerfScheduling. This approach
// makes it possible to reuse the same test cases in different configurations.
//
// Optional
Threshold float64
Threshold thresholds
// ThresholdMetricSelector defines to what metric the Threshold should be compared.
// If nil, the metric is set to DefaultThresholdMetricSelector of the testCase.
// If DefaultThresholdMetricSelector is nil, the metric is set to "SchedulingThroughput".
@@ -346,8 +352,13 @@ type workload struct {
}
func (w *workload) isValid(mcc *metricsCollectorConfig) error {
if w.Threshold < 0 {
return fmt.Errorf("invalid Threshold=%f; should be non-negative", w.Threshold)
if w.Threshold.value < 0 {
return fmt.Errorf("invalid Threshold=%f; should be non-negative", w.Threshold.value)
}
for topicName, value := range w.Threshold.valuesByTopic {
if value < 0 {
return fmt.Errorf("invalid Threshold=%f for topic %q; should be non-negative", value, topicName)
}
}
return w.ThresholdMetricSelector.isValid(mcc)
@@ -367,6 +378,28 @@ func (w *workload) setDefaults(testCaseThresholdMetricSelector *thresholdMetricS
}
}
type thresholds struct {
value float64
valuesByTopic map[string]float64
}
func (t *thresholds) UnmarshalJSON(text []byte) error {
if errFloat64 := json.Unmarshal(text, &t.value); errFloat64 != nil {
// Not a plain number. Let's try as map.
if errMap := json.Unmarshal(text, &t.valuesByTopic); errMap != nil {
return fmt.Errorf("expected either float64 or topic name -> float64 map: %w, %w", errFloat64, errMap)
}
}
return nil
}
func (t *thresholds) Get(topicName string) float64 {
if value, ok := t.valuesByTopic[topicName]; ok {
return value
}
return t.value
}
// thresholdMetricSelector defines the name and labels of metric to compare with threshold.
type thresholdMetricSelector struct {
// Name of the metric is compared to "Metric" field in DataItem labels.
@@ -1232,7 +1265,7 @@ func RunBenchmarkPerfScheduling(b *testing.B, configFile string, topicName strin
}
}
results, err := runWorkload(tCtx, tc, w, scheduler, informerFactory)
results, err := runWorkload(tCtx, tc, w, topicName, scheduler, informerFactory)
if err != nil {
tCtx.Fatalf("Error running workload %s: %s", w.Name, err)
}
@@ -1299,7 +1332,7 @@ func RunBenchmarkPerfScheduling(b *testing.B, configFile string, topicName strin
}
})
}
if err := dataItems2JSONFile(dataItems, b.Name()+"_benchmark_"+topicName); err != nil {
if err := dataItems2JSONFile(dataItems, strings.ReplaceAll(b.Name(), "/", "_")+"_benchmark_"+topicName); err != nil {
b.Fatalf("unable to write measured data %+v: %v", dataItems, err)
}
}
@@ -1333,7 +1366,7 @@ func RunIntegrationPerfScheduling(t *testing.T, configFile string) {
t.Fatalf("workload %s is not valid: %v", w.Name, err)
}
_, err = runWorkload(tCtx, tc, w, scheduler, informerFactory)
_, err = runWorkload(tCtx, tc, w, "" /* topic name not relevant */, scheduler, informerFactory)
if err != nil {
tCtx.Fatalf("Error running workload %s: %s", w.Name, err)
}
@@ -1522,10 +1555,11 @@ type WorkloadExecutor struct {
throughputErrorMargin float64
testCase *testCase
workload *workload
topicName string
nextNodeIndex int
}
func runWorkload(tCtx ktesting.TContext, tc *testCase, w *workload, scheduler *scheduler.Scheduler, informerFactory informers.SharedInformerFactory) ([]DataItem, error) {
func runWorkload(tCtx ktesting.TContext, tc *testCase, w *workload, topicName string, scheduler *scheduler.Scheduler, informerFactory informers.SharedInformerFactory) ([]DataItem, error) {
b, benchmarking := tCtx.TB().(*testing.B)
if benchmarking {
start := time.Now()
@@ -1565,6 +1599,7 @@ func runWorkload(tCtx ktesting.TContext, tc *testCase, w *workload, scheduler *s
throughputErrorMargin: throughputErrorMargin,
testCase: tc,
workload: w,
topicName: topicName,
}
tCtx.TB().Cleanup(func() {
@@ -1702,7 +1737,7 @@ func (e *WorkloadExecutor) runSleepOp(op *sleepOp) error {
}
func (e *WorkloadExecutor) runStopCollectingMetrics(opIndex int) error {
items, err := stopCollectingMetrics(e.tCtx, e.collectorCtx, &e.collectorWG, e.workload.Threshold, *e.workload.ThresholdMetricSelector, opIndex, e.collectors)
items, err := stopCollectingMetrics(e.tCtx, e.collectorCtx, &e.collectorWG, e.workload.Threshold.Get(e.topicName), *e.workload.ThresholdMetricSelector, opIndex, e.collectors)
if err != nil {
return err
}
@@ -1756,7 +1791,7 @@ func (e *WorkloadExecutor) runCreatePodsOp(opIndex int, op *createPodsOp) error
// CollectMetrics and SkipWaitToCompletion can never be true at the
// same time, so if we're here, it means that all pods have been
// scheduled.
items, err := stopCollectingMetrics(e.tCtx, e.collectorCtx, &e.collectorWG, e.workload.Threshold, *e.workload.ThresholdMetricSelector, opIndex, e.collectors)
items, err := stopCollectingMetrics(e.tCtx, e.collectorCtx, &e.collectorWG, e.workload.Threshold.Get(e.topicName), *e.workload.ThresholdMetricSelector, opIndex, e.collectors)
if err != nil {
return err
}