From d132afc87ee34fe865f00cb85687d98aee27c61b Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 3 Nov 2025 15:22:32 +0100 Subject: [PATCH] DRA scheduler_perf: benchmark baseline test cases with different allocators When adding new features to the "experimental" allocator we are not directly breaking "normal" usage of DRA because that uses the "incubating" allocator. But we still want to know about potential performance problems also for those test cases. If there is one, we might not be able to implement an experimental feature in a way that allows promoting it later. This is achieved by running the benchmarks twice as sub-tests, with different topic names. The topic names then can be used to select different thresholds, using a new extension of the scheduler_pref JSON config format. --- .../scheduler_perf/dra/dra_test.go | 15 ++++++ .../dra/performance-config.yaml | 40 ++++++++------ .../scheduler_perf/scheduler_perf.go | 53 +++++++++++++++---- 3 files changed, 83 insertions(+), 25 deletions(-) diff --git a/test/integration/scheduler_perf/dra/dra_test.go b/test/integration/scheduler_perf/dra/dra_test.go index 2ca40591565..bf64abb0877 100644 --- a/test/integration/scheduler_perf/dra/dra_test.go +++ b/test/integration/scheduler_perf/dra/dra_test.go @@ -69,10 +69,25 @@ func TestSchedulerPerf(t *testing.T) { } } +// These benchmarks have to contain the word BenchmarkPerfScheduling to be picked up +// by benchmark jobs. +// +// Sub-tests should have worked, too, but gotestsum was unhappy. + func BenchmarkPerfScheduling(b *testing.B) { // Restrict benchmarking to the default allocator. structured.EnableAllocators("incubating") defer structured.EnableAllocators() + // "dra" is how this was called traditionally. + // It's kept to avoid changing perf-dash results. perf.RunBenchmarkPerfScheduling(b, "performance-config.yaml", "dra", nil) } + +func BenchmarkPerfSchedulingExperimental(b *testing.B) { + // And now the experimental allocator. + structured.EnableAllocators("experimental") + defer structured.EnableAllocators() + + perf.RunBenchmarkPerfScheduling(b, "performance-config.yaml", "dra_experimental", nil) +} diff --git a/test/integration/scheduler_perf/dra/performance-config.yaml b/test/integration/scheduler_perf/dra/performance-config.yaml index 663f4cd183b..432c05e3253 100644 --- a/test/integration/scheduler_perf/dra/performance-config.yaml +++ b/test/integration/scheduler_perf/dra/performance-config.yaml @@ -130,8 +130,9 @@ featureGates: SchedulerQueueingHints: false labels: [performance] - # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSchedulingWithResourceClaimTemplate%2F5000pods_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable - threshold: 40 # typically above 51 + threshold: + # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSchedulingWithResourceClaimTemplate%2F5000pods_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable + dra: 40 # typically above 51 params: nodesWithDRA: 500 nodesWithoutDRA: 0 @@ -142,8 +143,9 @@ featureGates: SchedulerQueueingHints: true labels: [performance] - # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSchedulingWithResourceClaimTemplate%2F5000pods_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable - threshold: 56 # typically above 70 + threshold: + # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSchedulingWithResourceClaimTemplate%2F5000pods_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable + dra: 56 # typically above 70 params: nodesWithDRA: 500 nodesWithoutDRA: 0 @@ -208,8 +210,9 @@ featureGates: SchedulerQueueingHints: false labels: [performance] - # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fempty_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable - threshold: 64 # typically above 81 + threshold: + # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fempty_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable + dra: 64 # typically above 81 params: nodesWithDRA: 500 nodesWithoutDRA: 0 @@ -220,8 +223,9 @@ featureGates: SchedulerQueueingHints: true labels: [performance] - # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fempty_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable - threshold: 64 # typically above 81 + threshold: + # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fempty_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable + dra: 64 # typically above 81 params: nodesWithDRA: 500 nodesWithoutDRA: 0 @@ -247,8 +251,9 @@ featureGates: SchedulerQueueingHints: false labels: [performance] - # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fhalf_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable - threshold: 70 # typically over 78 + threshold: + # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fhalf_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable + dra: 70 # typically over 78 params: nodesWithDRA: 500 nodesWithoutDRA: 0 @@ -259,8 +264,9 @@ featureGates: SchedulerQueueingHints: true labels: [performance] - # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fhalf_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable - threshold: 60 # typically over 75 + threshold: + # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fhalf_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable + dra: 60 # typically over 75 params: nodesWithDRA: 500 nodesWithoutDRA: 0 @@ -286,8 +292,9 @@ featureGates: SchedulerQueueingHints: false labels: [performance] - # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Ffull_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable - threshold: 60 # typically over 75 + threshold: + # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Ffull_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable + dra: 60 # typically over 75 params: nodesWithDRA: 500 nodesWithoutDRA: 0 @@ -298,8 +305,9 @@ featureGates: SchedulerQueueingHints: true labels: [performance] - # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Ffull_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable - threshold: 60 # typically over 75 + threshold: + # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Ffull_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable + dra: 60 # typically over 75 params: nodesWithDRA: 500 nodesWithoutDRA: 0 diff --git a/test/integration/scheduler_perf/scheduler_perf.go b/test/integration/scheduler_perf/scheduler_perf.go index 57d3f7553b6..ba3e2143090 100644 --- a/test/integration/scheduler_perf/scheduler_perf.go +++ b/test/integration/scheduler_perf/scheduler_perf.go @@ -332,8 +332,14 @@ type workload struct { // The comparison is performed for op with CollectMetrics set to true. // If the measured value is below the threshold, the workload's test case will fail. // If set to zero, the threshold check is disabled. + // + // May contain a single value or map of topic name to value. + // The single value is used if there is no entry in the map for the topic name. + // Topic names are passed to RunBenchmarkPerfScheduling. This approach + // makes it possible to reuse the same test cases in different configurations. + // // Optional - Threshold float64 + Threshold thresholds // ThresholdMetricSelector defines to what metric the Threshold should be compared. // If nil, the metric is set to DefaultThresholdMetricSelector of the testCase. // If DefaultThresholdMetricSelector is nil, the metric is set to "SchedulingThroughput". @@ -346,8 +352,13 @@ type workload struct { } func (w *workload) isValid(mcc *metricsCollectorConfig) error { - if w.Threshold < 0 { - return fmt.Errorf("invalid Threshold=%f; should be non-negative", w.Threshold) + if w.Threshold.value < 0 { + return fmt.Errorf("invalid Threshold=%f; should be non-negative", w.Threshold.value) + } + for topicName, value := range w.Threshold.valuesByTopic { + if value < 0 { + return fmt.Errorf("invalid Threshold=%f for topic %q; should be non-negative", value, topicName) + } } return w.ThresholdMetricSelector.isValid(mcc) @@ -367,6 +378,28 @@ func (w *workload) setDefaults(testCaseThresholdMetricSelector *thresholdMetricS } } +type thresholds struct { + value float64 + valuesByTopic map[string]float64 +} + +func (t *thresholds) UnmarshalJSON(text []byte) error { + if errFloat64 := json.Unmarshal(text, &t.value); errFloat64 != nil { + // Not a plain number. Let's try as map. + if errMap := json.Unmarshal(text, &t.valuesByTopic); errMap != nil { + return fmt.Errorf("expected either float64 or topic name -> float64 map: %w, %w", errFloat64, errMap) + } + } + return nil +} + +func (t *thresholds) Get(topicName string) float64 { + if value, ok := t.valuesByTopic[topicName]; ok { + return value + } + return t.value +} + // thresholdMetricSelector defines the name and labels of metric to compare with threshold. type thresholdMetricSelector struct { // Name of the metric is compared to "Metric" field in DataItem labels. @@ -1232,7 +1265,7 @@ func RunBenchmarkPerfScheduling(b *testing.B, configFile string, topicName strin } } - results, err := runWorkload(tCtx, tc, w, scheduler, informerFactory) + results, err := runWorkload(tCtx, tc, w, topicName, scheduler, informerFactory) if err != nil { tCtx.Fatalf("Error running workload %s: %s", w.Name, err) } @@ -1299,7 +1332,7 @@ func RunBenchmarkPerfScheduling(b *testing.B, configFile string, topicName strin } }) } - if err := dataItems2JSONFile(dataItems, b.Name()+"_benchmark_"+topicName); err != nil { + if err := dataItems2JSONFile(dataItems, strings.ReplaceAll(b.Name(), "/", "_")+"_benchmark_"+topicName); err != nil { b.Fatalf("unable to write measured data %+v: %v", dataItems, err) } } @@ -1333,7 +1366,7 @@ func RunIntegrationPerfScheduling(t *testing.T, configFile string) { t.Fatalf("workload %s is not valid: %v", w.Name, err) } - _, err = runWorkload(tCtx, tc, w, scheduler, informerFactory) + _, err = runWorkload(tCtx, tc, w, "" /* topic name not relevant */, scheduler, informerFactory) if err != nil { tCtx.Fatalf("Error running workload %s: %s", w.Name, err) } @@ -1522,10 +1555,11 @@ type WorkloadExecutor struct { throughputErrorMargin float64 testCase *testCase workload *workload + topicName string nextNodeIndex int } -func runWorkload(tCtx ktesting.TContext, tc *testCase, w *workload, scheduler *scheduler.Scheduler, informerFactory informers.SharedInformerFactory) ([]DataItem, error) { +func runWorkload(tCtx ktesting.TContext, tc *testCase, w *workload, topicName string, scheduler *scheduler.Scheduler, informerFactory informers.SharedInformerFactory) ([]DataItem, error) { b, benchmarking := tCtx.TB().(*testing.B) if benchmarking { start := time.Now() @@ -1565,6 +1599,7 @@ func runWorkload(tCtx ktesting.TContext, tc *testCase, w *workload, scheduler *s throughputErrorMargin: throughputErrorMargin, testCase: tc, workload: w, + topicName: topicName, } tCtx.TB().Cleanup(func() { @@ -1702,7 +1737,7 @@ func (e *WorkloadExecutor) runSleepOp(op *sleepOp) error { } func (e *WorkloadExecutor) runStopCollectingMetrics(opIndex int) error { - items, err := stopCollectingMetrics(e.tCtx, e.collectorCtx, &e.collectorWG, e.workload.Threshold, *e.workload.ThresholdMetricSelector, opIndex, e.collectors) + items, err := stopCollectingMetrics(e.tCtx, e.collectorCtx, &e.collectorWG, e.workload.Threshold.Get(e.topicName), *e.workload.ThresholdMetricSelector, opIndex, e.collectors) if err != nil { return err } @@ -1756,7 +1791,7 @@ func (e *WorkloadExecutor) runCreatePodsOp(opIndex int, op *createPodsOp) error // CollectMetrics and SkipWaitToCompletion can never be true at the // same time, so if we're here, it means that all pods have been // scheduled. - items, err := stopCollectingMetrics(e.tCtx, e.collectorCtx, &e.collectorWG, e.workload.Threshold, *e.workload.ThresholdMetricSelector, opIndex, e.collectors) + items, err := stopCollectingMetrics(e.tCtx, e.collectorCtx, &e.collectorWG, e.workload.Threshold.Get(e.topicName), *e.workload.ThresholdMetricSelector, opIndex, e.collectors) if err != nil { return err }