diff --git a/test/integration/scheduler_perf/dra/dra_test.go b/test/integration/scheduler_perf/dra/dra_test.go index 2ca40591565..bf64abb0877 100644 --- a/test/integration/scheduler_perf/dra/dra_test.go +++ b/test/integration/scheduler_perf/dra/dra_test.go @@ -69,10 +69,25 @@ func TestSchedulerPerf(t *testing.T) { } } +// These benchmarks have to contain the word BenchmarkPerfScheduling to be picked up +// by benchmark jobs. +// +// Sub-tests should have worked, too, but gotestsum was unhappy. + func BenchmarkPerfScheduling(b *testing.B) { // Restrict benchmarking to the default allocator. structured.EnableAllocators("incubating") defer structured.EnableAllocators() + // "dra" is how this was called traditionally. + // It's kept to avoid changing perf-dash results. perf.RunBenchmarkPerfScheduling(b, "performance-config.yaml", "dra", nil) } + +func BenchmarkPerfSchedulingExperimental(b *testing.B) { + // And now the experimental allocator. + structured.EnableAllocators("experimental") + defer structured.EnableAllocators() + + perf.RunBenchmarkPerfScheduling(b, "performance-config.yaml", "dra_experimental", nil) +} diff --git a/test/integration/scheduler_perf/dra/performance-config.yaml b/test/integration/scheduler_perf/dra/performance-config.yaml index 663f4cd183b..432c05e3253 100644 --- a/test/integration/scheduler_perf/dra/performance-config.yaml +++ b/test/integration/scheduler_perf/dra/performance-config.yaml @@ -130,8 +130,9 @@ featureGates: SchedulerQueueingHints: false labels: [performance] - # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSchedulingWithResourceClaimTemplate%2F5000pods_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable - threshold: 40 # typically above 51 + threshold: + # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSchedulingWithResourceClaimTemplate%2F5000pods_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable + dra: 40 # typically above 51 params: nodesWithDRA: 500 nodesWithoutDRA: 0 @@ -142,8 +143,9 @@ featureGates: SchedulerQueueingHints: true labels: [performance] - # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSchedulingWithResourceClaimTemplate%2F5000pods_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable - threshold: 56 # typically above 70 + threshold: + # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSchedulingWithResourceClaimTemplate%2F5000pods_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable + dra: 56 # typically above 70 params: nodesWithDRA: 500 nodesWithoutDRA: 0 @@ -208,8 +210,9 @@ featureGates: SchedulerQueueingHints: false labels: [performance] - # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fempty_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable - threshold: 64 # typically above 81 + threshold: + # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fempty_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable + dra: 64 # typically above 81 params: nodesWithDRA: 500 nodesWithoutDRA: 0 @@ -220,8 +223,9 @@ featureGates: SchedulerQueueingHints: true labels: [performance] - # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fempty_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable - threshold: 64 # typically above 81 + threshold: + # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fempty_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable + dra: 64 # typically above 81 params: nodesWithDRA: 500 nodesWithoutDRA: 0 @@ -247,8 +251,9 @@ featureGates: SchedulerQueueingHints: false labels: [performance] - # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fhalf_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable - threshold: 70 # typically over 78 + threshold: + # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fhalf_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable + dra: 70 # typically over 78 params: nodesWithDRA: 500 nodesWithoutDRA: 0 @@ -259,8 +264,9 @@ featureGates: SchedulerQueueingHints: true labels: [performance] - # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fhalf_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable - threshold: 60 # typically over 75 + threshold: + # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Fhalf_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable + dra: 60 # typically over 75 params: nodesWithDRA: 500 nodesWithoutDRA: 0 @@ -286,8 +292,9 @@ featureGates: SchedulerQueueingHints: false labels: [performance] - # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Ffull_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable - threshold: 60 # typically over 75 + threshold: + # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Ffull_500nodes%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable + dra: 60 # typically over 75 params: nodesWithDRA: 500 nodesWithoutDRA: 0 @@ -298,8 +305,9 @@ featureGates: SchedulerQueueingHints: true labels: [performance] - # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Ffull_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable - threshold: 60 # typically over 75 + threshold: + # https://perf-dash.k8s.io/#/?jobname=scheduler-perf-benchmark&metriccategoryname=Scheduler&metricname=BenchmarkPerfScheduling&Metric=SchedulingThroughput&Name=BenchmarkPerfScheduling%2FSteadyStateClusterResourceClaimTemplate%2Ffull_500nodes_QueueingHintsEnabled%2Ftest&event=not%20applicable&extension_point=not%20applicable&plugin=not%20applicable&result=not%20applicable + dra: 60 # typically over 75 params: nodesWithDRA: 500 nodesWithoutDRA: 0 diff --git a/test/integration/scheduler_perf/scheduler_perf.go b/test/integration/scheduler_perf/scheduler_perf.go index 57d3f7553b6..ba3e2143090 100644 --- a/test/integration/scheduler_perf/scheduler_perf.go +++ b/test/integration/scheduler_perf/scheduler_perf.go @@ -332,8 +332,14 @@ type workload struct { // The comparison is performed for op with CollectMetrics set to true. // If the measured value is below the threshold, the workload's test case will fail. // If set to zero, the threshold check is disabled. + // + // May contain a single value or map of topic name to value. + // The single value is used if there is no entry in the map for the topic name. + // Topic names are passed to RunBenchmarkPerfScheduling. This approach + // makes it possible to reuse the same test cases in different configurations. + // // Optional - Threshold float64 + Threshold thresholds // ThresholdMetricSelector defines to what metric the Threshold should be compared. // If nil, the metric is set to DefaultThresholdMetricSelector of the testCase. // If DefaultThresholdMetricSelector is nil, the metric is set to "SchedulingThroughput". @@ -346,8 +352,13 @@ type workload struct { } func (w *workload) isValid(mcc *metricsCollectorConfig) error { - if w.Threshold < 0 { - return fmt.Errorf("invalid Threshold=%f; should be non-negative", w.Threshold) + if w.Threshold.value < 0 { + return fmt.Errorf("invalid Threshold=%f; should be non-negative", w.Threshold.value) + } + for topicName, value := range w.Threshold.valuesByTopic { + if value < 0 { + return fmt.Errorf("invalid Threshold=%f for topic %q; should be non-negative", value, topicName) + } } return w.ThresholdMetricSelector.isValid(mcc) @@ -367,6 +378,28 @@ func (w *workload) setDefaults(testCaseThresholdMetricSelector *thresholdMetricS } } +type thresholds struct { + value float64 + valuesByTopic map[string]float64 +} + +func (t *thresholds) UnmarshalJSON(text []byte) error { + if errFloat64 := json.Unmarshal(text, &t.value); errFloat64 != nil { + // Not a plain number. Let's try as map. + if errMap := json.Unmarshal(text, &t.valuesByTopic); errMap != nil { + return fmt.Errorf("expected either float64 or topic name -> float64 map: %w, %w", errFloat64, errMap) + } + } + return nil +} + +func (t *thresholds) Get(topicName string) float64 { + if value, ok := t.valuesByTopic[topicName]; ok { + return value + } + return t.value +} + // thresholdMetricSelector defines the name and labels of metric to compare with threshold. type thresholdMetricSelector struct { // Name of the metric is compared to "Metric" field in DataItem labels. @@ -1232,7 +1265,7 @@ func RunBenchmarkPerfScheduling(b *testing.B, configFile string, topicName strin } } - results, err := runWorkload(tCtx, tc, w, scheduler, informerFactory) + results, err := runWorkload(tCtx, tc, w, topicName, scheduler, informerFactory) if err != nil { tCtx.Fatalf("Error running workload %s: %s", w.Name, err) } @@ -1299,7 +1332,7 @@ func RunBenchmarkPerfScheduling(b *testing.B, configFile string, topicName strin } }) } - if err := dataItems2JSONFile(dataItems, b.Name()+"_benchmark_"+topicName); err != nil { + if err := dataItems2JSONFile(dataItems, strings.ReplaceAll(b.Name(), "/", "_")+"_benchmark_"+topicName); err != nil { b.Fatalf("unable to write measured data %+v: %v", dataItems, err) } } @@ -1333,7 +1366,7 @@ func RunIntegrationPerfScheduling(t *testing.T, configFile string) { t.Fatalf("workload %s is not valid: %v", w.Name, err) } - _, err = runWorkload(tCtx, tc, w, scheduler, informerFactory) + _, err = runWorkload(tCtx, tc, w, "" /* topic name not relevant */, scheduler, informerFactory) if err != nil { tCtx.Fatalf("Error running workload %s: %s", w.Name, err) } @@ -1522,10 +1555,11 @@ type WorkloadExecutor struct { throughputErrorMargin float64 testCase *testCase workload *workload + topicName string nextNodeIndex int } -func runWorkload(tCtx ktesting.TContext, tc *testCase, w *workload, scheduler *scheduler.Scheduler, informerFactory informers.SharedInformerFactory) ([]DataItem, error) { +func runWorkload(tCtx ktesting.TContext, tc *testCase, w *workload, topicName string, scheduler *scheduler.Scheduler, informerFactory informers.SharedInformerFactory) ([]DataItem, error) { b, benchmarking := tCtx.TB().(*testing.B) if benchmarking { start := time.Now() @@ -1565,6 +1599,7 @@ func runWorkload(tCtx ktesting.TContext, tc *testCase, w *workload, scheduler *s throughputErrorMargin: throughputErrorMargin, testCase: tc, workload: w, + topicName: topicName, } tCtx.TB().Cleanup(func() { @@ -1702,7 +1737,7 @@ func (e *WorkloadExecutor) runSleepOp(op *sleepOp) error { } func (e *WorkloadExecutor) runStopCollectingMetrics(opIndex int) error { - items, err := stopCollectingMetrics(e.tCtx, e.collectorCtx, &e.collectorWG, e.workload.Threshold, *e.workload.ThresholdMetricSelector, opIndex, e.collectors) + items, err := stopCollectingMetrics(e.tCtx, e.collectorCtx, &e.collectorWG, e.workload.Threshold.Get(e.topicName), *e.workload.ThresholdMetricSelector, opIndex, e.collectors) if err != nil { return err } @@ -1756,7 +1791,7 @@ func (e *WorkloadExecutor) runCreatePodsOp(opIndex int, op *createPodsOp) error // CollectMetrics and SkipWaitToCompletion can never be true at the // same time, so if we're here, it means that all pods have been // scheduled. - items, err := stopCollectingMetrics(e.tCtx, e.collectorCtx, &e.collectorWG, e.workload.Threshold, *e.workload.ThresholdMetricSelector, opIndex, e.collectors) + items, err := stopCollectingMetrics(e.tCtx, e.collectorCtx, &e.collectorWG, e.workload.Threshold.Get(e.topicName), *e.workload.ThresholdMetricSelector, opIndex, e.collectors) if err != nil { return err }