From e22a30a13e00e0479adb418e48119d6bbf57084c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Skocze=C5=84?= Date: Wed, 1 Oct 2025 10:14:16 +0000 Subject: [PATCH 1/2] Add scheduling duration collector to scheduler_perf --- .../scheduler_perf/scheduler_perf.go | 1 + test/integration/scheduler_perf/util.go | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/test/integration/scheduler_perf/scheduler_perf.go b/test/integration/scheduler_perf/scheduler_perf.go index 5f0b903b80e..381d8f8bc29 100644 --- a/test/integration/scheduler_perf/scheduler_perf.go +++ b/test/integration/scheduler_perf/scheduler_perf.go @@ -2018,6 +2018,7 @@ func getTestDataCollectors(podInformer coreinformers.PodInformer, name string, n newThroughputCollector(podInformer, map[string]string{"Name": name}, labelSelector, namespaces, throughputErrorMargin), newMetricsCollector(mcc, map[string]string{"Name": name}), newMemoryCollector(map[string]string{"Name": name}, 500*time.Millisecond), + newSchedulingDurationCollector(map[string]string{"Name": name}), } } diff --git a/test/integration/scheduler_perf/util.go b/test/integration/scheduler_perf/util.go index 85c99d84d7f..21e16400805 100644 --- a/test/integration/scheduler_perf/util.go +++ b/test/integration/scheduler_perf/util.go @@ -779,3 +779,38 @@ func (mc *memoryCollector) collect() []DataItem { growthItem, } } + +// schedulingDurationCollector calculates the total duration of the scheduling phase, including pod creation. +type schedulingDurationCollector struct { + resultLabels map[string]string + duration time.Duration +} + +func newSchedulingDurationCollector(resultLabels map[string]string) *schedulingDurationCollector { + return &schedulingDurationCollector{ + resultLabels: resultLabels, + } +} + +func (sdc *schedulingDurationCollector) init() error { + return nil +} + +func (sdc *schedulingDurationCollector) run(tCtx ktesting.TContext) { + start := time.Now() + // Wait for the scheduling to finish + <-tCtx.Done() + sdc.duration = time.Since(start) +} + +func (sdc *schedulingDurationCollector) collect() []DataItem { + labels := maps.Clone(sdc.resultLabels) + labels["Metric"] = "SchedulingDuration" + return []DataItem{{ + Labels: labels, + Data: map[string]float64{ + "Duration": sdc.duration.Seconds(), + }, + Unit: "s", + }} +} From bfc44a42d55168ad99ecb02087c02361c0d150c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Skocze=C5=84?= Date: Fri, 14 Nov 2025 10:36:52 +0000 Subject: [PATCH 2/2] Allow to change scheduler_perf threshold data bucket --- .../scheduler_perf/scheduler_perf.go | 26 ++- .../scheduler_perf/scheduler_perf_test.go | 193 ++++++++++++++++++ 2 files changed, 213 insertions(+), 6 deletions(-) diff --git a/test/integration/scheduler_perf/scheduler_perf.go b/test/integration/scheduler_perf/scheduler_perf.go index 381d8f8bc29..86e88fca48f 100644 --- a/test/integration/scheduler_perf/scheduler_perf.go +++ b/test/integration/scheduler_perf/scheduler_perf.go @@ -293,7 +293,7 @@ type testCase struct { Labels []string // DefaultThresholdMetricSelector defines default metric used for threshold comparison. // It is only populated to workloads without their ThresholdMetricSelector set. - // If nil, the default metric is set to "SchedulingThroughput". + // If nil, the default metric is set to "SchedulingThroughput" with "Average" data bucket. // Optional DefaultThresholdMetricSelector *thresholdMetricSelector } @@ -372,9 +372,10 @@ func (w *workload) setDefaults(testCaseThresholdMetricSelector *thresholdMetricS w.ThresholdMetricSelector = testCaseThresholdMetricSelector return } - // By default, SchedulingThroughput should be compared with the threshold. + // By default, SchedulingThroughput Average should be compared with the threshold. w.ThresholdMetricSelector = &thresholdMetricSelector{ - Name: "SchedulingThroughput", + Name: "SchedulingThroughput", + DataBucket: "Average", } } @@ -406,6 +407,8 @@ type thresholdMetricSelector struct { Name string // Labels of the metric. All of them needs to match the metric's labels to assume equality. Labels map[string]string + // DataBucket specifies which data bucket should be compared against the threshold. + DataBucket string // ExpectLower defines whether the threshold should denote the maximum allowable value of the metric. // If false, the threshold defines minimum allowable value. // Optional @@ -413,6 +416,9 @@ type thresholdMetricSelector struct { } func (ms thresholdMetricSelector) isValid(mcc *metricsCollectorConfig) error { + if ms.DataBucket == "" { + return fmt.Errorf("dataBucket should be set for metric %v", ms.Name) + } if ms.Name == "SchedulingThroughput" { return nil } @@ -1467,12 +1473,20 @@ func compareMetricWithThreshold(items []DataItem, threshold float64, metricSelec if threshold == 0 { return nil } + dataBucket := metricSelector.DataBucket for _, item := range items { - if item.Labels["Metric"] == metricSelector.Name && labelsMatch(item.Labels, metricSelector.Labels) && !valueWithinThreshold(item.Data["Average"], threshold, metricSelector.ExpectLower) { + if item.Labels["Metric"] != metricSelector.Name || !labelsMatch(item.Labels, metricSelector.Labels) { + continue + } + dataItem, ok := item.Data[dataBucket] + if !ok { + return fmt.Errorf("%s: no data present for %q metric %q bucket", item.Labels["Name"], metricSelector.Name, dataBucket) + } + if !valueWithinThreshold(dataItem, threshold, metricSelector.ExpectLower) { if metricSelector.ExpectLower { - return fmt.Errorf("%s: expected %s Average to be lower: got %f, want %f", item.Labels["Name"], metricSelector.Name, item.Data["Average"], threshold) + return fmt.Errorf("%s: expected %q %q to be lower: got %f, want %f", item.Labels["Name"], metricSelector.Name, dataBucket, dataItem, threshold) } - return fmt.Errorf("%s: expected %s Average to be higher: got %f, want %f", item.Labels["Name"], metricSelector.Name, item.Data["Average"], threshold) + return fmt.Errorf("%s: expected %q %q to be higher: got %f, want %f", item.Labels["Name"], metricSelector.Name, dataBucket, dataItem, threshold) } } return nil diff --git a/test/integration/scheduler_perf/scheduler_perf_test.go b/test/integration/scheduler_perf/scheduler_perf_test.go index 0deccf942d5..92c448a25e5 100644 --- a/test/integration/scheduler_perf/scheduler_perf_test.go +++ b/test/integration/scheduler_perf/scheduler_perf_test.go @@ -496,3 +496,196 @@ func TestFeatureGatesMerge(t *testing.T) { }) } } + +func TestCompareMetricWithThreshold(t *testing.T) { + tests := []struct { + name string + items []DataItem + threshold float64 + selector thresholdMetricSelector + wantErr bool + }{ + { + name: "no items, should pass", + items: []DataItem{}, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "TargetMetric", + DataBucket: "Average", + }, + wantErr: false, + }, + { + name: "zero threshold, should always pass", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "TargetMetric"}, + Data: map[string]float64{"Average": 10}, + }, + }, + threshold: 0, + selector: thresholdMetricSelector{ + Name: "TargetMetric", + DataBucket: "Average", + ExpectLower: true, + }, + wantErr: false, + }, + { + name: "metric not found in items, should pass (ignored)", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "OtherMetric"}, + Data: map[string]float64{"Average": 10}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "TargetMetric", + DataBucket: "Average", + }, + wantErr: false, + }, + { + name: "labels do not match, should pass (ignored)", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "TargetMetric", "plugin": "foo"}, + Data: map[string]float64{"Average": 10}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "TargetMetric", + Labels: map[string]string{"plugin": "bar"}, + DataBucket: "Average", + }, + wantErr: false, + }, + { + name: "labels match, value lower than threshold (ExpectLower=false), should fail", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "TargetMetric", "plugin": "foo"}, + Data: map[string]float64{"Average": 10}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "TargetMetric", + Labels: map[string]string{"plugin": "foo"}, + DataBucket: "Average", + }, + wantErr: true, + }, + { + name: "missing data bucket in item, should fail", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "TargetMetric"}, + Data: map[string]float64{"Average": 100}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "TargetMetric", + DataBucket: "99Perc", + }, + wantErr: true, + }, + { + name: "value higher than threshold (ExpectLower=false), should pass", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "Throughput"}, + Data: map[string]float64{"Average": 100}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "Throughput", + DataBucket: "Average", + ExpectLower: false, + }, + wantErr: false, + }, + { + name: "value lower than threshold (ExpectLower=false), should fail", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "Throughput"}, + Data: map[string]float64{"Average": 10}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "Throughput", + DataBucket: "Average", + ExpectLower: false, + }, + wantErr: true, + }, + { + name: "value lower than threshold (ExpectLower=true), should pass", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "Latency"}, + Data: map[string]float64{"Average": 10}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "Latency", + DataBucket: "Average", + ExpectLower: true, + }, + wantErr: false, + }, + { + name: "value higher than threshold (ExpectLower=true), should fail", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "Latency"}, + Data: map[string]float64{"Average": 100}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "Latency", + DataBucket: "Average", + ExpectLower: true, + }, + wantErr: true, + }, + { + name: "value exactly equals threshold, should fail", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "Throughput"}, + Data: map[string]float64{"Average": 50}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "Throughput", + DataBucket: "Average", + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := compareMetricWithThreshold(tt.items, tt.threshold, tt.selector) + if err != nil { + if !tt.wantErr { + t.Errorf("Expected no error in compareMetricWithThreshold, but got: %v", err) + } + } else { + if tt.wantErr { + t.Errorf("Expected error %v in compareMetricWithThreshold, but got nil", tt.wantErr) + } + } + }) + } +}