diff --git a/test/integration/scheduler_perf/scheduler_perf.go b/test/integration/scheduler_perf/scheduler_perf.go index 5a2d4031876..51ceed15790 100644 --- a/test/integration/scheduler_perf/scheduler_perf.go +++ b/test/integration/scheduler_perf/scheduler_perf.go @@ -293,7 +293,7 @@ type testCase struct { Labels []string // DefaultThresholdMetricSelector defines default metric used for threshold comparison. // It is only populated to workloads without their ThresholdMetricSelector set. - // If nil, the default metric is set to "SchedulingThroughput". + // If nil, the default metric is set to "SchedulingThroughput" with "Average" data bucket. // Optional DefaultThresholdMetricSelector *thresholdMetricSelector } @@ -372,9 +372,10 @@ func (w *workload) setDefaults(testCaseThresholdMetricSelector *thresholdMetricS w.ThresholdMetricSelector = testCaseThresholdMetricSelector return } - // By default, SchedulingThroughput should be compared with the threshold. + // By default, SchedulingThroughput Average should be compared with the threshold. w.ThresholdMetricSelector = &thresholdMetricSelector{ - Name: "SchedulingThroughput", + Name: "SchedulingThroughput", + DataBucket: "Average", } } @@ -406,6 +407,8 @@ type thresholdMetricSelector struct { Name string // Labels of the metric. All of them needs to match the metric's labels to assume equality. Labels map[string]string + // DataBucket specifies which data bucket should be compared against the threshold. + DataBucket string // ExpectLower defines whether the threshold should denote the maximum allowable value of the metric. // If false, the threshold defines minimum allowable value. // Optional @@ -413,6 +416,9 @@ type thresholdMetricSelector struct { } func (ms thresholdMetricSelector) isValid(mcc *metricsCollectorConfig) error { + if ms.DataBucket == "" { + return fmt.Errorf("dataBucket should be set for metric %v", ms.Name) + } if ms.Name == "SchedulingThroughput" { return nil } @@ -1470,12 +1476,20 @@ func compareMetricWithThreshold(items []DataItem, threshold float64, metricSelec if threshold == 0 { return nil } + dataBucket := metricSelector.DataBucket for _, item := range items { - if item.Labels["Metric"] == metricSelector.Name && labelsMatch(item.Labels, metricSelector.Labels) && !valueWithinThreshold(item.Data["Average"], threshold, metricSelector.ExpectLower) { + if item.Labels["Metric"] != metricSelector.Name || !labelsMatch(item.Labels, metricSelector.Labels) { + continue + } + dataItem, ok := item.Data[dataBucket] + if !ok { + return fmt.Errorf("%s: no data present for %q metric %q bucket", item.Labels["Name"], metricSelector.Name, dataBucket) + } + if !valueWithinThreshold(dataItem, threshold, metricSelector.ExpectLower) { if metricSelector.ExpectLower { - return fmt.Errorf("%s: expected %s Average to be lower: got %f, want %f", item.Labels["Name"], metricSelector.Name, item.Data["Average"], threshold) + return fmt.Errorf("%s: expected %q %q to be lower: got %f, want %f", item.Labels["Name"], metricSelector.Name, dataBucket, dataItem, threshold) } - return fmt.Errorf("%s: expected %s Average to be higher: got %f, want %f", item.Labels["Name"], metricSelector.Name, item.Data["Average"], threshold) + return fmt.Errorf("%s: expected %q %q to be higher: got %f, want %f", item.Labels["Name"], metricSelector.Name, dataBucket, dataItem, threshold) } } return nil @@ -2021,6 +2035,7 @@ func getTestDataCollectors(podInformer coreinformers.PodInformer, name string, n newThroughputCollector(podInformer, map[string]string{"Name": name}, labelSelector, namespaces, throughputErrorMargin), newMetricsCollector(mcc, map[string]string{"Name": name}), newMemoryCollector(map[string]string{"Name": name}, 500*time.Millisecond), + newSchedulingDurationCollector(map[string]string{"Name": name}), } } diff --git a/test/integration/scheduler_perf/scheduler_perf_test.go b/test/integration/scheduler_perf/scheduler_perf_test.go index 0deccf942d5..92c448a25e5 100644 --- a/test/integration/scheduler_perf/scheduler_perf_test.go +++ b/test/integration/scheduler_perf/scheduler_perf_test.go @@ -496,3 +496,196 @@ func TestFeatureGatesMerge(t *testing.T) { }) } } + +func TestCompareMetricWithThreshold(t *testing.T) { + tests := []struct { + name string + items []DataItem + threshold float64 + selector thresholdMetricSelector + wantErr bool + }{ + { + name: "no items, should pass", + items: []DataItem{}, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "TargetMetric", + DataBucket: "Average", + }, + wantErr: false, + }, + { + name: "zero threshold, should always pass", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "TargetMetric"}, + Data: map[string]float64{"Average": 10}, + }, + }, + threshold: 0, + selector: thresholdMetricSelector{ + Name: "TargetMetric", + DataBucket: "Average", + ExpectLower: true, + }, + wantErr: false, + }, + { + name: "metric not found in items, should pass (ignored)", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "OtherMetric"}, + Data: map[string]float64{"Average": 10}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "TargetMetric", + DataBucket: "Average", + }, + wantErr: false, + }, + { + name: "labels do not match, should pass (ignored)", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "TargetMetric", "plugin": "foo"}, + Data: map[string]float64{"Average": 10}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "TargetMetric", + Labels: map[string]string{"plugin": "bar"}, + DataBucket: "Average", + }, + wantErr: false, + }, + { + name: "labels match, value lower than threshold (ExpectLower=false), should fail", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "TargetMetric", "plugin": "foo"}, + Data: map[string]float64{"Average": 10}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "TargetMetric", + Labels: map[string]string{"plugin": "foo"}, + DataBucket: "Average", + }, + wantErr: true, + }, + { + name: "missing data bucket in item, should fail", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "TargetMetric"}, + Data: map[string]float64{"Average": 100}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "TargetMetric", + DataBucket: "99Perc", + }, + wantErr: true, + }, + { + name: "value higher than threshold (ExpectLower=false), should pass", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "Throughput"}, + Data: map[string]float64{"Average": 100}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "Throughput", + DataBucket: "Average", + ExpectLower: false, + }, + wantErr: false, + }, + { + name: "value lower than threshold (ExpectLower=false), should fail", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "Throughput"}, + Data: map[string]float64{"Average": 10}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "Throughput", + DataBucket: "Average", + ExpectLower: false, + }, + wantErr: true, + }, + { + name: "value lower than threshold (ExpectLower=true), should pass", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "Latency"}, + Data: map[string]float64{"Average": 10}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "Latency", + DataBucket: "Average", + ExpectLower: true, + }, + wantErr: false, + }, + { + name: "value higher than threshold (ExpectLower=true), should fail", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "Latency"}, + Data: map[string]float64{"Average": 100}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "Latency", + DataBucket: "Average", + ExpectLower: true, + }, + wantErr: true, + }, + { + name: "value exactly equals threshold, should fail", + items: []DataItem{ + { + Labels: map[string]string{"Metric": "Throughput"}, + Data: map[string]float64{"Average": 50}, + }, + }, + threshold: 50, + selector: thresholdMetricSelector{ + Name: "Throughput", + DataBucket: "Average", + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := compareMetricWithThreshold(tt.items, tt.threshold, tt.selector) + if err != nil { + if !tt.wantErr { + t.Errorf("Expected no error in compareMetricWithThreshold, but got: %v", err) + } + } else { + if tt.wantErr { + t.Errorf("Expected error %v in compareMetricWithThreshold, but got nil", tt.wantErr) + } + } + }) + } +} diff --git a/test/integration/scheduler_perf/util.go b/test/integration/scheduler_perf/util.go index 85c99d84d7f..21e16400805 100644 --- a/test/integration/scheduler_perf/util.go +++ b/test/integration/scheduler_perf/util.go @@ -779,3 +779,38 @@ func (mc *memoryCollector) collect() []DataItem { growthItem, } } + +// schedulingDurationCollector calculates the total duration of the scheduling phase, including pod creation. +type schedulingDurationCollector struct { + resultLabels map[string]string + duration time.Duration +} + +func newSchedulingDurationCollector(resultLabels map[string]string) *schedulingDurationCollector { + return &schedulingDurationCollector{ + resultLabels: resultLabels, + } +} + +func (sdc *schedulingDurationCollector) init() error { + return nil +} + +func (sdc *schedulingDurationCollector) run(tCtx ktesting.TContext) { + start := time.Now() + // Wait for the scheduling to finish + <-tCtx.Done() + sdc.duration = time.Since(start) +} + +func (sdc *schedulingDurationCollector) collect() []DataItem { + labels := maps.Clone(sdc.resultLabels) + labels["Metric"] = "SchedulingDuration" + return []DataItem{{ + Labels: labels, + Data: map[string]float64{ + "Duration": sdc.duration.Seconds(), + }, + Unit: "s", + }} +}