Merge pull request #134350 from macsko/add_scheduling_duration_collector

Add scheduling duration collector to scheduler_perf

test/integration/scheduler_perf/scheduler_perf.go

@@ -293,7 +293,7 @@ type testCase struct {
 	Labels []string
 	// DefaultThresholdMetricSelector defines default metric used for threshold comparison.
 	// It is only populated to workloads without their ThresholdMetricSelector set.
-	// If nil, the default metric is set to "SchedulingThroughput".
+	// If nil, the default metric is set to "SchedulingThroughput" with "Average" data bucket.
 	// Optional
 	DefaultThresholdMetricSelector *thresholdMetricSelector
 }
@@ -372,9 +372,10 @@ func (w *workload) setDefaults(testCaseThresholdMetricSelector *thresholdMetricS
 		w.ThresholdMetricSelector = testCaseThresholdMetricSelector
 		return
 	}
-	// By default, SchedulingThroughput should be compared with the threshold.
+	// By default, SchedulingThroughput Average should be compared with the threshold.
 	w.ThresholdMetricSelector = &thresholdMetricSelector{
-		Name: "SchedulingThroughput",
+		Name:       "SchedulingThroughput",
+		DataBucket: "Average",
 	}
 }
 
@@ -406,6 +407,8 @@ type thresholdMetricSelector struct {
 	Name string
 	// Labels of the metric. All of them needs to match the metric's labels to assume equality.
 	Labels map[string]string
+	// DataBucket specifies which data bucket should be compared against the threshold.
+	DataBucket string
 	// ExpectLower defines whether the threshold should denote the maximum allowable value of the metric.
 	// If false, the threshold defines minimum allowable value.
 	// Optional
@@ -413,6 +416,9 @@ type thresholdMetricSelector struct {
 }
 
 func (ms thresholdMetricSelector) isValid(mcc *metricsCollectorConfig) error {
+	if ms.DataBucket == "" {
+		return fmt.Errorf("dataBucket should be set for metric %v", ms.Name)
+	}
 	if ms.Name == "SchedulingThroughput" {
 		return nil
 	}
@@ -1470,12 +1476,20 @@ func compareMetricWithThreshold(items []DataItem, threshold float64, metricSelec
 	if threshold == 0 {
 		return nil
 	}
+	dataBucket := metricSelector.DataBucket
 	for _, item := range items {
-		if item.Labels["Metric"] == metricSelector.Name && labelsMatch(item.Labels, metricSelector.Labels) && !valueWithinThreshold(item.Data["Average"], threshold, metricSelector.ExpectLower) {
+		if item.Labels["Metric"] != metricSelector.Name || !labelsMatch(item.Labels, metricSelector.Labels) {
+			continue
+		}
+		dataItem, ok := item.Data[dataBucket]
+		if !ok {
+			return fmt.Errorf("%s: no data present for %q metric %q bucket", item.Labels["Name"], metricSelector.Name, dataBucket)
+		}
+		if !valueWithinThreshold(dataItem, threshold, metricSelector.ExpectLower) {
 			if metricSelector.ExpectLower {
-				return fmt.Errorf("%s: expected %s Average to be lower: got %f, want %f", item.Labels["Name"], metricSelector.Name, item.Data["Average"], threshold)
+				return fmt.Errorf("%s: expected %q %q to be lower: got %f, want %f", item.Labels["Name"], metricSelector.Name, dataBucket, dataItem, threshold)
 			}
-			return fmt.Errorf("%s: expected %s Average to be higher: got %f, want %f", item.Labels["Name"], metricSelector.Name, item.Data["Average"], threshold)
+			return fmt.Errorf("%s: expected %q %q to be higher: got %f, want %f", item.Labels["Name"], metricSelector.Name, dataBucket, dataItem, threshold)
 		}
 	}
 	return nil
@@ -2021,6 +2035,7 @@ func getTestDataCollectors(podInformer coreinformers.PodInformer, name string, n
 		newThroughputCollector(podInformer, map[string]string{"Name": name}, labelSelector, namespaces, throughputErrorMargin),
 		newMetricsCollector(mcc, map[string]string{"Name": name}),
 		newMemoryCollector(map[string]string{"Name": name}, 500*time.Millisecond),
+		newSchedulingDurationCollector(map[string]string{"Name": name}),
 	}
 }
 

test/integration/scheduler_perf/scheduler_perf_test.go

@@ -496,3 +496,196 @@ func TestFeatureGatesMerge(t *testing.T) {
 		})
 	}
 }
+
+func TestCompareMetricWithThreshold(t *testing.T) {
+	tests := []struct {
+		name      string
+		items     []DataItem
+		threshold float64
+		selector  thresholdMetricSelector
+		wantErr   bool
+	}{
+		{
+			name:      "no items, should pass",
+			items:     []DataItem{},
+			threshold: 50,
+			selector: thresholdMetricSelector{
+				Name:       "TargetMetric",
+				DataBucket: "Average",
+			},
+			wantErr: false,
+		},
+		{
+			name: "zero threshold, should always pass",
+			items: []DataItem{
+				{
+					Labels: map[string]string{"Metric": "TargetMetric"},
+					Data:   map[string]float64{"Average": 10},
+				},
+			},
+			threshold: 0,
+			selector: thresholdMetricSelector{
+				Name:        "TargetMetric",
+				DataBucket:  "Average",
+				ExpectLower: true,
+			},
+			wantErr: false,
+		},
+		{
+			name: "metric not found in items, should pass (ignored)",
+			items: []DataItem{
+				{
+					Labels: map[string]string{"Metric": "OtherMetric"},
+					Data:   map[string]float64{"Average": 10},
+				},
+			},
+			threshold: 50,
+			selector: thresholdMetricSelector{
+				Name:       "TargetMetric",
+				DataBucket: "Average",
+			},
+			wantErr: false,
+		},
+		{
+			name: "labels do not match, should pass (ignored)",
+			items: []DataItem{
+				{
+					Labels: map[string]string{"Metric": "TargetMetric", "plugin": "foo"},
+					Data:   map[string]float64{"Average": 10},
+				},
+			},
+			threshold: 50,
+			selector: thresholdMetricSelector{
+				Name:       "TargetMetric",
+				Labels:     map[string]string{"plugin": "bar"},
+				DataBucket: "Average",
+			},
+			wantErr: false,
+		},
+		{
+			name: "labels match, value lower than threshold (ExpectLower=false), should fail",
+			items: []DataItem{
+				{
+					Labels: map[string]string{"Metric": "TargetMetric", "plugin": "foo"},
+					Data:   map[string]float64{"Average": 10},
+				},
+			},
+			threshold: 50,
+			selector: thresholdMetricSelector{
+				Name:       "TargetMetric",
+				Labels:     map[string]string{"plugin": "foo"},
+				DataBucket: "Average",
+			},
+			wantErr: true,
+		},
+		{
+			name: "missing data bucket in item, should fail",
+			items: []DataItem{
+				{
+					Labels: map[string]string{"Metric": "TargetMetric"},
+					Data:   map[string]float64{"Average": 100},
+				},
+			},
+			threshold: 50,
+			selector: thresholdMetricSelector{
+				Name:       "TargetMetric",
+				DataBucket: "99Perc",
+			},
+			wantErr: true,
+		},
+		{
+			name: "value higher than threshold (ExpectLower=false), should pass",
+			items: []DataItem{
+				{
+					Labels: map[string]string{"Metric": "Throughput"},
+					Data:   map[string]float64{"Average": 100},
+				},
+			},
+			threshold: 50,
+			selector: thresholdMetricSelector{
+				Name:        "Throughput",
+				DataBucket:  "Average",
+				ExpectLower: false,
+			},
+			wantErr: false,
+		},
+		{
+			name: "value lower than threshold (ExpectLower=false), should fail",
+			items: []DataItem{
+				{
+					Labels: map[string]string{"Metric": "Throughput"},
+					Data:   map[string]float64{"Average": 10},
+				},
+			},
+			threshold: 50,
+			selector: thresholdMetricSelector{
+				Name:        "Throughput",
+				DataBucket:  "Average",
+				ExpectLower: false,
+			},
+			wantErr: true,
+		},
+		{
+			name: "value lower than threshold (ExpectLower=true), should pass",
+			items: []DataItem{
+				{
+					Labels: map[string]string{"Metric": "Latency"},
+					Data:   map[string]float64{"Average": 10},
+				},
+			},
+			threshold: 50,
+			selector: thresholdMetricSelector{
+				Name:        "Latency",
+				DataBucket:  "Average",
+				ExpectLower: true,
+			},
+			wantErr: false,
+		},
+		{
+			name: "value higher than threshold (ExpectLower=true), should fail",
+			items: []DataItem{
+				{
+					Labels: map[string]string{"Metric": "Latency"},
+					Data:   map[string]float64{"Average": 100},
+				},
+			},
+			threshold: 50,
+			selector: thresholdMetricSelector{
+				Name:        "Latency",
+				DataBucket:  "Average",
+				ExpectLower: true,
+			},
+			wantErr: true,
+		},
+		{
+			name: "value exactly equals threshold, should fail",
+			items: []DataItem{
+				{
+					Labels: map[string]string{"Metric": "Throughput"},
+					Data:   map[string]float64{"Average": 50},
+				},
+			},
+			threshold: 50,
+			selector: thresholdMetricSelector{
+				Name:       "Throughput",
+				DataBucket: "Average",
+			},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := compareMetricWithThreshold(tt.items, tt.threshold, tt.selector)
+			if err != nil {
+				if !tt.wantErr {
+					t.Errorf("Expected no error in compareMetricWithThreshold, but got: %v", err)
+				}
+			} else {
+				if tt.wantErr {
+					t.Errorf("Expected error %v in compareMetricWithThreshold, but got nil", tt.wantErr)
+				}
+			}
+		})
+	}
+}

test/integration/scheduler_perf/util.go

@@ -779,3 +779,38 @@ func (mc *memoryCollector) collect() []DataItem {
 		growthItem,
 	}
 }
+
+// schedulingDurationCollector calculates the total duration of the scheduling phase, including pod creation.
+type schedulingDurationCollector struct {
+	resultLabels map[string]string
+	duration     time.Duration
+}
+
+func newSchedulingDurationCollector(resultLabels map[string]string) *schedulingDurationCollector {
+	return &schedulingDurationCollector{
+		resultLabels: resultLabels,
+	}
+}
+
+func (sdc *schedulingDurationCollector) init() error {
+	return nil
+}
+
+func (sdc *schedulingDurationCollector) run(tCtx ktesting.TContext) {
+	start := time.Now()
+	// Wait for the scheduling to finish
+	<-tCtx.Done()
+	sdc.duration = time.Since(start)
+}
+
+func (sdc *schedulingDurationCollector) collect() []DataItem {
+	labels := maps.Clone(sdc.resultLabels)
+	labels["Metric"] = "SchedulingDuration"
+	return []DataItem{{
+		Labels: labels,
+		Data: map[string]float64{
+			"Duration": sdc.duration.Seconds(),
+		},
+		Unit: "s",
+	}}
+}