Merge pull request #104391 from chendave/metric_new

Add the metric data for different extension points

staging/src/k8s.io/component-base/metrics/testutil/metrics.go

@@ -269,10 +269,6 @@ func GetHistogramVecFromGatherer(gatherer metrics.Gatherer, metricName string, l
 		return nil, fmt.Errorf("metric %q not found", metricName)
 	}
 
-	if metricFamily.GetMetric() == nil {
-		return nil, fmt.Errorf("metric %q is empty", metricName)
-	}
-
 	if len(metricFamily.GetMetric()) == 0 {
 		return nil, fmt.Errorf("metric %q is empty", metricName)
 	}

staging/src/k8s.io/component-base/metrics/testutil/metrics_test.go

@@ -20,11 +20,14 @@ import (
 	"fmt"
 	"math"
 	"reflect"
+	"sync"
 	"testing"
 
-	"k8s.io/utils/pointer"
+	"github.com/google/go-cmp/cmp"
-
 	dto "github.com/prometheus/client_model/go"
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/utils/pointer"
 )
 
 func samples2Histogram(samples []float64, upperBounds []float64) Histogram {
@@ -512,3 +515,80 @@ func TestHistogramVec_Validate(t *testing.T) {
 		})
 	}
 }
+
+func TestGetHistogramVecFromGatherer(t *testing.T) {
+	var registerMetrics sync.Once
+	tests := []struct {
+		name    string
+		lvMap   map[string]string
+		wantVec HistogramVec
+	}{
+		{
+			name:  "filter with one label",
+			lvMap: map[string]string{"label1": "value1-0"},
+			wantVec: HistogramVec{
+				&Histogram{&dto.Histogram{
+					SampleCount: uint64Ptr(1),
+					SampleSum:   pointer.Float64Ptr(1.5),
+					Bucket: []*dto.Bucket{
+						{CumulativeCount: uint64Ptr(0), UpperBound: pointer.Float64Ptr(0.5)},
+						{CumulativeCount: uint64Ptr(1), UpperBound: pointer.Float64Ptr(2.0)},
+						{CumulativeCount: uint64Ptr(1), UpperBound: pointer.Float64Ptr(5.0)},
+					},
+				}},
+				&Histogram{&dto.Histogram{
+					SampleCount: uint64Ptr(1),
+					SampleSum:   pointer.Float64Ptr(2.5),
+					Bucket: []*dto.Bucket{
+						{CumulativeCount: uint64Ptr(0), UpperBound: pointer.Float64Ptr(0.5)},
+						{CumulativeCount: uint64Ptr(0), UpperBound: pointer.Float64Ptr(2.0)},
+						{CumulativeCount: uint64Ptr(1), UpperBound: pointer.Float64Ptr(5.0)},
+					},
+				}},
+			},
+		},
+		{
+			name:  "filter with two labels",
+			lvMap: map[string]string{"label1": "value1-0", "label2": "value2-1"},
+			wantVec: HistogramVec{
+				&Histogram{&dto.Histogram{
+					SampleCount: uint64Ptr(1),
+					SampleSum:   pointer.Float64Ptr(2.5),
+					Bucket: []*dto.Bucket{
+						{CumulativeCount: uint64Ptr(0), UpperBound: pointer.Float64Ptr(0.5)},
+						{CumulativeCount: uint64Ptr(0), UpperBound: pointer.Float64Ptr(2.0)},
+						{CumulativeCount: uint64Ptr(1), UpperBound: pointer.Float64Ptr(5.0)},
+					},
+				}},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			buckets := []float64{.5, 2, 5}
+			// HistogramVec has two labels defined.
+			labels := []string{"label1", "label2"}
+			HistogramOpts := &metrics.HistogramOpts{
+				Namespace: "namespace",
+				Name:      "metric_test_name",
+				Subsystem: "subsystem",
+				Help:      "histogram help message",
+				Buckets:   buckets,
+			}
+			vec := metrics.NewHistogramVec(HistogramOpts, labels)
+			registerMetrics.Do(func() {
+				legacyregistry.MustRegister(vec)
+			})
+			// Observe two metrics with same value for label1 but different value of label2.
+			vec.WithLabelValues("value1-0", "value2-0").Observe(1.5)
+			vec.WithLabelValues("value1-0", "value2-1").Observe(2.5)
+			vec.WithLabelValues("value1-1", "value2-0").Observe(3.5)
+			vec.WithLabelValues("value1-1", "value2-1").Observe(4.5)
+			metricName := fmt.Sprintf("%s_%s_%s", HistogramOpts.Namespace, HistogramOpts.Subsystem, HistogramOpts.Name)
+			histogramVec, _ := GetHistogramVecFromGatherer(legacyregistry.DefaultGatherer, metricName, tt.lvMap)
+			if diff := cmp.Diff(tt.wantVec, histogramVec); diff != "" {
+				t.Errorf("Got unexpected HistogramVec (-want +got):\n%s", diff)
+			}
+		})
+	}
+}

test/integration/scheduler_perf/scheduler_perf_test.go

@@ -53,13 +53,14 @@ import (
 )
 
 const (
-	configFile             = "config/performance-config.yaml"
+	configFile               = "config/performance-config.yaml"
-	createNodesOpcode      = "createNodes"
+	createNodesOpcode        = "createNodes"
-	createNamespacesOpcode = "createNamespaces"
+	createNamespacesOpcode   = "createNamespaces"
-	createPodsOpcode       = "createPods"
+	createPodsOpcode         = "createPods"
-	createPodSetsOpcode    = "createPodSets"
+	createPodSetsOpcode      = "createPodSets"
-	churnOpcode            = "churn"
+	churnOpcode              = "churn"
-	barrierOpcode          = "barrier"
+	barrierOpcode            = "barrier"
+	extensionPointsLabelName = "extension_point"
 
 	// Two modes supported in "churn" operator.
 
@@ -71,11 +72,13 @@ const (
 
 var (
 	defaultMetricsCollectorConfig = metricsCollectorConfig{
-		Metrics: []string{
+		Metrics: map[string]*labelValues{
-			"scheduler_scheduling_algorithm_predicate_evaluation_seconds",
+			"scheduler_framework_extension_point_duration_seconds": {
-			"scheduler_scheduling_algorithm_priority_evaluation_seconds",
+				label:  extensionPointsLabelName,
-			"scheduler_e2e_scheduling_duration_seconds",
+				values: []string{"Filter", "Score"},
-			"scheduler_pod_scheduling_duration_seconds",
+			},
+			"scheduler_e2e_scheduling_duration_seconds": nil,
+			"scheduler_pod_scheduling_duration_seconds": nil,
 		},
 	}
 )

test/integration/scheduler_perf/util.go

@@ -177,9 +177,15 @@ func dataItems2JSONFile(dataItems DataItems, namePrefix string) error {
 	return ioutil.WriteFile(destFile, b, 0644)
 }
 
+type labelValues struct {
+	label  string
+	values []string
+}
+
 // metricsCollectorConfig is the config to be marshalled to YAML config file.
+// NOTE: The mapping here means only one filter is supported, either value in the list of `values` is able to be collected.
 type metricsCollectorConfig struct {
-	Metrics []string
+	Metrics map[string]*labelValues
 }
 
 // metricsCollector collects metrics from legacyregistry.DefaultGatherer.Gather() endpoint.
@@ -202,17 +208,29 @@ func (*metricsCollector) run(ctx context.Context) {
 
 func (pc *metricsCollector) collect() []DataItem {
 	var dataItems []DataItem
-	for _, metric := range pc.Metrics {
+	for metric, labelVals := range pc.Metrics {
-		dataItem := collectHistogramVec(metric, pc.labels)
+		// no filter is specified, aggregate all the metrics within the same metricFamily.
-		if dataItem != nil {
+		if labelVals == nil {
-			dataItems = append(dataItems, *dataItem)
+			dataItem := collectHistogramVec(metric, pc.labels, nil)
+			if dataItem != nil {
+				dataItems = append(dataItems, *dataItem)
+			}
+		} else {
+			// fetch the metric from metricFamily which match each of the lvMap.
+			for _, value := range labelVals.values {
+				lvMap := map[string]string{labelVals.label: value}
+				dataItem := collectHistogramVec(metric, pc.labels, lvMap)
+				if dataItem != nil {
+					dataItems = append(dataItems, *dataItem)
+				}
+			}
 		}
 	}
 	return dataItems
 }
 
-func collectHistogramVec(metric string, labels map[string]string) *DataItem {
+func collectHistogramVec(metric string, labels map[string]string, lvMap map[string]string) *DataItem {
-	vec, err := testutil.GetHistogramVecFromGatherer(legacyregistry.DefaultGatherer, metric, nil)
+	vec, err := testutil.GetHistogramVecFromGatherer(legacyregistry.DefaultGatherer, metric, lvMap)
 	if err != nil {
 		klog.Error(err)
 		return nil
@@ -236,6 +254,9 @@ func collectHistogramVec(metric string, labels map[string]string) *DataItem {
 	for k, v := range labels {
 		labelMap[k] = v
 	}
+	for k, v := range lvMap {
+		labelMap[k] = v
+	}
 	return &DataItem{
 		Labels: labelMap,
 		Data: map[string]float64{