diff --git a/staging/src/k8s.io/component-base/metrics/testutil/metrics.go b/staging/src/k8s.io/component-base/metrics/testutil/metrics.go index 0cfe476a31b..e016b826b21 100644 --- a/staging/src/k8s.io/component-base/metrics/testutil/metrics.go +++ b/staging/src/k8s.io/component-base/metrics/testutil/metrics.go @@ -249,10 +249,10 @@ func (vec HistogramVec) Validate() error { return nil } -// GetHistogramVecFromGatherer collects a metric, that matches the input labelValue map, +// GetMetricFamilyFromGatherer collects a metric family, that matches the input metricName, // from a gatherer implementing k8s.io/component-base/metrics.Gatherer interface. // Used only for testing purposes where we need to gather metrics directly from a running binary (without metrics endpoint). -func GetHistogramVecFromGatherer(gatherer metrics.Gatherer, metricName string, lvMap map[string]string) (HistogramVec, error) { +func GetMetricFamilyFromGatherer(gatherer metrics.Gatherer, metricName string) (*dto.MetricFamily, error) { var metricFamily *dto.MetricFamily m, err := gatherer.Gather() if err != nil { @@ -269,14 +269,15 @@ func GetHistogramVecFromGatherer(gatherer metrics.Gatherer, metricName string, l return nil, fmt.Errorf("metric %q not found", metricName) } - if metricFamily.GetMetric() == nil { - return nil, fmt.Errorf("metric %q is empty", metricName) - } - if len(metricFamily.GetMetric()) == 0 { return nil, fmt.Errorf("metric %q is empty", metricName) } + return metricFamily, nil +} + +// GetHistogramVec collects a vector of Histogram from a metric family, that matches the input labelValue map. +func GetHistogramVec(metricFamily *dto.MetricFamily, lvMap map[string]string) HistogramVec { vec := make(HistogramVec, 0) for _, metric := range metricFamily.GetMetric() { if LabelsMatch(metric, lvMap) { @@ -285,7 +286,7 @@ func GetHistogramVecFromGatherer(gatherer metrics.Gatherer, metricName string, l } } } - return vec, nil + return vec } func uint64Ptr(u uint64) *uint64 { diff --git a/staging/src/k8s.io/component-base/metrics/testutil/metrics_test.go b/staging/src/k8s.io/component-base/metrics/testutil/metrics_test.go index e0437333fa7..dd792854eaf 100644 --- a/staging/src/k8s.io/component-base/metrics/testutil/metrics_test.go +++ b/staging/src/k8s.io/component-base/metrics/testutil/metrics_test.go @@ -20,11 +20,14 @@ import ( "fmt" "math" "reflect" + "sync" "testing" - "k8s.io/utils/pointer" - + "github.com/google/go-cmp/cmp" dto "github.com/prometheus/client_model/go" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" + "k8s.io/utils/pointer" ) func samples2Histogram(samples []float64, upperBounds []float64) Histogram { @@ -512,3 +515,81 @@ func TestHistogramVec_Validate(t *testing.T) { }) } } + +func TestGetHistogramVecFromGatherer(t *testing.T) { + var registerMetrics sync.Once + tests := []struct { + name string + lvMap map[string]string + wantVec HistogramVec + }{ + { + name: "filter with one label", + lvMap: map[string]string{"label1": "value1-0"}, + wantVec: HistogramVec{ + &Histogram{&dto.Histogram{ + SampleCount: uint64Ptr(1), + SampleSum: pointer.Float64Ptr(1.5), + Bucket: []*dto.Bucket{ + {CumulativeCount: uint64Ptr(0), UpperBound: pointer.Float64Ptr(0.5)}, + {CumulativeCount: uint64Ptr(1), UpperBound: pointer.Float64Ptr(2.0)}, + {CumulativeCount: uint64Ptr(1), UpperBound: pointer.Float64Ptr(5.0)}, + }, + }}, + &Histogram{&dto.Histogram{ + SampleCount: uint64Ptr(1), + SampleSum: pointer.Float64Ptr(2.5), + Bucket: []*dto.Bucket{ + {CumulativeCount: uint64Ptr(0), UpperBound: pointer.Float64Ptr(0.5)}, + {CumulativeCount: uint64Ptr(0), UpperBound: pointer.Float64Ptr(2.0)}, + {CumulativeCount: uint64Ptr(1), UpperBound: pointer.Float64Ptr(5.0)}, + }, + }}, + }, + }, + { + name: "filter with two labels", + lvMap: map[string]string{"label1": "value1-0", "label2": "value2-1"}, + wantVec: HistogramVec{ + &Histogram{&dto.Histogram{ + SampleCount: uint64Ptr(1), + SampleSum: pointer.Float64Ptr(2.5), + Bucket: []*dto.Bucket{ + {CumulativeCount: uint64Ptr(0), UpperBound: pointer.Float64Ptr(0.5)}, + {CumulativeCount: uint64Ptr(0), UpperBound: pointer.Float64Ptr(2.0)}, + {CumulativeCount: uint64Ptr(1), UpperBound: pointer.Float64Ptr(5.0)}, + }, + }}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buckets := []float64{.5, 2, 5} + // HistogramVec has two labels defined. + labels := []string{"label1", "label2"} + HistogramOpts := &metrics.HistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "histogram help message", + Buckets: buckets, + } + vec := metrics.NewHistogramVec(HistogramOpts, labels) + registerMetrics.Do(func() { + legacyregistry.MustRegister(vec) + }) + // Observe two metrics with same value for label1 but different value of label2. + vec.WithLabelValues("value1-0", "value2-0").Observe(1.5) + vec.WithLabelValues("value1-0", "value2-1").Observe(2.5) + vec.WithLabelValues("value1-1", "value2-0").Observe(3.5) + vec.WithLabelValues("value1-1", "value2-1").Observe(4.5) + metricName := fmt.Sprintf("%s_%s_%s", HistogramOpts.Namespace, HistogramOpts.Subsystem, HistogramOpts.Name) + metricFamily, _ := GetMetricFamilyFromGatherer(legacyregistry.DefaultGatherer, metricName) + histogramVec := GetHistogramVec(metricFamily, tt.lvMap) + if diff := cmp.Diff(tt.wantVec, histogramVec); diff != "" { + t.Errorf("Got unexpected HistogramVec (-want +got):\n%s", diff) + } + }) + } +} diff --git a/test/integration/scheduler_perf/scheduler_perf_test.go b/test/integration/scheduler_perf/scheduler_perf_test.go index 085fa693e3f..7ae2a60db4a 100644 --- a/test/integration/scheduler_perf/scheduler_perf_test.go +++ b/test/integration/scheduler_perf/scheduler_perf_test.go @@ -53,13 +53,14 @@ import ( ) const ( - configFile = "config/performance-config.yaml" - createNodesOpcode = "createNodes" - createNamespacesOpcode = "createNamespaces" - createPodsOpcode = "createPods" - createPodSetsOpcode = "createPodSets" - churnOpcode = "churn" - barrierOpcode = "barrier" + configFile = "config/performance-config.yaml" + createNodesOpcode = "createNodes" + createNamespacesOpcode = "createNamespaces" + createPodsOpcode = "createPods" + createPodSetsOpcode = "createPodSets" + churnOpcode = "churn" + barrierOpcode = "barrier" + extensionPointsLabelName = "extension_point" // Two modes supported in "churn" operator. @@ -71,11 +72,13 @@ const ( var ( defaultMetricsCollectorConfig = metricsCollectorConfig{ - Metrics: []string{ - "scheduler_scheduling_algorithm_predicate_evaluation_seconds", - "scheduler_scheduling_algorithm_priority_evaluation_seconds", - "scheduler_e2e_scheduling_duration_seconds", - "scheduler_pod_scheduling_duration_seconds", + Metrics: map[string]*labelValues{ + "scheduler_framework_extension_point_duration_seconds": { + label: extensionPointsLabelName, + values: []string{"Filter", "Score"}, + }, + "scheduler_e2e_scheduling_duration_seconds": nil, + "scheduler_pod_scheduling_duration_seconds": nil, }, } ) diff --git a/test/integration/scheduler_perf/util.go b/test/integration/scheduler_perf/util.go index f43e29e78e3..91eb385b90f 100644 --- a/test/integration/scheduler_perf/util.go +++ b/test/integration/scheduler_perf/util.go @@ -177,9 +177,15 @@ func dataItems2JSONFile(dataItems DataItems, namePrefix string) error { return ioutil.WriteFile(destFile, b, 0644) } +type labelValues struct { + label string + values []string +} + // metricsCollectorConfig is the config to be marshalled to YAML config file. +// NOTE: The mapping here means only one filter is supported, either value in the list of `values` is able to be collected. type metricsCollectorConfig struct { - Metrics []string + Metrics map[string]*labelValues } // metricsCollector collects metrics from legacyregistry.DefaultGatherer.Gather() endpoint. @@ -202,22 +208,38 @@ func (*metricsCollector) run(ctx context.Context) { func (pc *metricsCollector) collect() []DataItem { var dataItems []DataItem - for _, metric := range pc.Metrics { - dataItem := collectHistogramVec(metric, pc.labels) - if dataItem != nil { - dataItems = append(dataItems, *dataItem) + for name, labelVals := range pc.Metrics { + metricFamily, err := testutil.GetMetricFamilyFromGatherer(legacyregistry.DefaultGatherer, name) + if err != nil { + klog.Error(err) + return dataItems + } + // no filter is specified, aggregate all the metrics within the same metricFamily. + if labelVals == nil { + vec := testutil.GetHistogramVec(metricFamily, nil) + dataItem := collectHistogramData(vec, pc.labels, nil, name) + if dataItem != nil { + dataItems = append(dataItems, *dataItem) + } + } else { + // fetch the metric from metricFamily which match each of the lvMap. + for _, value := range labelVals.values { + lvMap := map[string]string{labelVals.label: value} + vec := testutil.GetHistogramVec(metricFamily, lvMap) + dataItem := collectHistogramData(vec, pc.labels, lvMap, name) + if dataItem != nil { + dataItems = append(dataItems, *dataItem) + } + } } } return dataItems } -func collectHistogramVec(metric string, labels map[string]string) *DataItem { - vec, err := testutil.GetHistogramVecFromGatherer(legacyregistry.DefaultGatherer, metric, nil) - if err != nil { - klog.Error(err) +func collectHistogramData(vec testutil.HistogramVec, labels, lvMap map[string]string, metric string) *DataItem { + if len(vec) == 0 { return nil } - if err := vec.Validate(); err != nil { klog.Error(err) return nil @@ -236,6 +258,9 @@ func collectHistogramVec(metric string, labels map[string]string) *DataItem { for k, v := range labels { labelMap[k] = v } + for k, v := range lvMap { + labelMap[k] = v + } return &DataItem{ Labels: labelMap, Data: map[string]float64{