HPA: expose the metrics "metric_computation_duration_seconds" and "metric_computation_total" from HPA controller

pkg/controller/podautoscaler/horizontal.go

@@ -424,6 +424,29 @@ func (a *HorizontalController) validateAndParseSelector(hpa *autoscalingv2.Horiz
 func (a *HorizontalController) computeReplicasForMetric(ctx context.Context, hpa *autoscalingv2.HorizontalPodAutoscaler, spec autoscalingv2.MetricSpec,
 	specReplicas, statusReplicas int32, selector labels.Selector, status *autoscalingv2.MetricStatus) (replicaCountProposal int32, metricNameProposal string,
 	timestampProposal time.Time, condition autoscalingv2.HorizontalPodAutoscalerCondition, err error) {
+	// actionLabel is used to report which actions this reconciliation has taken.
+	start := time.Now()
+	defer func() {
+		actionLabel := monitor.ActionLabelNone
+		switch {
+		case replicaCountProposal > hpa.Status.CurrentReplicas:
+			actionLabel = monitor.ActionLabelScaleUp
+		case replicaCountProposal < hpa.Status.CurrentReplicas:
+			actionLabel = monitor.ActionLabelScaleDown
+		}
+
+		errorLabel := monitor.ErrorLabelNone
+		if err != nil {
+			// In case of error, set "internal" as default.
+			errorLabel = monitor.ErrorLabelInternal
+			actionLabel = monitor.ActionLabelNone
+		}
+		if errors.Is(err, errSpec) {
+			errorLabel = monitor.ErrorLabelSpec
+		}
+
+		a.monitor.ObserveMetricComputationResult(actionLabel, errorLabel, time.Since(start), spec.Type)
+	}()
 
 	switch spec.Type {
 	case autoscalingv2.ObjectMetricSourceType:

pkg/controller/podautoscaler/monitor/metrics.go

@@ -46,10 +46,27 @@ var (
 			Buckets:        metrics.ExponentialBuckets(0.001, 2, 15),
 			StabilityLevel: metrics.ALPHA,
 		}, []string{"action", "error"})
+	metricComputationTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Subsystem:      hpaControllerSubsystem,
+			Name:           "metric_computation_total",
+			Help:           "Number of metric computations. The label 'action' should be either 'scale_down', 'scale_up', or 'none'. Also, the label 'error' should be either 'spec', 'internal', or 'none'. The label 'metric_type' corresponds to HPA.spec.metrics[*].type",
+			StabilityLevel: metrics.ALPHA,
+		}, []string{"action", "error", "metric_type"})
+	metricComputationDuration = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Subsystem:      hpaControllerSubsystem,
+			Name:           "metric_computation_duration_seconds",
+			Help:           "The time(seconds) that the HPA controller takes to calculate one metric. The label 'action' should be either 'scale_down', 'scale_up', or 'none'. The label 'error' should be either 'spec', 'internal', or 'none'. The label 'metric_type' corresponds to HPA.spec.metrics[*].type",
+			Buckets:        metrics.ExponentialBuckets(0.001, 2, 15),
+			StabilityLevel: metrics.ALPHA,
+		}, []string{"action", "error", "metric_type"})
 
 	metricsList = []metrics.Registerable{
 		reconciliationsTotal,
 		reconciliationsDuration,
+		metricComputationTotal,
+		metricComputationDuration,
 	}
 )
 

pkg/controller/podautoscaler/monitor/monitor.go

@@ -16,7 +16,11 @@ limitations under the License.
 
 package monitor
 
-import "time"
+import (
+	"time"
+
+	v2 "k8s.io/api/autoscaling/v2"
+)
 
 type ActionLabel string
 type ErrorLabel string
@@ -36,6 +40,7 @@ const (
 // Monitor records some metrics so that people can monitor HPA controller.
 type Monitor interface {
 	ObserveReconciliationResult(action ActionLabel, err ErrorLabel, duration time.Duration)
+	ObserveMetricComputationResult(action ActionLabel, err ErrorLabel, duration time.Duration, metricType v2.MetricSourceType)
 }
 
 type monitor struct{}
@@ -49,3 +54,9 @@ func (r *monitor) ObserveReconciliationResult(action ActionLabel, err ErrorLabel
 	reconciliationsTotal.WithLabelValues(string(action), string(err)).Inc()
 	reconciliationsDuration.WithLabelValues(string(action), string(err)).Observe(duration.Seconds())
 }
+
+// ObserveMetricComputationResult observes some metrics from a metric computation result.
+func (r *monitor) ObserveMetricComputationResult(action ActionLabel, err ErrorLabel, duration time.Duration, metricType v2.MetricSourceType) {
+	metricComputationTotal.WithLabelValues(string(action), string(err), string(metricType)).Inc()
+	metricComputationDuration.WithLabelValues(string(action), string(err), string(metricType)).Observe(duration.Seconds())
+}