From 6c451da1273689dd348ddd8c4b4a491944c31c67 Mon Sep 17 00:00:00 2001 From: Han Kang Date: Tue, 16 Aug 2022 09:19:26 -0700 Subject: [PATCH] add counter metric for dashpole Change-Id: I4a235c0d8b936da960cf5ce25cdd992e94130391 --- .../metrics/prometheus/health/metrics.go | 14 ++++++++++++++ .../metrics/prometheus/health/metrics_test.go | 14 +++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/staging/src/k8s.io/component-base/metrics/prometheus/health/metrics.go b/staging/src/k8s.io/component-base/metrics/prometheus/health/metrics.go index e27166fb191..4be16d7cb0b 100644 --- a/staging/src/k8s.io/component-base/metrics/prometheus/health/metrics.go +++ b/staging/src/k8s.io/component-base/metrics/prometheus/health/metrics.go @@ -51,6 +51,17 @@ var ( }, []string{"name", "type", "status"}, ) + + // healthchecksTotal is a Prometheus Counter metrics used for counting the results of a k8s healthcheck. + healthchecksTotal = k8smetrics.NewCounterVec( + &k8smetrics.CounterOpts{ + Namespace: "k8s", + Name: "healthchecks_total", + Help: "This metric records the results of all healthcheck.", + StabilityLevel: k8smetrics.ALPHA, + }, + []string{"name", "type", "status"}, + ) statuses = []HealthcheckStatus{Success, Error, Pending} statusSet = map[HealthcheckStatus]struct{}{Success: {}, Error: {}, Pending: {}} checkSet = map[HealthcheckType]struct{}{Livez: {}, Readyz: {}, Healthz: {}} @@ -58,10 +69,12 @@ var ( func init() { legacyregistry.MustRegister(healthcheck) + legacyregistry.MustRegister(healthchecksTotal) } func ResetHealthMetrics() { healthcheck.Reset() + healthchecksTotal.Reset() } func ObserveHealthcheck(ctx context.Context, name string, healthcheckType HealthcheckType, status HealthcheckStatus) error { @@ -76,6 +89,7 @@ func ObserveHealthcheck(ctx context.Context, name string, healthcheckType Health healthcheck.WithContext(ctx).WithLabelValues(name, string(healthcheckType), string(s)).Set(0) } } + healthchecksTotal.WithContext(ctx).WithLabelValues(name, string(healthcheckType), string(status)).Inc() healthcheck.WithContext(ctx).WithLabelValues(name, string(healthcheckType), string(status)).Set(1) return nil } diff --git a/staging/src/k8s.io/component-base/metrics/prometheus/health/metrics_test.go b/staging/src/k8s.io/component-base/metrics/prometheus/health/metrics_test.go index 84715b28496..9ba52a3099a 100644 --- a/staging/src/k8s.io/component-base/metrics/prometheus/health/metrics_test.go +++ b/staging/src/k8s.io/component-base/metrics/prometheus/health/metrics_test.go @@ -26,7 +26,7 @@ import ( ) var ( - testedMetrics = []string{"k8s_healthcheck"} + testedMetrics = []string{"k8s_healthcheck", "k8s_healthchecks_total"} ) func TestObserveHealthcheck(t *testing.T) { @@ -40,6 +40,9 @@ func TestObserveHealthcheck(t *testing.T) { k8s_healthcheck{name="healthcheck-a",status="error",type="healthz"} 1 k8s_healthcheck{name="healthcheck-a",status="pending",type="healthz"} 0 k8s_healthcheck{name="healthcheck-a",status="success",type="healthz"} 0 + # HELP k8s_healthchecks_total [ALPHA] This metric records the results of all healthcheck. + # TYPE k8s_healthchecks_total counter + k8s_healthchecks_total{name="healthcheck-a",status="error",type="healthz"} 1 ` testCases := []struct { desc string @@ -59,6 +62,10 @@ func TestObserveHealthcheck(t *testing.T) { k8s_healthcheck{name="healthcheck-a",status="error",type="healthz"} 0 k8s_healthcheck{name="healthcheck-a",status="pending",type="healthz"} 1 k8s_healthcheck{name="healthcheck-a",status="success",type="healthz"} 0 + # HELP k8s_healthchecks_total [ALPHA] This metric records the results of all healthcheck. + # TYPE k8s_healthchecks_total counter + k8s_healthchecks_total{name="healthcheck-a",status="error",type="healthz"} 1 + k8s_healthchecks_total{name="healthcheck-a",status="pending",type="healthz"} 1 `, }, { @@ -72,12 +79,17 @@ func TestObserveHealthcheck(t *testing.T) { k8s_healthcheck{name="healthcheck-a",status="error",type="healthz"} 0 k8s_healthcheck{name="healthcheck-a",status="pending",type="healthz"} 0 k8s_healthcheck{name="healthcheck-a",status="success",type="healthz"} 1 + # HELP k8s_healthchecks_total [ALPHA] This metric records the results of all healthcheck. + # TYPE k8s_healthchecks_total counter + k8s_healthchecks_total{name="healthcheck-a",status="error",type="healthz"} 1 + k8s_healthchecks_total{name="healthcheck-a",status="success",type="healthz"} 1 `, }, } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { + defer ResetHealthMetrics() // let's first record an error as initial state err := ObserveHealthcheck(context.Background(), test.name, test.hcType, initialState) if err != nil {