add counter metric for dashpole

Change-Id: I4a235c0d8b936da960cf5ce25cdd992e94130391
This commit is contained in:
Han Kang 2022-08-16 09:19:26 -07:00
parent 822c52c220
commit 6c451da127
2 changed files with 27 additions and 1 deletions

View File

@ -51,6 +51,17 @@ var (
}, },
[]string{"name", "type", "status"}, []string{"name", "type", "status"},
) )
// healthchecksTotal is a Prometheus Counter metrics used for counting the results of a k8s healthcheck.
healthchecksTotal = k8smetrics.NewCounterVec(
&k8smetrics.CounterOpts{
Namespace: "k8s",
Name: "healthchecks_total",
Help: "This metric records the results of all healthcheck.",
StabilityLevel: k8smetrics.ALPHA,
},
[]string{"name", "type", "status"},
)
statuses = []HealthcheckStatus{Success, Error, Pending} statuses = []HealthcheckStatus{Success, Error, Pending}
statusSet = map[HealthcheckStatus]struct{}{Success: {}, Error: {}, Pending: {}} statusSet = map[HealthcheckStatus]struct{}{Success: {}, Error: {}, Pending: {}}
checkSet = map[HealthcheckType]struct{}{Livez: {}, Readyz: {}, Healthz: {}} checkSet = map[HealthcheckType]struct{}{Livez: {}, Readyz: {}, Healthz: {}}
@ -58,10 +69,12 @@ var (
func init() { func init() {
legacyregistry.MustRegister(healthcheck) legacyregistry.MustRegister(healthcheck)
legacyregistry.MustRegister(healthchecksTotal)
} }
func ResetHealthMetrics() { func ResetHealthMetrics() {
healthcheck.Reset() healthcheck.Reset()
healthchecksTotal.Reset()
} }
func ObserveHealthcheck(ctx context.Context, name string, healthcheckType HealthcheckType, status HealthcheckStatus) error { func ObserveHealthcheck(ctx context.Context, name string, healthcheckType HealthcheckType, status HealthcheckStatus) error {
@ -76,6 +89,7 @@ func ObserveHealthcheck(ctx context.Context, name string, healthcheckType Health
healthcheck.WithContext(ctx).WithLabelValues(name, string(healthcheckType), string(s)).Set(0) healthcheck.WithContext(ctx).WithLabelValues(name, string(healthcheckType), string(s)).Set(0)
} }
} }
healthchecksTotal.WithContext(ctx).WithLabelValues(name, string(healthcheckType), string(status)).Inc()
healthcheck.WithContext(ctx).WithLabelValues(name, string(healthcheckType), string(status)).Set(1) healthcheck.WithContext(ctx).WithLabelValues(name, string(healthcheckType), string(status)).Set(1)
return nil return nil
} }

View File

@ -26,7 +26,7 @@ import (
) )
var ( var (
testedMetrics = []string{"k8s_healthcheck"} testedMetrics = []string{"k8s_healthcheck", "k8s_healthchecks_total"}
) )
func TestObserveHealthcheck(t *testing.T) { func TestObserveHealthcheck(t *testing.T) {
@ -40,6 +40,9 @@ func TestObserveHealthcheck(t *testing.T) {
k8s_healthcheck{name="healthcheck-a",status="error",type="healthz"} 1 k8s_healthcheck{name="healthcheck-a",status="error",type="healthz"} 1
k8s_healthcheck{name="healthcheck-a",status="pending",type="healthz"} 0 k8s_healthcheck{name="healthcheck-a",status="pending",type="healthz"} 0
k8s_healthcheck{name="healthcheck-a",status="success",type="healthz"} 0 k8s_healthcheck{name="healthcheck-a",status="success",type="healthz"} 0
# HELP k8s_healthchecks_total [ALPHA] This metric records the results of all healthcheck.
# TYPE k8s_healthchecks_total counter
k8s_healthchecks_total{name="healthcheck-a",status="error",type="healthz"} 1
` `
testCases := []struct { testCases := []struct {
desc string desc string
@ -59,6 +62,10 @@ func TestObserveHealthcheck(t *testing.T) {
k8s_healthcheck{name="healthcheck-a",status="error",type="healthz"} 0 k8s_healthcheck{name="healthcheck-a",status="error",type="healthz"} 0
k8s_healthcheck{name="healthcheck-a",status="pending",type="healthz"} 1 k8s_healthcheck{name="healthcheck-a",status="pending",type="healthz"} 1
k8s_healthcheck{name="healthcheck-a",status="success",type="healthz"} 0 k8s_healthcheck{name="healthcheck-a",status="success",type="healthz"} 0
# HELP k8s_healthchecks_total [ALPHA] This metric records the results of all healthcheck.
# TYPE k8s_healthchecks_total counter
k8s_healthchecks_total{name="healthcheck-a",status="error",type="healthz"} 1
k8s_healthchecks_total{name="healthcheck-a",status="pending",type="healthz"} 1
`, `,
}, },
{ {
@ -72,12 +79,17 @@ func TestObserveHealthcheck(t *testing.T) {
k8s_healthcheck{name="healthcheck-a",status="error",type="healthz"} 0 k8s_healthcheck{name="healthcheck-a",status="error",type="healthz"} 0
k8s_healthcheck{name="healthcheck-a",status="pending",type="healthz"} 0 k8s_healthcheck{name="healthcheck-a",status="pending",type="healthz"} 0
k8s_healthcheck{name="healthcheck-a",status="success",type="healthz"} 1 k8s_healthcheck{name="healthcheck-a",status="success",type="healthz"} 1
# HELP k8s_healthchecks_total [ALPHA] This metric records the results of all healthcheck.
# TYPE k8s_healthchecks_total counter
k8s_healthchecks_total{name="healthcheck-a",status="error",type="healthz"} 1
k8s_healthchecks_total{name="healthcheck-a",status="success",type="healthz"} 1
`, `,
}, },
} }
for _, test := range testCases { for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) { t.Run(test.desc, func(t *testing.T) {
defer ResetHealthMetrics()
// let's first record an error as initial state // let's first record an error as initial state
err := ObserveHealthcheck(context.Background(), test.name, test.hcType, initialState) err := ObserveHealthcheck(context.Background(), test.name, test.hcType, initialState)
if err != nil { if err != nil {