add counter metric for dashpole

Change-Id: I4a235c0d8b936da960cf5ce25cdd992e94130391
This commit is contained in:
Han Kang 2022-08-16 09:19:26 -07:00
parent 822c52c220
commit 6c451da127
2 changed files with 27 additions and 1 deletions

View File

@ -51,6 +51,17 @@ var (
},
[]string{"name", "type", "status"},
)
// healthchecksTotal is a Prometheus Counter metrics used for counting the results of a k8s healthcheck.
healthchecksTotal = k8smetrics.NewCounterVec(
&k8smetrics.CounterOpts{
Namespace: "k8s",
Name: "healthchecks_total",
Help: "This metric records the results of all healthcheck.",
StabilityLevel: k8smetrics.ALPHA,
},
[]string{"name", "type", "status"},
)
statuses = []HealthcheckStatus{Success, Error, Pending}
statusSet = map[HealthcheckStatus]struct{}{Success: {}, Error: {}, Pending: {}}
checkSet = map[HealthcheckType]struct{}{Livez: {}, Readyz: {}, Healthz: {}}
@ -58,10 +69,12 @@ var (
func init() {
legacyregistry.MustRegister(healthcheck)
legacyregistry.MustRegister(healthchecksTotal)
}
func ResetHealthMetrics() {
healthcheck.Reset()
healthchecksTotal.Reset()
}
func ObserveHealthcheck(ctx context.Context, name string, healthcheckType HealthcheckType, status HealthcheckStatus) error {
@ -76,6 +89,7 @@ func ObserveHealthcheck(ctx context.Context, name string, healthcheckType Health
healthcheck.WithContext(ctx).WithLabelValues(name, string(healthcheckType), string(s)).Set(0)
}
}
healthchecksTotal.WithContext(ctx).WithLabelValues(name, string(healthcheckType), string(status)).Inc()
healthcheck.WithContext(ctx).WithLabelValues(name, string(healthcheckType), string(status)).Set(1)
return nil
}

View File

@ -26,7 +26,7 @@ import (
)
var (
testedMetrics = []string{"k8s_healthcheck"}
testedMetrics = []string{"k8s_healthcheck", "k8s_healthchecks_total"}
)
func TestObserveHealthcheck(t *testing.T) {
@ -40,6 +40,9 @@ func TestObserveHealthcheck(t *testing.T) {
k8s_healthcheck{name="healthcheck-a",status="error",type="healthz"} 1
k8s_healthcheck{name="healthcheck-a",status="pending",type="healthz"} 0
k8s_healthcheck{name="healthcheck-a",status="success",type="healthz"} 0
# HELP k8s_healthchecks_total [ALPHA] This metric records the results of all healthcheck.
# TYPE k8s_healthchecks_total counter
k8s_healthchecks_total{name="healthcheck-a",status="error",type="healthz"} 1
`
testCases := []struct {
desc string
@ -59,6 +62,10 @@ func TestObserveHealthcheck(t *testing.T) {
k8s_healthcheck{name="healthcheck-a",status="error",type="healthz"} 0
k8s_healthcheck{name="healthcheck-a",status="pending",type="healthz"} 1
k8s_healthcheck{name="healthcheck-a",status="success",type="healthz"} 0
# HELP k8s_healthchecks_total [ALPHA] This metric records the results of all healthcheck.
# TYPE k8s_healthchecks_total counter
k8s_healthchecks_total{name="healthcheck-a",status="error",type="healthz"} 1
k8s_healthchecks_total{name="healthcheck-a",status="pending",type="healthz"} 1
`,
},
{
@ -72,12 +79,17 @@ func TestObserveHealthcheck(t *testing.T) {
k8s_healthcheck{name="healthcheck-a",status="error",type="healthz"} 0
k8s_healthcheck{name="healthcheck-a",status="pending",type="healthz"} 0
k8s_healthcheck{name="healthcheck-a",status="success",type="healthz"} 1
# HELP k8s_healthchecks_total [ALPHA] This metric records the results of all healthcheck.
# TYPE k8s_healthchecks_total counter
k8s_healthchecks_total{name="healthcheck-a",status="error",type="healthz"} 1
k8s_healthchecks_total{name="healthcheck-a",status="success",type="healthz"} 1
`,
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
defer ResetHealthMetrics()
// let's first record an error as initial state
err := ObserveHealthcheck(context.Background(), test.name, test.hcType, initialState)
if err != nil {