diff --git a/pkg/kubelet/prober/prober_manager.go b/pkg/kubelet/prober/prober_manager.go index 62106b95dba..147082721cb 100644 --- a/pkg/kubelet/prober/prober_manager.go +++ b/pkg/kubelet/prober/prober_manager.go @@ -32,17 +32,16 @@ import ( "k8s.io/kubernetes/pkg/kubelet/util/format" ) -// ProberResults stores the results of a probe as prometheus metrics. -var ProberResults = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ +// ProberResults stores the cumulative number of a probe by result as prometheus metrics. +var ProberResults = prometheus.NewCounterVec( + prometheus.CounterOpts{ Subsystem: "prober", - Name: "probe_result", - Help: "The result of a liveness or readiness probe for a container.", + Name: "probe_total", + Help: "Cumulative number of a liveness or readiness probe for a container by result.", }, []string{"probe_type", - "container_name", + "result", "container", - "pod_name", "pod", "namespace", "pod_uid"}, @@ -130,6 +129,10 @@ type probeType int const ( liveness probeType = iota readiness + + probeResultSuccessful string = "successful" + probeResultFailed string = "failed" + probeResultUnknown string = "unknown" ) // For debugging. diff --git a/pkg/kubelet/prober/worker.go b/pkg/kubelet/prober/worker.go index 295f7c40173..c502e0245c2 100644 --- a/pkg/kubelet/prober/worker.go +++ b/pkg/kubelet/prober/worker.go @@ -68,8 +68,10 @@ type worker struct { onHold bool // proberResultsMetricLabels holds the labels attached to this worker - // for the ProberResults metric. - proberResultsMetricLabels prometheus.Labels + // for the ProberResults metric by result. + proberResultsSuccessfulMetricLabels prometheus.Labels + proberResultsFailedMetricLabels prometheus.Labels + proberResultsUnknownMetricLabels prometheus.Labels } // Creates and starts a new probe worker. @@ -98,16 +100,23 @@ func newWorker( w.initialValue = results.Success } - w.proberResultsMetricLabels = prometheus.Labels{ - "probe_type": w.probeType.String(), - "container_name": w.container.Name, - "container": w.container.Name, - "pod_name": w.pod.Name, - "pod": w.pod.Name, - "namespace": w.pod.Namespace, - "pod_uid": string(w.pod.UID), + basicMetricLabels := prometheus.Labels{ + "probe_type": w.probeType.String(), + "container": w.container.Name, + "pod": w.pod.Name, + "namespace": w.pod.Namespace, + "pod_uid": string(w.pod.UID), } + w.proberResultsSuccessfulMetricLabels = deepCopyPrometheusLabels(basicMetricLabels) + w.proberResultsSuccessfulMetricLabels["result"] = probeResultSuccessful + + w.proberResultsFailedMetricLabels = deepCopyPrometheusLabels(basicMetricLabels) + w.proberResultsFailedMetricLabels["result"] = probeResultFailed + + w.proberResultsUnknownMetricLabels = deepCopyPrometheusLabels(basicMetricLabels) + w.proberResultsUnknownMetricLabels["result"] = probeResultUnknown + return w } @@ -129,7 +138,9 @@ func (w *worker) run() { } w.probeManager.removeWorker(w.pod.UID, w.container.Name, w.probeType) - ProberResults.Delete(w.proberResultsMetricLabels) + ProberResults.Delete(w.proberResultsSuccessfulMetricLabels) + ProberResults.Delete(w.proberResultsFailedMetricLabels) + ProberResults.Delete(w.proberResultsUnknownMetricLabels) }() probeLoop: @@ -220,6 +231,15 @@ func (w *worker) doProbe() (keepGoing bool) { return true } + switch result { + case results.Success: + ProberResults.With(w.proberResultsSuccessfulMetricLabels).Inc() + case results.Failure: + ProberResults.With(w.proberResultsFailedMetricLabels).Inc() + default: + ProberResults.With(w.proberResultsUnknownMetricLabels).Inc() + } + if w.lastResult == result { w.resultRun++ } else { @@ -234,7 +254,6 @@ func (w *worker) doProbe() (keepGoing bool) { } w.resultsManager.Set(w.containerID, result, w.pod) - ProberResults.With(w.proberResultsMetricLabels).Set(result.ToPrometheusType()) if w.probeType == liveness && result == results.Failure { // The container fails a liveness check, it will need to be restarted. @@ -247,3 +266,11 @@ func (w *worker) doProbe() (keepGoing bool) { return true } + +func deepCopyPrometheusLabels(m prometheus.Labels) prometheus.Labels { + ret := make(prometheus.Labels, len(m)) + for k, v := range m { + ret[k] = v + } + return ret +}