github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-09 03:57:41 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #76074 from danielqsj/probe

Browse Source 
change kubelet probe metrics to counter
This commit is contained in:
Kubernetes Prow Robot 2019-04-12 03:23:04 -07:00 committed by GitHub
commit b7858e31d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 49 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
pkg/kubelet/prober/prober_manager.go
View File

@ -32,17 +32,16 @@ import (
"k8s.io/kubernetes/pkg/kubelet/util/format" "k8s.io/kubernetes/pkg/kubelet/util/format"
) )
// ProberResults stores the results of a probe as prometheus metrics. // ProberResults stores the cumulative number of a probe by result as prometheus metrics.
var ProberResults = prometheus.NewGaugeVec( var ProberResults = prometheus.NewCounterVec(
prometheus.GaugeOpts{ prometheus.CounterOpts{
Subsystem: "prober", Subsystem: "prober",
Name: "probe_result", Name: "probe_total",
Help: "The result of a liveness or readiness probe for a container.", Help: "Cumulative number of a liveness or readiness probe for a container by result.",
}, },
[]string{"probe_type", []string{"probe_type",
"container_name", "result",
"container", "container",
"pod_name",
"pod", "pod",
"namespace", "namespace",
"pod_uid"}, "pod_uid"},
@ -130,6 +129,10 @@ type probeType int
const ( const (
liveness probeType = iota liveness probeType = iota
readiness readiness
probeResultSuccessful string = "successful"
probeResultFailed string = "failed"
probeResultUnknown string = "unknown"
) )
// For debugging. // For debugging.

51
pkg/kubelet/prober/worker.go
View File

@ -68,8 +68,10 @@ type worker struct {
onHold bool onHold bool
// proberResultsMetricLabels holds the labels attached to this worker // proberResultsMetricLabels holds the labels attached to this worker
// for the ProberResults metric. // for the ProberResults metric by result.
proberResultsMetricLabels prometheus.Labels proberResultsSuccessfulMetricLabels prometheus.Labels
proberResultsFailedMetricLabels prometheus.Labels
proberResultsUnknownMetricLabels prometheus.Labels
} }
// Creates and starts a new probe worker. // Creates and starts a new probe worker.
@ -98,16 +100,23 @@ func newWorker(
w.initialValue = results.Success w.initialValue = results.Success
} }
w.proberResultsMetricLabels = prometheus.Labels{ basicMetricLabels := prometheus.Labels{
"probe_type": w.probeType.String(), "probe_type": w.probeType.String(),
"container_name": w.container.Name, "container": w.container.Name,
"container": w.container.Name, "pod": w.pod.Name,
"pod_name": w.pod.Name, "namespace": w.pod.Namespace,
"pod": w.pod.Name, "pod_uid": string(w.pod.UID),
"namespace": w.pod.Namespace,
"pod_uid": string(w.pod.UID),
} }
w.proberResultsSuccessfulMetricLabels = deepCopyPrometheusLabels(basicMetricLabels)
w.proberResultsSuccessfulMetricLabels["result"] = probeResultSuccessful
w.proberResultsFailedMetricLabels = deepCopyPrometheusLabels(basicMetricLabels)
w.proberResultsFailedMetricLabels["result"] = probeResultFailed
w.proberResultsUnknownMetricLabels = deepCopyPrometheusLabels(basicMetricLabels)
w.proberResultsUnknownMetricLabels["result"] = probeResultUnknown
return w return w
} }
@ -129,7 +138,9 @@ func (w *worker) run() {
} }
w.probeManager.removeWorker(w.pod.UID, w.container.Name, w.probeType) w.probeManager.removeWorker(w.pod.UID, w.container.Name, w.probeType)
ProberResults.Delete(w.proberResultsMetricLabels) ProberResults.Delete(w.proberResultsSuccessfulMetricLabels)
ProberResults.Delete(w.proberResultsFailedMetricLabels)
ProberResults.Delete(w.proberResultsUnknownMetricLabels)
}() }()
probeLoop: probeLoop:
@ -220,6 +231,15 @@ func (w *worker) doProbe() (keepGoing bool) {
return true return true
} }
switch result {
case results.Success:
ProberResults.With(w.proberResultsSuccessfulMetricLabels).Inc()
case results.Failure:
ProberResults.With(w.proberResultsFailedMetricLabels).Inc()
default:
ProberResults.With(w.proberResultsUnknownMetricLabels).Inc()
}
if w.lastResult == result { if w.lastResult == result {
w.resultRun++ w.resultRun++
} else { } else {
@ -234,7 +254,6 @@ func (w *worker) doProbe() (keepGoing bool) {
} }
w.resultsManager.Set(w.containerID, result, w.pod) w.resultsManager.Set(w.containerID, result, w.pod)
ProberResults.With(w.proberResultsMetricLabels).Set(result.ToPrometheusType())
if w.probeType == liveness && result == results.Failure { if w.probeType == liveness && result == results.Failure {
// The container fails a liveness check, it will need to be restarted. // The container fails a liveness check, it will need to be restarted.
@ -247,3 +266,11 @@ func (w *worker) doProbe() (keepGoing bool) {
return true return true
} }
func deepCopyPrometheusLabels(m prometheus.Labels) prometheus.Labels {
ret := make(prometheus.Labels, len(m))
for k, v := range m {
ret[k] = v
}
return ret
}