github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-08 11:38:15 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #76074 from danielqsj/probe

Browse Source 
change kubelet probe metrics to counter
This commit is contained in:
Kubernetes Prow Robot 2019-04-12 03:23:04 -07:00 committed by GitHub
commit b7858e31d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 49 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
pkg/kubelet/prober/prober_manager.go
View File

@ -32,17 +32,16 @@ import (
"k8s.io/kubernetes/pkg/kubelet/util/format"
)
// ProberResults stores the results of a probe as prometheus metrics.
var ProberResults = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
// ProberResults stores the cumulative number of a probe by result as prometheus metrics.
var ProberResults = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: "prober",
Name: "probe_result",
Help: "The result of a liveness or readiness probe for a container.",
Name: "probe_total",
Help: "Cumulative number of a liveness or readiness probe for a container by result.",
},
[]string{"probe_type",
"container_name",
"result",
"container",
"pod_name",
"pod",
"namespace",
"pod_uid"},
@ -130,6 +129,10 @@ type probeType int
const (
liveness probeType = iota
readiness
probeResultSuccessful string = "successful"
probeResultFailed string = "failed"
probeResultUnknown string = "unknown"
)
// For debugging.

51
pkg/kubelet/prober/worker.go
View File

@ -68,8 +68,10 @@ type worker struct {
onHold bool
// proberResultsMetricLabels holds the labels attached to this worker
// for the ProberResults metric.
proberResultsMetricLabels prometheus.Labels
// for the ProberResults metric by result.
proberResultsSuccessfulMetricLabels prometheus.Labels
proberResultsFailedMetricLabels prometheus.Labels
proberResultsUnknownMetricLabels prometheus.Labels
}
// Creates and starts a new probe worker.
@ -98,16 +100,23 @@ func newWorker(
w.initialValue = results.Success
}
w.proberResultsMetricLabels = prometheus.Labels{
"probe_type": w.probeType.String(),
"container_name": w.container.Name,
"container": w.container.Name,
"pod_name": w.pod.Name,
"pod": w.pod.Name,
"namespace": w.pod.Namespace,
"pod_uid": string(w.pod.UID),
basicMetricLabels := prometheus.Labels{
"probe_type": w.probeType.String(),
"container": w.container.Name,
"pod": w.pod.Name,
"namespace": w.pod.Namespace,
"pod_uid": string(w.pod.UID),
}
w.proberResultsSuccessfulMetricLabels = deepCopyPrometheusLabels(basicMetricLabels)
w.proberResultsSuccessfulMetricLabels["result"] = probeResultSuccessful
w.proberResultsFailedMetricLabels = deepCopyPrometheusLabels(basicMetricLabels)
w.proberResultsFailedMetricLabels["result"] = probeResultFailed
w.proberResultsUnknownMetricLabels = deepCopyPrometheusLabels(basicMetricLabels)
w.proberResultsUnknownMetricLabels["result"] = probeResultUnknown
return w
}
@ -129,7 +138,9 @@ func (w *worker) run() {
}
w.probeManager.removeWorker(w.pod.UID, w.container.Name, w.probeType)
ProberResults.Delete(w.proberResultsMetricLabels)
ProberResults.Delete(w.proberResultsSuccessfulMetricLabels)
ProberResults.Delete(w.proberResultsFailedMetricLabels)
ProberResults.Delete(w.proberResultsUnknownMetricLabels)
}()
probeLoop:
@ -220,6 +231,15 @@ func (w *worker) doProbe() (keepGoing bool) {
return true
}
switch result {
case results.Success:
ProberResults.With(w.proberResultsSuccessfulMetricLabels).Inc()
case results.Failure:
ProberResults.With(w.proberResultsFailedMetricLabels).Inc()
default:
ProberResults.With(w.proberResultsUnknownMetricLabels).Inc()
}
if w.lastResult == result {
w.resultRun++
} else {
@ -234,7 +254,6 @@ func (w *worker) doProbe() (keepGoing bool) {
}
w.resultsManager.Set(w.containerID, result, w.pod)
ProberResults.With(w.proberResultsMetricLabels).Set(result.ToPrometheusType())
if w.probeType == liveness && result == results.Failure {
// The container fails a liveness check, it will need to be restarted.
@ -247,3 +266,11 @@ func (w *worker) doProbe() (keepGoing bool) {
return true
}
func deepCopyPrometheusLabels(m prometheus.Labels) prometheus.Labels {
ret := make(prometheus.Labels, len(m))
for k, v := range m {
ret[k] = v
}
return ret
}