From c0c5566c25aa1e118d77e1f1bce7489cd53b9016 Mon Sep 17 00:00:00 2001 From: Rohit Ramkumar Date: Mon, 19 Mar 2018 13:20:40 -0700 Subject: [PATCH] Expose kubelet health checks using new prometheus endpoint --- pkg/kubelet/prober/BUILD | 1 + pkg/kubelet/prober/prober_manager.go | 11 +++++++++++ pkg/kubelet/prober/results/results_manager.go | 12 ++++++++++++ pkg/kubelet/prober/worker.go | 15 +++++++++++++++ pkg/kubelet/server/BUILD | 1 + pkg/kubelet/server/server.go | 9 +++++++++ 6 files changed, 49 insertions(+) diff --git a/pkg/kubelet/prober/BUILD b/pkg/kubelet/prober/BUILD index e208ff07b4a..0807bfa08d5 100644 --- a/pkg/kubelet/prober/BUILD +++ b/pkg/kubelet/prober/BUILD @@ -26,6 +26,7 @@ go_library( "//pkg/probe/http:go_default_library", "//pkg/probe/tcp:go_default_library", "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", diff --git a/pkg/kubelet/prober/prober_manager.go b/pkg/kubelet/prober/prober_manager.go index fcbad24fe13..0e53e094076 100644 --- a/pkg/kubelet/prober/prober_manager.go +++ b/pkg/kubelet/prober/prober_manager.go @@ -20,6 +20,7 @@ import ( "sync" "github.com/golang/glog" + "github.com/prometheus/client_golang/prometheus" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" @@ -31,6 +32,16 @@ import ( "k8s.io/kubernetes/pkg/kubelet/util/format" ) +// ProberResults stores the results of a probe as prometheus metrics. +var ProberResults = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "prober", + Name: "probe_result", + Help: "The result of a liveness or readiness probe for a container.", + }, + []string{"probe_type", "container_name", "pod_name", "namespace", "pod_uid"}, +) + // Manager manages pod probing. It creates a probe "worker" for every container that specifies a // probe (AddPod). The worker periodically probes its assigned container and caches the results. The // manager use the cached probe results to set the appropriate Ready state in the PodStatus when diff --git a/pkg/kubelet/prober/results/results_manager.go b/pkg/kubelet/prober/results/results_manager.go index 2b01be7ba2f..f66a472c381 100644 --- a/pkg/kubelet/prober/results/results_manager.go +++ b/pkg/kubelet/prober/results/results_manager.go @@ -58,6 +58,18 @@ func (r Result) String() string { } } +// ToPrometheusType translates a Result to a form which is better understood by prometheus. +func (r Result) ToPrometheusType() float64 { + switch r { + case Success: + return 0 + case Failure: + return 1 + default: + return -1 + } +} + // Update is an enum of the types of updates sent over the Updates channel. type Update struct { ContainerID kubecontainer.ContainerID diff --git a/pkg/kubelet/prober/worker.go b/pkg/kubelet/prober/worker.go index d96365a2edc..30aa113d38e 100644 --- a/pkg/kubelet/prober/worker.go +++ b/pkg/kubelet/prober/worker.go @@ -21,6 +21,7 @@ import ( "time" "github.com/golang/glog" + "github.com/prometheus/client_golang/prometheus" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/runtime" podutil "k8s.io/kubernetes/pkg/api/v1/pod" @@ -65,6 +66,10 @@ type worker struct { // If set, skip probing. onHold bool + + // proberResultsMetricLabels holds the labels attached to this worker + // for the ProberResults metric. + proberResultsMetricLabels prometheus.Labels } // Creates and starts a new probe worker. @@ -93,6 +98,14 @@ func newWorker( w.initialValue = results.Success } + w.proberResultsMetricLabels = prometheus.Labels{ + "probe_type": w.probeType.String(), + "container_name": w.container.Name, + "pod_name": w.pod.Name, + "namespace": w.pod.Namespace, + "pod_uid": string(w.pod.UID), + } + return w } @@ -114,6 +127,7 @@ func (w *worker) run() { } w.probeManager.removeWorker(w.pod.UID, w.container.Name, w.probeType) + ProberResults.Delete(w.proberResultsMetricLabels) }() probeLoop: @@ -218,6 +232,7 @@ func (w *worker) doProbe() (keepGoing bool) { } w.resultsManager.Set(w.containerID, result, w.pod) + ProberResults.With(w.proberResultsMetricLabels).Set(result.ToPrometheusType()) if w.probeType == liveness && result == results.Failure { // The container fails a liveness check, it will need to be restarted. diff --git a/pkg/kubelet/server/BUILD b/pkg/kubelet/server/BUILD index a310a7d5f80..0d8138acf2d 100644 --- a/pkg/kubelet/server/BUILD +++ b/pkg/kubelet/server/BUILD @@ -19,6 +19,7 @@ go_library( "//pkg/apis/core:go_default_library", "//pkg/apis/core/v1/validation:go_default_library", "//pkg/kubelet/container:go_default_library", + "//pkg/kubelet/prober:go_default_library", "//pkg/kubelet/server/portforward:go_default_library", "//pkg/kubelet/server/remotecommand:go_default_library", "//pkg/kubelet/server/stats:go_default_library", diff --git a/pkg/kubelet/server/server.go b/pkg/kubelet/server/server.go index d22096e48d2..a57033f2044 100644 --- a/pkg/kubelet/server/server.go +++ b/pkg/kubelet/server/server.go @@ -54,6 +54,7 @@ import ( api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/core/v1/validation" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/kubelet/prober" "k8s.io/kubernetes/pkg/kubelet/server/portforward" remotecommandserver "k8s.io/kubernetes/pkg/kubelet/server/remotecommand" "k8s.io/kubernetes/pkg/kubelet/server/stats" @@ -66,6 +67,7 @@ import ( const ( metricsPath = "/metrics" cadvisorMetricsPath = "/metrics/cadvisor" + proberMetricsPath = "/metrics/probes" specPath = "/spec/" statsPath = "/stats/" logsPath = "/logs/" @@ -281,6 +283,13 @@ func (s *Server) InstallDefaultHandlers() { promhttp.HandlerFor(r, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}), ) + // prober metrics are exposed under a different endpoint + p := prometheus.NewRegistry() + p.MustRegister(prober.ProberResults) + s.restfulCont.Handle(proberMetricsPath, + promhttp.HandlerFor(p, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}), + ) + ws = new(restful.WebService) ws. Path(specPath).