Expose kubelet health checks using new prometheus endpoint

This commit is contained in:
Rohit Ramkumar 2018-03-19 13:20:40 -07:00
parent 32db4e1afe
commit c0c5566c25
6 changed files with 49 additions and 0 deletions

View File

@ -26,6 +26,7 @@ go_library(
"//pkg/probe/http:go_default_library", "//pkg/probe/http:go_default_library",
"//pkg/probe/tcp:go_default_library", "//pkg/probe/tcp:go_default_library",
"//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",

View File

@ -20,6 +20,7 @@ import (
"sync" "sync"
"github.com/golang/glog" "github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/sets"
@ -31,6 +32,16 @@ import (
"k8s.io/kubernetes/pkg/kubelet/util/format" "k8s.io/kubernetes/pkg/kubelet/util/format"
) )
// ProberResults stores the results of a probe as prometheus metrics.
var ProberResults = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Subsystem: "prober",
Name: "probe_result",
Help: "The result of a liveness or readiness probe for a container.",
},
[]string{"probe_type", "container_name", "pod_name", "namespace", "pod_uid"},
)
// Manager manages pod probing. It creates a probe "worker" for every container that specifies a // Manager manages pod probing. It creates a probe "worker" for every container that specifies a
// probe (AddPod). The worker periodically probes its assigned container and caches the results. The // probe (AddPod). The worker periodically probes its assigned container and caches the results. The
// manager use the cached probe results to set the appropriate Ready state in the PodStatus when // manager use the cached probe results to set the appropriate Ready state in the PodStatus when

View File

@ -58,6 +58,18 @@ func (r Result) String() string {
} }
} }
// ToPrometheusType translates a Result to a form which is better understood by prometheus.
func (r Result) ToPrometheusType() float64 {
switch r {
case Success:
return 0
case Failure:
return 1
default:
return -1
}
}
// Update is an enum of the types of updates sent over the Updates channel. // Update is an enum of the types of updates sent over the Updates channel.
type Update struct { type Update struct {
ContainerID kubecontainer.ContainerID ContainerID kubecontainer.ContainerID

View File

@ -21,6 +21,7 @@ import (
"time" "time"
"github.com/golang/glog" "github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/runtime"
podutil "k8s.io/kubernetes/pkg/api/v1/pod" podutil "k8s.io/kubernetes/pkg/api/v1/pod"
@ -65,6 +66,10 @@ type worker struct {
// If set, skip probing. // If set, skip probing.
onHold bool onHold bool
// proberResultsMetricLabels holds the labels attached to this worker
// for the ProberResults metric.
proberResultsMetricLabels prometheus.Labels
} }
// Creates and starts a new probe worker. // Creates and starts a new probe worker.
@ -93,6 +98,14 @@ func newWorker(
w.initialValue = results.Success w.initialValue = results.Success
} }
w.proberResultsMetricLabels = prometheus.Labels{
"probe_type": w.probeType.String(),
"container_name": w.container.Name,
"pod_name": w.pod.Name,
"namespace": w.pod.Namespace,
"pod_uid": string(w.pod.UID),
}
return w return w
} }
@ -114,6 +127,7 @@ func (w *worker) run() {
} }
w.probeManager.removeWorker(w.pod.UID, w.container.Name, w.probeType) w.probeManager.removeWorker(w.pod.UID, w.container.Name, w.probeType)
ProberResults.Delete(w.proberResultsMetricLabels)
}() }()
probeLoop: probeLoop:
@ -218,6 +232,7 @@ func (w *worker) doProbe() (keepGoing bool) {
} }
w.resultsManager.Set(w.containerID, result, w.pod) w.resultsManager.Set(w.containerID, result, w.pod)
ProberResults.With(w.proberResultsMetricLabels).Set(result.ToPrometheusType())
if w.probeType == liveness && result == results.Failure { if w.probeType == liveness && result == results.Failure {
// The container fails a liveness check, it will need to be restarted. // The container fails a liveness check, it will need to be restarted.

View File

@ -19,6 +19,7 @@ go_library(
"//pkg/apis/core:go_default_library", "//pkg/apis/core:go_default_library",
"//pkg/apis/core/v1/validation:go_default_library", "//pkg/apis/core/v1/validation:go_default_library",
"//pkg/kubelet/container:go_default_library", "//pkg/kubelet/container:go_default_library",
"//pkg/kubelet/prober:go_default_library",
"//pkg/kubelet/server/portforward:go_default_library", "//pkg/kubelet/server/portforward:go_default_library",
"//pkg/kubelet/server/remotecommand:go_default_library", "//pkg/kubelet/server/remotecommand:go_default_library",
"//pkg/kubelet/server/stats:go_default_library", "//pkg/kubelet/server/stats:go_default_library",

View File

@ -54,6 +54,7 @@ import (
api "k8s.io/kubernetes/pkg/apis/core" api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/apis/core/v1/validation" "k8s.io/kubernetes/pkg/apis/core/v1/validation"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/prober"
"k8s.io/kubernetes/pkg/kubelet/server/portforward" "k8s.io/kubernetes/pkg/kubelet/server/portforward"
remotecommandserver "k8s.io/kubernetes/pkg/kubelet/server/remotecommand" remotecommandserver "k8s.io/kubernetes/pkg/kubelet/server/remotecommand"
"k8s.io/kubernetes/pkg/kubelet/server/stats" "k8s.io/kubernetes/pkg/kubelet/server/stats"
@ -66,6 +67,7 @@ import (
const ( const (
metricsPath = "/metrics" metricsPath = "/metrics"
cadvisorMetricsPath = "/metrics/cadvisor" cadvisorMetricsPath = "/metrics/cadvisor"
proberMetricsPath = "/metrics/probes"
specPath = "/spec/" specPath = "/spec/"
statsPath = "/stats/" statsPath = "/stats/"
logsPath = "/logs/" logsPath = "/logs/"
@ -281,6 +283,13 @@ func (s *Server) InstallDefaultHandlers() {
promhttp.HandlerFor(r, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}), promhttp.HandlerFor(r, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}),
) )
// prober metrics are exposed under a different endpoint
p := prometheus.NewRegistry()
p.MustRegister(prober.ProberResults)
s.restfulCont.Handle(proberMetricsPath,
promhttp.HandlerFor(p, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}),
)
ws = new(restful.WebService) ws = new(restful.WebService)
ws. ws.
Path(specPath). Path(specPath).