mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-22 19:31:44 +00:00
Expose kubelet health checks using new prometheus endpoint
This commit is contained in:
parent
32db4e1afe
commit
c0c5566c25
@ -26,6 +26,7 @@ go_library(
|
|||||||
"//pkg/probe/http:go_default_library",
|
"//pkg/probe/http:go_default_library",
|
||||||
"//pkg/probe/tcp:go_default_library",
|
"//pkg/probe/tcp:go_default_library",
|
||||||
"//vendor/github.com/golang/glog:go_default_library",
|
"//vendor/github.com/golang/glog:go_default_library",
|
||||||
|
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
|
||||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||||
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
|
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
|
||||||
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
|
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
|
||||||
|
@ -20,6 +20,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/golang/glog"
|
"github.com/golang/glog"
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
"k8s.io/api/core/v1"
|
"k8s.io/api/core/v1"
|
||||||
"k8s.io/apimachinery/pkg/types"
|
"k8s.io/apimachinery/pkg/types"
|
||||||
"k8s.io/apimachinery/pkg/util/sets"
|
"k8s.io/apimachinery/pkg/util/sets"
|
||||||
@ -31,6 +32,16 @@ import (
|
|||||||
"k8s.io/kubernetes/pkg/kubelet/util/format"
|
"k8s.io/kubernetes/pkg/kubelet/util/format"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ProberResults stores the results of a probe as prometheus metrics.
|
||||||
|
var ProberResults = prometheus.NewGaugeVec(
|
||||||
|
prometheus.GaugeOpts{
|
||||||
|
Subsystem: "prober",
|
||||||
|
Name: "probe_result",
|
||||||
|
Help: "The result of a liveness or readiness probe for a container.",
|
||||||
|
},
|
||||||
|
[]string{"probe_type", "container_name", "pod_name", "namespace", "pod_uid"},
|
||||||
|
)
|
||||||
|
|
||||||
// Manager manages pod probing. It creates a probe "worker" for every container that specifies a
|
// Manager manages pod probing. It creates a probe "worker" for every container that specifies a
|
||||||
// probe (AddPod). The worker periodically probes its assigned container and caches the results. The
|
// probe (AddPod). The worker periodically probes its assigned container and caches the results. The
|
||||||
// manager use the cached probe results to set the appropriate Ready state in the PodStatus when
|
// manager use the cached probe results to set the appropriate Ready state in the PodStatus when
|
||||||
|
@ -58,6 +58,18 @@ func (r Result) String() string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ToPrometheusType translates a Result to a form which is better understood by prometheus.
|
||||||
|
func (r Result) ToPrometheusType() float64 {
|
||||||
|
switch r {
|
||||||
|
case Success:
|
||||||
|
return 0
|
||||||
|
case Failure:
|
||||||
|
return 1
|
||||||
|
default:
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Update is an enum of the types of updates sent over the Updates channel.
|
// Update is an enum of the types of updates sent over the Updates channel.
|
||||||
type Update struct {
|
type Update struct {
|
||||||
ContainerID kubecontainer.ContainerID
|
ContainerID kubecontainer.ContainerID
|
||||||
|
@ -21,6 +21,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/golang/glog"
|
"github.com/golang/glog"
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
"k8s.io/api/core/v1"
|
"k8s.io/api/core/v1"
|
||||||
"k8s.io/apimachinery/pkg/util/runtime"
|
"k8s.io/apimachinery/pkg/util/runtime"
|
||||||
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
|
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
|
||||||
@ -65,6 +66,10 @@ type worker struct {
|
|||||||
|
|
||||||
// If set, skip probing.
|
// If set, skip probing.
|
||||||
onHold bool
|
onHold bool
|
||||||
|
|
||||||
|
// proberResultsMetricLabels holds the labels attached to this worker
|
||||||
|
// for the ProberResults metric.
|
||||||
|
proberResultsMetricLabels prometheus.Labels
|
||||||
}
|
}
|
||||||
|
|
||||||
// Creates and starts a new probe worker.
|
// Creates and starts a new probe worker.
|
||||||
@ -93,6 +98,14 @@ func newWorker(
|
|||||||
w.initialValue = results.Success
|
w.initialValue = results.Success
|
||||||
}
|
}
|
||||||
|
|
||||||
|
w.proberResultsMetricLabels = prometheus.Labels{
|
||||||
|
"probe_type": w.probeType.String(),
|
||||||
|
"container_name": w.container.Name,
|
||||||
|
"pod_name": w.pod.Name,
|
||||||
|
"namespace": w.pod.Namespace,
|
||||||
|
"pod_uid": string(w.pod.UID),
|
||||||
|
}
|
||||||
|
|
||||||
return w
|
return w
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -114,6 +127,7 @@ func (w *worker) run() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
w.probeManager.removeWorker(w.pod.UID, w.container.Name, w.probeType)
|
w.probeManager.removeWorker(w.pod.UID, w.container.Name, w.probeType)
|
||||||
|
ProberResults.Delete(w.proberResultsMetricLabels)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
probeLoop:
|
probeLoop:
|
||||||
@ -218,6 +232,7 @@ func (w *worker) doProbe() (keepGoing bool) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
w.resultsManager.Set(w.containerID, result, w.pod)
|
w.resultsManager.Set(w.containerID, result, w.pod)
|
||||||
|
ProberResults.With(w.proberResultsMetricLabels).Set(result.ToPrometheusType())
|
||||||
|
|
||||||
if w.probeType == liveness && result == results.Failure {
|
if w.probeType == liveness && result == results.Failure {
|
||||||
// The container fails a liveness check, it will need to be restarted.
|
// The container fails a liveness check, it will need to be restarted.
|
||||||
|
@ -19,6 +19,7 @@ go_library(
|
|||||||
"//pkg/apis/core:go_default_library",
|
"//pkg/apis/core:go_default_library",
|
||||||
"//pkg/apis/core/v1/validation:go_default_library",
|
"//pkg/apis/core/v1/validation:go_default_library",
|
||||||
"//pkg/kubelet/container:go_default_library",
|
"//pkg/kubelet/container:go_default_library",
|
||||||
|
"//pkg/kubelet/prober:go_default_library",
|
||||||
"//pkg/kubelet/server/portforward:go_default_library",
|
"//pkg/kubelet/server/portforward:go_default_library",
|
||||||
"//pkg/kubelet/server/remotecommand:go_default_library",
|
"//pkg/kubelet/server/remotecommand:go_default_library",
|
||||||
"//pkg/kubelet/server/stats:go_default_library",
|
"//pkg/kubelet/server/stats:go_default_library",
|
||||||
|
@ -54,6 +54,7 @@ import (
|
|||||||
api "k8s.io/kubernetes/pkg/apis/core"
|
api "k8s.io/kubernetes/pkg/apis/core"
|
||||||
"k8s.io/kubernetes/pkg/apis/core/v1/validation"
|
"k8s.io/kubernetes/pkg/apis/core/v1/validation"
|
||||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/prober"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/server/portforward"
|
"k8s.io/kubernetes/pkg/kubelet/server/portforward"
|
||||||
remotecommandserver "k8s.io/kubernetes/pkg/kubelet/server/remotecommand"
|
remotecommandserver "k8s.io/kubernetes/pkg/kubelet/server/remotecommand"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/server/stats"
|
"k8s.io/kubernetes/pkg/kubelet/server/stats"
|
||||||
@ -66,6 +67,7 @@ import (
|
|||||||
const (
|
const (
|
||||||
metricsPath = "/metrics"
|
metricsPath = "/metrics"
|
||||||
cadvisorMetricsPath = "/metrics/cadvisor"
|
cadvisorMetricsPath = "/metrics/cadvisor"
|
||||||
|
proberMetricsPath = "/metrics/probes"
|
||||||
specPath = "/spec/"
|
specPath = "/spec/"
|
||||||
statsPath = "/stats/"
|
statsPath = "/stats/"
|
||||||
logsPath = "/logs/"
|
logsPath = "/logs/"
|
||||||
@ -281,6 +283,13 @@ func (s *Server) InstallDefaultHandlers() {
|
|||||||
promhttp.HandlerFor(r, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}),
|
promhttp.HandlerFor(r, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// prober metrics are exposed under a different endpoint
|
||||||
|
p := prometheus.NewRegistry()
|
||||||
|
p.MustRegister(prober.ProberResults)
|
||||||
|
s.restfulCont.Handle(proberMetricsPath,
|
||||||
|
promhttp.HandlerFor(p, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}),
|
||||||
|
)
|
||||||
|
|
||||||
ws = new(restful.WebService)
|
ws = new(restful.WebService)
|
||||||
ws.
|
ws.
|
||||||
Path(specPath).
|
Path(specPath).
|
||||||
|
Loading…
Reference in New Issue
Block a user