From f5f989382423871059579db083187ff6a1434eaf Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Tue, 18 Jul 2017 01:12:24 -0400 Subject: [PATCH] Restore cAdvisor prometheus metrics to the main port But under a new path - `/metrics/cadvisor`. This ensures a secure port still exists for metrics while getting the benefit of separating out container metrics from the kubelet's metrics. --- pkg/kubelet/kubelet_cadvisor.go | 5 +++ pkg/kubelet/server/BUILD | 3 ++ pkg/kubelet/server/server.go | 62 +++++++++++++++++++++++++++++-- pkg/kubelet/server/server_test.go | 6 ++- 4 files changed, 71 insertions(+), 5 deletions(-) diff --git a/pkg/kubelet/kubelet_cadvisor.go b/pkg/kubelet/kubelet_cadvisor.go index 779095f5675..f87330cd0c5 100644 --- a/pkg/kubelet/kubelet_cadvisor.go +++ b/pkg/kubelet/kubelet_cadvisor.go @@ -76,6 +76,11 @@ func (kl *Kubelet) GetRawContainerInfo(containerName string, req *cadvisorapi.Co } } +// GetVersionInfo returns information about the version of cAdvisor in use. +func (kl *Kubelet) GetVersionInfo() (*cadvisorapi.VersionInfo, error) { + return kl.cadvisor.VersionInfo() +} + // GetCachedMachineInfo assumes that the machine info can't change without a reboot func (kl *Kubelet) GetCachedMachineInfo() (*cadvisorapi.MachineInfo, error) { if kl.machineInfo == nil { diff --git a/pkg/kubelet/server/BUILD b/pkg/kubelet/server/BUILD index 95a76a2f80e..94381c2f3c7 100644 --- a/pkg/kubelet/server/BUILD +++ b/pkg/kubelet/server/BUILD @@ -25,6 +25,7 @@ go_library( "//pkg/kubelet/server/remotecommand:go_default_library", "//pkg/kubelet/server/stats:go_default_library", "//pkg/kubelet/server/streaming:go_default_library", + "//pkg/kubelet/types:go_default_library", "//pkg/util/configz:go_default_library", "//pkg/util/limitwriter:go_default_library", "//pkg/volume:go_default_library", @@ -32,7 +33,9 @@ go_library( "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/info/v2:go_default_library", + "//vendor/github.com/google/cadvisor/metrics:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus/promhttp:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/pkg/kubelet/server/server.go b/pkg/kubelet/server/server.go index b9f33a967f9..e728daaf784 100644 --- a/pkg/kubelet/server/server.go +++ b/pkg/kubelet/server/server.go @@ -34,7 +34,9 @@ import ( "github.com/golang/glog" cadvisorapi "github.com/google/cadvisor/info/v1" cadvisorapiv2 "github.com/google/cadvisor/info/v2" + "github.com/google/cadvisor/metrics" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" @@ -58,16 +60,18 @@ import ( remotecommandserver "k8s.io/kubernetes/pkg/kubelet/server/remotecommand" "k8s.io/kubernetes/pkg/kubelet/server/stats" "k8s.io/kubernetes/pkg/kubelet/server/streaming" + kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/util/configz" "k8s.io/kubernetes/pkg/util/limitwriter" "k8s.io/kubernetes/pkg/volume" ) const ( - metricsPath = "/metrics" - specPath = "/spec/" - statsPath = "/stats/" - logsPath = "/logs/" + metricsPath = "/metrics" + cadvisorMetricsPath = "/metrics/cadvisor" + specPath = "/spec/" + statsPath = "/stats/" + logsPath = "/logs/" ) // Server is a http.Handler which exposes kubelet functionality over HTTP. @@ -169,6 +173,7 @@ type HostInterface interface { GetContainerInfo(podFullName string, uid types.UID, containerName string, req *cadvisorapi.ContainerInfoRequest) (*cadvisorapi.ContainerInfo, error) GetContainerInfoV2(name string, options cadvisorapiv2.RequestOptions) (map[string]cadvisorapiv2.ContainerInfo, error) GetRawContainerInfo(containerName string, req *cadvisorapi.ContainerInfoRequest, subcontainers bool) (map[string]*cadvisorapi.ContainerInfo, error) + GetVersionInfo() (*cadvisorapi.VersionInfo, error) GetCachedMachineInfo() (*cadvisorapi.MachineInfo, error) GetPods() []*v1.Pod GetRunningPods() ([]*v1.Pod, error) @@ -279,6 +284,13 @@ func (s *Server) InstallDefaultHandlers() { s.restfulCont.Add(stats.CreateHandlers(statsPath, s.host, s.resourceAnalyzer)) s.restfulCont.Handle(metricsPath, prometheus.Handler()) + // cAdvisor metrics are exposed under the secured handler as well + r := prometheus.NewRegistry() + r.MustRegister(metrics.NewPrometheusCollector(prometheusHostAdapter{s.host}, containerPrometheusLabels)) + s.restfulCont.Handle(cadvisorMetricsPath, + promhttp.HandlerFor(r, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}), + ) + ws = new(restful.WebService) ws. Path(specPath). @@ -778,3 +790,45 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { ).Log() s.restfulCont.ServeHTTP(w, req) } + +// prometheusHostAdapter adapts the HostInterface to the interface expected by the +// cAdvisor prometheus collector. +type prometheusHostAdapter struct { + host HostInterface +} + +func (a prometheusHostAdapter) SubcontainersInfo(containerName string, query *cadvisorapi.ContainerInfoRequest) ([]*cadvisorapi.ContainerInfo, error) { + all, err := a.host.GetRawContainerInfo(containerName, query, true) + items := make([]*cadvisorapi.ContainerInfo, 0, len(all)) + for _, v := range all { + items = append(items, v) + } + return items, err +} +func (a prometheusHostAdapter) GetVersionInfo() (*cadvisorapi.VersionInfo, error) { + return a.host.GetVersionInfo() +} +func (a prometheusHostAdapter) GetMachineInfo() (*cadvisorapi.MachineInfo, error) { + return a.host.GetCachedMachineInfo() +} + +// containerPrometheusLabels maps cAdvisor labels to prometheus labels. +func containerPrometheusLabels(c *cadvisorapi.ContainerInfo) map[string]string { + set := map[string]string{metrics.LabelID: c.Name} + if len(c.Aliases) > 0 { + set[metrics.LabelName] = c.Aliases[0] + } + if image := c.Spec.Image; len(image) > 0 { + set[metrics.LabelImage] = image + } + if v, ok := c.Spec.Labels[kubelettypes.KubernetesPodNameLabel]; ok { + set["pod_name"] = v + } + if v, ok := c.Spec.Labels[kubelettypes.KubernetesPodNamespaceLabel]; ok { + set["namespace"] = v + } + if v, ok := c.Spec.Labels[kubelettypes.KubernetesContainerNameLabel]; ok { + set["container_name"] = v + } + return set +} diff --git a/pkg/kubelet/server/server_test.go b/pkg/kubelet/server/server_test.go index 04ff198efc9..015947ddac4 100644 --- a/pkg/kubelet/server/server_test.go +++ b/pkg/kubelet/server/server_test.go @@ -110,6 +110,10 @@ func (fk *fakeKubelet) GetCachedMachineInfo() (*cadvisorapi.MachineInfo, error) return fk.machineInfoFunc() } +func (_ *fakeKubelet) GetVersionInfo() (*cadvisorapi.VersionInfo, error) { + return &cadvisorapi.VersionInfo{}, nil +} + func (fk *fakeKubelet) GetPods() []*v1.Pod { return fk.podsFunc() } @@ -592,7 +596,7 @@ func TestAuthFilters(t *testing.T) { // This is a sanity check that the Handle->HandleWithFilter() delegation is working // Ideally, these would move to registered web services and this list would get shorter - expectedPaths := []string{"/healthz", "/metrics"} + expectedPaths := []string{"/healthz", "/metrics", "/metrics/cadvisor"} paths := sets.NewString(fw.serverUnderTest.restfulCont.RegisteredHandlePaths()...) for _, expectedPath := range expectedPaths { if !paths.Has(expectedPath) {