Merge pull request #49079 from smarterclayton/restore_metrics

Automatic merge from submit-queue

Restore cAdvisor prometheus metrics to the main port

But under a new path - `/metrics/cadvisor`. This ensures a secure port still exists for metrics while getting the benefit of separating out container metrics from the kubelet's metrics as recommended in the linked issue.

Fixes #48483

```release-note-action-required
Restored cAdvisor prometheus metrics to the main port -- a regression that existed in v1.7.0-v1.7.2
cAdvisor metrics can now be scraped from `/metrics/cadvisor` on the kubelet ports.
Note that you have to update your scraping jobs to get kubelet-only metrics from `/metrics` and `container_*` metrics from `/metrics/cadvisor`
```

pkg/kubelet/kubelet_cadvisor.go

@@ -76,6 +76,11 @@ func (kl *Kubelet) GetRawContainerInfo(containerName string, req *cadvisorapi.Co
 	}
 }
 
+// GetVersionInfo returns information about the version of cAdvisor in use.
+func (kl *Kubelet) GetVersionInfo() (*cadvisorapi.VersionInfo, error) {
+	return kl.cadvisor.VersionInfo()
+}
+
 // GetCachedMachineInfo assumes that the machine info can't change without a reboot
 func (kl *Kubelet) GetCachedMachineInfo() (*cadvisorapi.MachineInfo, error) {
 	if kl.machineInfo == nil {

pkg/kubelet/server/BUILD

@@ -25,6 +25,7 @@ go_library(
         "//pkg/kubelet/server/remotecommand:go_default_library",
         "//pkg/kubelet/server/stats:go_default_library",
         "//pkg/kubelet/server/streaming:go_default_library",
+        "//pkg/kubelet/types:go_default_library",
         "//pkg/util/configz:go_default_library",
         "//pkg/util/limitwriter:go_default_library",
         "//pkg/volume:go_default_library",
@@ -32,7 +33,9 @@ go_library(
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v2:go_default_library",
+        "//vendor/github.com/google/cadvisor/metrics:go_default_library",
         "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus/promhttp:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

pkg/kubelet/server/server.go

@@ -34,7 +34,9 @@ import (
 	"github.com/golang/glog"
 	cadvisorapi "github.com/google/cadvisor/info/v1"
 	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
+	"github.com/google/cadvisor/metrics"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	"k8s.io/api/core/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
@@ -58,16 +60,18 @@ import (
 	remotecommandserver "k8s.io/kubernetes/pkg/kubelet/server/remotecommand"
 	"k8s.io/kubernetes/pkg/kubelet/server/stats"
 	"k8s.io/kubernetes/pkg/kubelet/server/streaming"
+	kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/pkg/util/configz"
 	"k8s.io/kubernetes/pkg/util/limitwriter"
 	"k8s.io/kubernetes/pkg/volume"
 )
 
 const (
-	metricsPath = "/metrics"
-	specPath    = "/spec/"
-	statsPath   = "/stats/"
-	logsPath    = "/logs/"
+	metricsPath         = "/metrics"
+	cadvisorMetricsPath = "/metrics/cadvisor"
+	specPath            = "/spec/"
+	statsPath           = "/stats/"
+	logsPath            = "/logs/"
 )
 
 // Server is a http.Handler which exposes kubelet functionality over HTTP.
@@ -169,6 +173,7 @@ type HostInterface interface {
 	GetContainerInfo(podFullName string, uid types.UID, containerName string, req *cadvisorapi.ContainerInfoRequest) (*cadvisorapi.ContainerInfo, error)
 	GetContainerInfoV2(name string, options cadvisorapiv2.RequestOptions) (map[string]cadvisorapiv2.ContainerInfo, error)
 	GetRawContainerInfo(containerName string, req *cadvisorapi.ContainerInfoRequest, subcontainers bool) (map[string]*cadvisorapi.ContainerInfo, error)
+	GetVersionInfo() (*cadvisorapi.VersionInfo, error)
 	GetCachedMachineInfo() (*cadvisorapi.MachineInfo, error)
 	GetPods() []*v1.Pod
 	GetRunningPods() ([]*v1.Pod, error)
@@ -279,6 +284,13 @@ func (s *Server) InstallDefaultHandlers() {
 	s.restfulCont.Add(stats.CreateHandlers(statsPath, s.host, s.resourceAnalyzer))
 	s.restfulCont.Handle(metricsPath, prometheus.Handler())
 
+	// cAdvisor metrics are exposed under the secured handler as well
+	r := prometheus.NewRegistry()
+	r.MustRegister(metrics.NewPrometheusCollector(prometheusHostAdapter{s.host}, containerPrometheusLabels))
+	s.restfulCont.Handle(cadvisorMetricsPath,
+		promhttp.HandlerFor(r, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}),
+	)
+
 	ws = new(restful.WebService)
 	ws.
 		Path(specPath).
@@ -778,3 +790,45 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	).Log()
 	s.restfulCont.ServeHTTP(w, req)
 }
+
+// prometheusHostAdapter adapts the HostInterface to the interface expected by the
+// cAdvisor prometheus collector.
+type prometheusHostAdapter struct {
+	host HostInterface
+}
+
+func (a prometheusHostAdapter) SubcontainersInfo(containerName string, query *cadvisorapi.ContainerInfoRequest) ([]*cadvisorapi.ContainerInfo, error) {
+	all, err := a.host.GetRawContainerInfo(containerName, query, true)
+	items := make([]*cadvisorapi.ContainerInfo, 0, len(all))
+	for _, v := range all {
+		items = append(items, v)
+	}
+	return items, err
+}
+func (a prometheusHostAdapter) GetVersionInfo() (*cadvisorapi.VersionInfo, error) {
+	return a.host.GetVersionInfo()
+}
+func (a prometheusHostAdapter) GetMachineInfo() (*cadvisorapi.MachineInfo, error) {
+	return a.host.GetCachedMachineInfo()
+}
+
+// containerPrometheusLabels maps cAdvisor labels to prometheus labels.
+func containerPrometheusLabels(c *cadvisorapi.ContainerInfo) map[string]string {
+	set := map[string]string{metrics.LabelID: c.Name}
+	if len(c.Aliases) > 0 {
+		set[metrics.LabelName] = c.Aliases[0]
+	}
+	if image := c.Spec.Image; len(image) > 0 {
+		set[metrics.LabelImage] = image
+	}
+	if v, ok := c.Spec.Labels[kubelettypes.KubernetesPodNameLabel]; ok {
+		set["pod_name"] = v
+	}
+	if v, ok := c.Spec.Labels[kubelettypes.KubernetesPodNamespaceLabel]; ok {
+		set["namespace"] = v
+	}
+	if v, ok := c.Spec.Labels[kubelettypes.KubernetesContainerNameLabel]; ok {
+		set["container_name"] = v
+	}
+	return set
+}

pkg/kubelet/server/server_test.go

@@ -110,6 +110,10 @@ func (fk *fakeKubelet) GetCachedMachineInfo() (*cadvisorapi.MachineInfo, error)
 	return fk.machineInfoFunc()
 }
 
+func (_ *fakeKubelet) GetVersionInfo() (*cadvisorapi.VersionInfo, error) {
+	return &cadvisorapi.VersionInfo{}, nil
+}
+
 func (fk *fakeKubelet) GetPods() []*v1.Pod {
 	return fk.podsFunc()
 }
@@ -592,7 +596,7 @@ func TestAuthFilters(t *testing.T) {
 
 	// This is a sanity check that the Handle->HandleWithFilter() delegation is working
 	// Ideally, these would move to registered web services and this list would get shorter
-	expectedPaths := []string{"/healthz", "/metrics"}
+	expectedPaths := []string{"/healthz", "/metrics", "/metrics/cadvisor"}
 	paths := sets.NewString(fw.serverUnderTest.restfulCont.RegisteredHandlePaths()...)
 	for _, expectedPath := range expectedPaths {
 		if !paths.Has(expectedPath) {