github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 11:50:44 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #51473 from bboreham/cadvisor-consistent-labels

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 51471, 50561, 50435, 51473, 51436)

Fix inconsistent Prometheus cAdvisor metrics

**What this PR does / why we need it**:

We need this because otherwise kubelet is exposing different sets of Prometheus metrics that randomly include or do not include containers.

See also https://github.com/google/cadvisor/issues/1704; quoting here:

Prometheus requires that all metrics in the same family have the same labels, so we arrange to supply blank strings for missing labels

The function `containerPrometheusLabels()` conditionally adds various metric labels from container labels - pod name, image, etc. However, when it receives the metrics, Prometheus [checks](https://github.com/prometheus/client_golang/blob/master/prometheus/registry.go#L665) that all metrics in the same family have the same label set, and [rejects](https://github.com/prometheus/client_golang/blob/master/prometheus/registry.go#L497) those that do not.

Since containers are collected in (somewhat) random order, depending on which kind is seen first you get one set of metrics or the other.

Changing the container labels function to always add the same set of labels, adding `""` when it doesn't have a real value, eliminates the issue in my testing.

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #

Fixes #50151

**Special notes for your reviewer**:

I have made the same fix in two places.  I am 98% sure the one in `cadvisor_linux.go` isn't used and indeed cannot be used, but have not gone fully down that rabbit-hole.

**Release note**:
```release-note
Fix inconsistent Prometheus cAdvisor metrics
```
This commit is contained in:
Kubernetes Submit Queue 2017-08-29 02:22:16 -07:00 committed by GitHub
commit cc557e61cc
2 changed files with 32 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
pkg/kubelet/cadvisor/cadvisor_linux.go
View File

@ -78,21 +78,29 @@ func init() {
}
func containerLabels(c *cadvisorapi.ContainerInfo) map[string]string {
set := map[string]string{metrics.LabelID: c.Name}
// Prometheus requires that all metrics in the same family have the same labels,
// so we arrange to supply blank strings for missing labels
var name, image, podName, namespace, containerName string
if len(c.Aliases) > 0 {
set[metrics.LabelName] = c.Aliases[0]
}
if image := c.Spec.Image; len(image) > 0 {
set[metrics.LabelImage] = image
name = c.Aliases[0]
}
image = c.Spec.Image
if v, ok := c.Spec.Labels[types.KubernetesPodNameLabel]; ok {
set["pod_name"] = v
podName = v
}
if v, ok := c.Spec.Labels[types.KubernetesPodNamespaceLabel]; ok {
set["namespace"] = v
namespace = v
}
if v, ok := c.Spec.Labels[types.KubernetesContainerNameLabel]; ok {
set["container_name"] = v
containerName = v
}
set := map[string]string{
metrics.LabelID: c.Name,
metrics.LabelName: name,
metrics.LabelImage: image,
"pod_name": podName,
"namespace": namespace,
"container_name": containerName,
}
return set
}

24
pkg/kubelet/server/server.go
View File

@ -801,21 +801,29 @@ func (a prometheusHostAdapter) GetMachineInfo() (*cadvisorapi.MachineInfo, error
// containerPrometheusLabels maps cAdvisor labels to prometheus labels.
func containerPrometheusLabels(c *cadvisorapi.ContainerInfo) map[string]string {
set := map[string]string{metrics.LabelID: c.Name}
// Prometheus requires that all metrics in the same family have the same labels,
// so we arrange to supply blank strings for missing labels
var name, image, podName, namespace, containerName string
if len(c.Aliases) > 0 {
set[metrics.LabelName] = c.Aliases[0]
}
if image := c.Spec.Image; len(image) > 0 {
set[metrics.LabelImage] = image
name = c.Aliases[0]
}
image = c.Spec.Image
if v, ok := c.Spec.Labels[kubelettypes.KubernetesPodNameLabel]; ok {
set["pod_name"] = v
podName = v
}
if v, ok := c.Spec.Labels[kubelettypes.KubernetesPodNamespaceLabel]; ok {
set["namespace"] = v
namespace = v
}
if v, ok := c.Spec.Labels[kubelettypes.KubernetesContainerNameLabel]; ok {
set["container_name"] = v
containerName = v
}
set := map[string]string{
metrics.LabelID: c.Name,
metrics.LabelName: name,
metrics.LabelImage: image,
"pod_name": podName,
"namespace": namespace,
"container_name": containerName,
}
return set
}