diff --git a/pkg/kubelet/stats/cadvisor_stats_provider.go b/pkg/kubelet/stats/cadvisor_stats_provider.go index 7fa1dc64d1f..6e4655fdcde 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider.go @@ -91,12 +91,11 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { if err != nil { return nil, fmt.Errorf("failed to get container info from cadvisor: %v", err) } - // removeTerminatedContainerInfo will also remove pod level cgroups, so save the infos into allInfos first - allInfos := infos - infos = removeTerminatedContainerInfo(infos) + + filteredInfos, allInfos := filterTerminatedContainerInfoAndAssembleByPodCgroupKey(infos) // Map each container to a pod and update the PodStats with container data. podToStats := map[statsapi.PodReference]*statsapi.PodStats{} - for key, cinfo := range infos { + for key, cinfo := range filteredInfos { // On systemd using devicemapper each mount into the container has an // associated cgroup. We ignore them to ensure we do not get duplicate // entries in our summary. For details on .mount units: @@ -186,12 +185,10 @@ func (p *cadvisorStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, if err != nil { return nil, fmt.Errorf("failed to get container info from cadvisor: %v", err) } - // removeTerminatedContainerInfo will also remove pod level cgroups, so save the infos into allInfos first - allInfos := infos - infos = removeTerminatedContainerInfo(infos) + filteredInfos, allInfos := filterTerminatedContainerInfoAndAssembleByPodCgroupKey(infos) // Map each container to a pod and update the PodStats with container data. podToStats := map[statsapi.PodReference]*statsapi.PodStats{} - for key, cinfo := range infos { + for key, cinfo := range filteredInfos { // On systemd using devicemapper each mount into the container has an // associated cgroup. We ignore them to ensure we do not get duplicate // entries in our summary. For details on .mount units: @@ -303,30 +300,32 @@ func isPodManagedContainer(cinfo *cadvisorapiv2.ContainerInfo) bool { // getCadvisorPodInfoFromPodUID returns a pod cgroup information by matching the podUID with its CgroupName identifier base name func getCadvisorPodInfoFromPodUID(podUID types.UID, infos map[string]cadvisorapiv2.ContainerInfo) *cadvisorapiv2.ContainerInfo { - for key, info := range infos { - if cm.IsSystemdStyleName(key) { - // Convert to internal cgroup name and take the last component only. - internalCgroupName := cm.ParseSystemdToCgroupName(key) - key = internalCgroupName[len(internalCgroupName)-1] - } else { - // Take last component only. - key = path.Base(key) - } - if cm.GetPodCgroupNameSuffix(podUID) == key { - return &info - } + if info, found := infos[cm.GetPodCgroupNameSuffix(podUID)]; found { + return &info } return nil } -// removeTerminatedContainerInfo returns the specified containerInfo but with -// the stats of the terminated containers removed. -// +// filterTerminatedContainerInfoAndAssembleByPodCgroupKey returns the specified containerInfo but with +// the stats of the terminated containers removed and all containerInfos assembled by pod cgroup key. +// the first return map is container cgroup name <-> ContainerInfo and +// the second return map is pod cgroup key <-> ContainerInfo. // A ContainerInfo is considered to be of a terminated container if it has an // older CreationTime and zero CPU instantaneous and memory RSS usage. -func removeTerminatedContainerInfo(containerInfo map[string]cadvisorapiv2.ContainerInfo) map[string]cadvisorapiv2.ContainerInfo { +func filterTerminatedContainerInfoAndAssembleByPodCgroupKey(containerInfo map[string]cadvisorapiv2.ContainerInfo) (map[string]cadvisorapiv2.ContainerInfo, map[string]cadvisorapiv2.ContainerInfo) { cinfoMap := make(map[containerID][]containerInfoWithCgroup) + cinfosByPodCgroupKey := make(map[string]cadvisorapiv2.ContainerInfo) for key, cinfo := range containerInfo { + var podCgroupKey string + if cm.IsSystemdStyleName(key) { + // Convert to internal cgroup name and take the last component only. + internalCgroupName := cm.ParseSystemdToCgroupName(key) + podCgroupKey = internalCgroupName[len(internalCgroupName)-1] + } else { + // Take last component only. + podCgroupKey = path.Base(key) + } + cinfosByPodCgroupKey[podCgroupKey] = cinfo if !isPodManagedContainer(&cinfo) { continue } @@ -353,7 +352,7 @@ func removeTerminatedContainerInfo(containerInfo map[string]cadvisorapiv2.Contai } } } - return result + return result, cinfosByPodCgroupKey } // ByCreationTime implements sort.Interface for []containerInfoWithCgroup based diff --git a/pkg/kubelet/stats/cadvisor_stats_provider_test.go b/pkg/kubelet/stats/cadvisor_stats_provider_test.go index fd3c38c739e..8e18376e29f 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider_test.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider_test.go @@ -34,7 +34,7 @@ import ( statustest "k8s.io/kubernetes/pkg/kubelet/status/testing" ) -func TestRemoveTerminatedContainerInfo(t *testing.T) { +func TestFilterTerminatedContainerInfoAndAssembleByPodCgroupKey(t *testing.T) { const ( seedPastPod0Infra = 1000 seedPastPod0Container0 = 2000 @@ -63,10 +63,16 @@ func TestRemoveTerminatedContainerInfo(t *testing.T) { "/pod0-i": getTestContainerInfo(seedPod0Infra, pName0, namespace, leaky.PodInfraContainerName), "/pod0-c0": getTestContainerInfo(seedPod0Container0, pName0, namespace, cName00), } - output := removeTerminatedContainerInfo(infos) - assert.Len(t, output, 2) + filteredInfos, allInfos := filterTerminatedContainerInfoAndAssembleByPodCgroupKey(infos) + assert.Len(t, filteredInfos, 2) + assert.Len(t, allInfos, 6) for _, c := range []string{"/pod0-i", "/pod0-c0"} { - if _, found := output[c]; !found { + if _, found := filteredInfos[c]; !found { + t.Errorf("%q is expected to be in the output\n", c) + } + } + for _, c := range []string{"pod0-i-terminated-1", "pod0-c0-terminated-1", "pod0-i-terminated-2", "pod0-c0-terminated-2", "pod0-i", "pod0-c0"} { + if _, found := allInfos[c]; !found { t.Errorf("%q is expected to be in the output\n", c) } } diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go index 9c959c3c26d..e68aa7c42ac 100644 --- a/pkg/kubelet/stats/cri_stats_provider.go +++ b/pkg/kubelet/stats/cri_stats_provider.go @@ -161,7 +161,7 @@ func (p *criStatsProvider) listPodStats(updateCPUNanoCoreUsage bool) ([]statsapi if err != nil { return nil, fmt.Errorf("failed to fetch cadvisor stats: %v", err) } - caInfos := getCRICadvisorStats(allInfos) + caInfos, allInfos := getCRICadvisorStats(allInfos) // get network stats for containers. // This is only used on Windows. For other platforms, (nil, nil) should be returned. @@ -255,7 +255,7 @@ func (p *criStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, erro if err != nil { return nil, fmt.Errorf("failed to fetch cadvisor stats: %v", err) } - caInfos := getCRICadvisorStats(allInfos) + caInfos, allInfos := getCRICadvisorStats(allInfos) for _, stats := range resp { containerID := stats.Attributes.Id @@ -811,10 +811,10 @@ func (p *criStatsProvider) addCadvisorContainerCPUAndMemoryStats( } } -func getCRICadvisorStats(infos map[string]cadvisorapiv2.ContainerInfo) map[string]cadvisorapiv2.ContainerInfo { +func getCRICadvisorStats(infos map[string]cadvisorapiv2.ContainerInfo) (map[string]cadvisorapiv2.ContainerInfo, map[string]cadvisorapiv2.ContainerInfo) { stats := make(map[string]cadvisorapiv2.ContainerInfo) - infos = removeTerminatedContainerInfo(infos) - for key, info := range infos { + filteredInfos, cinfosByPodCgroupKey := filterTerminatedContainerInfoAndAssembleByPodCgroupKey(infos) + for key, info := range filteredInfos { // On systemd using devicemapper each mount into the container has an // associated cgroup. We ignore them to ensure we do not get duplicate // entries in our summary. For details on .mount units: @@ -828,7 +828,7 @@ func getCRICadvisorStats(infos map[string]cadvisorapiv2.ContainerInfo) map[strin } stats[extractIDFromCgroupPath(key)] = info } - return stats + return stats, cinfosByPodCgroupKey } func extractIDFromCgroupPath(cgroupPath string) string {