From 6d9264247d491e4be2b135b25bfa8c8b2f8efc6b Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Mon, 21 Mar 2022 15:10:07 -0400 Subject: [PATCH 1/2] kubelet/stats: deduplicate makePodStorageStats Signed-off-by: Peter Hunt --- pkg/kubelet/stats/cadvisor_stats_provider.go | 20 ++----------- pkg/kubelet/stats/cri_stats_provider.go | 30 ++------------------ pkg/kubelet/stats/helper.go | 28 ++++++++++++++++++ 3 files changed, 32 insertions(+), 46 deletions(-) diff --git a/pkg/kubelet/stats/cadvisor_stats_provider.go b/pkg/kubelet/stats/cadvisor_stats_provider.go index f58cd43e3d9..f6a8680a39f 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider.go @@ -142,25 +142,9 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { // Add each PodStats to the result. result := make([]statsapi.PodStats, 0, len(podToStats)) for _, podStats := range podToStats { - // Lookup the volume stats for each pod. + makePodStorageStats(podStats, &rootFsInfo, p.resourceAnalyzer, p.hostStatsProvider, false) + podUID := types.UID(podStats.PodRef.UID) - var ephemeralStats []statsapi.VolumeStats - if vstats, found := p.resourceAnalyzer.GetPodVolumeStats(podUID); found { - ephemeralStats = make([]statsapi.VolumeStats, len(vstats.EphemeralVolumes)) - copy(ephemeralStats, vstats.EphemeralVolumes) - podStats.VolumeStats = append(append([]statsapi.VolumeStats{}, vstats.EphemeralVolumes...), vstats.PersistentVolumes...) - } - - logStats, err := p.hostStatsProvider.getPodLogStats(podStats.PodRef.Namespace, podStats.PodRef.Name, podUID, &rootFsInfo) - if err != nil { - klog.ErrorS(err, "Unable to fetch pod log stats", "pod", klog.KRef(podStats.PodRef.Namespace, podStats.PodRef.Name)) - } - etcHostsStats, err := p.hostStatsProvider.getPodEtcHostsStats(podUID, &rootFsInfo) - if err != nil { - klog.ErrorS(err, "Unable to fetch pod etc hosts stats", "pod", klog.KRef(podStats.PodRef.Namespace, podStats.PodRef.Name)) - } - - podStats.EphemeralStorage = calcEphemeralStorage(podStats.Containers, ephemeralStats, &rootFsInfo, logStats, etcHostsStats, false) // Lookup the pod-level cgroup's CPU and memory stats podInfo := getCadvisorPodInfoFromPodUID(podUID, allInfos) if podInfo != nil { diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go index fa9c64e468f..c220795fec6 100644 --- a/pkg/kubelet/stats/cri_stats_provider.go +++ b/pkg/kubelet/stats/cri_stats_provider.go @@ -220,7 +220,7 @@ func (p *criStatsProvider) listPodStatsPartiallyFromCRI(updateCPUNanoCoreUsage b result := make([]statsapi.PodStats, 0, len(sandboxIDToPodStats)) for _, s := range sandboxIDToPodStats { - p.makePodStorageStats(s, rootFsInfo) + makePodStorageStats(s, rootFsInfo, p.resourceAnalyzer, p.hostStatsProvider, true) result = append(result, *s) } return result, nil @@ -257,7 +257,7 @@ func (p *criStatsProvider) listPodStatsStrictlyFromCRI(updateCPUNanoCoreUsage bo addCRIPodCPUStats(ps, criSandboxStat) addCRIPodMemoryStats(ps, criSandboxStat) addCRIPodProcessStats(ps, criSandboxStat) - p.makePodStorageStats(ps, rootFsInfo) + makePodStorageStats(ps, rootFsInfo, p.resourceAnalyzer, p.hostStatsProvider, true) summarySandboxStats = append(summarySandboxStats, *ps) } return summarySandboxStats, nil @@ -469,32 +469,6 @@ func buildPodStats(podSandbox *runtimeapi.PodSandbox) *statsapi.PodStats { } } -func (p *criStatsProvider) makePodStorageStats(s *statsapi.PodStats, rootFsInfo *cadvisorapiv2.FsInfo) { - podNs := s.PodRef.Namespace - podName := s.PodRef.Name - podUID := types.UID(s.PodRef.UID) - vstats, found := p.resourceAnalyzer.GetPodVolumeStats(podUID) - if !found { - return - } - logStats, err := p.hostStatsProvider.getPodLogStats(podNs, podName, podUID, rootFsInfo) - if err != nil { - klog.ErrorS(err, "Unable to fetch pod log stats", "pod", klog.KRef(podNs, podName)) - // If people do in-place upgrade, there might be pods still using - // the old log path. For those pods, no pod log stats is returned. - // We should continue generating other stats in that case. - // calcEphemeralStorage tolerants logStats == nil. - } - etcHostsStats, err := p.hostStatsProvider.getPodEtcHostsStats(podUID, rootFsInfo) - if err != nil { - klog.ErrorS(err, "Unable to fetch pod etc hosts stats", "pod", klog.KRef(podNs, podName)) - } - ephemeralStats := make([]statsapi.VolumeStats, len(vstats.EphemeralVolumes)) - copy(ephemeralStats, vstats.EphemeralVolumes) - s.VolumeStats = append(append([]statsapi.VolumeStats{}, vstats.EphemeralVolumes...), vstats.PersistentVolumes...) - s.EphemeralStorage = calcEphemeralStorage(s.Containers, ephemeralStats, rootFsInfo, logStats, etcHostsStats, true) -} - func (p *criStatsProvider) addPodNetworkStats( ps *statsapi.PodStats, podSandboxID string, diff --git a/pkg/kubelet/stats/helper.go b/pkg/kubelet/stats/helper.go index fdab200a005..ff9ec874dfa 100644 --- a/pkg/kubelet/stats/helper.go +++ b/pkg/kubelet/stats/helper.go @@ -23,9 +23,11 @@ import ( cadvisorapiv1 "github.com/google/cadvisor/info/v1" cadvisorapiv2 "github.com/google/cadvisor/info/v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" + "k8s.io/kubernetes/pkg/kubelet/server/stats" ) // defaultNetworkInterfaceName is used for collectng network stats. @@ -415,3 +417,29 @@ func addUsage(first, second *uint64) *uint64 { total := *first + *second return &total } + +func makePodStorageStats(s *statsapi.PodStats, rootFsInfo *cadvisorapiv2.FsInfo, resourceAnalyzer stats.ResourceAnalyzer, hostStatsProvider HostStatsProvider, isCRIStatsProvider bool) { + podNs := s.PodRef.Namespace + podName := s.PodRef.Name + podUID := types.UID(s.PodRef.UID) + var ephemeralStats []statsapi.VolumeStats + if vstats, found := resourceAnalyzer.GetPodVolumeStats(podUID); found { + ephemeralStats = make([]statsapi.VolumeStats, len(vstats.EphemeralVolumes)) + copy(ephemeralStats, vstats.EphemeralVolumes) + s.VolumeStats = append(append([]statsapi.VolumeStats{}, vstats.EphemeralVolumes...), vstats.PersistentVolumes...) + + } + logStats, err := hostStatsProvider.getPodLogStats(podNs, podName, podUID, rootFsInfo) + if err != nil { + klog.ErrorS(err, "Unable to fetch pod log stats", "pod", klog.KRef(podNs, podName)) + // If people do in-place upgrade, there might be pods still using + // the old log path. For those pods, no pod log stats is returned. + // We should continue generating other stats in that case. + // calcEphemeralStorage tolerants logStats == nil. + } + etcHostsStats, err := hostStatsProvider.getPodEtcHostsStats(podUID, rootFsInfo) + if err != nil { + klog.ErrorS(err, "Unable to fetch pod etc hosts stats", "pod", klog.KRef(podNs, podName)) + } + s.EphemeralStorage = calcEphemeralStorage(s.Containers, ephemeralStats, rootFsInfo, logStats, etcHostsStats, isCRIStatsProvider) +} From d6ffca04c9293627daa8c34e08774af2dc5ef6fd Mon Sep 17 00:00:00 2001 From: Peter Hunt~ Date: Tue, 16 Aug 2022 16:45:19 -0400 Subject: [PATCH 2/2] kubelet/stats: drop makePodStorageStats errors to V(6) and by doing so, fix a bug where the stats providers report a directory is not found after a pod's storage is removed Signed-off-by: Peter Hunt --- pkg/kubelet/stats/helper.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/kubelet/stats/helper.go b/pkg/kubelet/stats/helper.go index ff9ec874dfa..dd968b06eea 100644 --- a/pkg/kubelet/stats/helper.go +++ b/pkg/kubelet/stats/helper.go @@ -431,7 +431,7 @@ func makePodStorageStats(s *statsapi.PodStats, rootFsInfo *cadvisorapiv2.FsInfo, } logStats, err := hostStatsProvider.getPodLogStats(podNs, podName, podUID, rootFsInfo) if err != nil { - klog.ErrorS(err, "Unable to fetch pod log stats", "pod", klog.KRef(podNs, podName)) + klog.V(6).ErrorS(err, "Unable to fetch pod log stats", "pod", klog.KRef(podNs, podName)) // If people do in-place upgrade, there might be pods still using // the old log path. For those pods, no pod log stats is returned. // We should continue generating other stats in that case. @@ -439,7 +439,7 @@ func makePodStorageStats(s *statsapi.PodStats, rootFsInfo *cadvisorapiv2.FsInfo, } etcHostsStats, err := hostStatsProvider.getPodEtcHostsStats(podUID, rootFsInfo) if err != nil { - klog.ErrorS(err, "Unable to fetch pod etc hosts stats", "pod", klog.KRef(podNs, podName)) + klog.V(6).ErrorS(err, "Unable to fetch pod etc hosts stats", "pod", klog.KRef(podNs, podName)) } s.EphemeralStorage = calcEphemeralStorage(s.Containers, ephemeralStats, rootFsInfo, logStats, etcHostsStats, isCRIStatsProvider) }