From 14952cef5b78bb3d78cc887f409f9cc4a6a25dbe Mon Sep 17 00:00:00 2001 From: mansikulkarni96 Date: Wed, 29 Mar 2023 00:45:22 -0400 Subject: [PATCH 1/2] kubelet: Move Linux stats to a Linux specific file This commit moves the Linux stats from CRI to a linux specific file and adds unimplemented methods for platform other than Linux and Windows. --- pkg/kubelet/stats/cri_stats_provider.go | 63 +---------- pkg/kubelet/stats/cri_stats_provider_linux.go | 105 ++++++++++++++++++ .../stats/cri_stats_provider_others.go | 25 ++++- 3 files changed, 129 insertions(+), 64 deletions(-) create mode 100644 pkg/kubelet/stats/cri_stats_provider_linux.go diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go index d1e1d90b275..72b65af3547 100644 --- a/pkg/kubelet/stats/cri_stats_provider.go +++ b/pkg/kubelet/stats/cri_stats_provider.go @@ -247,15 +247,7 @@ func (p *criStatsProvider) listPodStatsStrictlyFromCRI(ctx context.Context, upda continue } ps := buildPodStats(podSandbox) - for _, criContainerStat := range criSandboxStat.Linux.Containers { - container, found := containerMap[criContainerStat.Attributes.Id] - if !found { - continue - } - // Fill available stats for full set of required pod stats - cs := p.makeContainerStats(criContainerStat, container, rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata(), updateCPUNanoCoreUsage) - ps.Containers = append(ps.Containers, *cs) - } + p.addCRIPodContainerStats(criSandboxStat, ps, fsIDtoInfo, containerMap, podSandbox, rootFsInfo, updateCPUNanoCoreUsage) addCRIPodNetworkStats(ps, criSandboxStat) addCRIPodCPUStats(ps, criSandboxStat) addCRIPodMemoryStats(ps, criSandboxStat) @@ -914,22 +906,6 @@ func extractIDFromCgroupPath(cgroupPath string) string { return id } -func addCRIPodNetworkStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) { - if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Network == nil { - return - } - criNetwork := criPodStat.Linux.Network - iStats := statsapi.NetworkStats{ - Time: metav1.NewTime(time.Unix(0, criNetwork.Timestamp)), - InterfaceStats: criInterfaceToSummary(criNetwork.DefaultInterface), - Interfaces: make([]statsapi.InterfaceStats, 0, len(criNetwork.Interfaces)), - } - for _, iface := range criNetwork.Interfaces { - iStats.Interfaces = append(iStats.Interfaces, criInterfaceToSummary(iface)) - } - ps.Network = &iStats -} - func criInterfaceToSummary(criIface *runtimeapi.NetworkInterfaceUsage) statsapi.InterfaceStats { return statsapi.InterfaceStats{ Name: criIface.Name, @@ -940,43 +916,6 @@ func criInterfaceToSummary(criIface *runtimeapi.NetworkInterfaceUsage) statsapi. } } -func addCRIPodCPUStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) { - if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Cpu == nil { - return - } - criCPU := criPodStat.Linux.Cpu - ps.CPU = &statsapi.CPUStats{ - Time: metav1.NewTime(time.Unix(0, criCPU.Timestamp)), - UsageNanoCores: valueOfUInt64Value(criCPU.UsageNanoCores), - UsageCoreNanoSeconds: valueOfUInt64Value(criCPU.UsageCoreNanoSeconds), - } -} - -func addCRIPodMemoryStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) { - if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Memory == nil { - return - } - criMemory := criPodStat.Linux.Memory - ps.Memory = &statsapi.MemoryStats{ - Time: metav1.NewTime(time.Unix(0, criMemory.Timestamp)), - AvailableBytes: valueOfUInt64Value(criMemory.AvailableBytes), - UsageBytes: valueOfUInt64Value(criMemory.UsageBytes), - WorkingSetBytes: valueOfUInt64Value(criMemory.WorkingSetBytes), - RSSBytes: valueOfUInt64Value(criMemory.RssBytes), - PageFaults: valueOfUInt64Value(criMemory.PageFaults), - MajorPageFaults: valueOfUInt64Value(criMemory.MajorPageFaults), - } -} - -func addCRIPodProcessStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) { - if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Process == nil { - return - } - ps.ProcessStats = &statsapi.ProcessStats{ - ProcessCount: valueOfUInt64Value(criPodStat.Linux.Process.ProcessCount), - } -} - func valueOfUInt64Value(value *runtimeapi.UInt64Value) *uint64 { if value == nil { return nil diff --git a/pkg/kubelet/stats/cri_stats_provider_linux.go b/pkg/kubelet/stats/cri_stats_provider_linux.go new file mode 100644 index 00000000000..2bfa0fd48f3 --- /dev/null +++ b/pkg/kubelet/stats/cri_stats_provider_linux.go @@ -0,0 +1,105 @@ +//go:build linux +// +build linux + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + "time" + + cadvisorapiv2 "github.com/google/cadvisor/info/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1" +) + +func (p *criStatsProvider) addCRIPodContainerStats(criSandboxStat *runtimeapi.PodSandboxStats, + ps *statsapi.PodStats, fsIDtoInfo map[runtimeapi.FilesystemIdentifier]*cadvisorapiv2.FsInfo, + containerMap map[string]*runtimeapi.Container, + podSandbox *runtimeapi.PodSandbox, + rootFsInfo *cadvisorapiv2.FsInfo, updateCPUNanoCoreUsage bool) { + for _, criContainerStat := range criSandboxStat.Linux.Containers { + container, found := containerMap[criContainerStat.Attributes.Id] + if !found { + continue + } + // Fill available stats for full set of required pod stats + cs := p.makeContainerStats(criContainerStat, container, rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata(), + updateCPUNanoCoreUsage) + ps.Containers = append(ps.Containers, *cs) + } +} + +func addCRIPodNetworkStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) { + if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Network == nil { + return + } + criNetwork := criPodStat.Linux.Network + iStats := statsapi.NetworkStats{ + Time: metav1.NewTime(time.Unix(0, criNetwork.Timestamp)), + InterfaceStats: criInterfaceToSummary(criNetwork.DefaultInterface), + Interfaces: make([]statsapi.InterfaceStats, 0, len(criNetwork.Interfaces)), + } + for _, iface := range criNetwork.Interfaces { + iStats.Interfaces = append(iStats.Interfaces, criInterfaceToSummary(iface)) + } + ps.Network = &iStats +} + +func addCRIPodMemoryStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) { + if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Memory == nil { + return + } + criMemory := criPodStat.Linux.Memory + ps.Memory = &statsapi.MemoryStats{ + Time: metav1.NewTime(time.Unix(0, criMemory.Timestamp)), + AvailableBytes: valueOfUInt64Value(criMemory.AvailableBytes), + UsageBytes: valueOfUInt64Value(criMemory.UsageBytes), + WorkingSetBytes: valueOfUInt64Value(criMemory.WorkingSetBytes), + RSSBytes: valueOfUInt64Value(criMemory.RssBytes), + PageFaults: valueOfUInt64Value(criMemory.PageFaults), + MajorPageFaults: valueOfUInt64Value(criMemory.MajorPageFaults), + } +} + +func addCRIPodCPUStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) { + if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Cpu == nil { + return + } + criCPU := criPodStat.Linux.Cpu + ps.CPU = &statsapi.CPUStats{ + Time: metav1.NewTime(time.Unix(0, criCPU.Timestamp)), + UsageNanoCores: valueOfUInt64Value(criCPU.UsageNanoCores), + UsageCoreNanoSeconds: valueOfUInt64Value(criCPU.UsageCoreNanoSeconds), + } +} + +func addCRIPodProcessStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) { + if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Process == nil { + return + } + ps.ProcessStats = &statsapi.ProcessStats{ + ProcessCount: valueOfUInt64Value(criPodStat.Linux.Process.ProcessCount), + } +} + +// listContainerNetworkStats returns the network stats of all the running containers. +// It should return (nil, nil) for platforms other than Windows. +func (p *criStatsProvider) listContainerNetworkStats() (map[string]*statsapi.NetworkStats, error) { + return nil, nil +} diff --git a/pkg/kubelet/stats/cri_stats_provider_others.go b/pkg/kubelet/stats/cri_stats_provider_others.go index 80762538f4c..fb18e89f16c 100644 --- a/pkg/kubelet/stats/cri_stats_provider_others.go +++ b/pkg/kubelet/stats/cri_stats_provider_others.go @@ -1,5 +1,5 @@ -//go:build !windows -// +build !windows +//go:build !linux && !windows +// +build !linux,!windows /* Copyright 2019 The Kubernetes Authors. @@ -20,6 +20,8 @@ limitations under the License. package stats import ( + cadvisorapiv2 "github.com/google/cadvisor/info/v2" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1" ) @@ -28,3 +30,22 @@ import ( func (p *criStatsProvider) listContainerNetworkStats() (map[string]*statsapi.NetworkStats, error) { return nil, nil } + +func (p *criStatsProvider) addCRIPodContainerStats(criSandboxStat *runtimeapi.PodSandboxStats, + ps *statsapi.PodStats, fsIDtoInfo map[runtimeapi.FilesystemIdentifier]*cadvisorapiv2.FsInfo, + containerMap map[string]*runtimeapi.Container, + podSandbox *runtimeapi.PodSandbox, + rootFsInfo *cadvisorapiv2.FsInfo, updateCPUNanoCoreUsage bool) { +} + +func addCRIPodNetworkStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) { +} + +func addCRIPodMemoryStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) { +} + +func addCRIPodCPUStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) { +} + +func addCRIPodProcessStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) { +} From 79266560383b446c26e5da3c62c1c17a3b19a19d Mon Sep 17 00:00:00 2001 From: mansikulkarni96 Date: Wed, 29 Mar 2023 00:56:07 -0400 Subject: [PATCH 2/2] kubelet: Implement support for Windows podAndContainerStatsFromCRI Part of kubernetes/enhancements#2371 Follow up to the initial work introducing CRI API fields for Windows metrics collection #110754 Windows equivalent work for adding support for Windows podAndContainerStatsFromCRI #103095, which will allow users to get Windows pod and container stats only from CRI. Signed-off-by: mansikulkarni96 --- .../stats/cri_stats_provider_windows.go | 159 +++++++++++++++++- 1 file changed, 157 insertions(+), 2 deletions(-) diff --git a/pkg/kubelet/stats/cri_stats_provider_windows.go b/pkg/kubelet/stats/cri_stats_provider_windows.go index a32ffe9e981..e64da34c4db 100644 --- a/pkg/kubelet/stats/cri_stats_provider_windows.go +++ b/pkg/kubelet/stats/cri_stats_provider_windows.go @@ -22,10 +22,12 @@ package stats import ( "time" - "k8s.io/klog/v2" - "github.com/Microsoft/hcsshim" + cadvisorapiv2 "github.com/google/cadvisor/info/v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + "k8s.io/klog/v2" statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1" ) @@ -79,6 +81,101 @@ func (p *criStatsProvider) listContainerNetworkStats() (map[string]*statsapi.Net return networkStats, nil } +func (p *criStatsProvider) addCRIPodContainerStats(criSandboxStat *runtimeapi.PodSandboxStats, + ps *statsapi.PodStats, fsIDtoInfo map[runtimeapi.FilesystemIdentifier]*cadvisorapiv2.FsInfo, + containerMap map[string]*runtimeapi.Container, + podSandbox *runtimeapi.PodSandbox, + rootFsInfo *cadvisorapiv2.FsInfo, + updateCPUNanoCoreUsage bool) { + for _, criContainerStat := range criSandboxStat.Windows.Containers { + container, found := containerMap[criContainerStat.Attributes.Id] + if !found { + continue + } + // Fill available stats for full set of required pod stats + cs := p.makeWinContainerStats(criContainerStat, container, rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata()) + ps.Containers = append(ps.Containers, *cs) + } +} + +func (p *criStatsProvider) makeWinContainerStats( + stats *runtimeapi.WindowsContainerStats, + container *runtimeapi.Container, + rootFsInfo *cadvisorapiv2.FsInfo, + fsIDtoInfo map[runtimeapi.FilesystemIdentifier]*cadvisorapiv2.FsInfo, + meta *runtimeapi.PodSandboxMetadata) *statsapi.ContainerStats { + result := &statsapi.ContainerStats{ + Name: stats.Attributes.Metadata.Name, + // The StartTime in the summary API is the container creation time. + StartTime: metav1.NewTime(time.Unix(0, container.CreatedAt)), + CPU: &statsapi.CPUStats{}, + Memory: &statsapi.MemoryStats{}, + Rootfs: &statsapi.FsStats{}, + // UserDefinedMetrics is not supported by CRI. + } + if stats.Cpu != nil { + result.CPU.Time = metav1.NewTime(time.Unix(0, stats.Cpu.Timestamp)) + if stats.Cpu.UsageCoreNanoSeconds != nil { + result.CPU.UsageCoreNanoSeconds = &stats.Cpu.UsageCoreNanoSeconds.Value + } + if stats.Cpu.UsageNanoCores != nil { + result.CPU.UsageNanoCores = &stats.Cpu.UsageNanoCores.Value + } + } else { + result.CPU.Time = metav1.NewTime(time.Unix(0, time.Now().UnixNano())) + result.CPU.UsageCoreNanoSeconds = uint64Ptr(0) + result.CPU.UsageNanoCores = uint64Ptr(0) + } + if stats.Memory != nil { + result.Memory.Time = metav1.NewTime(time.Unix(0, stats.Memory.Timestamp)) + if stats.Memory.WorkingSetBytes != nil { + result.Memory.WorkingSetBytes = &stats.Memory.WorkingSetBytes.Value + } + if stats.Memory.AvailableBytes != nil { + result.Memory.AvailableBytes = &stats.Memory.AvailableBytes.Value + } + if stats.Memory.PageFaults != nil { + result.Memory.AvailableBytes = &stats.Memory.PageFaults.Value + } + } else { + result.Memory.Time = metav1.NewTime(time.Unix(0, time.Now().UnixNano())) + result.Memory.WorkingSetBytes = uint64Ptr(0) + result.Memory.AvailableBytes = uint64Ptr(0) + result.Memory.PageFaults = uint64Ptr(0) + } + if stats.WritableLayer != nil { + result.Rootfs.Time = metav1.NewTime(time.Unix(0, stats.WritableLayer.Timestamp)) + if stats.WritableLayer.UsedBytes != nil { + result.Rootfs.UsedBytes = &stats.WritableLayer.UsedBytes.Value + } + } + fsID := stats.GetWritableLayer().GetFsId() + if fsID != nil { + imageFsInfo, found := fsIDtoInfo[*fsID] + if !found { + imageFsInfo = p.getFsInfo(fsID) + fsIDtoInfo[*fsID] = imageFsInfo + } + if imageFsInfo != nil { + // The image filesystem id is unknown to the local node or there's + // an error on retrieving the stats. In these cases, we omit those stats + // and return the best-effort partial result. See + // https://github.com/kubernetes/heapster/issues/1793. + result.Rootfs.AvailableBytes = &imageFsInfo.Available + result.Rootfs.CapacityBytes = &imageFsInfo.Capacity + } + } + // NOTE: This doesn't support the old pod log path, `/var/log/pods/UID`. For containers + // using old log path, empty log stats are returned. This is fine, because we don't + // officially support in-place upgrade anyway. + var err error + result.Logs, err = p.hostStatsProvider.getPodContainerLogStats(meta.GetNamespace(), meta.GetName(), types.UID(meta.GetUid()), container.GetMetadata().GetName(), rootFsInfo) + if err != nil { + klog.ErrorS(err, "Unable to fetch container log stats", "containerName", container.GetMetadata().GetName()) + } + return result +} + // hcsStatsToNetworkStats converts hcsshim.Statistics.Network to statsapi.NetworkStats func hcsStatsToNetworkStats(timestamp time.Time, hcsStats *hcsshim.HNSEndpointStats, endpointName string) *statsapi.NetworkStats { result := &statsapi.NetworkStats{ @@ -104,6 +201,64 @@ func hcsStatToInterfaceStat(hcsStats *hcsshim.HNSEndpointStats, endpointName str return iStat } +func addCRIPodCPUStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) { + if criPodStat == nil || criPodStat.Windows == nil || criPodStat.Windows.Cpu == nil { + return + } + criCPU := criPodStat.Windows.Cpu + ps.CPU = &statsapi.CPUStats{ + Time: metav1.NewTime(time.Unix(0, criCPU.Timestamp)), + UsageNanoCores: valueOfUInt64Value(criCPU.UsageNanoCores), + UsageCoreNanoSeconds: valueOfUInt64Value(criCPU.UsageCoreNanoSeconds), + } +} + +func addCRIPodMemoryStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) { + if criPodStat == nil || criPodStat.Windows == nil || criPodStat.Windows.Memory == nil { + return + } + criMemory := criPodStat.Windows.Memory + ps.Memory = &statsapi.MemoryStats{ + Time: metav1.NewTime(time.Unix(0, criMemory.Timestamp)), + AvailableBytes: valueOfUInt64Value(criMemory.AvailableBytes), + WorkingSetBytes: valueOfUInt64Value(criMemory.WorkingSetBytes), + PageFaults: valueOfUInt64Value(criMemory.PageFaults), + } +} + +func addCRIPodProcessStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) { + if criPodStat == nil || criPodStat.Windows == nil || criPodStat.Windows.Process == nil { + return + } + ps.ProcessStats = &statsapi.ProcessStats{ + ProcessCount: valueOfUInt64Value(criPodStat.Windows.Process.ProcessCount), + } +} + +func addCRIPodNetworkStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) { + if criPodStat == nil || criPodStat.Windows == nil || criPodStat.Windows.Network == nil { + return + } + criNetwork := criPodStat.Windows.Network + iStats := statsapi.NetworkStats{ + Time: metav1.NewTime(time.Unix(0, criNetwork.Timestamp)), + InterfaceStats: criInterfaceToWinSummary(criNetwork.DefaultInterface), + Interfaces: make([]statsapi.InterfaceStats, 0, len(criNetwork.Interfaces)), + } + for _, iface := range criNetwork.Interfaces { + iStats.Interfaces = append(iStats.Interfaces, criInterfaceToWinSummary(iface)) + } + ps.Network = &iStats +} + +func criInterfaceToWinSummary(criIface *runtimeapi.WindowsNetworkInterfaceUsage) statsapi.InterfaceStats { + return statsapi.InterfaceStats{ + Name: criIface.Name, + RxBytes: valueOfUInt64Value(criIface.RxBytes), + TxBytes: valueOfUInt64Value(criIface.TxBytes), + } +} + // newNetworkStatsProvider uses the real windows hcsshim if not provided otherwise if the interface is provided // by the cristatsprovider in testing scenarios it uses that one func newNetworkStatsProvider(p *criStatsProvider) windowsNetworkStatsProvider {