Merge pull request #116968 from mansikulkarni96/windowsStatsFromCRI

kubelet: Implement support for Windows podAndContainerStatsFromCRI

pkg/kubelet/stats/cri_stats_provider.go

@@ -248,15 +248,7 @@ func (p *criStatsProvider) listPodStatsStrictlyFromCRI(ctx context.Context, upda
 			continue
 		}
 		ps := buildPodStats(podSandbox)
-		for _, criContainerStat := range criSandboxStat.Linux.Containers {
+		p.addCRIPodContainerStats(criSandboxStat, ps, fsIDtoInfo, containerMap, podSandbox, rootFsInfo, updateCPUNanoCoreUsage)
-			container, found := containerMap[criContainerStat.Attributes.Id]
-			if !found {
-				continue
-			}
-			// Fill available stats for full set of required pod stats
-			cs := p.makeContainerStats(criContainerStat, container, rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata(), updateCPUNanoCoreUsage)
-			ps.Containers = append(ps.Containers, *cs)
-		}
 		addCRIPodNetworkStats(ps, criSandboxStat)
 		addCRIPodCPUStats(ps, criSandboxStat)
 		addCRIPodMemoryStats(ps, criSandboxStat)
@@ -954,22 +946,6 @@ func extractIDFromCgroupPath(cgroupPath string) string {
 	return id
 }
 
-func addCRIPodNetworkStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
-	if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Network == nil {
-		return
-	}
-	criNetwork := criPodStat.Linux.Network
-	iStats := statsapi.NetworkStats{
-		Time:           metav1.NewTime(time.Unix(0, criNetwork.Timestamp)),
-		InterfaceStats: criInterfaceToSummary(criNetwork.DefaultInterface),
-		Interfaces:     make([]statsapi.InterfaceStats, 0, len(criNetwork.Interfaces)),
-	}
-	for _, iface := range criNetwork.Interfaces {
-		iStats.Interfaces = append(iStats.Interfaces, criInterfaceToSummary(iface))
-	}
-	ps.Network = &iStats
-}
-
 func criInterfaceToSummary(criIface *runtimeapi.NetworkInterfaceUsage) statsapi.InterfaceStats {
 	return statsapi.InterfaceStats{
 		Name:     criIface.Name,
@@ -980,43 +956,6 @@ func criInterfaceToSummary(criIface *runtimeapi.NetworkInterfaceUsage) statsapi.
 	}
 }
 
-func addCRIPodCPUStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
-	if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Cpu == nil {
-		return
-	}
-	criCPU := criPodStat.Linux.Cpu
-	ps.CPU = &statsapi.CPUStats{
-		Time:                 metav1.NewTime(time.Unix(0, criCPU.Timestamp)),
-		UsageNanoCores:       valueOfUInt64Value(criCPU.UsageNanoCores),
-		UsageCoreNanoSeconds: valueOfUInt64Value(criCPU.UsageCoreNanoSeconds),
-	}
-}
-
-func addCRIPodMemoryStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
-	if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Memory == nil {
-		return
-	}
-	criMemory := criPodStat.Linux.Memory
-	ps.Memory = &statsapi.MemoryStats{
-		Time:            metav1.NewTime(time.Unix(0, criMemory.Timestamp)),
-		AvailableBytes:  valueOfUInt64Value(criMemory.AvailableBytes),
-		UsageBytes:      valueOfUInt64Value(criMemory.UsageBytes),
-		WorkingSetBytes: valueOfUInt64Value(criMemory.WorkingSetBytes),
-		RSSBytes:        valueOfUInt64Value(criMemory.RssBytes),
-		PageFaults:      valueOfUInt64Value(criMemory.PageFaults),
-		MajorPageFaults: valueOfUInt64Value(criMemory.MajorPageFaults),
-	}
-}
-
-func addCRIPodProcessStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
-	if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Process == nil {
-		return
-	}
-	ps.ProcessStats = &statsapi.ProcessStats{
-		ProcessCount: valueOfUInt64Value(criPodStat.Linux.Process.ProcessCount),
-	}
-}
-
 func valueOfUInt64Value(value *runtimeapi.UInt64Value) *uint64 {
 	if value == nil {
 		return nil

pkg/kubelet/stats/cri_stats_provider_linux.go

@@ -0,0 +1,105 @@
+//go:build linux
+// +build linux
+
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stats
+
+import (
+	"time"
+
+	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+	statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
+)
+
+func (p *criStatsProvider) addCRIPodContainerStats(criSandboxStat *runtimeapi.PodSandboxStats,
+	ps *statsapi.PodStats, fsIDtoInfo map[runtimeapi.FilesystemIdentifier]*cadvisorapiv2.FsInfo,
+	containerMap map[string]*runtimeapi.Container,
+	podSandbox *runtimeapi.PodSandbox,
+	rootFsInfo *cadvisorapiv2.FsInfo, updateCPUNanoCoreUsage bool) {
+	for _, criContainerStat := range criSandboxStat.Linux.Containers {
+		container, found := containerMap[criContainerStat.Attributes.Id]
+		if !found {
+			continue
+		}
+		// Fill available stats for full set of required pod stats
+		cs := p.makeContainerStats(criContainerStat, container, rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata(),
+			updateCPUNanoCoreUsage)
+		ps.Containers = append(ps.Containers, *cs)
+	}
+}
+
+func addCRIPodNetworkStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
+	if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Network == nil {
+		return
+	}
+	criNetwork := criPodStat.Linux.Network
+	iStats := statsapi.NetworkStats{
+		Time:           metav1.NewTime(time.Unix(0, criNetwork.Timestamp)),
+		InterfaceStats: criInterfaceToSummary(criNetwork.DefaultInterface),
+		Interfaces:     make([]statsapi.InterfaceStats, 0, len(criNetwork.Interfaces)),
+	}
+	for _, iface := range criNetwork.Interfaces {
+		iStats.Interfaces = append(iStats.Interfaces, criInterfaceToSummary(iface))
+	}
+	ps.Network = &iStats
+}
+
+func addCRIPodMemoryStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
+	if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Memory == nil {
+		return
+	}
+	criMemory := criPodStat.Linux.Memory
+	ps.Memory = &statsapi.MemoryStats{
+		Time:            metav1.NewTime(time.Unix(0, criMemory.Timestamp)),
+		AvailableBytes:  valueOfUInt64Value(criMemory.AvailableBytes),
+		UsageBytes:      valueOfUInt64Value(criMemory.UsageBytes),
+		WorkingSetBytes: valueOfUInt64Value(criMemory.WorkingSetBytes),
+		RSSBytes:        valueOfUInt64Value(criMemory.RssBytes),
+		PageFaults:      valueOfUInt64Value(criMemory.PageFaults),
+		MajorPageFaults: valueOfUInt64Value(criMemory.MajorPageFaults),
+	}
+}
+
+func addCRIPodCPUStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
+	if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Cpu == nil {
+		return
+	}
+	criCPU := criPodStat.Linux.Cpu
+	ps.CPU = &statsapi.CPUStats{
+		Time:                 metav1.NewTime(time.Unix(0, criCPU.Timestamp)),
+		UsageNanoCores:       valueOfUInt64Value(criCPU.UsageNanoCores),
+		UsageCoreNanoSeconds: valueOfUInt64Value(criCPU.UsageCoreNanoSeconds),
+	}
+}
+
+func addCRIPodProcessStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
+	if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Process == nil {
+		return
+	}
+	ps.ProcessStats = &statsapi.ProcessStats{
+		ProcessCount: valueOfUInt64Value(criPodStat.Linux.Process.ProcessCount),
+	}
+}
+
+// listContainerNetworkStats returns the network stats of all the running containers.
+// It should return (nil, nil) for platforms other than Windows.
+func (p *criStatsProvider) listContainerNetworkStats() (map[string]*statsapi.NetworkStats, error) {
+	return nil, nil
+}

pkg/kubelet/stats/cri_stats_provider_others.go

@@ -1,5 +1,5 @@
-//go:build !windows
+//go:build !linux && !windows
-// +build !windows
+// +build !linux,!windows
 
 /*
 Copyright 2019 The Kubernetes Authors.
@@ -20,6 +20,8 @@ limitations under the License.
 package stats
 
 import (
+	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
 	statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 )
 
@@ -28,3 +30,22 @@ import (
 func (p *criStatsProvider) listContainerNetworkStats() (map[string]*statsapi.NetworkStats, error) {
 	return nil, nil
 }
+
+func (p *criStatsProvider) addCRIPodContainerStats(criSandboxStat *runtimeapi.PodSandboxStats,
+	ps *statsapi.PodStats, fsIDtoInfo map[runtimeapi.FilesystemIdentifier]*cadvisorapiv2.FsInfo,
+	containerMap map[string]*runtimeapi.Container,
+	podSandbox *runtimeapi.PodSandbox,
+	rootFsInfo *cadvisorapiv2.FsInfo, updateCPUNanoCoreUsage bool) {
+}
+
+func addCRIPodNetworkStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
+}
+
+func addCRIPodMemoryStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
+}
+
+func addCRIPodCPUStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
+}
+
+func addCRIPodProcessStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
+}

pkg/kubelet/stats/cri_stats_provider_windows.go

@@ -22,10 +22,12 @@ package stats
 import (
 	"time"
 
-	"k8s.io/klog/v2"
-
 	"github.com/Microsoft/hcsshim"
+	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+	"k8s.io/klog/v2"
 	statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 )
 
@@ -79,6 +81,101 @@ func (p *criStatsProvider) listContainerNetworkStats() (map[string]*statsapi.Net
 	return networkStats, nil
 }
 
+func (p *criStatsProvider) addCRIPodContainerStats(criSandboxStat *runtimeapi.PodSandboxStats,
+	ps *statsapi.PodStats, fsIDtoInfo map[runtimeapi.FilesystemIdentifier]*cadvisorapiv2.FsInfo,
+	containerMap map[string]*runtimeapi.Container,
+	podSandbox *runtimeapi.PodSandbox,
+	rootFsInfo *cadvisorapiv2.FsInfo,
+	updateCPUNanoCoreUsage bool) {
+	for _, criContainerStat := range criSandboxStat.Windows.Containers {
+		container, found := containerMap[criContainerStat.Attributes.Id]
+		if !found {
+			continue
+		}
+		// Fill available stats for full set of required pod stats
+		cs := p.makeWinContainerStats(criContainerStat, container, rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata())
+		ps.Containers = append(ps.Containers, *cs)
+	}
+}
+
+func (p *criStatsProvider) makeWinContainerStats(
+	stats *runtimeapi.WindowsContainerStats,
+	container *runtimeapi.Container,
+	rootFsInfo *cadvisorapiv2.FsInfo,
+	fsIDtoInfo map[runtimeapi.FilesystemIdentifier]*cadvisorapiv2.FsInfo,
+	meta *runtimeapi.PodSandboxMetadata) *statsapi.ContainerStats {
+	result := &statsapi.ContainerStats{
+		Name: stats.Attributes.Metadata.Name,
+		// The StartTime in the summary API is the container creation time.
+		StartTime: metav1.NewTime(time.Unix(0, container.CreatedAt)),
+		CPU:       &statsapi.CPUStats{},
+		Memory:    &statsapi.MemoryStats{},
+		Rootfs:    &statsapi.FsStats{},
+		// UserDefinedMetrics is not supported by CRI.
+	}
+	if stats.Cpu != nil {
+		result.CPU.Time = metav1.NewTime(time.Unix(0, stats.Cpu.Timestamp))
+		if stats.Cpu.UsageCoreNanoSeconds != nil {
+			result.CPU.UsageCoreNanoSeconds = &stats.Cpu.UsageCoreNanoSeconds.Value
+		}
+		if stats.Cpu.UsageNanoCores != nil {
+			result.CPU.UsageNanoCores = &stats.Cpu.UsageNanoCores.Value
+		}
+	} else {
+		result.CPU.Time = metav1.NewTime(time.Unix(0, time.Now().UnixNano()))
+		result.CPU.UsageCoreNanoSeconds = uint64Ptr(0)
+		result.CPU.UsageNanoCores = uint64Ptr(0)
+	}
+	if stats.Memory != nil {
+		result.Memory.Time = metav1.NewTime(time.Unix(0, stats.Memory.Timestamp))
+		if stats.Memory.WorkingSetBytes != nil {
+			result.Memory.WorkingSetBytes = &stats.Memory.WorkingSetBytes.Value
+		}
+		if stats.Memory.AvailableBytes != nil {
+			result.Memory.AvailableBytes = &stats.Memory.AvailableBytes.Value
+		}
+		if stats.Memory.PageFaults != nil {
+			result.Memory.AvailableBytes = &stats.Memory.PageFaults.Value
+		}
+	} else {
+		result.Memory.Time = metav1.NewTime(time.Unix(0, time.Now().UnixNano()))
+		result.Memory.WorkingSetBytes = uint64Ptr(0)
+		result.Memory.AvailableBytes = uint64Ptr(0)
+		result.Memory.PageFaults = uint64Ptr(0)
+	}
+	if stats.WritableLayer != nil {
+		result.Rootfs.Time = metav1.NewTime(time.Unix(0, stats.WritableLayer.Timestamp))
+		if stats.WritableLayer.UsedBytes != nil {
+			result.Rootfs.UsedBytes = &stats.WritableLayer.UsedBytes.Value
+		}
+	}
+	fsID := stats.GetWritableLayer().GetFsId()
+	if fsID != nil {
+		imageFsInfo, found := fsIDtoInfo[*fsID]
+		if !found {
+			imageFsInfo = p.getFsInfo(fsID)
+			fsIDtoInfo[*fsID] = imageFsInfo
+		}
+		if imageFsInfo != nil {
+			// The image filesystem id is unknown to the local node or there's
+			// an error on retrieving the stats. In these cases, we omit those stats
+			// and return the best-effort partial result. See
+			// https://github.com/kubernetes/heapster/issues/1793.
+			result.Rootfs.AvailableBytes = &imageFsInfo.Available
+			result.Rootfs.CapacityBytes = &imageFsInfo.Capacity
+		}
+	}
+	// NOTE: This doesn't support the old pod log path, `/var/log/pods/UID`. For containers
+	// using old log path, empty log stats are returned. This is fine, because we don't
+	// officially support in-place upgrade anyway.
+	var err error
+	result.Logs, err = p.hostStatsProvider.getPodContainerLogStats(meta.GetNamespace(), meta.GetName(), types.UID(meta.GetUid()), container.GetMetadata().GetName(), rootFsInfo)
+	if err != nil {
+		klog.ErrorS(err, "Unable to fetch container log stats", "containerName", container.GetMetadata().GetName())
+	}
+	return result
+}
+
 // hcsStatsToNetworkStats converts hcsshim.Statistics.Network to statsapi.NetworkStats
 func hcsStatsToNetworkStats(timestamp time.Time, hcsStats *hcsshim.HNSEndpointStats, endpointName string) *statsapi.NetworkStats {
 	result := &statsapi.NetworkStats{
@@ -104,6 +201,64 @@ func hcsStatToInterfaceStat(hcsStats *hcsshim.HNSEndpointStats, endpointName str
 	return iStat
 }
 
+func addCRIPodCPUStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
+	if criPodStat == nil || criPodStat.Windows == nil || criPodStat.Windows.Cpu == nil {
+		return
+	}
+	criCPU := criPodStat.Windows.Cpu
+	ps.CPU = &statsapi.CPUStats{
+		Time:                 metav1.NewTime(time.Unix(0, criCPU.Timestamp)),
+		UsageNanoCores:       valueOfUInt64Value(criCPU.UsageNanoCores),
+		UsageCoreNanoSeconds: valueOfUInt64Value(criCPU.UsageCoreNanoSeconds),
+	}
+}
+
+func addCRIPodMemoryStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
+	if criPodStat == nil || criPodStat.Windows == nil || criPodStat.Windows.Memory == nil {
+		return
+	}
+	criMemory := criPodStat.Windows.Memory
+	ps.Memory = &statsapi.MemoryStats{
+		Time:            metav1.NewTime(time.Unix(0, criMemory.Timestamp)),
+		AvailableBytes:  valueOfUInt64Value(criMemory.AvailableBytes),
+		WorkingSetBytes: valueOfUInt64Value(criMemory.WorkingSetBytes),
+		PageFaults:      valueOfUInt64Value(criMemory.PageFaults),
+	}
+}
+
+func addCRIPodProcessStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
+	if criPodStat == nil || criPodStat.Windows == nil || criPodStat.Windows.Process == nil {
+		return
+	}
+	ps.ProcessStats = &statsapi.ProcessStats{
+		ProcessCount: valueOfUInt64Value(criPodStat.Windows.Process.ProcessCount),
+	}
+}
+
+func addCRIPodNetworkStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
+	if criPodStat == nil || criPodStat.Windows == nil || criPodStat.Windows.Network == nil {
+		return
+	}
+	criNetwork := criPodStat.Windows.Network
+	iStats := statsapi.NetworkStats{
+		Time:           metav1.NewTime(time.Unix(0, criNetwork.Timestamp)),
+		InterfaceStats: criInterfaceToWinSummary(criNetwork.DefaultInterface),
+		Interfaces:     make([]statsapi.InterfaceStats, 0, len(criNetwork.Interfaces)),
+	}
+	for _, iface := range criNetwork.Interfaces {
+		iStats.Interfaces = append(iStats.Interfaces, criInterfaceToWinSummary(iface))
+	}
+	ps.Network = &iStats
+}
+
+func criInterfaceToWinSummary(criIface *runtimeapi.WindowsNetworkInterfaceUsage) statsapi.InterfaceStats {
+	return statsapi.InterfaceStats{
+		Name:    criIface.Name,
+		RxBytes: valueOfUInt64Value(criIface.RxBytes),
+		TxBytes: valueOfUInt64Value(criIface.TxBytes),
+	}
+}
+
 // newNetworkStatsProvider uses the real windows hcsshim if not provided otherwise if the interface is provided
 // by the cristatsprovider in testing scenarios it uses that one
 func newNetworkStatsProvider(p *criStatsProvider) windowsNetworkStatsProvider {