github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-21 10:51:29 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #130701 from roycaihw/psi-metrics

Browse Source 
Surface Pressure Stall Information (PSI) metrics
This commit is contained in:
Kubernetes Prow Robot 2025-03-24 10:38:33 -07:00 committed by GitHub
commit 62555cadc7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 2111 additions and 532 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/features/kube_features.go
View File

@ -398,6 +398,12 @@ const (
// Enable POD resources API with Get method
KubeletPodResourcesGet featuregate.Feature = "KubeletPodResourcesGet"
// KubeletPSI enables Kubelet to surface PSI metrics
// owner: @roycaihw
// kep: https://kep.k8s.io/4205
// alpha: v1.33
KubeletPSI featuregate.Feature = "KubeletPSI"
// owner: @kannon92
// kep: https://kep.k8s.io/4191
//
@ -1421,6 +1427,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
},
KubeletPSI: {
{Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Alpha},
},
KubeletRegistrationGetOnExistsOnly: {
{Version: version.MustParse("1.0"), Default: true, PreRelease: featuregate.GA},
{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Deprecated},

6
pkg/kubelet/cadvisor/cadvisor_linux.go
View File

@ -39,7 +39,9 @@ import (
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
"github.com/google/cadvisor/manager"
"github.com/google/cadvisor/utils/sysfs"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/features"
"k8s.io/utils/ptr"
)
@ -93,6 +95,10 @@ func New(imageFsInfoProvider ImageFsInfoProvider, rootPath string, cgroupRoots [
cadvisormetrics.OOMMetrics: struct{}{},
}
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
includedMetrics[cadvisormetrics.PressureMetrics] = struct{}{}
}
if usingLegacyStats || localStorageCapacityIsolation {
includedMetrics[cadvisormetrics.DiskUsageMetrics] = struct{}{}
}

5
pkg/kubelet/server/server.go
View File

@ -455,6 +455,11 @@ func (s *Server) InstallAuthNotRequiredHandlers() {
cadvisormetrics.ProcessMetrics: struct{}{},
cadvisormetrics.OOMMetrics: struct{}{},
}
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
includedMetrics[cadvisormetrics.PressureMetrics] = struct{}{}
}
// cAdvisor metrics are exposed under the secured handler as well
r := compbasemetrics.NewKubeRegistry()
r.RawMustRegister(metrics.NewPrometheusMachineCollector(prometheusHostAdapter{s.host}, includedMetrics))

5
pkg/kubelet/server/stats/summary.go
View File

@ -24,7 +24,9 @@ import (
"k8s.io/klog/v2"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/util"
)
@ -113,6 +115,9 @@ func (sp *summaryProviderImpl) Get(ctx context.Context, updateStats bool) (*stat
Rlimit: rlimit,
SystemContainers: sp.GetSystemContainersStats(nodeConfig, podStats, updateStats),
}
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
nodeStats.IO = rootStats.IO
}
summary := statsapi.Summary{
Node: nodeStats,
Pods: podStats,

17
pkg/kubelet/server/stats/summary_test.go
View File

@ -29,7 +29,10 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/cm"
statstest "k8s.io/kubernetes/pkg/kubelet/server/stats/testing"
)
@ -48,6 +51,7 @@ var (
)
func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true)
ctx := context.Background()
assert := assert.New(t)
@ -98,6 +102,7 @@ func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) {
assert.Equal(summary.Node.CPU, cgroupStatsMap["/"].cs.CPU)
assert.Equal(summary.Node.Memory, cgroupStatsMap["/"].cs.Memory)
assert.Equal(summary.Node.Swap, cgroupStatsMap["/"].cs.Swap)
assert.Equal(summary.Node.IO, cgroupStatsMap["/"].cs.IO)
assert.Equal(summary.Node.Network, cgroupStatsMap["/"].ns)
assert.Equal(summary.Node.Fs, rootFsStats)
assert.Equal(&statsapi.RuntimeStats{ContainerFs: imageFsStats, ImageFs: imageFsStats}, summary.Node.Runtime)
@ -111,6 +116,7 @@ func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) {
Accelerators: cgroupStatsMap["/kubelet"].cs.Accelerators,
UserDefinedMetrics: cgroupStatsMap["/kubelet"].cs.UserDefinedMetrics,
Swap: cgroupStatsMap["/kubelet"].cs.Swap,
IO: cgroupStatsMap["/kubelet"].cs.IO,
})
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
Name: "misc",
@ -120,6 +126,7 @@ func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) {
Accelerators: cgroupStatsMap["/misc"].cs.Accelerators,
UserDefinedMetrics: cgroupStatsMap["/misc"].cs.UserDefinedMetrics,
Swap: cgroupStatsMap["/misc"].cs.Swap,
IO: cgroupStatsMap["/misc"].cs.IO,
})
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
Name: "runtime",
@ -129,6 +136,7 @@ func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) {
Accelerators: cgroupStatsMap["/runtime"].cs.Accelerators,
UserDefinedMetrics: cgroupStatsMap["/runtime"].cs.UserDefinedMetrics,
Swap: cgroupStatsMap["/runtime"].cs.Swap,
IO: cgroupStatsMap["/runtime"].cs.IO,
})
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
Name: "pods",
@ -138,11 +146,13 @@ func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) {
Accelerators: cgroupStatsMap["/pods"].cs.Accelerators,
UserDefinedMetrics: cgroupStatsMap["/pods"].cs.UserDefinedMetrics,
Swap: cgroupStatsMap["/pods"].cs.Swap,
IO: cgroupStatsMap["/pods"].cs.IO,
})
assert.Equal(summary.Pods, podStats)
}
func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true)
ctx := context.Background()
assert := assert.New(t)
@ -194,6 +204,7 @@ func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) {
assert.Equal(summary.Node.CPU, cgroupStatsMap["/"].cs.CPU)
assert.Equal(summary.Node.Memory, cgroupStatsMap["/"].cs.Memory)
assert.Equal(summary.Node.Swap, cgroupStatsMap["/"].cs.Swap)
assert.Equal(summary.Node.IO, cgroupStatsMap["/"].cs.IO)
assert.Equal(summary.Node.Network, cgroupStatsMap["/"].ns)
assert.Equal(summary.Node.Fs, rootFsStats)
// Since we are a split filesystem we want root filesystem to be container fs and image to be image filesystem
@ -208,6 +219,7 @@ func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) {
Accelerators: cgroupStatsMap["/kubelet"].cs.Accelerators,
UserDefinedMetrics: cgroupStatsMap["/kubelet"].cs.UserDefinedMetrics,
Swap: cgroupStatsMap["/kubelet"].cs.Swap,
IO: cgroupStatsMap["/kubelet"].cs.IO,
})
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
Name: "misc",
@ -217,6 +229,7 @@ func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) {
Accelerators: cgroupStatsMap["/misc"].cs.Accelerators,
UserDefinedMetrics: cgroupStatsMap["/misc"].cs.UserDefinedMetrics,
Swap: cgroupStatsMap["/misc"].cs.Swap,
IO: cgroupStatsMap["/misc"].cs.IO,
})
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
Name: "runtime",
@ -226,6 +239,7 @@ func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) {
Accelerators: cgroupStatsMap["/runtime"].cs.Accelerators,
UserDefinedMetrics: cgroupStatsMap["/runtime"].cs.UserDefinedMetrics,
Swap: cgroupStatsMap["/runtime"].cs.Swap,
IO: cgroupStatsMap["/runtime"].cs.IO,
})
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
Name: "pods",
@ -235,11 +249,13 @@ func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) {
Accelerators: cgroupStatsMap["/pods"].cs.Accelerators,
UserDefinedMetrics: cgroupStatsMap["/pods"].cs.UserDefinedMetrics,
Swap: cgroupStatsMap["/pods"].cs.Swap,
IO: cgroupStatsMap["/pods"].cs.IO,
})
assert.Equal(summary.Pods, podStats)
}
func TestSummaryProviderGetCPUAndMemoryStats(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true)
ctx := context.Background()
assert := assert.New(t)
@ -283,6 +299,7 @@ func TestSummaryProviderGetCPUAndMemoryStats(t *testing.T) {
assert.Nil(summary.Node.Network)
assert.Nil(summary.Node.Fs)
assert.Nil(summary.Node.Runtime)
assert.Nil(summary.Node.IO)
assert.Len(summary.Node.SystemContainers, 4)
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{

3
pkg/kubelet/stats/cadvisor_stats_provider.go
View File

@ -156,6 +156,9 @@ func (p *cadvisorStatsProvider) ListPodStats(ctx context.Context) ([]statsapi.Po
podStats.CPU = cpu
podStats.Memory = memory
podStats.Swap = cadvisorInfoToSwapStats(podInfo)
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
podStats.IO = cadvisorInfoToIOStats(podInfo)
}
// ProcessStats were accumulated as the containers were iterated.
}

15
pkg/kubelet/stats/cadvisor_stats_provider_test.go
View File

@ -114,6 +114,7 @@ func TestFilterTerminatedContainerInfoAndAssembleByPodCgroupKey(t *testing.T) {
}
func TestCadvisorListPodStats(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true)
ctx := context.Background()
const (
namespace0 = "test0"
@ -295,12 +296,14 @@ func TestCadvisorListPodStats(t *testing.T) {
checkCPUStats(t, "Pod0Container0", seedPod0Container0, con.CPU)
checkMemoryStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.Memory)
checkSwapStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.Swap)
checkIOStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.IO)
con = indexCon[cName01]
assert.EqualValues(t, testTime(creationTime, seedPod0Container1).Unix(), con.StartTime.Time.Unix())
checkCPUStats(t, "Pod0Container1", seedPod0Container1, con.CPU)
checkMemoryStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.Memory)
checkSwapStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.Swap)
checkIOStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.IO)
assert.EqualValues(t, p0Time.Unix(), ps.StartTime.Time.Unix())
checkNetworkStats(t, "Pod0", seedPod0Infra, ps.Network)
@ -315,6 +318,9 @@ func TestCadvisorListPodStats(t *testing.T) {
checkSwapStats(t, "Pod0", seedPod0Infra, infos["/pod0-i"], ps.Swap)
checkContainersSwapStats(t, ps, infos["/pod0-c0"], infos["/pod0-c1"])
}
if ps.IO != nil {
checkIOStats(t, "Pod0", seedPod0Infra, infos["/pod0-i"], ps.IO)
}
// Validate Pod1 Results
ps, found = indexPods[prf1]
@ -325,6 +331,7 @@ func TestCadvisorListPodStats(t *testing.T) {
checkCPUStats(t, "Pod1Container0", seedPod1Container, con.CPU)
checkMemoryStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.Memory)
checkSwapStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.Swap)
checkIOStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.IO)
checkNetworkStats(t, "Pod1", seedPod1Infra, ps.Network)
checkContainersSwapStats(t, ps, infos["/pod1-c0"])
@ -337,6 +344,7 @@ func TestCadvisorListPodStats(t *testing.T) {
checkCPUStats(t, "Pod2Container0", seedPod2Container, con.CPU)
checkMemoryStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.Memory)
checkSwapStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.Swap)
checkIOStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.IO)
checkNetworkStats(t, "Pod2", seedPod2Infra, ps.Network)
checkContainersSwapStats(t, ps, infos["/pod2-c0"])
@ -355,10 +363,12 @@ func TestCadvisorListPodStats(t *testing.T) {
checkCPUStats(t, "Pod3Container1", seedPod3Container1, con.CPU)
checkMemoryStats(t, "Pod3Container1", seedPod3Container1, infos["/pod3-c1"], con.Memory)
checkSwapStats(t, "Pod3Container1", seedPod3Container1, infos["/pod3-c1"], con.Swap)
checkIOStats(t, "Pod3Container1", seedPod3Container1, infos["/pod3-c1"], con.IO)
checkContainersSwapStats(t, ps, infos["/pod3-c1"])
}
func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true)
ctx := context.Background()
const (
namespace0 = "test0"
@ -476,6 +486,7 @@ func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) {
assert.Nil(t, con.Logs)
assert.Nil(t, con.Accelerators)
assert.Nil(t, con.UserDefinedMetrics)
assert.Nil(t, con.IO)
con = indexCon[cName01]
assert.EqualValues(t, testTime(creationTime, seedPod0Container1).Unix(), con.StartTime.Time.Unix())
@ -486,11 +497,13 @@ func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) {
assert.Nil(t, con.Logs)
assert.Nil(t, con.Accelerators)
assert.Nil(t, con.UserDefinedMetrics)
assert.Nil(t, con.IO)
assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix())
assert.Nil(t, ps.EphemeralStorage)
assert.Nil(t, ps.VolumeStats)
assert.Nil(t, ps.Network)
assert.Nil(t, con.IO)
if ps.CPU != nil {
checkCPUStats(t, "Pod0", seedPod0Infra, ps.CPU)
}
@ -515,6 +528,7 @@ func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) {
assert.Nil(t, ps.EphemeralStorage)
assert.Nil(t, ps.VolumeStats)
assert.Nil(t, ps.Network)
assert.Nil(t, con.IO)
// Validate Pod2 Results
ps, found = indexPods[prf2]
@ -529,6 +543,7 @@ func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) {
assert.Nil(t, ps.EphemeralStorage)
assert.Nil(t, ps.VolumeStats)
assert.Nil(t, ps.Network)
assert.Nil(t, con.IO)
}
func TestCadvisorImagesFsStatsKubeletSeparateDiskOff(t *testing.T) {

81
pkg/kubelet/stats/cri_stats_provider.go
View File

@ -33,11 +33,13 @@ import (
"google.golang.org/grpc/status"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
utilfeature "k8s.io/apiserver/pkg/util/feature"
internalapi "k8s.io/cri-api/pkg/apis"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"k8s.io/klog/v2"
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
kubetypes "k8s.io/kubelet/pkg/types"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
"k8s.io/kubernetes/pkg/kubelet/server/stats"
"k8s.io/utils/clock"
@ -211,6 +213,7 @@ func (p *criStatsProvider) listPodStatsPartiallyFromCRI(ctx context.Context, upd
p.addPodNetworkStats(ps, podSandboxID, caInfos, cs, containerNetworkStats[podSandboxID])
p.addPodCPUMemoryStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos, cs)
p.addSwapStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos, cs)
p.addIOStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos, cs)
// If cadvisor stats is available for the container, use it to populate
// container stats
@ -260,6 +263,7 @@ func (p *criStatsProvider) listPodStatsStrictlyFromCRI(ctx context.Context, upda
addCRIPodCPUStats(ps, criSandboxStat)
addCRIPodMemoryStats(ps, criSandboxStat)
addCRIPodProcessStats(ps, criSandboxStat)
addCRIPodIOStats(ps, criSandboxStat)
makePodStorageStats(ps, rootFsInfo, p.resourceAnalyzer, p.hostStatsProvider, true)
summarySandboxStats = append(summarySandboxStats, *ps)
}
@ -535,6 +539,7 @@ func (p *criStatsProvider) addPodCPUMemoryStats(
usageNanoCores := getUint64Value(cs.CPU.UsageNanoCores) + getUint64Value(ps.CPU.UsageNanoCores)
ps.CPU.UsageCoreNanoSeconds = &usageCoreNanoSeconds
ps.CPU.UsageNanoCores = &usageNanoCores
// Pod level PSI stats cannot be calculated from container level
}
if cs.Memory != nil {
@ -555,6 +560,7 @@ func (p *criStatsProvider) addPodCPUMemoryStats(
ps.Memory.RSSBytes = &rSSBytes
ps.Memory.PageFaults = &pageFaults
ps.Memory.MajorPageFaults = &majorPageFaults
// Pod level PSI stats cannot be calculated from container level
}
}
@ -564,14 +570,14 @@ func (p *criStatsProvider) addSwapStats(
allInfos map[string]cadvisorapiv2.ContainerInfo,
cs *statsapi.ContainerStats,
) {
// try get cpu and memory stats from cadvisor first.
// try get swap stats from cadvisor first.
podCgroupInfo := getCadvisorPodInfoFromPodUID(podUID, allInfos)
if podCgroupInfo != nil {
ps.Swap = cadvisorInfoToSwapStats(podCgroupInfo)
return
}
// Sum Pod cpu and memory stats from containers stats.
// Sum Pod swap stats from containers stats.
if cs.Swap != nil {
if ps.Swap == nil {
ps.Swap = &statsapi.SwapStats{Time: cs.Swap.Time}
@ -583,6 +589,30 @@ func (p *criStatsProvider) addSwapStats(
}
}
func (p *criStatsProvider) addIOStats(
ps *statsapi.PodStats,
podUID types.UID,
allInfos map[string]cadvisorapiv2.ContainerInfo,
cs *statsapi.ContainerStats,
) {
if !utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
return
}
// try get IO stats from cadvisor first.
podCgroupInfo := getCadvisorPodInfoFromPodUID(podUID, allInfos)
if podCgroupInfo != nil {
ps.IO = cadvisorInfoToIOStats(podCgroupInfo)
return
}
if cs.IO != nil {
if ps.IO == nil {
ps.IO = &statsapi.IOStats{Time: cs.IO.Time}
}
// Pod level PSI stats cannot be calculated from container level
}
}
func (p *criStatsProvider) addProcessStats(
ps *statsapi.PodStats,
container *cadvisorapiv2.ContainerInfo,
@ -624,6 +654,7 @@ func (p *criStatsProvider) makeContainerStats(
if usageNanoCores != nil {
result.CPU.UsageNanoCores = usageNanoCores
}
result.CPU.PSI = makePSIStats(stats.Cpu.Psi)
} else {
result.CPU.Time = metav1.NewTime(time.Unix(0, time.Now().UnixNano()))
result.CPU.UsageCoreNanoSeconds = uint64Ptr(0)
@ -634,6 +665,7 @@ func (p *criStatsProvider) makeContainerStats(
if stats.Memory.WorkingSetBytes != nil {
result.Memory.WorkingSetBytes = &stats.Memory.WorkingSetBytes.Value
}
result.Memory.PSI = makePSIStats(stats.Memory.Psi)
} else {
result.Memory.Time = metav1.NewTime(time.Unix(0, time.Now().UnixNano()))
result.Memory.WorkingSetBytes = uint64Ptr(0)
@ -651,6 +683,15 @@ func (p *criStatsProvider) makeContainerStats(
result.Swap.SwapUsageBytes = uint64Ptr(0)
result.Swap.SwapAvailableBytes = uint64Ptr(0)
}
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
result.IO = &statsapi.IOStats{}
if stats.Io != nil {
result.IO.Time = metav1.NewTime(time.Unix(0, stats.Io.Timestamp))
result.IO.PSI = makePSIStats(stats.Io.Psi)
} else {
result.IO.Time = metav1.NewTime(time.Unix(0, time.Now().UnixNano()))
}
}
if stats.WritableLayer != nil {
result.Rootfs.Time = metav1.NewTime(time.Unix(0, stats.WritableLayer.Timestamp))
if stats.WritableLayer.UsedBytes != nil {
@ -714,6 +755,7 @@ func (p *criStatsProvider) makeContainerCPUAndMemoryStats(
if usageNanoCores != nil {
result.CPU.UsageNanoCores = usageNanoCores
}
result.CPU.PSI = makePSIStats(stats.Cpu.Psi)
} else {
result.CPU.Time = metav1.NewTime(time.Unix(0, time.Now().UnixNano()))
result.CPU.UsageCoreNanoSeconds = uint64Ptr(0)
@ -724,6 +766,7 @@ func (p *criStatsProvider) makeContainerCPUAndMemoryStats(
if stats.Memory.WorkingSetBytes != nil {
result.Memory.WorkingSetBytes = &stats.Memory.WorkingSetBytes.Value
}
result.Memory.PSI = makePSIStats(stats.Memory.Psi)
} else {
result.Memory.Time = metav1.NewTime(time.Unix(0, time.Now().UnixNano()))
result.Memory.WorkingSetBytes = uint64Ptr(0)
@ -732,6 +775,33 @@ func (p *criStatsProvider) makeContainerCPUAndMemoryStats(
return result
}
func makePSIStats(stats *runtimeapi.PsiStats) *statsapi.PSIStats {
if !utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
return nil
}
if stats == nil {
return nil
}
result := &statsapi.PSIStats{}
if stats.Full != nil {
result.Full = statsapi.PSIData{
Total: stats.Full.Total,
Avg10: stats.Full.Avg10,
Avg60: stats.Full.Avg60,
Avg300: stats.Full.Avg300,
}
}
if stats.Some != nil {
result.Some = statsapi.PSIData{
Total: stats.Some.Total,
Avg10: stats.Some.Avg10,
Avg60: stats.Some.Avg60,
Avg300: stats.Some.Avg300,
}
}
return result
}
// getContainerUsageNanoCores first attempts to get the usage nano cores from the stats reported
// by the CRI. If it is unable to, it gets the information from the cache instead.
func (p *criStatsProvider) getContainerUsageNanoCores(stats *runtimeapi.ContainerStats) *uint64 {
@ -910,6 +980,13 @@ func (p *criStatsProvider) addCadvisorContainerStats(
if swap != nil {
cs.Swap = swap
}
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
io := cadvisorInfoToIOStats(caPodStats)
if io != nil {
cs.IO = io
}
}
}
func (p *criStatsProvider) addCadvisorContainerCPUAndMemoryStats(

18
pkg/kubelet/stats/cri_stats_provider_linux.go
View File

@ -25,8 +25,10 @@ import (
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
"k8s.io/kubernetes/pkg/features"
)
func (p *criStatsProvider) addCRIPodContainerStats(criSandboxStat *runtimeapi.PodSandboxStats,
@ -79,6 +81,7 @@ func addCRIPodMemoryStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandb
RSSBytes: valueOfUInt64Value(criMemory.RssBytes),
PageFaults: valueOfUInt64Value(criMemory.PageFaults),
MajorPageFaults: valueOfUInt64Value(criMemory.MajorPageFaults),
PSI: makePSIStats(criMemory.Psi),
}
}
@ -91,6 +94,21 @@ func addCRIPodCPUStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxS
Time: metav1.NewTime(time.Unix(0, criCPU.Timestamp)),
UsageNanoCores: valueOfUInt64Value(criCPU.UsageNanoCores),
UsageCoreNanoSeconds: valueOfUInt64Value(criCPU.UsageCoreNanoSeconds),
PSI: makePSIStats(criCPU.Psi),
}
}
func addCRIPodIOStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
if !utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
return
}
if criPodStat == nil || criPodStat.Linux == nil || criPodStat.Linux.Io == nil {
return
}
criIO := criPodStat.Linux.Io
ps.IO = &statsapi.IOStats{
Time: metav1.NewTime(time.Unix(0, criIO.Timestamp)),
PSI: makePSIStats(criIO.Psi),
}
}

3
pkg/kubelet/stats/cri_stats_provider_others.go
View File

@ -50,3 +50,6 @@ func addCRIPodCPUStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxS
func addCRIPodProcessStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
}
func addCRIPodIOStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
}

167
pkg/kubelet/stats/cri_stats_provider_test.go
View File

@ -27,16 +27,20 @@ import (
"time"
cadvisorfs "github.com/google/cadvisor/fs"
cadvisorapiv1 "github.com/google/cadvisor/info/v1"
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
"github.com/stretchr/testify/assert"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/uuid"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
critest "k8s.io/cri-api/pkg/apis/testing"
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
kubelettypes "k8s.io/kubelet/pkg/types"
"k8s.io/kubernetes/pkg/features"
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
"k8s.io/kubernetes/pkg/kubelet/cm"
kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
@ -91,6 +95,7 @@ const (
const testPodLogDirectory = "/var/log/kube/pods/" // Use non-default path to ensure stats are collected properly
func TestCRIListPodStats(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true)
ctx := context.Background()
var (
imageFsMountpoint = "/test/mount/point"
@ -265,6 +270,7 @@ func TestCRIListPodStats(t *testing.T) {
c0 := containerStatsMap[cName0]
assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano())
checkCRICPUAndMemoryStats(assert, c0, infos[container0.ContainerStatus.Id].Stats[0])
checkCRIIOStats(assert, c0, infos[container0.ContainerStatus.Id].Stats[0])
assert.Nil(c0.Accelerators)
checkCRIRootfsStats(assert, c0, containerStats0, &imageFsInfo)
checkCRILogsStats(assert, c0, &rootFsInfo, containerLogStats0)
@ -272,12 +278,14 @@ func TestCRIListPodStats(t *testing.T) {
c1 := containerStatsMap[cName1]
assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano())
checkCRICPUAndMemoryStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0])
checkCRIIOStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0])
assert.Nil(c0.Accelerators)
checkCRIRootfsStats(assert, c1, containerStats1, nil)
checkCRILogsStats(assert, c1, &rootFsInfo, containerLogStats1)
checkCRINetworkStats(assert, p0.Network, infos[sandbox0.PodSandboxStatus.Id].Stats[0].Network)
checkCRIPodCPUAndMemoryStats(assert, p0, infos[sandbox0Cgroup].Stats[0])
checkCRIPodSwapStats(assert, p0, infos[sandbox0Cgroup].Stats[0])
checkCRIPodIOStats(assert, p0, infos[sandbox0Cgroup].Stats[0])
checkContainersSwapStats(t, p0, infos[container0.Id], infos[container1.Id])
@ -291,12 +299,14 @@ func TestCRIListPodStats(t *testing.T) {
assert.Equal(cName2, c2.Name)
assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano())
checkCRICPUAndMemoryStats(assert, c2, infos[container2.ContainerStatus.Id].Stats[0])
checkCRIIOStats(assert, c2, infos[container2.ContainerStatus.Id].Stats[0])
assert.Nil(c0.Accelerators)
checkCRIRootfsStats(assert, c2, containerStats2, &imageFsInfo)
checkCRILogsStats(assert, c2, &rootFsInfo, containerLogStats2)
checkCRINetworkStats(assert, p1.Network, infos[sandbox1.PodSandboxStatus.Id].Stats[0].Network)
checkCRIPodCPUAndMemoryStats(assert, p1, infos[sandbox1Cgroup].Stats[0])
checkCRIPodSwapStats(assert, p1, infos[sandbox1Cgroup].Stats[0])
checkCRIPodIOStats(assert, p1, infos[sandbox1Cgroup].Stats[0])
checkContainersSwapStats(t, p1, infos[container2.Id])
@ -311,6 +321,7 @@ func TestCRIListPodStats(t *testing.T) {
assert.Equal(cName3, c3.Name)
assert.Equal(container4.CreatedAt, c3.StartTime.UnixNano())
checkCRICPUAndMemoryStats(assert, c3, infos[container4.ContainerStatus.Id].Stats[0])
checkCRIIOStats(assert, c3, infos[container4.ContainerStatus.Id].Stats[0])
assert.Nil(c0.Accelerators)
checkCRIRootfsStats(assert, c3, containerStats4, &imageFsInfo)
@ -318,6 +329,7 @@ func TestCRIListPodStats(t *testing.T) {
checkCRINetworkStats(assert, p2.Network, infos[sandbox2.PodSandboxStatus.Id].Stats[0].Network)
checkCRIPodCPUAndMemoryStats(assert, p2, infos[sandbox2Cgroup].Stats[0])
checkCRIPodSwapStats(assert, p2, infos[sandbox2Cgroup].Stats[0])
checkCRIPodIOStats(assert, p2, infos[sandbox2Cgroup].Stats[0])
checkContainersSwapStats(t, p2, infos[container4.Id])
@ -332,9 +344,11 @@ func TestCRIListPodStats(t *testing.T) {
assert.NotNil(c8.Memory.Time)
checkCRIPodCPUAndMemoryStats(assert, p3, infos[sandbox3Cgroup].Stats[0])
checkCRIPodSwapStats(assert, p3, infos[sandbox3Cgroup].Stats[0])
checkCRIPodIOStats(assert, p3, infos[sandbox3Cgroup].Stats[0])
}
func TestListPodStatsStrictlyFromCRI(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true)
if runtime.GOOS == "windows" {
// TODO: remove skip once the failing test has been fixed.
t.Skip("Skip failing test on Windows.")
@ -494,6 +508,7 @@ func TestListPodStatsStrictlyFromCRI(t *testing.T) {
c0 := containerStatsMap[cName0]
assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano())
checkCRICPUAndMemoryStatsForStrictlyFromCRI(assert, c0, exceptedContainerStatsMap[cName0])
checkCRIIOStatsForStrictlyFromCRI(assert, c0, exceptedContainerStatsMap[cName0])
assert.Nil(c0.Accelerators)
checkCRIRootfsStats(assert, c0, containerStats0, &imageFsInfo)
checkCRILogsStats(assert, c0, &rootFsInfo, containerLogStats0)
@ -501,10 +516,12 @@ func TestListPodStatsStrictlyFromCRI(t *testing.T) {
c1 := containerStatsMap[cName1]
assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano())
checkCRICPUAndMemoryStatsForStrictlyFromCRI(assert, c1, exceptedContainerStatsMap[cName1])
checkCRIIOStatsForStrictlyFromCRI(assert, c1, exceptedContainerStatsMap[cName1])
assert.Nil(c0.Accelerators)
checkCRIRootfsStats(assert, c1, containerStats1, nil)
checkCRILogsStats(assert, c1, &rootFsInfo, containerLogStats1)
checkCRIPodCPUAndMemoryStatsStrictlyFromCRI(assert, p0, exceptedPodStatsMap[prf0])
checkCRIPodIOStatsStrictlyFromCRI(assert, p0, exceptedPodStatsMap[prf0])
assert.NotNil(cadvisorInfos[sandbox0Cgroup].Stats[0].Cpu)
assert.NotNil(cadvisorInfos[sandbox0Cgroup].Stats[0].Memory)
@ -518,10 +535,12 @@ func TestListPodStatsStrictlyFromCRI(t *testing.T) {
assert.Equal(cName2, c2.Name)
assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano())
checkCRICPUAndMemoryStatsForStrictlyFromCRI(assert, c2, exceptedContainerStatsMap[cName2])
checkCRIIOStatsForStrictlyFromCRI(assert, c2, exceptedContainerStatsMap[cName2])
assert.Nil(c0.Accelerators)
checkCRIRootfsStats(assert, c2, containerStats2, &imageFsInfo)
checkCRILogsStats(assert, c2, &rootFsInfo, containerLogStats2)
checkCRIPodCPUAndMemoryStatsStrictlyFromCRI(assert, p1, exceptedPodStatsMap[prf1])
checkCRIPodIOStatsStrictlyFromCRI(assert, p1, exceptedPodStatsMap[prf1])
if runtime.GOOS == "linux" {
if _, ok := cadvisorInfos[sandbox1Cgroup]; ok {
@ -530,6 +549,7 @@ func TestListPodStatsStrictlyFromCRI(t *testing.T) {
}
}
func TestCRIListPodCPUAndMemoryStats(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true)
ctx := context.Background()
var (
@ -656,6 +676,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) {
assert.Nil(p0.EphemeralStorage)
assert.Nil(p0.VolumeStats)
assert.Nil(p0.Network)
assert.Nil(p0.IO)
checkCRIPodCPUAndMemoryStats(assert, p0, infos[sandbox0Cgroup].Stats[0])
containerStatsMap := make(map[string]statsapi.ContainerStats)
@ -670,6 +691,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) {
assert.Nil(c0.Logs)
assert.Nil(c0.Accelerators)
assert.Nil(c0.UserDefinedMetrics)
assert.Nil(c0.IO)
c1 := containerStatsMap[cName1]
assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano())
checkCRICPUAndMemoryStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0])
@ -677,6 +699,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) {
assert.Nil(c1.Logs)
assert.Nil(c1.Accelerators)
assert.Nil(c1.UserDefinedMetrics)
assert.Nil(c1.IO)
p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}]
assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
@ -684,6 +707,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) {
assert.Nil(p1.EphemeralStorage)
assert.Nil(p1.VolumeStats)
assert.Nil(p1.Network)
assert.Nil(p1.IO)
checkCRIPodCPUAndMemoryStats(assert, p1, infos[sandbox1Cgroup].Stats[0])
c2 := p1.Containers[0]
@ -694,6 +718,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) {
assert.Nil(c2.Logs)
assert.Nil(c2.Accelerators)
assert.Nil(c2.UserDefinedMetrics)
assert.Nil(c2.IO)
p2 := podStatsMap[statsapi.PodReference{Name: "sandbox2-name", UID: "sandbox2-uid", Namespace: "sandbox2-ns"}]
assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano())
@ -701,6 +726,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) {
assert.Nil(p2.EphemeralStorage)
assert.Nil(p2.VolumeStats)
assert.Nil(p2.Network)
assert.Nil(p2.IO)
checkCRIPodCPUAndMemoryStats(assert, p2, infos[sandbox2Cgroup].Stats[0])
c3 := p2.Containers[0]
@ -711,6 +737,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) {
assert.Nil(c2.Logs)
assert.Nil(c2.Accelerators)
assert.Nil(c2.UserDefinedMetrics)
assert.Nil(c2.IO)
p3 := podStatsMap[statsapi.PodReference{Name: "sandbox3-name", UID: "sandbox3-uid", Namespace: "sandbox3-ns"}]
assert.Equal(sandbox3.CreatedAt, p3.StartTime.UnixNano())
@ -881,14 +908,21 @@ func makeFakeContainerStatsStrictlyFromCRI(seed int, container *critest.FakeCont
if container.State == runtimeapi.ContainerState_CONTAINER_EXITED {
containerStats.Cpu = nil
containerStats.Memory = nil
containerStats.Io = nil
} else {
containerStats.Cpu = &runtimeapi.CpuUsage{
Timestamp: timestamp.UnixNano(),
UsageCoreNanoSeconds: &runtimeapi.UInt64Value{Value: uint64(seed + offsetCRI + offsetCPUUsageCoreSeconds)},
Psi: getCRITestPSIStats(seed),
}
containerStats.Memory = &runtimeapi.MemoryUsage{
Timestamp: timestamp.UnixNano(),
WorkingSetBytes: &runtimeapi.UInt64Value{Value: uint64(seed + offsetCRI + offsetMemWorkingSetBytes)},
Psi: getCRITestPSIStats(seed),
}
containerStats.Io = &runtimeapi.IoUsage{
Timestamp: timestamp.UnixNano(),
Psi: getCRITestPSIStats(seed),
}
}
return containerStats
@ -906,19 +940,44 @@ func makeFakePodSandboxStatsStrictlyFromCRI(seed int, podSandbox *critest.FakePo
if podSandbox.State == runtimeapi.PodSandboxState_SANDBOX_NOTREADY {
podSandboxStats.Linux.Cpu = nil
podSandboxStats.Linux.Memory = nil
podSandboxStats.Linux.Io = nil
} else {
podSandboxStats.Linux.Cpu = &runtimeapi.CpuUsage{
Timestamp: timestamp.UnixNano(),
UsageCoreNanoSeconds: &runtimeapi.UInt64Value{Value: uint64(seed + offsetCRI + offsetCPUUsageCoreSeconds)},
Psi: getCRITestPSIStats(seed),
}
podSandboxStats.Linux.Memory = &runtimeapi.MemoryUsage{
Timestamp: timestamp.UnixNano(),
WorkingSetBytes: &runtimeapi.UInt64Value{Value: uint64(seed + offsetCRI + offsetMemWorkingSetBytes)},
Psi: getCRITestPSIStats(seed),
}
podSandboxStats.Linux.Io = &runtimeapi.IoUsage{
Timestamp: timestamp.UnixNano(),
Psi: getCRITestPSIStats(seed),
}
}
return podSandboxStats
}
func getCRITestPSIStats(seed int) *runtimeapi.PsiStats {
return &runtimeapi.PsiStats{
Full: getCRITestPSIData(seed),
Some: getCRITestPSIData(seed),
}
}
func getCRITestPSIData(seed int) *runtimeapi.PsiData {
return &runtimeapi.PsiData{
Total: uint64(seed + offsetPSIDataTotal),
Avg10: float64(10),
Avg60: float64(10),
Avg300: float64(10),
}
}
func getPodSandboxStatsStrictlyFromCRI(seed int, podSandbox *critest.FakePodSandbox) statsapi.PodStats {
psi := getTestPSIStats(seed)
podStats := statsapi.PodStats{
PodRef: statsapi.PodReference{
Name: podSandbox.Metadata.Name,
@ -931,16 +990,23 @@ func getPodSandboxStatsStrictlyFromCRI(seed int, podSandbox *critest.FakePodSand
if podSandbox.State == runtimeapi.PodSandboxState_SANDBOX_NOTREADY {
podStats.CPU = nil
podStats.Memory = nil
podStats.IO = nil
} else {
usageCoreNanoSeconds := uint64(seed + offsetCRI + offsetCPUUsageCoreSeconds)
workingSetBytes := uint64(seed + offsetCRI + offsetMemWorkingSetBytes)
podStats.CPU = &statsapi.CPUStats{
Time: metav1.NewTime(timestamp),
UsageCoreNanoSeconds: &usageCoreNanoSeconds,
PSI: cadvisorPSIToStatsPSI(&psi),
}
podStats.Memory = &statsapi.MemoryStats{
Time: metav1.NewTime(timestamp),
WorkingSetBytes: &workingSetBytes,
PSI: cadvisorPSIToStatsPSI(&psi),
}
podStats.IO = &statsapi.IOStats{
Time: metav1.NewTime(timestamp),
PSI: cadvisorPSIToStatsPSI(&psi),
}
}
@ -992,12 +1058,34 @@ func checkCRICPUAndMemoryStats(assert *assert.Assertions, actual statsapi.Contai
assert.Equal(cs.Memory.RSS, *actual.Memory.RSSBytes)
assert.Equal(cs.Memory.ContainerData.Pgfault, *actual.Memory.PageFaults)
assert.Equal(cs.Memory.ContainerData.Pgmajfault, *actual.Memory.MajorPageFaults)
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
checkCRIPSIStats(assert, &cs.Cpu.PSI, actual.CPU.PSI)
checkCRIPSIStats(assert, &cs.Memory.PSI, actual.Memory.PSI)
}
}
func checkCRICPUAndMemoryStatsForStrictlyFromCRI(assert *assert.Assertions, actual statsapi.ContainerStats, excepted statsapi.ContainerStats) {
assert.Equal(excepted.CPU.Time.UnixNano(), actual.CPU.Time.UnixNano())
assert.Equal(*excepted.CPU.UsageCoreNanoSeconds, *actual.CPU.UsageCoreNanoSeconds)
assert.Equal(*excepted.Memory.WorkingSetBytes, *actual.Memory.WorkingSetBytes)
func checkCRIIOStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *cadvisorapiv2.ContainerStats) {
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
checkCRIPSIStats(assert, &cs.DiskIo.PSI, actual.IO.PSI)
}
}
func checkCRICPUAndMemoryStatsForStrictlyFromCRI(assert *assert.Assertions, actual statsapi.ContainerStats, expected statsapi.ContainerStats) {
assert.Equal(expected.CPU.Time.UnixNano(), actual.CPU.Time.UnixNano())
assert.Equal(*expected.CPU.UsageCoreNanoSeconds, *actual.CPU.UsageCoreNanoSeconds)
assert.Equal(*expected.Memory.WorkingSetBytes, *actual.Memory.WorkingSetBytes)
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
checkCRIPSIStatsStrictlyFromCRI(assert, expected.CPU.PSI, actual.CPU.PSI)
checkCRIPSIStatsStrictlyFromCRI(assert, expected.Memory.PSI, actual.Memory.PSI)
}
}
func checkCRIIOStatsForStrictlyFromCRI(assert *assert.Assertions, actual statsapi.ContainerStats, expected statsapi.ContainerStats) {
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
checkCRIPSIStatsStrictlyFromCRI(assert, expected.IO.PSI, actual.IO.PSI)
}
}
func checkCRIRootfsStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *runtimeapi.ContainerStats, imageFsInfo *cadvisorapiv2.FsInfo) {
@ -1078,6 +1166,48 @@ func checkCRIPodCPUAndMemoryStats(assert *assert.Assertions, actual statsapi.Pod
assert.Equal(cs.Memory.RSS, *actual.Memory.RSSBytes)
assert.Equal(cs.Memory.ContainerData.Pgfault, *actual.Memory.PageFaults)
assert.Equal(cs.Memory.ContainerData.Pgmajfault, *actual.Memory.MajorPageFaults)
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
checkCRIPSIStats(assert, &cs.Cpu.PSI, actual.CPU.PSI)
checkCRIPSIStats(assert, &cs.Memory.PSI, actual.Memory.PSI)
}
}
func checkCRIPodIOStats(assert *assert.Assertions, actual statsapi.PodStats, cs *cadvisorapiv2.ContainerStats) {
if runtime.GOOS != "linux" {
return
}
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
checkCRIPSIStats(assert, &cs.DiskIo.PSI, actual.IO.PSI)
}
}
func checkCRIPSIStats(assert *assert.Assertions, want *cadvisorapiv1.PSIStats, got *statsapi.PSIStats) {
assert.NotNil(want)
assert.NotNil(got)
checkCRIPSIData(assert, want.Full, got.Full)
checkCRIPSIData(assert, want.Some, got.Some)
}
func checkCRIPSIData(assert *assert.Assertions, want cadvisorapiv1.PSIData, got statsapi.PSIData) {
assert.Equal(want.Total, got.Total)
assert.InDelta(want.Avg10, got.Avg10, 0.01)
assert.InDelta(want.Avg60, got.Avg60, 0.01)
assert.InDelta(want.Avg300, got.Avg300, 0.01)
}
func checkCRIPSIStatsStrictlyFromCRI(assert *assert.Assertions, want, got *statsapi.PSIStats) {
assert.NotNil(want)
assert.NotNil(got)
checkCRIPSIDataStrictlyFromCRI(assert, want.Full, got.Full)
checkCRIPSIDataStrictlyFromCRI(assert, want.Some, got.Some)
}
func checkCRIPSIDataStrictlyFromCRI(assert *assert.Assertions, want, got statsapi.PSIData) {
assert.Equal(want.Total, got.Total)
assert.InDelta(want.Avg10, got.Avg10, 0.01)
assert.InDelta(want.Avg60, got.Avg60, 0.01)
assert.InDelta(want.Avg300, got.Avg300, 0.01)
}
func checkCRIPodSwapStats(assert *assert.Assertions, actual statsapi.PodStats, cs *cadvisorapiv2.ContainerStats) {
@ -1089,13 +1219,27 @@ func checkCRIPodSwapStats(assert *assert.Assertions, actual statsapi.PodStats, c
assert.Equal(cs.Memory.Swap, *actual.Swap.SwapUsageBytes)
}
func checkCRIPodCPUAndMemoryStatsStrictlyFromCRI(assert *assert.Assertions, actual statsapi.PodStats, excepted statsapi.PodStats) {
func checkCRIPodCPUAndMemoryStatsStrictlyFromCRI(assert *assert.Assertions, actual statsapi.PodStats, expected statsapi.PodStats) {
if runtime.GOOS != "linux" {
return
}
assert.Equal(excepted.CPU.Time.UnixNano(), actual.CPU.Time.UnixNano())
assert.Equal(*excepted.CPU.UsageCoreNanoSeconds, *actual.CPU.UsageCoreNanoSeconds)
assert.Equal(*excepted.Memory.WorkingSetBytes, *actual.Memory.WorkingSetBytes)
assert.Equal(expected.CPU.Time.UnixNano(), actual.CPU.Time.UnixNano())
assert.Equal(*expected.CPU.UsageCoreNanoSeconds, *actual.CPU.UsageCoreNanoSeconds)
assert.Equal(*expected.Memory.WorkingSetBytes, *actual.Memory.WorkingSetBytes)
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
checkCRIPSIStatsStrictlyFromCRI(assert, expected.CPU.PSI, actual.CPU.PSI)
checkCRIPSIStatsStrictlyFromCRI(assert, expected.Memory.PSI, actual.Memory.PSI)
}
}
func checkCRIPodIOStatsStrictlyFromCRI(assert *assert.Assertions, actual statsapi.PodStats, expected statsapi.PodStats) {
if runtime.GOOS != "linux" {
return
}
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
checkCRIPSIStatsStrictlyFromCRI(assert, expected.IO.PSI, actual.IO.PSI)
}
}
func makeFakeLogStats(seed int) *volume.Metrics {
@ -1303,6 +1447,7 @@ func TestExtractIDFromCgroupPath(t *testing.T) {
}
func getCRIContainerStatsStrictlyFromCRI(seed int, containerName string) statsapi.ContainerStats {
psi := getTestPSIStats(seed)
result := statsapi.ContainerStats{
Name: containerName,
StartTime: metav1.NewTime(timestamp),
@ -1310,15 +1455,21 @@ func getCRIContainerStatsStrictlyFromCRI(seed int, containerName string) statsap
Memory: &statsapi.MemoryStats{},
// UserDefinedMetrics is not supported by CRI.
Rootfs: &statsapi.FsStats{},
IO: &statsapi.IOStats{},
}
result.CPU.Time = metav1.NewTime(timestamp)
usageCoreNanoSeconds := uint64(seed + offsetCRI + offsetCPUUsageCoreSeconds)
result.CPU.UsageCoreNanoSeconds = &usageCoreNanoSeconds
result.CPU.PSI = cadvisorPSIToStatsPSI(&psi)
result.Memory.Time = metav1.NewTime(timestamp)
workingSetBytes := uint64(seed + offsetCRI + offsetMemWorkingSetBytes)
result.Memory.WorkingSetBytes = &workingSetBytes
result.Memory.PSI = cadvisorPSIToStatsPSI(&psi)
result.IO.Time = metav1.NewTime(timestamp)
result.IO.PSI = cadvisorPSIToStatsPSI(&psi)
result.Rootfs.Time = metav1.NewTime(timestamp)
usedBytes := uint64(seed + offsetCRI + offsetFsUsage)

3
pkg/kubelet/stats/cri_stats_provider_windows.go
View File

@ -236,6 +236,9 @@ func addCRIPodMemoryStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandb
}
}
func addCRIPodIOStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
}
func addCRIPodProcessStats(ps *statsapi.PodStats, criPodStat *runtimeapi.PodSandboxStats) {
if criPodStat == nil || criPodStat.Windows == nil || criPodStat.Windows.Process == nil {
return

49
pkg/kubelet/stats/helper.go
View File

@ -24,8 +24,10 @@ import (
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
"k8s.io/kubernetes/pkg/kubelet/server/stats"
)
@ -53,6 +55,9 @@ func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsa
}
if cstat.Cpu != nil {
cpuStats.UsageCoreNanoSeconds = &cstat.Cpu.Usage.Total
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
cpuStats.PSI = cadvisorPSIToStatsPSI(&cstat.Cpu.PSI)
}
}
}
if info.Spec.HasMemory && cstat.Memory != nil {
@ -71,6 +76,9 @@ func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsa
availableBytes := info.Spec.Memory.Limit - cstat.Memory.WorkingSet
memoryStats.AvailableBytes = &availableBytes
}
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
memoryStats.PSI = cadvisorPSIToStatsPSI(&cstat.Memory.PSI)
}
} else {
memoryStats = &statsapi.MemoryStats{
Time: metav1.NewTime(cstat.Timestamp),
@ -96,6 +104,9 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo
result.CPU = cpu
result.Memory = memory
result.Swap = cadvisorInfoToSwapStats(info)
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
result.IO = cadvisorInfoToIOStats(info)
}
// NOTE: if they can be found, log stats will be overwritten
// by the caller, as it knows more information about the pod,
@ -307,6 +318,24 @@ func cadvisorInfoToSwapStats(info *cadvisorapiv2.ContainerInfo) *statsapi.SwapSt
return swapStats
}
func cadvisorInfoToIOStats(info *cadvisorapiv2.ContainerInfo) *statsapi.IOStats {
cstat, found := latestContainerStats(info)
if !found {
return nil
}
var ioStats *statsapi.IOStats
if info.Spec.HasDiskIo && cstat.DiskIo != nil {
ioStats = &statsapi.IOStats{
Time: metav1.NewTime(cstat.Timestamp),
PSI: cadvisorPSIToStatsPSI(&cstat.DiskIo.PSI),
}
}
return ioStats
}
// latestContainerStats returns the latest container stats from cadvisor, or nil if none exist
func latestContainerStats(info *cadvisorapiv2.ContainerInfo) (*cadvisorapiv2.ContainerStats, bool) {
stats := info.Stats
@ -493,3 +522,23 @@ func makePodStorageStats(s *statsapi.PodStats, rootFsInfo *cadvisorapiv2.FsInfo,
}
s.EphemeralStorage = calcEphemeralStorage(s.Containers, ephemeralStats, rootFsInfo, logStats, etcHostsStats, isCRIStatsProvider)
}
func cadvisorPSIToStatsPSI(psi *cadvisorapiv1.PSIStats) *statsapi.PSIStats {
if psi == nil {
return nil
}
return &statsapi.PSIStats{
Full: statsapi.PSIData{
Total: psi.Full.Total,
Avg10: psi.Full.Avg10,
Avg60: psi.Full.Avg60,
Avg300: psi.Full.Avg300,
},
Some: statsapi.PSIData{
Total: psi.Some.Total,
Avg10: psi.Some.Avg10,
Avg60: psi.Some.Avg60,
Avg300: psi.Some.Avg300,
},
}
}

36
pkg/kubelet/stats/helper_test.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package stats
import (
"reflect"
"testing"
"time"
@ -26,6 +27,7 @@ import (
"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
"k8s.io/utils/ptr"
)
@ -140,3 +142,37 @@ func TestMergeProcessStats(t *testing.T) {
})
}
}
// TestCadvisorPSIStruct checks the fields in cadvisor PSI structs. If cadvisor
// PSI structs change, the conversion between cadvisor PSI structs and kubelet stats API structs needs to be re-evaluated and updated.
func TestCadvisorPSIStructs(t *testing.T) {
psiStatsFields := sets.New("Full", "Some")
s := cadvisorapiv1.PSIStats{}
st := reflect.TypeOf(s)
for i := 0; i < st.NumField(); i++ {
field := st.Field(i)
if !psiStatsFields.Has(field.Name) {
t.Errorf("cadvisorapiv1.PSIStats contains unknown field: %s. The conversion between cadvisor PSIStats and kubelet stats API PSIStats needs to be re-evaluated and updated.", field.Name)
}
}
psiDataFields := map[string]reflect.Kind{
"Total": reflect.Uint64,
"Avg10": reflect.Float64,
"Avg60": reflect.Float64,
"Avg300": reflect.Float64,
}
d := cadvisorapiv1.PSIData{}
dt := reflect.TypeOf(d)
for i := 0; i < dt.NumField(); i++ {
field := dt.Field(i)
wantKind, fieldExist := psiDataFields[field.Name]
if !fieldExist {
t.Errorf("cadvisorapiv1.PSIData contains unknown field: %s. The conversion between cadvisor PSIData and kubelet stats API PSIData needs to be re-evaluated and updated.", field.Name)
}
if field.Type.Kind() != wantKind {
t.Errorf("unexpected cadvisorapiv1.PSIStats field %s type, want: %s, got: %s. The conversion between cadvisor PSIStats and kubelet stats API PSIStats needs to be re-evaluated and updated.", field.Name, wantKind, field.Type.Kind())
}
}
}

51
pkg/kubelet/stats/provider_test.go
View File

@ -32,7 +32,9 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
utilfeature "k8s.io/apiserver/pkg/util/feature"
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
"k8s.io/kubernetes/pkg/features"
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing"
@ -63,6 +65,7 @@ const (
offsetFsInodeUsage
offsetAcceleratorDutyCycle
offsetMemSwapUsageBytes
offsetPSIDataTotal
)
var (
@ -265,6 +268,7 @@ func getTestContainerInfo(seed int, podName string, podNamespace string, contain
CreationTime: testTime(creationTime, seed),
HasCpu: true,
HasMemory: true,
HasDiskIo: true,
HasNetwork: true,
Labels: labels,
Memory: cadvisorapiv2.MemorySpec{
@ -280,7 +284,9 @@ func getTestContainerInfo(seed int, podName string, podNamespace string, contain
stats := cadvisorapiv2.ContainerStats{
Timestamp: testTime(timestamp, seed),
Cpu: &cadvisorapiv1.CpuStats{},
Cpu: &cadvisorapiv1.CpuStats{
PSI: getTestPSIStats(seed),
},
CpuInst: &cadvisorapiv2.CpuInstStats{},
Memory: &cadvisorapiv1.MemoryStats{
Usage: uint64(seed + offsetMemUsageBytes),
@ -291,6 +297,7 @@ func getTestContainerInfo(seed int, podName string, podNamespace string, contain
Pgmajfault: uint64(seed + offsetMemMajorPageFaults),
},
Swap: uint64(seed + offsetMemSwapUsageBytes),
PSI: getTestPSIStats(seed),
},
Network: &cadvisorapiv2.NetworkStats{
Interfaces: []cadvisorapiv1.InterfaceStats{{
@ -323,6 +330,9 @@ func getTestContainerInfo(seed int, podName string, podNamespace string, contain
DutyCycle: uint64(seed + offsetAcceleratorDutyCycle),
},
},
DiskIo: &cadvisorapiv1.DiskIoStats{
PSI: getTestPSIStats(seed),
},
}
stats.Cpu.Usage.Total = uint64(seed + offsetCPUUsageCoreSeconds)
stats.CpuInst.Usage.Total = uint64(seed + offsetCPUUsageCores)
@ -332,6 +342,22 @@ func getTestContainerInfo(seed int, podName string, podNamespace string, contain
}
}
func getTestPSIStats(seed int) cadvisorapiv1.PSIStats {
return cadvisorapiv1.PSIStats{
Full: getTestPSIData(seed),
Some: getTestPSIData(seed),
}
}
func getTestPSIData(seed int) cadvisorapiv1.PSIData {
return cadvisorapiv1.PSIData{
Total: uint64(seed + offsetPSIDataTotal),
Avg10: float64(10),
Avg60: float64(10),
Avg300: float64(10),
}
}
func getTestFsInfo(seed int) cadvisorapiv2.FsInfo {
var (
inodes = uint64(seed + offsetFsInodes)
@ -469,6 +495,7 @@ func checkCPUStats(t *testing.T, label string, seed int, stats *statsapi.CPUStat
assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".CPU.Time")
assert.EqualValues(t, seed+offsetCPUUsageCores, *stats.UsageNanoCores, label+".CPU.UsageCores")
assert.EqualValues(t, seed+offsetCPUUsageCoreSeconds, *stats.UsageCoreNanoSeconds, label+".CPU.UsageCoreSeconds")
checkPSIStats(t, label+".CPU", seed, stats.PSI)
}
func checkMemoryStats(t *testing.T, label string, seed int, info cadvisorapiv2.ContainerInfo, stats *statsapi.MemoryStats) {
@ -484,6 +511,28 @@ func checkMemoryStats(t *testing.T, label string, seed int, info cadvisorapiv2.C
expected := info.Spec.Memory.Limit - *stats.WorkingSetBytes
assert.EqualValues(t, expected, *stats.AvailableBytes, label+".Mem.AvailableBytes")
}
checkPSIStats(t, label+".Mem", seed, stats.PSI)
}
func checkIOStats(t *testing.T, label string, seed int, info cadvisorapiv2.ContainerInfo, stats *statsapi.IOStats) {
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".Mem.Time")
checkPSIStats(t, label+".IO", seed, stats.PSI)
}
}
func checkPSIStats(t *testing.T, label string, seed int, stats *statsapi.PSIStats) {
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
require.NotNil(t, stats, label+".PSI")
assert.EqualValues(t, seed+offsetPSIDataTotal, stats.Full.Total, label+".PSI.Full.Total")
assert.InDelta(t, 10, stats.Full.Avg10, 0.01, label+".PSI.Full.Avg10")
assert.InDelta(t, 10, stats.Full.Avg60, 0.01, label+".PSI.Full.Avg60")
assert.InDelta(t, 10, stats.Full.Avg300, 0.01, label+".PSI.Full.Avg300")
assert.EqualValues(t, seed+offsetPSIDataTotal, stats.Some.Total, label+".PSI.Some.Total")
assert.InDelta(t, 10, stats.Some.Avg10, 0.01, label+".PSI.Some.Avg10")
assert.InDelta(t, 10, stats.Some.Avg60, 0.01, label+".PSI.Some.Avg60")
assert.InDelta(t, 10, stats.Some.Avg300, 0.01, label+".PSI.Some.Avg300")
}
}
func checkSwapStats(t *testing.T, label string, seed int, info cadvisorapiv2.ContainerInfo, stats *statsapi.SwapStats) {

2009
staging/src/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go
View File

File diff suppressed because it is too large Load Diff

36
staging/src/k8s.io/cri-api/pkg/apis/runtime/v1/api.proto
View File

@ -748,6 +748,8 @@ message LinuxPodSandboxStats {
ProcessUsage process = 4;
// Stats of containers in the measured pod sandbox.
repeated ContainerStats containers = 5;
// IO usage gathered for the pod sandbox.
IoUsage io = 6;
}
// WindowsPodSandboxStats provides the resource usage statistics for a pod sandbox on windows
@ -1793,6 +1795,8 @@ message ContainerStats {
FilesystemUsage writable_layer = 4;
// Swap usage gathered from the container.
SwapUsage swap = 5;
// IO usage gathered from the container.
IoUsage io = 6;
}
// WindowsContainerStats provides the resource usage statistics for a container specific for Windows
@ -1807,6 +1811,27 @@ message WindowsContainerStats {
WindowsFilesystemUsage writable_layer = 4;
}
// PSI statistics for an individual resource.
message PsiStats {
// PSI data for all tasks in the cgroup.
PsiData Full = 1;
// PSI data for some tasks in the cgroup.
PsiData Some = 2;
}
// PSI data for an individual resource.
message PsiData {
// Total time duration for tasks in the cgroup have waited due to congestion.
// Unit: nanoseconds.
uint64 Total = 1;
// The average (in %) tasks have waited due to congestion over a 10 second window.
double Avg10 = 2;
// The average (in %) tasks have waited due to congestion over a 60 second window.
double Avg60 = 3;
// The average (in %) tasks have waited due to congestion over a 300 second window.
double Avg300 = 4;
}
// CpuUsage provides the CPU usage information.
message CpuUsage {
// Timestamp in nanoseconds at which the information were collected. Must be > 0.
@ -1816,6 +1841,8 @@ message CpuUsage {
// Total CPU usage (sum of all cores) averaged over the sample window.
// The "core" unit can be interpreted as CPU core-nanoseconds per second.
UInt64Value usage_nano_cores = 3;
// CPU PSI statistics.
PsiStats psi = 4;
}
// WindowsCpuUsage provides the CPU usage information specific to Windows
@ -1845,6 +1872,15 @@ message MemoryUsage {
UInt64Value page_faults = 6;
// Cumulative number of major page faults.
UInt64Value major_page_faults = 7;
// Memory PSI statistics.
PsiStats psi = 8;
}
message IoUsage {
// Timestamp in nanoseconds at which the information were collected. Must be > 0.
int64 timestamp = 1;
// IO PSI statistics.
PsiStats psi = 2;
}
message SwapUsage {

45
staging/src/k8s.io/kubelet/pkg/apis/stats/v1alpha1/types.go
View File

@ -46,6 +46,9 @@ type NodeStats struct {
// Stats pertaining to memory (RAM) resources.
// +optional
Memory *MemoryStats `json:"memory,omitempty"`
// Stats pertaining to IO resources.
// +optional
IO *IOStats `json:"io,omitempty"`
// Stats pertaining to network resources.
// +optional
Network *NetworkStats `json:"network,omitempty"`
@ -127,6 +130,9 @@ type PodStats struct {
// Stats pertaining to memory (RAM) resources consumed by pod cgroup (which includes all containers' resource usage and pod overhead).
// +optional
Memory *MemoryStats `json:"memory,omitempty"`
// Stats pertaining to IO resources consumed by pod cgroup (which includes all containers' resource usage and pod overhead).
// +optional
IO *IOStats `json:"io,omitempty"`
// Stats pertaining to network resources.
// +optional
Network *NetworkStats `json:"network,omitempty"`
@ -159,6 +165,9 @@ type ContainerStats struct {
// Stats pertaining to memory (RAM) resources.
// +optional
Memory *MemoryStats `json:"memory,omitempty"`
// Stats pertaining to IO resources.
// +optional
IO *IOStats `json:"io,omitempty"`
// Metrics for Accelerators. Each Accelerator corresponds to one element in the array.
Accelerators []AcceleratorStats `json:"accelerators,omitempty"`
// Stats pertaining to container rootfs usage of filesystem resources.
@ -225,6 +234,9 @@ type CPUStats struct {
// Cumulative CPU usage (sum of all cores) since object creation.
// +optional
UsageCoreNanoSeconds *uint64 `json:"usageCoreNanoSeconds,omitempty"`
// CPU PSI stats.
// +optional
PSI *PSIStats `json:"psi,omitempty"`
}
// MemoryStats contains data about memory usage.
@ -252,6 +264,39 @@ type MemoryStats struct {
// Cumulative number of major page faults.
// +optional
MajorPageFaults *uint64 `json:"majorPageFaults,omitempty"`
// Memory PSI stats.
// +optional
PSI *PSIStats `json:"psi,omitempty"`
}
// IOStats contains data about IO usage.
type IOStats struct {
// The time at which these stats were updated.
Time metav1.Time `json:"time"`
// IO PSI stats.
// +optional
PSI *PSIStats `json:"psi,omitempty"`
}
// PSI statistics for an individual resource.
type PSIStats struct {
// PSI data for all tasks in the cgroup.
Full PSIData `json:"full"`
// PSI data for some tasks in the cgroup.
Some PSIData `json:"some"`
}
// PSI data for an individual resource.
type PSIData struct {
// Total time duration for tasks in the cgroup have waited due to congestion.
// Unit: nanoseconds.
Total uint64 `json:"total"`
// The average (in %) tasks have waited due to congestion over a 10 second window.
Avg10 float64 `json:"avg10"`
// The average (in %) tasks have waited due to congestion over a 60 second window.
Avg60 float64 `json:"avg60"`
// The average (in %) tasks have waited due to congestion over a 300 second window.
Avg300 float64 `json:"avg300"`
}
// SwapStats contains data about memory usage

6
test/compatibility_lifecycle/reference/versioned_feature_list.yaml
View File

@ -709,6 +709,12 @@
lockToDefault: false
preRelease: Alpha
version: "1.27"
- name: KubeletPSI
versionedSpecs:
- default: false
lockToDefault: false
preRelease: Alpha
version: "1.33"
- name: KubeletRegistrationGetOnExistsOnly
versionedSpecs:
- default: true

52
test/e2e_node/summary_test.go
View File

@ -19,6 +19,7 @@ package e2enode
import (
"context"
"fmt"
"math"
"os"
"strings"
"time"
@ -26,7 +27,9 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
kubeletstatsv1alpha1 "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/test/e2e/framework"
e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@ -100,6 +103,7 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() {
// for more information.
"UsageNanoCores": gomega.SatisfyAny(gstruct.PointTo(gomega.BeZero()), bounded(10000, 2e9)),
"UsageCoreNanoSeconds": bounded(10000000, 1e15),
"PSI": psiExpectation(),
}),
"Memory": ptrMatchAllFields(gstruct.Fields{
"Time": recent(maxStatsAge),
@ -111,7 +115,9 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() {
"RSSBytes": bounded(1*e2evolume.Mb, memoryLimit),
"PageFaults": bounded(1000, 1e9),
"MajorPageFaults": bounded(0, 1e9),
"PSI": psiExpectation(),
}),
"IO": ioExpectation(maxStatsAge),
"Swap": swapExpectation(memoryLimit),
"Accelerators": gomega.BeEmpty(),
"Rootfs": gomega.BeNil(),
@ -138,6 +144,7 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() {
"RSSBytes": bounded(1*e2evolume.Kb, memoryLimit),
"PageFaults": bounded(0, expectedPageFaultsUpperBound),
"MajorPageFaults": bounded(0, expectedMajorPageFaultsUpperBound),
"PSI": psiExpectation(),
})
runtimeContExpectations := sysContExpectations().(*gstruct.FieldsMatcher)
systemContainers := gstruct.Elements{
@ -159,6 +166,7 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() {
"RSSBytes": bounded(100*e2evolume.Kb, memoryLimit),
"PageFaults": bounded(1000, 1e9),
"MajorPageFaults": bounded(0, 1e9),
"PSI": psiExpectation(),
})
systemContainers["misc"] = miscContExpectations
}
@ -174,6 +182,7 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() {
"Time": recent(maxStatsAge),
"UsageNanoCores": bounded(10000, 1e9),
"UsageCoreNanoSeconds": bounded(10000000, 1e11),
"PSI": psiExpectation(),
}),
"Memory": ptrMatchAllFields(gstruct.Fields{
"Time": recent(maxStatsAge),
@ -183,7 +192,9 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() {
"RSSBytes": bounded(1*e2evolume.Kb, 80*e2evolume.Mb),
"PageFaults": bounded(100, expectedPageFaultsUpperBound),
"MajorPageFaults": bounded(0, expectedMajorPageFaultsUpperBound),
"PSI": psiExpectation(),
}),
"IO": ioExpectation(maxStatsAge),
"Swap": swapExpectation(memoryLimit),
"Accelerators": gomega.BeEmpty(),
"Rootfs": ptrMatchAllFields(gstruct.Fields{
@ -222,6 +233,7 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() {
"Time": recent(maxStatsAge),
"UsageNanoCores": bounded(10000, 1e9),
"UsageCoreNanoSeconds": bounded(10000000, 1e11),
"PSI": psiExpectation(),
}),
"Memory": ptrMatchAllFields(gstruct.Fields{
"Time": recent(maxStatsAge),
@ -231,7 +243,9 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() {
"RSSBytes": bounded(1*e2evolume.Kb, 80*e2evolume.Mb),
"PageFaults": bounded(0, expectedPageFaultsUpperBound),
"MajorPageFaults": bounded(0, expectedMajorPageFaultsUpperBound),
"PSI": psiExpectation(),
}),
"IO": ioExpectation(maxStatsAge),
"Swap": swapExpectation(memoryLimit),
"VolumeStats": gstruct.MatchAllElements(summaryObjectID, gstruct.Elements{
"test-empty-dir": gstruct.MatchAllFields(gstruct.Fields{
@ -272,6 +286,7 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() {
"Time": recent(maxStatsAge),
"UsageNanoCores": bounded(100e3, 2e9),
"UsageCoreNanoSeconds": bounded(1e9, 1e15),
"PSI": psiExpectation(),
}),
"Memory": ptrMatchAllFields(gstruct.Fields{
"Time": recent(maxStatsAge),
@ -282,7 +297,9 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() {
"RSSBytes": bounded(1*e2evolume.Kb, memoryLimit),
"PageFaults": bounded(1000, 1e9),
"MajorPageFaults": bounded(0, 1e9),
"PSI": psiExpectation(),
}),
"IO": ioExpectation(maxStatsAge),
"Swap": swapExpectation(memoryLimit),
// TODO(#28407): Handle non-eth0 network interface names.
"Network": ptrMatchAllFields(gstruct.Fields{
@ -419,9 +436,13 @@ func ptrMatchAllFields(fields gstruct.Fields) types.GomegaMatcher {
}
func bounded(lower, upper interface{}) types.GomegaMatcher {
return gstruct.PointTo(gomega.And(
return gstruct.PointTo(boundedValue(lower, upper))
}
func boundedValue(lower, upper interface{}) types.GomegaMatcher {
return gomega.And(
gomega.BeNumerically(">=", lower),
gomega.BeNumerically("<=", upper)))
gomega.BeNumerically("<=", upper))
}
func swapExpectation(upper interface{}) types.GomegaMatcher {
@ -492,3 +513,30 @@ func recordSystemCgroupProcesses(ctx context.Context) {
}
}
}
func psiExpectation() types.GomegaMatcher {
if !utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
return gomega.BeNil()
}
psiDataExpectation := gstruct.MatchAllFields(gstruct.Fields{
"Total": boundedValue(0, uint64(math.MaxUint64)),
"Avg10": boundedValue(0, 100),
"Avg60": boundedValue(0, 100),
"Avg300": boundedValue(0, 100),
})
return ptrMatchAllFields(gstruct.Fields{
"Full": psiDataExpectation,
"Some": psiDataExpectation,
})
}
func ioExpectation(maxStatsAge time.Duration) types.GomegaMatcher {
if !utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
return gomega.BeNil()
}
return gomega.Or(gomega.BeNil(),
ptrMatchAllFields(gstruct.Fields{
"Time": recent(maxStatsAge),
"PSI": psiExpectation(),
}))
}