diff --git a/pkg/kubelet/server/stats/summary_test.go b/pkg/kubelet/server/stats/summary_test.go index 81286856ea1..0fbdd30e888 100644 --- a/pkg/kubelet/server/stats/summary_test.go +++ b/pkg/kubelet/server/stats/summary_test.go @@ -29,7 +29,10 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/cm" statstest "k8s.io/kubernetes/pkg/kubelet/server/stats/testing" ) @@ -48,6 +51,7 @@ var ( ) func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true) ctx := context.Background() assert := assert.New(t) @@ -98,6 +102,7 @@ func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) { assert.Equal(summary.Node.CPU, cgroupStatsMap["/"].cs.CPU) assert.Equal(summary.Node.Memory, cgroupStatsMap["/"].cs.Memory) assert.Equal(summary.Node.Swap, cgroupStatsMap["/"].cs.Swap) + assert.Equal(summary.Node.IO, cgroupStatsMap["/"].cs.IO) assert.Equal(summary.Node.Network, cgroupStatsMap["/"].ns) assert.Equal(summary.Node.Fs, rootFsStats) assert.Equal(&statsapi.RuntimeStats{ContainerFs: imageFsStats, ImageFs: imageFsStats}, summary.Node.Runtime) @@ -111,6 +116,7 @@ func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) { Accelerators: cgroupStatsMap["/kubelet"].cs.Accelerators, UserDefinedMetrics: cgroupStatsMap["/kubelet"].cs.UserDefinedMetrics, Swap: cgroupStatsMap["/kubelet"].cs.Swap, + IO: cgroupStatsMap["/kubelet"].cs.IO, }) assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{ Name: "misc", @@ -120,6 +126,7 @@ func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) { Accelerators: cgroupStatsMap["/misc"].cs.Accelerators, UserDefinedMetrics: cgroupStatsMap["/misc"].cs.UserDefinedMetrics, Swap: cgroupStatsMap["/misc"].cs.Swap, + IO: cgroupStatsMap["/misc"].cs.IO, }) assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{ Name: "runtime", @@ -129,6 +136,7 @@ func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) { Accelerators: cgroupStatsMap["/runtime"].cs.Accelerators, UserDefinedMetrics: cgroupStatsMap["/runtime"].cs.UserDefinedMetrics, Swap: cgroupStatsMap["/runtime"].cs.Swap, + IO: cgroupStatsMap["/runtime"].cs.IO, }) assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{ Name: "pods", @@ -138,11 +146,13 @@ func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) { Accelerators: cgroupStatsMap["/pods"].cs.Accelerators, UserDefinedMetrics: cgroupStatsMap["/pods"].cs.UserDefinedMetrics, Swap: cgroupStatsMap["/pods"].cs.Swap, + IO: cgroupStatsMap["/pods"].cs.IO, }) assert.Equal(summary.Pods, podStats) } func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true) ctx := context.Background() assert := assert.New(t) @@ -194,6 +204,7 @@ func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) { assert.Equal(summary.Node.CPU, cgroupStatsMap["/"].cs.CPU) assert.Equal(summary.Node.Memory, cgroupStatsMap["/"].cs.Memory) assert.Equal(summary.Node.Swap, cgroupStatsMap["/"].cs.Swap) + assert.Equal(summary.Node.IO, cgroupStatsMap["/"].cs.IO) assert.Equal(summary.Node.Network, cgroupStatsMap["/"].ns) assert.Equal(summary.Node.Fs, rootFsStats) // Since we are a split filesystem we want root filesystem to be container fs and image to be image filesystem @@ -208,6 +219,7 @@ func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) { Accelerators: cgroupStatsMap["/kubelet"].cs.Accelerators, UserDefinedMetrics: cgroupStatsMap["/kubelet"].cs.UserDefinedMetrics, Swap: cgroupStatsMap["/kubelet"].cs.Swap, + IO: cgroupStatsMap["/kubelet"].cs.IO, }) assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{ Name: "misc", @@ -217,6 +229,7 @@ func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) { Accelerators: cgroupStatsMap["/misc"].cs.Accelerators, UserDefinedMetrics: cgroupStatsMap["/misc"].cs.UserDefinedMetrics, Swap: cgroupStatsMap["/misc"].cs.Swap, + IO: cgroupStatsMap["/misc"].cs.IO, }) assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{ Name: "runtime", @@ -226,6 +239,7 @@ func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) { Accelerators: cgroupStatsMap["/runtime"].cs.Accelerators, UserDefinedMetrics: cgroupStatsMap["/runtime"].cs.UserDefinedMetrics, Swap: cgroupStatsMap["/runtime"].cs.Swap, + IO: cgroupStatsMap["/runtime"].cs.IO, }) assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{ Name: "pods", @@ -235,11 +249,13 @@ func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) { Accelerators: cgroupStatsMap["/pods"].cs.Accelerators, UserDefinedMetrics: cgroupStatsMap["/pods"].cs.UserDefinedMetrics, Swap: cgroupStatsMap["/pods"].cs.Swap, + IO: cgroupStatsMap["/pods"].cs.IO, }) assert.Equal(summary.Pods, podStats) } func TestSummaryProviderGetCPUAndMemoryStats(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true) ctx := context.Background() assert := assert.New(t) @@ -283,6 +299,7 @@ func TestSummaryProviderGetCPUAndMemoryStats(t *testing.T) { assert.Nil(summary.Node.Network) assert.Nil(summary.Node.Fs) assert.Nil(summary.Node.Runtime) + assert.Nil(summary.Node.IO) assert.Len(summary.Node.SystemContainers, 4) assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{ diff --git a/pkg/kubelet/stats/cadvisor_stats_provider_test.go b/pkg/kubelet/stats/cadvisor_stats_provider_test.go index 11a0bc8dd1a..9a8eabae4f0 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider_test.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider_test.go @@ -114,6 +114,7 @@ func TestFilterTerminatedContainerInfoAndAssembleByPodCgroupKey(t *testing.T) { } func TestCadvisorListPodStats(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true) ctx := context.Background() const ( namespace0 = "test0" @@ -295,12 +296,14 @@ func TestCadvisorListPodStats(t *testing.T) { checkCPUStats(t, "Pod0Container0", seedPod0Container0, con.CPU) checkMemoryStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.Memory) checkSwapStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.Swap) + checkIOStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.IO) con = indexCon[cName01] assert.EqualValues(t, testTime(creationTime, seedPod0Container1).Unix(), con.StartTime.Time.Unix()) checkCPUStats(t, "Pod0Container1", seedPod0Container1, con.CPU) checkMemoryStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.Memory) checkSwapStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.Swap) + checkIOStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.IO) assert.EqualValues(t, p0Time.Unix(), ps.StartTime.Time.Unix()) checkNetworkStats(t, "Pod0", seedPod0Infra, ps.Network) @@ -315,6 +318,9 @@ func TestCadvisorListPodStats(t *testing.T) { checkSwapStats(t, "Pod0", seedPod0Infra, infos["/pod0-i"], ps.Swap) checkContainersSwapStats(t, ps, infos["/pod0-c0"], infos["/pod0-c1"]) } + if ps.IO != nil { + checkIOStats(t, "Pod0", seedPod0Infra, infos["/pod0-i"], ps.IO) + } // Validate Pod1 Results ps, found = indexPods[prf1] @@ -325,6 +331,7 @@ func TestCadvisorListPodStats(t *testing.T) { checkCPUStats(t, "Pod1Container0", seedPod1Container, con.CPU) checkMemoryStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.Memory) checkSwapStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.Swap) + checkIOStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.IO) checkNetworkStats(t, "Pod1", seedPod1Infra, ps.Network) checkContainersSwapStats(t, ps, infos["/pod1-c0"]) @@ -337,6 +344,7 @@ func TestCadvisorListPodStats(t *testing.T) { checkCPUStats(t, "Pod2Container0", seedPod2Container, con.CPU) checkMemoryStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.Memory) checkSwapStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.Swap) + checkIOStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.IO) checkNetworkStats(t, "Pod2", seedPod2Infra, ps.Network) checkContainersSwapStats(t, ps, infos["/pod2-c0"]) @@ -355,10 +363,12 @@ func TestCadvisorListPodStats(t *testing.T) { checkCPUStats(t, "Pod3Container1", seedPod3Container1, con.CPU) checkMemoryStats(t, "Pod3Container1", seedPod3Container1, infos["/pod3-c1"], con.Memory) checkSwapStats(t, "Pod3Container1", seedPod3Container1, infos["/pod3-c1"], con.Swap) + checkIOStats(t, "Pod3Container1", seedPod3Container1, infos["/pod3-c1"], con.IO) checkContainersSwapStats(t, ps, infos["/pod3-c1"]) } func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true) ctx := context.Background() const ( namespace0 = "test0" @@ -476,6 +486,7 @@ func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) { assert.Nil(t, con.Logs) assert.Nil(t, con.Accelerators) assert.Nil(t, con.UserDefinedMetrics) + assert.Nil(t, con.IO) con = indexCon[cName01] assert.EqualValues(t, testTime(creationTime, seedPod0Container1).Unix(), con.StartTime.Time.Unix()) @@ -486,11 +497,13 @@ func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) { assert.Nil(t, con.Logs) assert.Nil(t, con.Accelerators) assert.Nil(t, con.UserDefinedMetrics) + assert.Nil(t, con.IO) assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix()) assert.Nil(t, ps.EphemeralStorage) assert.Nil(t, ps.VolumeStats) assert.Nil(t, ps.Network) + assert.Nil(t, con.IO) if ps.CPU != nil { checkCPUStats(t, "Pod0", seedPod0Infra, ps.CPU) } @@ -515,6 +528,7 @@ func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) { assert.Nil(t, ps.EphemeralStorage) assert.Nil(t, ps.VolumeStats) assert.Nil(t, ps.Network) + assert.Nil(t, con.IO) // Validate Pod2 Results ps, found = indexPods[prf2] @@ -529,6 +543,7 @@ func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) { assert.Nil(t, ps.EphemeralStorage) assert.Nil(t, ps.VolumeStats) assert.Nil(t, ps.Network) + assert.Nil(t, con.IO) } func TestCadvisorImagesFsStatsKubeletSeparateDiskOff(t *testing.T) { diff --git a/pkg/kubelet/stats/cri_stats_provider_test.go b/pkg/kubelet/stats/cri_stats_provider_test.go index 1011b05f0a5..e8694183a89 100644 --- a/pkg/kubelet/stats/cri_stats_provider_test.go +++ b/pkg/kubelet/stats/cri_stats_provider_test.go @@ -27,16 +27,20 @@ import ( "time" cadvisorfs "github.com/google/cadvisor/fs" + cadvisorapiv1 "github.com/google/cadvisor/info/v1" cadvisorapiv2 "github.com/google/cadvisor/info/v2" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" critest "k8s.io/cri-api/pkg/apis/testing" statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1" kubelettypes "k8s.io/kubelet/pkg/types" + "k8s.io/kubernetes/pkg/features" cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" "k8s.io/kubernetes/pkg/kubelet/cm" kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing" @@ -91,6 +95,7 @@ const ( const testPodLogDirectory = "/var/log/kube/pods/" // Use non-default path to ensure stats are collected properly func TestCRIListPodStats(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true) ctx := context.Background() var ( imageFsMountpoint = "/test/mount/point" @@ -265,6 +270,7 @@ func TestCRIListPodStats(t *testing.T) { c0 := containerStatsMap[cName0] assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano()) checkCRICPUAndMemoryStats(assert, c0, infos[container0.ContainerStatus.Id].Stats[0]) + checkCRIIOStats(assert, c0, infos[container0.ContainerStatus.Id].Stats[0]) assert.Nil(c0.Accelerators) checkCRIRootfsStats(assert, c0, containerStats0, &imageFsInfo) checkCRILogsStats(assert, c0, &rootFsInfo, containerLogStats0) @@ -272,12 +278,14 @@ func TestCRIListPodStats(t *testing.T) { c1 := containerStatsMap[cName1] assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano()) checkCRICPUAndMemoryStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0]) + checkCRIIOStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0]) assert.Nil(c0.Accelerators) checkCRIRootfsStats(assert, c1, containerStats1, nil) checkCRILogsStats(assert, c1, &rootFsInfo, containerLogStats1) checkCRINetworkStats(assert, p0.Network, infos[sandbox0.PodSandboxStatus.Id].Stats[0].Network) checkCRIPodCPUAndMemoryStats(assert, p0, infos[sandbox0Cgroup].Stats[0]) checkCRIPodSwapStats(assert, p0, infos[sandbox0Cgroup].Stats[0]) + checkCRIPodIOStats(assert, p0, infos[sandbox0Cgroup].Stats[0]) checkContainersSwapStats(t, p0, infos[container0.Id], infos[container1.Id]) @@ -291,12 +299,14 @@ func TestCRIListPodStats(t *testing.T) { assert.Equal(cName2, c2.Name) assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano()) checkCRICPUAndMemoryStats(assert, c2, infos[container2.ContainerStatus.Id].Stats[0]) + checkCRIIOStats(assert, c2, infos[container2.ContainerStatus.Id].Stats[0]) assert.Nil(c0.Accelerators) checkCRIRootfsStats(assert, c2, containerStats2, &imageFsInfo) checkCRILogsStats(assert, c2, &rootFsInfo, containerLogStats2) checkCRINetworkStats(assert, p1.Network, infos[sandbox1.PodSandboxStatus.Id].Stats[0].Network) checkCRIPodCPUAndMemoryStats(assert, p1, infos[sandbox1Cgroup].Stats[0]) checkCRIPodSwapStats(assert, p1, infos[sandbox1Cgroup].Stats[0]) + checkCRIPodIOStats(assert, p1, infos[sandbox1Cgroup].Stats[0]) checkContainersSwapStats(t, p1, infos[container2.Id]) @@ -311,6 +321,7 @@ func TestCRIListPodStats(t *testing.T) { assert.Equal(cName3, c3.Name) assert.Equal(container4.CreatedAt, c3.StartTime.UnixNano()) checkCRICPUAndMemoryStats(assert, c3, infos[container4.ContainerStatus.Id].Stats[0]) + checkCRIIOStats(assert, c3, infos[container4.ContainerStatus.Id].Stats[0]) assert.Nil(c0.Accelerators) checkCRIRootfsStats(assert, c3, containerStats4, &imageFsInfo) @@ -318,6 +329,7 @@ func TestCRIListPodStats(t *testing.T) { checkCRINetworkStats(assert, p2.Network, infos[sandbox2.PodSandboxStatus.Id].Stats[0].Network) checkCRIPodCPUAndMemoryStats(assert, p2, infos[sandbox2Cgroup].Stats[0]) checkCRIPodSwapStats(assert, p2, infos[sandbox2Cgroup].Stats[0]) + checkCRIPodIOStats(assert, p2, infos[sandbox2Cgroup].Stats[0]) checkContainersSwapStats(t, p2, infos[container4.Id]) @@ -332,9 +344,11 @@ func TestCRIListPodStats(t *testing.T) { assert.NotNil(c8.Memory.Time) checkCRIPodCPUAndMemoryStats(assert, p3, infos[sandbox3Cgroup].Stats[0]) checkCRIPodSwapStats(assert, p3, infos[sandbox3Cgroup].Stats[0]) + checkCRIPodIOStats(assert, p3, infos[sandbox3Cgroup].Stats[0]) } func TestListPodStatsStrictlyFromCRI(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true) if runtime.GOOS == "windows" { // TODO: remove skip once the failing test has been fixed. t.Skip("Skip failing test on Windows.") @@ -494,6 +508,7 @@ func TestListPodStatsStrictlyFromCRI(t *testing.T) { c0 := containerStatsMap[cName0] assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano()) checkCRICPUAndMemoryStatsForStrictlyFromCRI(assert, c0, exceptedContainerStatsMap[cName0]) + checkCRIIOStatsForStrictlyFromCRI(assert, c0, exceptedContainerStatsMap[cName0]) assert.Nil(c0.Accelerators) checkCRIRootfsStats(assert, c0, containerStats0, &imageFsInfo) checkCRILogsStats(assert, c0, &rootFsInfo, containerLogStats0) @@ -501,10 +516,12 @@ func TestListPodStatsStrictlyFromCRI(t *testing.T) { c1 := containerStatsMap[cName1] assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano()) checkCRICPUAndMemoryStatsForStrictlyFromCRI(assert, c1, exceptedContainerStatsMap[cName1]) + checkCRIIOStatsForStrictlyFromCRI(assert, c1, exceptedContainerStatsMap[cName1]) assert.Nil(c0.Accelerators) checkCRIRootfsStats(assert, c1, containerStats1, nil) checkCRILogsStats(assert, c1, &rootFsInfo, containerLogStats1) checkCRIPodCPUAndMemoryStatsStrictlyFromCRI(assert, p0, exceptedPodStatsMap[prf0]) + checkCRIPodIOStatsStrictlyFromCRI(assert, p0, exceptedPodStatsMap[prf0]) assert.NotNil(cadvisorInfos[sandbox0Cgroup].Stats[0].Cpu) assert.NotNil(cadvisorInfos[sandbox0Cgroup].Stats[0].Memory) @@ -518,10 +535,12 @@ func TestListPodStatsStrictlyFromCRI(t *testing.T) { assert.Equal(cName2, c2.Name) assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano()) checkCRICPUAndMemoryStatsForStrictlyFromCRI(assert, c2, exceptedContainerStatsMap[cName2]) + checkCRIIOStatsForStrictlyFromCRI(assert, c2, exceptedContainerStatsMap[cName2]) assert.Nil(c0.Accelerators) checkCRIRootfsStats(assert, c2, containerStats2, &imageFsInfo) checkCRILogsStats(assert, c2, &rootFsInfo, containerLogStats2) checkCRIPodCPUAndMemoryStatsStrictlyFromCRI(assert, p1, exceptedPodStatsMap[prf1]) + checkCRIPodIOStatsStrictlyFromCRI(assert, p1, exceptedPodStatsMap[prf1]) if runtime.GOOS == "linux" { if _, ok := cadvisorInfos[sandbox1Cgroup]; ok { @@ -530,6 +549,7 @@ func TestListPodStatsStrictlyFromCRI(t *testing.T) { } } func TestCRIListPodCPUAndMemoryStats(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true) ctx := context.Background() var ( @@ -656,6 +676,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) { assert.Nil(p0.EphemeralStorage) assert.Nil(p0.VolumeStats) assert.Nil(p0.Network) + assert.Nil(p0.IO) checkCRIPodCPUAndMemoryStats(assert, p0, infos[sandbox0Cgroup].Stats[0]) containerStatsMap := make(map[string]statsapi.ContainerStats) @@ -670,6 +691,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) { assert.Nil(c0.Logs) assert.Nil(c0.Accelerators) assert.Nil(c0.UserDefinedMetrics) + assert.Nil(c0.IO) c1 := containerStatsMap[cName1] assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano()) checkCRICPUAndMemoryStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0]) @@ -677,6 +699,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) { assert.Nil(c1.Logs) assert.Nil(c1.Accelerators) assert.Nil(c1.UserDefinedMetrics) + assert.Nil(c1.IO) p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}] assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano()) @@ -684,6 +707,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) { assert.Nil(p1.EphemeralStorage) assert.Nil(p1.VolumeStats) assert.Nil(p1.Network) + assert.Nil(p1.IO) checkCRIPodCPUAndMemoryStats(assert, p1, infos[sandbox1Cgroup].Stats[0]) c2 := p1.Containers[0] @@ -694,6 +718,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) { assert.Nil(c2.Logs) assert.Nil(c2.Accelerators) assert.Nil(c2.UserDefinedMetrics) + assert.Nil(c2.IO) p2 := podStatsMap[statsapi.PodReference{Name: "sandbox2-name", UID: "sandbox2-uid", Namespace: "sandbox2-ns"}] assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano()) @@ -701,6 +726,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) { assert.Nil(p2.EphemeralStorage) assert.Nil(p2.VolumeStats) assert.Nil(p2.Network) + assert.Nil(p2.IO) checkCRIPodCPUAndMemoryStats(assert, p2, infos[sandbox2Cgroup].Stats[0]) c3 := p2.Containers[0] @@ -711,6 +737,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) { assert.Nil(c2.Logs) assert.Nil(c2.Accelerators) assert.Nil(c2.UserDefinedMetrics) + assert.Nil(c2.IO) p3 := podStatsMap[statsapi.PodReference{Name: "sandbox3-name", UID: "sandbox3-uid", Namespace: "sandbox3-ns"}] assert.Equal(sandbox3.CreatedAt, p3.StartTime.UnixNano()) @@ -881,14 +908,21 @@ func makeFakeContainerStatsStrictlyFromCRI(seed int, container *critest.FakeCont if container.State == runtimeapi.ContainerState_CONTAINER_EXITED { containerStats.Cpu = nil containerStats.Memory = nil + containerStats.Io = nil } else { containerStats.Cpu = &runtimeapi.CpuUsage{ Timestamp: timestamp.UnixNano(), UsageCoreNanoSeconds: &runtimeapi.UInt64Value{Value: uint64(seed + offsetCRI + offsetCPUUsageCoreSeconds)}, + Psi: getCRITestPSIStats(seed), } containerStats.Memory = &runtimeapi.MemoryUsage{ Timestamp: timestamp.UnixNano(), WorkingSetBytes: &runtimeapi.UInt64Value{Value: uint64(seed + offsetCRI + offsetMemWorkingSetBytes)}, + Psi: getCRITestPSIStats(seed), + } + containerStats.Io = &runtimeapi.IoUsage{ + Timestamp: timestamp.UnixNano(), + Psi: getCRITestPSIStats(seed), } } return containerStats @@ -906,19 +940,44 @@ func makeFakePodSandboxStatsStrictlyFromCRI(seed int, podSandbox *critest.FakePo if podSandbox.State == runtimeapi.PodSandboxState_SANDBOX_NOTREADY { podSandboxStats.Linux.Cpu = nil podSandboxStats.Linux.Memory = nil + podSandboxStats.Linux.Io = nil } else { podSandboxStats.Linux.Cpu = &runtimeapi.CpuUsage{ Timestamp: timestamp.UnixNano(), UsageCoreNanoSeconds: &runtimeapi.UInt64Value{Value: uint64(seed + offsetCRI + offsetCPUUsageCoreSeconds)}, + Psi: getCRITestPSIStats(seed), } podSandboxStats.Linux.Memory = &runtimeapi.MemoryUsage{ Timestamp: timestamp.UnixNano(), WorkingSetBytes: &runtimeapi.UInt64Value{Value: uint64(seed + offsetCRI + offsetMemWorkingSetBytes)}, + Psi: getCRITestPSIStats(seed), + } + podSandboxStats.Linux.Io = &runtimeapi.IoUsage{ + Timestamp: timestamp.UnixNano(), + Psi: getCRITestPSIStats(seed), } } return podSandboxStats } + +func getCRITestPSIStats(seed int) *runtimeapi.PsiStats { + return &runtimeapi.PsiStats{ + Full: getCRITestPSIData(seed), + Some: getCRITestPSIData(seed), + } +} + +func getCRITestPSIData(seed int) *runtimeapi.PsiData { + return &runtimeapi.PsiData{ + Total: uint64(seed + offsetPSIDataTotal), + Avg10: float64(10), + Avg60: float64(10), + Avg300: float64(10), + } +} + func getPodSandboxStatsStrictlyFromCRI(seed int, podSandbox *critest.FakePodSandbox) statsapi.PodStats { + psi := getTestPSIStats(seed) podStats := statsapi.PodStats{ PodRef: statsapi.PodReference{ Name: podSandbox.Metadata.Name, @@ -931,16 +990,23 @@ func getPodSandboxStatsStrictlyFromCRI(seed int, podSandbox *critest.FakePodSand if podSandbox.State == runtimeapi.PodSandboxState_SANDBOX_NOTREADY { podStats.CPU = nil podStats.Memory = nil + podStats.IO = nil } else { usageCoreNanoSeconds := uint64(seed + offsetCRI + offsetCPUUsageCoreSeconds) workingSetBytes := uint64(seed + offsetCRI + offsetMemWorkingSetBytes) podStats.CPU = &statsapi.CPUStats{ Time: metav1.NewTime(timestamp), UsageCoreNanoSeconds: &usageCoreNanoSeconds, + PSI: cadvisorPSIToStatsPSI(&psi), } podStats.Memory = &statsapi.MemoryStats{ Time: metav1.NewTime(timestamp), WorkingSetBytes: &workingSetBytes, + PSI: cadvisorPSIToStatsPSI(&psi), + } + podStats.IO = &statsapi.IOStats{ + Time: metav1.NewTime(timestamp), + PSI: cadvisorPSIToStatsPSI(&psi), } } @@ -992,12 +1058,34 @@ func checkCRICPUAndMemoryStats(assert *assert.Assertions, actual statsapi.Contai assert.Equal(cs.Memory.RSS, *actual.Memory.RSSBytes) assert.Equal(cs.Memory.ContainerData.Pgfault, *actual.Memory.PageFaults) assert.Equal(cs.Memory.ContainerData.Pgmajfault, *actual.Memory.MajorPageFaults) + + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) { + checkCRIPSIStats(assert, &cs.Cpu.PSI, actual.CPU.PSI) + checkCRIPSIStats(assert, &cs.Memory.PSI, actual.Memory.PSI) + } } -func checkCRICPUAndMemoryStatsForStrictlyFromCRI(assert *assert.Assertions, actual statsapi.ContainerStats, excepted statsapi.ContainerStats) { - assert.Equal(excepted.CPU.Time.UnixNano(), actual.CPU.Time.UnixNano()) - assert.Equal(*excepted.CPU.UsageCoreNanoSeconds, *actual.CPU.UsageCoreNanoSeconds) - assert.Equal(*excepted.Memory.WorkingSetBytes, *actual.Memory.WorkingSetBytes) +func checkCRIIOStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *cadvisorapiv2.ContainerStats) { + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) { + checkCRIPSIStats(assert, &cs.DiskIo.PSI, actual.IO.PSI) + } +} + +func checkCRICPUAndMemoryStatsForStrictlyFromCRI(assert *assert.Assertions, actual statsapi.ContainerStats, expected statsapi.ContainerStats) { + assert.Equal(expected.CPU.Time.UnixNano(), actual.CPU.Time.UnixNano()) + assert.Equal(*expected.CPU.UsageCoreNanoSeconds, *actual.CPU.UsageCoreNanoSeconds) + assert.Equal(*expected.Memory.WorkingSetBytes, *actual.Memory.WorkingSetBytes) + + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) { + checkCRIPSIStatsStrictlyFromCRI(assert, expected.CPU.PSI, actual.CPU.PSI) + checkCRIPSIStatsStrictlyFromCRI(assert, expected.Memory.PSI, actual.Memory.PSI) + } +} + +func checkCRIIOStatsForStrictlyFromCRI(assert *assert.Assertions, actual statsapi.ContainerStats, expected statsapi.ContainerStats) { + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) { + checkCRIPSIStatsStrictlyFromCRI(assert, expected.IO.PSI, actual.IO.PSI) + } } func checkCRIRootfsStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *runtimeapi.ContainerStats, imageFsInfo *cadvisorapiv2.FsInfo) { @@ -1078,6 +1166,48 @@ func checkCRIPodCPUAndMemoryStats(assert *assert.Assertions, actual statsapi.Pod assert.Equal(cs.Memory.RSS, *actual.Memory.RSSBytes) assert.Equal(cs.Memory.ContainerData.Pgfault, *actual.Memory.PageFaults) assert.Equal(cs.Memory.ContainerData.Pgmajfault, *actual.Memory.MajorPageFaults) + + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) { + checkCRIPSIStats(assert, &cs.Cpu.PSI, actual.CPU.PSI) + checkCRIPSIStats(assert, &cs.Memory.PSI, actual.Memory.PSI) + } +} + +func checkCRIPodIOStats(assert *assert.Assertions, actual statsapi.PodStats, cs *cadvisorapiv2.ContainerStats) { + if runtime.GOOS != "linux" { + return + } + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) { + checkCRIPSIStats(assert, &cs.DiskIo.PSI, actual.IO.PSI) + } +} + +func checkCRIPSIStats(assert *assert.Assertions, want *cadvisorapiv1.PSIStats, got *statsapi.PSIStats) { + assert.NotNil(want) + assert.NotNil(got) + checkCRIPSIData(assert, want.Full, got.Full) + checkCRIPSIData(assert, want.Some, got.Some) +} + +func checkCRIPSIData(assert *assert.Assertions, want cadvisorapiv1.PSIData, got statsapi.PSIData) { + assert.Equal(want.Total, got.Total) + assert.InDelta(want.Avg10, got.Avg10, 0.01) + assert.InDelta(want.Avg60, got.Avg60, 0.01) + assert.InDelta(want.Avg300, got.Avg300, 0.01) +} + +func checkCRIPSIStatsStrictlyFromCRI(assert *assert.Assertions, want, got *statsapi.PSIStats) { + assert.NotNil(want) + assert.NotNil(got) + checkCRIPSIDataStrictlyFromCRI(assert, want.Full, got.Full) + checkCRIPSIDataStrictlyFromCRI(assert, want.Some, got.Some) +} + +func checkCRIPSIDataStrictlyFromCRI(assert *assert.Assertions, want, got statsapi.PSIData) { + assert.Equal(want.Total, got.Total) + assert.InDelta(want.Avg10, got.Avg10, 0.01) + assert.InDelta(want.Avg60, got.Avg60, 0.01) + assert.InDelta(want.Avg300, got.Avg300, 0.01) } func checkCRIPodSwapStats(assert *assert.Assertions, actual statsapi.PodStats, cs *cadvisorapiv2.ContainerStats) { @@ -1089,13 +1219,27 @@ func checkCRIPodSwapStats(assert *assert.Assertions, actual statsapi.PodStats, c assert.Equal(cs.Memory.Swap, *actual.Swap.SwapUsageBytes) } -func checkCRIPodCPUAndMemoryStatsStrictlyFromCRI(assert *assert.Assertions, actual statsapi.PodStats, excepted statsapi.PodStats) { +func checkCRIPodCPUAndMemoryStatsStrictlyFromCRI(assert *assert.Assertions, actual statsapi.PodStats, expected statsapi.PodStats) { if runtime.GOOS != "linux" { return } - assert.Equal(excepted.CPU.Time.UnixNano(), actual.CPU.Time.UnixNano()) - assert.Equal(*excepted.CPU.UsageCoreNanoSeconds, *actual.CPU.UsageCoreNanoSeconds) - assert.Equal(*excepted.Memory.WorkingSetBytes, *actual.Memory.WorkingSetBytes) + assert.Equal(expected.CPU.Time.UnixNano(), actual.CPU.Time.UnixNano()) + assert.Equal(*expected.CPU.UsageCoreNanoSeconds, *actual.CPU.UsageCoreNanoSeconds) + assert.Equal(*expected.Memory.WorkingSetBytes, *actual.Memory.WorkingSetBytes) + + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) { + checkCRIPSIStatsStrictlyFromCRI(assert, expected.CPU.PSI, actual.CPU.PSI) + checkCRIPSIStatsStrictlyFromCRI(assert, expected.Memory.PSI, actual.Memory.PSI) + } +} + +func checkCRIPodIOStatsStrictlyFromCRI(assert *assert.Assertions, actual statsapi.PodStats, expected statsapi.PodStats) { + if runtime.GOOS != "linux" { + return + } + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) { + checkCRIPSIStatsStrictlyFromCRI(assert, expected.IO.PSI, actual.IO.PSI) + } } func makeFakeLogStats(seed int) *volume.Metrics { @@ -1303,6 +1447,7 @@ func TestExtractIDFromCgroupPath(t *testing.T) { } func getCRIContainerStatsStrictlyFromCRI(seed int, containerName string) statsapi.ContainerStats { + psi := getTestPSIStats(seed) result := statsapi.ContainerStats{ Name: containerName, StartTime: metav1.NewTime(timestamp), @@ -1310,15 +1455,21 @@ func getCRIContainerStatsStrictlyFromCRI(seed int, containerName string) statsap Memory: &statsapi.MemoryStats{}, // UserDefinedMetrics is not supported by CRI. Rootfs: &statsapi.FsStats{}, + IO: &statsapi.IOStats{}, } result.CPU.Time = metav1.NewTime(timestamp) usageCoreNanoSeconds := uint64(seed + offsetCRI + offsetCPUUsageCoreSeconds) result.CPU.UsageCoreNanoSeconds = &usageCoreNanoSeconds + result.CPU.PSI = cadvisorPSIToStatsPSI(&psi) result.Memory.Time = metav1.NewTime(timestamp) workingSetBytes := uint64(seed + offsetCRI + offsetMemWorkingSetBytes) result.Memory.WorkingSetBytes = &workingSetBytes + result.Memory.PSI = cadvisorPSIToStatsPSI(&psi) + + result.IO.Time = metav1.NewTime(timestamp) + result.IO.PSI = cadvisorPSIToStatsPSI(&psi) result.Rootfs.Time = metav1.NewTime(timestamp) usedBytes := uint64(seed + offsetCRI + offsetFsUsage) diff --git a/pkg/kubelet/stats/helper_test.go b/pkg/kubelet/stats/helper_test.go index d0bc23b69fa..3c111363bc1 100644 --- a/pkg/kubelet/stats/helper_test.go +++ b/pkg/kubelet/stats/helper_test.go @@ -17,6 +17,7 @@ limitations under the License. package stats import ( + "reflect" "testing" "time" @@ -26,6 +27,7 @@ import ( "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1" "k8s.io/utils/ptr" ) @@ -140,3 +142,37 @@ func TestMergeProcessStats(t *testing.T) { }) } } + +// TestCadvisorPSIStruct checks the fields in cadvisor PSI structs. If cadvisor +// PSI structs change, the conversion between cadvisor PSI structs and kubelet stats API structs needs to be re-evaluated and updated. +func TestCadvisorPSIStructs(t *testing.T) { + psiStatsFields := sets.New("Full", "Some") + s := cadvisorapiv1.PSIStats{} + st := reflect.TypeOf(s) + for i := 0; i < st.NumField(); i++ { + field := st.Field(i) + if !psiStatsFields.Has(field.Name) { + t.Errorf("cadvisorapiv1.PSIStats contains unknown field: %s. The conversion between cadvisor PSIStats and kubelet stats API PSIStats needs to be re-evaluated and updated.", field.Name) + } + } + + psiDataFields := map[string]reflect.Kind{ + "Total": reflect.Uint64, + "Avg10": reflect.Float64, + "Avg60": reflect.Float64, + "Avg300": reflect.Float64, + } + d := cadvisorapiv1.PSIData{} + dt := reflect.TypeOf(d) + for i := 0; i < dt.NumField(); i++ { + field := dt.Field(i) + wantKind, fieldExist := psiDataFields[field.Name] + if !fieldExist { + t.Errorf("cadvisorapiv1.PSIData contains unknown field: %s. The conversion between cadvisor PSIData and kubelet stats API PSIData needs to be re-evaluated and updated.", field.Name) + } + if field.Type.Kind() != wantKind { + t.Errorf("unexpected cadvisorapiv1.PSIStats field %s type, want: %s, got: %s. The conversion between cadvisor PSIStats and kubelet stats API PSIStats needs to be re-evaluated and updated.", field.Name, wantKind, field.Type.Kind()) + } + } + +} diff --git a/pkg/kubelet/stats/provider_test.go b/pkg/kubelet/stats/provider_test.go index 0d52a1c468b..81f94c26734 100644 --- a/pkg/kubelet/stats/provider_test.go +++ b/pkg/kubelet/stats/provider_test.go @@ -32,7 +32,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1" + "k8s.io/kubernetes/pkg/features" cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing" kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing" @@ -63,6 +65,7 @@ const ( offsetFsInodeUsage offsetAcceleratorDutyCycle offsetMemSwapUsageBytes + offsetPSIDataTotal ) var ( @@ -265,6 +268,7 @@ func getTestContainerInfo(seed int, podName string, podNamespace string, contain CreationTime: testTime(creationTime, seed), HasCpu: true, HasMemory: true, + HasDiskIo: true, HasNetwork: true, Labels: labels, Memory: cadvisorapiv2.MemorySpec{ @@ -280,8 +284,10 @@ func getTestContainerInfo(seed int, podName string, podNamespace string, contain stats := cadvisorapiv2.ContainerStats{ Timestamp: testTime(timestamp, seed), - Cpu: &cadvisorapiv1.CpuStats{}, - CpuInst: &cadvisorapiv2.CpuInstStats{}, + Cpu: &cadvisorapiv1.CpuStats{ + PSI: getTestPSIStats(seed), + }, + CpuInst: &cadvisorapiv2.CpuInstStats{}, Memory: &cadvisorapiv1.MemoryStats{ Usage: uint64(seed + offsetMemUsageBytes), WorkingSet: uint64(seed + offsetMemWorkingSetBytes), @@ -291,6 +297,7 @@ func getTestContainerInfo(seed int, podName string, podNamespace string, contain Pgmajfault: uint64(seed + offsetMemMajorPageFaults), }, Swap: uint64(seed + offsetMemSwapUsageBytes), + PSI: getTestPSIStats(seed), }, Network: &cadvisorapiv2.NetworkStats{ Interfaces: []cadvisorapiv1.InterfaceStats{{ @@ -323,6 +330,9 @@ func getTestContainerInfo(seed int, podName string, podNamespace string, contain DutyCycle: uint64(seed + offsetAcceleratorDutyCycle), }, }, + DiskIo: &cadvisorapiv1.DiskIoStats{ + PSI: getTestPSIStats(seed), + }, } stats.Cpu.Usage.Total = uint64(seed + offsetCPUUsageCoreSeconds) stats.CpuInst.Usage.Total = uint64(seed + offsetCPUUsageCores) @@ -332,6 +342,22 @@ func getTestContainerInfo(seed int, podName string, podNamespace string, contain } } +func getTestPSIStats(seed int) cadvisorapiv1.PSIStats { + return cadvisorapiv1.PSIStats{ + Full: getTestPSIData(seed), + Some: getTestPSIData(seed), + } +} + +func getTestPSIData(seed int) cadvisorapiv1.PSIData { + return cadvisorapiv1.PSIData{ + Total: uint64(seed + offsetPSIDataTotal), + Avg10: float64(10), + Avg60: float64(10), + Avg300: float64(10), + } +} + func getTestFsInfo(seed int) cadvisorapiv2.FsInfo { var ( inodes = uint64(seed + offsetFsInodes) @@ -469,6 +495,7 @@ func checkCPUStats(t *testing.T, label string, seed int, stats *statsapi.CPUStat assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".CPU.Time") assert.EqualValues(t, seed+offsetCPUUsageCores, *stats.UsageNanoCores, label+".CPU.UsageCores") assert.EqualValues(t, seed+offsetCPUUsageCoreSeconds, *stats.UsageCoreNanoSeconds, label+".CPU.UsageCoreSeconds") + checkPSIStats(t, label+".CPU", seed, stats.PSI) } func checkMemoryStats(t *testing.T, label string, seed int, info cadvisorapiv2.ContainerInfo, stats *statsapi.MemoryStats) { @@ -484,6 +511,28 @@ func checkMemoryStats(t *testing.T, label string, seed int, info cadvisorapiv2.C expected := info.Spec.Memory.Limit - *stats.WorkingSetBytes assert.EqualValues(t, expected, *stats.AvailableBytes, label+".Mem.AvailableBytes") } + checkPSIStats(t, label+".Mem", seed, stats.PSI) +} + +func checkIOStats(t *testing.T, label string, seed int, info cadvisorapiv2.ContainerInfo, stats *statsapi.IOStats) { + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) { + assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".Mem.Time") + checkPSIStats(t, label+".IO", seed, stats.PSI) + } +} + +func checkPSIStats(t *testing.T, label string, seed int, stats *statsapi.PSIStats) { + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) { + require.NotNil(t, stats, label+".PSI") + assert.EqualValues(t, seed+offsetPSIDataTotal, stats.Full.Total, label+".PSI.Full.Total") + assert.InDelta(t, 10, stats.Full.Avg10, 0.01, label+".PSI.Full.Avg10") + assert.InDelta(t, 10, stats.Full.Avg60, 0.01, label+".PSI.Full.Avg60") + assert.InDelta(t, 10, stats.Full.Avg300, 0.01, label+".PSI.Full.Avg300") + assert.EqualValues(t, seed+offsetPSIDataTotal, stats.Some.Total, label+".PSI.Some.Total") + assert.InDelta(t, 10, stats.Some.Avg10, 0.01, label+".PSI.Some.Avg10") + assert.InDelta(t, 10, stats.Some.Avg60, 0.01, label+".PSI.Some.Avg60") + assert.InDelta(t, 10, stats.Some.Avg300, 0.01, label+".PSI.Some.Avg300") + } } func checkSwapStats(t *testing.T, label string, seed int, info cadvisorapiv2.ContainerInfo, stats *statsapi.SwapStats) { diff --git a/test/e2e_node/summary_test.go b/test/e2e_node/summary_test.go index 77c2fa394a6..d36a662751d 100644 --- a/test/e2e_node/summary_test.go +++ b/test/e2e_node/summary_test.go @@ -19,6 +19,7 @@ package e2enode import ( "context" "fmt" + "math" "os" "strings" "time" @@ -26,7 +27,9 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" kubeletstatsv1alpha1 "k8s.io/kubelet/pkg/apis/stats/v1alpha1" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/test/e2e/framework" e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -100,6 +103,7 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() { // for more information. "UsageNanoCores": gomega.SatisfyAny(gstruct.PointTo(gomega.BeZero()), bounded(10000, 2e9)), "UsageCoreNanoSeconds": bounded(10000000, 1e15), + "PSI": psiExpectation(), }), "Memory": ptrMatchAllFields(gstruct.Fields{ "Time": recent(maxStatsAge), @@ -111,7 +115,9 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() { "RSSBytes": bounded(1*e2evolume.Mb, memoryLimit), "PageFaults": bounded(1000, 1e9), "MajorPageFaults": bounded(0, 1e9), + "PSI": psiExpectation(), }), + "IO": ioExpectation(maxStatsAge), "Swap": swapExpectation(memoryLimit), "Accelerators": gomega.BeEmpty(), "Rootfs": gomega.BeNil(), @@ -138,6 +144,7 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() { "RSSBytes": bounded(1*e2evolume.Kb, memoryLimit), "PageFaults": bounded(0, expectedPageFaultsUpperBound), "MajorPageFaults": bounded(0, expectedMajorPageFaultsUpperBound), + "PSI": psiExpectation(), }) runtimeContExpectations := sysContExpectations().(*gstruct.FieldsMatcher) systemContainers := gstruct.Elements{ @@ -159,6 +166,7 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() { "RSSBytes": bounded(100*e2evolume.Kb, memoryLimit), "PageFaults": bounded(1000, 1e9), "MajorPageFaults": bounded(0, 1e9), + "PSI": psiExpectation(), }) systemContainers["misc"] = miscContExpectations } @@ -174,6 +182,7 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() { "Time": recent(maxStatsAge), "UsageNanoCores": bounded(10000, 1e9), "UsageCoreNanoSeconds": bounded(10000000, 1e11), + "PSI": psiExpectation(), }), "Memory": ptrMatchAllFields(gstruct.Fields{ "Time": recent(maxStatsAge), @@ -183,7 +192,9 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() { "RSSBytes": bounded(1*e2evolume.Kb, 80*e2evolume.Mb), "PageFaults": bounded(100, expectedPageFaultsUpperBound), "MajorPageFaults": bounded(0, expectedMajorPageFaultsUpperBound), + "PSI": psiExpectation(), }), + "IO": ioExpectation(maxStatsAge), "Swap": swapExpectation(memoryLimit), "Accelerators": gomega.BeEmpty(), "Rootfs": ptrMatchAllFields(gstruct.Fields{ @@ -222,6 +233,7 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() { "Time": recent(maxStatsAge), "UsageNanoCores": bounded(10000, 1e9), "UsageCoreNanoSeconds": bounded(10000000, 1e11), + "PSI": psiExpectation(), }), "Memory": ptrMatchAllFields(gstruct.Fields{ "Time": recent(maxStatsAge), @@ -231,7 +243,9 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() { "RSSBytes": bounded(1*e2evolume.Kb, 80*e2evolume.Mb), "PageFaults": bounded(0, expectedPageFaultsUpperBound), "MajorPageFaults": bounded(0, expectedMajorPageFaultsUpperBound), + "PSI": psiExpectation(), }), + "IO": ioExpectation(maxStatsAge), "Swap": swapExpectation(memoryLimit), "VolumeStats": gstruct.MatchAllElements(summaryObjectID, gstruct.Elements{ "test-empty-dir": gstruct.MatchAllFields(gstruct.Fields{ @@ -272,6 +286,7 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() { "Time": recent(maxStatsAge), "UsageNanoCores": bounded(100e3, 2e9), "UsageCoreNanoSeconds": bounded(1e9, 1e15), + "PSI": psiExpectation(), }), "Memory": ptrMatchAllFields(gstruct.Fields{ "Time": recent(maxStatsAge), @@ -282,7 +297,9 @@ var _ = SIGDescribe("Summary API", framework.WithNodeConformance(), func() { "RSSBytes": bounded(1*e2evolume.Kb, memoryLimit), "PageFaults": bounded(1000, 1e9), "MajorPageFaults": bounded(0, 1e9), + "PSI": psiExpectation(), }), + "IO": ioExpectation(maxStatsAge), "Swap": swapExpectation(memoryLimit), // TODO(#28407): Handle non-eth0 network interface names. "Network": ptrMatchAllFields(gstruct.Fields{ @@ -419,9 +436,13 @@ func ptrMatchAllFields(fields gstruct.Fields) types.GomegaMatcher { } func bounded(lower, upper interface{}) types.GomegaMatcher { - return gstruct.PointTo(gomega.And( + return gstruct.PointTo(boundedValue(lower, upper)) +} + +func boundedValue(lower, upper interface{}) types.GomegaMatcher { + return gomega.And( gomega.BeNumerically(">=", lower), - gomega.BeNumerically("<=", upper))) + gomega.BeNumerically("<=", upper)) } func swapExpectation(upper interface{}) types.GomegaMatcher { @@ -492,3 +513,30 @@ func recordSystemCgroupProcesses(ctx context.Context) { } } } + +func psiExpectation() types.GomegaMatcher { + if !utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) { + return gomega.BeNil() + } + psiDataExpectation := gstruct.MatchAllFields(gstruct.Fields{ + "Total": boundedValue(0, uint64(math.MaxUint64)), + "Avg10": boundedValue(0, 100), + "Avg60": boundedValue(0, 100), + "Avg300": boundedValue(0, 100), + }) + return ptrMatchAllFields(gstruct.Fields{ + "Full": psiDataExpectation, + "Some": psiDataExpectation, + }) +} + +func ioExpectation(maxStatsAge time.Duration) types.GomegaMatcher { + if !utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) { + return gomega.BeNil() + } + return gomega.Or(gomega.BeNil(), + ptrMatchAllFields(gstruct.Fields{ + "Time": recent(maxStatsAge), + "PSI": psiExpectation(), + })) +}