github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-07-23 02:00:12 +00:00
Code Issues Projects Releases Wiki Activity

record 'kubelet_container_requested_resizes_total' metric

Browse Source
This commit is contained in:
Natasha Sarkar
2025-07-11 21:07:02 +00:00
parent 468aa5e30c
commit 0595ddaafa
3 changed files with 712 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
pkg/kubelet/allocation/allocation_manager.go
View File

@@ -105,7 +105,8 @@ type Manager interface {
Run(ctx context.Context)
// PushPendingResize queues a pod with a pending resize request for later reevaluation.
PushPendingResize(uid types.UID)
// Returns true if the pending resize was added to the queue, false if it was already present.
PushPendingResize(uid types.UID) bool
// RetryPendingResizes retries all pending resizes. It returns a list of successful resizes.
RetryPendingResizes() []*v1.Pod
@@ -311,20 +312,21 @@ func (m *manager) RetryPendingResizes() []*v1.Pod {
}
func (m *manager) PushPendingResize(uid types.UID) {
func (m *manager) PushPendingResize(uid types.UID) bool {
m.allocationMutex.Lock()
defer m.allocationMutex.Unlock()
for _, p := range m.podsWithPendingResizes {
if p == uid {
// Pod is already in the pending resizes queue.
return
return false
}
}
// Add the pod to the pending resizes list and sort by priority.
m.podsWithPendingResizes = append(m.podsWithPendingResizes, uid)
m.sortPendingResizes()
return true
}
// sortPendingResizes sorts the list of pending resizes:
@@ -772,7 +774,7 @@ func (m *manager) CheckPodResizeInProgress(allocatedPod *v1.Pod, podStatus *kube
func (m *manager) isPodResizeInProgress(allocatedPod *v1.Pod, podStatus *kubecontainer.PodStatus) bool {
return !podutil.VisitContainers(&allocatedPod.Spec, podutil.InitContainers|podutil.Containers,
func(allocatedContainer *v1.Container, containerType podutil.ContainerType) (shouldContinue bool) {
if !isResizableContainer(allocatedContainer, containerType) {
if !IsResizableContainer(allocatedContainer, containerType) {
return true
}
@@ -804,7 +806,7 @@ func (m *manager) getAllocatedPods(activePods []*v1.Pod) []*v1.Pod {
return allocatedPods
}
func isResizableContainer(container *v1.Container, containerType podutil.ContainerType) bool {
func IsResizableContainer(container *v1.Container, containerType podutil.ContainerType) bool {
switch containerType {
case podutil.InitContainers:
return podutil.IsRestartableInitContainer(container)

66
pkg/kubelet/kubelet.go
View File

@@ -51,6 +51,7 @@ import (
"k8s.io/utils/ptr"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
@@ -2674,9 +2675,19 @@ func (kl *Kubelet) HandlePodUpdates(pods []*v1.Pod) {
}
if utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling) {
_, updatedFromAllocation := kl.allocationManager.UpdatePodFromAllocation(pod)
allocatedPod, updatedFromAllocation := kl.allocationManager.UpdatePodFromAllocation(pod)
if updatedFromAllocation {
kl.allocationManager.PushPendingResize(pod.UID)
if kl.allocationManager.PushPendingResize(pod.UID) {
// We only record the metric if the resize is not already in the pending queue.
// This will cause us to miss incrementing the counter when a previously
// pending resize is overwritten with a new resize, but will prevent us from
// erroneously incrementing the counter when there is a non-resize related update
// of a pod that currently has a pending resize.
// TODO (natasha41575): See if there is an easy way to detect when a pending resize is
// being overwritten with a new one.
recordContainerResizeOperations(allocatedPod, pod)
}
// TODO (natasha41575): If the resize is immediately actuated, it will trigger a pod sync
// and we will end up calling UpdatePod twice. Figure out if there is a way to avoid this.
kl.allocationManager.RetryPendingResizes()
@@ -2696,6 +2707,57 @@ func (kl *Kubelet) HandlePodUpdates(pods []*v1.Pod) {
}
}
// recordContainerResizeOperations records if any of the pod's containers needs to be resized.
func recordContainerResizeOperations(oldPod, newPod *v1.Pod) {
for oldContainer, containerType := range podutil.ContainerIter(&oldPod.Spec, podutil.InitContainers|podutil.Containers) {
if !allocation.IsResizableContainer(oldContainer, containerType) {
continue
}
var newContainer *v1.Container
for new, newType := range podutil.ContainerIter(&newPod.Spec, podutil.InitContainers|podutil.Containers) {
if !allocation.IsResizableContainer(new, newType) {
continue
}
if new.Name == oldContainer.Name && containerType == newType {
newContainer = new
}
}
newResources := newContainer.Resources
oldResources := oldContainer.Resources
if op := resizeOperationForResources(newResources.Requests.Memory(), oldResources.Requests.Memory()); op != "" {
metrics.ContainerRequestedResizes.WithLabelValues("memory", "requests", op).Inc()
}
if op := resizeOperationForResources(newResources.Limits.Memory(), oldResources.Limits.Memory()); op != "" {
metrics.ContainerRequestedResizes.WithLabelValues("memory", "limits", op).Inc()
}
if op := resizeOperationForResources(newResources.Requests.Cpu(), oldResources.Requests.Cpu()); op != "" {
metrics.ContainerRequestedResizes.WithLabelValues("cpu", "requests", op).Inc()
}
if op := resizeOperationForResources(newResources.Limits.Cpu(), oldResources.Limits.Cpu()); op != "" {
metrics.ContainerRequestedResizes.WithLabelValues("cpu", "limits", op).Inc()
}
}
}
func resizeOperationForResources(new, old *resource.Quantity) string {
if new.IsZero() && !old.IsZero() {
return "remove"
}
if old.IsZero() && !new.IsZero() {
return "add"
}
if new.Cmp(*old) < 0 {
return "decrease"
}
if new.Cmp(*old) > 0 {
return "increase"
}
return ""
}
// HandlePodRemoves is the callback in the SyncHandler interface for pods
// being removed from a config source.
func (kl *Kubelet) HandlePodRemoves(pods []*v1.Pod) {

641
pkg/kubelet/kubelet_test.go
View File

@@ -24,6 +24,7 @@ import (
"os"
"path/filepath"
"reflect"
"regexp"
goruntime "runtime"
"sort"
"strconv"
@@ -35,6 +36,7 @@ import (
"go.opentelemetry.io/otel/sdk/trace/tracetest"
oteltrace "go.opentelemetry.io/otel/trace"
noopoteltrace "go.opentelemetry.io/otel/trace/noop"
"k8s.io/component-base/metrics/legacyregistry"
cadvisorapi "github.com/google/cadvisor/info/v1"
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
@@ -3583,3 +3585,642 @@ func TestSyncPodWithErrorsDuringInPlacePodResize(t *testing.T) {
})
}
}
func TestHandlePodUpdates_RecordContainerRequestedResizes(t *testing.T) {
metrics.Register()
metrics.ContainerRequestedResizes.Reset()
type expectedMetricsStruct struct {
memoryLimitsCounter map[string]int
memoryRequestsCounter map[string]int
cpuLimitsCounter map[string]int
cpuRequestsCounter map[string]int
}
expectedMetrics := expectedMetricsStruct{
memoryLimitsCounter: make(map[string]int),
memoryRequestsCounter: make(map[string]int),
cpuLimitsCounter: make(map[string]int),
cpuRequestsCounter: make(map[string]int),
}
for _, tc := range []struct {
name string
initialAllocation *v1.Pod
updatedPod *v1.Pod
updateExpectedFunc func(*expectedMetricsStruct)
}{
// Memory requests
{
name: "add memory requests",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{Containers: []v1.Container{{}}},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("90"),
},
},
}},
},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.memoryRequestsCounter["add"]++
},
},
{
name: "remove memory requests",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("90"),
},
},
}},
},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{Containers: []v1.Container{{}}},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.memoryRequestsCounter["remove"]++
},
},
{
name: "increase memory requests",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("90"),
},
}},
},
},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("110"),
},
},
}},
},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.memoryRequestsCounter["increase"]++
},
},
{
name: "decrease memory requests",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("110"),
},
},
}},
},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("90"),
},
},
}},
},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.memoryRequestsCounter["decrease"]++
},
},
// Memory limits
{
name: "add memory limits",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{Containers: []v1.Container{{}}},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("90"),
},
},
}},
},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.memoryLimitsCounter["add"]++
},
},
{
name: "remove memory limits",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("90"),
},
},
}},
},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{Containers: []v1.Container{{}}},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.memoryLimitsCounter["remove"]++
},
},
{
name: "increase memory limits",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("90"),
},
},
}},
},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("110"),
},
},
}},
},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.memoryLimitsCounter["increase"]++
},
},
{
name: "decrease memory limits",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("110"),
},
},
}},
},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("90"),
},
},
}},
},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.memoryLimitsCounter["decrease"]++
},
},
// CPU requests
{
name: "add cpu requests",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{Containers: []v1.Container{{}}},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("90"),
},
},
}},
},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.cpuRequestsCounter["add"]++
},
},
{
name: "remove cpu requests",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("90"),
},
},
}},
},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{Containers: []v1.Container{{}}},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.cpuRequestsCounter["remove"]++
},
},
{
name: "increase cpu requests",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("90"),
},
},
}},
},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("110"),
},
},
}},
},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.cpuRequestsCounter["increase"]++
},
},
{
name: "decrease cpu requests",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("110"),
},
},
}},
},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("90"),
},
},
}},
},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.cpuRequestsCounter["decrease"]++
},
},
// CPU limits
{
name: "add cpu limits",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{Containers: []v1.Container{{}}},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("90"),
},
},
}},
},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.cpuLimitsCounter["add"]++
},
},
{
name: "remove cpu limits",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("90"),
},
},
}},
},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{Containers: []v1.Container{{}}},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.cpuLimitsCounter["remove"]++
},
},
{
name: "increase cpu limits",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("90"),
},
},
}},
},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("110"),
},
},
}},
},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.cpuLimitsCounter["increase"]++
},
},
{
name: "decrease cpu limits",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("110"),
},
},
}},
},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("90"),
},
},
}},
},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.cpuLimitsCounter["decrease"]++
},
},
// Some combinations of things
{
name: "add cpu limits + increase memory requests",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("90"),
},
},
}},
},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("110"),
},
Requests: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("110"),
},
},
}},
},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.cpuLimitsCounter["add"]++
e.memoryRequestsCounter["increase"]++
},
},
{
name: "remove memory limits + decrease cpu requests",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("90"),
},
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("110"),
},
},
}},
},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("90"),
},
},
}},
},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.memoryLimitsCounter["remove"]++
e.cpuRequestsCounter["decrease"]++
},
},
{
name: "increase cpu requests + memory requests",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("90"),
v1.ResourceMemory: resource.MustParse("90"),
},
},
}},
},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("110"),
v1.ResourceMemory: resource.MustParse("110"),
},
},
}},
},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.cpuRequestsCounter["increase"]++
e.memoryRequestsCounter["increase"]++
},
},
{
name: "decrease all possible values",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("110"),
v1.ResourceMemory: resource.MustParse("110"),
},
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("110"),
v1.ResourceMemory: resource.MustParse("110"),
},
},
}},
},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("90"),
v1.ResourceMemory: resource.MustParse("90"),
},
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("90"),
v1.ResourceMemory: resource.MustParse("90"),
},
},
}},
},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {
e.cpuLimitsCounter["decrease"]++
e.cpuRequestsCounter["decrease"]++
e.memoryLimitsCounter["decrease"]++
e.memoryRequestsCounter["decrease"]++
},
},
{
name: "no resize request",
initialAllocation: &v1.Pod{
Spec: v1.PodSpec{Containers: []v1.Container{{}}},
},
updatedPod: &v1.Pod{
Spec: v1.PodSpec{Containers: []v1.Container{{}}},
},
updateExpectedFunc: func(e *expectedMetricsStruct) {},
},
} {
t.Run(tc.name, func(t *testing.T) {
testPod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
UID: "12345",
},
}
initialPod := testPod.DeepCopy()
updatedPod := testPod.DeepCopy()
initialPod.Spec = tc.initialAllocation.Spec
updatedPod.Spec = tc.updatedPod.Spec
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
t.Cleanup(func() { testKubelet.Cleanup() })
kubelet := testKubelet.kubelet
require.NoError(t, kubelet.allocationManager.SetAllocatedResources(initialPod))
kubelet.HandlePodUpdates([]*v1.Pod{updatedPod})
tc.updateExpectedFunc(&expectedMetrics)
expectedFormat := `
# HELP kubelet_container_requested_resizes_total [ALPHA] Number of requested resizes, counted at the container level. Different resources on the same container are counted separately. The 'requirement' label refers to 'memory' or 'limits'; the 'operation' label can be one of 'add', 'remove', 'increase' or 'decrease'.
# TYPE kubelet_container_requested_resizes_total counter
kubelet_container_requested_resizes_total{operation="add",requirement="requests",resource="memory"} %d
kubelet_container_requested_resizes_total{operation="add",requirement="requests",resource="cpu"} %d
kubelet_container_requested_resizes_total{operation="add",requirement="limits",resource="memory"} %d
kubelet_container_requested_resizes_total{operation="add",requirement="limits",resource="cpu"} %d
kubelet_container_requested_resizes_total{operation="decrease",requirement="requests",resource="memory"} %d
kubelet_container_requested_resizes_total{operation="decrease",requirement="requests",resource="cpu"} %d
kubelet_container_requested_resizes_total{operation="decrease",requirement="limits",resource="memory"} %d
kubelet_container_requested_resizes_total{operation="decrease",requirement="limits",resource="cpu"} %d
kubelet_container_requested_resizes_total{operation="increase",requirement="requests",resource="memory"} %d
kubelet_container_requested_resizes_total{operation="increase",requirement="requests",resource="cpu"} %d
kubelet_container_requested_resizes_total{operation="increase",requirement="limits",resource="memory"} %d
kubelet_container_requested_resizes_total{operation="increase",requirement="limits",resource="cpu"} %d
kubelet_container_requested_resizes_total{operation="remove",requirement="requests",resource="memory"} %d
kubelet_container_requested_resizes_total{operation="remove",requirement="requests",resource="cpu"} %d
kubelet_container_requested_resizes_total{operation="remove",requirement="limits",resource="memory"} %d
kubelet_container_requested_resizes_total{operation="remove",requirement="limits",resource="cpu"} %d
`
expected := fmt.Sprintf(expectedFormat,
expectedMetrics.memoryRequestsCounter["add"],
expectedMetrics.cpuRequestsCounter["add"],
expectedMetrics.memoryLimitsCounter["add"],
expectedMetrics.cpuLimitsCounter["add"],
expectedMetrics.memoryRequestsCounter["decrease"],
expectedMetrics.cpuRequestsCounter["decrease"],
expectedMetrics.memoryLimitsCounter["decrease"],
expectedMetrics.cpuLimitsCounter["decrease"],
expectedMetrics.memoryRequestsCounter["increase"],
expectedMetrics.cpuRequestsCounter["increase"],
expectedMetrics.memoryLimitsCounter["increase"],
expectedMetrics.cpuLimitsCounter["increase"],
expectedMetrics.memoryRequestsCounter["remove"],
expectedMetrics.cpuRequestsCounter["remove"],
expectedMetrics.memoryLimitsCounter["remove"],
expectedMetrics.cpuLimitsCounter["remove"],
)
// Omit lines from the expected metrics string where the count is "0"
re := regexp.MustCompile("(?m)[\r\n]+^.*} 0.*$")
expected = re.ReplaceAllString(expected, "")
require.NoError(t, testutil.GatherAndCompare(
legacyregistry.DefaultGatherer, strings.NewReader(expected), "kubelet_container_requested_resizes_total",
))
})
}
}