diff --git a/pkg/kubelet/allocation/allocation_manager.go b/pkg/kubelet/allocation/allocation_manager.go index 96910d7f7a0..f8ad19bcd0a 100644 --- a/pkg/kubelet/allocation/allocation_manager.go +++ b/pkg/kubelet/allocation/allocation_manager.go @@ -105,7 +105,8 @@ type Manager interface { Run(ctx context.Context) // PushPendingResize queues a pod with a pending resize request for later reevaluation. - PushPendingResize(uid types.UID) + // Returns true if the pending resize was added to the queue, false if it was already present. + PushPendingResize(uid types.UID) bool // RetryPendingResizes retries all pending resizes. It returns a list of successful resizes. RetryPendingResizes() []*v1.Pod @@ -311,20 +312,21 @@ func (m *manager) RetryPendingResizes() []*v1.Pod { } -func (m *manager) PushPendingResize(uid types.UID) { +func (m *manager) PushPendingResize(uid types.UID) bool { m.allocationMutex.Lock() defer m.allocationMutex.Unlock() for _, p := range m.podsWithPendingResizes { if p == uid { // Pod is already in the pending resizes queue. - return + return false } } // Add the pod to the pending resizes list and sort by priority. m.podsWithPendingResizes = append(m.podsWithPendingResizes, uid) m.sortPendingResizes() + return true } // sortPendingResizes sorts the list of pending resizes: @@ -772,7 +774,7 @@ func (m *manager) CheckPodResizeInProgress(allocatedPod *v1.Pod, podStatus *kube func (m *manager) isPodResizeInProgress(allocatedPod *v1.Pod, podStatus *kubecontainer.PodStatus) bool { return !podutil.VisitContainers(&allocatedPod.Spec, podutil.InitContainers|podutil.Containers, func(allocatedContainer *v1.Container, containerType podutil.ContainerType) (shouldContinue bool) { - if !isResizableContainer(allocatedContainer, containerType) { + if !IsResizableContainer(allocatedContainer, containerType) { return true } @@ -804,7 +806,7 @@ func (m *manager) getAllocatedPods(activePods []*v1.Pod) []*v1.Pod { return allocatedPods } -func isResizableContainer(container *v1.Container, containerType podutil.ContainerType) bool { +func IsResizableContainer(container *v1.Container, containerType podutil.ContainerType) bool { switch containerType { case podutil.InitContainers: return podutil.IsRestartableInitContainer(container) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 1eb28b2421f..8a5f964b1e0 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -51,6 +51,7 @@ import ( "k8s.io/utils/ptr" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" @@ -2674,9 +2675,19 @@ func (kl *Kubelet) HandlePodUpdates(pods []*v1.Pod) { } if utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling) { - _, updatedFromAllocation := kl.allocationManager.UpdatePodFromAllocation(pod) + allocatedPod, updatedFromAllocation := kl.allocationManager.UpdatePodFromAllocation(pod) if updatedFromAllocation { - kl.allocationManager.PushPendingResize(pod.UID) + if kl.allocationManager.PushPendingResize(pod.UID) { + // We only record the metric if the resize is not already in the pending queue. + // This will cause us to miss incrementing the counter when a previously + // pending resize is overwritten with a new resize, but will prevent us from + // erroneously incrementing the counter when there is a non-resize related update + // of a pod that currently has a pending resize. + // TODO (natasha41575): See if there is an easy way to detect when a pending resize is + // being overwritten with a new one. + recordContainerResizeOperations(allocatedPod, pod) + } + // TODO (natasha41575): If the resize is immediately actuated, it will trigger a pod sync // and we will end up calling UpdatePod twice. Figure out if there is a way to avoid this. kl.allocationManager.RetryPendingResizes() @@ -2696,6 +2707,57 @@ func (kl *Kubelet) HandlePodUpdates(pods []*v1.Pod) { } } +// recordContainerResizeOperations records if any of the pod's containers needs to be resized. +func recordContainerResizeOperations(oldPod, newPod *v1.Pod) { + for oldContainer, containerType := range podutil.ContainerIter(&oldPod.Spec, podutil.InitContainers|podutil.Containers) { + if !allocation.IsResizableContainer(oldContainer, containerType) { + continue + } + + var newContainer *v1.Container + for new, newType := range podutil.ContainerIter(&newPod.Spec, podutil.InitContainers|podutil.Containers) { + if !allocation.IsResizableContainer(new, newType) { + continue + } + if new.Name == oldContainer.Name && containerType == newType { + newContainer = new + } + } + + newResources := newContainer.Resources + oldResources := oldContainer.Resources + + if op := resizeOperationForResources(newResources.Requests.Memory(), oldResources.Requests.Memory()); op != "" { + metrics.ContainerRequestedResizes.WithLabelValues("memory", "requests", op).Inc() + } + if op := resizeOperationForResources(newResources.Limits.Memory(), oldResources.Limits.Memory()); op != "" { + metrics.ContainerRequestedResizes.WithLabelValues("memory", "limits", op).Inc() + } + if op := resizeOperationForResources(newResources.Requests.Cpu(), oldResources.Requests.Cpu()); op != "" { + metrics.ContainerRequestedResizes.WithLabelValues("cpu", "requests", op).Inc() + } + if op := resizeOperationForResources(newResources.Limits.Cpu(), oldResources.Limits.Cpu()); op != "" { + metrics.ContainerRequestedResizes.WithLabelValues("cpu", "limits", op).Inc() + } + } +} + +func resizeOperationForResources(new, old *resource.Quantity) string { + if new.IsZero() && !old.IsZero() { + return "remove" + } + if old.IsZero() && !new.IsZero() { + return "add" + } + if new.Cmp(*old) < 0 { + return "decrease" + } + if new.Cmp(*old) > 0 { + return "increase" + } + return "" +} + // HandlePodRemoves is the callback in the SyncHandler interface for pods // being removed from a config source. func (kl *Kubelet) HandlePodRemoves(pods []*v1.Pod) { diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 79d2de3ef0c..9a9fe986b6e 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -24,6 +24,7 @@ import ( "os" "path/filepath" "reflect" + "regexp" goruntime "runtime" "sort" "strconv" @@ -35,6 +36,7 @@ import ( "go.opentelemetry.io/otel/sdk/trace/tracetest" oteltrace "go.opentelemetry.io/otel/trace" noopoteltrace "go.opentelemetry.io/otel/trace/noop" + "k8s.io/component-base/metrics/legacyregistry" cadvisorapi "github.com/google/cadvisor/info/v1" cadvisorapiv2 "github.com/google/cadvisor/info/v2" @@ -3583,3 +3585,642 @@ func TestSyncPodWithErrorsDuringInPlacePodResize(t *testing.T) { }) } } + +func TestHandlePodUpdates_RecordContainerRequestedResizes(t *testing.T) { + metrics.Register() + metrics.ContainerRequestedResizes.Reset() + + type expectedMetricsStruct struct { + memoryLimitsCounter map[string]int + memoryRequestsCounter map[string]int + cpuLimitsCounter map[string]int + cpuRequestsCounter map[string]int + } + expectedMetrics := expectedMetricsStruct{ + memoryLimitsCounter: make(map[string]int), + memoryRequestsCounter: make(map[string]int), + cpuLimitsCounter: make(map[string]int), + cpuRequestsCounter: make(map[string]int), + } + + for _, tc := range []struct { + name string + initialAllocation *v1.Pod + updatedPod *v1.Pod + updateExpectedFunc func(*expectedMetricsStruct) + }{ + // Memory requests + { + name: "add memory requests", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{Containers: []v1.Container{{}}}, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.memoryRequestsCounter["add"]++ + }, + }, + { + name: "remove memory requests", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{Containers: []v1.Container{{}}}, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.memoryRequestsCounter["remove"]++ + }, + }, + { + name: "increase memory requests", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("90"), + }, + }}, + }, + }, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("110"), + }, + }, + }}, + }, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.memoryRequestsCounter["increase"]++ + }, + }, + { + name: "decrease memory requests", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("110"), + }, + }, + }}, + }, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.memoryRequestsCounter["decrease"]++ + }, + }, + + // Memory limits + { + name: "add memory limits", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{Containers: []v1.Container{{}}}, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.memoryLimitsCounter["add"]++ + }, + }, + { + name: "remove memory limits", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{Containers: []v1.Container{{}}}, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.memoryLimitsCounter["remove"]++ + }, + }, + { + name: "increase memory limits", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("110"), + }, + }, + }}, + }, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.memoryLimitsCounter["increase"]++ + }, + }, + { + name: "decrease memory limits", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("110"), + }, + }, + }}, + }, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.memoryLimitsCounter["decrease"]++ + }, + }, + + // CPU requests + { + name: "add cpu requests", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{Containers: []v1.Container{{}}}, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.cpuRequestsCounter["add"]++ + }, + }, + { + name: "remove cpu requests", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{Containers: []v1.Container{{}}}, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.cpuRequestsCounter["remove"]++ + }, + }, + { + name: "increase cpu requests", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("110"), + }, + }, + }}, + }, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.cpuRequestsCounter["increase"]++ + }, + }, + { + name: "decrease cpu requests", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("110"), + }, + }, + }}, + }, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.cpuRequestsCounter["decrease"]++ + }, + }, + + // CPU limits + { + name: "add cpu limits", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{Containers: []v1.Container{{}}}, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.cpuLimitsCounter["add"]++ + }, + }, + { + name: "remove cpu limits", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{Containers: []v1.Container{{}}}, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.cpuLimitsCounter["remove"]++ + }, + }, + { + name: "increase cpu limits", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("110"), + }, + }, + }}, + }, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.cpuLimitsCounter["increase"]++ + }, + }, + { + name: "decrease cpu limits", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("110"), + }, + }, + }}, + }, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.cpuLimitsCounter["decrease"]++ + }, + }, + + // Some combinations of things + { + name: "add cpu limits + increase memory requests", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("110"), + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("110"), + }, + }, + }}, + }, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.cpuLimitsCounter["add"]++ + e.memoryRequestsCounter["increase"]++ + }, + }, + { + name: "remove memory limits + decrease cpu requests", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("90"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("110"), + }, + }, + }}, + }, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.memoryLimitsCounter["remove"]++ + e.cpuRequestsCounter["decrease"]++ + }, + }, + { + name: "increase cpu requests + memory requests", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("90"), + v1.ResourceMemory: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("110"), + v1.ResourceMemory: resource.MustParse("110"), + }, + }, + }}, + }, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.cpuRequestsCounter["increase"]++ + e.memoryRequestsCounter["increase"]++ + }, + }, + { + name: "decrease all possible values", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("110"), + v1.ResourceMemory: resource.MustParse("110"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("110"), + v1.ResourceMemory: resource.MustParse("110"), + }, + }, + }}, + }, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("90"), + v1.ResourceMemory: resource.MustParse("90"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("90"), + v1.ResourceMemory: resource.MustParse("90"), + }, + }, + }}, + }, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) { + e.cpuLimitsCounter["decrease"]++ + e.cpuRequestsCounter["decrease"]++ + e.memoryLimitsCounter["decrease"]++ + e.memoryRequestsCounter["decrease"]++ + }, + }, + { + name: "no resize request", + initialAllocation: &v1.Pod{ + Spec: v1.PodSpec{Containers: []v1.Container{{}}}, + }, + updatedPod: &v1.Pod{ + Spec: v1.PodSpec{Containers: []v1.Container{{}}}, + }, + updateExpectedFunc: func(e *expectedMetricsStruct) {}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + testPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + UID: "12345", + }, + } + initialPod := testPod.DeepCopy() + updatedPod := testPod.DeepCopy() + + initialPod.Spec = tc.initialAllocation.Spec + updatedPod.Spec = tc.updatedPod.Spec + + testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) + t.Cleanup(func() { testKubelet.Cleanup() }) + kubelet := testKubelet.kubelet + + require.NoError(t, kubelet.allocationManager.SetAllocatedResources(initialPod)) + kubelet.HandlePodUpdates([]*v1.Pod{updatedPod}) + + tc.updateExpectedFunc(&expectedMetrics) + + expectedFormat := ` + # HELP kubelet_container_requested_resizes_total [ALPHA] Number of requested resizes, counted at the container level. Different resources on the same container are counted separately. The 'requirement' label refers to 'memory' or 'limits'; the 'operation' label can be one of 'add', 'remove', 'increase' or 'decrease'. + # TYPE kubelet_container_requested_resizes_total counter + kubelet_container_requested_resizes_total{operation="add",requirement="requests",resource="memory"} %d + kubelet_container_requested_resizes_total{operation="add",requirement="requests",resource="cpu"} %d + kubelet_container_requested_resizes_total{operation="add",requirement="limits",resource="memory"} %d + kubelet_container_requested_resizes_total{operation="add",requirement="limits",resource="cpu"} %d + kubelet_container_requested_resizes_total{operation="decrease",requirement="requests",resource="memory"} %d + kubelet_container_requested_resizes_total{operation="decrease",requirement="requests",resource="cpu"} %d + kubelet_container_requested_resizes_total{operation="decrease",requirement="limits",resource="memory"} %d + kubelet_container_requested_resizes_total{operation="decrease",requirement="limits",resource="cpu"} %d + kubelet_container_requested_resizes_total{operation="increase",requirement="requests",resource="memory"} %d + kubelet_container_requested_resizes_total{operation="increase",requirement="requests",resource="cpu"} %d + kubelet_container_requested_resizes_total{operation="increase",requirement="limits",resource="memory"} %d + kubelet_container_requested_resizes_total{operation="increase",requirement="limits",resource="cpu"} %d + kubelet_container_requested_resizes_total{operation="remove",requirement="requests",resource="memory"} %d + kubelet_container_requested_resizes_total{operation="remove",requirement="requests",resource="cpu"} %d + kubelet_container_requested_resizes_total{operation="remove",requirement="limits",resource="memory"} %d + kubelet_container_requested_resizes_total{operation="remove",requirement="limits",resource="cpu"} %d + ` + + expected := fmt.Sprintf(expectedFormat, + expectedMetrics.memoryRequestsCounter["add"], + expectedMetrics.cpuRequestsCounter["add"], + expectedMetrics.memoryLimitsCounter["add"], + expectedMetrics.cpuLimitsCounter["add"], + + expectedMetrics.memoryRequestsCounter["decrease"], + expectedMetrics.cpuRequestsCounter["decrease"], + expectedMetrics.memoryLimitsCounter["decrease"], + expectedMetrics.cpuLimitsCounter["decrease"], + + expectedMetrics.memoryRequestsCounter["increase"], + expectedMetrics.cpuRequestsCounter["increase"], + expectedMetrics.memoryLimitsCounter["increase"], + expectedMetrics.cpuLimitsCounter["increase"], + + expectedMetrics.memoryRequestsCounter["remove"], + expectedMetrics.cpuRequestsCounter["remove"], + expectedMetrics.memoryLimitsCounter["remove"], + expectedMetrics.cpuLimitsCounter["remove"], + ) + + // Omit lines from the expected metrics string where the count is "0" + re := regexp.MustCompile("(?m)[\r\n]+^.*} 0.*$") + expected = re.ReplaceAllString(expected, "") + + require.NoError(t, testutil.GatherAndCompare( + legacyregistry.DefaultGatherer, strings.NewReader(expected), "kubelet_container_requested_resizes_total", + )) + }) + } +}