From 4db8e8cc1dc2e5683c878b3ef29cb2e0fbe70f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Marc=20Fran=C3=A7ois?= Date: Fri, 21 Mar 2025 17:42:25 -0400 Subject: [PATCH] Add configurable tolerance e2e test. --- .../horizontal_pod_autoscaling_behavior.go | 107 +++++++++++++----- test/e2e/feature/feature.go | 4 + .../autoscaling/autoscaling_utils.go | 7 ++ 3 files changed, 90 insertions(+), 28 deletions(-) diff --git a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go index d457af524c2..88ab6fa5207 100644 --- a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go +++ b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go @@ -21,6 +21,7 @@ import ( "time" autoscalingv2 "k8s.io/api/autoscaling/v2" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2eautoscaling "k8s.io/kubernetes/test/e2e/framework/autoscaling" @@ -30,38 +31,30 @@ import ( "github.com/onsi/gomega" ) +const ( + hpaName = "consumer" + + podCPURequest = 500 + targetCPUUtilizationPercent = 25 + + fullWindowOfNewUsage = 30 * time.Second + windowWithOldUsagePasses = 30 * time.Second + newPodMetricsDelay = 15 * time.Second + metricsAvailableDelay = fullWindowOfNewUsage + windowWithOldUsagePasses + newPodMetricsDelay + + hpaReconciliationInterval = 15 * time.Second + actuationDelay = 10 * time.Second + maxHPAReactionTime = metricsAvailableDelay + hpaReconciliationInterval + actuationDelay + + maxConsumeCPUDelay = 30 * time.Second + waitForReplicasPollInterval = 20 * time.Second + maxResourceConsumerDelay = maxConsumeCPUDelay + waitForReplicasPollInterval +) + var _ = SIGDescribe(feature.HPA, framework.WithSerial(), framework.WithSlow(), "Horizontal pod autoscaling (non-default behavior)", func() { f := framework.NewDefaultFramework("horizontal-pod-autoscaling") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged - hpaName := "consumer" - - podCPURequest := 500 - targetCPUUtilizationPercent := 25 - - // usageForReplicas returns usage for (n - 0.5) replicas as if they would consume all CPU - // under the target. The 0.5 replica reduction is to accommodate for the deviation between - // the actual consumed cpu and requested usage by the ResourceConsumer. - // HPA rounds up the recommendations. So, if the usage is e.g. for 3.5 replicas, - // the recommended replica number will be 4. - usageForReplicas := func(replicas int) int { - usagePerReplica := podCPURequest * targetCPUUtilizationPercent / 100 - return replicas*usagePerReplica - usagePerReplica/2 - } - - fullWindowOfNewUsage := 30 * time.Second - windowWithOldUsagePasses := 30 * time.Second - newPodMetricsDelay := 15 * time.Second - metricsAvailableDelay := fullWindowOfNewUsage + windowWithOldUsagePasses + newPodMetricsDelay - - hpaReconciliationInterval := 15 * time.Second - actuationDelay := 10 * time.Second - maxHPAReactionTime := metricsAvailableDelay + hpaReconciliationInterval + actuationDelay - - maxConsumeCPUDelay := 30 * time.Second - waitForReplicasPollInterval := 20 * time.Second - maxResourceConsumerDelay := maxConsumeCPUDelay + waitForReplicasPollInterval - waitBuffer := 1 * time.Minute ginkgo.Describe("with short downscale stabilization window", func() { @@ -505,3 +498,61 @@ var _ = SIGDescribe(feature.HPA, framework.WithSerial(), framework.WithSlow(), " }) }) }) + +var _ = SIGDescribe(feature.HPAConfigurableTolerance, framework.WithFeatureGate(features.HPAConfigurableTolerance), + framework.WithSerial(), framework.WithSlow(), "Horizontal pod autoscaling (configurable tolerance)", func() { + f := framework.NewDefaultFramework("horizontal-pod-autoscaling") + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + + waitBuffer := 1 * time.Minute + + ginkgo.Describe("with large configurable tolerance", func() { + ginkgo.It("should not scale", func(ctx context.Context) { + ginkgo.By("setting up resource consumer and HPA") + initPods := 1 + initCPUUsageTotal := usageForReplicas(initPods) + + rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, + hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, + initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, + f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, + ) + ginkgo.DeferCleanup(rc.CleanUp) + + scaleRule := e2eautoscaling.HPAScalingRuleWithToleranceMilli(10000) + hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, + rc, int32(targetCPUUtilizationPercent), 1, 10, + e2eautoscaling.HPABehaviorWithScaleUpAndDownRules(scaleRule, scaleRule), + ) + ginkgo.DeferCleanup(e2eautoscaling.DeleteHPAWithBehavior, rc, hpa.Name) + + waitDeadline := maxHPAReactionTime + maxResourceConsumerDelay + waitBuffer + + ginkgo.By("trying to trigger scale up") + rc.ConsumeCPU(usageForReplicas(8)) + waitStart := time.Now() + + rc.EnsureDesiredReplicasInRange(ctx, initPods, initPods, waitDeadline, hpa.Name) + timeWaited := time.Since(waitStart) + + ginkgo.By("verifying time waited for a scale up") + framework.Logf("time waited for scale up: %s", timeWaited) + gomega.Expect(timeWaited).To(gomega.BeNumerically(">", waitDeadline), "waited %s, wanted to wait more than %s", timeWaited, waitDeadline) + + ginkgo.By("verifying number of replicas") + replicas, err := rc.GetReplicas(ctx) + framework.ExpectNoError(err) + gomega.Expect(replicas).To(gomega.BeNumerically("==", initPods), "had %s replicas, still have %s replicas after time deadline", initPods, replicas) + }) + }) + }) + +// usageForReplicas returns usage for (n - 0.5) replicas as if they would consume all CPU +// under the target. The 0.5 replica reduction is to accommodate for the deviation between +// the actual consumed cpu and requested usage by the ResourceConsumer. +// HPA rounds up the recommendations. So, if the usage is e.g. for 3.5 replicas, +// the recommended replica number will be 4. +func usageForReplicas(replicas int) int { + usagePerReplica := podCPURequest * targetCPUUtilizationPercent / 100 + return replicas*usagePerReplica - usagePerReplica/2 +} diff --git a/test/e2e/feature/feature.go b/test/e2e/feature/feature.go index b785aeee439..9baa3227f60 100644 --- a/test/e2e/feature/feature.go +++ b/test/e2e/feature/feature.go @@ -219,6 +219,10 @@ var ( // TODO: document the feature (owning SIG, when to use this feature for a test) HPA = framework.WithFeature(framework.ValidFeatures.Add("HPA")) + // OWNER: sig-autoscaling + // Marks tests that require HPA configurable tolerance (https://kep.k8s.io/4951). + HPAConfigurableTolerance = framework.WithFeature(framework.ValidFeatures.Add("HPAConfigurableTolerance")) + // owner: sig-node HostAccess = framework.WithFeature(framework.ValidFeatures.Add("HostAccess")) diff --git a/test/e2e/framework/autoscaling/autoscaling_utils.go b/test/e2e/framework/autoscaling/autoscaling_utils.go index b90965ebbcc..d015275f71f 100644 --- a/test/e2e/framework/autoscaling/autoscaling_utils.go +++ b/test/e2e/framework/autoscaling/autoscaling_utils.go @@ -880,6 +880,13 @@ func HPAScalingRuleWithScalingPolicy(policyType autoscalingv2.HPAScalingPolicyTy } } +func HPAScalingRuleWithToleranceMilli(toleranceMilli int64) *autoscalingv2.HPAScalingRules { + quantity := resource.NewMilliQuantity(toleranceMilli, resource.DecimalSI) + return &autoscalingv2.HPAScalingRules{ + Tolerance: quantity, + } +} + func HPABehaviorWithStabilizationWindows(upscaleStabilization, downscaleStabilization time.Duration) *autoscalingv2.HorizontalPodAutoscalerBehavior { scaleUpRule := HPAScalingRuleWithStabilizationWindow(int32(upscaleStabilization.Seconds())) scaleDownRule := HPAScalingRuleWithStabilizationWindow(int32(downscaleStabilization.Seconds()))