From 1c455c10de8947f5508ad683a09a659f9b5c5063 Mon Sep 17 00:00:00 2001 From: Piotr Nosek Date: Tue, 19 Jul 2022 16:53:47 +0000 Subject: [PATCH] Add e2e HPA Behavior tests: scale down stabilisation, scale up disabled, scale down disabled. --- .../horizontal_pod_autoscaling_behavior.go | 160 ++++++++++++++++-- .../autoscaling/autoscaling_utils.go | 49 +++++- 2 files changed, 187 insertions(+), 22 deletions(-) diff --git a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go index 0cf9a6f2c6d..555453bd871 100644 --- a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go +++ b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go @@ -30,6 +30,21 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n f := framework.NewDefaultFramework("horizontal-pod-autoscaling") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + fullWindowOfNewUsage := 30 * time.Second + windowWithOldUsagePasses := 30 * time.Second + newPodMetricsDelay := 15 * time.Second + metricsAvailableDelay := fullWindowOfNewUsage + windowWithOldUsagePasses + newPodMetricsDelay + + hpaReconciliationInterval := 15 * time.Second + actuationDelay := 10 * time.Second + maxHPAReactionTime := metricsAvailableDelay + hpaReconciliationInterval + actuationDelay + + maxConsumeCPUDelay := 30 * time.Second + waitForReplicasPollInterval := 20 * time.Second + maxResourceConsumerDelay := maxConsumeCPUDelay + waitForReplicasPollInterval + + waitBuffer := 1 * time.Minute + ginkgo.Describe("with short downscale stabilization window", func() { ginkgo.It("should scale down soon after the stabilization period", func() { ginkgo.By("setting up resource consumer and HPA") @@ -38,6 +53,7 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n usageForSingleReplica := 110 initPods := 1 initCPUUsageTotal := initPods * usageForSingleReplica + upScaleStabilization := 0 * time.Minute downScaleStabilization := 1 * time.Minute rc := e2eautoscaling.NewDynamicResourceConsumer( @@ -48,25 +64,11 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n defer rc.CleanUp() hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior( - rc, int32(targetCPUUtilizationPercent), 1, 5, int32(downScaleStabilization.Seconds()), + rc, int32(targetCPUUtilizationPercent), 1, 5, + e2eautoscaling.HPABehaviorWithStabilizationWindows(upScaleStabilization, downScaleStabilization), ) defer e2eautoscaling.DeleteHPAWithBehavior(rc, hpa.Name) - fullWindowOfNewUsage := 30 * time.Second - windowWithOldUsagePasses := 30 * time.Second - newPodMetricsDelay := 15 * time.Second - metricsAvailableDelay := fullWindowOfNewUsage + windowWithOldUsagePasses + newPodMetricsDelay - - hpaReconciliationInterval := 15 * time.Second - actuationDelay := 10 * time.Second - maxHPAReactionTime := metricsAvailableDelay + hpaReconciliationInterval + actuationDelay - - maxConsumeCPUDelay := 30 * time.Second - waitForReplicasPollInterval := 20 * time.Second - maxResourceConsumerDelay := maxConsumeCPUDelay + waitForReplicasPollInterval - - waitBuffer := 1 * time.Minute - // making sure HPA is ready, doing its job and already has a recommendation recorded // for stabilization logic before lowering the consumption ginkgo.By("triggering scale up to record a recommendation") @@ -86,4 +88,130 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n framework.ExpectEqual(timeWaited < deadline, true, "waited %s, wanted less than %s", timeWaited, deadline) }) }) + + ginkgo.Describe("with long upscale stabilization window", func() { + ginkgo.It("should scale up only after the stabilization period", func() { + ginkgo.By("setting up resource consumer and HPA") + podCPURequest := 500 + targetCPUUtilizationPercent := 25 + usageForSingleReplica := 110 + initPods := 2 + initCPUUsageTotal := initPods * usageForSingleReplica + upScaleStabilization := 3 * time.Minute + downScaleStabilization := 0 * time.Minute + + rc := e2eautoscaling.NewDynamicResourceConsumer( + "consumer", f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, + initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, + f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, + ) + defer rc.CleanUp() + + hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior( + rc, int32(targetCPUUtilizationPercent), 1, 10, + e2eautoscaling.HPABehaviorWithStabilizationWindows(upScaleStabilization, downScaleStabilization), + ) + defer e2eautoscaling.DeleteHPAWithBehavior(rc, hpa.Name) + + // making sure HPA is ready, doing its job and already has a recommendation recorded + // for stabilization logic before increasing the consumption + ginkgo.By("triggering scale down to record a recommendation") + rc.ConsumeCPU(1 * usageForSingleReplica) + rc.WaitForReplicas(1, maxHPAReactionTime+maxResourceConsumerDelay+waitBuffer) + + ginkgo.By("triggering scale up by increasing consumption") + rc.ConsumeCPU(3 * usageForSingleReplica) + waitStart := time.Now() + rc.WaitForReplicas(3, upScaleStabilization+maxHPAReactionTime+maxResourceConsumerDelay+waitBuffer) + timeWaited := time.Now().Sub(waitStart) + + ginkgo.By("verifying time waited for a scale up") + framework.Logf("time waited for scale up: %s", timeWaited) + framework.ExpectEqual(timeWaited > upScaleStabilization, true, "waited %s, wanted more than %s", timeWaited, upScaleStabilization) + deadline := upScaleStabilization + maxHPAReactionTime + maxResourceConsumerDelay + framework.ExpectEqual(timeWaited < deadline, true, "waited %s, wanted less than %s", timeWaited, deadline) + }) + }) + + ginkgo.Describe("with upscale autoscaling disabled", func() { + ginkgo.It("shouldn't scale up", func() { + ginkgo.By("setting up resource consumer and HPA") + podCPURequest := 500 + targetCPUUtilizationPercent := 25 + usageForSingleReplica := 110 + initPods := 1 + initCPUUsageTotal := initPods * usageForSingleReplica + + rc := e2eautoscaling.NewDynamicResourceConsumer( + "consumer", f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, + initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, + f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, + ) + defer rc.CleanUp() + + hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior( + rc, int32(targetCPUUtilizationPercent), 1, 10, e2eautoscaling.HPABehaviorWithScaleUpDisabled(), + ) + defer e2eautoscaling.DeleteHPAWithBehavior(rc, hpa.Name) + + waitDeadline := maxHPAReactionTime + maxResourceConsumerDelay + waitBuffer + + ginkgo.By("trying to trigger scale up") + rc.ConsumeCPU(8 * usageForSingleReplica) + waitStart := time.Now() + + rc.EnsureDesiredReplicasInRange(initPods, initPods, waitDeadline, hpa.Name) + timeWaited := time.Now().Sub(waitStart) + + ginkgo.By("verifying time waited for a scale up") + framework.Logf("time waited for scale up: %s", timeWaited) + framework.ExpectEqual(timeWaited > waitDeadline, true, "waited %s, wanted to wait more than %s", timeWaited, waitDeadline) + + ginkgo.By("verifying number of replicas") + replicas := rc.GetReplicas() + framework.ExpectEqual(replicas == initPods, true, "had %s replicas, still have %s replicas after time deadline", initPods, replicas) + }) + }) + + ginkgo.Describe("with downscale autoscaling disabled", func() { + ginkgo.It("shouldn't scale down", func() { + ginkgo.By("setting up resource consumer and HPA") + podCPURequest := 500 + targetCPUUtilizationPercent := 25 + usageForSingleReplica := 110 + initPods := 3 + initCPUUsageTotal := initPods * usageForSingleReplica + + rc := e2eautoscaling.NewDynamicResourceConsumer( + "consumer", f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, + initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, + f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, + ) + defer rc.CleanUp() + + hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior( + rc, int32(targetCPUUtilizationPercent), 1, 10, e2eautoscaling.HPABehaviorWithScaleDownDisabled(), + ) + defer e2eautoscaling.DeleteHPAWithBehavior(rc, hpa.Name) + + defaultDownscaleStabilisation := 5 * time.Minute + waitDeadline := maxHPAReactionTime + maxResourceConsumerDelay + defaultDownscaleStabilisation + + ginkgo.By("trying to trigger scale down") + rc.ConsumeCPU(1 * usageForSingleReplica) + waitStart := time.Now() + + rc.EnsureDesiredReplicasInRange(initPods, initPods, waitDeadline, hpa.Name) + timeWaited := time.Now().Sub(waitStart) + + ginkgo.By("verifying time waited for a scale down") + framework.Logf("time waited for scale down: %s", timeWaited) + framework.ExpectEqual(timeWaited > waitDeadline, true, "waited %s, wanted to wait more than %s", timeWaited, waitDeadline) + + ginkgo.By("verifying number of replicas") + replicas := rc.GetReplicas() + framework.ExpectEqual(replicas == initPods, true, "had %s replicas, still have %s replicas after time deadline", initPods, replicas) + }) + }) + }) diff --git a/test/e2e/framework/autoscaling/autoscaling_utils.go b/test/e2e/framework/autoscaling/autoscaling_utils.go index 5fbbb7585ef..118cc51d15a 100644 --- a/test/e2e/framework/autoscaling/autoscaling_utils.go +++ b/test/e2e/framework/autoscaling/autoscaling_utils.go @@ -684,7 +684,7 @@ func DeleteContainerResourceHPA(rc *ResourceConsumer, autoscalerName string) { rc.clientSet.AutoscalingV2().HorizontalPodAutoscalers(rc.nsName).Delete(context.TODO(), autoscalerName, metav1.DeleteOptions{}) } -func CreateCPUHorizontalPodAutoscalerWithBehavior(rc *ResourceConsumer, cpu, minReplicas, maxRepl, downscaleStabilizationSeconds int32) *autoscalingv2.HorizontalPodAutoscaler { +func CreateCPUHorizontalPodAutoscalerWithBehavior(rc *ResourceConsumer, cpu int32, minReplicas int32, maxRepl int32, behavior *autoscalingv2.HorizontalPodAutoscalerBehavior) *autoscalingv2.HorizontalPodAutoscaler { hpa := &autoscalingv2.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: rc.name, @@ -710,11 +710,7 @@ func CreateCPUHorizontalPodAutoscalerWithBehavior(rc *ResourceConsumer, cpu, min }, }, }, - Behavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{ - ScaleDown: &autoscalingv2.HPAScalingRules{ - StabilizationWindowSeconds: &downscaleStabilizationSeconds, - }, - }, + Behavior: behavior, }, } hpa, errHPA := rc.clientSet.AutoscalingV2().HorizontalPodAutoscalers(rc.nsName).Create(context.TODO(), hpa, metav1.CreateOptions{}) @@ -722,6 +718,47 @@ func CreateCPUHorizontalPodAutoscalerWithBehavior(rc *ResourceConsumer, cpu, min return hpa } +func HPABehaviorWithScaleUpAndDownRules(scaleUpRule, scaleDownRule *autoscalingv2.HPAScalingRules) *autoscalingv2.HorizontalPodAutoscalerBehavior { + return &autoscalingv2.HorizontalPodAutoscalerBehavior{ + ScaleUp: scaleUpRule, + ScaleDown: scaleDownRule, + } +} + +func HPAStabilizationWindowScalingRule(stabilizationDuration int32) *autoscalingv2.HPAScalingRules { + return &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: &stabilizationDuration, + } +} + +func HPAPolicyDisabledScalingRule() *autoscalingv2.HPAScalingRules { + disabledPolicy := autoscalingv2.DisabledPolicySelect + return &autoscalingv2.HPAScalingRules{ + SelectPolicy: &disabledPolicy, + } +} + +func HPABehaviorWithStabilizationWindows(upscaleStabilization, downscaleStabilization time.Duration) *autoscalingv2.HorizontalPodAutoscalerBehavior { + return HPABehaviorWithScaleUpAndDownRules( + /*scaleUpRule=*/ HPAStabilizationWindowScalingRule(int32(upscaleStabilization.Seconds())), + /*scaleDownRule=*/ HPAStabilizationWindowScalingRule(int32(downscaleStabilization.Seconds())), + ) +} + +func HPABehaviorWithScaleUpDisabled() *autoscalingv2.HorizontalPodAutoscalerBehavior { + return HPABehaviorWithScaleUpAndDownRules( + /*scaleUpRule=*/ HPAPolicyDisabledScalingRule(), + /*scaleDownRule=*/ nil, + ) +} + +func HPABehaviorWithScaleDownDisabled() *autoscalingv2.HorizontalPodAutoscalerBehavior { + return HPABehaviorWithScaleUpAndDownRules( + /*scaleUpRule=*/ nil, + /*scaleDownRule=*/ HPAPolicyDisabledScalingRule(), + ) +} + func DeleteHPAWithBehavior(rc *ResourceConsumer, autoscalerName string) { rc.clientSet.AutoscalingV2().HorizontalPodAutoscalers(rc.nsName).Delete(context.TODO(), autoscalerName, metav1.DeleteOptions{}) }