diff --git a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go index 555453bd871..583877ea73b 100644 --- a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go +++ b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go @@ -30,6 +30,12 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n f := framework.NewDefaultFramework("horizontal-pod-autoscaling") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + hpaName := "consumer" + + podCPURequest := 500 + targetCPUUtilizationPercent := 25 + usageForSingleReplica := 110 + fullWindowOfNewUsage := 30 * time.Second windowWithOldUsagePasses := 30 * time.Second newPodMetricsDelay := 15 * time.Second @@ -48,16 +54,13 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n ginkgo.Describe("with short downscale stabilization window", func() { ginkgo.It("should scale down soon after the stabilization period", func() { ginkgo.By("setting up resource consumer and HPA") - podCPURequest := 500 - targetCPUUtilizationPercent := 25 - usageForSingleReplica := 110 initPods := 1 initCPUUsageTotal := initPods * usageForSingleReplica upScaleStabilization := 0 * time.Minute downScaleStabilization := 1 * time.Minute rc := e2eautoscaling.NewDynamicResourceConsumer( - "consumer", f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, + hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, ) @@ -92,16 +95,13 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n ginkgo.Describe("with long upscale stabilization window", func() { ginkgo.It("should scale up only after the stabilization period", func() { ginkgo.By("setting up resource consumer and HPA") - podCPURequest := 500 - targetCPUUtilizationPercent := 25 - usageForSingleReplica := 110 initPods := 2 initCPUUsageTotal := initPods * usageForSingleReplica upScaleStabilization := 3 * time.Minute downScaleStabilization := 0 * time.Minute rc := e2eautoscaling.NewDynamicResourceConsumer( - "consumer", f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, + hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, ) @@ -133,24 +133,21 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n }) }) - ginkgo.Describe("with upscale autoscaling disabled", func() { + ginkgo.Describe("with autoscaling disabled", func() { ginkgo.It("shouldn't scale up", func() { ginkgo.By("setting up resource consumer and HPA") - podCPURequest := 500 - targetCPUUtilizationPercent := 25 - usageForSingleReplica := 110 initPods := 1 initCPUUsageTotal := initPods * usageForSingleReplica rc := e2eautoscaling.NewDynamicResourceConsumer( - "consumer", f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, + hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, ) defer rc.CleanUp() hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior( - rc, int32(targetCPUUtilizationPercent), 1, 10, e2eautoscaling.HPABehaviorWithScaleUpDisabled(), + rc, int32(targetCPUUtilizationPercent), 1, 10, e2eautoscaling.HPABehaviorWithScaleDisabled(e2eautoscaling.ScaleUpDirection), ) defer e2eautoscaling.DeleteHPAWithBehavior(rc, hpa.Name) @@ -171,26 +168,21 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n replicas := rc.GetReplicas() framework.ExpectEqual(replicas == initPods, true, "had %s replicas, still have %s replicas after time deadline", initPods, replicas) }) - }) - ginkgo.Describe("with downscale autoscaling disabled", func() { ginkgo.It("shouldn't scale down", func() { ginkgo.By("setting up resource consumer and HPA") - podCPURequest := 500 - targetCPUUtilizationPercent := 25 - usageForSingleReplica := 110 initPods := 3 initCPUUsageTotal := initPods * usageForSingleReplica rc := e2eautoscaling.NewDynamicResourceConsumer( - "consumer", f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, + hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, ) defer rc.CleanUp() hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior( - rc, int32(targetCPUUtilizationPercent), 1, 10, e2eautoscaling.HPABehaviorWithScaleDownDisabled(), + rc, int32(targetCPUUtilizationPercent), 1, 10, e2eautoscaling.HPABehaviorWithScaleDisabled(e2eautoscaling.ScaleDownDirection), ) defer e2eautoscaling.DeleteHPAWithBehavior(rc, hpa.Name) @@ -212,6 +204,178 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n replicas := rc.GetReplicas() framework.ExpectEqual(replicas == initPods, true, "had %s replicas, still have %s replicas after time deadline", initPods, replicas) }) + }) + ginkgo.Describe("with scale limited by number of Pods rate", func() { + ginkgo.It("should scale up no more than given number of Pods per minute", func() { + ginkgo.By("setting up resource consumer and HPA") + initPods := 1 + initCPUUsageTotal := initPods * usageForSingleReplica + limitWindowLength := 1 * time.Minute + podsLimitPerMinute := 2 + + rc := e2eautoscaling.NewDynamicResourceConsumer( + hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, + initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, + f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, + ) + defer rc.CleanUp() + + hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior( + rc, int32(targetCPUUtilizationPercent), 1, 10, + e2eautoscaling.HPABehaviorWithScaleLimitedByNumberOfPods(e2eautoscaling.ScaleUpDirection, int32(podsLimitPerMinute), int32(limitWindowLength.Seconds())), + ) + defer e2eautoscaling.DeleteHPAWithBehavior(rc, hpa.Name) + + ginkgo.By("triggering scale up by increasing consumption") + rc.ConsumeCPU(5 * usageForSingleReplica) + + waitStart := time.Now() + rc.WaitForReplicas(3, maxHPAReactionTime+maxResourceConsumerDelay+limitWindowLength) + timeWaitedFor3 := time.Now().Sub(waitStart) + + waitStart = time.Now() + rc.WaitForReplicas(5, maxHPAReactionTime+maxResourceConsumerDelay+limitWindowLength) + timeWaitedFor5 := time.Now().Sub(waitStart) + + ginkgo.By("verifying time waited for a scale up to 3 replicas") + deadline := limitWindowLength + maxHPAReactionTime + maxResourceConsumerDelay + // First scale event can happen right away, as there were no scale events in the past. + framework.ExpectEqual(timeWaitedFor3 < deadline, true, "waited %s, wanted less than %s", timeWaitedFor3, deadline) + + ginkgo.By("verifying time waited for a scale up to 5 replicas") + // Second scale event needs to respect limit window. + framework.ExpectEqual(timeWaitedFor5 > limitWindowLength, true, "waited %s, wanted to wait more than %s", timeWaitedFor5, limitWindowLength) + framework.ExpectEqual(timeWaitedFor5 < deadline, true, "waited %s, wanted less than %s", timeWaitedFor5, deadline) + }) + + ginkgo.It("should scale down no more than given number of Pods per minute", func() { + ginkgo.By("setting up resource consumer and HPA") + initPods := 6 + initCPUUsageTotal := initPods * usageForSingleReplica + limitWindowLength := 1 * time.Minute + podsLimitPerMinute := 2 + + rc := e2eautoscaling.NewDynamicResourceConsumer( + hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, + initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, + f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, + ) + defer rc.CleanUp() + + hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior( + rc, int32(targetCPUUtilizationPercent), 1, 10, + e2eautoscaling.HPABehaviorWithScaleLimitedByNumberOfPods(e2eautoscaling.ScaleDownDirection, int32(podsLimitPerMinute), int32(limitWindowLength.Seconds())), + ) + defer e2eautoscaling.DeleteHPAWithBehavior(rc, hpa.Name) + + ginkgo.By("triggering scale down by lowering consumption") + rc.ConsumeCPU(1 * usageForSingleReplica) + + waitStart := time.Now() + rc.WaitForReplicas(4, maxHPAReactionTime+maxResourceConsumerDelay+limitWindowLength) + timeWaitedFor4 := time.Now().Sub(waitStart) + + waitStart = time.Now() + rc.WaitForReplicas(2, maxHPAReactionTime+maxResourceConsumerDelay+limitWindowLength) + timeWaitedFor2 := time.Now().Sub(waitStart) + + ginkgo.By("verifying time waited for a scale down to 4 replicas") + deadline := limitWindowLength + maxHPAReactionTime + maxResourceConsumerDelay + // First scale event can happen right away, as there were no scale events in the past. + framework.ExpectEqual(timeWaitedFor4 < deadline, true, "waited %s, wanted less than %s", timeWaitedFor4, deadline) + + ginkgo.By("verifying time waited for a scale down to 2 replicas") + // Second scale event needs to respect limit window. + framework.ExpectEqual(timeWaitedFor2 > limitWindowLength, true, "waited %s, wanted more than %s", timeWaitedFor2, limitWindowLength) + framework.ExpectEqual(timeWaitedFor2 < deadline, true, "waited %s, wanted less than %s", timeWaitedFor2, deadline) + }) + }) + + ginkgo.Describe("with scale limited by percentage", func() { + ginkgo.It("should scale up no more than given percentage of current Pods per minute", func() { + ginkgo.By("setting up resource consumer and HPA") + initPods := 4 + initCPUUsageTotal := initPods * usageForSingleReplica + limitWindowLength := 1 * time.Minute + percentageLimitPerMinute := 50 + + rc := e2eautoscaling.NewDynamicResourceConsumer( + hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, + initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, + f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, + ) + defer rc.CleanUp() + + hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior( + rc, int32(targetCPUUtilizationPercent), 1, 10, + e2eautoscaling.HPABehaviorWithScaleLimitedByPercentage(e2eautoscaling.ScaleUpDirection, int32(percentageLimitPerMinute), int32(limitWindowLength.Seconds())), + ) + defer e2eautoscaling.DeleteHPAWithBehavior(rc, hpa.Name) + + ginkgo.By("triggering scale up by increasing consumption") + rc.ConsumeCPU(10 * usageForSingleReplica) + + waitStart := time.Now() + rc.WaitForReplicas(6, maxHPAReactionTime+maxResourceConsumerDelay+limitWindowLength) + timeWaitedFor6 := time.Now().Sub(waitStart) + + waitStart = time.Now() + rc.WaitForReplicas(9, maxHPAReactionTime+maxResourceConsumerDelay+limitWindowLength) + timeWaitedFor9 := time.Now().Sub(waitStart) + + ginkgo.By("verifying time waited for a scale up to 6 replicas") + deadline := limitWindowLength + maxHPAReactionTime + maxResourceConsumerDelay + // First scale event can happen right away, as there were no scale events in the past. + framework.ExpectEqual(timeWaitedFor6 < deadline, true, "waited %s, wanted less than %s", timeWaitedFor6, deadline) + + ginkgo.By("verifying time waited for a scale up to 9 replicas") + // Second scale event needs to respect limit window. + framework.ExpectEqual(timeWaitedFor9 > limitWindowLength, true, "waited %s, wanted to wait more than %s", timeWaitedFor9, limitWindowLength) + framework.ExpectEqual(timeWaitedFor9 < deadline, true, "waited %s, wanted less than %s", timeWaitedFor9, deadline) + }) + + ginkgo.It("should scale down no more than given percentage of current Pods per minute", func() { + ginkgo.By("setting up resource consumer and HPA") + initPods := 8 + initCPUUsageTotal := initPods * usageForSingleReplica + limitWindowLength := 1 * time.Minute + percentageLimitPerMinute := 50 + + rc := e2eautoscaling.NewDynamicResourceConsumer( + hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, + initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, + f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, + ) + defer rc.CleanUp() + + hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior( + rc, int32(targetCPUUtilizationPercent), 1, 10, + e2eautoscaling.HPABehaviorWithScaleLimitedByPercentage(e2eautoscaling.ScaleDownDirection, int32(percentageLimitPerMinute), int32(limitWindowLength.Seconds())), + ) + defer e2eautoscaling.DeleteHPAWithBehavior(rc, hpa.Name) + + ginkgo.By("triggering scale down by lowering consumption") + rc.ConsumeCPU(1 * usageForSingleReplica) + + waitStart := time.Now() + rc.WaitForReplicas(4, maxHPAReactionTime+maxResourceConsumerDelay+limitWindowLength) + timeWaitedFor4 := time.Now().Sub(waitStart) + + waitStart = time.Now() + rc.WaitForReplicas(2, maxHPAReactionTime+maxResourceConsumerDelay+limitWindowLength) + timeWaitedFor2 := time.Now().Sub(waitStart) + + ginkgo.By("verifying time waited for a scale down to 4 replicas") + deadline := limitWindowLength + maxHPAReactionTime + maxResourceConsumerDelay + // First scale event can happen right away, as there were no scale events in the past. + framework.ExpectEqual(timeWaitedFor4 < deadline, true, "waited %s, wanted less than %s", timeWaitedFor4, deadline) + + ginkgo.By("verifying time waited for a scale down to 2 replicas") + // Second scale event needs to respect limit window. + framework.ExpectEqual(timeWaitedFor2 > limitWindowLength, true, "waited %s, wanted more than %s", timeWaitedFor2, limitWindowLength) + framework.ExpectEqual(timeWaitedFor2 < deadline, true, "waited %s, wanted less than %s", timeWaitedFor2, deadline) + }) + }) }) diff --git a/test/e2e/framework/autoscaling/autoscaling_utils.go b/test/e2e/framework/autoscaling/autoscaling_utils.go index eb90996daab..7d7fb99c5a7 100644 --- a/test/e2e/framework/autoscaling/autoscaling_utils.go +++ b/test/e2e/framework/autoscaling/autoscaling_utils.go @@ -78,6 +78,15 @@ var ( KindReplicaSet = schema.GroupVersionKind{Group: "apps", Version: "v1beta2", Kind: "ReplicaSet"} ) +// ScalingDirection identifies the scale direction for HPA Behavior. +type ScalingDirection int + +const ( + DirectionUnknown ScalingDirection = iota + ScaleUpDirection + ScaleDownDirection +) + /* ResourceConsumer is a tool for testing. It helps create specified usage of CPU or memory (Warning: memory not supported) typical use case: @@ -725,38 +734,65 @@ func HPABehaviorWithScaleUpAndDownRules(scaleUpRule, scaleDownRule *autoscalingv } } -func HPAStabilizationWindowScalingRule(stabilizationDuration int32) *autoscalingv2.HPAScalingRules { +func HPABehaviorWithScalingRuleInDirection(scalingDirection ScalingDirection, rule *autoscalingv2.HPAScalingRules) *autoscalingv2.HorizontalPodAutoscalerBehavior { + var scaleUpRule, scaleDownRule *autoscalingv2.HPAScalingRules + if scalingDirection == ScaleUpDirection { + scaleUpRule = rule + } + if scalingDirection == ScaleDownDirection { + scaleDownRule = rule + } + return HPABehaviorWithScaleUpAndDownRules(scaleUpRule, scaleDownRule) +} + +func HPAScalingRuleWithStabilizationWindow(stabilizationDuration int32) *autoscalingv2.HPAScalingRules { return &autoscalingv2.HPAScalingRules{ StabilizationWindowSeconds: &stabilizationDuration, } } -func HPAPolicyDisabledScalingRule() *autoscalingv2.HPAScalingRules { +func HPAScalingRuleWithPolicyDisabled() *autoscalingv2.HPAScalingRules { disabledPolicy := autoscalingv2.DisabledPolicySelect return &autoscalingv2.HPAScalingRules{ SelectPolicy: &disabledPolicy, } } +func HPAScalingRuleWithScalingPolicy(policyType autoscalingv2.HPAScalingPolicyType, value, periodSeconds int32) *autoscalingv2.HPAScalingRules { + stabilizationWindowDisabledDuration := int32(0) + selectPolicy := autoscalingv2.MaxChangePolicySelect + return &autoscalingv2.HPAScalingRules{ + Policies: []autoscalingv2.HPAScalingPolicy{ + { + Type: policyType, + Value: value, + PeriodSeconds: periodSeconds, + }, + }, + SelectPolicy: &selectPolicy, + StabilizationWindowSeconds: &stabilizationWindowDisabledDuration, + } +} + func HPABehaviorWithStabilizationWindows(upscaleStabilization, downscaleStabilization time.Duration) *autoscalingv2.HorizontalPodAutoscalerBehavior { - return HPABehaviorWithScaleUpAndDownRules( - /*scaleUpRule=*/ HPAStabilizationWindowScalingRule(int32(upscaleStabilization.Seconds())), - /*scaleDownRule=*/ HPAStabilizationWindowScalingRule(int32(downscaleStabilization.Seconds())), - ) + scaleUpRule := HPAScalingRuleWithStabilizationWindow(int32(upscaleStabilization.Seconds())) + scaleDownRule := HPAScalingRuleWithStabilizationWindow(int32(downscaleStabilization.Seconds())) + return HPABehaviorWithScaleUpAndDownRules(scaleUpRule, scaleDownRule) } -func HPABehaviorWithScaleUpDisabled() *autoscalingv2.HorizontalPodAutoscalerBehavior { - return HPABehaviorWithScaleUpAndDownRules( - /*scaleUpRule=*/ HPAPolicyDisabledScalingRule(), - /*scaleDownRule=*/ nil, - ) +func HPABehaviorWithScaleDisabled(scalingDirection ScalingDirection) *autoscalingv2.HorizontalPodAutoscalerBehavior { + scalingRule := HPAScalingRuleWithPolicyDisabled() + return HPABehaviorWithScalingRuleInDirection(scalingDirection, scalingRule) } -func HPABehaviorWithScaleDownDisabled() *autoscalingv2.HorizontalPodAutoscalerBehavior { - return HPABehaviorWithScaleUpAndDownRules( - /*scaleUpRule=*/ nil, - /*scaleDownRule=*/ HPAPolicyDisabledScalingRule(), - ) +func HPABehaviorWithScaleLimitedByNumberOfPods(scalingDirection ScalingDirection, numberOfPods, periodSeconds int32) *autoscalingv2.HorizontalPodAutoscalerBehavior { + scalingRule := HPAScalingRuleWithScalingPolicy(autoscalingv2.PodsScalingPolicy, numberOfPods, periodSeconds) + return HPABehaviorWithScalingRuleInDirection(scalingDirection, scalingRule) +} + +func HPABehaviorWithScaleLimitedByPercentage(scalingDirection ScalingDirection, percentage, periodSeconds int32) *autoscalingv2.HorizontalPodAutoscalerBehavior { + scalingRule := HPAScalingRuleWithScalingPolicy(autoscalingv2.PercentScalingPolicy, percentage, periodSeconds) + return HPABehaviorWithScalingRuleInDirection(scalingDirection, scalingRule) } func DeleteHPAWithBehavior(rc *ResourceConsumer, autoscalerName string) {