From a724818ad0430cabe117c5e64afdaecfb8dd51c8 Mon Sep 17 00:00:00 2001 From: Piotr Nosek Date: Fri, 3 Jun 2022 14:21:56 +0000 Subject: [PATCH] Add e2e test for HPA behavior: scale up delay --- .../autoscaling/horizontal_pod_autoscaling.go | 3 +- .../horizontal_pod_autoscaling_behavior.go | 59 ++++++++++++++++++- .../autoscaling/autoscaling_utils.go | 24 ++++++-- 3 files changed, 78 insertions(+), 8 deletions(-) diff --git a/test/e2e/autoscaling/horizontal_pod_autoscaling.go b/test/e2e/autoscaling/horizontal_pod_autoscaling.go index d518cabb1b2..5f57095e47e 100644 --- a/test/e2e/autoscaling/horizontal_pod_autoscaling.go +++ b/test/e2e/autoscaling/horizontal_pod_autoscaling.go @@ -17,9 +17,10 @@ limitations under the License. package autoscaling import ( - "k8s.io/pod-security-admission/api" "time" + "k8s.io/pod-security-admission/api" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/kubernetes/test/e2e/framework" e2eautoscaling "k8s.io/kubernetes/test/e2e/framework/autoscaling" diff --git a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go index 396bfa07221..7550e1eef94 100644 --- a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go +++ b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go @@ -48,7 +48,7 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n defer rc.CleanUp() hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior( - rc, int32(targetCPUUtilizationPercent), 1, 5, int32(downScaleStabilization.Seconds()), + rc, int32(targetCPUUtilizationPercent), 1, 5, e2eautoscaling.HPABehaviorWithDownscaleStabilizationSeconds(int32(downScaleStabilization.Seconds())), ) defer e2eautoscaling.DeleteHPAWithBehavior(rc, hpa.Name) @@ -86,4 +86,61 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n framework.ExpectEqual(timeWaited < deadline, true, "waited %s, wanted less than %s", timeWaited, deadline) }) }) + + ginkgo.Describe("with long upscale stabilization window", func() { + ginkgo.It("should scale up only after the stabilization period", func() { + ginkgo.By("setting up resource consumer and HPA") + podCPURequest := 500 + targetCPUUtilizationPercent := 25 + usageForSingleReplica := 110 + initPods := 4 + initCPUUsageTotal := initPods * usageForSingleReplica + upScaleStabilization := 3 * time.Minute + + rc := e2eautoscaling.NewDynamicResourceConsumer( + "consumer", f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, + initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, + f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, + ) + defer rc.CleanUp() + + hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior( + rc, int32(targetCPUUtilizationPercent), 1, 5, e2eautoscaling.HPABehaviorWithUpscaleStabilizationSeconds(int32(upScaleStabilization.Seconds())), + ) + defer e2eautoscaling.DeleteHPAWithBehavior(rc, hpa.Name) + + fullWindowOfNewUsage := 30 * time.Second + windowWithOldUsagePasses := 30 * time.Second + newPodMetricsDelay := 15 * time.Second + metricsAvailableDelay := fullWindowOfNewUsage + windowWithOldUsagePasses + newPodMetricsDelay + + hpaReconciliationInterval := 15 * time.Second + actuationDelay := 10 * time.Second + maxHPAReactionTime := metricsAvailableDelay + hpaReconciliationInterval + actuationDelay + + maxConsumeCPUDelay := 30 * time.Second + waitForReplicasPollInterval := 20 * time.Second + maxResourceConsumerDelay := maxConsumeCPUDelay + waitForReplicasPollInterval + + waitBuffer := 1 * time.Minute + + // making sure HPA is ready, doing its job and already has a recommendation recorded + // for stabilization logic before increasing the consumption + ginkgo.By("triggering scale down to record a recommendation") + rc.ConsumeCPU(2 * usageForSingleReplica) + rc.WaitForReplicas(2, maxHPAReactionTime+maxResourceConsumerDelay+waitBuffer) + + ginkgo.By("triggering scale up by increasing consumption") + rc.ConsumeCPU(5 * usageForSingleReplica) + waitStart := time.Now() + rc.WaitForReplicas(5, upScaleStabilization+maxHPAReactionTime+maxResourceConsumerDelay+waitBuffer) + timeWaited := time.Now().Sub(waitStart) + + ginkgo.By("verifying time waited for a scale up") + framework.Logf("time waited for scale up: %s", timeWaited) + framework.ExpectEqual(timeWaited > upScaleStabilization, true, "waited %s, wanted more than %s", timeWaited, upScaleStabilization) + deadline := upScaleStabilization + maxHPAReactionTime + maxResourceConsumerDelay + framework.ExpectEqual(timeWaited < deadline, true, "waited %s, wanted less than %s", timeWaited, deadline) + }) + }) }) diff --git a/test/e2e/framework/autoscaling/autoscaling_utils.go b/test/e2e/framework/autoscaling/autoscaling_utils.go index e2d1bd2b852..451c381670a 100644 --- a/test/e2e/framework/autoscaling/autoscaling_utils.go +++ b/test/e2e/framework/autoscaling/autoscaling_utils.go @@ -667,7 +667,7 @@ func DeleteContainerResourceHPA(rc *ResourceConsumer, autoscalerName string) { rc.clientSet.AutoscalingV2().HorizontalPodAutoscalers(rc.nsName).Delete(context.TODO(), autoscalerName, metav1.DeleteOptions{}) } -func CreateCPUHorizontalPodAutoscalerWithBehavior(rc *ResourceConsumer, cpu, minReplicas, maxRepl, downscaleStabilizationSeconds int32) *autoscalingv2.HorizontalPodAutoscaler { +func CreateCPUHorizontalPodAutoscalerWithBehavior(rc *ResourceConsumer, cpu int32, minReplicas int32, maxRepl int32, behavior *autoscalingv2.HorizontalPodAutoscalerBehavior) *autoscalingv2.HorizontalPodAutoscaler { hpa := &autoscalingv2.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: rc.name, @@ -693,11 +693,7 @@ func CreateCPUHorizontalPodAutoscalerWithBehavior(rc *ResourceConsumer, cpu, min }, }, }, - Behavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{ - ScaleDown: &autoscalingv2.HPAScalingRules{ - StabilizationWindowSeconds: &downscaleStabilizationSeconds, - }, - }, + Behavior: behavior, }, } hpa, errHPA := rc.clientSet.AutoscalingV2().HorizontalPodAutoscalers(rc.nsName).Create(context.TODO(), hpa, metav1.CreateOptions{}) @@ -709,6 +705,22 @@ func DeleteHPAWithBehavior(rc *ResourceConsumer, autoscalerName string) { rc.clientSet.AutoscalingV2().HorizontalPodAutoscalers(rc.nsName).Delete(context.TODO(), autoscalerName, metav1.DeleteOptions{}) } +func HPABehaviorWithDownscaleStabilizationSeconds(downscaleStabilizationSeconds int32) *autoscalingv2.HorizontalPodAutoscalerBehavior { + return &autoscalingv2.HorizontalPodAutoscalerBehavior{ + ScaleDown: &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: &downscaleStabilizationSeconds, + }, + } +} + +func HPABehaviorWithUpscaleStabilizationSeconds(upscaleStabilizationSeconds int32) *autoscalingv2.HorizontalPodAutoscalerBehavior { + return &autoscalingv2.HorizontalPodAutoscalerBehavior{ + ScaleUp: &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: &upscaleStabilizationSeconds, + }, + } +} + //SidecarStatusType type for sidecar status type SidecarStatusType bool