From 50200dc54fc603de3f6b4cd2abb8a6b2e14a5f4d Mon Sep 17 00:00:00 2001 From: Piotr Betkier Date: Mon, 21 Feb 2022 23:12:09 +0100 Subject: [PATCH 1/2] Add e2e test for HPA behavior: decreased downscale stabilization --- .../horizontal_pod_autoscaling_behavior.go | 84 +++++++++++++++++++ .../autoscaling/autoscaling_utils.go | 42 ++++++++++ 2 files changed, 126 insertions(+) create mode 100644 test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go diff --git a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go new file mode 100644 index 00000000000..25abc162180 --- /dev/null +++ b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go @@ -0,0 +1,84 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoscaling + +import ( + "time" + + "k8s.io/kubernetes/test/e2e/framework" + e2eautoscaling "k8s.io/kubernetes/test/e2e/framework/autoscaling" + + "github.com/onsi/ginkgo" +) + +var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (non-default behavior)", func() { + f := framework.NewDefaultFramework("horizontal-pod-autoscaling") + + ginkgo.Describe("with low downscale stabilization window", func() { + ginkgo.It("should scale down soon after the stabilization period", func() { + ginkgo.By("setting up resource consumer and HPA") + podCPURequest := 500 + targetCPUUtilizationPercent := 25 + usageForSingleReplica := 110 + initPods := 1 + initCPUUsageTotal := initPods * usageForSingleReplica + downScaleStabilization := 1 * time.Minute + + rc := e2eautoscaling.NewDynamicResourceConsumer( + "consumer", f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, + initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, + f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, + ) + defer rc.CleanUp() + + hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior( + rc, int32(targetCPUUtilizationPercent), 1, 5, int32(downScaleStabilization.Seconds()), + ) + defer e2eautoscaling.DeleteHPAWithBehavior(rc, hpa.Name) + + // 30s of metrics window and + // up to 30s until the old metrics window finishes + // and additional 15s until metrics available on a new pod + metricsAvailableDelay := 75 * time.Second + hpaReconciliationInterval := 15 * time.Second + actuationDelay := 10 * time.Second + maxHPAReactionTime := metricsAvailableDelay + hpaReconciliationInterval + actuationDelay + + maxConsumeCPUDelay := 30 * time.Second + waitForReplicasPollInterval := 20 * time.Second + maxResourceConsumerDelay := maxConsumeCPUDelay + waitForReplicasPollInterval + + waitBuffer := 1 * time.Minute + + ginkgo.By("triggering scale up to record a recommendation") + rc.ConsumeCPU(3 * usageForSingleReplica) + rc.WaitForReplicas(3, maxHPAReactionTime+maxResourceConsumerDelay+waitBuffer) + + ginkgo.By("triggering scale down by lowering consumption") + rc.ConsumeCPU(2 * usageForSingleReplica) + waitStart := time.Now() + rc.WaitForReplicas(2, downScaleStabilization+maxHPAReactionTime+maxResourceConsumerDelay+waitBuffer) + timeWaited := time.Now().Sub(waitStart) + + ginkgo.By("verifying time waited for a scale down") + framework.Logf("time waited for scale down: %s", timeWaited) + framework.ExpectEqual(timeWaited > downScaleStabilization, true, "waited %s, wanted more than %s", timeWaited, downScaleStabilization) + deadline := downScaleStabilization + maxHPAReactionTime + maxResourceConsumerDelay + framework.ExpectEqual(timeWaited < deadline, true, "waited %s, wanted less than %s", timeWaited, deadline) + }) + }) +}) diff --git a/test/e2e/framework/autoscaling/autoscaling_utils.go b/test/e2e/framework/autoscaling/autoscaling_utils.go index 2ac29f8a03f..e2d1bd2b852 100644 --- a/test/e2e/framework/autoscaling/autoscaling_utils.go +++ b/test/e2e/framework/autoscaling/autoscaling_utils.go @@ -667,6 +667,48 @@ func DeleteContainerResourceHPA(rc *ResourceConsumer, autoscalerName string) { rc.clientSet.AutoscalingV2().HorizontalPodAutoscalers(rc.nsName).Delete(context.TODO(), autoscalerName, metav1.DeleteOptions{}) } +func CreateCPUHorizontalPodAutoscalerWithBehavior(rc *ResourceConsumer, cpu, minReplicas, maxRepl, downscaleStabilizationSeconds int32) *autoscalingv2.HorizontalPodAutoscaler { + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: rc.name, + Namespace: rc.nsName, + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: rc.kind.GroupVersion().String(), + Kind: rc.kind.Kind, + Name: rc.name, + }, + MinReplicas: &minReplicas, + MaxReplicas: maxRepl, + Metrics: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: v1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: &cpu, + }, + }, + }, + }, + Behavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{ + ScaleDown: &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: &downscaleStabilizationSeconds, + }, + }, + }, + } + hpa, errHPA := rc.clientSet.AutoscalingV2().HorizontalPodAutoscalers(rc.nsName).Create(context.TODO(), hpa, metav1.CreateOptions{}) + framework.ExpectNoError(errHPA) + return hpa +} + +func DeleteHPAWithBehavior(rc *ResourceConsumer, autoscalerName string) { + rc.clientSet.AutoscalingV2().HorizontalPodAutoscalers(rc.nsName).Delete(context.TODO(), autoscalerName, metav1.DeleteOptions{}) +} + //SidecarStatusType type for sidecar status type SidecarStatusType bool From 13c3f091a7d6e0dec6e329c9d511875aae337661 Mon Sep 17 00:00:00 2001 From: Piotr Betkier Date: Fri, 25 Feb 2022 14:12:36 +0100 Subject: [PATCH 2/2] Small review fixes --- .../horizontal_pod_autoscaling_behavior.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go index 25abc162180..a86f4649d4e 100644 --- a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go +++ b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go @@ -1,5 +1,5 @@ /* -Copyright 2015 The Kubernetes Authors. +Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -28,7 +28,7 @@ import ( var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (non-default behavior)", func() { f := framework.NewDefaultFramework("horizontal-pod-autoscaling") - ginkgo.Describe("with low downscale stabilization window", func() { + ginkgo.Describe("with short downscale stabilization window", func() { ginkgo.It("should scale down soon after the stabilization period", func() { ginkgo.By("setting up resource consumer and HPA") podCPURequest := 500 @@ -50,10 +50,11 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n ) defer e2eautoscaling.DeleteHPAWithBehavior(rc, hpa.Name) - // 30s of metrics window and - // up to 30s until the old metrics window finishes - // and additional 15s until metrics available on a new pod - metricsAvailableDelay := 75 * time.Second + fullWindowOfNewUsage := 30 * time.Second + windowWithOldUsagePasses := 30 * time.Second + newPodMetricsDelay := 15 * time.Second + metricsAvailableDelay := fullWindowOfNewUsage + windowWithOldUsagePasses + newPodMetricsDelay + hpaReconciliationInterval := 15 * time.Second actuationDelay := 10 * time.Second maxHPAReactionTime := metricsAvailableDelay + hpaReconciliationInterval + actuationDelay @@ -64,6 +65,8 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n waitBuffer := 1 * time.Minute + // making sure HPA is ready, doing its job and already has a recommendation recorded + // for stabilization logic before lowering the consumption ginkgo.By("triggering scale up to record a recommendation") rc.ConsumeCPU(3 * usageForSingleReplica) rc.WaitForReplicas(3, maxHPAReactionTime+maxResourceConsumerDelay+waitBuffer)