From 463b15b9b222bb1c72aa199016daa9eda64271a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Marc=20Fran=C3=A7ois?= Date: Fri, 14 Mar 2025 10:43:11 -0400 Subject: [PATCH 1/7] Add the HPAConfigurableTolerance feature gate. --- pkg/features/kube_features.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index e0b09608030..da9b369479a 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -293,6 +293,12 @@ const ( // Make the kubelet use shutdown configuration based on pod priority values for graceful shutdown. GracefulNodeShutdownBasedOnPodPriority featuregate.Feature = "GracefulNodeShutdownBasedOnPodPriority" + // owner: @jm-franc + // kep: https://kep.k8s.io/4951 + // + // Enables support of configurable HPA scale-up and scale-down tolerances. + HPAConfigurableTolerance featuregate.Feature = "HPAConfigurableTolerance" + // owner: @dxist // // Enables support of HPA scaling to zero pods when an object or custom metric is configured. @@ -1341,6 +1347,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.36 }, + HPAConfigurableTolerance: { + {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Alpha}, + }, + HPAScaleToZero: { {Version: version.MustParse("1.16"), Default: false, PreRelease: featuregate.Alpha}, }, From a41284d9fa3a3d5a5e8760db6e9fd4f7e5e6fca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Marc=20Fran=C3=A7ois?= Date: Fri, 21 Mar 2025 16:47:44 -0400 Subject: [PATCH 2/7] Add the HorizontalPodAutoscaler tolerance field. Includes v2beta2 HPA round-trip conversion, defaulting, and validation. --- pkg/apis/autoscaling/annotations.go | 8 + pkg/apis/autoscaling/helpers.go | 10 +- pkg/apis/autoscaling/types.go | 27 +- pkg/apis/autoscaling/v2/defaults.go | 10 +- pkg/apis/autoscaling/v2/defaults_test.go | 148 +++++++++++ pkg/apis/autoscaling/v2beta2/conversion.go | 67 ++++- .../autoscaling/v2beta2/conversion_test.go | 124 +++++++++ .../v2beta2/zz_generated.conversion.go | 91 +++++-- pkg/apis/autoscaling/validation/validation.go | 47 +++- .../autoscaling/validation/validation_test.go | 239 +++++++++++++++++- .../src/k8s.io/api/autoscaling/v2/types.go | 26 +- 11 files changed, 745 insertions(+), 52 deletions(-) create mode 100644 pkg/apis/autoscaling/v2beta2/conversion_test.go diff --git a/pkg/apis/autoscaling/annotations.go b/pkg/apis/autoscaling/annotations.go index e49aaab2eca..d793cd4aa38 100644 --- a/pkg/apis/autoscaling/annotations.go +++ b/pkg/apis/autoscaling/annotations.go @@ -36,3 +36,11 @@ const DefaultCPUUtilization = 80 // BehaviorSpecsAnnotation is the annotation which holds the HPA constraints specs // when converting the `Behavior` field from autoscaling/v2beta2 const BehaviorSpecsAnnotation = "autoscaling.alpha.kubernetes.io/behavior" + +// ToleranceScaleDownAnnotation is the annotation which holds the HPA tolerance specs +// when converting the `ScaleDown.Tolerance` field from autoscaling/v2 +const ToleranceScaleDownAnnotation = "autoscaling.alpha.kubernetes.io/scale-down-tolerance" + +// ToleranceScaleUpAnnotation is the annotation which holds the HPA tolerance specs +// when converting the `ScaleUp.Tolerance` field from autoscaling/v2 +const ToleranceScaleUpAnnotation = "autoscaling.alpha.kubernetes.io/scale-up-tolerance" diff --git a/pkg/apis/autoscaling/helpers.go b/pkg/apis/autoscaling/helpers.go index f66a12f4def..67853dc29a0 100644 --- a/pkg/apis/autoscaling/helpers.go +++ b/pkg/apis/autoscaling/helpers.go @@ -16,8 +16,10 @@ limitations under the License. package autoscaling -// DropRoundTripHorizontalPodAutoscalerAnnotations removes any annotations used to serialize round-tripped fields from later API versions, +// DropRoundTripHorizontalPodAutoscalerAnnotations removes any annotations used to +// serialize round-tripped fields from HorizontalPodAutoscaler later API versions, // and returns false if no changes were made and the original input object was returned. +// // It should always be called when converting internal -> external versions, prior // to setting any of the custom annotations: // @@ -34,12 +36,16 @@ package autoscaling func DropRoundTripHorizontalPodAutoscalerAnnotations(in map[string]string) (out map[string]string, copied bool) { _, hasMetricsSpecs := in[MetricSpecsAnnotation] _, hasBehaviorSpecs := in[BehaviorSpecsAnnotation] + _, hasToleranceScaleDown := in[ToleranceScaleDownAnnotation] + _, hasToleranceScaleUp := in[ToleranceScaleUpAnnotation] _, hasMetricsStatuses := in[MetricStatusesAnnotation] _, hasConditions := in[HorizontalPodAutoscalerConditionsAnnotation] - if hasMetricsSpecs || hasBehaviorSpecs || hasMetricsStatuses || hasConditions { + if hasMetricsSpecs || hasBehaviorSpecs || hasToleranceScaleDown || hasToleranceScaleUp || hasMetricsStatuses || hasConditions { out = DeepCopyStringMap(in) delete(out, MetricSpecsAnnotation) delete(out, BehaviorSpecsAnnotation) + delete(out, ToleranceScaleDownAnnotation) + delete(out, ToleranceScaleUpAnnotation) delete(out, MetricStatusesAnnotation) delete(out, HorizontalPodAutoscalerConditionsAnnotation) return out, true diff --git a/pkg/apis/autoscaling/types.go b/pkg/apis/autoscaling/types.go index 7f254442b13..aa7e0ca0159 100644 --- a/pkg/apis/autoscaling/types.go +++ b/pkg/apis/autoscaling/types.go @@ -138,12 +138,18 @@ const ( DisabledPolicySelect ScalingPolicySelect = "Disabled" ) -// HPAScalingRules configures the scaling behavior for one direction. -// These Rules are applied after calculating DesiredReplicas from metrics for the HPA. +// HPAScalingRules configures the scaling behavior for one direction via +// scaling Policy Rules and a configurable metric tolerance. +// +// Scaling Policy Rules are applied after calculating DesiredReplicas from metrics for the HPA. // They can limit the scaling velocity by specifying scaling policies. // They can prevent flapping by specifying the stabilization window, so that the // number of replicas is not set instantly, instead, the safest value from the stabilization // window is chosen. +// +// The tolerance is applied to the metric values and prevents scaling too +// eagerly for small metric variations. (Note that setting a tolerance requires +// enabling the alpha HPAConfigurableTolerance feature gate.) type HPAScalingRules struct { // StabilizationWindowSeconds is the number of seconds for which past recommendations should be // considered while scaling up or scaling down. @@ -157,10 +163,23 @@ type HPAScalingRules struct { // If not set, the default value MaxPolicySelect is used. // +optional SelectPolicy *ScalingPolicySelect - // policies is a list of potential scaling polices which can used during scaling. - // At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + // policies is a list of potential scaling polices which can be used during scaling. + // If not set, use the default values: + // - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + // - For scale down: allow all pods to be removed in a 15s window. // +optional Policies []HPAScalingPolicy + // tolerance is the tolerance on the ratio between the current and desired + // metric value under which no updates are made to the desired number of + // replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + // set, the default cluster-wide tolerance is applied (by default 10%). + // + // This is an alpha field and requires enabling the HPAConfigurableTolerance + // feature gate. + // + // +featureGate=HPAConfigurableTolerance + // +optional + Tolerance *resource.Quantity } // HPAScalingPolicyType is the type of the policy which could be used while making scaling decisions. diff --git a/pkg/apis/autoscaling/v2/defaults.go b/pkg/apis/autoscaling/v2/defaults.go index 2ae50ae287d..2917dd5ff05 100644 --- a/pkg/apis/autoscaling/v2/defaults.go +++ b/pkg/apis/autoscaling/v2/defaults.go @@ -91,9 +91,12 @@ func SetDefaults_HorizontalPodAutoscaler(obj *autoscalingv2.HorizontalPodAutosca SetDefaults_HorizontalPodAutoscalerBehavior(obj) } -// SetDefaults_HorizontalPodAutoscalerBehavior fills the behavior if it is not null +// SetDefaults_HorizontalPodAutoscalerBehavior fills the behavior if it contains +// at least one scaling rule policy (for scale-up or scale-down) func SetDefaults_HorizontalPodAutoscalerBehavior(obj *autoscalingv2.HorizontalPodAutoscaler) { - // if behavior is specified, we should fill all the 'nil' values with the default ones + // If behavior contains a scaling rule policy (either for scale-up, scale-down, or both), we + // should fill all the unset scaling policy fields (i.e. StabilizationWindowSeconds, + // SelectPolicy, Policies) with default values if obj.Spec.Behavior != nil { obj.Spec.Behavior.ScaleUp = GenerateHPAScaleUpRules(obj.Spec.Behavior.ScaleUp) obj.Spec.Behavior.ScaleDown = GenerateHPAScaleDownRules(obj.Spec.Behavior.ScaleDown) @@ -129,5 +132,8 @@ func copyHPAScalingRules(from, to *autoscalingv2.HPAScalingRules) *autoscalingv2 if from.Policies != nil { to.Policies = from.Policies } + if from.Tolerance != nil { + to.Tolerance = from.Tolerance + } return to } diff --git a/pkg/apis/autoscaling/v2/defaults_test.go b/pkg/apis/autoscaling/v2/defaults_test.go index ad95076d7e5..cd8cfa0bc42 100644 --- a/pkg/apis/autoscaling/v2/defaults_test.go +++ b/pkg/apis/autoscaling/v2/defaults_test.go @@ -20,8 +20,12 @@ import ( "reflect" "testing" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" + "k8s.io/kubernetes/pkg/features" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" @@ -132,14 +136,17 @@ func TestGenerateScaleUpRules(t *testing.T) { rateUpPercentPeriodSeconds int32 stabilizationSeconds *int32 selectPolicy *autoscalingv2.ScalingPolicySelect + tolerance *resource.Quantity expectedPolicies []autoscalingv2.HPAScalingPolicy expectedStabilization *int32 expectedSelectPolicy string + expectedTolerance *resource.Quantity annotation string } maxPolicy := autoscalingv2.MaxChangePolicySelect minPolicy := autoscalingv2.MinChangePolicySelect + sampleTolerance := resource.MustParse("0.5") tests := []TestCase{ { annotation: "Default values", @@ -208,12 +215,25 @@ func TestGenerateScaleUpRules(t *testing.T) { expectedStabilization: utilpointer.Int32(25), expectedSelectPolicy: string(autoscalingv2.MaxChangePolicySelect), }, + { + annotation: "Percent policy and tolerance is specified", + rateUpPercent: 7, + rateUpPercentPeriodSeconds: 10, + tolerance: &sampleTolerance, + expectedPolicies: []autoscalingv2.HPAScalingPolicy{ + {Type: autoscalingv2.PercentScalingPolicy, Value: 7, PeriodSeconds: 10}, + }, + expectedStabilization: utilpointer.Int32(0), + expectedSelectPolicy: string(autoscalingv2.MaxChangePolicySelect), + expectedTolerance: &sampleTolerance, + }, } for _, tc := range tests { t.Run(tc.annotation, func(t *testing.T) { scaleUpRules := &autoscalingv2.HPAScalingRules{ StabilizationWindowSeconds: tc.stabilizationSeconds, SelectPolicy: tc.selectPolicy, + Tolerance: tc.tolerance, } if tc.rateUpPods != 0 || tc.rateUpPodsPeriodSeconds != 0 { scaleUpRules.Policies = append(scaleUpRules.Policies, autoscalingv2.HPAScalingPolicy{ @@ -234,10 +254,138 @@ func TestGenerateScaleUpRules(t *testing.T) { } assert.Equal(t, autoscalingv2.ScalingPolicySelect(tc.expectedSelectPolicy), *up.SelectPolicy) + assert.Equal(t, tc.expectedTolerance, up.Tolerance) }) } } +func TestSetBehaviorDefaults(t *testing.T) { + sampleTolerance := resource.MustParse("0.5") + maxPolicy := autoscalingv2.MaxChangePolicySelect + policies := []autoscalingv2.HPAScalingPolicy{ + {Type: autoscalingv2.PercentScalingPolicy, Value: 7, PeriodSeconds: 10}, + } + type TestCase struct { + behavior *autoscalingv2.HorizontalPodAutoscalerBehavior + expectedBehavior *autoscalingv2.HorizontalPodAutoscalerBehavior + annotation string + } + + tests := []TestCase{ + { + annotation: "Nil behavior", + behavior: nil, + expectedBehavior: nil, + }, + { + annotation: "Behavior with stabilizationWindowSeconds and tolerance", + behavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{ + ScaleUp: &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: utilpointer.Int32(100), + Tolerance: &sampleTolerance, + }, + }, + expectedBehavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{ + ScaleDown: &autoscalingv2.HPAScalingRules{ + SelectPolicy: &maxPolicy, + Policies: []autoscalingv2.HPAScalingPolicy{ + {Type: autoscalingv2.PercentScalingPolicy, Value: 100, PeriodSeconds: 15}, + }, + }, + ScaleUp: &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: utilpointer.Int32(100), + SelectPolicy: &maxPolicy, + Policies: []autoscalingv2.HPAScalingPolicy{ + {Type: autoscalingv2.PodsScalingPolicy, Value: 4, PeriodSeconds: 15}, + {Type: autoscalingv2.PercentScalingPolicy, Value: 100, PeriodSeconds: 15}, + }, + Tolerance: &sampleTolerance, + }, + }, + }, + { + annotation: "Behavior with policy, without tolerance", + behavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{ + ScaleDown: &autoscalingv2.HPAScalingRules{ + Policies: policies, + }, + }, + expectedBehavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{ + ScaleDown: &autoscalingv2.HPAScalingRules{ + SelectPolicy: &maxPolicy, + Policies: policies, + }, + ScaleUp: &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: utilpointer.Int32(0), + SelectPolicy: &maxPolicy, + Policies: []autoscalingv2.HPAScalingPolicy{ + {Type: autoscalingv2.PodsScalingPolicy, Value: 4, PeriodSeconds: 15}, + {Type: autoscalingv2.PercentScalingPolicy, Value: 100, PeriodSeconds: 15}, + }, + }, + }, + }, + } + for _, tc := range tests { + t.Run(tc.annotation, func(t *testing.T) { + hpa := autoscalingv2.HorizontalPodAutoscaler{ + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + Behavior: tc.behavior, + }, + } + expectedHPA := autoscalingv2.HorizontalPodAutoscaler{ + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + Behavior: tc.expectedBehavior, + }, + } + SetDefaults_HorizontalPodAutoscalerBehavior(&hpa) + assert.Equal(t, expectedHPA, hpa) + }) + } +} + +func TestSetBehaviorDefaultsConfigurableToleranceEnabled(t *testing.T) { + // Enable HPAConfigurableTolerance feature gate. + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAConfigurableTolerance, true) + + // Verify that the tolerance field is left unset. + maxPolicy := autoscalingv2.MaxChangePolicySelect + + hpa := autoscalingv2.HorizontalPodAutoscaler{ + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + Behavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{ + ScaleUp: &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: utilpointer.Int32(100), + }, + }, + }, + } + + expectedHPA := autoscalingv2.HorizontalPodAutoscaler{ + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + Behavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{ + ScaleDown: &autoscalingv2.HPAScalingRules{ + SelectPolicy: &maxPolicy, + Policies: []autoscalingv2.HPAScalingPolicy{ + {Type: autoscalingv2.PercentScalingPolicy, Value: 100, PeriodSeconds: 15}, + }, + }, + ScaleUp: &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: utilpointer.Int32(100), + SelectPolicy: &maxPolicy, + Policies: []autoscalingv2.HPAScalingPolicy{ + {Type: autoscalingv2.PodsScalingPolicy, Value: 4, PeriodSeconds: 15}, + {Type: autoscalingv2.PercentScalingPolicy, Value: 100, PeriodSeconds: 15}, + }, + }, + }, + }, + } + + SetDefaults_HorizontalPodAutoscalerBehavior(&hpa) + assert.Equal(t, expectedHPA, hpa) +} + func TestHorizontalPodAutoscalerAnnotations(t *testing.T) { tests := []struct { hpa autoscalingv2.HorizontalPodAutoscaler diff --git a/pkg/apis/autoscaling/v2beta2/conversion.go b/pkg/apis/autoscaling/v2beta2/conversion.go index 3a555e8c872..a152442605a 100644 --- a/pkg/apis/autoscaling/v2beta2/conversion.go +++ b/pkg/apis/autoscaling/v2beta2/conversion.go @@ -17,8 +17,11 @@ limitations under the License. package v2beta2 import ( + "fmt" + autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/conversion" "k8s.io/kubernetes/pkg/apis/autoscaling" ) @@ -27,8 +30,29 @@ func Convert_autoscaling_HorizontalPodAutoscaler_To_v2beta2_HorizontalPodAutosca if err := autoConvert_autoscaling_HorizontalPodAutoscaler_To_v2beta2_HorizontalPodAutoscaler(in, out, s); err != nil { return err } - // v2beta2 round-trips to internal without any serialized annotations, make sure any from other versions don't get serialized - out.Annotations, _ = autoscaling.DropRoundTripHorizontalPodAutoscalerAnnotations(out.Annotations) + // Ensure old round-trips annotations are discarded + annotations, copiedAnnotations := autoscaling.DropRoundTripHorizontalPodAutoscalerAnnotations(out.Annotations) + out.Annotations = annotations + + behavior := in.Spec.Behavior + if behavior == nil { + return nil + } + // Save the tolerance fields in annotations for round-trip + if behavior.ScaleDown != nil && behavior.ScaleDown.Tolerance != nil { + if !copiedAnnotations { + copiedAnnotations = true + out.Annotations = autoscaling.DeepCopyStringMap(out.Annotations) + } + out.Annotations[autoscaling.ToleranceScaleDownAnnotation] = behavior.ScaleDown.Tolerance.String() + } + if behavior.ScaleUp != nil && behavior.ScaleUp.Tolerance != nil { + if !copiedAnnotations { + copiedAnnotations = true + out.Annotations = autoscaling.DeepCopyStringMap(out.Annotations) + } + out.Annotations[autoscaling.ToleranceScaleUpAnnotation] = behavior.ScaleUp.Tolerance.String() + } return nil } @@ -36,7 +60,44 @@ func Convert_v2beta2_HorizontalPodAutoscaler_To_autoscaling_HorizontalPodAutosca if err := autoConvert_v2beta2_HorizontalPodAutoscaler_To_autoscaling_HorizontalPodAutoscaler(in, out, s); err != nil { return err } - // v2beta2 round-trips to internal without any serialized annotations, make sure any from other versions don't get serialized + // Restore the tolerance fields from annotations for round-trip + if tolerance, ok := out.Annotations[autoscaling.ToleranceScaleDownAnnotation]; ok { + if out.Spec.Behavior == nil { + out.Spec.Behavior = &autoscaling.HorizontalPodAutoscalerBehavior{} + } + if out.Spec.Behavior.ScaleDown == nil { + out.Spec.Behavior.ScaleDown = &autoscaling.HPAScalingRules{} + } + q, err := resource.ParseQuantity(tolerance) + if err != nil { + return fmt.Errorf("failed to parse annotation %q: %w", autoscaling.ToleranceScaleDownAnnotation, err) + } + out.Spec.Behavior.ScaleDown.Tolerance = &q + } + if tolerance, ok := out.Annotations[autoscaling.ToleranceScaleUpAnnotation]; ok { + if out.Spec.Behavior == nil { + out.Spec.Behavior = &autoscaling.HorizontalPodAutoscalerBehavior{} + } + if out.Spec.Behavior.ScaleUp == nil { + out.Spec.Behavior.ScaleUp = &autoscaling.HPAScalingRules{} + } + q, err := resource.ParseQuantity(tolerance) + if err != nil { + return fmt.Errorf("failed to parse annotation %q: %w", autoscaling.ToleranceScaleUpAnnotation, err) + } + out.Spec.Behavior.ScaleUp.Tolerance = &q + } + // Do not save round-trip annotations in internal resource out.Annotations, _ = autoscaling.DropRoundTripHorizontalPodAutoscalerAnnotations(out.Annotations) return nil } + +func Convert_v2beta2_HPAScalingRules_To_autoscaling_HPAScalingRules(in *autoscalingv2beta2.HPAScalingRules, out *autoscaling.HPAScalingRules, s conversion.Scope) error { + // Tolerance field is handled in the HorizontalPodAutoscaler conversion function. + return autoConvert_v2beta2_HPAScalingRules_To_autoscaling_HPAScalingRules(in, out, s) +} + +func Convert_autoscaling_HPAScalingRules_To_v2beta2_HPAScalingRules(in *autoscaling.HPAScalingRules, out *autoscalingv2beta2.HPAScalingRules, s conversion.Scope) error { + // Tolerance field is handled in the HorizontalPodAutoscaler conversion function. + return autoConvert_autoscaling_HPAScalingRules_To_v2beta2_HPAScalingRules(in, out, s) +} diff --git a/pkg/apis/autoscaling/v2beta2/conversion_test.go b/pkg/apis/autoscaling/v2beta2/conversion_test.go new file mode 100644 index 00000000000..faacf410fdd --- /dev/null +++ b/pkg/apis/autoscaling/v2beta2/conversion_test.go @@ -0,0 +1,124 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v2beta2 + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/resource" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/apis/autoscaling" + api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/utils/ptr" +) + +func TestConvertRoundTrip(t *testing.T) { + tolerance1 := resource.MustParse("0.1") + tolerance2 := resource.MustParse("0.2") + tests := []struct { + name string + internalHPA *autoscaling.HorizontalPodAutoscaler + }{ + { + "Complete HPA with scale-up tolerance", + &autoscaling.HorizontalPodAutoscaler{ + ObjectMeta: v1.ObjectMeta{ + Name: "hpa", + Namespace: "hpa-ns", + Annotations: map[string]string{"key": "value"}, + }, + Spec: autoscaling.HorizontalPodAutoscalerSpec{ + MinReplicas: ptr.To(int32(1)), + MaxReplicas: 3, + Metrics: []autoscaling.MetricSpec{ + { + Type: autoscaling.ResourceMetricSourceType, + Resource: &autoscaling.ResourceMetricSource{ + Name: api.ResourceCPU, + Target: autoscaling.MetricTarget{ + Type: autoscaling.AverageValueMetricType, + AverageValue: resource.NewMilliQuantity(300, resource.DecimalSI), + }, + }, + }, + }, + Behavior: &autoscaling.HorizontalPodAutoscalerBehavior{ + ScaleUp: &autoscaling.HPAScalingRules{ + Policies: []autoscaling.HPAScalingPolicy{ + { + Type: autoscaling.PodsScalingPolicy, + Value: 1, + PeriodSeconds: 2, + }, + }, + Tolerance: &tolerance1, + }, + }, + }, + }, + }, + { + "Scale-down and scale-up tolerances", + &autoscaling.HorizontalPodAutoscaler{ + Spec: autoscaling.HorizontalPodAutoscalerSpec{ + MinReplicas: ptr.To(int32(1)), + MaxReplicas: 3, + Behavior: &autoscaling.HorizontalPodAutoscalerBehavior{ + ScaleUp: &autoscaling.HPAScalingRules{ + Tolerance: &tolerance1, + }, + ScaleDown: &autoscaling.HPAScalingRules{ + Tolerance: &tolerance2, + }, + }, + }, + }, + }, + { + "Scale-down tolerance only", + &autoscaling.HorizontalPodAutoscaler{ + Spec: autoscaling.HorizontalPodAutoscalerSpec{ + MinReplicas: ptr.To(int32(1)), + MaxReplicas: 3, + Behavior: &autoscaling.HorizontalPodAutoscalerBehavior{ + ScaleDown: &autoscaling.HPAScalingRules{ + Tolerance: &tolerance2, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v2beta2HPA := &autoscalingv2beta2.HorizontalPodAutoscaler{} + if err := Convert_autoscaling_HorizontalPodAutoscaler_To_v2beta2_HorizontalPodAutoscaler(tt.internalHPA, v2beta2HPA, nil); err != nil { + t.Errorf("Convert_autoscaling_HorizontalPodAutoscaler_To_v2beta2_HorizontalPodAutoscaler() error = %v", err) + } + roundtripHPA := &autoscaling.HorizontalPodAutoscaler{} + if err := Convert_v2beta2_HorizontalPodAutoscaler_To_autoscaling_HorizontalPodAutoscaler(v2beta2HPA, roundtripHPA, nil); err != nil { + t.Errorf("Convert_v2beta2_HorizontalPodAutoscaler_To_autoscaling_HorizontalPodAutoscaler() error = %v", err) + } + if !apiequality.Semantic.DeepEqual(tt.internalHPA, roundtripHPA) { + t.Errorf("HPA is not equivalent after round-trip: mismatch (-want +got):\n%s", cmp.Diff(tt.internalHPA, roundtripHPA)) + } + }) + } +} diff --git a/pkg/apis/autoscaling/v2beta2/zz_generated.conversion.go b/pkg/apis/autoscaling/v2beta2/zz_generated.conversion.go index e567f74cc14..7f544a0864f 100644 --- a/pkg/apis/autoscaling/v2beta2/zz_generated.conversion.go +++ b/pkg/apis/autoscaling/v2beta2/zz_generated.conversion.go @@ -101,16 +101,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*autoscalingv2beta2.HPAScalingRules)(nil), (*autoscaling.HPAScalingRules)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v2beta2_HPAScalingRules_To_autoscaling_HPAScalingRules(a.(*autoscalingv2beta2.HPAScalingRules), b.(*autoscaling.HPAScalingRules), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*autoscaling.HPAScalingRules)(nil), (*autoscalingv2beta2.HPAScalingRules)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_autoscaling_HPAScalingRules_To_v2beta2_HPAScalingRules(a.(*autoscaling.HPAScalingRules), b.(*autoscalingv2beta2.HPAScalingRules), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*autoscalingv2beta2.HorizontalPodAutoscalerBehavior)(nil), (*autoscaling.HorizontalPodAutoscalerBehavior)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v2beta2_HorizontalPodAutoscalerBehavior_To_autoscaling_HorizontalPodAutoscalerBehavior(a.(*autoscalingv2beta2.HorizontalPodAutoscalerBehavior), b.(*autoscaling.HorizontalPodAutoscalerBehavior), scope) }); err != nil { @@ -271,11 +261,21 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*autoscaling.HPAScalingRules)(nil), (*autoscalingv2beta2.HPAScalingRules)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_autoscaling_HPAScalingRules_To_v2beta2_HPAScalingRules(a.(*autoscaling.HPAScalingRules), b.(*autoscalingv2beta2.HPAScalingRules), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*autoscaling.HorizontalPodAutoscaler)(nil), (*autoscalingv2beta2.HorizontalPodAutoscaler)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_autoscaling_HorizontalPodAutoscaler_To_v2beta2_HorizontalPodAutoscaler(a.(*autoscaling.HorizontalPodAutoscaler), b.(*autoscalingv2beta2.HorizontalPodAutoscaler), scope) }); err != nil { return err } + if err := s.AddConversionFunc((*autoscalingv2beta2.HPAScalingRules)(nil), (*autoscaling.HPAScalingRules)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v2beta2_HPAScalingRules_To_autoscaling_HPAScalingRules(a.(*autoscalingv2beta2.HPAScalingRules), b.(*autoscaling.HPAScalingRules), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*autoscalingv2beta2.HorizontalPodAutoscaler)(nil), (*autoscaling.HorizontalPodAutoscaler)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v2beta2_HorizontalPodAutoscaler_To_autoscaling_HorizontalPodAutoscaler(a.(*autoscalingv2beta2.HorizontalPodAutoscaler), b.(*autoscaling.HorizontalPodAutoscaler), scope) }); err != nil { @@ -455,23 +455,14 @@ func autoConvert_v2beta2_HPAScalingRules_To_autoscaling_HPAScalingRules(in *auto return nil } -// Convert_v2beta2_HPAScalingRules_To_autoscaling_HPAScalingRules is an autogenerated conversion function. -func Convert_v2beta2_HPAScalingRules_To_autoscaling_HPAScalingRules(in *autoscalingv2beta2.HPAScalingRules, out *autoscaling.HPAScalingRules, s conversion.Scope) error { - return autoConvert_v2beta2_HPAScalingRules_To_autoscaling_HPAScalingRules(in, out, s) -} - func autoConvert_autoscaling_HPAScalingRules_To_v2beta2_HPAScalingRules(in *autoscaling.HPAScalingRules, out *autoscalingv2beta2.HPAScalingRules, s conversion.Scope) error { out.StabilizationWindowSeconds = (*int32)(unsafe.Pointer(in.StabilizationWindowSeconds)) out.SelectPolicy = (*autoscalingv2beta2.ScalingPolicySelect)(unsafe.Pointer(in.SelectPolicy)) out.Policies = *(*[]autoscalingv2beta2.HPAScalingPolicy)(unsafe.Pointer(&in.Policies)) + // WARNING: in.Tolerance requires manual conversion: does not exist in peer-type return nil } -// Convert_autoscaling_HPAScalingRules_To_v2beta2_HPAScalingRules is an autogenerated conversion function. -func Convert_autoscaling_HPAScalingRules_To_v2beta2_HPAScalingRules(in *autoscaling.HPAScalingRules, out *autoscalingv2beta2.HPAScalingRules, s conversion.Scope) error { - return autoConvert_autoscaling_HPAScalingRules_To_v2beta2_HPAScalingRules(in, out, s) -} - func autoConvert_v2beta2_HorizontalPodAutoscaler_To_autoscaling_HorizontalPodAutoscaler(in *autoscalingv2beta2.HorizontalPodAutoscaler, out *autoscaling.HorizontalPodAutoscaler, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v2beta2_HorizontalPodAutoscalerSpec_To_autoscaling_HorizontalPodAutoscalerSpec(&in.Spec, &out.Spec, s); err != nil { @@ -495,8 +486,24 @@ func autoConvert_autoscaling_HorizontalPodAutoscaler_To_v2beta2_HorizontalPodAut } func autoConvert_v2beta2_HorizontalPodAutoscalerBehavior_To_autoscaling_HorizontalPodAutoscalerBehavior(in *autoscalingv2beta2.HorizontalPodAutoscalerBehavior, out *autoscaling.HorizontalPodAutoscalerBehavior, s conversion.Scope) error { - out.ScaleUp = (*autoscaling.HPAScalingRules)(unsafe.Pointer(in.ScaleUp)) - out.ScaleDown = (*autoscaling.HPAScalingRules)(unsafe.Pointer(in.ScaleDown)) + if in.ScaleUp != nil { + in, out := &in.ScaleUp, &out.ScaleUp + *out = new(autoscaling.HPAScalingRules) + if err := Convert_v2beta2_HPAScalingRules_To_autoscaling_HPAScalingRules(*in, *out, s); err != nil { + return err + } + } else { + out.ScaleUp = nil + } + if in.ScaleDown != nil { + in, out := &in.ScaleDown, &out.ScaleDown + *out = new(autoscaling.HPAScalingRules) + if err := Convert_v2beta2_HPAScalingRules_To_autoscaling_HPAScalingRules(*in, *out, s); err != nil { + return err + } + } else { + out.ScaleDown = nil + } return nil } @@ -506,8 +513,24 @@ func Convert_v2beta2_HorizontalPodAutoscalerBehavior_To_autoscaling_HorizontalPo } func autoConvert_autoscaling_HorizontalPodAutoscalerBehavior_To_v2beta2_HorizontalPodAutoscalerBehavior(in *autoscaling.HorizontalPodAutoscalerBehavior, out *autoscalingv2beta2.HorizontalPodAutoscalerBehavior, s conversion.Scope) error { - out.ScaleUp = (*autoscalingv2beta2.HPAScalingRules)(unsafe.Pointer(in.ScaleUp)) - out.ScaleDown = (*autoscalingv2beta2.HPAScalingRules)(unsafe.Pointer(in.ScaleDown)) + if in.ScaleUp != nil { + in, out := &in.ScaleUp, &out.ScaleUp + *out = new(autoscalingv2beta2.HPAScalingRules) + if err := Convert_autoscaling_HPAScalingRules_To_v2beta2_HPAScalingRules(*in, *out, s); err != nil { + return err + } + } else { + out.ScaleUp = nil + } + if in.ScaleDown != nil { + in, out := &in.ScaleDown, &out.ScaleDown + *out = new(autoscalingv2beta2.HPAScalingRules) + if err := Convert_autoscaling_HPAScalingRules_To_v2beta2_HPAScalingRules(*in, *out, s); err != nil { + return err + } + } else { + out.ScaleDown = nil + } return nil } @@ -603,7 +626,15 @@ func autoConvert_v2beta2_HorizontalPodAutoscalerSpec_To_autoscaling_HorizontalPo } else { out.Metrics = nil } - out.Behavior = (*autoscaling.HorizontalPodAutoscalerBehavior)(unsafe.Pointer(in.Behavior)) + if in.Behavior != nil { + in, out := &in.Behavior, &out.Behavior + *out = new(autoscaling.HorizontalPodAutoscalerBehavior) + if err := Convert_v2beta2_HorizontalPodAutoscalerBehavior_To_autoscaling_HorizontalPodAutoscalerBehavior(*in, *out, s); err != nil { + return err + } + } else { + out.Behavior = nil + } return nil } @@ -629,7 +660,15 @@ func autoConvert_autoscaling_HorizontalPodAutoscalerSpec_To_v2beta2_HorizontalPo } else { out.Metrics = nil } - out.Behavior = (*autoscalingv2beta2.HorizontalPodAutoscalerBehavior)(unsafe.Pointer(in.Behavior)) + if in.Behavior != nil { + in, out := &in.Behavior, &out.Behavior + *out = new(autoscalingv2beta2.HorizontalPodAutoscalerBehavior) + if err := Convert_autoscaling_HorizontalPodAutoscalerBehavior_To_v2beta2_HorizontalPodAutoscalerBehavior(*in, *out, s); err != nil { + return err + } + } else { + out.Behavior = nil + } return nil } diff --git a/pkg/apis/autoscaling/validation/validation.go b/pkg/apis/autoscaling/validation/validation.go index bd986816cfd..8b5ac06c442 100644 --- a/pkg/apis/autoscaling/validation/validation.go +++ b/pkg/apis/autoscaling/validation/validation.go @@ -53,12 +53,12 @@ func ValidateScale(scale *autoscaling.Scale) field.ErrorList { // Prefix indicates this name will be used as part of generation, in which case trailing dashes are allowed. var ValidateHorizontalPodAutoscalerName = apivalidation.ValidateReplicationControllerName -func validateHorizontalPodAutoscalerSpec(autoscaler autoscaling.HorizontalPodAutoscalerSpec, fldPath *field.Path, minReplicasLowerBound int32) field.ErrorList { +func validateHorizontalPodAutoscalerSpec(autoscaler autoscaling.HorizontalPodAutoscalerSpec, fldPath *field.Path, opts HorizontalPodAutoscalerSpecValidationOptions) field.ErrorList { allErrs := field.ErrorList{} - if autoscaler.MinReplicas != nil && *autoscaler.MinReplicas < minReplicasLowerBound { + if autoscaler.MinReplicas != nil && *autoscaler.MinReplicas < opts.MinReplicasLowerBound { allErrs = append(allErrs, field.Invalid(fldPath.Child("minReplicas"), *autoscaler.MinReplicas, - fmt.Sprintf("must be greater than or equal to %d", minReplicasLowerBound))) + fmt.Sprintf("must be greater than or equal to %d", opts.MinReplicasLowerBound))) } if autoscaler.MaxReplicas < 1 { allErrs = append(allErrs, field.Invalid(fldPath.Child("maxReplicas"), autoscaler.MaxReplicas, "must be greater than 0")) @@ -72,7 +72,7 @@ func validateHorizontalPodAutoscalerSpec(autoscaler autoscaling.HorizontalPodAut if refErrs := validateMetrics(autoscaler.Metrics, fldPath.Child("metrics"), autoscaler.MinReplicas); len(refErrs) > 0 { allErrs = append(allErrs, refErrs...) } - if refErrs := validateBehavior(autoscaler.Behavior, fldPath.Child("behavior")); len(refErrs) > 0 { + if refErrs := validateBehavior(autoscaler.Behavior, fldPath.Child("behavior"), opts); len(refErrs) > 0 { allErrs = append(allErrs, refErrs...) } return allErrs @@ -115,7 +115,13 @@ func ValidateHorizontalPodAutoscaler(autoscaler *autoscaling.HorizontalPodAutosc } else { minReplicasLowerBound = 1 } - allErrs = append(allErrs, validateHorizontalPodAutoscalerSpec(autoscaler.Spec, field.NewPath("spec"), minReplicasLowerBound)...) + + opts := HorizontalPodAutoscalerSpecValidationOptions{ + AllowTolerance: utilfeature.DefaultMutableFeatureGate.Enabled(features.HPAConfigurableTolerance), + MinReplicasLowerBound: minReplicasLowerBound, + } + + allErrs = append(allErrs, validateHorizontalPodAutoscalerSpec(autoscaler.Spec, field.NewPath("spec"), opts)...) return allErrs } @@ -134,7 +140,12 @@ func ValidateHorizontalPodAutoscalerUpdate(newAutoscaler, oldAutoscaler *autosca minReplicasLowerBound = 1 } - allErrs = append(allErrs, validateHorizontalPodAutoscalerSpec(newAutoscaler.Spec, field.NewPath("spec"), minReplicasLowerBound)...) + opts := HorizontalPodAutoscalerSpecValidationOptions{ + AllowTolerance: utilfeature.DefaultMutableFeatureGate.Enabled(features.HPAConfigurableTolerance), + MinReplicasLowerBound: minReplicasLowerBound, + } + + allErrs = append(allErrs, validateHorizontalPodAutoscalerSpec(newAutoscaler.Spec, field.NewPath("spec"), opts)...) return allErrs } @@ -148,6 +159,15 @@ func ValidateHorizontalPodAutoscalerStatusUpdate(newAutoscaler, oldAutoscaler *a return allErrs } +// HorizontalPodAutoscalerSpecValidationOptions contains the different settings for +// HorizontalPodAutoscaler spec validation. +type HorizontalPodAutoscalerSpecValidationOptions struct { + // Allow setting a tolerance on HPAScalingRules. + AllowTolerance bool + // The minimum value for minReplicas. + MinReplicasLowerBound int32 +} + func validateMetrics(metrics []autoscaling.MetricSpec, fldPath *field.Path, minReplicas *int32) field.ErrorList { allErrs := field.ErrorList{} hasObjectMetrics := false @@ -175,13 +195,13 @@ func validateMetrics(metrics []autoscaling.MetricSpec, fldPath *field.Path, minR return allErrs } -func validateBehavior(behavior *autoscaling.HorizontalPodAutoscalerBehavior, fldPath *field.Path) field.ErrorList { +func validateBehavior(behavior *autoscaling.HorizontalPodAutoscalerBehavior, fldPath *field.Path, opts HorizontalPodAutoscalerSpecValidationOptions) field.ErrorList { allErrs := field.ErrorList{} if behavior != nil { - if scaleUpErrs := validateScalingRules(behavior.ScaleUp, fldPath.Child("scaleUp")); len(scaleUpErrs) > 0 { + if scaleUpErrs := validateScalingRules(behavior.ScaleUp, fldPath.Child("scaleUp"), opts); len(scaleUpErrs) > 0 { allErrs = append(allErrs, scaleUpErrs...) } - if scaleDownErrs := validateScalingRules(behavior.ScaleDown, fldPath.Child("scaleDown")); len(scaleDownErrs) > 0 { + if scaleDownErrs := validateScalingRules(behavior.ScaleDown, fldPath.Child("scaleDown"), opts); len(scaleDownErrs) > 0 { allErrs = append(allErrs, scaleDownErrs...) } } @@ -191,7 +211,7 @@ func validateBehavior(behavior *autoscaling.HorizontalPodAutoscalerBehavior, fld var validSelectPolicyTypes = sets.NewString(string(autoscaling.MaxPolicySelect), string(autoscaling.MinPolicySelect), string(autoscaling.DisabledPolicySelect)) var validSelectPolicyTypesList = validSelectPolicyTypes.List() -func validateScalingRules(rules *autoscaling.HPAScalingRules, fldPath *field.Path) field.ErrorList { +func validateScalingRules(rules *autoscaling.HPAScalingRules, fldPath *field.Path, opts HorizontalPodAutoscalerSpecValidationOptions) field.ErrorList { allErrs := field.ErrorList{} if rules != nil { if rules.StabilizationWindowSeconds != nil && *rules.StabilizationWindowSeconds < 0 { @@ -214,6 +234,13 @@ func validateScalingRules(rules *autoscaling.HPAScalingRules, fldPath *field.Pat allErrs = append(allErrs, policyErrs...) } } + + if rules.Tolerance != nil && !opts.AllowTolerance { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("tolerance"), "tolerance is not supported when the HPAConfigurableTolerance feature gate is not enabled")) + } + if rules.Tolerance != nil && rules.Tolerance.Sign() < 0 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("tolerance"), rules.Tolerance, "must be greater or equal to zero")) + } } return allErrs } diff --git a/pkg/apis/autoscaling/validation/validation_test.go b/pkg/apis/autoscaling/validation/validation_test.go index 443ee39199c..003e97c1edc 100644 --- a/pkg/apis/autoscaling/validation/validation_test.go +++ b/pkg/apis/autoscaling/validation/validation_test.go @@ -28,6 +28,7 @@ import ( api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/features" utilpointer "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ) func TestValidateScale(t *testing.T) { @@ -367,8 +368,9 @@ func TestValidateBehavior(t *testing.T) { func prepareHPAWithBehavior(b autoscaling.HorizontalPodAutoscalerBehavior) autoscaling.HorizontalPodAutoscaler { return autoscaling.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: "myautoscaler", - Namespace: metav1.NamespaceDefault, + Name: "myautoscaler", + Namespace: metav1.NamespaceDefault, + ResourceVersion: "1", }, Spec: autoscaling.HorizontalPodAutoscalerSpec{ ScaleTargetRef: autoscaling.CrossVersionObjectReference{ @@ -1652,3 +1654,236 @@ func TestValidateHorizontalPodAutoscalerScaleToZeroUpdateDisabled(t *testing.T) } } } + +func TestValidateHorizontalPodAutoscalerConfigurableToleranceEnabled(t *testing.T) { + // Enable HPAConfigurableTolerance feature gate. + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAConfigurableTolerance, true) + + policiesList := []autoscaling.HPAScalingPolicy{{ + Type: autoscaling.PodsScalingPolicy, + Value: 1, + PeriodSeconds: 1800, + }} + + successCases := []autoscaling.HPAScalingRules{ + { + Policies: policiesList, + Tolerance: ptr.To(resource.MustParse("0.1")), + }, + { + Policies: policiesList, + Tolerance: ptr.To(resource.MustParse("10")), + }, + { + Policies: policiesList, + Tolerance: ptr.To(resource.MustParse("0")), + }, + { + Policies: policiesList, + Tolerance: resource.NewMilliQuantity(100, resource.DecimalSI), + }, + { + Policies: policiesList, + Tolerance: resource.NewScaledQuantity(1, resource.Milli), + }, + { + Policies: policiesList, + }, + } + for _, c := range successCases { + b := autoscaling.HorizontalPodAutoscalerBehavior{ + ScaleDown: &c, + } + hpa := prepareHPAWithBehavior(b) + if errs := ValidateHorizontalPodAutoscaler(&hpa); len(errs) != 0 { + t.Errorf("expected success: %v", errs) + } + } + + failureCases := []struct { + rule autoscaling.HPAScalingRules + msg string + }{ + { + rule: autoscaling.HPAScalingRules{}, + msg: "at least one Policy", + }, + { + rule: autoscaling.HPAScalingRules{ + Policies: policiesList, + Tolerance: ptr.To(resource.MustParse("-0.001")), + }, + msg: "greater or equal to zero", + }, + { + rule: autoscaling.HPAScalingRules{ + Policies: policiesList, + Tolerance: resource.NewMilliQuantity(-10, resource.DecimalSI), + }, + msg: "greater or equal to zero", + }, + { + rule: autoscaling.HPAScalingRules{ + StabilizationWindowSeconds: utilpointer.Int32(60), + }, + msg: "at least one Policy", + }, + { + rule: autoscaling.HPAScalingRules{ + Tolerance: resource.NewMilliQuantity(1, resource.DecimalSI), + StabilizationWindowSeconds: utilpointer.Int32(60), + }, + msg: "at least one Policy", + }, + } + for _, c := range failureCases { + b := autoscaling.HorizontalPodAutoscalerBehavior{ + ScaleUp: &c.rule, + } + hpa := prepareHPAWithBehavior(b) + errs := ValidateHorizontalPodAutoscaler(&hpa) + if len(errs) != 1 { + t.Fatalf("expected exactly one error, got: %v", errs) + } + if !strings.Contains(errs[0].Error(), c.msg) { + t.Errorf("unexpected error: %q, expected: %q", errs[0], c.msg) + } + } +} + +func TestValidateHorizontalPodAutoscalerConfigurableToleranceDisabled(t *testing.T) { + // Disable HPAConfigurableTolerance feature gate. + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAConfigurableTolerance, false) + + maxPolicy := autoscaling.MaxPolicySelect + policiesList := []autoscaling.HPAScalingPolicy{{ + Type: autoscaling.PodsScalingPolicy, + Value: 1, + PeriodSeconds: 1800, + }} + + successCases := []autoscaling.HPAScalingRules{ + { + Policies: policiesList, + }, + { + SelectPolicy: &maxPolicy, + Policies: policiesList, + }, + { + StabilizationWindowSeconds: utilpointer.Int32(60), + Policies: policiesList, + }, + } + for _, c := range successCases { + b := autoscaling.HorizontalPodAutoscalerBehavior{ + ScaleDown: &c, + } + hpa := prepareHPAWithBehavior(b) + if errs := ValidateHorizontalPodAutoscaler(&hpa); len(errs) != 0 { + t.Errorf("expected success: %v", errs) + } + } + + failureCases := []struct { + rule autoscaling.HPAScalingRules + msg string + }{ + { + rule: autoscaling.HPAScalingRules{ + Policies: policiesList, + Tolerance: resource.NewMilliQuantity(1, resource.DecimalSI), + }, + msg: "not supported", + }, + { + rule: autoscaling.HPAScalingRules{}, + msg: "at least one Policy", + }, + { + rule: autoscaling.HPAScalingRules{ + StabilizationWindowSeconds: utilpointer.Int32(60), + }, + msg: "at least one Policy", + }, + } + for _, c := range failureCases { + b := autoscaling.HorizontalPodAutoscalerBehavior{ + ScaleUp: &c.rule, + } + hpa := prepareHPAWithBehavior(b) + errs := ValidateHorizontalPodAutoscaler(&hpa) + if len(errs) != 1 { + t.Fatalf("expected exactly one error, got: %v", errs) + } + if !strings.Contains(errs[0].Error(), c.msg) { + t.Errorf("unexpected error: %q, expected: %q", errs[0], c.msg) + } + } +} + +func TestValidateHorizontalPodAutoscalerUpdateConfigurableToleranceEnabled(t *testing.T) { + // Enable HPAConfigurableTolerance feature gate. + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAConfigurableTolerance, true) + + policiesList := []autoscaling.HPAScalingPolicy{{ + Type: autoscaling.PodsScalingPolicy, + Value: 1, + PeriodSeconds: 1800, + }} + + withToleranceHPA := prepareHPAWithBehavior(autoscaling.HorizontalPodAutoscalerBehavior{ + ScaleUp: &autoscaling.HPAScalingRules{ + Policies: policiesList, + Tolerance: resource.NewMilliQuantity(10, resource.DecimalSI), + }}) + withoutToleranceHPA := prepareHPAWithBehavior(autoscaling.HorizontalPodAutoscalerBehavior{ + ScaleUp: &autoscaling.HPAScalingRules{ + Policies: policiesList, + }}) + + if errs := ValidateHorizontalPodAutoscalerUpdate(&withToleranceHPA, &withoutToleranceHPA); len(errs) != 0 { + t.Errorf("expected success: %v", errs) + } + + if errs := ValidateHorizontalPodAutoscalerUpdate(&withoutToleranceHPA, &withToleranceHPA); len(errs) != 0 { + t.Errorf("expected success: %v", errs) + } +} + +func TestValidateHorizontalPodAutoscalerConfigurableToleranceUpdateDisabled(t *testing.T) { + // Disable HPAConfigurableTolerance feature gate. + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAConfigurableTolerance, false) + + policiesList := []autoscaling.HPAScalingPolicy{{ + Type: autoscaling.PodsScalingPolicy, + Value: 1, + PeriodSeconds: 1800, + }} + + withToleranceHPA := prepareHPAWithBehavior(autoscaling.HorizontalPodAutoscalerBehavior{ + ScaleUp: &autoscaling.HPAScalingRules{ + Policies: policiesList, + Tolerance: resource.NewMilliQuantity(10, resource.DecimalSI), + }}) + withoutToleranceHPA := prepareHPAWithBehavior(autoscaling.HorizontalPodAutoscalerBehavior{ + ScaleUp: &autoscaling.HPAScalingRules{ + Policies: policiesList, + }}) + notSupportedErrorMsg := "not supported" + + errs := ValidateHorizontalPodAutoscalerUpdate(&withToleranceHPA, &withoutToleranceHPA) + if len(errs) == 0 { + t.Errorf("expected failure for %q", notSupportedErrorMsg) + } else if !strings.Contains(errs[0].Error(), notSupportedErrorMsg) { + t.Errorf("unexpected error: %q, expected: %q", errs[0], notSupportedErrorMsg) + } + + if errs := ValidateHorizontalPodAutoscalerUpdate(&withoutToleranceHPA, &withoutToleranceHPA); len(errs) != 0 { + t.Errorf("expected success: %v", errs) + } + + if errs := ValidateHorizontalPodAutoscalerUpdate(&withoutToleranceHPA, &withToleranceHPA); len(errs) != 0 { + t.Errorf("expected success: %v", errs) + } +} diff --git a/staging/src/k8s.io/api/autoscaling/v2/types.go b/staging/src/k8s.io/api/autoscaling/v2/types.go index 99e8db09dc8..e6b9a5fc205 100644 --- a/staging/src/k8s.io/api/autoscaling/v2/types.go +++ b/staging/src/k8s.io/api/autoscaling/v2/types.go @@ -171,12 +171,18 @@ const ( DisabledPolicySelect ScalingPolicySelect = "Disabled" ) -// HPAScalingRules configures the scaling behavior for one direction. -// These Rules are applied after calculating DesiredReplicas from metrics for the HPA. +// HPAScalingRules configures the scaling behavior for one direction via +// scaling Policy Rules and a configurable metric tolerance. +// +// Scaling Policy Rules are applied after calculating DesiredReplicas from metrics for the HPA. // They can limit the scaling velocity by specifying scaling policies. // They can prevent flapping by specifying the stabilization window, so that the // number of replicas is not set instantly, instead, the safest value from the stabilization // window is chosen. +// +// The tolerance is applied to the metric values and prevents scaling too +// eagerly for small metric variations. (Note that setting a tolerance requires +// enabling the alpha HPAConfigurableTolerance feature gate.) type HPAScalingRules struct { // stabilizationWindowSeconds is the number of seconds for which past recommendations should be // considered while scaling up or scaling down. @@ -193,10 +199,24 @@ type HPAScalingRules struct { SelectPolicy *ScalingPolicySelect `json:"selectPolicy,omitempty" protobuf:"bytes,1,opt,name=selectPolicy"` // policies is a list of potential scaling polices which can be used during scaling. - // At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + // If not set, use the default values: + // - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + // - For scale down: allow all pods to be removed in a 15s window. // +listType=atomic // +optional Policies []HPAScalingPolicy `json:"policies,omitempty" listType:"atomic" protobuf:"bytes,2,rep,name=policies"` + + // tolerance is the tolerance on the ratio between the current and desired + // metric value under which no updates are made to the desired number of + // replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + // set, the default cluster-wide tolerance is applied (by default 10%). + // + // This is an alpha field and requires enabling the HPAConfigurableTolerance + // feature gate. + // + // +featureGate=HPAConfigurableTolerance + // +optional + Tolerance *resource.Quantity `json:"tolerance,omitempty" protobuf:"bytes,4,opt,name=tolerance"` } // HPAScalingPolicyType is the type of the policy which could be used while making scaling decisions. From 11b6e2a3752adf92a328b2a5b55c467644e6639a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Marc=20Fran=C3=A7ois?= Date: Fri, 21 Mar 2025 16:48:30 -0400 Subject: [PATCH 3/7] Update test fixture to account for new tolerance field. --- ...utoscaling.v2.HorizontalPodAutoscaler.json | 6 ++++-- .../autoscaling.v2.HorizontalPodAutoscaler.pb | Bin 1570 -> 1580 bytes ...utoscaling.v2.HorizontalPodAutoscaler.yaml | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/staging/src/k8s.io/api/testdata/HEAD/autoscaling.v2.HorizontalPodAutoscaler.json b/staging/src/k8s.io/api/testdata/HEAD/autoscaling.v2.HorizontalPodAutoscaler.json index f03e36623e8..4bc9fceaf70 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/autoscaling.v2.HorizontalPodAutoscaler.json +++ b/staging/src/k8s.io/api/testdata/HEAD/autoscaling.v2.HorizontalPodAutoscaler.json @@ -165,7 +165,8 @@ "value": 2, "periodSeconds": 3 } - ] + ], + "tolerance": "0" }, "scaleDown": { "stabilizationWindowSeconds": 3, @@ -176,7 +177,8 @@ "value": 2, "periodSeconds": 3 } - ] + ], + "tolerance": "0" } } }, diff --git a/staging/src/k8s.io/api/testdata/HEAD/autoscaling.v2.HorizontalPodAutoscaler.pb b/staging/src/k8s.io/api/testdata/HEAD/autoscaling.v2.HorizontalPodAutoscaler.pb index 3d1507519883ea000ab7d4157647b5a5e6af632d..0d57f89ae3c6564f0c16a2cbaadbf3f25542218d 100644 GIT binary patch delta 56 zcmZ3)vxaAa0pqughN+Cq|5>;u&t}ose30=sBXbm&_T>G{YV1nPT#N=nU^Xk5y_uIq GlnDU!)efxy delta 39 scmZ3(vxsMc0ppvEhN+B Date: Fri, 21 Mar 2025 16:57:31 -0400 Subject: [PATCH 4/7] Add configurable tolerance logic. --- pkg/controller/podautoscaler/horizontal.go | 50 ++++- .../podautoscaler/horizontal_test.go | 104 +++++++++ .../podautoscaler/replica_calculator.go | 58 +++-- .../podautoscaler/replica_calculator_test.go | 209 +++++++++++++++++- 4 files changed, 377 insertions(+), 44 deletions(-) diff --git a/pkg/controller/podautoscaler/horizontal.go b/pkg/controller/podautoscaler/horizontal.go index 57bea5e1372..67f78b4b1a8 100644 --- a/pkg/controller/podautoscaler/horizontal.go +++ b/pkg/controller/podautoscaler/horizontal.go @@ -37,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" autoscalinginformers "k8s.io/client-go/informers/autoscaling/v2" coreinformers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes/scheme" @@ -53,6 +54,7 @@ import ( metricsclient "k8s.io/kubernetes/pkg/controller/podautoscaler/metrics" "k8s.io/kubernetes/pkg/controller/podautoscaler/monitor" "k8s.io/kubernetes/pkg/controller/util/selectors" + "k8s.io/kubernetes/pkg/features" ) var ( @@ -86,6 +88,7 @@ type HorizontalController struct { hpaNamespacer autoscalingclient.HorizontalPodAutoscalersGetter mapper apimeta.RESTMapper + tolerance float64 replicaCalc *ReplicaCalculator eventRecorder record.EventRecorder @@ -146,6 +149,7 @@ func NewHorizontalController( eventRecorder: recorder, scaleNamespacer: scaleNamespacer, hpaNamespacer: hpaNamespacer, + tolerance: tolerance, downscaleStabilisationWindow: downscaleStabilisationWindow, monitor: monitor.New(), queue: workqueue.NewTypedRateLimitingQueueWithConfig( @@ -181,7 +185,6 @@ func NewHorizontalController( replicaCalc := NewReplicaCalculator( metricsClient, hpaController.podLister, - tolerance, cpuInitializationPeriod, delayOfInitialReadinessStatus, ) @@ -539,8 +542,9 @@ func (a *HorizontalController) computeStatusForObjectMetric(specReplicas, status }, }, } + tolerances := a.tolerancesForHpa(hpa) if metricSpec.Object.Target.Type == autoscalingv2.ValueMetricType && metricSpec.Object.Target.Value != nil { - replicaCountProposal, usageProposal, timestampProposal, err := a.replicaCalc.GetObjectMetricReplicas(specReplicas, metricSpec.Object.Target.Value.MilliValue(), metricSpec.Object.Metric.Name, hpa.Namespace, &metricSpec.Object.DescribedObject, selector, metricSelector) + replicaCountProposal, usageProposal, timestampProposal, err := a.replicaCalc.GetObjectMetricReplicas(specReplicas, metricSpec.Object.Target.Value.MilliValue(), metricSpec.Object.Metric.Name, tolerances, hpa.Namespace, &metricSpec.Object.DescribedObject, selector, metricSelector) if err != nil { condition := a.getUnableComputeReplicaCountCondition(hpa, "FailedGetObjectMetric", err) return 0, timestampProposal, "", condition, err @@ -549,7 +553,7 @@ func (a *HorizontalController) computeStatusForObjectMetric(specReplicas, status *status = metricStatus return replicaCountProposal, timestampProposal, fmt.Sprintf("%s metric %s", metricSpec.Object.DescribedObject.Kind, metricSpec.Object.Metric.Name), autoscalingv2.HorizontalPodAutoscalerCondition{}, nil } else if metricSpec.Object.Target.Type == autoscalingv2.AverageValueMetricType && metricSpec.Object.Target.AverageValue != nil { - replicaCountProposal, usageProposal, timestampProposal, err := a.replicaCalc.GetObjectPerPodMetricReplicas(statusReplicas, metricSpec.Object.Target.AverageValue.MilliValue(), metricSpec.Object.Metric.Name, hpa.Namespace, &metricSpec.Object.DescribedObject, metricSelector) + replicaCountProposal, usageProposal, timestampProposal, err := a.replicaCalc.GetObjectPerPodMetricReplicas(statusReplicas, metricSpec.Object.Target.AverageValue.MilliValue(), metricSpec.Object.Metric.Name, tolerances, hpa.Namespace, &metricSpec.Object.DescribedObject, metricSelector) if err != nil { condition := a.getUnableComputeReplicaCountCondition(hpa, "FailedGetObjectMetric", err) return 0, time.Time{}, "", condition, fmt.Errorf("failed to get %s object metric: %v", metricSpec.Object.Metric.Name, err) @@ -566,7 +570,8 @@ func (a *HorizontalController) computeStatusForObjectMetric(specReplicas, status // computeStatusForPodsMetric computes the desired number of replicas for the specified metric of type PodsMetricSourceType. func (a *HorizontalController) computeStatusForPodsMetric(currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus, metricSelector labels.Selector) (replicaCountProposal int32, timestampProposal time.Time, metricNameProposal string, condition autoscalingv2.HorizontalPodAutoscalerCondition, err error) { - replicaCountProposal, usageProposal, timestampProposal, err := a.replicaCalc.GetMetricReplicas(currentReplicas, metricSpec.Pods.Target.AverageValue.MilliValue(), metricSpec.Pods.Metric.Name, hpa.Namespace, selector, metricSelector) + tolerances := a.tolerancesForHpa(hpa) + replicaCountProposal, usageProposal, timestampProposal, err := a.replicaCalc.GetMetricReplicas(currentReplicas, metricSpec.Pods.Target.AverageValue.MilliValue(), metricSpec.Pods.Metric.Name, tolerances, hpa.Namespace, selector, metricSelector) if err != nil { condition = a.getUnableComputeReplicaCountCondition(hpa, "FailedGetPodsMetric", err) return 0, timestampProposal, "", condition, err @@ -588,12 +593,14 @@ func (a *HorizontalController) computeStatusForPodsMetric(currentReplicas int32, } func (a *HorizontalController) computeStatusForResourceMetricGeneric(ctx context.Context, currentReplicas int32, target autoscalingv2.MetricTarget, - resourceName v1.ResourceName, namespace string, container string, selector labels.Selector, sourceType autoscalingv2.MetricSourceType) (replicaCountProposal int32, + resourceName v1.ResourceName, hpa *autoscalingv2.HorizontalPodAutoscaler, container string, selector labels.Selector, sourceType autoscalingv2.MetricSourceType) (replicaCountProposal int32, metricStatus *autoscalingv2.MetricValueStatus, timestampProposal time.Time, metricNameProposal string, condition autoscalingv2.HorizontalPodAutoscalerCondition, err error) { + namespace := hpa.Namespace + tolerances := a.tolerancesForHpa(hpa) if target.AverageValue != nil { var rawProposal int64 - replicaCountProposal, rawProposal, timestampProposal, err := a.replicaCalc.GetRawResourceReplicas(ctx, currentReplicas, target.AverageValue.MilliValue(), resourceName, namespace, selector, container) + replicaCountProposal, rawProposal, timestampProposal, err := a.replicaCalc.GetRawResourceReplicas(ctx, currentReplicas, target.AverageValue.MilliValue(), resourceName, tolerances, namespace, selector, container) if err != nil { return 0, nil, time.Time{}, "", condition, fmt.Errorf("failed to get %s usage: %v", resourceName, err) } @@ -610,7 +617,7 @@ func (a *HorizontalController) computeStatusForResourceMetricGeneric(ctx context } targetUtilization := *target.AverageUtilization - replicaCountProposal, percentageProposal, rawProposal, timestampProposal, err := a.replicaCalc.GetResourceReplicas(ctx, currentReplicas, targetUtilization, resourceName, namespace, selector, container) + replicaCountProposal, percentageProposal, rawProposal, timestampProposal, err := a.replicaCalc.GetResourceReplicas(ctx, currentReplicas, targetUtilization, resourceName, tolerances, namespace, selector, container) if err != nil { return 0, nil, time.Time{}, "", condition, fmt.Errorf("failed to get %s utilization: %v", resourceName, err) } @@ -630,7 +637,7 @@ func (a *HorizontalController) computeStatusForResourceMetricGeneric(ctx context func (a *HorizontalController) computeStatusForResourceMetric(ctx context.Context, currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (replicaCountProposal int32, timestampProposal time.Time, metricNameProposal string, condition autoscalingv2.HorizontalPodAutoscalerCondition, err error) { - replicaCountProposal, metricValueStatus, timestampProposal, metricNameProposal, condition, err := a.computeStatusForResourceMetricGeneric(ctx, currentReplicas, metricSpec.Resource.Target, metricSpec.Resource.Name, hpa.Namespace, "", selector, autoscalingv2.ResourceMetricSourceType) + replicaCountProposal, metricValueStatus, timestampProposal, metricNameProposal, condition, err := a.computeStatusForResourceMetricGeneric(ctx, currentReplicas, metricSpec.Resource.Target, metricSpec.Resource.Name, hpa, "", selector, autoscalingv2.ResourceMetricSourceType) if err != nil { condition = a.getUnableComputeReplicaCountCondition(hpa, "FailedGetResourceMetric", err) return replicaCountProposal, timestampProposal, metricNameProposal, condition, err @@ -649,7 +656,7 @@ func (a *HorizontalController) computeStatusForResourceMetric(ctx context.Contex func (a *HorizontalController) computeStatusForContainerResourceMetric(ctx context.Context, currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (replicaCountProposal int32, timestampProposal time.Time, metricNameProposal string, condition autoscalingv2.HorizontalPodAutoscalerCondition, err error) { - replicaCountProposal, metricValueStatus, timestampProposal, metricNameProposal, condition, err := a.computeStatusForResourceMetricGeneric(ctx, currentReplicas, metricSpec.ContainerResource.Target, metricSpec.ContainerResource.Name, hpa.Namespace, metricSpec.ContainerResource.Container, selector, autoscalingv2.ContainerResourceMetricSourceType) + replicaCountProposal, metricValueStatus, timestampProposal, metricNameProposal, condition, err := a.computeStatusForResourceMetricGeneric(ctx, currentReplicas, metricSpec.ContainerResource.Target, metricSpec.ContainerResource.Name, hpa, metricSpec.ContainerResource.Container, selector, autoscalingv2.ContainerResourceMetricSourceType) if err != nil { condition = a.getUnableComputeReplicaCountCondition(hpa, "FailedGetContainerResourceMetric", err) return replicaCountProposal, timestampProposal, metricNameProposal, condition, err @@ -667,8 +674,9 @@ func (a *HorizontalController) computeStatusForContainerResourceMetric(ctx conte // computeStatusForExternalMetric computes the desired number of replicas for the specified metric of type ExternalMetricSourceType. func (a *HorizontalController) computeStatusForExternalMetric(specReplicas, statusReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (replicaCountProposal int32, timestampProposal time.Time, metricNameProposal string, condition autoscalingv2.HorizontalPodAutoscalerCondition, err error) { + tolerances := a.tolerancesForHpa(hpa) if metricSpec.External.Target.AverageValue != nil { - replicaCountProposal, usageProposal, timestampProposal, err := a.replicaCalc.GetExternalPerPodMetricReplicas(statusReplicas, metricSpec.External.Target.AverageValue.MilliValue(), metricSpec.External.Metric.Name, hpa.Namespace, metricSpec.External.Metric.Selector) + replicaCountProposal, usageProposal, timestampProposal, err := a.replicaCalc.GetExternalPerPodMetricReplicas(statusReplicas, metricSpec.External.Target.AverageValue.MilliValue(), metricSpec.External.Metric.Name, tolerances, hpa.Namespace, metricSpec.External.Metric.Selector) if err != nil { condition = a.getUnableComputeReplicaCountCondition(hpa, "FailedGetExternalMetric", err) return 0, time.Time{}, "", condition, fmt.Errorf("failed to get %s external metric: %v", metricSpec.External.Metric.Name, err) @@ -688,7 +696,7 @@ func (a *HorizontalController) computeStatusForExternalMetric(specReplicas, stat return replicaCountProposal, timestampProposal, fmt.Sprintf("external metric %s(%+v)", metricSpec.External.Metric.Name, metricSpec.External.Metric.Selector), autoscalingv2.HorizontalPodAutoscalerCondition{}, nil } if metricSpec.External.Target.Value != nil { - replicaCountProposal, usageProposal, timestampProposal, err := a.replicaCalc.GetExternalMetricReplicas(specReplicas, metricSpec.External.Target.Value.MilliValue(), metricSpec.External.Metric.Name, hpa.Namespace, metricSpec.External.Metric.Selector, selector) + replicaCountProposal, usageProposal, timestampProposal, err := a.replicaCalc.GetExternalMetricReplicas(specReplicas, metricSpec.External.Target.Value.MilliValue(), metricSpec.External.Metric.Name, tolerances, hpa.Namespace, metricSpec.External.Metric.Selector, selector) if err != nil { condition = a.getUnableComputeReplicaCountCondition(hpa, "FailedGetExternalMetric", err) return 0, time.Time{}, "", condition, fmt.Errorf("failed to get external metric %s: %v", metricSpec.External.Metric.Name, err) @@ -835,6 +843,7 @@ func (a *HorizontalController) reconcileAutoscaler(ctx context.Context, hpaShare logger.V(4).Info("Proposing desired replicas", "desiredReplicas", metricDesiredReplicas, "metric", metricName, + "tolerances", a.tolerancesForHpa(hpa), "timestamp", metricTimestamp, "scaleTarget", reference) @@ -1384,6 +1393,25 @@ func (a *HorizontalController) updateStatus(ctx context.Context, hpa *autoscalin return nil } +// tolerancesForHpa returns the metrics usage ratio tolerances for a given HPA. +// It ignores configurable tolerances set in the HPA spec.behavior field if the +// HPAConfigurableTolerance feature gate is disabled. +func (a *HorizontalController) tolerancesForHpa(hpa *autoscalingv2.HorizontalPodAutoscaler) Tolerances { + t := Tolerances{a.tolerance, a.tolerance} + behavior := hpa.Spec.Behavior + allowConfigurableTolerances := utilfeature.DefaultFeatureGate.Enabled(features.HPAConfigurableTolerance) + if behavior == nil || !allowConfigurableTolerances { + return t + } + if behavior.ScaleDown != nil && behavior.ScaleDown.Tolerance != nil { + t.scaleDown = behavior.ScaleDown.Tolerance.AsApproximateFloat64() + } + if behavior.ScaleUp != nil && behavior.ScaleUp.Tolerance != nil { + t.scaleUp = behavior.ScaleUp.Tolerance.AsApproximateFloat64() + } + return t +} + // setCondition sets the specific condition type on the given HPA to the specified value with the given reason // and message. The message and args are treated like a format string. The condition will be added if it is // not present. diff --git a/pkg/controller/podautoscaler/horizontal_test.go b/pkg/controller/podautoscaler/horizontal_test.go index 250e45d7682..0492bf47e77 100644 --- a/pkg/controller/podautoscaler/horizontal_test.go +++ b/pkg/controller/podautoscaler/horizontal_test.go @@ -37,16 +37,19 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" scalefake "k8s.io/client-go/scale/fake" core "k8s.io/client-go/testing" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" autoscalingapiv2 "k8s.io/kubernetes/pkg/apis/autoscaling/v2" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller/podautoscaler/metrics" "k8s.io/kubernetes/pkg/controller/podautoscaler/monitor" "k8s.io/kubernetes/pkg/controller/util/selectors" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/test/utils/ktesting" cmapi "k8s.io/metrics/pkg/apis/custom_metrics/v1beta2" emapi "k8s.io/metrics/pkg/apis/external_metrics/v1beta1" @@ -2216,6 +2219,107 @@ func TestTolerance(t *testing.T) { tc.runTest(t) } +func TestConfigurableTolerance(t *testing.T) { + onePercentQuantity := resource.MustParse("0.01") + ninetyPercentQuantity := resource.MustParse("0.9") + + testCases := []struct { + name string + configurableToleranceGate bool + replicas int32 + scaleUpRules *autoscalingv2.HPAScalingRules + scaleDownRules *autoscalingv2.HPAScalingRules + reportedLevels []uint64 + reportedCPURequests []resource.Quantity + expectedDesiredReplicas int32 + expectedConditionReason string + expectedActionLabel monitor.ActionLabel + }{ + { + name: "Scaling up because of a 1% configurable tolerance", + configurableToleranceGate: true, + replicas: 3, + scaleUpRules: &autoscalingv2.HPAScalingRules{ + Tolerance: &onePercentQuantity, + }, + reportedLevels: []uint64{1010, 1030, 1020}, + reportedCPURequests: []resource.Quantity{resource.MustParse("0.9"), resource.MustParse("1.0"), resource.MustParse("1.1")}, + expectedDesiredReplicas: 4, + expectedConditionReason: "SucceededRescale", + expectedActionLabel: monitor.ActionLabelScaleUp, + }, + { + name: "No scale-down because of a 90% configurable tolerance", + configurableToleranceGate: true, + replicas: 3, + scaleDownRules: &autoscalingv2.HPAScalingRules{ + Tolerance: &ninetyPercentQuantity, + }, + reportedLevels: []uint64{300, 300, 300}, + reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, + expectedDesiredReplicas: 3, + expectedConditionReason: "ReadyForNewScale", + expectedActionLabel: monitor.ActionLabelNone, + }, + { + name: "No scaling because of the large default tolerance", + configurableToleranceGate: true, + replicas: 3, + reportedLevels: []uint64{1010, 1030, 1020}, + reportedCPURequests: []resource.Quantity{resource.MustParse("0.9"), resource.MustParse("1.0"), resource.MustParse("1.1")}, + expectedDesiredReplicas: 3, + expectedConditionReason: "ReadyForNewScale", + expectedActionLabel: monitor.ActionLabelNone, + }, + { + name: "No scaling because the configurable tolerance is ignored as the feature gate is disabled", + configurableToleranceGate: false, + replicas: 3, + scaleUpRules: &autoscalingv2.HPAScalingRules{ + Tolerance: &onePercentQuantity, + }, + reportedLevels: []uint64{1010, 1030, 1020}, + reportedCPURequests: []resource.Quantity{resource.MustParse("0.9"), resource.MustParse("1.0"), resource.MustParse("1.1")}, + expectedDesiredReplicas: 3, + expectedConditionReason: "ReadyForNewScale", + expectedActionLabel: monitor.ActionLabelNone, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAConfigurableTolerance, tc.configurableToleranceGate) + tc := testCase{ + minReplicas: 1, + maxReplicas: 5, + specReplicas: tc.replicas, + statusReplicas: tc.replicas, + scaleDownRules: tc.scaleDownRules, + scaleUpRules: tc.scaleUpRules, + expectedDesiredReplicas: tc.expectedDesiredReplicas, + CPUTarget: 100, + reportedLevels: tc.reportedLevels, + reportedCPURequests: tc.reportedCPURequests, + useMetricsAPI: true, + expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{ + Type: autoscalingv2.AbleToScale, + Status: v1.ConditionTrue, + Reason: tc.expectedConditionReason, + }), + expectedReportedReconciliationActionLabel: tc.expectedActionLabel, + expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone, + expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{ + autoscalingv2.ResourceMetricSourceType: tc.expectedActionLabel, + }, + expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{ + autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone, + }, + } + tc.runTest(t) + }) + } +} + func TestToleranceCM(t *testing.T) { averageValue := resource.MustParse("20.0") tc := testCase{ diff --git a/pkg/controller/podautoscaler/replica_calculator.go b/pkg/controller/podautoscaler/replica_calculator.go index f4f89b544a6..97bf1ead639 100644 --- a/pkg/controller/podautoscaler/replica_calculator.go +++ b/pkg/controller/podautoscaler/replica_calculator.go @@ -40,21 +40,33 @@ const ( defaultTestingDelayOfInitialReadinessStatus = 10 * time.Second ) +// Tolerances contains metric usage ratio scale-up and scale-down tolerances. +type Tolerances struct { + scaleDown float64 + scaleUp float64 +} + +func (t Tolerances) String() string { + return fmt.Sprintf("[down:%.1f%%, up:%.1f%%]", t.scaleDown*100., t.scaleUp*100.) +} + +func (t Tolerances) isWithin(usageRatio float64) bool { + return (1.0-t.scaleDown) <= usageRatio && usageRatio <= (1.0+t.scaleUp) +} + // ReplicaCalculator bundles all needed information to calculate the target amount of replicas type ReplicaCalculator struct { metricsClient metricsclient.MetricsClient podLister corelisters.PodLister - tolerance float64 cpuInitializationPeriod time.Duration delayOfInitialReadinessStatus time.Duration } // NewReplicaCalculator creates a new ReplicaCalculator and passes all necessary information to the new instance -func NewReplicaCalculator(metricsClient metricsclient.MetricsClient, podLister corelisters.PodLister, tolerance float64, cpuInitializationPeriod, delayOfInitialReadinessStatus time.Duration) *ReplicaCalculator { +func NewReplicaCalculator(metricsClient metricsclient.MetricsClient, podLister corelisters.PodLister, cpuInitializationPeriod, delayOfInitialReadinessStatus time.Duration) *ReplicaCalculator { return &ReplicaCalculator{ metricsClient: metricsClient, podLister: podLister, - tolerance: tolerance, cpuInitializationPeriod: cpuInitializationPeriod, delayOfInitialReadinessStatus: delayOfInitialReadinessStatus, } @@ -62,7 +74,7 @@ func NewReplicaCalculator(metricsClient metricsclient.MetricsClient, podLister c // GetResourceReplicas calculates the desired replica count based on a target resource utilization percentage // of the given resource for pods matching the given selector in the given namespace, and the current replica count -func (c *ReplicaCalculator) GetResourceReplicas(ctx context.Context, currentReplicas int32, targetUtilization int32, resource v1.ResourceName, namespace string, selector labels.Selector, container string) (replicaCount int32, utilization int32, rawUtilization int64, timestamp time.Time, err error) { +func (c *ReplicaCalculator) GetResourceReplicas(ctx context.Context, currentReplicas int32, targetUtilization int32, resource v1.ResourceName, tolerances Tolerances, namespace string, selector labels.Selector, container string) (replicaCount int32, utilization int32, rawUtilization int64, timestamp time.Time, err error) { metrics, timestamp, err := c.metricsClient.GetResourceMetric(ctx, resource, namespace, selector, container) if err != nil { return 0, 0, 0, time.Time{}, fmt.Errorf("unable to get metrics for resource %s: %v", resource, err) @@ -94,7 +106,7 @@ func (c *ReplicaCalculator) GetResourceReplicas(ctx context.Context, currentRepl scaleUpWithUnready := len(unreadyPods) > 0 && usageRatio > 1.0 if !scaleUpWithUnready && len(missingPods) == 0 { - if math.Abs(1.0-usageRatio) <= c.tolerance { + if tolerances.isWithin(usageRatio) { // return the current replicas if the change would be too small return currentReplicas, utilization, rawUtilization, timestamp, nil } @@ -132,7 +144,7 @@ func (c *ReplicaCalculator) GetResourceReplicas(ctx context.Context, currentRepl return 0, utilization, rawUtilization, time.Time{}, err } - if math.Abs(1.0-newUsageRatio) <= c.tolerance || (usageRatio < 1.0 && newUsageRatio > 1.0) || (usageRatio > 1.0 && newUsageRatio < 1.0) { + if tolerances.isWithin(newUsageRatio) || (usageRatio < 1.0 && newUsageRatio > 1.0) || (usageRatio > 1.0 && newUsageRatio < 1.0) { // return the current replicas if the change would be too small, // or if the new usage ratio would cause a change in scale direction return currentReplicas, utilization, rawUtilization, timestamp, nil @@ -151,31 +163,31 @@ func (c *ReplicaCalculator) GetResourceReplicas(ctx context.Context, currentRepl // GetRawResourceReplicas calculates the desired replica count based on a target resource usage (as a raw milli-value) // for pods matching the given selector in the given namespace, and the current replica count -func (c *ReplicaCalculator) GetRawResourceReplicas(ctx context.Context, currentReplicas int32, targetUsage int64, resource v1.ResourceName, namespace string, selector labels.Selector, container string) (replicaCount int32, usage int64, timestamp time.Time, err error) { +func (c *ReplicaCalculator) GetRawResourceReplicas(ctx context.Context, currentReplicas int32, targetUsage int64, resource v1.ResourceName, tolerances Tolerances, namespace string, selector labels.Selector, container string) (replicaCount int32, usage int64, timestamp time.Time, err error) { metrics, timestamp, err := c.metricsClient.GetResourceMetric(ctx, resource, namespace, selector, container) if err != nil { return 0, 0, time.Time{}, fmt.Errorf("unable to get metrics for resource %s: %v", resource, err) } - replicaCount, usage, err = c.calcPlainMetricReplicas(metrics, currentReplicas, targetUsage, namespace, selector, resource) + replicaCount, usage, err = c.calcPlainMetricReplicas(metrics, currentReplicas, targetUsage, tolerances, namespace, selector, resource) return replicaCount, usage, timestamp, err } // GetMetricReplicas calculates the desired replica count based on a target metric usage // (as a milli-value) for pods matching the given selector in the given namespace, and the // current replica count -func (c *ReplicaCalculator) GetMetricReplicas(currentReplicas int32, targetUsage int64, metricName string, namespace string, selector labels.Selector, metricSelector labels.Selector) (replicaCount int32, usage int64, timestamp time.Time, err error) { +func (c *ReplicaCalculator) GetMetricReplicas(currentReplicas int32, targetUsage int64, metricName string, tolerances Tolerances, namespace string, selector labels.Selector, metricSelector labels.Selector) (replicaCount int32, usage int64, timestamp time.Time, err error) { metrics, timestamp, err := c.metricsClient.GetRawMetric(metricName, namespace, selector, metricSelector) if err != nil { return 0, 0, time.Time{}, fmt.Errorf("unable to get metric %s: %v", metricName, err) } - replicaCount, usage, err = c.calcPlainMetricReplicas(metrics, currentReplicas, targetUsage, namespace, selector, v1.ResourceName("")) + replicaCount, usage, err = c.calcPlainMetricReplicas(metrics, currentReplicas, targetUsage, tolerances, namespace, selector, v1.ResourceName("")) return replicaCount, usage, timestamp, err } // calcPlainMetricReplicas calculates the desired replicas for plain (i.e. non-utilization percentage) metrics. -func (c *ReplicaCalculator) calcPlainMetricReplicas(metrics metricsclient.PodMetricsInfo, currentReplicas int32, targetUsage int64, namespace string, selector labels.Selector, resource v1.ResourceName) (replicaCount int32, usage int64, err error) { +func (c *ReplicaCalculator) calcPlainMetricReplicas(metrics metricsclient.PodMetricsInfo, currentReplicas int32, targetUsage int64, tolerances Tolerances, namespace string, selector labels.Selector, resource v1.ResourceName) (replicaCount int32, usage int64, err error) { podList, err := c.podLister.Pods(namespace).List(selector) if err != nil { @@ -199,7 +211,7 @@ func (c *ReplicaCalculator) calcPlainMetricReplicas(metrics metricsclient.PodMet scaleUpWithUnready := len(unreadyPods) > 0 && usageRatio > 1.0 if !scaleUpWithUnready && len(missingPods) == 0 { - if math.Abs(1.0-usageRatio) <= c.tolerance { + if tolerances.isWithin(usageRatio) { // return the current replicas if the change would be too small return currentReplicas, usage, nil } @@ -232,7 +244,7 @@ func (c *ReplicaCalculator) calcPlainMetricReplicas(metrics metricsclient.PodMet // re-run the usage calculation with our new numbers newUsageRatio, _ := metricsclient.GetMetricUsageRatio(metrics, targetUsage) - if math.Abs(1.0-newUsageRatio) <= c.tolerance || (usageRatio < 1.0 && newUsageRatio > 1.0) || (usageRatio > 1.0 && newUsageRatio < 1.0) { + if tolerances.isWithin(newUsageRatio) || (usageRatio < 1.0 && newUsageRatio > 1.0) || (usageRatio > 1.0 && newUsageRatio < 1.0) { // return the current replicas if the change would be too small, // or if the new usage ratio would cause a change in scale direction return currentReplicas, usage, nil @@ -251,22 +263,22 @@ func (c *ReplicaCalculator) calcPlainMetricReplicas(metrics metricsclient.PodMet // GetObjectMetricReplicas calculates the desired replica count based on a target metric usage (as a milli-value) // for the given object in the given namespace, and the current replica count. -func (c *ReplicaCalculator) GetObjectMetricReplicas(currentReplicas int32, targetUsage int64, metricName string, namespace string, objectRef *autoscaling.CrossVersionObjectReference, selector labels.Selector, metricSelector labels.Selector) (replicaCount int32, usage int64, timestamp time.Time, err error) { +func (c *ReplicaCalculator) GetObjectMetricReplicas(currentReplicas int32, targetUsage int64, metricName string, tolerances Tolerances, namespace string, objectRef *autoscaling.CrossVersionObjectReference, selector labels.Selector, metricSelector labels.Selector) (replicaCount int32, usage int64, timestamp time.Time, err error) { usage, _, err = c.metricsClient.GetObjectMetric(metricName, namespace, objectRef, metricSelector) if err != nil { return 0, 0, time.Time{}, fmt.Errorf("unable to get metric %s: %v on %s %s/%s", metricName, objectRef.Kind, namespace, objectRef.Name, err) } usageRatio := float64(usage) / float64(targetUsage) - replicaCount, timestamp, err = c.getUsageRatioReplicaCount(currentReplicas, usageRatio, namespace, selector) + replicaCount, timestamp, err = c.getUsageRatioReplicaCount(currentReplicas, usageRatio, tolerances, namespace, selector) return replicaCount, usage, timestamp, err } // getUsageRatioReplicaCount calculates the desired replica count based on usageRatio and ready pods count. // For currentReplicas=0 doesn't take into account ready pods count and tolerance to support scaling to zero pods. -func (c *ReplicaCalculator) getUsageRatioReplicaCount(currentReplicas int32, usageRatio float64, namespace string, selector labels.Selector) (replicaCount int32, timestamp time.Time, err error) { +func (c *ReplicaCalculator) getUsageRatioReplicaCount(currentReplicas int32, usageRatio float64, tolerances Tolerances, namespace string, selector labels.Selector) (replicaCount int32, timestamp time.Time, err error) { if currentReplicas != 0 { - if math.Abs(1.0-usageRatio) <= c.tolerance { + if tolerances.isWithin(usageRatio) { // return the current replicas if the change would be too small return currentReplicas, timestamp, nil } @@ -286,7 +298,7 @@ func (c *ReplicaCalculator) getUsageRatioReplicaCount(currentReplicas int32, usa // GetObjectPerPodMetricReplicas calculates the desired replica count based on a target metric usage (as a milli-value) // for the given object in the given namespace, and the current replica count. -func (c *ReplicaCalculator) GetObjectPerPodMetricReplicas(statusReplicas int32, targetAverageUsage int64, metricName string, namespace string, objectRef *autoscaling.CrossVersionObjectReference, metricSelector labels.Selector) (replicaCount int32, usage int64, timestamp time.Time, err error) { +func (c *ReplicaCalculator) GetObjectPerPodMetricReplicas(statusReplicas int32, targetAverageUsage int64, metricName string, tolerances Tolerances, namespace string, objectRef *autoscaling.CrossVersionObjectReference, metricSelector labels.Selector) (replicaCount int32, usage int64, timestamp time.Time, err error) { usage, timestamp, err = c.metricsClient.GetObjectMetric(metricName, namespace, objectRef, metricSelector) if err != nil { return 0, 0, time.Time{}, fmt.Errorf("unable to get metric %s: %v on %s %s/%s", metricName, objectRef.Kind, namespace, objectRef.Name, err) @@ -294,7 +306,7 @@ func (c *ReplicaCalculator) GetObjectPerPodMetricReplicas(statusReplicas int32, replicaCount = statusReplicas usageRatio := float64(usage) / (float64(targetAverageUsage) * float64(replicaCount)) - if math.Abs(1.0-usageRatio) > c.tolerance { + if !tolerances.isWithin(usageRatio) { // update number of replicas if change is large enough replicaCount = int32(math.Ceil(float64(usage) / float64(targetAverageUsage))) } @@ -329,7 +341,7 @@ func (c *ReplicaCalculator) getReadyPodsCount(namespace string, selector labels. // GetExternalMetricReplicas calculates the desired replica count based on a // target metric value (as a milli-value) for the external metric in the given // namespace, and the current replica count. -func (c *ReplicaCalculator) GetExternalMetricReplicas(currentReplicas int32, targetUsage int64, metricName, namespace string, metricSelector *metav1.LabelSelector, podSelector labels.Selector) (replicaCount int32, usage int64, timestamp time.Time, err error) { +func (c *ReplicaCalculator) GetExternalMetricReplicas(currentReplicas int32, targetUsage int64, metricName string, tolerances Tolerances, namespace string, metricSelector *metav1.LabelSelector, podSelector labels.Selector) (replicaCount int32, usage int64, timestamp time.Time, err error) { metricLabelSelector, err := metav1.LabelSelectorAsSelector(metricSelector) if err != nil { return 0, 0, time.Time{}, err @@ -344,14 +356,14 @@ func (c *ReplicaCalculator) GetExternalMetricReplicas(currentReplicas int32, tar } usageRatio := float64(usage) / float64(targetUsage) - replicaCount, timestamp, err = c.getUsageRatioReplicaCount(currentReplicas, usageRatio, namespace, podSelector) + replicaCount, timestamp, err = c.getUsageRatioReplicaCount(currentReplicas, usageRatio, tolerances, namespace, podSelector) return replicaCount, usage, timestamp, err } // GetExternalPerPodMetricReplicas calculates the desired replica count based on a // target metric value per pod (as a milli-value) for the external metric in the // given namespace, and the current replica count. -func (c *ReplicaCalculator) GetExternalPerPodMetricReplicas(statusReplicas int32, targetUsagePerPod int64, metricName, namespace string, metricSelector *metav1.LabelSelector) (replicaCount int32, usage int64, timestamp time.Time, err error) { +func (c *ReplicaCalculator) GetExternalPerPodMetricReplicas(statusReplicas int32, targetUsagePerPod int64, metricName string, tolerances Tolerances, namespace string, metricSelector *metav1.LabelSelector) (replicaCount int32, usage int64, timestamp time.Time, err error) { metricLabelSelector, err := metav1.LabelSelectorAsSelector(metricSelector) if err != nil { return 0, 0, time.Time{}, err @@ -367,7 +379,7 @@ func (c *ReplicaCalculator) GetExternalPerPodMetricReplicas(statusReplicas int32 replicaCount = statusReplicas usageRatio := float64(usage) / (float64(targetUsagePerPod) * float64(replicaCount)) - if math.Abs(1.0-usageRatio) > c.tolerance { + if !tolerances.isWithin(usageRatio) { // update number of replicas if the change is large enough replicaCount = int32(math.Ceil(float64(usage) / float64(targetUsagePerPod))) } diff --git a/pkg/controller/podautoscaler/replica_calculator_test.go b/pkg/controller/podautoscaler/replica_calculator_test.go index 2ca484d96e7..a91ffa4a09f 100644 --- a/pkg/controller/podautoscaler/replica_calculator_test.go +++ b/pkg/controller/podautoscaler/replica_calculator_test.go @@ -90,9 +90,10 @@ type replicaCalcTestCase struct { timestamp time.Time - resource *resourceInfo - metric *metricInfo - container string + tolerances *Tolerances + resource *resourceInfo + metric *metricInfo + container string podReadiness []v1.ConditionStatus podStartTime []metav1.Time @@ -343,7 +344,7 @@ func (tc *replicaCalcTestCase) runTest(t *testing.T) { informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc()) informer := informerFactory.Core().V1().Pods() - replicaCalc := NewReplicaCalculator(metricsClient, informer.Lister(), defaultTestingTolerance, defaultTestingCPUInitializationPeriod, defaultTestingDelayOfInitialReadinessStatus) + replicaCalc := NewReplicaCalculator(metricsClient, informer.Lister(), defaultTestingCPUInitializationPeriod, defaultTestingDelayOfInitialReadinessStatus) stop := make(chan struct{}) defer close(stop) @@ -357,8 +358,14 @@ func (tc *replicaCalcTestCase) runTest(t *testing.T) { }) require.NoError(t, err, "something went horribly wrong...") + // Use default if tolerances are not specified in the test case. + tolerances := Tolerances{defaultTestingTolerance, defaultTestingTolerance} + if tc.tolerances != nil { + tolerances = *tc.tolerances + } + if tc.resource != nil { - outReplicas, outUtilization, outRawValue, outTimestamp, err := replicaCalc.GetResourceReplicas(context.TODO(), tc.currentReplicas, tc.resource.targetUtilization, tc.resource.name, testNamespace, selector, tc.container) + outReplicas, outUtilization, outRawValue, outTimestamp, err := replicaCalc.GetResourceReplicas(context.TODO(), tc.currentReplicas, tc.resource.targetUtilization, tc.resource.name, tolerances, testNamespace, selector, tc.container) if tc.expectedError != nil { require.Error(t, err, "there should be an error calculating the replica count") @@ -381,12 +388,12 @@ func (tc *replicaCalcTestCase) runTest(t *testing.T) { if tc.metric.singleObject == nil { t.Fatal("Metric specified as objectMetric but metric.singleObject is nil.") } - outReplicas, outUsage, outTimestamp, err = replicaCalc.GetObjectMetricReplicas(tc.currentReplicas, tc.metric.targetUsage, tc.metric.name, testNamespace, tc.metric.singleObject, selector, nil) + outReplicas, outUsage, outTimestamp, err = replicaCalc.GetObjectMetricReplicas(tc.currentReplicas, tc.metric.targetUsage, tc.metric.name, tolerances, testNamespace, tc.metric.singleObject, selector, nil) case objectPerPodMetric: if tc.metric.singleObject == nil { t.Fatal("Metric specified as objectMetric but metric.singleObject is nil.") } - outReplicas, outUsage, outTimestamp, err = replicaCalc.GetObjectPerPodMetricReplicas(tc.currentReplicas, tc.metric.perPodTargetUsage, tc.metric.name, testNamespace, tc.metric.singleObject, nil) + outReplicas, outUsage, outTimestamp, err = replicaCalc.GetObjectPerPodMetricReplicas(tc.currentReplicas, tc.metric.perPodTargetUsage, tc.metric.name, tolerances, testNamespace, tc.metric.singleObject, nil) case externalMetric: if tc.metric.selector == nil { t.Fatal("Metric specified as externalMetric but metric.selector is nil.") @@ -394,7 +401,7 @@ func (tc *replicaCalcTestCase) runTest(t *testing.T) { if tc.metric.targetUsage <= 0 { t.Fatalf("Metric specified as externalMetric but metric.targetUsage is %d which is <=0.", tc.metric.targetUsage) } - outReplicas, outUsage, outTimestamp, err = replicaCalc.GetExternalMetricReplicas(tc.currentReplicas, tc.metric.targetUsage, tc.metric.name, testNamespace, tc.metric.selector, selector) + outReplicas, outUsage, outTimestamp, err = replicaCalc.GetExternalMetricReplicas(tc.currentReplicas, tc.metric.targetUsage, tc.metric.name, tolerances, testNamespace, tc.metric.selector, selector) case externalPerPodMetric: if tc.metric.selector == nil { t.Fatal("Metric specified as externalPerPodMetric but metric.selector is nil.") @@ -403,9 +410,9 @@ func (tc *replicaCalcTestCase) runTest(t *testing.T) { t.Fatalf("Metric specified as externalPerPodMetric but metric.perPodTargetUsage is %d which is <=0.", tc.metric.perPodTargetUsage) } - outReplicas, outUsage, outTimestamp, err = replicaCalc.GetExternalPerPodMetricReplicas(tc.currentReplicas, tc.metric.perPodTargetUsage, tc.metric.name, testNamespace, tc.metric.selector) + outReplicas, outUsage, outTimestamp, err = replicaCalc.GetExternalPerPodMetricReplicas(tc.currentReplicas, tc.metric.perPodTargetUsage, tc.metric.name, tolerances, testNamespace, tc.metric.selector) case podMetric: - outReplicas, outUsage, outTimestamp, err = replicaCalc.GetMetricReplicas(tc.currentReplicas, tc.metric.targetUsage, tc.metric.name, testNamespace, selector, nil) + outReplicas, outUsage, outTimestamp, err = replicaCalc.GetMetricReplicas(tc.currentReplicas, tc.metric.targetUsage, tc.metric.name, tolerances, testNamespace, selector, nil) default: t.Fatalf("Unknown metric type: %d", tc.metric.metricType) } @@ -1263,6 +1270,188 @@ func TestReplicaCalcTolerancePerPodCMExternal(t *testing.T) { tc.runTest(t) } +func TestReplicaCalcConfigurableTolerance(t *testing.T) { + testCases := []struct { + name string + replicaCalcTestCase + }{ + { + name: "Outside of a 0% tolerance", + replicaCalcTestCase: replicaCalcTestCase{ + tolerances: &Tolerances{0., 0.}, + currentReplicas: 3, + expectedReplicas: 4, + resource: &resourceInfo{ + name: v1.ResourceCPU, + requests: []resource.Quantity{resource.MustParse("0.9"), resource.MustParse("1.0"), resource.MustParse("1.1")}, + levels: makePodMetricLevels(909, 1010, 1111), + targetUtilization: 100, + expectedUtilization: 101, + expectedValue: numContainersPerPod * 1010, + }, + }, + }, + { + name: "Within a 200% scale-up tolerance", + replicaCalcTestCase: replicaCalcTestCase{ + tolerances: &Tolerances{defaultTestingTolerance, 2.}, + currentReplicas: 3, + expectedReplicas: 3, + resource: &resourceInfo{ + name: v1.ResourceCPU, + requests: []resource.Quantity{resource.MustParse("0.9"), resource.MustParse("1.0"), resource.MustParse("1.1")}, + levels: makePodMetricLevels(1890, 1910, 1900), + targetUtilization: 100, + expectedUtilization: 190, + expectedValue: numContainersPerPod * 1900, + }, + }, + }, + { + name: "Outside 8% scale-up tolerance (and superfuous scale-down tolerance)", + replicaCalcTestCase: replicaCalcTestCase{ + tolerances: &Tolerances{2., .08}, + currentReplicas: 3, + expectedReplicas: 4, + resource: &resourceInfo{ + name: v1.ResourceCPU, + requests: []resource.Quantity{resource.MustParse("0.9"), resource.MustParse("1.0"), resource.MustParse("1.1")}, + levels: makePodMetricLevels(1100, 1080, 1090), + targetUtilization: 100, + expectedUtilization: 109, + expectedValue: numContainersPerPod * 1090, + }, + }, + }, + { + name: "Within a 36% scale-down tolerance", + replicaCalcTestCase: replicaCalcTestCase{ + tolerances: &Tolerances{.36, defaultTestingTolerance}, + currentReplicas: 3, + expectedReplicas: 3, + resource: &resourceInfo{ + name: v1.ResourceCPU, + requests: []resource.Quantity{resource.MustParse("0.9"), resource.MustParse("1.0"), resource.MustParse("1.1")}, + levels: makePodMetricLevels(660, 640, 650), + targetUtilization: 100, + expectedUtilization: 65, + expectedValue: numContainersPerPod * 650, + }, + }, + }, + { + name: "Outside a 34% scale-down tolerance", + replicaCalcTestCase: replicaCalcTestCase{ + tolerances: &Tolerances{.34, defaultTestingTolerance}, + currentReplicas: 3, + expectedReplicas: 2, + resource: &resourceInfo{ + name: v1.ResourceCPU, + requests: []resource.Quantity{resource.MustParse("0.9"), resource.MustParse("1.0"), resource.MustParse("1.1")}, + levels: makePodMetricLevels(660, 640, 650), + targetUtilization: 100, + expectedUtilization: 65, + expectedValue: numContainersPerPod * 650, + }, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, tc.runTest) + } +} + +func TestReplicaCalcConfigurableToleranceCM(t *testing.T) { + tc := replicaCalcTestCase{ + tolerances: &Tolerances{defaultTestingTolerance, .01}, + currentReplicas: 3, + expectedReplicas: 4, + metric: &metricInfo{ + name: "qps", + levels: []int64{20000, 21000, 21000}, + targetUsage: 20000, + expectedUsage: 20666, + metricType: podMetric, + }, + } + tc.runTest(t) +} + +func TestReplicaCalcConfigurableToleranceCMObject(t *testing.T) { + tc := replicaCalcTestCase{ + tolerances: &Tolerances{defaultTestingTolerance, .01}, + currentReplicas: 3, + expectedReplicas: 4, + metric: &metricInfo{ + name: "qps", + levels: []int64{20666}, + targetUsage: 20000, + expectedUsage: 20666, + singleObject: &autoscalingv2.CrossVersionObjectReference{ + Kind: "Deployment", + APIVersion: "apps/v1", + Name: "some-deployment", + }, + }, + } + tc.runTest(t) +} + +func TestReplicaCalcConfigurableTolerancePerPodCMObject(t *testing.T) { + tc := replicaCalcTestCase{ + tolerances: &Tolerances{defaultTestingTolerance, .01}, + currentReplicas: 4, + expectedReplicas: 5, + metric: &metricInfo{ + metricType: objectPerPodMetric, + name: "qps", + levels: []int64{20208}, + perPodTargetUsage: 5000, + expectedUsage: 5052, + singleObject: &autoscalingv2.CrossVersionObjectReference{ + Kind: "Deployment", + APIVersion: "apps/v1", + Name: "some-deployment", + }, + }, + } + tc.runTest(t) +} + +func TestReplicaCalcConfigurableToleranceCMExternal(t *testing.T) { + tc := replicaCalcTestCase{ + tolerances: &Tolerances{defaultTestingTolerance, .01}, + currentReplicas: 3, + expectedReplicas: 4, + metric: &metricInfo{ + name: "qps", + levels: []int64{8900}, + targetUsage: 8800, + expectedUsage: 8900, + selector: &metav1.LabelSelector{MatchLabels: map[string]string{"label": "value"}}, + metricType: externalMetric, + }, + } + tc.runTest(t) +} + +func TestReplicaCalcConfigurableTolerancePerPodCMExternal(t *testing.T) { + tc := replicaCalcTestCase{ + tolerances: &Tolerances{defaultTestingTolerance, .01}, + currentReplicas: 3, + expectedReplicas: 4, + metric: &metricInfo{ + name: "qps", + levels: []int64{8600}, + perPodTargetUsage: 2800, + expectedUsage: 2867, + selector: &metav1.LabelSelector{MatchLabels: map[string]string{"label": "value"}}, + metricType: externalPerPodMetric, + }, + } + tc.runTest(t) +} + func TestReplicaCalcSuperfluousMetrics(t *testing.T) { tc := replicaCalcTestCase{ currentReplicas: 4, From dc1696d8074edacf3e2d3857a5767eed66526386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Marc=20Fran=C3=A7ois?= Date: Fri, 21 Mar 2025 17:14:12 -0400 Subject: [PATCH 5/7] Fix API doc and tolerance field handling when gate is flipped. --- pkg/apis/autoscaling/types.go | 4 + pkg/apis/autoscaling/validation/validation.go | 48 +----- .../autoscaling/validation/validation_test.go | 121 ++++--------- .../horizontalpodautoscaler/strategy.go | 59 ++++++- .../horizontalpodautoscaler/strategy_test.go | 161 ++++++++++++++++++ .../src/k8s.io/api/autoscaling/v2/types.go | 4 + 6 files changed, 259 insertions(+), 138 deletions(-) create mode 100644 pkg/registry/autoscaling/horizontalpodautoscaler/strategy_test.go diff --git a/pkg/apis/autoscaling/types.go b/pkg/apis/autoscaling/types.go index aa7e0ca0159..db9ac44083b 100644 --- a/pkg/apis/autoscaling/types.go +++ b/pkg/apis/autoscaling/types.go @@ -174,6 +174,10 @@ type HPAScalingRules struct { // replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not // set, the default cluster-wide tolerance is applied (by default 10%). // + // For example, if autoscaling is configured with a memory consumption target of 100Mi, + // and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + // triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + // // This is an alpha field and requires enabling the HPAConfigurableTolerance // feature gate. // diff --git a/pkg/apis/autoscaling/validation/validation.go b/pkg/apis/autoscaling/validation/validation.go index 8b5ac06c442..aa643694a55 100644 --- a/pkg/apis/autoscaling/validation/validation.go +++ b/pkg/apis/autoscaling/validation/validation.go @@ -23,11 +23,9 @@ import ( pathvalidation "k8s.io/apimachinery/pkg/api/validation/path" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/apis/autoscaling" corevalidation "k8s.io/kubernetes/pkg/apis/core/v1/validation" apivalidation "k8s.io/kubernetes/pkg/apis/core/validation" - "k8s.io/kubernetes/pkg/features" ) const ( @@ -103,48 +101,16 @@ func ValidateCrossVersionObjectReference(ref autoscaling.CrossVersionObjectRefer // ValidateHorizontalPodAutoscaler validates a HorizontalPodAutoscaler and returns an // ErrorList with any errors. -func ValidateHorizontalPodAutoscaler(autoscaler *autoscaling.HorizontalPodAutoscaler) field.ErrorList { +func ValidateHorizontalPodAutoscaler(autoscaler *autoscaling.HorizontalPodAutoscaler, opts HorizontalPodAutoscalerSpecValidationOptions) field.ErrorList { allErrs := apivalidation.ValidateObjectMeta(&autoscaler.ObjectMeta, true, ValidateHorizontalPodAutoscalerName, field.NewPath("metadata")) - - // MinReplicasLowerBound represents a minimum value for minReplicas - // 0 when HPA scale-to-zero feature is enabled - var minReplicasLowerBound int32 - - if utilfeature.DefaultFeatureGate.Enabled(features.HPAScaleToZero) { - minReplicasLowerBound = 0 - } else { - minReplicasLowerBound = 1 - } - - opts := HorizontalPodAutoscalerSpecValidationOptions{ - AllowTolerance: utilfeature.DefaultMutableFeatureGate.Enabled(features.HPAConfigurableTolerance), - MinReplicasLowerBound: minReplicasLowerBound, - } - allErrs = append(allErrs, validateHorizontalPodAutoscalerSpec(autoscaler.Spec, field.NewPath("spec"), opts)...) return allErrs } // ValidateHorizontalPodAutoscalerUpdate validates an update to a HorizontalPodAutoscaler and returns an // ErrorList with any errors. -func ValidateHorizontalPodAutoscalerUpdate(newAutoscaler, oldAutoscaler *autoscaling.HorizontalPodAutoscaler) field.ErrorList { +func ValidateHorizontalPodAutoscalerUpdate(newAutoscaler, oldAutoscaler *autoscaling.HorizontalPodAutoscaler, opts HorizontalPodAutoscalerSpecValidationOptions) field.ErrorList { allErrs := apivalidation.ValidateObjectMetaUpdate(&newAutoscaler.ObjectMeta, &oldAutoscaler.ObjectMeta, field.NewPath("metadata")) - - // minReplicasLowerBound represents a minimum value for minReplicas - // 0 when HPA scale-to-zero feature is enabled or HPA object already has minReplicas=0 - var minReplicasLowerBound int32 - - if utilfeature.DefaultFeatureGate.Enabled(features.HPAScaleToZero) || (oldAutoscaler.Spec.MinReplicas != nil && *oldAutoscaler.Spec.MinReplicas == 0) { - minReplicasLowerBound = 0 - } else { - minReplicasLowerBound = 1 - } - - opts := HorizontalPodAutoscalerSpecValidationOptions{ - AllowTolerance: utilfeature.DefaultMutableFeatureGate.Enabled(features.HPAConfigurableTolerance), - MinReplicasLowerBound: minReplicasLowerBound, - } - allErrs = append(allErrs, validateHorizontalPodAutoscalerSpec(newAutoscaler.Spec, field.NewPath("spec"), opts)...) return allErrs } @@ -162,8 +128,6 @@ func ValidateHorizontalPodAutoscalerStatusUpdate(newAutoscaler, oldAutoscaler *a // HorizontalPodAutoscalerSpecValidationOptions contains the different settings for // HorizontalPodAutoscaler spec validation. type HorizontalPodAutoscalerSpecValidationOptions struct { - // Allow setting a tolerance on HPAScalingRules. - AllowTolerance bool // The minimum value for minReplicas. MinReplicasLowerBound int32 } @@ -234,12 +198,8 @@ func validateScalingRules(rules *autoscaling.HPAScalingRules, fldPath *field.Pat allErrs = append(allErrs, policyErrs...) } } - - if rules.Tolerance != nil && !opts.AllowTolerance { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("tolerance"), "tolerance is not supported when the HPAConfigurableTolerance feature gate is not enabled")) - } - if rules.Tolerance != nil && rules.Tolerance.Sign() < 0 { - allErrs = append(allErrs, field.Invalid(fldPath.Child("tolerance"), rules.Tolerance, "must be greater or equal to zero")) + if rules.Tolerance != nil { + allErrs = append(allErrs, apivalidation.ValidateNonnegativeQuantity(*rules.Tolerance, fldPath.Child("tolerance"))...) } } return allErrs diff --git a/pkg/apis/autoscaling/validation/validation_test.go b/pkg/apis/autoscaling/validation/validation_test.go index 003e97c1edc..aeac3c46381 100644 --- a/pkg/apis/autoscaling/validation/validation_test.go +++ b/pkg/apis/autoscaling/validation/validation_test.go @@ -22,15 +22,21 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/apis/autoscaling" api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/features" utilpointer "k8s.io/utils/pointer" "k8s.io/utils/ptr" ) +var ( + hpaSpecValidationOpts = HorizontalPodAutoscalerSpecValidationOptions{ + MinReplicasLowerBound: 1, + } + hpaScaleToZeroSpecValidationOpts = HorizontalPodAutoscalerSpecValidationOptions{ + MinReplicasLowerBound: 0, + } +) + func TestValidateScale(t *testing.T) { successCases := []autoscaling.Scale{{ ObjectMeta: metav1.ObjectMeta{ @@ -163,7 +169,7 @@ func TestValidateBehavior(t *testing.T) { }} for _, behavior := range successCases { hpa := prepareHPAWithBehavior(behavior) - if errs := ValidateHorizontalPodAutoscaler(&hpa); len(errs) != 0 { + if errs := ValidateHorizontalPodAutoscaler(&hpa, hpaSpecValidationOpts); len(errs) != 0 { t.Errorf("expected success: %v", errs) } } @@ -357,7 +363,7 @@ func TestValidateBehavior(t *testing.T) { }} for _, c := range errorCases { hpa := prepareHPAWithBehavior(c.behavior) - if errs := ValidateHorizontalPodAutoscaler(&hpa); len(errs) == 0 { + if errs := ValidateHorizontalPodAutoscaler(&hpa, hpaSpecValidationOpts); len(errs) == 0 { t.Errorf("expected failure for %s", c.msg) } else if !strings.Contains(errs[0].Error(), c.msg) { t.Errorf("unexpected error: %v, expected: %s", errs[0], c.msg) @@ -615,7 +621,7 @@ func TestValidateHorizontalPodAutoscaler(t *testing.T) { }, }} for _, successCase := range successCases { - if errs := ValidateHorizontalPodAutoscaler(&successCase); len(errs) != 0 { + if errs := ValidateHorizontalPodAutoscaler(&successCase, hpaSpecValidationOpts); len(errs) != 0 { t.Errorf("expected success: %v", errs) } } @@ -1419,7 +1425,7 @@ func TestValidateHorizontalPodAutoscaler(t *testing.T) { } for _, c := range errorCases { - errs := ValidateHorizontalPodAutoscaler(&c.horizontalPodAutoscaler) + errs := ValidateHorizontalPodAutoscaler(&c.horizontalPodAutoscaler, hpaSpecValidationOpts) if len(errs) == 0 { t.Errorf("expected failure for %q", c.msg) } else if !strings.Contains(errs[0].Error(), c.msg) { @@ -1490,7 +1496,7 @@ func TestValidateHorizontalPodAutoscaler(t *testing.T) { MinReplicas: utilpointer.Int32(1), MaxReplicas: 5, Metrics: []autoscaling.MetricSpec{spec}, }, - }) + }, hpaSpecValidationOpts) expectedMsg := "must populate information for the given metric source" @@ -1570,26 +1576,20 @@ func prepareMinReplicasCases(t *testing.T, minReplicas int32) []autoscaling.Hori } func TestValidateHorizontalPodAutoscalerScaleToZeroEnabled(t *testing.T) { - // Enable HPAScaleToZero feature gate. - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, true) - zeroMinReplicasCases := prepareMinReplicasCases(t, 0) for _, successCase := range zeroMinReplicasCases { - if errs := ValidateHorizontalPodAutoscaler(&successCase); len(errs) != 0 { + if errs := ValidateHorizontalPodAutoscaler(&successCase, hpaScaleToZeroSpecValidationOpts); len(errs) != 0 { t.Errorf("expected success: %v", errs) } } } func TestValidateHorizontalPodAutoscalerScaleToZeroDisabled(t *testing.T) { - // Disable HPAScaleToZero feature gate. - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, false) - zeroMinReplicasCases := prepareMinReplicasCases(t, 0) errorMsg := "must be greater than or equal to 1" for _, errorCase := range zeroMinReplicasCases { - errs := ValidateHorizontalPodAutoscaler(&errorCase) + errs := ValidateHorizontalPodAutoscaler(&errorCase, hpaSpecValidationOpts) if len(errs) == 0 { t.Errorf("expected failure for %q", errorMsg) } else if !strings.Contains(errs[0].Error(), errorMsg) { @@ -1601,43 +1601,37 @@ func TestValidateHorizontalPodAutoscalerScaleToZeroDisabled(t *testing.T) { for _, successCase := range nonZeroMinReplicasCases { successCase.Spec.MinReplicas = utilpointer.Int32(1) - if errs := ValidateHorizontalPodAutoscaler(&successCase); len(errs) != 0 { + if errs := ValidateHorizontalPodAutoscaler(&successCase, hpaSpecValidationOpts); len(errs) != 0 { t.Errorf("expected success: %v", errs) } } } func TestValidateHorizontalPodAutoscalerUpdateScaleToZeroEnabled(t *testing.T) { - // Enable HPAScaleToZero feature gate. - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, true) - zeroMinReplicasCases := prepareMinReplicasCases(t, 0) nonZeroMinReplicasCases := prepareMinReplicasCases(t, 1) for i, zeroCase := range zeroMinReplicasCases { nonZeroCase := nonZeroMinReplicasCases[i] - if errs := ValidateHorizontalPodAutoscalerUpdate(&nonZeroCase, &zeroCase); len(errs) != 0 { + if errs := ValidateHorizontalPodAutoscalerUpdate(&nonZeroCase, &zeroCase, hpaScaleToZeroSpecValidationOpts); len(errs) != 0 { t.Errorf("expected success: %v", errs) } - if errs := ValidateHorizontalPodAutoscalerUpdate(&zeroCase, &nonZeroCase); len(errs) != 0 { + if errs := ValidateHorizontalPodAutoscalerUpdate(&zeroCase, &nonZeroCase, hpaScaleToZeroSpecValidationOpts); len(errs) != 0 { t.Errorf("expected success: %v", errs) } } } func TestValidateHorizontalPodAutoscalerScaleToZeroUpdateDisabled(t *testing.T) { - // Disable HPAScaleToZero feature gate. - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, false) - zeroMinReplicasCases := prepareMinReplicasCases(t, 0) nonZeroMinReplicasCases := prepareMinReplicasCases(t, 1) errorMsg := "must be greater than or equal to 1" for i, zeroCase := range zeroMinReplicasCases { nonZeroCase := nonZeroMinReplicasCases[i] - errs := ValidateHorizontalPodAutoscalerUpdate(&zeroCase, &nonZeroCase) + errs := ValidateHorizontalPodAutoscalerUpdate(&zeroCase, &nonZeroCase, hpaSpecValidationOpts) if len(errs) == 0 { t.Errorf("expected failure for %q", errorMsg) @@ -1645,20 +1639,13 @@ func TestValidateHorizontalPodAutoscalerScaleToZeroUpdateDisabled(t *testing.T) t.Errorf("unexpected error: %q, expected: %q", errs[0], errorMsg) } - if errs := ValidateHorizontalPodAutoscalerUpdate(&zeroCase, &zeroCase); len(errs) != 0 { - t.Errorf("expected success: %v", errs) - } - - if errs := ValidateHorizontalPodAutoscalerUpdate(&nonZeroCase, &zeroCase); len(errs) != 0 { + if errs := ValidateHorizontalPodAutoscalerUpdate(&nonZeroCase, &zeroCase, hpaSpecValidationOpts); len(errs) != 0 { t.Errorf("expected success: %v", errs) } } } func TestValidateHorizontalPodAutoscalerConfigurableToleranceEnabled(t *testing.T) { - // Enable HPAConfigurableTolerance feature gate. - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAConfigurableTolerance, true) - policiesList := []autoscaling.HPAScalingPolicy{{ Type: autoscaling.PodsScalingPolicy, Value: 1, @@ -1695,7 +1682,7 @@ func TestValidateHorizontalPodAutoscalerConfigurableToleranceEnabled(t *testing. ScaleDown: &c, } hpa := prepareHPAWithBehavior(b) - if errs := ValidateHorizontalPodAutoscaler(&hpa); len(errs) != 0 { + if errs := ValidateHorizontalPodAutoscaler(&hpa, hpaScaleToZeroSpecValidationOpts); len(errs) != 0 { t.Errorf("expected success: %v", errs) } } @@ -1713,14 +1700,14 @@ func TestValidateHorizontalPodAutoscalerConfigurableToleranceEnabled(t *testing. Policies: policiesList, Tolerance: ptr.To(resource.MustParse("-0.001")), }, - msg: "greater or equal to zero", + msg: "greater than or equal to 0", }, { rule: autoscaling.HPAScalingRules{ Policies: policiesList, Tolerance: resource.NewMilliQuantity(-10, resource.DecimalSI), }, - msg: "greater or equal to zero", + msg: "greater than or equal to 0", }, { rule: autoscaling.HPAScalingRules{ @@ -1741,7 +1728,7 @@ func TestValidateHorizontalPodAutoscalerConfigurableToleranceEnabled(t *testing. ScaleUp: &c.rule, } hpa := prepareHPAWithBehavior(b) - errs := ValidateHorizontalPodAutoscaler(&hpa) + errs := ValidateHorizontalPodAutoscaler(&hpa, hpaScaleToZeroSpecValidationOpts) if len(errs) != 1 { t.Fatalf("expected exactly one error, got: %v", errs) } @@ -1752,9 +1739,6 @@ func TestValidateHorizontalPodAutoscalerConfigurableToleranceEnabled(t *testing. } func TestValidateHorizontalPodAutoscalerConfigurableToleranceDisabled(t *testing.T) { - // Disable HPAConfigurableTolerance feature gate. - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAConfigurableTolerance, false) - maxPolicy := autoscaling.MaxPolicySelect policiesList := []autoscaling.HPAScalingPolicy{{ Type: autoscaling.PodsScalingPolicy, @@ -1780,7 +1764,7 @@ func TestValidateHorizontalPodAutoscalerConfigurableToleranceDisabled(t *testing ScaleDown: &c, } hpa := prepareHPAWithBehavior(b) - if errs := ValidateHorizontalPodAutoscaler(&hpa); len(errs) != 0 { + if errs := ValidateHorizontalPodAutoscaler(&hpa, hpaSpecValidationOpts); len(errs) != 0 { t.Errorf("expected success: %v", errs) } } @@ -1789,13 +1773,6 @@ func TestValidateHorizontalPodAutoscalerConfigurableToleranceDisabled(t *testing rule autoscaling.HPAScalingRules msg string }{ - { - rule: autoscaling.HPAScalingRules{ - Policies: policiesList, - Tolerance: resource.NewMilliQuantity(1, resource.DecimalSI), - }, - msg: "not supported", - }, { rule: autoscaling.HPAScalingRules{}, msg: "at least one Policy", @@ -1812,7 +1789,7 @@ func TestValidateHorizontalPodAutoscalerConfigurableToleranceDisabled(t *testing ScaleUp: &c.rule, } hpa := prepareHPAWithBehavior(b) - errs := ValidateHorizontalPodAutoscaler(&hpa) + errs := ValidateHorizontalPodAutoscaler(&hpa, hpaSpecValidationOpts) if len(errs) != 1 { t.Fatalf("expected exactly one error, got: %v", errs) } @@ -1823,9 +1800,6 @@ func TestValidateHorizontalPodAutoscalerConfigurableToleranceDisabled(t *testing } func TestValidateHorizontalPodAutoscalerUpdateConfigurableToleranceEnabled(t *testing.T) { - // Enable HPAConfigurableTolerance feature gate. - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAConfigurableTolerance, true) - policiesList := []autoscaling.HPAScalingPolicy{{ Type: autoscaling.PodsScalingPolicy, Value: 1, @@ -1842,48 +1816,11 @@ func TestValidateHorizontalPodAutoscalerUpdateConfigurableToleranceEnabled(t *te Policies: policiesList, }}) - if errs := ValidateHorizontalPodAutoscalerUpdate(&withToleranceHPA, &withoutToleranceHPA); len(errs) != 0 { + if errs := ValidateHorizontalPodAutoscalerUpdate(&withToleranceHPA, &withoutToleranceHPA, hpaScaleToZeroSpecValidationOpts); len(errs) != 0 { t.Errorf("expected success: %v", errs) } - if errs := ValidateHorizontalPodAutoscalerUpdate(&withoutToleranceHPA, &withToleranceHPA); len(errs) != 0 { - t.Errorf("expected success: %v", errs) - } -} - -func TestValidateHorizontalPodAutoscalerConfigurableToleranceUpdateDisabled(t *testing.T) { - // Disable HPAConfigurableTolerance feature gate. - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAConfigurableTolerance, false) - - policiesList := []autoscaling.HPAScalingPolicy{{ - Type: autoscaling.PodsScalingPolicy, - Value: 1, - PeriodSeconds: 1800, - }} - - withToleranceHPA := prepareHPAWithBehavior(autoscaling.HorizontalPodAutoscalerBehavior{ - ScaleUp: &autoscaling.HPAScalingRules{ - Policies: policiesList, - Tolerance: resource.NewMilliQuantity(10, resource.DecimalSI), - }}) - withoutToleranceHPA := prepareHPAWithBehavior(autoscaling.HorizontalPodAutoscalerBehavior{ - ScaleUp: &autoscaling.HPAScalingRules{ - Policies: policiesList, - }}) - notSupportedErrorMsg := "not supported" - - errs := ValidateHorizontalPodAutoscalerUpdate(&withToleranceHPA, &withoutToleranceHPA) - if len(errs) == 0 { - t.Errorf("expected failure for %q", notSupportedErrorMsg) - } else if !strings.Contains(errs[0].Error(), notSupportedErrorMsg) { - t.Errorf("unexpected error: %q, expected: %q", errs[0], notSupportedErrorMsg) - } - - if errs := ValidateHorizontalPodAutoscalerUpdate(&withoutToleranceHPA, &withoutToleranceHPA); len(errs) != 0 { - t.Errorf("expected success: %v", errs) - } - - if errs := ValidateHorizontalPodAutoscalerUpdate(&withoutToleranceHPA, &withToleranceHPA); len(errs) != 0 { + if errs := ValidateHorizontalPodAutoscalerUpdate(&withoutToleranceHPA, &withToleranceHPA, hpaScaleToZeroSpecValidationOpts); len(errs) != 0 { t.Errorf("expected success: %v", errs) } } diff --git a/pkg/registry/autoscaling/horizontalpodautoscaler/strategy.go b/pkg/registry/autoscaling/horizontalpodautoscaler/strategy.go index 8558440f7aa..ddd374de51e 100644 --- a/pkg/registry/autoscaling/horizontalpodautoscaler/strategy.go +++ b/pkg/registry/autoscaling/horizontalpodautoscaler/strategy.go @@ -22,9 +22,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/storage/names" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/autoscaling" "k8s.io/kubernetes/pkg/apis/autoscaling/validation" + "k8s.io/kubernetes/pkg/features" "sigs.k8s.io/structured-merge-diff/v4/fieldpath" ) @@ -70,12 +72,15 @@ func (autoscalerStrategy) PrepareForCreate(ctx context.Context, obj runtime.Obje // create cannot set status newHPA.Status = autoscaling.HorizontalPodAutoscalerStatus{} + + dropDisabledFields(newHPA, nil) } // Validate validates a new autoscaler. func (autoscalerStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList { autoscaler := obj.(*autoscaling.HorizontalPodAutoscaler) - return validation.ValidateHorizontalPodAutoscaler(autoscaler) + opts := validationOptionsForHorizontalPodAutoscaler(autoscaler, nil) + return validation.ValidateHorizontalPodAutoscaler(autoscaler, opts) } // WarningsOnCreate returns warnings for the creation of the given object. @@ -98,11 +103,16 @@ func (autoscalerStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime oldHPA := old.(*autoscaling.HorizontalPodAutoscaler) // Update is not allowed to set status newHPA.Status = oldHPA.Status + + dropDisabledFields(newHPA, oldHPA) } // ValidateUpdate is the default update validation for an end user. func (autoscalerStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList { - return validation.ValidateHorizontalPodAutoscalerUpdate(obj.(*autoscaling.HorizontalPodAutoscaler), old.(*autoscaling.HorizontalPodAutoscaler)) + newHPA := obj.(*autoscaling.HorizontalPodAutoscaler) + oldHPA := old.(*autoscaling.HorizontalPodAutoscaler) + opts := validationOptionsForHorizontalPodAutoscaler(newHPA, oldHPA) + return validation.ValidateHorizontalPodAutoscalerUpdate(newHPA, oldHPA, opts) } // WarningsOnUpdate returns warnings for the given update. @@ -157,3 +167,48 @@ func (autoscalerStatusStrategy) ValidateUpdate(ctx context.Context, obj, old run func (autoscalerStatusStrategy) WarningsOnUpdate(ctx context.Context, obj, old runtime.Object) []string { return nil } + +func validationOptionsForHorizontalPodAutoscaler(newHPA, oldHPA *autoscaling.HorizontalPodAutoscaler) validation.HorizontalPodAutoscalerSpecValidationOptions { + opts := validation.HorizontalPodAutoscalerSpecValidationOptions{ + MinReplicasLowerBound: 1, + } + + oldHasZeroMinReplicas := oldHPA != nil && (oldHPA.Spec.MinReplicas != nil && *oldHPA.Spec.MinReplicas == 0) + if utilfeature.DefaultFeatureGate.Enabled(features.HPAScaleToZero) || oldHasZeroMinReplicas { + opts.MinReplicasLowerBound = 0 + } + return opts +} + +// dropDisabledFields will drop any disabled fields that have not previously been +// set on the old HPA. oldHPA is ignored if nil. +func dropDisabledFields(newHPA, oldHPA *autoscaling.HorizontalPodAutoscaler) { + if utilfeature.DefaultFeatureGate.Enabled(features.HPAConfigurableTolerance) { + return + } + if toleranceInUse(oldHPA) { + return + } + newBehavior := newHPA.Spec.Behavior + if newBehavior == nil { + return + } + + for _, sr := range []*autoscaling.HPAScalingRules{newBehavior.ScaleDown, newBehavior.ScaleUp} { + if sr != nil { + sr.Tolerance = nil + } + } +} + +func toleranceInUse(hpa *autoscaling.HorizontalPodAutoscaler) bool { + if hpa == nil || hpa.Spec.Behavior == nil { + return false + } + for _, sr := range []*autoscaling.HPAScalingRules{hpa.Spec.Behavior.ScaleDown, hpa.Spec.Behavior.ScaleUp} { + if sr != nil && sr.Tolerance != nil { + return true + } + } + return false +} diff --git a/pkg/registry/autoscaling/horizontalpodautoscaler/strategy_test.go b/pkg/registry/autoscaling/horizontalpodautoscaler/strategy_test.go new file mode 100644 index 00000000000..ba055629100 --- /dev/null +++ b/pkg/registry/autoscaling/horizontalpodautoscaler/strategy_test.go @@ -0,0 +1,161 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package horizontalpodautoscaler + +import ( + "context" + "testing" + + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/apis/autoscaling" + api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/features" + "k8s.io/utils/ptr" +) + +type toleranceSet bool +type zeroMinReplicasSet bool + +const ( + withTolerance toleranceSet = true + withoutTolerance = false + zeroMinReplicas zeroMinReplicasSet = true + oneMinReplicas = false +) + +func TestPrepareForCreateConfigurableToleranceEnabled(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAConfigurableTolerance, true) + hpa := prepareHPA(oneMinReplicas, withTolerance) + + Strategy.PrepareForCreate(context.Background(), &hpa) + if hpa.Spec.Behavior.ScaleUp.Tolerance == nil { + t.Error("Expected tolerance field, got none") + } +} + +func TestPrepareForCreateConfigurableToleranceDisabled(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAConfigurableTolerance, false) + hpa := prepareHPA(oneMinReplicas, withTolerance) + + Strategy.PrepareForCreate(context.Background(), &hpa) + if hpa.Spec.Behavior.ScaleUp.Tolerance != nil { + t.Errorf("Expected tolerance field wiped out, got %v", hpa.Spec.Behavior.ScaleUp.Tolerance) + } +} + +func TestPrepareForUpdateConfigurableToleranceEnabled(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAConfigurableTolerance, true) + newHPA := prepareHPA(oneMinReplicas, withTolerance) + oldHPA := prepareHPA(oneMinReplicas, withTolerance) + + Strategy.PrepareForUpdate(context.Background(), &newHPA, &oldHPA) + if newHPA.Spec.Behavior.ScaleUp.Tolerance == nil { + t.Error("Expected tolerance field, got none") + } +} + +func TestPrepareForUpdateConfigurableToleranceDisabled(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAConfigurableTolerance, false) + newHPA := prepareHPA(oneMinReplicas, withTolerance) + oldHPA := prepareHPA(oneMinReplicas, withoutTolerance) + + Strategy.PrepareForUpdate(context.Background(), &newHPA, &oldHPA) + if newHPA.Spec.Behavior.ScaleUp.Tolerance != nil { + t.Errorf("Expected tolerance field wiped out, got %v", newHPA.Spec.Behavior.ScaleUp.Tolerance) + } + + newHPA = prepareHPA(oneMinReplicas, withTolerance) + oldHPA = prepareHPA(oneMinReplicas, withTolerance) + Strategy.PrepareForUpdate(context.Background(), &newHPA, &oldHPA) + if newHPA.Spec.Behavior.ScaleUp.Tolerance == nil { + t.Errorf("Expected tolerance field not wiped out, got nil") + } +} + +func TestValidateOptionsScaleToZeroEnabled(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, true) + oneReplicasHPA := prepareHPA(oneMinReplicas, withoutTolerance) + + opts := validationOptionsForHorizontalPodAutoscaler(&oneReplicasHPA, &oneReplicasHPA) + if opts.MinReplicasLowerBound != 0 { + t.Errorf("Expected zero minReplicasLowerBound, got %v", opts.MinReplicasLowerBound) + } +} + +func TestValidateOptionsScaleToZeroDisabled(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, false) + zeroReplicasHPA := prepareHPA(zeroMinReplicas, withoutTolerance) + oneReplicasHPA := prepareHPA(oneMinReplicas, withoutTolerance) + + // MinReplicas should be 0 despite the gate being disabled since the old HPA + // had MinReplicas set to 0 already. + opts := validationOptionsForHorizontalPodAutoscaler(&zeroReplicasHPA, &zeroReplicasHPA) + if opts.MinReplicasLowerBound != 0 { + t.Errorf("Expected zero minReplicasLowerBound, got %v", opts.MinReplicasLowerBound) + } + + opts = validationOptionsForHorizontalPodAutoscaler(&zeroReplicasHPA, &oneReplicasHPA) + if opts.MinReplicasLowerBound == 0 { + t.Errorf("Expected non-zero minReplicasLowerBound, got 0") + } +} + +func prepareHPA(hasZeroMinReplicas zeroMinReplicasSet, hasTolerance toleranceSet) autoscaling.HorizontalPodAutoscaler { + tolerance := ptr.To(resource.MustParse("0.1")) + if !hasTolerance { + tolerance = nil + } + + minReplicas := int32(0) + if !hasZeroMinReplicas { + minReplicas = 1 + } + + return autoscaling.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "myautoscaler", + Namespace: metav1.NamespaceDefault, + ResourceVersion: "1", + }, + Spec: autoscaling.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + Kind: "ReplicationController", + Name: "myrc", + }, + MinReplicas: &minReplicas, + MaxReplicas: 5, + Metrics: []autoscaling.MetricSpec{{ + Type: autoscaling.ResourceMetricSourceType, + Resource: &autoscaling.ResourceMetricSource{ + Name: api.ResourceCPU, + Target: autoscaling.MetricTarget{ + Type: autoscaling.UtilizationMetricType, + AverageUtilization: ptr.To(int32(70)), + }, + }, + }}, + Behavior: &autoscaling.HorizontalPodAutoscalerBehavior{ + ScaleUp: &autoscaling.HPAScalingRules{ + Tolerance: tolerance, + }, + }, + }, + } +} diff --git a/staging/src/k8s.io/api/autoscaling/v2/types.go b/staging/src/k8s.io/api/autoscaling/v2/types.go index e6b9a5fc205..9ce69b1edc8 100644 --- a/staging/src/k8s.io/api/autoscaling/v2/types.go +++ b/staging/src/k8s.io/api/autoscaling/v2/types.go @@ -211,6 +211,10 @@ type HPAScalingRules struct { // replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not // set, the default cluster-wide tolerance is applied (by default 10%). // + // For example, if autoscaling is configured with a memory consumption target of 100Mi, + // and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + // triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + // // This is an alpha field and requires enabling the HPAConfigurableTolerance // feature gate. // From ac107137ce85166464465ea3954092752ca88326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Marc=20Fran=C3=A7ois?= Date: Fri, 21 Mar 2025 17:41:50 -0400 Subject: [PATCH 6/7] Refresh autogenerated files following the configurable tolerance updates. --- api/openapi-spec/swagger.json | 8 +- .../v3/apis__autoscaling__v2_openapi.json | 12 +- .../autoscaling/v2/zz_generated.conversion.go | 2 + pkg/apis/autoscaling/zz_generated.deepcopy.go | 5 + pkg/generated/openapi/zz_generated.openapi.go | 12 +- .../k8s.io/api/autoscaling/v2/generated.pb.go | 272 +++++++++++------- .../k8s.io/api/autoscaling/v2/generated.proto | 30 +- .../v2/types_swagger_doc_generated.go | 5 +- .../autoscaling/v2/zz_generated.deepcopy.go | 5 + .../autoscaling/v2/hpascalingrules.go | 10 + .../applyconfigurations/internal/internal.go | 3 + .../reference/versioned_feature_list.yaml | 6 + 12 files changed, 249 insertions(+), 121 deletions(-) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 0c8a508d8ec..bfefac3c9e6 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -3934,10 +3934,10 @@ "type": "object" }, "io.k8s.api.autoscaling.v2.HPAScalingRules": { - "description": "HPAScalingRules configures the scaling behavior for one direction. These Rules are applied after calculating DesiredReplicas from metrics for the HPA. They can limit the scaling velocity by specifying scaling policies. They can prevent flapping by specifying the stabilization window, so that the number of replicas is not set instantly, instead, the safest value from the stabilization window is chosen.", + "description": "HPAScalingRules configures the scaling behavior for one direction via scaling Policy Rules and a configurable metric tolerance.\n\nScaling Policy Rules are applied after calculating DesiredReplicas from metrics for the HPA. They can limit the scaling velocity by specifying scaling policies. They can prevent flapping by specifying the stabilization window, so that the number of replicas is not set instantly, instead, the safest value from the stabilization window is chosen.\n\nThe tolerance is applied to the metric values and prevents scaling too eagerly for small metric variations. (Note that setting a tolerance requires enabling the alpha HPAConfigurableTolerance feature gate.)", "properties": { "policies": { - "description": "policies is a list of potential scaling polices which can be used during scaling. At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid", + "description": "policies is a list of potential scaling polices which can be used during scaling. If not set, use the default values: - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. - For scale down: allow all pods to be removed in a 15s window.", "items": { "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HPAScalingPolicy" }, @@ -3952,6 +3952,10 @@ "description": "stabilizationWindowSeconds is the number of seconds for which past recommendations should be considered while scaling up or scaling down. StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). If not set, use the default values: - For scale up: 0 (i.e. no stabilization is done). - For scale down: 300 (i.e. the stabilization window is 300 seconds long).", "format": "int32", "type": "integer" + }, + "tolerance": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity", + "description": "tolerance is the tolerance on the ratio between the current and desired metric value under which no updates are made to the desired number of replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not set, the default cluster-wide tolerance is applied (by default 10%).\n\nFor example, if autoscaling is configured with a memory consumption target of 100Mi, and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be triggered when the actual consumption falls below 95Mi or exceeds 101Mi.\n\nThis is an alpha field and requires enabling the HPAConfigurableTolerance feature gate." } }, "type": "object" diff --git a/api/openapi-spec/v3/apis__autoscaling__v2_openapi.json b/api/openapi-spec/v3/apis__autoscaling__v2_openapi.json index 33700788ec9..7d58245683c 100644 --- a/api/openapi-spec/v3/apis__autoscaling__v2_openapi.json +++ b/api/openapi-spec/v3/apis__autoscaling__v2_openapi.json @@ -170,10 +170,10 @@ "type": "object" }, "io.k8s.api.autoscaling.v2.HPAScalingRules": { - "description": "HPAScalingRules configures the scaling behavior for one direction. These Rules are applied after calculating DesiredReplicas from metrics for the HPA. They can limit the scaling velocity by specifying scaling policies. They can prevent flapping by specifying the stabilization window, so that the number of replicas is not set instantly, instead, the safest value from the stabilization window is chosen.", + "description": "HPAScalingRules configures the scaling behavior for one direction via scaling Policy Rules and a configurable metric tolerance.\n\nScaling Policy Rules are applied after calculating DesiredReplicas from metrics for the HPA. They can limit the scaling velocity by specifying scaling policies. They can prevent flapping by specifying the stabilization window, so that the number of replicas is not set instantly, instead, the safest value from the stabilization window is chosen.\n\nThe tolerance is applied to the metric values and prevents scaling too eagerly for small metric variations. (Note that setting a tolerance requires enabling the alpha HPAConfigurableTolerance feature gate.)", "properties": { "policies": { - "description": "policies is a list of potential scaling polices which can be used during scaling. At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid", + "description": "policies is a list of potential scaling polices which can be used during scaling. If not set, use the default values: - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. - For scale down: allow all pods to be removed in a 15s window.", "items": { "allOf": [ { @@ -193,6 +193,14 @@ "description": "stabilizationWindowSeconds is the number of seconds for which past recommendations should be considered while scaling up or scaling down. StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). If not set, use the default values: - For scale up: 0 (i.e. no stabilization is done). - For scale down: 300 (i.e. the stabilization window is 300 seconds long).", "format": "int32", "type": "integer" + }, + "tolerance": { + "allOf": [ + { + "$ref": "#/components/schemas/io.k8s.apimachinery.pkg.api.resource.Quantity" + } + ], + "description": "tolerance is the tolerance on the ratio between the current and desired metric value under which no updates are made to the desired number of replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not set, the default cluster-wide tolerance is applied (by default 10%).\n\nFor example, if autoscaling is configured with a memory consumption target of 100Mi, and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be triggered when the actual consumption falls below 95Mi or exceeds 101Mi.\n\nThis is an alpha field and requires enabling the HPAConfigurableTolerance feature gate." } }, "type": "object" diff --git a/pkg/apis/autoscaling/v2/zz_generated.conversion.go b/pkg/apis/autoscaling/v2/zz_generated.conversion.go index cb885417843..42ffadee898 100644 --- a/pkg/apis/autoscaling/v2/zz_generated.conversion.go +++ b/pkg/apis/autoscaling/v2/zz_generated.conversion.go @@ -452,6 +452,7 @@ func autoConvert_v2_HPAScalingRules_To_autoscaling_HPAScalingRules(in *autoscali out.StabilizationWindowSeconds = (*int32)(unsafe.Pointer(in.StabilizationWindowSeconds)) out.SelectPolicy = (*autoscaling.ScalingPolicySelect)(unsafe.Pointer(in.SelectPolicy)) out.Policies = *(*[]autoscaling.HPAScalingPolicy)(unsafe.Pointer(&in.Policies)) + out.Tolerance = (*resource.Quantity)(unsafe.Pointer(in.Tolerance)) return nil } @@ -464,6 +465,7 @@ func autoConvert_autoscaling_HPAScalingRules_To_v2_HPAScalingRules(in *autoscali out.StabilizationWindowSeconds = (*int32)(unsafe.Pointer(in.StabilizationWindowSeconds)) out.SelectPolicy = (*autoscalingv2.ScalingPolicySelect)(unsafe.Pointer(in.SelectPolicy)) out.Policies = *(*[]autoscalingv2.HPAScalingPolicy)(unsafe.Pointer(&in.Policies)) + out.Tolerance = (*resource.Quantity)(unsafe.Pointer(in.Tolerance)) return nil } diff --git a/pkg/apis/autoscaling/zz_generated.deepcopy.go b/pkg/apis/autoscaling/zz_generated.deepcopy.go index 9f2d6c2e542..9e2ac0f0083 100644 --- a/pkg/apis/autoscaling/zz_generated.deepcopy.go +++ b/pkg/apis/autoscaling/zz_generated.deepcopy.go @@ -146,6 +146,11 @@ func (in *HPAScalingRules) DeepCopyInto(out *HPAScalingRules) { *out = make([]HPAScalingPolicy, len(*in)) copy(*out, *in) } + if in.Tolerance != nil { + in, out := &in.Tolerance, &out.Tolerance + x := (*in).DeepCopy() + *out = &x + } return } diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 60ea7d11952..fef070a5049 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -14693,7 +14693,7 @@ func schema_k8sio_api_autoscaling_v2_HPAScalingRules(ref common.ReferenceCallbac return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "HPAScalingRules configures the scaling behavior for one direction. These Rules are applied after calculating DesiredReplicas from metrics for the HPA. They can limit the scaling velocity by specifying scaling policies. They can prevent flapping by specifying the stabilization window, so that the number of replicas is not set instantly, instead, the safest value from the stabilization window is chosen.", + Description: "HPAScalingRules configures the scaling behavior for one direction via scaling Policy Rules and a configurable metric tolerance.\n\nScaling Policy Rules are applied after calculating DesiredReplicas from metrics for the HPA. They can limit the scaling velocity by specifying scaling policies. They can prevent flapping by specifying the stabilization window, so that the number of replicas is not set instantly, instead, the safest value from the stabilization window is chosen.\n\nThe tolerance is applied to the metric values and prevents scaling too eagerly for small metric variations. (Note that setting a tolerance requires enabling the alpha HPAConfigurableTolerance feature gate.)", Type: []string{"object"}, Properties: map[string]spec.Schema{ "stabilizationWindowSeconds": { @@ -14717,7 +14717,7 @@ func schema_k8sio_api_autoscaling_v2_HPAScalingRules(ref common.ReferenceCallbac }, }, SchemaProps: spec.SchemaProps{ - Description: "policies is a list of potential scaling polices which can be used during scaling. At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid", + Description: "policies is a list of potential scaling polices which can be used during scaling. If not set, use the default values: - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. - For scale down: allow all pods to be removed in a 15s window.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -14729,11 +14729,17 @@ func schema_k8sio_api_autoscaling_v2_HPAScalingRules(ref common.ReferenceCallbac }, }, }, + "tolerance": { + SchemaProps: spec.SchemaProps{ + Description: "tolerance is the tolerance on the ratio between the current and desired metric value under which no updates are made to the desired number of replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not set, the default cluster-wide tolerance is applied (by default 10%).\n\nFor example, if autoscaling is configured with a memory consumption target of 100Mi, and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be triggered when the actual consumption falls below 95Mi or exceeds 101Mi.\n\nThis is an alpha field and requires enabling the HPAConfigurableTolerance feature gate.", + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, }, }, }, Dependencies: []string{ - "k8s.io/api/autoscaling/v2.HPAScalingPolicy"}, + "k8s.io/api/autoscaling/v2.HPAScalingPolicy", "k8s.io/apimachinery/pkg/api/resource.Quantity"}, } } diff --git a/staging/src/k8s.io/api/autoscaling/v2/generated.pb.go b/staging/src/k8s.io/api/autoscaling/v2/generated.pb.go index ece6dedadb1..40b60ebeca4 100644 --- a/staging/src/k8s.io/api/autoscaling/v2/generated.pb.go +++ b/staging/src/k8s.io/api/autoscaling/v2/generated.pb.go @@ -751,115 +751,116 @@ func init() { } var fileDescriptor_4d5f2c8767749221 = []byte{ - // 1722 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x59, 0xcb, 0x8f, 0x1b, 0x49, - 0x19, 0x9f, 0xb6, 0x3d, 0xaf, 0xf2, 0x3c, 0x2b, 0x2f, 0x67, 0xa2, 0xd8, 0xa3, 0x26, 0x90, 0x07, - 0xa4, 0x4d, 0x4c, 0x88, 0x22, 0x72, 0x40, 0xd3, 0x13, 0x20, 0xa3, 0xcc, 0x30, 0x4e, 0x39, 0xc9, - 0x00, 0x02, 0x94, 0x72, 0x77, 0x8d, 0xa7, 0x18, 0xbb, 0xdb, 0xea, 0x6e, 0x3b, 0x99, 0x48, 0x48, - 0x5c, 0xb8, 0x23, 0x50, 0x84, 0xf8, 0x1f, 0x22, 0x4e, 0xa0, 0x70, 0x00, 0x09, 0x69, 0xf7, 0x90, - 0xcb, 0x4a, 0x39, 0xec, 0x21, 0x27, 0x6b, 0xe3, 0x95, 0xf6, 0xb8, 0x7f, 0x40, 0x4e, 0xab, 0x7a, - 0xf4, 0xd3, 0xaf, 0x71, 0x76, 0x32, 0xd2, 0xdc, 0x5c, 0x55, 0xdf, 0xf7, 0xfb, 0x1e, 0xf5, 0xbd, - 0xaa, 0x0d, 0xae, 0xee, 0xdf, 0x76, 0x35, 0x6a, 0x17, 0x71, 0x93, 0x16, 0x71, 0xcb, 0xb3, 0x5d, - 0x03, 0xd7, 0xa9, 0x55, 0x2b, 0xb6, 0x4b, 0xc5, 0x1a, 0xb1, 0x88, 0x83, 0x3d, 0x62, 0x6a, 0x4d, - 0xc7, 0xf6, 0x6c, 0x78, 0x5e, 0x90, 0x6a, 0xb8, 0x49, 0xb5, 0x08, 0xa9, 0xd6, 0x2e, 0xad, 0x5c, - 0xaf, 0x51, 0x6f, 0xaf, 0x55, 0xd5, 0x0c, 0xbb, 0x51, 0xac, 0xd9, 0x35, 0xbb, 0xc8, 0x39, 0xaa, - 0xad, 0x5d, 0xbe, 0xe2, 0x0b, 0xfe, 0x4b, 0x20, 0xad, 0xa8, 0x11, 0xa1, 0x86, 0xed, 0x90, 0x62, - 0xfb, 0x46, 0x52, 0xda, 0xca, 0xcd, 0x90, 0xa6, 0x81, 0x8d, 0x3d, 0x6a, 0x11, 0xe7, 0xa0, 0xd8, - 0xdc, 0xaf, 0x71, 0x26, 0x87, 0xb8, 0x76, 0xcb, 0x31, 0xc8, 0x58, 0x5c, 0x6e, 0xb1, 0x41, 0x3c, - 0xdc, 0x4f, 0x56, 0x71, 0x10, 0x97, 0xd3, 0xb2, 0x3c, 0xda, 0xe8, 0x15, 0x73, 0x6b, 0x14, 0x83, - 0x6b, 0xec, 0x91, 0x06, 0x4e, 0xf2, 0xa9, 0x5f, 0x29, 0xe0, 0xe2, 0xba, 0x6d, 0x79, 0x98, 0x71, - 0x20, 0x69, 0xc4, 0x16, 0xf1, 0x1c, 0x6a, 0x54, 0xf8, 0x6f, 0xb8, 0x0e, 0x32, 0x16, 0x6e, 0x90, - 0x9c, 0xb2, 0xaa, 0x5c, 0x99, 0xd5, 0x8b, 0xaf, 0x3b, 0x85, 0x89, 0x6e, 0xa7, 0x90, 0xf9, 0x25, - 0x6e, 0x90, 0xf7, 0x9d, 0x42, 0xa1, 0xd7, 0x71, 0x9a, 0x0f, 0xc3, 0x48, 0x10, 0x67, 0x86, 0xdb, - 0x60, 0xca, 0xc3, 0x4e, 0x8d, 0x78, 0xb9, 0xd4, 0xaa, 0x72, 0x25, 0x5b, 0xba, 0xac, 0x0d, 0xbc, - 0x3a, 0x4d, 0x48, 0x7f, 0xc8, 0xc9, 0xf5, 0x05, 0x29, 0x6f, 0x4a, 0xac, 0x91, 0x84, 0x81, 0x45, - 0x30, 0x6b, 0xf8, 0x6a, 0xe7, 0xd2, 0x5c, 0xb5, 0x65, 0x49, 0x3a, 0x1b, 0xda, 0x13, 0xd2, 0xa8, - 0x5f, 0x0f, 0x31, 0xd4, 0xc3, 0x5e, 0xcb, 0x3d, 0x1a, 0x43, 0x77, 0xc0, 0xb4, 0xd1, 0x72, 0x1c, - 0x62, 0xf9, 0x96, 0xfe, 0x60, 0xa4, 0xa5, 0x8f, 0x71, 0xbd, 0x45, 0x84, 0x0e, 0xfa, 0xa2, 0x94, - 0x3a, 0xbd, 0x2e, 0x40, 0x90, 0x8f, 0x36, 0xbe, 0xc1, 0x2f, 0x14, 0x70, 0x61, 0xdd, 0xb1, 0x5d, - 0xf7, 0x31, 0x71, 0x5c, 0x6a, 0x5b, 0xdb, 0xd5, 0x3f, 0x10, 0xc3, 0x43, 0x64, 0x97, 0x38, 0xc4, - 0x32, 0x08, 0x5c, 0x05, 0x99, 0x7d, 0x6a, 0x99, 0xd2, 0xdc, 0x39, 0xdf, 0xdc, 0xfb, 0xd4, 0x32, - 0x11, 0x3f, 0x61, 0x14, 0xdc, 0x21, 0xa9, 0x38, 0x45, 0xc4, 0xda, 0x12, 0x00, 0xb8, 0x49, 0xa5, - 0x00, 0xa9, 0x15, 0x94, 0x74, 0x60, 0xad, 0xbc, 0x21, 0x4f, 0x50, 0x84, 0x4a, 0xfd, 0xaf, 0x02, - 0x4e, 0xff, 0xec, 0x99, 0x47, 0x1c, 0x0b, 0xd7, 0x63, 0x81, 0x56, 0x01, 0x53, 0x0d, 0xbe, 0xe6, - 0x2a, 0x65, 0x4b, 0xdf, 0x1f, 0xe9, 0xb9, 0x0d, 0x93, 0x58, 0x1e, 0xdd, 0xa5, 0xc4, 0x09, 0xe3, - 0x44, 0x9c, 0x20, 0x09, 0x75, 0xe4, 0x81, 0xa7, 0x7e, 0xda, 0xab, 0xbe, 0x08, 0x9f, 0x8f, 0xa2, - 0xfe, 0xc7, 0x0a, 0x27, 0xf5, 0x9f, 0x0a, 0x58, 0xba, 0x57, 0x5e, 0xab, 0x08, 0xee, 0xb2, 0x5d, - 0xa7, 0xc6, 0x01, 0xbc, 0x0d, 0x32, 0xde, 0x41, 0xd3, 0xcf, 0x80, 0x4b, 0xfe, 0x85, 0x3f, 0x3c, - 0x68, 0xb2, 0x0c, 0x38, 0x9d, 0xa4, 0x67, 0xfb, 0x88, 0x73, 0xc0, 0xef, 0x80, 0xc9, 0x36, 0x93, - 0xcb, 0xb5, 0x9c, 0xd4, 0xe7, 0x25, 0xeb, 0x24, 0x57, 0x06, 0x89, 0x33, 0x78, 0x07, 0xcc, 0x37, - 0x89, 0x43, 0x6d, 0xb3, 0x42, 0x0c, 0xdb, 0x32, 0x5d, 0x1e, 0x30, 0x93, 0xfa, 0x19, 0x49, 0x3c, - 0x5f, 0x8e, 0x1e, 0xa2, 0x38, 0xad, 0xfa, 0x8f, 0x14, 0x58, 0x0c, 0x15, 0x40, 0xad, 0x3a, 0x71, - 0xe1, 0xef, 0xc1, 0x8a, 0xeb, 0xe1, 0x2a, 0xad, 0xd3, 0xe7, 0xd8, 0xa3, 0xb6, 0xb5, 0x43, 0x2d, - 0xd3, 0x7e, 0x1a, 0x47, 0xcf, 0x77, 0x3b, 0x85, 0x95, 0xca, 0x40, 0x2a, 0x34, 0x04, 0x01, 0xde, - 0x07, 0x73, 0x2e, 0xa9, 0x13, 0xc3, 0x13, 0xf6, 0x4a, 0xbf, 0x5c, 0xee, 0x76, 0x0a, 0x73, 0x95, - 0xc8, 0xfe, 0xfb, 0x4e, 0xe1, 0x54, 0xcc, 0x31, 0xe2, 0x10, 0xc5, 0x98, 0xe1, 0xaf, 0xc1, 0x4c, - 0x93, 0xfd, 0xa2, 0xc4, 0xcd, 0xa5, 0x56, 0xd3, 0x23, 0x22, 0x24, 0xe9, 0x6b, 0x7d, 0x49, 0x7a, - 0x69, 0xa6, 0x2c, 0x41, 0x50, 0x00, 0xa7, 0xbe, 0x4a, 0x81, 0x73, 0xf7, 0x6c, 0x87, 0x3e, 0x67, - 0xc9, 0x5f, 0x2f, 0xdb, 0xe6, 0x9a, 0x04, 0x23, 0x0e, 0x7c, 0x02, 0x66, 0x58, 0x93, 0x31, 0xb1, - 0x87, 0x65, 0x60, 0xfe, 0x30, 0x22, 0x36, 0xe8, 0x15, 0x5a, 0x73, 0xbf, 0xc6, 0x36, 0x5c, 0x8d, - 0x51, 0x6b, 0xed, 0x1b, 0x9a, 0xa8, 0x17, 0x5b, 0xc4, 0xc3, 0x61, 0x4a, 0x87, 0x7b, 0x28, 0x40, - 0x85, 0xbf, 0x02, 0x19, 0xb7, 0x49, 0x0c, 0x19, 0xa0, 0xb7, 0x86, 0x19, 0xd5, 0x5f, 0xc7, 0x4a, - 0x93, 0x18, 0x61, 0x79, 0x61, 0x2b, 0xc4, 0x11, 0xe1, 0x13, 0x30, 0xe5, 0xf2, 0x40, 0xe6, 0x77, - 0x99, 0x2d, 0xdd, 0xfe, 0x00, 0x6c, 0x91, 0x08, 0x41, 0x7e, 0x89, 0x35, 0x92, 0xb8, 0xea, 0x67, - 0x0a, 0x28, 0x0c, 0xe0, 0xd4, 0xc9, 0x1e, 0x6e, 0x53, 0xdb, 0x81, 0x0f, 0xc0, 0x34, 0xdf, 0x79, - 0xd4, 0x94, 0x0e, 0xbc, 0x76, 0xa8, 0x7b, 0xe3, 0x21, 0xaa, 0x67, 0x59, 0xf6, 0x55, 0x04, 0x3b, - 0xf2, 0x71, 0xe0, 0x0e, 0x98, 0xe5, 0x3f, 0xef, 0xda, 0x4f, 0x2d, 0xe9, 0xb7, 0x71, 0x40, 0xe7, - 0x59, 0xd1, 0xaf, 0xf8, 0x00, 0x28, 0xc4, 0x52, 0xff, 0x9c, 0x06, 0xab, 0x03, 0xec, 0x59, 0xb7, - 0x2d, 0x93, 0xb2, 0x18, 0x87, 0xf7, 0x62, 0x69, 0x7e, 0x33, 0x91, 0xe6, 0x97, 0x46, 0xf1, 0x47, - 0xd2, 0x7e, 0x33, 0xb8, 0xa0, 0x54, 0x0c, 0x4b, 0xba, 0xf9, 0x7d, 0xa7, 0xd0, 0x67, 0xb0, 0xd2, - 0x02, 0xa4, 0xf8, 0x65, 0xc0, 0x36, 0x80, 0x75, 0xec, 0x7a, 0x0f, 0x1d, 0x6c, 0xb9, 0x42, 0x12, - 0x6d, 0x10, 0x79, 0xf5, 0xd7, 0x0e, 0x17, 0xb4, 0x8c, 0x43, 0x5f, 0x91, 0x5a, 0xc0, 0xcd, 0x1e, - 0x34, 0xd4, 0x47, 0x02, 0xfc, 0x1e, 0x98, 0x72, 0x08, 0x76, 0x6d, 0x2b, 0x97, 0xe1, 0x56, 0x04, - 0xc1, 0x82, 0xf8, 0x2e, 0x92, 0xa7, 0xf0, 0x2a, 0x98, 0x6e, 0x10, 0xd7, 0xc5, 0x35, 0x92, 0x9b, - 0xe4, 0x84, 0x41, 0x79, 0xdd, 0x12, 0xdb, 0xc8, 0x3f, 0x57, 0x3f, 0x57, 0xc0, 0x85, 0x01, 0x7e, - 0xdc, 0xa4, 0xae, 0x07, 0x7f, 0xdb, 0x93, 0x95, 0xda, 0xe1, 0x0c, 0x64, 0xdc, 0x3c, 0x27, 0x83, - 0x7a, 0xe0, 0xef, 0x44, 0x32, 0x72, 0x07, 0x4c, 0x52, 0x8f, 0x34, 0xfc, 0x3a, 0x53, 0x1a, 0x3f, - 0x6d, 0xc2, 0x0a, 0xbe, 0xc1, 0x80, 0x90, 0xc0, 0x53, 0x5f, 0xa5, 0x07, 0x9a, 0xc5, 0xd2, 0x16, - 0xb6, 0xc1, 0x02, 0x5f, 0xc9, 0x9e, 0x49, 0x76, 0xa5, 0x71, 0xc3, 0x8a, 0xc2, 0x90, 0x19, 0x45, - 0x3f, 0x2b, 0xb5, 0x58, 0xa8, 0xc4, 0x50, 0x51, 0x42, 0x0a, 0xbc, 0x01, 0xb2, 0x0d, 0x6a, 0x21, - 0xd2, 0xac, 0x53, 0x03, 0xbb, 0xb2, 0x09, 0x2d, 0x76, 0x3b, 0x85, 0xec, 0x56, 0xb8, 0x8d, 0xa2, - 0x34, 0xf0, 0xc7, 0x20, 0xdb, 0xc0, 0xcf, 0x02, 0x16, 0xd1, 0x2c, 0x4e, 0x49, 0x79, 0xd9, 0xad, - 0xf0, 0x08, 0x45, 0xe9, 0x60, 0x99, 0xc5, 0x00, 0x6b, 0xb3, 0x6e, 0x2e, 0xc3, 0x9d, 0xfb, 0xdd, - 0x91, 0x0d, 0x99, 0x97, 0xb7, 0x48, 0xa8, 0x70, 0x6e, 0xe4, 0xc3, 0x40, 0x13, 0xcc, 0x54, 0x65, - 0xa9, 0xe1, 0x61, 0x95, 0x2d, 0xfd, 0xe4, 0x03, 0xee, 0x4b, 0x22, 0xe8, 0x73, 0x2c, 0x24, 0xfc, - 0x15, 0x0a, 0x90, 0xd5, 0x97, 0x19, 0x70, 0x71, 0x68, 0x89, 0x84, 0x3f, 0x07, 0xd0, 0xae, 0xba, - 0xc4, 0x69, 0x13, 0xf3, 0x17, 0xe2, 0x91, 0xc0, 0x66, 0x3a, 0x76, 0x7f, 0x69, 0xfd, 0x2c, 0xcb, - 0xa6, 0xed, 0x9e, 0x53, 0xd4, 0x87, 0x03, 0x1a, 0x60, 0x9e, 0xe5, 0x98, 0xb8, 0x31, 0x2a, 0xc7, - 0xc7, 0xf1, 0x12, 0x78, 0x99, 0x4d, 0x03, 0x9b, 0x51, 0x10, 0x14, 0xc7, 0x84, 0x6b, 0x60, 0x51, - 0x4e, 0x32, 0x89, 0x1b, 0x3c, 0x27, 0xfd, 0xbc, 0xb8, 0x1e, 0x3f, 0x46, 0x49, 0x7a, 0x06, 0x61, - 0x12, 0x97, 0x3a, 0xc4, 0x0c, 0x20, 0x32, 0x71, 0x88, 0xbb, 0xf1, 0x63, 0x94, 0xa4, 0x87, 0x35, - 0xb0, 0x20, 0x51, 0xe5, 0xad, 0xe6, 0x26, 0x79, 0x4c, 0x8c, 0x1e, 0x32, 0x65, 0x5b, 0x0a, 0xe2, - 0x7b, 0x3d, 0x06, 0x83, 0x12, 0xb0, 0xd0, 0x06, 0xc0, 0xf0, 0x8b, 0xa6, 0x9b, 0x9b, 0xe2, 0x42, - 0xee, 0x8c, 0x1f, 0x25, 0x41, 0xe1, 0x0d, 0x3b, 0x7a, 0xb0, 0xe5, 0xa2, 0x88, 0x08, 0xf5, 0x6f, - 0x0a, 0x58, 0x4a, 0x0e, 0xa9, 0xc1, 0x7b, 0x40, 0x19, 0xf8, 0x1e, 0xf8, 0x1d, 0x98, 0x11, 0x33, - 0x8f, 0xed, 0xc8, 0x6b, 0xff, 0xd1, 0x21, 0xcb, 0x1a, 0xae, 0x92, 0x7a, 0x45, 0xb2, 0x8a, 0x20, - 0xf6, 0x57, 0x28, 0x80, 0x54, 0x5f, 0x64, 0x00, 0x08, 0x73, 0x0a, 0xde, 0x8c, 0xf5, 0xb1, 0xd5, - 0x44, 0x1f, 0x5b, 0x8a, 0x3e, 0x2e, 0x22, 0x3d, 0xeb, 0x01, 0x98, 0xb2, 0x79, 0x99, 0x91, 0x1a, - 0x5e, 0x1f, 0xe2, 0xc7, 0x60, 0xde, 0x09, 0x80, 0x74, 0xc0, 0x1a, 0x83, 0xac, 0x53, 0x12, 0x08, - 0x6e, 0x80, 0x4c, 0xd3, 0x36, 0xfd, 0x29, 0x65, 0xd8, 0x58, 0x57, 0xb6, 0x4d, 0x37, 0x06, 0x37, - 0xc3, 0x34, 0x66, 0xbb, 0x88, 0x43, 0xb0, 0x29, 0xd1, 0xff, 0x94, 0xc0, 0xc3, 0x31, 0x5b, 0x2a, - 0x0e, 0x81, 0xeb, 0xf7, 0x60, 0x17, 0xde, 0xf3, 0x4f, 0x50, 0x00, 0x07, 0xff, 0x08, 0x96, 0x8d, - 0xe4, 0x03, 0x38, 0x37, 0x3d, 0x72, 0xb0, 0x1a, 0xfa, 0x75, 0x40, 0x3f, 0xd3, 0xed, 0x14, 0x96, - 0x7b, 0x48, 0x50, 0xaf, 0x24, 0x66, 0x19, 0x91, 0xef, 0x26, 0x59, 0xe7, 0x86, 0x59, 0xd6, 0xef, - 0x85, 0x28, 0x2c, 0xf3, 0x4f, 0x50, 0x00, 0xa7, 0xfe, 0x3d, 0x03, 0xe6, 0x62, 0x6f, 0xb1, 0x63, - 0x8e, 0x0c, 0x91, 0xcc, 0x47, 0x16, 0x19, 0x02, 0xee, 0x48, 0x23, 0x43, 0x40, 0x1e, 0x53, 0x64, - 0x08, 0x61, 0xc7, 0x14, 0x19, 0x11, 0xcb, 0xfa, 0x44, 0xc6, 0x27, 0x29, 0x3f, 0x32, 0xc4, 0xb0, - 0x70, 0xb8, 0xc8, 0x10, 0xb4, 0x91, 0xc8, 0xd8, 0x8e, 0x3e, 0x6f, 0x47, 0xcc, 0x6a, 0x9a, 0xef, - 0x56, 0xed, 0x41, 0x0b, 0x5b, 0x1e, 0xf5, 0x0e, 0xf4, 0xd9, 0x9e, 0xa7, 0xb0, 0x09, 0xe6, 0x70, - 0x9b, 0x38, 0xb8, 0x46, 0xf8, 0xb6, 0x8c, 0x8f, 0x71, 0x71, 0x97, 0xd8, 0x4b, 0x74, 0x2d, 0x82, - 0x83, 0x62, 0xa8, 0xac, 0xa5, 0xcb, 0xf5, 0x23, 0x2f, 0x78, 0xe2, 0xca, 0x2e, 0xc7, 0x5b, 0xfa, - 0x5a, 0xcf, 0x29, 0xea, 0xc3, 0xa1, 0xfe, 0x35, 0x05, 0x96, 0x7b, 0x3e, 0x2e, 0x84, 0x4e, 0x51, - 0x3e, 0x92, 0x53, 0x52, 0xc7, 0xe8, 0x94, 0xf4, 0xd8, 0x4e, 0xf9, 0x77, 0x0a, 0xc0, 0xde, 0xfe, - 0x00, 0x0f, 0xf8, 0x58, 0x61, 0x38, 0xb4, 0x4a, 0x4c, 0x71, 0xfc, 0x2d, 0x67, 0xe0, 0xe8, 0x38, - 0x12, 0x85, 0x45, 0x49, 0x39, 0x47, 0xff, 0x91, 0x35, 0xfc, 0xa4, 0x95, 0x3e, 0xb2, 0x4f, 0x5a, - 0xea, 0xff, 0x92, 0x7e, 0x3b, 0x81, 0x9f, 0xcf, 0xfa, 0xdd, 0x72, 0xfa, 0x78, 0x6e, 0x59, 0xfd, - 0x8f, 0x02, 0x96, 0x92, 0x63, 0xc4, 0x09, 0xf9, 0x76, 0xfa, 0xff, 0xb8, 0xea, 0x27, 0xf1, 0xbb, - 0xe9, 0x4b, 0x05, 0x9c, 0x3e, 0x39, 0x7f, 0x93, 0xa8, 0xff, 0xea, 0x55, 0xf7, 0x04, 0xfc, 0xd9, - 0xa1, 0xff, 0xf4, 0xf5, 0xbb, 0xfc, 0xc4, 0x9b, 0x77, 0xf9, 0x89, 0xb7, 0xef, 0xf2, 0x13, 0x7f, - 0xea, 0xe6, 0x95, 0xd7, 0xdd, 0xbc, 0xf2, 0xa6, 0x9b, 0x57, 0xde, 0x76, 0xf3, 0xca, 0x17, 0xdd, - 0xbc, 0xf2, 0x97, 0x2f, 0xf3, 0x13, 0xbf, 0x39, 0x3f, 0xf0, 0x9f, 0xc2, 0x6f, 0x02, 0x00, 0x00, - 0xff, 0xff, 0xca, 0x8b, 0x47, 0xba, 0x45, 0x1c, 0x00, 0x00, + // 1742 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x59, 0xc9, 0x8f, 0x1b, 0x4b, + 0x19, 0x9f, 0xb6, 0x3d, 0x5b, 0x79, 0xd6, 0xca, 0xe6, 0x4c, 0x14, 0x7b, 0xd4, 0x04, 0xb2, 0x40, + 0xda, 0xc4, 0x84, 0x28, 0x22, 0x07, 0x34, 0x3d, 0x01, 0x32, 0xca, 0x0c, 0xe3, 0x94, 0x27, 0x19, + 0x76, 0xa5, 0xdc, 0x5d, 0xe3, 0x29, 0xc6, 0xee, 0xb6, 0xba, 0xdb, 0x4e, 0x26, 0x12, 0x12, 0x17, + 0xee, 0x08, 0x14, 0xf1, 0x4f, 0x44, 0x9c, 0x40, 0xe1, 0x00, 0x12, 0x12, 0x1c, 0x72, 0x41, 0xca, + 0x81, 0x43, 0x4e, 0x16, 0x31, 0xd2, 0x3b, 0xbe, 0xe3, 0x3b, 0xe4, 0xf4, 0x54, 0x4b, 0xaf, 0xde, + 0xc6, 0x79, 0x93, 0x91, 0xe6, 0xe6, 0xaa, 0xfa, 0xbe, 0xdf, 0xb7, 0xd4, 0xb7, 0x55, 0x1b, 0x5c, + 0x3f, 0xb8, 0xeb, 0x6a, 0xd4, 0x2e, 0xe2, 0x26, 0x2d, 0xe2, 0x96, 0x67, 0xbb, 0x06, 0xae, 0x53, + 0xab, 0x56, 0x6c, 0x97, 0x8a, 0x35, 0x62, 0x11, 0x07, 0x7b, 0xc4, 0xd4, 0x9a, 0x8e, 0xed, 0xd9, + 0xf0, 0xa2, 0x20, 0xd5, 0x70, 0x93, 0x6a, 0x11, 0x52, 0xad, 0x5d, 0x5a, 0xb9, 0x59, 0xa3, 0xde, + 0x7e, 0xab, 0xaa, 0x19, 0x76, 0xa3, 0x58, 0xb3, 0x6b, 0x76, 0x91, 0x73, 0x54, 0x5b, 0x7b, 0x7c, + 0xc5, 0x17, 0xfc, 0x97, 0x40, 0x5a, 0x51, 0x23, 0x42, 0x0d, 0xdb, 0x21, 0xc5, 0xf6, 0xad, 0xa4, + 0xb4, 0x95, 0xdb, 0x21, 0x4d, 0x03, 0x1b, 0xfb, 0xd4, 0x22, 0xce, 0x61, 0xb1, 0x79, 0x50, 0xe3, + 0x4c, 0x0e, 0x71, 0xed, 0x96, 0x63, 0x90, 0xb1, 0xb8, 0xdc, 0x62, 0x83, 0x78, 0xb8, 0x9f, 0xac, + 0xe2, 0x20, 0x2e, 0xa7, 0x65, 0x79, 0xb4, 0xd1, 0x2b, 0xe6, 0xce, 0x28, 0x06, 0xd7, 0xd8, 0x27, + 0x0d, 0x9c, 0xe4, 0x53, 0x3f, 0x53, 0xc0, 0xe5, 0x75, 0xdb, 0xf2, 0x30, 0xe3, 0x40, 0xd2, 0x88, + 0x2d, 0xe2, 0x39, 0xd4, 0xa8, 0xf0, 0xdf, 0x70, 0x1d, 0x64, 0x2c, 0xdc, 0x20, 0x39, 0x65, 0x55, + 0xb9, 0x36, 0xab, 0x17, 0xdf, 0x74, 0x0a, 0x13, 0xdd, 0x4e, 0x21, 0xf3, 0x63, 0xdc, 0x20, 0x1f, + 0x3a, 0x85, 0x42, 0xaf, 0xe3, 0x34, 0x1f, 0x86, 0x91, 0x20, 0xce, 0x0c, 0xb7, 0xc1, 0x94, 0x87, + 0x9d, 0x1a, 0xf1, 0x72, 0xa9, 0x55, 0xe5, 0x5a, 0xb6, 0x74, 0x55, 0x1b, 0x78, 0x75, 0x9a, 0x90, + 0xbe, 0xc3, 0xc9, 0xf5, 0x05, 0x29, 0x6f, 0x4a, 0xac, 0x91, 0x84, 0x81, 0x45, 0x30, 0x6b, 0xf8, + 0x6a, 0xe7, 0xd2, 0x5c, 0xb5, 0x65, 0x49, 0x3a, 0x1b, 0xda, 0x13, 0xd2, 0xa8, 0x9f, 0x0f, 0x31, + 0xd4, 0xc3, 0x5e, 0xcb, 0x3d, 0x1e, 0x43, 0x77, 0xc1, 0xb4, 0xd1, 0x72, 0x1c, 0x62, 0xf9, 0x96, + 0x7e, 0x6b, 0xa4, 0xa5, 0x4f, 0x70, 0xbd, 0x45, 0x84, 0x0e, 0xfa, 0xa2, 0x94, 0x3a, 0xbd, 0x2e, + 0x40, 0x90, 0x8f, 0x36, 0xbe, 0xc1, 0x2f, 0x15, 0x70, 0x69, 0xdd, 0xb1, 0x5d, 0xf7, 0x09, 0x71, + 0x5c, 0x6a, 0x5b, 0xdb, 0xd5, 0x5f, 0x13, 0xc3, 0x43, 0x64, 0x8f, 0x38, 0xc4, 0x32, 0x08, 0x5c, + 0x05, 0x99, 0x03, 0x6a, 0x99, 0xd2, 0xdc, 0x39, 0xdf, 0xdc, 0x87, 0xd4, 0x32, 0x11, 0x3f, 0x61, + 0x14, 0xdc, 0x21, 0xa9, 0x38, 0x45, 0xc4, 0xda, 0x12, 0x00, 0xb8, 0x49, 0xa5, 0x00, 0xa9, 0x15, + 0x94, 0x74, 0x60, 0xad, 0xbc, 0x21, 0x4f, 0x50, 0x84, 0x4a, 0xfd, 0xbb, 0x02, 0xce, 0xfe, 0xe0, + 0xb9, 0x47, 0x1c, 0x0b, 0xd7, 0x63, 0x81, 0x56, 0x01, 0x53, 0x0d, 0xbe, 0xe6, 0x2a, 0x65, 0x4b, + 0xdf, 0x1c, 0xe9, 0xb9, 0x0d, 0x93, 0x58, 0x1e, 0xdd, 0xa3, 0xc4, 0x09, 0xe3, 0x44, 0x9c, 0x20, + 0x09, 0x75, 0xec, 0x81, 0xa7, 0xfe, 0xbb, 0x57, 0x7d, 0x11, 0x3e, 0x9f, 0x44, 0xfd, 0x4f, 0x15, + 0x4e, 0xea, 0x9f, 0x15, 0xb0, 0xf4, 0xa0, 0xbc, 0x56, 0x11, 0xdc, 0x65, 0xbb, 0x4e, 0x8d, 0x43, + 0x78, 0x17, 0x64, 0xbc, 0xc3, 0xa6, 0x9f, 0x01, 0x57, 0xfc, 0x0b, 0xdf, 0x39, 0x6c, 0xb2, 0x0c, + 0x38, 0x9b, 0xa4, 0x67, 0xfb, 0x88, 0x73, 0xc0, 0xaf, 0x81, 0xc9, 0x36, 0x93, 0xcb, 0xb5, 0x9c, + 0xd4, 0xe7, 0x25, 0xeb, 0x24, 0x57, 0x06, 0x89, 0x33, 0x78, 0x0f, 0xcc, 0x37, 0x89, 0x43, 0x6d, + 0xb3, 0x42, 0x0c, 0xdb, 0x32, 0x5d, 0x1e, 0x30, 0x93, 0xfa, 0x39, 0x49, 0x3c, 0x5f, 0x8e, 0x1e, + 0xa2, 0x38, 0xad, 0xfa, 0x45, 0x0a, 0x2c, 0x86, 0x0a, 0xa0, 0x56, 0x9d, 0xb8, 0xf0, 0x57, 0x60, + 0xc5, 0xf5, 0x70, 0x95, 0xd6, 0xe9, 0x0b, 0xec, 0x51, 0xdb, 0xda, 0xa5, 0x96, 0x69, 0x3f, 0x8b, + 0xa3, 0xe7, 0xbb, 0x9d, 0xc2, 0x4a, 0x65, 0x20, 0x15, 0x1a, 0x82, 0x00, 0x1f, 0x82, 0x39, 0x97, + 0xd4, 0x89, 0xe1, 0x09, 0x7b, 0xa5, 0x5f, 0xae, 0x76, 0x3b, 0x85, 0xb9, 0x4a, 0x64, 0xff, 0x43, + 0xa7, 0x70, 0x26, 0xe6, 0x18, 0x71, 0x88, 0x62, 0xcc, 0xf0, 0xa7, 0x60, 0xa6, 0xc9, 0x7e, 0x51, + 0xe2, 0xe6, 0x52, 0xab, 0xe9, 0x11, 0x11, 0x92, 0xf4, 0xb5, 0xbe, 0x24, 0xbd, 0x34, 0x53, 0x96, + 0x20, 0x28, 0x80, 0x83, 0x3f, 0x07, 0xb3, 0x9e, 0x5d, 0x27, 0x0e, 0xb6, 0x0c, 0x92, 0xcb, 0xf0, + 0x38, 0xd1, 0x22, 0xd8, 0x41, 0x43, 0xd0, 0x9a, 0x07, 0x35, 0x2e, 0xcc, 0xef, 0x56, 0xda, 0xa3, + 0x16, 0xb6, 0x3c, 0xea, 0x1d, 0xea, 0xf3, 0xac, 0x8e, 0xec, 0xf8, 0x20, 0x28, 0xc4, 0x53, 0x5f, + 0xa7, 0xc0, 0x85, 0x07, 0xb6, 0x43, 0x5f, 0xb0, 0xca, 0x52, 0x2f, 0xdb, 0xe6, 0x9a, 0xd4, 0x94, + 0x38, 0xf0, 0x29, 0x98, 0x61, 0x1d, 0xcc, 0xc4, 0x1e, 0x96, 0x51, 0xff, 0xed, 0x61, 0x72, 0x5d, + 0x8d, 0x51, 0x6b, 0xed, 0x5b, 0x9a, 0x28, 0x46, 0x5b, 0xc4, 0xc3, 0x61, 0xbd, 0x08, 0xf7, 0x50, + 0x80, 0x0a, 0x7f, 0x02, 0x32, 0x6e, 0x93, 0x18, 0x32, 0xfa, 0xef, 0x0c, 0xf3, 0x58, 0x7f, 0x1d, + 0x2b, 0x4d, 0x62, 0x84, 0xb5, 0x8b, 0xad, 0x10, 0x47, 0x84, 0x4f, 0xc1, 0x94, 0xcb, 0xb3, 0x84, + 0x07, 0x4a, 0xb6, 0x74, 0xf7, 0x23, 0xb0, 0x45, 0x96, 0x05, 0xc9, 0x2b, 0xd6, 0x48, 0xe2, 0xaa, + 0xff, 0x51, 0x40, 0x61, 0x00, 0xa7, 0x4e, 0xf6, 0x71, 0x9b, 0xda, 0x0e, 0x7c, 0x04, 0xa6, 0xf9, + 0xce, 0xe3, 0xa6, 0x74, 0xe0, 0x8d, 0x23, 0x05, 0x05, 0x8f, 0x7f, 0x3d, 0xcb, 0x52, 0xbb, 0x22, + 0xd8, 0x91, 0x8f, 0x03, 0x77, 0xc1, 0x2c, 0xff, 0x79, 0xdf, 0x7e, 0x66, 0x49, 0xbf, 0x8d, 0x03, + 0xca, 0x23, 0xa1, 0xe2, 0x03, 0xa0, 0x10, 0x4b, 0xfd, 0x5d, 0x1a, 0xac, 0x0e, 0xb0, 0x67, 0xdd, + 0xb6, 0x4c, 0xca, 0x12, 0x08, 0x3e, 0x88, 0xd5, 0x90, 0xdb, 0x89, 0x1a, 0x72, 0x65, 0x14, 0x7f, + 0xa4, 0xa6, 0x6c, 0x06, 0x17, 0x94, 0x8a, 0x61, 0x49, 0x37, 0x7f, 0xe8, 0x14, 0xfa, 0x4c, 0x6d, + 0x5a, 0x80, 0x14, 0xbf, 0x0c, 0xd8, 0x06, 0xb0, 0x8e, 0x5d, 0x6f, 0xc7, 0xc1, 0x96, 0x2b, 0x24, + 0xd1, 0x06, 0x91, 0x57, 0x7f, 0xe3, 0x68, 0x41, 0xcb, 0x38, 0xf4, 0x15, 0xa9, 0x05, 0xdc, 0xec, + 0x41, 0x43, 0x7d, 0x24, 0xc0, 0x6f, 0x80, 0x29, 0x87, 0x60, 0xd7, 0xb6, 0x78, 0x62, 0xce, 0x86, + 0xc1, 0x82, 0xf8, 0x2e, 0x92, 0xa7, 0xf0, 0x3a, 0x98, 0x6e, 0x10, 0xd7, 0xc5, 0x35, 0x92, 0x9b, + 0xe4, 0x84, 0x41, 0xed, 0xde, 0x12, 0xdb, 0xc8, 0x3f, 0x57, 0xff, 0xab, 0x80, 0x4b, 0x03, 0xfc, + 0xb8, 0x49, 0x5d, 0x0f, 0xfe, 0xa2, 0x27, 0x2b, 0xb5, 0xa3, 0x19, 0xc8, 0xb8, 0x79, 0x4e, 0x06, + 0xc5, 0xc6, 0xdf, 0x89, 0x64, 0xe4, 0x2e, 0x98, 0xa4, 0x1e, 0x69, 0xf8, 0x45, 0xac, 0x34, 0x7e, + 0xda, 0x84, 0xed, 0x61, 0x83, 0x01, 0x21, 0x81, 0xa7, 0xbe, 0x4e, 0x0f, 0x34, 0x8b, 0xa5, 0x2d, + 0x6c, 0x83, 0x05, 0xbe, 0x92, 0x0d, 0x99, 0xec, 0x49, 0xe3, 0x86, 0x15, 0x85, 0x21, 0x03, 0x90, + 0x7e, 0x5e, 0x6a, 0xb1, 0x50, 0x89, 0xa1, 0xa2, 0x84, 0x14, 0x78, 0x0b, 0x64, 0x1b, 0xd4, 0x42, + 0xa4, 0x59, 0xa7, 0x06, 0x76, 0x65, 0x87, 0x5b, 0xec, 0x76, 0x0a, 0xd9, 0xad, 0x70, 0x1b, 0x45, + 0x69, 0xe0, 0x77, 0x41, 0xb6, 0x81, 0x9f, 0x07, 0x2c, 0xa2, 0x13, 0x9d, 0x91, 0xf2, 0xb2, 0x5b, + 0xe1, 0x11, 0x8a, 0xd2, 0xc1, 0x32, 0x8b, 0x01, 0xd6, 0xc3, 0xdd, 0x5c, 0x86, 0x3b, 0xf7, 0xeb, + 0x23, 0xbb, 0x3d, 0x2f, 0x6f, 0x91, 0x50, 0xe1, 0xdc, 0xc8, 0x87, 0x81, 0x26, 0x98, 0xa9, 0xca, + 0x52, 0xc3, 0xc3, 0x2a, 0x5b, 0xfa, 0xde, 0x47, 0xdc, 0x97, 0x44, 0xd0, 0xe7, 0x58, 0x48, 0xf8, + 0x2b, 0x14, 0x20, 0xab, 0xaf, 0x32, 0xe0, 0xf2, 0xd0, 0x12, 0x09, 0x7f, 0x08, 0xa0, 0x5d, 0x75, + 0x89, 0xd3, 0x26, 0xe6, 0x8f, 0xc4, 0x0b, 0x84, 0x0d, 0x8c, 0xec, 0xfe, 0xd2, 0xfa, 0x79, 0x96, + 0x4d, 0xdb, 0x3d, 0xa7, 0xa8, 0x0f, 0x07, 0x34, 0xc0, 0x3c, 0xcb, 0x31, 0x71, 0x63, 0x54, 0xce, + 0xa6, 0xe3, 0x25, 0xf0, 0x32, 0x1b, 0x35, 0x36, 0xa3, 0x20, 0x28, 0x8e, 0x09, 0xd7, 0xc0, 0xa2, + 0x1c, 0x93, 0x12, 0x37, 0x78, 0x41, 0xfa, 0x79, 0x71, 0x3d, 0x7e, 0x8c, 0x92, 0xf4, 0x0c, 0xc2, + 0x24, 0x2e, 0x75, 0x88, 0x19, 0x40, 0x64, 0xe2, 0x10, 0xf7, 0xe3, 0xc7, 0x28, 0x49, 0x0f, 0x6b, + 0x60, 0x41, 0xa2, 0xca, 0x5b, 0xcd, 0x4d, 0xf2, 0x98, 0x18, 0x3d, 0xc1, 0xca, 0xb6, 0x14, 0xc4, + 0xf7, 0x7a, 0x0c, 0x06, 0x25, 0x60, 0xa1, 0x0d, 0x80, 0xe1, 0x17, 0x4d, 0x37, 0x37, 0xc5, 0x85, + 0xdc, 0x1b, 0x3f, 0x4a, 0x82, 0xc2, 0x1b, 0x76, 0xf4, 0x60, 0xcb, 0x45, 0x11, 0x11, 0xea, 0x1f, + 0x15, 0xb0, 0x94, 0x9c, 0x80, 0x83, 0xc7, 0x86, 0x32, 0xf0, 0xb1, 0xf1, 0x4b, 0x30, 0x23, 0x06, + 0x2a, 0xdb, 0x91, 0xd7, 0xfe, 0x9d, 0x23, 0x96, 0x35, 0x5c, 0x25, 0xf5, 0x8a, 0x64, 0x15, 0x41, + 0xec, 0xaf, 0x50, 0x00, 0xa9, 0xbe, 0xcc, 0x00, 0x10, 0xe6, 0x14, 0xbc, 0x1d, 0xeb, 0x63, 0xab, + 0x89, 0x3e, 0xb6, 0x14, 0x7d, 0xb9, 0x44, 0x7a, 0xd6, 0x23, 0x30, 0x65, 0xf3, 0x32, 0x23, 0x35, + 0xbc, 0x39, 0xc4, 0x8f, 0xc1, 0xbc, 0x13, 0x00, 0xe9, 0x80, 0x35, 0x06, 0x59, 0xa7, 0x24, 0x10, + 0xdc, 0x00, 0x99, 0xa6, 0x6d, 0xfa, 0x53, 0xca, 0xb0, 0x99, 0xb1, 0x6c, 0x9b, 0x6e, 0x0c, 0x6e, + 0x86, 0x69, 0xcc, 0x76, 0x11, 0x87, 0x60, 0x23, 0xa8, 0x3f, 0xf9, 0xc9, 0x31, 0xb1, 0x38, 0x04, + 0xae, 0xdf, 0xd7, 0x00, 0xe1, 0x3d, 0xff, 0x04, 0x05, 0x70, 0xf0, 0x37, 0x60, 0xd9, 0x48, 0xbe, + 0xae, 0x73, 0xd3, 0x23, 0x07, 0xab, 0xa1, 0x9f, 0x1e, 0xf4, 0x73, 0xdd, 0x4e, 0x61, 0xb9, 0x87, + 0x04, 0xf5, 0x4a, 0x62, 0x96, 0x11, 0xf9, 0x28, 0x93, 0x75, 0x6e, 0x98, 0x65, 0xfd, 0x9e, 0x9f, + 0xc2, 0x32, 0xff, 0x04, 0x05, 0x70, 0xea, 0x9f, 0x32, 0x60, 0x2e, 0xf6, 0xd0, 0x3b, 0xe1, 0xc8, + 0x10, 0xc9, 0x7c, 0x6c, 0x91, 0x21, 0xe0, 0x8e, 0x35, 0x32, 0x04, 0xe4, 0x09, 0x45, 0x86, 0x10, + 0x76, 0x42, 0x91, 0x11, 0xb1, 0xac, 0x4f, 0x64, 0xfc, 0x2b, 0xe5, 0x47, 0x86, 0x18, 0x16, 0x8e, + 0x16, 0x19, 0x82, 0x36, 0x12, 0x19, 0xdb, 0xd1, 0xb7, 0xf3, 0xf8, 0x2f, 0xb7, 0xd9, 0x9e, 0x77, + 0xb6, 0x09, 0xe6, 0x70, 0x9b, 0x38, 0xb8, 0x46, 0xf8, 0xb6, 0x8c, 0x8f, 0x71, 0x71, 0x97, 0xd8, + 0x33, 0x77, 0x2d, 0x82, 0x83, 0x62, 0xa8, 0xac, 0xa5, 0xcb, 0xf5, 0x63, 0x2f, 0x78, 0x3f, 0xcb, + 0x2e, 0xc7, 0x5b, 0xfa, 0x5a, 0xcf, 0x29, 0xea, 0xc3, 0xa1, 0xfe, 0x21, 0x05, 0x96, 0x7b, 0xbe, + 0x5c, 0x84, 0x4e, 0x51, 0x3e, 0x91, 0x53, 0x52, 0x27, 0xe8, 0x94, 0xf4, 0xd8, 0x4e, 0xf9, 0x6b, + 0x0a, 0xc0, 0xde, 0xfe, 0x00, 0x0f, 0xf9, 0x58, 0x61, 0x38, 0xb4, 0x4a, 0x4c, 0x71, 0xfc, 0x15, + 0x67, 0xe0, 0xe8, 0x38, 0x12, 0x85, 0x45, 0x49, 0x39, 0xc7, 0xff, 0x05, 0x37, 0xfc, 0x5e, 0x96, + 0x3e, 0xb6, 0xef, 0x65, 0xea, 0x3f, 0x92, 0x7e, 0x3b, 0x85, 0xdf, 0xe6, 0xfa, 0xdd, 0x72, 0xfa, + 0x64, 0x6e, 0x59, 0xfd, 0x9b, 0x02, 0x96, 0x92, 0x63, 0xc4, 0x29, 0xf9, 0x30, 0xfb, 0xcf, 0xb8, + 0xea, 0xa7, 0xf1, 0xa3, 0xec, 0x2b, 0x05, 0x9c, 0x3d, 0x3d, 0xff, 0xc1, 0xa8, 0x7f, 0xe9, 0x55, + 0xf7, 0x14, 0xfc, 0x93, 0xa2, 0x7f, 0xff, 0xcd, 0xfb, 0xfc, 0xc4, 0xdb, 0xf7, 0xf9, 0x89, 0x77, + 0xef, 0xf3, 0x13, 0xbf, 0xed, 0xe6, 0x95, 0x37, 0xdd, 0xbc, 0xf2, 0xb6, 0x9b, 0x57, 0xde, 0x75, + 0xf3, 0xca, 0xff, 0xba, 0x79, 0xe5, 0xf7, 0xff, 0xcf, 0x4f, 0xfc, 0xec, 0xe2, 0xc0, 0xbf, 0x21, + 0xbf, 0x0c, 0x00, 0x00, 0xff, 0xff, 0xbe, 0x23, 0xae, 0x54, 0xa2, 0x1c, 0x00, 0x00, } func (m *ContainerResourceMetricSource) Marshal() (dAtA []byte, err error) { @@ -1126,6 +1127,18 @@ func (m *HPAScalingRules) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Tolerance != nil { + { + size, err := m.Tolerance.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } if m.StabilizationWindowSeconds != nil { i = encodeVarintGenerated(dAtA, i, uint64(*m.StabilizationWindowSeconds)) i-- @@ -2203,6 +2216,10 @@ func (m *HPAScalingRules) Size() (n int) { if m.StabilizationWindowSeconds != nil { n += 1 + sovGenerated(uint64(*m.StabilizationWindowSeconds)) } + if m.Tolerance != nil { + l = m.Tolerance.Size() + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -2619,6 +2636,7 @@ func (this *HPAScalingRules) String() string { `SelectPolicy:` + valueToStringGenerated(this.SelectPolicy) + `,`, `Policies:` + repeatedStringForPolicies + `,`, `StabilizationWindowSeconds:` + valueToStringGenerated(this.StabilizationWindowSeconds) + `,`, + `Tolerance:` + strings.Replace(fmt.Sprintf("%v", this.Tolerance), "Quantity", "resource.Quantity", 1) + `,`, `}`, }, "") return s @@ -3770,6 +3788,42 @@ func (m *HPAScalingRules) Unmarshal(dAtA []byte) error { } } m.StabilizationWindowSeconds = &v + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Tolerance", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Tolerance == nil { + m.Tolerance = &resource.Quantity{} + } + if err := m.Tolerance.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/staging/src/k8s.io/api/autoscaling/v2/generated.proto b/staging/src/k8s.io/api/autoscaling/v2/generated.proto index 4e6dc0592aa..04c34d6e168 100644 --- a/staging/src/k8s.io/api/autoscaling/v2/generated.proto +++ b/staging/src/k8s.io/api/autoscaling/v2/generated.proto @@ -112,12 +112,18 @@ message HPAScalingPolicy { optional int32 periodSeconds = 3; } -// HPAScalingRules configures the scaling behavior for one direction. -// These Rules are applied after calculating DesiredReplicas from metrics for the HPA. +// HPAScalingRules configures the scaling behavior for one direction via +// scaling Policy Rules and a configurable metric tolerance. +// +// Scaling Policy Rules are applied after calculating DesiredReplicas from metrics for the HPA. // They can limit the scaling velocity by specifying scaling policies. // They can prevent flapping by specifying the stabilization window, so that the // number of replicas is not set instantly, instead, the safest value from the stabilization // window is chosen. +// +// The tolerance is applied to the metric values and prevents scaling too +// eagerly for small metric variations. (Note that setting a tolerance requires +// enabling the alpha HPAConfigurableTolerance feature gate.) message HPAScalingRules { // stabilizationWindowSeconds is the number of seconds for which past recommendations should be // considered while scaling up or scaling down. @@ -134,10 +140,28 @@ message HPAScalingRules { optional string selectPolicy = 1; // policies is a list of potential scaling polices which can be used during scaling. - // At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + // If not set, use the default values: + // - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + // - For scale down: allow all pods to be removed in a 15s window. // +listType=atomic // +optional repeated HPAScalingPolicy policies = 2; + + // tolerance is the tolerance on the ratio between the current and desired + // metric value under which no updates are made to the desired number of + // replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + // set, the default cluster-wide tolerance is applied (by default 10%). + // + // For example, if autoscaling is configured with a memory consumption target of 100Mi, + // and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + // triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + // + // This is an alpha field and requires enabling the HPAConfigurableTolerance + // feature gate. + // + // +featureGate=HPAConfigurableTolerance + // +optional + optional .k8s.io.apimachinery.pkg.api.resource.Quantity tolerance = 4; } // HorizontalPodAutoscaler is the configuration for a horizontal pod diff --git a/staging/src/k8s.io/api/autoscaling/v2/types_swagger_doc_generated.go b/staging/src/k8s.io/api/autoscaling/v2/types_swagger_doc_generated.go index 649cd04a03c..017fefcde72 100644 --- a/staging/src/k8s.io/api/autoscaling/v2/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/autoscaling/v2/types_swagger_doc_generated.go @@ -92,10 +92,11 @@ func (HPAScalingPolicy) SwaggerDoc() map[string]string { } var map_HPAScalingRules = map[string]string{ - "": "HPAScalingRules configures the scaling behavior for one direction. These Rules are applied after calculating DesiredReplicas from metrics for the HPA. They can limit the scaling velocity by specifying scaling policies. They can prevent flapping by specifying the stabilization window, so that the number of replicas is not set instantly, instead, the safest value from the stabilization window is chosen.", + "": "HPAScalingRules configures the scaling behavior for one direction via scaling Policy Rules and a configurable metric tolerance.\n\nScaling Policy Rules are applied after calculating DesiredReplicas from metrics for the HPA. They can limit the scaling velocity by specifying scaling policies. They can prevent flapping by specifying the stabilization window, so that the number of replicas is not set instantly, instead, the safest value from the stabilization window is chosen.\n\nThe tolerance is applied to the metric values and prevents scaling too eagerly for small metric variations. (Note that setting a tolerance requires enabling the alpha HPAConfigurableTolerance feature gate.)", "stabilizationWindowSeconds": "stabilizationWindowSeconds is the number of seconds for which past recommendations should be considered while scaling up or scaling down. StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). If not set, use the default values: - For scale up: 0 (i.e. no stabilization is done). - For scale down: 300 (i.e. the stabilization window is 300 seconds long).", "selectPolicy": "selectPolicy is used to specify which policy should be used. If not set, the default value Max is used.", - "policies": "policies is a list of potential scaling polices which can be used during scaling. At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid", + "policies": "policies is a list of potential scaling polices which can be used during scaling. If not set, use the default values: - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. - For scale down: allow all pods to be removed in a 15s window.", + "tolerance": "tolerance is the tolerance on the ratio between the current and desired metric value under which no updates are made to the desired number of replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not set, the default cluster-wide tolerance is applied (by default 10%).\n\nFor example, if autoscaling is configured with a memory consumption target of 100Mi, and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be triggered when the actual consumption falls below 95Mi or exceeds 101Mi.\n\nThis is an alpha field and requires enabling the HPAConfigurableTolerance feature gate.", } func (HPAScalingRules) SwaggerDoc() map[string]string { diff --git a/staging/src/k8s.io/api/autoscaling/v2/zz_generated.deepcopy.go b/staging/src/k8s.io/api/autoscaling/v2/zz_generated.deepcopy.go index 125708d6fda..5fbcf9f807c 100644 --- a/staging/src/k8s.io/api/autoscaling/v2/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/autoscaling/v2/zz_generated.deepcopy.go @@ -146,6 +146,11 @@ func (in *HPAScalingRules) DeepCopyInto(out *HPAScalingRules) { *out = make([]HPAScalingPolicy, len(*in)) copy(*out, *in) } + if in.Tolerance != nil { + in, out := &in.Tolerance, &out.Tolerance + x := (*in).DeepCopy() + *out = &x + } return } diff --git a/staging/src/k8s.io/client-go/applyconfigurations/autoscaling/v2/hpascalingrules.go b/staging/src/k8s.io/client-go/applyconfigurations/autoscaling/v2/hpascalingrules.go index 6a6a2655f5e..6fd0f25cc1d 100644 --- a/staging/src/k8s.io/client-go/applyconfigurations/autoscaling/v2/hpascalingrules.go +++ b/staging/src/k8s.io/client-go/applyconfigurations/autoscaling/v2/hpascalingrules.go @@ -20,6 +20,7 @@ package v2 import ( autoscalingv2 "k8s.io/api/autoscaling/v2" + resource "k8s.io/apimachinery/pkg/api/resource" ) // HPAScalingRulesApplyConfiguration represents a declarative configuration of the HPAScalingRules type for use @@ -28,6 +29,7 @@ type HPAScalingRulesApplyConfiguration struct { StabilizationWindowSeconds *int32 `json:"stabilizationWindowSeconds,omitempty"` SelectPolicy *autoscalingv2.ScalingPolicySelect `json:"selectPolicy,omitempty"` Policies []HPAScalingPolicyApplyConfiguration `json:"policies,omitempty"` + Tolerance *resource.Quantity `json:"tolerance,omitempty"` } // HPAScalingRulesApplyConfiguration constructs a declarative configuration of the HPAScalingRules type for use with @@ -64,3 +66,11 @@ func (b *HPAScalingRulesApplyConfiguration) WithPolicies(values ...*HPAScalingPo } return b } + +// WithTolerance sets the Tolerance field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Tolerance field is set to the value of the last call. +func (b *HPAScalingRulesApplyConfiguration) WithTolerance(value resource.Quantity) *HPAScalingRulesApplyConfiguration { + b.Tolerance = &value + return b +} diff --git a/staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go b/staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go index 769de4016f9..d961f9f3f09 100644 --- a/staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go +++ b/staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go @@ -2934,6 +2934,9 @@ var schemaYAML = typed.YAMLObject(`types: - name: stabilizationWindowSeconds type: scalar: numeric + - name: tolerance + type: + namedType: io.k8s.apimachinery.pkg.api.resource.Quantity - name: io.k8s.api.autoscaling.v2.HorizontalPodAutoscaler map: fields: diff --git a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml index 1b748bcc5fd..7de737cbe21 100644 --- a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml +++ b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml @@ -539,6 +539,12 @@ lockToDefault: true preRelease: GA version: "1.33" +- name: HPAConfigurableTolerance + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.33" - name: HPAScaleToZero versionedSpecs: - default: false From 4db8e8cc1dc2e5683c878b3ef29cb2e0fbe70f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Marc=20Fran=C3=A7ois?= Date: Fri, 21 Mar 2025 17:42:25 -0400 Subject: [PATCH 7/7] Add configurable tolerance e2e test. --- .../horizontal_pod_autoscaling_behavior.go | 107 +++++++++++++----- test/e2e/feature/feature.go | 4 + .../autoscaling/autoscaling_utils.go | 7 ++ 3 files changed, 90 insertions(+), 28 deletions(-) diff --git a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go index d457af524c2..88ab6fa5207 100644 --- a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go +++ b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go @@ -21,6 +21,7 @@ import ( "time" autoscalingv2 "k8s.io/api/autoscaling/v2" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2eautoscaling "k8s.io/kubernetes/test/e2e/framework/autoscaling" @@ -30,38 +31,30 @@ import ( "github.com/onsi/gomega" ) +const ( + hpaName = "consumer" + + podCPURequest = 500 + targetCPUUtilizationPercent = 25 + + fullWindowOfNewUsage = 30 * time.Second + windowWithOldUsagePasses = 30 * time.Second + newPodMetricsDelay = 15 * time.Second + metricsAvailableDelay = fullWindowOfNewUsage + windowWithOldUsagePasses + newPodMetricsDelay + + hpaReconciliationInterval = 15 * time.Second + actuationDelay = 10 * time.Second + maxHPAReactionTime = metricsAvailableDelay + hpaReconciliationInterval + actuationDelay + + maxConsumeCPUDelay = 30 * time.Second + waitForReplicasPollInterval = 20 * time.Second + maxResourceConsumerDelay = maxConsumeCPUDelay + waitForReplicasPollInterval +) + var _ = SIGDescribe(feature.HPA, framework.WithSerial(), framework.WithSlow(), "Horizontal pod autoscaling (non-default behavior)", func() { f := framework.NewDefaultFramework("horizontal-pod-autoscaling") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged - hpaName := "consumer" - - podCPURequest := 500 - targetCPUUtilizationPercent := 25 - - // usageForReplicas returns usage for (n - 0.5) replicas as if they would consume all CPU - // under the target. The 0.5 replica reduction is to accommodate for the deviation between - // the actual consumed cpu and requested usage by the ResourceConsumer. - // HPA rounds up the recommendations. So, if the usage is e.g. for 3.5 replicas, - // the recommended replica number will be 4. - usageForReplicas := func(replicas int) int { - usagePerReplica := podCPURequest * targetCPUUtilizationPercent / 100 - return replicas*usagePerReplica - usagePerReplica/2 - } - - fullWindowOfNewUsage := 30 * time.Second - windowWithOldUsagePasses := 30 * time.Second - newPodMetricsDelay := 15 * time.Second - metricsAvailableDelay := fullWindowOfNewUsage + windowWithOldUsagePasses + newPodMetricsDelay - - hpaReconciliationInterval := 15 * time.Second - actuationDelay := 10 * time.Second - maxHPAReactionTime := metricsAvailableDelay + hpaReconciliationInterval + actuationDelay - - maxConsumeCPUDelay := 30 * time.Second - waitForReplicasPollInterval := 20 * time.Second - maxResourceConsumerDelay := maxConsumeCPUDelay + waitForReplicasPollInterval - waitBuffer := 1 * time.Minute ginkgo.Describe("with short downscale stabilization window", func() { @@ -505,3 +498,61 @@ var _ = SIGDescribe(feature.HPA, framework.WithSerial(), framework.WithSlow(), " }) }) }) + +var _ = SIGDescribe(feature.HPAConfigurableTolerance, framework.WithFeatureGate(features.HPAConfigurableTolerance), + framework.WithSerial(), framework.WithSlow(), "Horizontal pod autoscaling (configurable tolerance)", func() { + f := framework.NewDefaultFramework("horizontal-pod-autoscaling") + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + + waitBuffer := 1 * time.Minute + + ginkgo.Describe("with large configurable tolerance", func() { + ginkgo.It("should not scale", func(ctx context.Context) { + ginkgo.By("setting up resource consumer and HPA") + initPods := 1 + initCPUUsageTotal := usageForReplicas(initPods) + + rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, + hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, + initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, + f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, + ) + ginkgo.DeferCleanup(rc.CleanUp) + + scaleRule := e2eautoscaling.HPAScalingRuleWithToleranceMilli(10000) + hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, + rc, int32(targetCPUUtilizationPercent), 1, 10, + e2eautoscaling.HPABehaviorWithScaleUpAndDownRules(scaleRule, scaleRule), + ) + ginkgo.DeferCleanup(e2eautoscaling.DeleteHPAWithBehavior, rc, hpa.Name) + + waitDeadline := maxHPAReactionTime + maxResourceConsumerDelay + waitBuffer + + ginkgo.By("trying to trigger scale up") + rc.ConsumeCPU(usageForReplicas(8)) + waitStart := time.Now() + + rc.EnsureDesiredReplicasInRange(ctx, initPods, initPods, waitDeadline, hpa.Name) + timeWaited := time.Since(waitStart) + + ginkgo.By("verifying time waited for a scale up") + framework.Logf("time waited for scale up: %s", timeWaited) + gomega.Expect(timeWaited).To(gomega.BeNumerically(">", waitDeadline), "waited %s, wanted to wait more than %s", timeWaited, waitDeadline) + + ginkgo.By("verifying number of replicas") + replicas, err := rc.GetReplicas(ctx) + framework.ExpectNoError(err) + gomega.Expect(replicas).To(gomega.BeNumerically("==", initPods), "had %s replicas, still have %s replicas after time deadline", initPods, replicas) + }) + }) + }) + +// usageForReplicas returns usage for (n - 0.5) replicas as if they would consume all CPU +// under the target. The 0.5 replica reduction is to accommodate for the deviation between +// the actual consumed cpu and requested usage by the ResourceConsumer. +// HPA rounds up the recommendations. So, if the usage is e.g. for 3.5 replicas, +// the recommended replica number will be 4. +func usageForReplicas(replicas int) int { + usagePerReplica := podCPURequest * targetCPUUtilizationPercent / 100 + return replicas*usagePerReplica - usagePerReplica/2 +} diff --git a/test/e2e/feature/feature.go b/test/e2e/feature/feature.go index b785aeee439..9baa3227f60 100644 --- a/test/e2e/feature/feature.go +++ b/test/e2e/feature/feature.go @@ -219,6 +219,10 @@ var ( // TODO: document the feature (owning SIG, when to use this feature for a test) HPA = framework.WithFeature(framework.ValidFeatures.Add("HPA")) + // OWNER: sig-autoscaling + // Marks tests that require HPA configurable tolerance (https://kep.k8s.io/4951). + HPAConfigurableTolerance = framework.WithFeature(framework.ValidFeatures.Add("HPAConfigurableTolerance")) + // owner: sig-node HostAccess = framework.WithFeature(framework.ValidFeatures.Add("HostAccess")) diff --git a/test/e2e/framework/autoscaling/autoscaling_utils.go b/test/e2e/framework/autoscaling/autoscaling_utils.go index b90965ebbcc..d015275f71f 100644 --- a/test/e2e/framework/autoscaling/autoscaling_utils.go +++ b/test/e2e/framework/autoscaling/autoscaling_utils.go @@ -880,6 +880,13 @@ func HPAScalingRuleWithScalingPolicy(policyType autoscalingv2.HPAScalingPolicyTy } } +func HPAScalingRuleWithToleranceMilli(toleranceMilli int64) *autoscalingv2.HPAScalingRules { + quantity := resource.NewMilliQuantity(toleranceMilli, resource.DecimalSI) + return &autoscalingv2.HPAScalingRules{ + Tolerance: quantity, + } +} + func HPABehaviorWithStabilizationWindows(upscaleStabilization, downscaleStabilization time.Duration) *autoscalingv2.HorizontalPodAutoscalerBehavior { scaleUpRule := HPAScalingRuleWithStabilizationWindow(int32(upscaleStabilization.Seconds())) scaleDownRule := HPAScalingRuleWithStabilizationWindow(int32(downscaleStabilization.Seconds()))