HPA scale-to-zero for custom object/external metrics

Add support for scaling to zero pods

minReplicas is allowed to be zero

condition is set once

Based on https://github.com/kubernetes/kubernetes/pull/61423

set original valid condition

add scale to/from zero and invalid metric tests

Scaling up from zero pods ignores tolerance

validate metrics when minReplicas is 0

Document HPA behaviour when minReplicas is 0

Documented minReplicas field in autoscaling APIs

pkg/apis/autoscaling/types.go

@@ -76,8 +76,11 @@ type HorizontalPodAutoscalerSpec struct {
 	// ScaleTargetRef points to the target resource to scale, and is used to the pods for which metrics
 	// should be collected, as well as to actually change the replica count.
 	ScaleTargetRef CrossVersionObjectReference
-	// MinReplicas is the lower limit for the number of replicas to which the autoscaler can scale down.
-	// It defaults to 1 pod.
+	// minReplicas is the lower limit for the number of replicas to which the autoscaler
+	// can scale down.  It defaults to 1 pod.  minReplicas is allowed to be 0 if the
+	// alpha feature gate HPAScaleToZero is enabled and at least one Object or External
+	// metric is configured.  Scaling is active as long as at least one metric value is
+	// available.
 	// +optional
 	MinReplicas *int32
 	// MaxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up.

pkg/apis/autoscaling/validation/validation.go

@@ -17,14 +17,17 @@ limitations under the License.
 package validation
 
 import (
+	"fmt"
 	"strings"
 
 	apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation"
 	pathvalidation "k8s.io/apimachinery/pkg/api/validation/path"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/apis/autoscaling"
 	apivalidation "k8s.io/kubernetes/pkg/apis/core/validation"
+	"k8s.io/kubernetes/pkg/features"
 )
 
 func ValidateScale(scale *autoscaling.Scale) field.ErrorList {
@@ -42,10 +45,12 @@ func ValidateScale(scale *autoscaling.Scale) field.ErrorList {
 // Prefix indicates this name will be used as part of generation, in which case trailing dashes are allowed.
 var ValidateHorizontalPodAutoscalerName = apivalidation.ValidateReplicationControllerName
 
-func validateHorizontalPodAutoscalerSpec(autoscaler autoscaling.HorizontalPodAutoscalerSpec, fldPath *field.Path) field.ErrorList {
+func validateHorizontalPodAutoscalerSpec(autoscaler autoscaling.HorizontalPodAutoscalerSpec, fldPath *field.Path, minReplicasLowerBound int32) field.ErrorList {
 	allErrs := field.ErrorList{}
-	if autoscaler.MinReplicas != nil && *autoscaler.MinReplicas < 1 {
-		allErrs = append(allErrs, field.Invalid(fldPath.Child("minReplicas"), *autoscaler.MinReplicas, "must be greater than 0"))
+
+	if autoscaler.MinReplicas != nil && *autoscaler.MinReplicas < minReplicasLowerBound {
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("minReplicas"), *autoscaler.MinReplicas,
+			fmt.Sprintf("must be greater than or equal to %d", minReplicasLowerBound)))
 	}
 	if autoscaler.MaxReplicas < 1 {
 		allErrs = append(allErrs, field.Invalid(fldPath.Child("maxReplicas"), autoscaler.MaxReplicas, "must be greater than 0"))
@@ -56,7 +61,7 @@ func validateHorizontalPodAutoscalerSpec(autoscaler autoscaling.HorizontalPodAut
 	if refErrs := ValidateCrossVersionObjectReference(autoscaler.ScaleTargetRef, fldPath.Child("scaleTargetRef")); len(refErrs) > 0 {
 		allErrs = append(allErrs, refErrs...)
 	}
-	if refErrs := validateMetrics(autoscaler.Metrics, fldPath.Child("metrics")); len(refErrs) > 0 {
+	if refErrs := validateMetrics(autoscaler.Metrics, fldPath.Child("metrics"), autoscaler.MinReplicas); len(refErrs) > 0 {
 		allErrs = append(allErrs, refErrs...)
 	}
 	return allErrs
@@ -85,13 +90,34 @@ func ValidateCrossVersionObjectReference(ref autoscaling.CrossVersionObjectRefer
 
 func ValidateHorizontalPodAutoscaler(autoscaler *autoscaling.HorizontalPodAutoscaler) field.ErrorList {
 	allErrs := apivalidation.ValidateObjectMeta(&autoscaler.ObjectMeta, true, ValidateHorizontalPodAutoscalerName, field.NewPath("metadata"))
-	allErrs = append(allErrs, validateHorizontalPodAutoscalerSpec(autoscaler.Spec, field.NewPath("spec"))...)
+
+	// MinReplicasLowerBound represents a minimum value for minReplicas
+	// 0 when HPA scale-to-zero feature is enabled
+	var minReplicasLowerBound int32
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.HPAScaleToZero) {
+		minReplicasLowerBound = 0
+	} else {
+		minReplicasLowerBound = 1
+	}
+	allErrs = append(allErrs, validateHorizontalPodAutoscalerSpec(autoscaler.Spec, field.NewPath("spec"), minReplicasLowerBound)...)
 	return allErrs
 }
 
 func ValidateHorizontalPodAutoscalerUpdate(newAutoscaler, oldAutoscaler *autoscaling.HorizontalPodAutoscaler) field.ErrorList {
 	allErrs := apivalidation.ValidateObjectMetaUpdate(&newAutoscaler.ObjectMeta, &oldAutoscaler.ObjectMeta, field.NewPath("metadata"))
-	allErrs = append(allErrs, validateHorizontalPodAutoscalerSpec(newAutoscaler.Spec, field.NewPath("spec"))...)
+
+	// minReplicasLowerBound represents a minimum value for minReplicas
+	// 0 when HPA scale-to-zero feature is enabled or HPA object already has minReplicas=0
+	var minReplicasLowerBound int32
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.HPAScaleToZero) || (oldAutoscaler.Spec.MinReplicas != nil && *oldAutoscaler.Spec.MinReplicas == 0) {
+		minReplicasLowerBound = 0
+	} else {
+		minReplicasLowerBound = 1
+	}
+
+	allErrs = append(allErrs, validateHorizontalPodAutoscalerSpec(newAutoscaler.Spec, field.NewPath("spec"), minReplicasLowerBound)...)
 	return allErrs
 }
 
@@ -103,14 +129,28 @@ func ValidateHorizontalPodAutoscalerStatusUpdate(newAutoscaler, oldAutoscaler *a
 	return allErrs
 }
 
-func validateMetrics(metrics []autoscaling.MetricSpec, fldPath *field.Path) field.ErrorList {
+func validateMetrics(metrics []autoscaling.MetricSpec, fldPath *field.Path, minReplicas *int32) field.ErrorList {
 	allErrs := field.ErrorList{}
+	hasObjectMetrics := false
+	hasExternalMetrics := false
 
 	for i, metricSpec := range metrics {
 		idxPath := fldPath.Index(i)
 		if targetErrs := validateMetricSpec(metricSpec, idxPath); len(targetErrs) > 0 {
 			allErrs = append(allErrs, targetErrs...)
 		}
+		if metricSpec.Type == autoscaling.ObjectMetricSourceType {
+			hasObjectMetrics = true
+		}
+		if metricSpec.Type == autoscaling.ExternalMetricSourceType {
+			hasExternalMetrics = true
+		}
+	}
+
+	if minReplicas != nil && *minReplicas == 0 {
+		if !hasObjectMetrics && !hasExternalMetrics {
+			allErrs = append(allErrs, field.Forbidden(fldPath, "must specify at least one Object or External metric to support scaling to zero replicas"))
+		}
 	}
 
 	return allErrs

pkg/apis/autoscaling/validation/validation_test.go

@@ -22,8 +22,11 @@ import (
 
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	"k8s.io/kubernetes/pkg/apis/autoscaling"
 	api "k8s.io/kubernetes/pkg/apis/core"
+	"k8s.io/kubernetes/pkg/features"
 	utilpointer "k8s.io/utils/pointer"
 )
 
@@ -96,6 +99,7 @@ func TestValidateHorizontalPodAutoscaler(t *testing.T) {
 	if err != nil {
 		t.Errorf("unable to parse label selector: %v", err)
 	}
+
 	successCases := []autoscaling.HorizontalPodAutoscaler{
 		{
 			ObjectMeta: metav1.ObjectMeta{
@@ -396,7 +400,7 @@ func TestValidateHorizontalPodAutoscaler(t *testing.T) {
 					MaxReplicas:    5,
 				},
 			},
-			msg: "must be greater than 0",
+			msg: "must be greater than or equal to 1",
 		},
 		{
 			horizontalPodAutoscaler: autoscaling.HorizontalPodAutoscaler{
@@ -1002,3 +1006,162 @@ func TestValidateHorizontalPodAutoscaler(t *testing.T) {
 		}
 	}
 }
+
+func prepareMinReplicasCases(t *testing.T, minReplicas int32) []autoscaling.HorizontalPodAutoscaler {
+	metricLabelSelector, err := metav1.ParseToLabelSelector("label=value")
+	if err != nil {
+		t.Errorf("unable to parse label selector: %v", err)
+	}
+	minReplicasCases := []autoscaling.HorizontalPodAutoscaler{
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:            "myautoscaler",
+				Namespace:       metav1.NamespaceDefault,
+				ResourceVersion: "theversion",
+			},
+			Spec: autoscaling.HorizontalPodAutoscalerSpec{
+				ScaleTargetRef: autoscaling.CrossVersionObjectReference{
+					Kind: "ReplicationController",
+					Name: "myrc",
+				},
+				MinReplicas: utilpointer.Int32Ptr(minReplicas),
+				MaxReplicas: 5,
+				Metrics: []autoscaling.MetricSpec{
+					{
+						Type: autoscaling.ObjectMetricSourceType,
+						Object: &autoscaling.ObjectMetricSource{
+							DescribedObject: autoscaling.CrossVersionObjectReference{
+								Kind: "ReplicationController",
+								Name: "myrc",
+							},
+							Metric: autoscaling.MetricIdentifier{
+								Name: "somemetric",
+							},
+							Target: autoscaling.MetricTarget{
+								Type:  autoscaling.ValueMetricType,
+								Value: resource.NewMilliQuantity(300, resource.DecimalSI),
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:            "myautoscaler",
+				Namespace:       metav1.NamespaceDefault,
+				ResourceVersion: "theversion",
+			},
+			Spec: autoscaling.HorizontalPodAutoscalerSpec{
+				ScaleTargetRef: autoscaling.CrossVersionObjectReference{
+					Kind: "ReplicationController",
+					Name: "myrc",
+				},
+				MinReplicas: utilpointer.Int32Ptr(minReplicas),
+				MaxReplicas: 5,
+				Metrics: []autoscaling.MetricSpec{
+					{
+						Type: autoscaling.ExternalMetricSourceType,
+						External: &autoscaling.ExternalMetricSource{
+							Metric: autoscaling.MetricIdentifier{
+								Name:     "somemetric",
+								Selector: metricLabelSelector,
+							},
+							Target: autoscaling.MetricTarget{
+								Type:         autoscaling.AverageValueMetricType,
+								AverageValue: resource.NewMilliQuantity(300, resource.DecimalSI),
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	return minReplicasCases
+}
+
+func TestValidateHorizontalPodAutoscalerScaleToZeroEnabled(t *testing.T) {
+	// Enable HPAScaleToZero feature gate.
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, true)()
+
+	zeroMinReplicasCases := prepareMinReplicasCases(t, 0)
+	for _, successCase := range zeroMinReplicasCases {
+		if errs := ValidateHorizontalPodAutoscaler(&successCase); len(errs) != 0 {
+			t.Errorf("expected success: %v", errs)
+		}
+	}
+}
+
+func TestValidateHorizontalPodAutoscalerScaleToZeroDisabled(t *testing.T) {
+	// Disable HPAScaleToZero feature gate.
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, false)()
+
+	zeroMinReplicasCases := prepareMinReplicasCases(t, 0)
+	errorMsg := "must be greater than or equal to 1"
+
+	for _, errorCase := range zeroMinReplicasCases {
+		errs := ValidateHorizontalPodAutoscaler(&errorCase)
+		if len(errs) == 0 {
+			t.Errorf("expected failure for %q", errorMsg)
+		} else if !strings.Contains(errs[0].Error(), errorMsg) {
+			t.Errorf("unexpected error: %q, expected: %q", errs[0], errorMsg)
+		}
+	}
+
+	nonZeroMinReplicasCases := prepareMinReplicasCases(t, 1)
+
+	for _, successCase := range nonZeroMinReplicasCases {
+		successCase.Spec.MinReplicas = utilpointer.Int32Ptr(1)
+		if errs := ValidateHorizontalPodAutoscaler(&successCase); len(errs) != 0 {
+			t.Errorf("expected success: %v", errs)
+		}
+	}
+}
+
+func TestValidateHorizontalPodAutoscalerUpdateScaleToZeroEnabled(t *testing.T) {
+	// Enable HPAScaleToZero feature gate.
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, true)()
+
+	zeroMinReplicasCases := prepareMinReplicasCases(t, 0)
+	nonZeroMinReplicasCases := prepareMinReplicasCases(t, 1)
+
+	for i, zeroCase := range zeroMinReplicasCases {
+		nonZeroCase := nonZeroMinReplicasCases[i]
+
+		if errs := ValidateHorizontalPodAutoscalerUpdate(&nonZeroCase, &zeroCase); len(errs) != 0 {
+			t.Errorf("expected success: %v", errs)
+		}
+
+		if errs := ValidateHorizontalPodAutoscalerUpdate(&zeroCase, &nonZeroCase); len(errs) != 0 {
+			t.Errorf("expected success: %v", errs)
+		}
+	}
+}
+
+func TestValidateHorizontalPodAutoscalerScaleToZeroUpdateDisabled(t *testing.T) {
+	// Disable HPAScaleToZero feature gate.
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, false)()
+
+	zeroMinReplicasCases := prepareMinReplicasCases(t, 0)
+	nonZeroMinReplicasCases := prepareMinReplicasCases(t, 1)
+	errorMsg := "must be greater than or equal to 1"
+
+	for i, zeroCase := range zeroMinReplicasCases {
+		nonZeroCase := nonZeroMinReplicasCases[i]
+		errs := ValidateHorizontalPodAutoscalerUpdate(&zeroCase, &nonZeroCase)
+
+		if len(errs) == 0 {
+			t.Errorf("expected failure for %q", errorMsg)
+		} else if !strings.Contains(errs[0].Error(), errorMsg) {
+			t.Errorf("unexpected error: %q, expected: %q", errs[0], errorMsg)
+		}
+
+		if errs := ValidateHorizontalPodAutoscalerUpdate(&zeroCase, &zeroCase); len(errs) != 0 {
+			t.Errorf("expected success: %v", errs)
+		}
+
+		if errs := ValidateHorizontalPodAutoscalerUpdate(&nonZeroCase, &zeroCase); len(errs) != 0 {
+			t.Errorf("expected success: %v", errs)
+		}
+	}
+}