github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 11:50:44 +00:00
Code Issues Projects Releases Wiki Activity

hpa: Don't scale down if at least one metric was invalid

Browse Source 
Signed-off-by: Mikkel Oscar Lyderik Larsen <mikkel.larsen@zalando.de>
This commit is contained in:
Mikkel Oscar Lyderik Larsen 2021-02-27 00:29:19 +01:00
parent 5b0d0451ff
commit fef092b417
No known key found for this signature in database
GPG Key ID: 50AD98B2A0D8D4EF
2 changed files with 10 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/controller/podautoscaler/horizontal.go
View File

@ -288,8 +288,10 @@ func (a *HorizontalController) computeReplicasForMetrics(hpa *autoscalingv2.Hori
}
}
// If all metrics are invalid return error and set condition on hpa based on first invalid metric.
if invalidMetricsCount >= len(metricSpecs) {
// If all metrics are invalid or some are invalid and we would scale down,
// return an error and set the condition of the hpa based on the first invalid metric.
// Otherwise set the condition as scaling active as we're going to scale
if invalidMetricsCount >= len(metricSpecs) || (invalidMetricsCount > 0 && replicas < specReplicas) {
setCondition(hpa, invalidMetricCondition.Type, invalidMetricCondition.Status, invalidMetricCondition.Reason, invalidMetricCondition.Message)
return 0, "", statuses, time.Time{}, fmt.Errorf("invalid metrics (%v invalid out of %v), first error is: %v", invalidMetricsCount, len(metricSpecs), invalidMetricError)
}

70
pkg/controller/podautoscaler/horizontal_test.go
View File

@ -1679,64 +1679,6 @@ func TestScaleDownIncludeUnreadyPods(t *testing.T) {
tc.runTest(t)
}
func TestScaleDownOneMetricInvalid(t *testing.T) {
tc := testCase{
minReplicas: 2,
maxReplicas: 6,
specReplicas: 5,
statusReplicas: 5,
expectedDesiredReplicas: 3,
CPUTarget: 50,
metricsTarget: []autoscalingv2.MetricSpec{
{
Type: "CheddarCheese",
},
},
reportedLevels: []uint64{100, 300, 500, 250, 250},
reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
useMetricsAPI: true,
recommendations: []timestampedRecommendation{},
}
tc.runTest(t)
}
func TestScaleDownOneMetricEmpty(t *testing.T) {
tc := testCase{
minReplicas: 2,
maxReplicas: 6,
specReplicas: 5,
statusReplicas: 5,
expectedDesiredReplicas: 3,
CPUTarget: 50,
metricsTarget: []autoscalingv2.MetricSpec{
{
Type: autoscalingv2.ExternalMetricSourceType,
External: &autoscalingv2.ExternalMetricSource{
Metric: autoscalingv2.MetricIdentifier{
Name: "qps",
Selector: &metav1.LabelSelector{},
},
Target: autoscalingv2.MetricTarget{
Type: autoscalingv2.AverageValueMetricType,
AverageValue: resource.NewMilliQuantity(1000, resource.DecimalSI),
},
},
},
},
reportedLevels: []uint64{100, 300, 500, 250, 250},
reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
useMetricsAPI: true,
recommendations: []timestampedRecommendation{},
}
_, _, _, testEMClient, _ := tc.prepareTestClient(t)
testEMClient.PrependReactor("list", "*", func(action core.Action) (handled bool, ret runtime.Object, err error) {
return true, &emapi.ExternalMetricValueList{}, fmt.Errorf("something went wrong")
})
tc.testEMClient = testEMClient
tc.runTest(t)
}
func TestScaleDownIgnoreHotCpuPods(t *testing.T) {
tc := testCase{
minReplicas: 2,
@ -4113,10 +4055,10 @@ func TestNoScaleDownOneMetricInvalid(t *testing.T) {
reportedLevels: []uint64{100, 300, 500, 250, 250},
reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
useMetricsAPI: true,
recommendations: []timestampedRecommendation{},
expectedConditions: []autoscalingv1.HorizontalPodAutoscalerCondition{
{Type: autoscalingv1.AbleToScale, Status: v1.ConditionTrue, Reason: "ScaleDownStabilized"},
{Type: autoscalingv1.ScalingActive, Status: v1.ConditionTrue, Reason: "ValidMetricFound"},
{Type: autoscalingv1.ScalingLimited, Status: v1.ConditionFalse, Reason: "DesiredWithinRange"},
{Type: autoscalingv1.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededGetScale"},
{Type: autoscalingv1.ScalingActive, Status: v1.ConditionFalse, Reason: "InvalidMetricSourceType"},
},
}
@ -4148,10 +4090,10 @@ func TestNoScaleDownOneMetricEmpty(t *testing.T) {
reportedLevels: []uint64{100, 300, 500, 250, 250},
reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
useMetricsAPI: true,
recommendations: []timestampedRecommendation{},
expectedConditions: []autoscalingv1.HorizontalPodAutoscalerCondition{
{Type: autoscalingv1.AbleToScale, Status: v1.ConditionTrue, Reason: "ScaleDownStabilized"},
{Type: autoscalingv1.ScalingActive, Status: v1.ConditionTrue, Reason: "ValidMetricFound"},
{Type: autoscalingv1.ScalingLimited, Status: v1.ConditionFalse, Reason: "DesiredWithinRange"},
{Type: autoscalingv1.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededGetScale"},
{Type: autoscalingv1.ScalingActive, Status: v1.ConditionFalse, Reason: "FailedGetExternalMetric"},
},
}
_, _, _, testEMClient, _ := tc.prepareTestClient(t)