From 7d6676eab1b7d6a1ee241b9482e3f8dfc12a29bc Mon Sep 17 00:00:00 2001 From: Joachim Bartosik Date: Wed, 8 Aug 2018 15:00:17 +0200 Subject: [PATCH] Improve HPA sample sanitization After my previous changes HPA wasn't behaving correctly in the following situation: - Pods use a lot of CPU during initilization, become ready right after they initialize, - Scale up triggers, - When new pods become ready HPA counts their usage (even though it's not related to any work that needs doing), - Another scale up, even though existing pods can handle work, no problem. --- pkg/controller/podautoscaler/BUILD | 1 + .../podautoscaler/horizontal_test.go | 123 ++++++- .../podautoscaler/legacy_horizontal_test.go | 12 +- .../legacy_replica_calculator_test.go | 13 +- .../podautoscaler/replica_calculator.go | 153 ++++----- .../podautoscaler/replica_calculator_test.go | 321 +++++++++++++++--- 6 files changed, 467 insertions(+), 156 deletions(-) diff --git a/pkg/controller/podautoscaler/BUILD b/pkg/controller/podautoscaler/BUILD index 8924b2ddb89..80342236639 100644 --- a/pkg/controller/podautoscaler/BUILD +++ b/pkg/controller/podautoscaler/BUILD @@ -73,6 +73,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", diff --git a/pkg/controller/podautoscaler/horizontal_test.go b/pkg/controller/podautoscaler/horizontal_test.go index dc4015b08b4..5939dfe9eb1 100644 --- a/pkg/controller/podautoscaler/horizontal_test.go +++ b/pkg/controller/podautoscaler/horizontal_test.go @@ -103,6 +103,7 @@ type testCase struct { reportedLevels []uint64 reportedCPURequests []resource.Quantity reportedPodReadiness []v1.ConditionStatus + reportedPodStartTime []metav1.Time reportedPodPhase []v1.PodPhase scaleUpdated bool statusUpdated bool @@ -261,6 +262,10 @@ func (tc *testCase) prepareTestClient(t *testing.T) (*fake.Clientset, *metricsfa if tc.reportedPodReadiness != nil { podReadiness = tc.reportedPodReadiness[i] } + var podStartTime metav1.Time + if tc.reportedPodStartTime != nil { + podStartTime = tc.reportedPodStartTime[i] + } podPhase := v1.PodRunning if tc.reportedPodPhase != nil { @@ -283,6 +288,7 @@ func (tc *testCase) prepareTestClient(t *testing.T) (*fake.Clientset, *metricsfa Status: podReadiness, }, }, + StartTime: &podStartTime, }, ObjectMeta: metav1.ObjectMeta{ Name: podName, @@ -660,6 +666,14 @@ func (tc *testCase) setupController(t *testing.T) (*HorizontalController, inform return hpaController, informerFactory } +func hotCpuCreationTime() metav1.Time { + return metav1.Time{Time: time.Now()} +} + +func coolCpuCreationTime() metav1.Time { + return metav1.Time{Time: time.Now().Add(-3 * time.Minute)} +} + func (tc *testCase) runTestWithController(t *testing.T, hpaController *HorizontalController, informerFactory informers.SharedInformerFactory) { stop := make(chan struct{}) defer close(stop) @@ -716,6 +730,23 @@ func TestScaleUpUnreadyLessScale(t *testing.T) { tc.runTest(t) } +func TestScaleUpHotCpuLessScale(t *testing.T) { + tc := testCase{ + minReplicas: 2, + maxReplicas: 6, + initialReplicas: 3, + expectedDesiredReplicas: 4, + CPUTarget: 30, + CPUCurrent: 60, + verifyCPUCurrent: true, + reportedLevels: []uint64{300, 500, 700}, + reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, + reportedPodStartTime: []metav1.Time{hotCpuCreationTime(), coolCpuCreationTime(), coolCpuCreationTime()}, + useMetricsAPI: true, + } + tc.runTest(t) +} + func TestScaleUpUnreadyNoScale(t *testing.T) { tc := testCase{ minReplicas: 2, @@ -738,6 +769,29 @@ func TestScaleUpUnreadyNoScale(t *testing.T) { tc.runTest(t) } +func TestScaleUpHotCpuNoScale(t *testing.T) { + tc := testCase{ + minReplicas: 2, + maxReplicas: 6, + initialReplicas: 3, + expectedDesiredReplicas: 3, + CPUTarget: 30, + CPUCurrent: 40, + verifyCPUCurrent: true, + reportedLevels: []uint64{400, 500, 700}, + reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, + reportedPodReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse}, + reportedPodStartTime: []metav1.Time{coolCpuCreationTime(), hotCpuCreationTime(), hotCpuCreationTime()}, + useMetricsAPI: true, + expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{ + Type: autoscalingv2.AbleToScale, + Status: v1.ConditionTrue, + Reason: "ReadyForNewScale", + }), + } + tc.runTest(t) +} + func TestScaleUpIgnoresFailedPods(t *testing.T) { tc := testCase{ minReplicas: 2, @@ -818,12 +872,12 @@ func TestScaleUpCM(t *testing.T) { tc.runTest(t) } -func TestScaleUpCMUnreadyLessScale(t *testing.T) { +func TestScaleUpCMUnreadyAndHotCpuNoLessScale(t *testing.T) { tc := testCase{ minReplicas: 2, maxReplicas: 6, initialReplicas: 3, - expectedDesiredReplicas: 4, + expectedDesiredReplicas: 6, CPUTarget: 0, metricsTarget: []autoscalingv2.MetricSpec{ { @@ -836,17 +890,18 @@ func TestScaleUpCMUnreadyLessScale(t *testing.T) { }, reportedLevels: []uint64{50000, 10000, 30000}, reportedPodReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionTrue, v1.ConditionFalse}, + reportedPodStartTime: []metav1.Time{coolCpuCreationTime(), coolCpuCreationTime(), hotCpuCreationTime()}, reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, } tc.runTest(t) } -func TestScaleUpCMUnreadyNoScaleWouldScaleDown(t *testing.T) { +func TestScaleUpCMUnreadyandCpuHot(t *testing.T) { tc := testCase{ minReplicas: 2, maxReplicas: 6, initialReplicas: 3, - expectedDesiredReplicas: 3, + expectedDesiredReplicas: 6, CPUTarget: 0, metricsTarget: []autoscalingv2.MetricSpec{ { @@ -859,11 +914,48 @@ func TestScaleUpCMUnreadyNoScaleWouldScaleDown(t *testing.T) { }, reportedLevels: []uint64{50000, 15000, 30000}, reportedPodReadiness: []v1.ConditionStatus{v1.ConditionFalse, v1.ConditionTrue, v1.ConditionFalse}, + reportedPodStartTime: []metav1.Time{hotCpuCreationTime(), coolCpuCreationTime(), hotCpuCreationTime()}, reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{ Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, - Reason: "ReadyForNewScale", + Reason: "SucceededRescale", + }, autoscalingv2.HorizontalPodAutoscalerCondition{ + Type: autoscalingv2.ScalingLimited, + Status: v1.ConditionTrue, + Reason: "TooManyReplicas", + }), + } + tc.runTest(t) +} + +func TestScaleUpHotCpuNoScaleWouldScaleDown(t *testing.T) { + tc := testCase{ + minReplicas: 2, + maxReplicas: 6, + initialReplicas: 3, + expectedDesiredReplicas: 6, + CPUTarget: 0, + metricsTarget: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.PodsMetricSourceType, + Pods: &autoscalingv2.PodsMetricSource{ + MetricName: "qps", + TargetAverageValue: resource.MustParse("15.0"), + }, + }, + }, + reportedLevels: []uint64{50000, 15000, 30000}, + reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, + reportedPodStartTime: []metav1.Time{hotCpuCreationTime(), coolCpuCreationTime(), hotCpuCreationTime()}, + expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{ + Type: autoscalingv2.AbleToScale, + Status: v1.ConditionTrue, + Reason: "SucceededRescale", + }, autoscalingv2.HorizontalPodAutoscalerCondition{ + Type: autoscalingv2.ScalingLimited, + Status: v1.ConditionTrue, + Reason: "TooManyReplicas", }), } tc.runTest(t) @@ -1043,7 +1135,7 @@ func TestScaleDownPerPodCMExternal(t *testing.T) { tc.runTest(t) } -func TestScaleDownIgnoresUnreadyPods(t *testing.T) { +func TestScaleDownIncludeUnreadyPods(t *testing.T) { tc := testCase{ minReplicas: 2, maxReplicas: 6, @@ -1060,6 +1152,23 @@ func TestScaleDownIgnoresUnreadyPods(t *testing.T) { tc.runTest(t) } +func TestScaleDownIgnoreHotCpuPods(t *testing.T) { + tc := testCase{ + minReplicas: 2, + maxReplicas: 6, + initialReplicas: 5, + expectedDesiredReplicas: 2, + CPUTarget: 50, + CPUCurrent: 30, + verifyCPUCurrent: true, + reportedLevels: []uint64{100, 300, 500, 250, 250}, + reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, + useMetricsAPI: true, + reportedPodStartTime: []metav1.Time{coolCpuCreationTime(), coolCpuCreationTime(), coolCpuCreationTime(), hotCpuCreationTime(), hotCpuCreationTime()}, + } + tc.runTest(t) +} + func TestScaleDownIgnoresFailedPods(t *testing.T) { tc := testCase{ minReplicas: 2, @@ -1975,7 +2084,7 @@ func TestAvoidUncessaryUpdates(t *testing.T) { verifyCPUCurrent: true, reportedLevels: []uint64{400, 500, 700}, reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, - reportedPodReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse}, + reportedPodStartTime: []metav1.Time{coolCpuCreationTime(), hotCpuCreationTime(), hotCpuCreationTime()}, useMetricsAPI: true, } testClient, _, _, _, _ := tc.prepareTestClient(t) diff --git a/pkg/controller/podautoscaler/legacy_horizontal_test.go b/pkg/controller/podautoscaler/legacy_horizontal_test.go index 01fc6986234..3918d6dfbbb 100644 --- a/pkg/controller/podautoscaler/legacy_horizontal_test.go +++ b/pkg/controller/podautoscaler/legacy_horizontal_test.go @@ -222,7 +222,8 @@ func (tc *legacyTestCase) prepareTestClient(t *testing.T) (*fake.Clientset, *sca podName := fmt.Sprintf("%s-%d", podNamePrefix, i) pod := v1.Pod{ Status: v1.PodStatus{ - Phase: v1.PodRunning, + StartTime: &metav1.Time{Time: time.Now().Add(-3 * time.Minute)}, + Phase: v1.PodRunning, Conditions: []v1.PodCondition{ { Type: v1.PodReady, @@ -545,8 +546,7 @@ func TestLegacyScaleUpUnreadyLessScale(t *testing.T) { initialReplicas: 3, desiredReplicas: 4, CPUTarget: 30, - CPUCurrent: 60, - verifyCPUCurrent: true, + verifyCPUCurrent: false, reportedLevels: []uint64{300, 500, 700}, reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, reportedPodReadiness: []v1.ConditionStatus{v1.ConditionFalse, v1.ConditionTrue, v1.ConditionTrue}, @@ -634,12 +634,12 @@ func TestLegacyScaleUpCM(t *testing.T) { tc.runTest(t) } -func TestLegacyScaleUpCMUnreadyLessScale(t *testing.T) { +func TestLegacyScaleUpCMUnreadyNoLessScale(t *testing.T) { tc := legacyTestCase{ minReplicas: 2, maxReplicas: 6, initialReplicas: 3, - desiredReplicas: 4, + desiredReplicas: 6, CPUTarget: 0, metricsTarget: []autoscalingv2.MetricSpec{ { @@ -662,7 +662,7 @@ func TestLegacyScaleUpCMUnreadyNoScaleWouldScaleDown(t *testing.T) { minReplicas: 2, maxReplicas: 6, initialReplicas: 3, - desiredReplicas: 3, + desiredReplicas: 6, CPUTarget: 0, metricsTarget: []autoscalingv2.MetricSpec{ { diff --git a/pkg/controller/podautoscaler/legacy_replica_calculator_test.go b/pkg/controller/podautoscaler/legacy_replica_calculator_test.go index 7f3e872a502..f950ae97e3a 100644 --- a/pkg/controller/podautoscaler/legacy_replica_calculator_test.go +++ b/pkg/controller/podautoscaler/legacy_replica_calculator_test.go @@ -67,7 +67,8 @@ func (tc *legacyReplicaCalcTestCase) prepareTestClient(t *testing.T) *fake.Clien podName := fmt.Sprintf("%s-%d", podNamePrefix, i) pod := v1.Pod{ Status: v1.PodStatus{ - Phase: v1.PodRunning, + Phase: v1.PodRunning, + StartTime: &metav1.Time{Time: time.Now().Add(-3 * time.Minute)}, Conditions: []v1.PodCondition{ { Type: v1.PodReady, @@ -310,10 +311,10 @@ func TestLegacyReplicaCalcScaleUpCM(t *testing.T) { tc.runTest(t) } -func TestLegacyReplicaCalcScaleUpCMUnreadyLessScale(t *testing.T) { +func TestLegacyReplicaCalcScaleUpCMUnreadyNoLessScale(t *testing.T) { tc := legacyReplicaCalcTestCase{ currentReplicas: 3, - expectedReplicas: 4, + expectedReplicas: 6, podReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionTrue, v1.ConditionFalse}, metric: &metricInfo{ name: "qps", @@ -325,16 +326,16 @@ func TestLegacyReplicaCalcScaleUpCMUnreadyLessScale(t *testing.T) { tc.runTest(t) } -func TestLegacyReplicaCalcScaleUpCMUnreadyNoScaleWouldScaleDown(t *testing.T) { +func TestLegacyReplicaCalcScaleUpCMUnreadyScale(t *testing.T) { tc := legacyReplicaCalcTestCase{ currentReplicas: 3, - expectedReplicas: 3, + expectedReplicas: 7, podReadiness: []v1.ConditionStatus{v1.ConditionFalse, v1.ConditionTrue, v1.ConditionFalse}, metric: &metricInfo{ name: "qps", levels: []int64{50000, 15000, 30000}, targetUtilization: 15000, - expectedUtilization: 15000, + expectedUtilization: 31666, }, } tc.runTest(t) diff --git a/pkg/controller/podautoscaler/replica_calculator.go b/pkg/controller/podautoscaler/replica_calculator.go index 304c2afba82..65a469f7b64 100644 --- a/pkg/controller/podautoscaler/replica_calculator.go +++ b/pkg/controller/podautoscaler/replica_calculator.go @@ -73,41 +73,11 @@ func (c *ReplicaCalculator) GetResourceReplicas(currentReplicas int32, targetUti return 0, 0, 0, time.Time{}, fmt.Errorf("no pods returned by selector while calculating replica count") } - requests := make(map[string]int64, itemsLen) - readyPodCount := 0 - unreadyPods := sets.NewString() - missingPods := sets.NewString() - - for _, pod := range podList.Items { - podSum := int64(0) - for _, container := range pod.Spec.Containers { - if containerRequest, ok := container.Resources.Requests[resource]; ok { - podSum += containerRequest.MilliValue() - } else { - return 0, 0, 0, time.Time{}, fmt.Errorf("missing request for %s on container %s in pod %s/%s", resource, container.Name, namespace, pod.Name) - } - } - - requests[pod.Name] = podSum - - if pod.Status.Phase != v1.PodRunning || !podutil.IsPodReady(&pod) { - // save this pod name for later, but pretend it doesn't exist for now - if pod.Status.Phase != v1.PodFailed { - // Failed pods should not be counted as unready pods as they will - // not become running anymore. - unreadyPods.Insert(pod.Name) - } - delete(metrics, pod.Name) - continue - } - - if _, found := metrics[pod.Name]; !found { - // save this pod name for later, but pretend it doesn't exist for now - missingPods.Insert(pod.Name) - continue - } - - readyPodCount++ + readyPodCount, ignoredPods, missingPods := groupPods(podList.Items, metrics, resource) + removeMetricsForPods(metrics, ignoredPods) + requests, err := calculatePodRequests(podList.Items, resource) + if err != nil { + return 0, 0, 0, time.Time{}, err } if len(metrics) == 0 { @@ -119,8 +89,8 @@ func (c *ReplicaCalculator) GetResourceReplicas(currentReplicas int32, targetUti return 0, 0, 0, time.Time{}, err } - rebalanceUnready := len(unreadyPods) > 0 && usageRatio > 1.0 - if !rebalanceUnready && len(missingPods) == 0 { + rebalanceIgnored := len(ignoredPods) > 0 && usageRatio > 1.0 + if !rebalanceIgnored && len(missingPods) == 0 { if math.Abs(1.0-usageRatio) <= c.tolerance { // return the current replicas if the change would be too small return currentReplicas, utilization, rawUtilization, timestamp, nil @@ -144,9 +114,9 @@ func (c *ReplicaCalculator) GetResourceReplicas(currentReplicas int32, targetUti } } - if rebalanceUnready { + if rebalanceIgnored { // on a scale-up, treat unready pods as using 0% of the resource request - for podName := range unreadyPods { + for podName := range ignoredPods { metrics[podName] = 0 } } @@ -176,7 +146,7 @@ func (c *ReplicaCalculator) GetRawResourceReplicas(currentReplicas int32, target return 0, 0, time.Time{}, fmt.Errorf("unable to get metrics for resource %s: %v", resource, err) } - replicaCount, utilization, err = c.calcPlainMetricReplicas(metrics, currentReplicas, targetUtilization, namespace, selector) + replicaCount, utilization, err = c.calcPlainMetricReplicas(metrics, currentReplicas, targetUtilization, namespace, selector, resource) return replicaCount, utilization, timestamp, err } @@ -189,12 +159,12 @@ func (c *ReplicaCalculator) GetMetricReplicas(currentReplicas int32, targetUtili return 0, 0, time.Time{}, fmt.Errorf("unable to get metric %s: %v", metricName, err) } - replicaCount, utilization, err = c.calcPlainMetricReplicas(metrics, currentReplicas, targetUtilization, namespace, selector) + replicaCount, utilization, err = c.calcPlainMetricReplicas(metrics, currentReplicas, targetUtilization, namespace, selector, v1.ResourceName("")) return replicaCount, utilization, timestamp, err } // calcPlainMetricReplicas calculates the desired replicas for plain (i.e. non-utilization percentage) metrics. -func (c *ReplicaCalculator) calcPlainMetricReplicas(metrics metricsclient.PodMetricsInfo, currentReplicas int32, targetUtilization int64, namespace string, selector labels.Selector) (replicaCount int32, utilization int64, err error) { +func (c *ReplicaCalculator) calcPlainMetricReplicas(metrics metricsclient.PodMetricsInfo, currentReplicas int32, targetUtilization int64, namespace string, selector labels.Selector, resource v1.ResourceName) (replicaCount int32, utilization int64, err error) { podList, err := c.podsGetter.Pods(namespace).List(metav1.ListOptions{LabelSelector: selector.String()}) if err != nil { return 0, 0, fmt.Errorf("unable to get pods while calculating replica count: %v", err) @@ -204,26 +174,8 @@ func (c *ReplicaCalculator) calcPlainMetricReplicas(metrics metricsclient.PodMet return 0, 0, fmt.Errorf("no pods returned by selector while calculating replica count") } - readyPodCount := 0 - unreadyPods := sets.NewString() - missingPods := sets.NewString() - - for _, pod := range podList.Items { - if pod.Status.Phase != v1.PodRunning || !hasPodBeenReadyBefore(&pod) { - // save this pod name for later, but pretend it doesn't exist for now - unreadyPods.Insert(pod.Name) - delete(metrics, pod.Name) - continue - } - - if _, found := metrics[pod.Name]; !found { - // save this pod name for later, but pretend it doesn't exist for now - missingPods.Insert(pod.Name) - continue - } - - readyPodCount++ - } + readyPodCount, ignoredPods, missingPods := groupPods(podList.Items, metrics, resource) + removeMetricsForPods(metrics, ignoredPods) if len(metrics) == 0 { return 0, 0, fmt.Errorf("did not receive metrics for any ready pods") @@ -231,9 +183,9 @@ func (c *ReplicaCalculator) calcPlainMetricReplicas(metrics metricsclient.PodMet usageRatio, utilization := metricsclient.GetMetricUtilizationRatio(metrics, targetUtilization) - rebalanceUnready := len(unreadyPods) > 0 && usageRatio > 1.0 + rebalanceIgnored := len(ignoredPods) > 0 && usageRatio > 1.0 - if !rebalanceUnready && len(missingPods) == 0 { + if !rebalanceIgnored && len(missingPods) == 0 { if math.Abs(1.0-usageRatio) <= c.tolerance { // return the current replicas if the change would be too small return currentReplicas, utilization, nil @@ -257,9 +209,9 @@ func (c *ReplicaCalculator) calcPlainMetricReplicas(metrics metricsclient.PodMet } } - if rebalanceUnready { + if rebalanceIgnored { // on a scale-up, treat unready pods as using 0% of the resource request - for podName := range unreadyPods { + for podName := range ignoredPods { metrics[podName] = 0 } } @@ -386,21 +338,58 @@ func (c *ReplicaCalculator) GetExternalPerPodMetricReplicas(currentReplicas int3 return replicaCount, utilization, timestamp, nil } -// hasPodBeenReadyBefore returns true if the pod is ready or if it's not ready -func hasPodBeenReadyBefore(pod *v1.Pod) bool { - _, readyCondition := podutil.GetPodCondition(&pod.Status, v1.PodReady) - if readyCondition == nil { - return false +func groupPods(pods []v1.Pod, metrics metricsclient.PodMetricsInfo, resource v1.ResourceName) (readyPodCount int, ignoredPods sets.String, missingPods sets.String) { + missingPods = sets.NewString() + ignoredPods = sets.NewString() + for _, pod := range pods { + if pod.Status.Phase == v1.PodFailed { + continue + } + if _, found := metrics[pod.Name]; !found { + missingPods.Insert(pod.Name) + continue + } + if resource == v1.ResourceCPU { + var ignorePod bool + _, condition := podutil.GetPodCondition(&pod.Status, v1.PodReady) + if condition == nil || pod.Status.StartTime == nil { + ignorePod = true + } else { + if condition.Status == v1.ConditionTrue { + ignorePod = pod.Status.StartTime.Add(2 * time.Minute).After(time.Now()) + } else { + ignorePod = pod.Status.StartTime.Add(maxDelayOfInitialReadinessStatus).After(condition.LastTransitionTime.Time) + } + } + if ignorePod { + ignoredPods.Insert(pod.Name) + continue + } + } + + readyPodCount++ + } + return +} + +func calculatePodRequests(pods []v1.Pod, resource v1.ResourceName) (map[string]int64, error) { + requests := make(map[string]int64, len(pods)) + for _, pod := range pods { + podSum := int64(0) + for _, container := range pod.Spec.Containers { + if containerRequest, ok := container.Resources.Requests[resource]; ok { + podSum += containerRequest.MilliValue() + } else { + return nil, fmt.Errorf("missing request for %s", resource) + } + } + requests[pod.Name] = podSum + } + return requests, nil +} + +func removeMetricsForPods(metrics metricsclient.PodMetricsInfo, pods sets.String) { + for _, pod := range pods.UnsortedList() { + delete(metrics, pod) } - if readyCondition.Status == v1.ConditionTrue { - return true - } - lastReady := readyCondition.LastTransitionTime.Time - if pod.Status.StartTime == nil { - return false - } - started := pod.Status.StartTime.Time - // If last status change was longer than maxDelayOfInitialReadinessStatus after the pod was - // created assume it was ready in the past. - return lastReady.After(started.Add(maxDelayOfInitialReadinessStatus)) } diff --git a/pkg/controller/podautoscaler/replica_calculator_test.go b/pkg/controller/podautoscaler/replica_calculator_test.go index 0dc43b0696a..60e4d6a2d70 100644 --- a/pkg/controller/podautoscaler/replica_calculator_test.go +++ b/pkg/controller/podautoscaler/replica_calculator_test.go @@ -29,10 +29,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/controller/podautoscaler/metrics" + metricsclient "k8s.io/kubernetes/pkg/controller/podautoscaler/metrics" cmapi "k8s.io/metrics/pkg/apis/custom_metrics/v1beta1" emapi "k8s.io/metrics/pkg/apis/external_metrics/v1beta1" metricsapi "k8s.io/metrics/pkg/apis/metrics/v1beta1" @@ -88,6 +90,7 @@ type replicaCalcTestCase struct { metric *metricInfo podReadiness []v1.ConditionStatus + podStartTime []metav1.Time podPhase []v1.PodPhase } @@ -111,6 +114,10 @@ func (tc *replicaCalcTestCase) prepareTestClientSet() *fake.Clientset { if tc.podReadiness != nil && i < len(tc.podReadiness) { podReadiness = tc.podReadiness[i] } + var podStartTime metav1.Time + if tc.podStartTime != nil { + podStartTime = tc.podStartTime[i] + } podPhase := v1.PodRunning if tc.podPhase != nil { podPhase = tc.podPhase[i] @@ -118,7 +125,8 @@ func (tc *replicaCalcTestCase) prepareTestClientSet() *fake.Clientset { podName := fmt.Sprintf("%s-%d", podNamePrefix, i) pod := v1.Pod{ Status: v1.PodStatus{ - Phase: podPhase, + Phase: podPhase, + StartTime: &podStartTime, Conditions: []v1.PodCondition{ { Type: v1.PodReady, @@ -439,6 +447,24 @@ func TestReplicaCalcScaleUpUnreadyLessScale(t *testing.T) { tc.runTest(t) } +func TestReplicaCalcScaleUpHotCpuLessScale(t *testing.T) { + tc := replicaCalcTestCase{ + currentReplicas: 3, + expectedReplicas: 4, + podStartTime: []metav1.Time{hotCpuCreationTime(), coolCpuCreationTime(), coolCpuCreationTime()}, + resource: &resourceInfo{ + name: v1.ResourceCPU, + requests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, + levels: []int64{300, 500, 700}, + + targetUtilization: 30, + expectedUtilization: 60, + expectedValue: numContainersPerPod * 600, + }, + } + tc.runTest(t) +} + func TestReplicaCalcScaleUpUnreadyNoScale(t *testing.T) { tc := replicaCalcTestCase{ currentReplicas: 3, @@ -457,6 +483,25 @@ func TestReplicaCalcScaleUpUnreadyNoScale(t *testing.T) { tc.runTest(t) } +func TestReplicaCalcScaleHotCpuNoScale(t *testing.T) { + tc := replicaCalcTestCase{ + currentReplicas: 3, + expectedReplicas: 3, + podReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse}, + podStartTime: []metav1.Time{coolCpuCreationTime(), hotCpuCreationTime(), hotCpuCreationTime()}, + resource: &resourceInfo{ + name: v1.ResourceCPU, + requests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, + levels: []int64{400, 500, 700}, + + targetUtilization: 30, + expectedUtilization: 40, + expectedValue: numContainersPerPod * 400, + }, + } + tc.runTest(t) +} + func TestReplicaCalcScaleUpIgnoresFailedPods(t *testing.T) { tc := replicaCalcTestCase{ currentReplicas: 2, @@ -491,11 +536,12 @@ func TestReplicaCalcScaleUpCM(t *testing.T) { tc.runTest(t) } -func TestReplicaCalcScaleUpCMUnreadyLessScale(t *testing.T) { +func TestReplicaCalcScaleUpCMUnreadyHotCpuNoLessScale(t *testing.T) { tc := replicaCalcTestCase{ currentReplicas: 3, - expectedReplicas: 4, + expectedReplicas: 6, podReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionTrue, v1.ConditionFalse}, + podStartTime: []metav1.Time{coolCpuCreationTime(), coolCpuCreationTime(), hotCpuCreationTime()}, metric: &metricInfo{ name: "qps", levels: []int64{50000, 10000, 30000}, @@ -507,16 +553,17 @@ func TestReplicaCalcScaleUpCMUnreadyLessScale(t *testing.T) { tc.runTest(t) } -func TestReplicaCalcScaleUpCMUnreadyNoScaleWouldScaleDown(t *testing.T) { +func TestReplicaCalcScaleUpCMUnreadyHotCpuScaleWouldScaleDown(t *testing.T) { tc := replicaCalcTestCase{ currentReplicas: 3, - expectedReplicas: 3, + expectedReplicas: 7, podReadiness: []v1.ConditionStatus{v1.ConditionFalse, v1.ConditionTrue, v1.ConditionFalse}, + podStartTime: []metav1.Time{hotCpuCreationTime(), coolCpuCreationTime(), hotCpuCreationTime()}, metric: &metricInfo{ name: "qps", levels: []int64{50000, 15000, 30000}, targetUtilization: 15000, - expectedUtilization: 15000, + expectedUtilization: 31666, metricType: podMetric, }, } @@ -709,7 +756,7 @@ func TestReplicaCalcScaleDownPerPodCMExternal(t *testing.T) { tc.runTest(t) } -func TestReplicaCalcScaleDownIgnoresUnreadyPods(t *testing.T) { +func TestReplicaCalcScaleDownIncludeUnreadyPods(t *testing.T) { tc := replicaCalcTestCase{ currentReplicas: 5, expectedReplicas: 2, @@ -727,6 +774,24 @@ func TestReplicaCalcScaleDownIgnoresUnreadyPods(t *testing.T) { tc.runTest(t) } +func TestReplicaCalcScaleDownIgnoreHotCpuPods(t *testing.T) { + tc := replicaCalcTestCase{ + currentReplicas: 5, + expectedReplicas: 2, + podStartTime: []metav1.Time{coolCpuCreationTime(), coolCpuCreationTime(), coolCpuCreationTime(), hotCpuCreationTime(), hotCpuCreationTime()}, + resource: &resourceInfo{ + name: v1.ResourceCPU, + requests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, + levels: []int64{100, 300, 500, 250, 250}, + + targetUtilization: 50, + expectedUtilization: 30, + expectedValue: numContainersPerPod * 300, + }, + } + tc.runTest(t) +} + func TestReplicaCalcScaleDownIgnoresFailedPods(t *testing.T) { tc := replicaCalcTestCase{ currentReplicas: 5, @@ -943,7 +1008,7 @@ func TestReplicaCalcMissingMetricsNoChangeLt(t *testing.T) { tc.runTest(t) } -func TestReplicaCalcMissingMetricsUnreadyNoChange(t *testing.T) { +func TestReplicaCalcMissingMetricsUnreadyChange(t *testing.T) { tc := replicaCalcTestCase{ currentReplicas: 3, expectedReplicas: 3, @@ -961,6 +1026,24 @@ func TestReplicaCalcMissingMetricsUnreadyNoChange(t *testing.T) { tc.runTest(t) } +func TestReplicaCalcMissingMetricsHotCpuNoChange(t *testing.T) { + tc := replicaCalcTestCase{ + currentReplicas: 3, + expectedReplicas: 3, + podStartTime: []metav1.Time{hotCpuCreationTime(), coolCpuCreationTime(), coolCpuCreationTime()}, + resource: &resourceInfo{ + name: v1.ResourceCPU, + requests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, + levels: []int64{100, 450}, + + targetUtilization: 50, + expectedUtilization: 45, + expectedValue: numContainersPerPod * 450, + }, + } + tc.runTest(t) +} + func TestReplicaCalcMissingMetricsUnreadyScaleUp(t *testing.T) { tc := replicaCalcTestCase{ currentReplicas: 3, @@ -979,6 +1062,25 @@ func TestReplicaCalcMissingMetricsUnreadyScaleUp(t *testing.T) { tc.runTest(t) } +func TestReplicaCalcMissingMetricsHotCpuScaleUp(t *testing.T) { + tc := replicaCalcTestCase{ + currentReplicas: 3, + expectedReplicas: 4, + podReadiness: []v1.ConditionStatus{v1.ConditionFalse, v1.ConditionTrue, v1.ConditionTrue}, + podStartTime: []metav1.Time{hotCpuCreationTime(), coolCpuCreationTime(), coolCpuCreationTime()}, + resource: &resourceInfo{ + name: v1.ResourceCPU, + requests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, + levels: []int64{100, 2000}, + + targetUtilization: 50, + expectedUtilization: 200, + expectedValue: numContainersPerPod * 2000, + }, + } + tc.runTest(t) +} + func TestReplicaCalcMissingMetricsUnreadyScaleDown(t *testing.T) { tc := replicaCalcTestCase{ currentReplicas: 4, @@ -1069,74 +1171,183 @@ func TestReplicaCalcComputedToleranceAlgImplementation(t *testing.T) { tc.runTest(t) } -func TestHasPodBeenReadyBefore(t *testing.T) { +func TestGroupPods(t *testing.T) { tests := []struct { - name string - conditions []v1.PodCondition - started time.Time - expected bool + name string + pods []v1.Pod + metrics metricsclient.PodMetricsInfo + resource v1.ResourceName + expectReadyPodCount int + expectUnreadyPods sets.String + expectMissingPods sets.String }{ { - "initially unready", - []v1.PodCondition{ - { - Type: v1.PodReady, - LastTransitionTime: metav1.Time{ - Time: metav1.Date(2018, 7, 25, 17, 10, 0, 0, time.UTC).Time, - }, - Status: v1.ConditionFalse, - }, - }, - metav1.Date(2018, 7, 25, 17, 10, 0, 0, time.UTC).Time, - false, + "void", + []v1.Pod{}, + metricsclient.PodMetricsInfo{}, + v1.ResourceName(""), + 0, + sets.NewString(), + sets.NewString(), }, { - "currently unready", - []v1.PodCondition{ + "a ready pod", + []v1.Pod{ { - Type: v1.PodReady, - LastTransitionTime: metav1.Time{ - Time: metav1.Date(2018, 7, 25, 17, 10, 0, 0, time.UTC).Time, + ObjectMeta: metav1.ObjectMeta{ + Name: "bentham", + }, + Status: v1.PodStatus{ + Phase: v1.PodSucceeded, }, - Status: v1.ConditionFalse, }, }, - metav1.Date(2018, 7, 25, 17, 0, 0, 0, time.UTC).Time, - true, + metricsclient.PodMetricsInfo{ + "bentham": 1, + }, + v1.ResourceName("hedons"), + 1, + sets.NewString(), + sets.NewString(), }, { - "currently ready", - []v1.PodCondition{ + "an unready pod", + []v1.Pod{ { - Type: v1.PodReady, - LastTransitionTime: metav1.Time{ - Time: metav1.Date(2018, 7, 25, 17, 10, 0, 0, time.UTC).Time, + ObjectMeta: metav1.ObjectMeta{ + Name: "lucretius", + }, + Status: v1.PodStatus{ + Phase: v1.PodSucceeded, + StartTime: &metav1.Time{ + Time: time.Now(), + }, }, - Status: v1.ConditionTrue, }, }, - metav1.Date(2018, 7, 25, 17, 10, 0, 0, time.UTC).Time, - true, + metricsclient.PodMetricsInfo{ + "lucretius": 1, + }, + v1.ResourceCPU, + 0, + sets.NewString("lucretius"), + sets.NewString(), }, { - "no ready status", - []v1.PodCondition{}, - metav1.Date(2018, 7, 25, 17, 10, 0, 0, time.UTC).Time, - false, + "a ready cpu pod", + []v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "niccolo", + }, + Status: v1.PodStatus{ + Phase: v1.PodSucceeded, + StartTime: &metav1.Time{ + Time: time.Now().Add(-3 * time.Minute), + }, + Conditions: []v1.PodCondition{ + { + Type: v1.PodReady, + LastTransitionTime: metav1.Time{Time: time.Now().Add(-3 * time.Minute)}, + Status: v1.ConditionTrue, + }, + }, + }, + }, + }, + metricsclient.PodMetricsInfo{ + "niccolo": 1, + }, + v1.ResourceCPU, + 1, + sets.NewString(), + sets.NewString(), + }, + { + "a missing pod", + []v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "epicurus", + }, + Status: v1.PodStatus{ + Phase: v1.PodSucceeded, + StartTime: &metav1.Time{ + Time: time.Now().Add(-3 * time.Minute), + }, + }, + }, + }, + metricsclient.PodMetricsInfo{}, + v1.ResourceCPU, + 0, + sets.NewString(), + sets.NewString("epicurus"), + }, + { + "all together", + []v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "lucretius", + }, + Status: v1.PodStatus{ + Phase: v1.PodSucceeded, + StartTime: &metav1.Time{ + Time: time.Now(), + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "niccolo", + }, + Status: v1.PodStatus{ + Phase: v1.PodSucceeded, + StartTime: &metav1.Time{ + Time: time.Now().Add(-3 * time.Minute), + }, + Conditions: []v1.PodCondition{ + { + Type: v1.PodReady, + LastTransitionTime: metav1.Time{Time: time.Now().Add(-3 * time.Minute)}, + Status: v1.ConditionTrue, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "epicurus", + }, + Status: v1.PodStatus{ + Phase: v1.PodSucceeded, + StartTime: &metav1.Time{ + Time: time.Now().Add(-3 * time.Minute), + }, + }, + }, + }, + metricsclient.PodMetricsInfo{ + "lucretius": 1, + "niccolo": 1, + }, + v1.ResourceCPU, + 1, + sets.NewString("lucretius"), + sets.NewString("epicurus"), }, } for _, tc := range tests { - pod := &v1.Pod{ - Status: v1.PodStatus{ - Conditions: tc.conditions, - StartTime: &metav1.Time{ - Time: tc.started, - }, - }, + readyPodCount, unreadyPods, missingPods := groupPods(tc.pods, tc.metrics, tc.resource) + if readyPodCount != tc.expectReadyPodCount { + t.Errorf("%s got readyPodCount %d, expected %d", tc.name, readyPodCount, tc.expectReadyPodCount) } - got := hasPodBeenReadyBefore(pod) - if got != tc.expected { - t.Errorf("[TestHasPodBeenReadyBefore.%s] got %v, want %v", tc.name, got, tc.expected) + if !unreadyPods.Equal(tc.expectUnreadyPods) { + t.Errorf("%s got unreadyPods %v, expected %v", tc.name, unreadyPods, tc.expectUnreadyPods) + } + if !missingPods.Equal(tc.expectMissingPods) { + t.Errorf("%s got missingPods %v, expected %v", tc.name, missingPods, tc.expectMissingPods) } } }