From 965dab366b5c03fd2e9d5855355fd1235434b076 Mon Sep 17 00:00:00 2001 From: Dmitry1987 Date: Sat, 18 Feb 2017 10:32:38 +0000 Subject: [PATCH] make hpa upscale and downscale delay window configurable --- .../app/autoscaling.go | 2 + .../app/options/options.go | 70 ++++++++++--------- hack/verify-flags/known-flags.txt | 2 + pkg/apis/componentconfig/types.go | 4 ++ pkg/controller/podautoscaler/horizontal.go | 29 ++++---- .../podautoscaler/horizontal_test.go | 4 ++ .../podautoscaler/legacy_horizontal_test.go | 4 ++ 7 files changed, 70 insertions(+), 45 deletions(-) diff --git a/cmd/kube-controller-manager/app/autoscaling.go b/cmd/kube-controller-manager/app/autoscaling.go index 7a09659ad25..6a0328a507d 100644 --- a/cmd/kube-controller-manager/app/autoscaling.go +++ b/cmd/kube-controller-manager/app/autoscaling.go @@ -76,6 +76,8 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me replicaCalc, ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(), ctx.Options.HorizontalPodAutoscalerSyncPeriod.Duration, + ctx.Options.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, + ctx.Options.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, ).Run(ctx.Stop) return true, nil } diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index fe8affcbda0..917ea6a41f0 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -49,39 +49,41 @@ type CMServer struct { func NewCMServer() *CMServer { s := CMServer{ KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{ - Controllers: []string{"*"}, - Port: ports.ControllerManagerPort, - Address: "0.0.0.0", - ConcurrentEndpointSyncs: 5, - ConcurrentServiceSyncs: 1, - ConcurrentRCSyncs: 5, - ConcurrentRSSyncs: 5, - ConcurrentDaemonSetSyncs: 2, - ConcurrentJobSyncs: 5, - ConcurrentResourceQuotaSyncs: 5, - ConcurrentDeploymentSyncs: 5, - ConcurrentNamespaceSyncs: 2, - ConcurrentSATokenSyncs: 5, - LookupCacheSizeForRC: 4096, - LookupCacheSizeForRS: 4096, - LookupCacheSizeForDaemonSet: 1024, - ServiceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, - RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second}, - ResourceQuotaSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, - NamespaceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, - PVClaimBinderSyncPeriod: metav1.Duration{Duration: 15 * time.Second}, - HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, - DeploymentControllerSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, - MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour}, - RegisterRetryCount: 10, - PodEvictionTimeout: metav1.Duration{Duration: 5 * time.Minute}, - NodeMonitorGracePeriod: metav1.Duration{Duration: 40 * time.Second}, - NodeStartupGracePeriod: metav1.Duration{Duration: 60 * time.Second}, - NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second}, - ClusterName: "kubernetes", - NodeCIDRMaskSize: 24, - ConfigureCloudRoutes: true, - TerminatedPodGCThreshold: 12500, + Controllers: []string{"*"}, + Port: ports.ControllerManagerPort, + Address: "0.0.0.0", + ConcurrentEndpointSyncs: 5, + ConcurrentServiceSyncs: 1, + ConcurrentRCSyncs: 5, + ConcurrentRSSyncs: 5, + ConcurrentDaemonSetSyncs: 2, + ConcurrentJobSyncs: 5, + ConcurrentResourceQuotaSyncs: 5, + ConcurrentDeploymentSyncs: 5, + ConcurrentNamespaceSyncs: 2, + ConcurrentSATokenSyncs: 5, + LookupCacheSizeForRC: 4096, + LookupCacheSizeForRS: 4096, + LookupCacheSizeForDaemonSet: 1024, + ServiceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, + RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second}, + ResourceQuotaSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, + NamespaceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, + PVClaimBinderSyncPeriod: metav1.Duration{Duration: 15 * time.Second}, + HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, + HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 3 * time.Minute}, + HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute}, + DeploymentControllerSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, + MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour}, + RegisterRetryCount: 10, + PodEvictionTimeout: metav1.Duration{Duration: 5 * time.Minute}, + NodeMonitorGracePeriod: metav1.Duration{Duration: 40 * time.Second}, + NodeStartupGracePeriod: metav1.Duration{Duration: 60 * time.Second}, + NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second}, + ClusterName: "kubernetes", + NodeCIDRMaskSize: 24, + ConfigureCloudRoutes: true, + TerminatedPodGCThreshold: 12500, VolumeConfiguration: componentconfig.VolumeConfiguration{ EnableHostPathProvisioning: false, EnableDynamicProvisioning: true, @@ -161,6 +163,8 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabled fs.StringVar(&s.VolumeConfiguration.FlexVolumePluginDir, "flex-volume-plugin-dir", s.VolumeConfiguration.FlexVolumePluginDir, "Full path of the directory in which the flex volume plugin should search for additional third party volume plugins.") fs.Int32Var(&s.TerminatedPodGCThreshold, "terminated-pod-gc-threshold", s.TerminatedPodGCThreshold, "Number of terminated pods that can exist before the terminated pod garbage collector starts deleting terminated pods. If <= 0, the terminated pod garbage collector is disabled.") fs.DurationVar(&s.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", s.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.") + fs.DurationVar(&s.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-upscale-delay", s.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "The period since last upscale, before another upscale can be performed in horizontal pod autoscaler.") + fs.DurationVar(&s.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-downscale-delay", s.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "The period since last downscale, before another downscale can be performed in horizontal pod autoscaler.") fs.DurationVar(&s.DeploymentControllerSyncPeriod.Duration, "deployment-controller-sync-period", s.DeploymentControllerSyncPeriod.Duration, "Period for syncing the deployments.") fs.DurationVar(&s.PodEvictionTimeout.Duration, "pod-eviction-timeout", s.PodEvictionTimeout.Duration, "The grace period for deleting pods on failed nodes.") fs.Float32Var(&s.DeletingPodsQps, "deleting-pods-qps", 0.1, "Number of nodes per second on which pods are deleted in case of node failure.") diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 8ea118c0f6e..582dcef1c03 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -300,6 +300,8 @@ heapster-port heapster-scheme heapster-service horizontal-pod-autoscaler-sync-period +horizontal-pod-autoscaler-upscale-delay +horizontal-pod-autoscaler-downscale-delay host-cluster-context host-ipc-sources hostname-override diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index 919088b01a0..b166164c137 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -725,6 +725,10 @@ type KubeControllerManagerConfiguration struct { // horizontalPodAutoscalerSyncPeriod is the period for syncing the number of // pods in horizontal pod autoscaler. HorizontalPodAutoscalerSyncPeriod metav1.Duration + // horizontalPodAutoscalerUpscaleForbiddenWindow is a period after which next upscale allowed. + HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration + // horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed. + HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration // deploymentControllerSyncPeriod is the period for syncing the deployments. DeploymentControllerSyncPeriod metav1.Duration // podEvictionTimeout is the grace period for deleting pods on failed nodes. diff --git a/pkg/controller/podautoscaler/horizontal.go b/pkg/controller/podautoscaler/horizontal.go index 7cb334762ca..d42857fa383 100644 --- a/pkg/controller/podautoscaler/horizontal.go +++ b/pkg/controller/podautoscaler/horizontal.go @@ -85,6 +85,9 @@ type HorizontalController struct { replicaCalc *ReplicaCalculator eventRecorder record.EventRecorder + upscaleForbiddenWindow time.Duration + downscaleForbiddenWindow time.Duration + // hpaLister is able to list/get HPAs from the shared cache from the informer passed in to // NewHorizontalController. hpaLister autoscalinglisters.HorizontalPodAutoscalerLister @@ -94,9 +97,6 @@ type HorizontalController struct { queue workqueue.RateLimitingInterface } -var downscaleForbiddenWindow = 5 * time.Minute -var upscaleForbiddenWindow = 3 * time.Minute - func NewHorizontalController( evtNamespacer v1core.EventsGetter, scaleNamespacer extensionsclient.ScalesGetter, @@ -104,6 +104,9 @@ func NewHorizontalController( replicaCalc *ReplicaCalculator, hpaInformer autoscalinginformers.HorizontalPodAutoscalerInformer, resyncPeriod time.Duration, + upscaleForbiddenWindow time.Duration, + downscaleForbiddenWindow time.Duration, + ) *HorizontalController { broadcaster := record.NewBroadcaster() // TODO: remove the wrapper when every clients have moved to use the clientset. @@ -111,11 +114,13 @@ func NewHorizontalController( recorder := broadcaster.NewRecorder(api.Scheme, clientv1.EventSource{Component: "horizontal-pod-autoscaler"}) hpaController := &HorizontalController{ - replicaCalc: replicaCalc, - eventRecorder: recorder, - scaleNamespacer: scaleNamespacer, - hpaNamespacer: hpaNamespacer, - queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"), + replicaCalc: replicaCalc, + eventRecorder: recorder, + scaleNamespacer: scaleNamespacer, + hpaNamespacer: hpaNamespacer, + upscaleForbiddenWindow: upscaleForbiddenWindow, + downscaleForbiddenWindow: downscaleForbiddenWindow, + queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"), } hpaInformer.Informer().AddEventHandlerWithResyncPeriod( @@ -434,7 +439,7 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho desiredReplicas = calculateScaleUpLimit(currentReplicas) } - rescale = shouldScale(hpa, currentReplicas, desiredReplicas, timestamp) + rescale = a.shouldScale(hpa, currentReplicas, desiredReplicas, timestamp) } if rescale { @@ -455,7 +460,7 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho return a.updateStatus(hpa, currentReplicas, desiredReplicas, metricStatuses, rescale) } -func shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, timestamp time.Time) bool { +func (a *HorizontalController) shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, timestamp time.Time) bool { if desiredReplicas == currentReplicas { return false } @@ -466,13 +471,13 @@ func shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, de // Going down only if the usageRatio dropped significantly below the target // and there was no rescaling in the last downscaleForbiddenWindow. - if desiredReplicas < currentReplicas && hpa.Status.LastScaleTime.Add(downscaleForbiddenWindow).Before(timestamp) { + if desiredReplicas < currentReplicas && hpa.Status.LastScaleTime.Add(a.downscaleForbiddenWindow).Before(timestamp) { return true } // Going up only if the usage ratio increased significantly above the target // and there was no rescaling in the last upscaleForbiddenWindow. - if desiredReplicas > currentReplicas && hpa.Status.LastScaleTime.Add(upscaleForbiddenWindow).Before(timestamp) { + if desiredReplicas > currentReplicas && hpa.Status.LastScaleTime.Add(a.upscaleForbiddenWindow).Before(timestamp) { return true } diff --git a/pkg/controller/podautoscaler/horizontal_test.go b/pkg/controller/podautoscaler/horizontal_test.go index 62354f5c2cb..b90d5b7c85b 100644 --- a/pkg/controller/podautoscaler/horizontal_test.go +++ b/pkg/controller/podautoscaler/horizontal_test.go @@ -518,6 +518,8 @@ func (tc *testCase) runTest(t *testing.T) { } informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc()) + defaultUpscaleForbiddenWindow := 3 * time.Minute + defaultDownscaleForbiddenWindow := 5 * time.Minute hpaController := NewHorizontalController( eventClient.Core(), @@ -526,6 +528,8 @@ func (tc *testCase) runTest(t *testing.T) { replicaCalc, informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(), controller.NoResyncPeriodFunc(), + defaultUpscaleForbiddenWindow, + defaultDownscaleForbiddenWindow, ) hpaController.hpaListerSynced = alwaysReady diff --git a/pkg/controller/podautoscaler/legacy_horizontal_test.go b/pkg/controller/podautoscaler/legacy_horizontal_test.go index 81e5c5fad07..fe809a0bf56 100644 --- a/pkg/controller/podautoscaler/legacy_horizontal_test.go +++ b/pkg/controller/podautoscaler/legacy_horizontal_test.go @@ -489,6 +489,8 @@ func (tc *legacyTestCase) runTest(t *testing.T) { } informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc()) + defaultUpscaleForbiddenWindow := 3 * time.Minute + defaultDownscaleForbiddenWindow := 5 * time.Minute hpaController := NewHorizontalController( eventClient.Core(), @@ -497,6 +499,8 @@ func (tc *legacyTestCase) runTest(t *testing.T) { replicaCalc, informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(), controller.NoResyncPeriodFunc(), + defaultUpscaleForbiddenWindow, + defaultDownscaleForbiddenWindow, ) hpaController.hpaListerSynced = alwaysReady