github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-27 05:27:21 +00:00
Code Issues Projects Releases Wiki Activity

make hpa upscale and downscale delay window configurable

Browse Source
This commit is contained in:
Dmitry1987 2017-02-18 10:32:38 +00:00
parent 92f8d9be38
commit 965dab366b
7 changed files with 70 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kube-controller-manager/app/autoscaling.go
View File

@ -76,6 +76,8 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me
replicaCalc, replicaCalc,
ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(), ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
ctx.Options.HorizontalPodAutoscalerSyncPeriod.Duration, ctx.Options.HorizontalPodAutoscalerSyncPeriod.Duration,
ctx.Options.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration,
ctx.Options.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration,
).Run(ctx.Stop) ).Run(ctx.Stop)
return true, nil return true, nil
} }

70
cmd/kube-controller-manager/app/options/options.go
View File

@ -49,39 +49,41 @@ type CMServer struct {
func NewCMServer() *CMServer { func NewCMServer() *CMServer {
s := CMServer{ s := CMServer{
KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{ KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
Controllers: []string{"*"}, Controllers: []string{"*"},
Port: ports.ControllerManagerPort, Port: ports.ControllerManagerPort,
Address: "0.0.0.0", Address: "0.0.0.0",
ConcurrentEndpointSyncs: 5, ConcurrentEndpointSyncs: 5,
ConcurrentServiceSyncs: 1, ConcurrentServiceSyncs: 1,
ConcurrentRCSyncs: 5, ConcurrentRCSyncs: 5,
ConcurrentRSSyncs: 5, ConcurrentRSSyncs: 5,
ConcurrentDaemonSetSyncs: 2, ConcurrentDaemonSetSyncs: 2,
ConcurrentJobSyncs: 5, ConcurrentJobSyncs: 5,
ConcurrentResourceQuotaSyncs: 5, ConcurrentResourceQuotaSyncs: 5,
ConcurrentDeploymentSyncs: 5, ConcurrentDeploymentSyncs: 5,
ConcurrentNamespaceSyncs: 2, ConcurrentNamespaceSyncs: 2,
ConcurrentSATokenSyncs: 5, ConcurrentSATokenSyncs: 5,
LookupCacheSizeForRC: 4096, LookupCacheSizeForRC: 4096,
LookupCacheSizeForRS: 4096, LookupCacheSizeForRS: 4096,
LookupCacheSizeForDaemonSet: 1024, LookupCacheSizeForDaemonSet: 1024,
ServiceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, ServiceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second}, RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second},
ResourceQuotaSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, ResourceQuotaSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
NamespaceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, NamespaceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
PVClaimBinderSyncPeriod: metav1.Duration{Duration: 15 * time.Second}, PVClaimBinderSyncPeriod: metav1.Duration{Duration: 15 * time.Second},
HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 30 * time.Second},
DeploymentControllerSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 3 * time.Minute},
MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour}, HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute},
RegisterRetryCount: 10, DeploymentControllerSyncPeriod: metav1.Duration{Duration: 30 * time.Second},
PodEvictionTimeout: metav1.Duration{Duration: 5 * time.Minute}, MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour},
NodeMonitorGracePeriod: metav1.Duration{Duration: 40 * time.Second}, RegisterRetryCount: 10,
NodeStartupGracePeriod: metav1.Duration{Duration: 60 * time.Second}, PodEvictionTimeout: metav1.Duration{Duration: 5 * time.Minute},
NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second}, NodeMonitorGracePeriod: metav1.Duration{Duration: 40 * time.Second},
ClusterName: "kubernetes", NodeStartupGracePeriod: metav1.Duration{Duration: 60 * time.Second},
NodeCIDRMaskSize: 24, NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second},
ConfigureCloudRoutes: true, ClusterName: "kubernetes",
TerminatedPodGCThreshold: 12500, NodeCIDRMaskSize: 24,
ConfigureCloudRoutes: true,
TerminatedPodGCThreshold: 12500,
VolumeConfiguration: componentconfig.VolumeConfiguration{ VolumeConfiguration: componentconfig.VolumeConfiguration{
EnableHostPathProvisioning: false, EnableHostPathProvisioning: false,
EnableDynamicProvisioning: true, EnableDynamicProvisioning: true,
@ -161,6 +163,8 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabled
fs.StringVar(&s.VolumeConfiguration.FlexVolumePluginDir, "flex-volume-plugin-dir", s.VolumeConfiguration.FlexVolumePluginDir, "Full path of the directory in which the flex volume plugin should search for additional third party volume plugins.") fs.StringVar(&s.VolumeConfiguration.FlexVolumePluginDir, "flex-volume-plugin-dir", s.VolumeConfiguration.FlexVolumePluginDir, "Full path of the directory in which the flex volume plugin should search for additional third party volume plugins.")
fs.Int32Var(&s.TerminatedPodGCThreshold, "terminated-pod-gc-threshold", s.TerminatedPodGCThreshold, "Number of terminated pods that can exist before the terminated pod garbage collector starts deleting terminated pods. If <= 0, the terminated pod garbage collector is disabled.") fs.Int32Var(&s.TerminatedPodGCThreshold, "terminated-pod-gc-threshold", s.TerminatedPodGCThreshold, "Number of terminated pods that can exist before the terminated pod garbage collector starts deleting terminated pods. If <= 0, the terminated pod garbage collector is disabled.")
fs.DurationVar(&s.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", s.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.") fs.DurationVar(&s.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", s.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.")
fs.DurationVar(&s.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-upscale-delay", s.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "The period since last upscale, before another upscale can be performed in horizontal pod autoscaler.")
fs.DurationVar(&s.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-downscale-delay", s.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "The period since last downscale, before another downscale can be performed in horizontal pod autoscaler.")
fs.DurationVar(&s.DeploymentControllerSyncPeriod.Duration, "deployment-controller-sync-period", s.DeploymentControllerSyncPeriod.Duration, "Period for syncing the deployments.") fs.DurationVar(&s.DeploymentControllerSyncPeriod.Duration, "deployment-controller-sync-period", s.DeploymentControllerSyncPeriod.Duration, "Period for syncing the deployments.")
fs.DurationVar(&s.PodEvictionTimeout.Duration, "pod-eviction-timeout", s.PodEvictionTimeout.Duration, "The grace period for deleting pods on failed nodes.") fs.DurationVar(&s.PodEvictionTimeout.Duration, "pod-eviction-timeout", s.PodEvictionTimeout.Duration, "The grace period for deleting pods on failed nodes.")
fs.Float32Var(&s.DeletingPodsQps, "deleting-pods-qps", 0.1, "Number of nodes per second on which pods are deleted in case of node failure.") fs.Float32Var(&s.DeletingPodsQps, "deleting-pods-qps", 0.1, "Number of nodes per second on which pods are deleted in case of node failure.")

2
hack/verify-flags/known-flags.txt
View File

@ -300,6 +300,8 @@ heapster-port
heapster-scheme heapster-scheme
heapster-service heapster-service
horizontal-pod-autoscaler-sync-period horizontal-pod-autoscaler-sync-period
horizontal-pod-autoscaler-upscale-delay
horizontal-pod-autoscaler-downscale-delay
host-cluster-context host-cluster-context
host-ipc-sources host-ipc-sources
hostname-override hostname-override

4
pkg/apis/componentconfig/types.go
View File

@ -725,6 +725,10 @@ type KubeControllerManagerConfiguration struct {
// horizontalPodAutoscalerSyncPeriod is the period for syncing the number of // horizontalPodAutoscalerSyncPeriod is the period for syncing the number of
// pods in horizontal pod autoscaler. // pods in horizontal pod autoscaler.
HorizontalPodAutoscalerSyncPeriod metav1.Duration HorizontalPodAutoscalerSyncPeriod metav1.Duration
// horizontalPodAutoscalerUpscaleForbiddenWindow is a period after which next upscale allowed.
HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
// horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed.
HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
// deploymentControllerSyncPeriod is the period for syncing the deployments. // deploymentControllerSyncPeriod is the period for syncing the deployments.
DeploymentControllerSyncPeriod metav1.Duration DeploymentControllerSyncPeriod metav1.Duration
// podEvictionTimeout is the grace period for deleting pods on failed nodes. // podEvictionTimeout is the grace period for deleting pods on failed nodes.

29
pkg/controller/podautoscaler/horizontal.go
View File

@ -85,6 +85,9 @@ type HorizontalController struct {
replicaCalc *ReplicaCalculator replicaCalc *ReplicaCalculator
eventRecorder record.EventRecorder eventRecorder record.EventRecorder
upscaleForbiddenWindow time.Duration
downscaleForbiddenWindow time.Duration
// hpaLister is able to list/get HPAs from the shared cache from the informer passed in to // hpaLister is able to list/get HPAs from the shared cache from the informer passed in to
// NewHorizontalController. // NewHorizontalController.
hpaLister autoscalinglisters.HorizontalPodAutoscalerLister hpaLister autoscalinglisters.HorizontalPodAutoscalerLister
@ -94,9 +97,6 @@ type HorizontalController struct {
queue workqueue.RateLimitingInterface queue workqueue.RateLimitingInterface
} }
var downscaleForbiddenWindow = 5 * time.Minute
var upscaleForbiddenWindow = 3 * time.Minute
func NewHorizontalController( func NewHorizontalController(
evtNamespacer v1core.EventsGetter, evtNamespacer v1core.EventsGetter,
scaleNamespacer extensionsclient.ScalesGetter, scaleNamespacer extensionsclient.ScalesGetter,
@ -104,6 +104,9 @@ func NewHorizontalController(
replicaCalc *ReplicaCalculator, replicaCalc *ReplicaCalculator,
hpaInformer autoscalinginformers.HorizontalPodAutoscalerInformer, hpaInformer autoscalinginformers.HorizontalPodAutoscalerInformer,
resyncPeriod time.Duration, resyncPeriod time.Duration,
upscaleForbiddenWindow time.Duration,
downscaleForbiddenWindow time.Duration,
) *HorizontalController { ) *HorizontalController {
broadcaster := record.NewBroadcaster() broadcaster := record.NewBroadcaster()
// TODO: remove the wrapper when every clients have moved to use the clientset. // TODO: remove the wrapper when every clients have moved to use the clientset.
@ -111,11 +114,13 @@ func NewHorizontalController(
recorder := broadcaster.NewRecorder(api.Scheme, clientv1.EventSource{Component: "horizontal-pod-autoscaler"}) recorder := broadcaster.NewRecorder(api.Scheme, clientv1.EventSource{Component: "horizontal-pod-autoscaler"})
hpaController := &HorizontalController{ hpaController := &HorizontalController{
replicaCalc: replicaCalc, replicaCalc: replicaCalc,
eventRecorder: recorder, eventRecorder: recorder,
scaleNamespacer: scaleNamespacer, scaleNamespacer: scaleNamespacer,
hpaNamespacer: hpaNamespacer, hpaNamespacer: hpaNamespacer,
queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"), upscaleForbiddenWindow: upscaleForbiddenWindow,
downscaleForbiddenWindow: downscaleForbiddenWindow,
queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"),
} }
hpaInformer.Informer().AddEventHandlerWithResyncPeriod( hpaInformer.Informer().AddEventHandlerWithResyncPeriod(
@ -434,7 +439,7 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
desiredReplicas = calculateScaleUpLimit(currentReplicas) desiredReplicas = calculateScaleUpLimit(currentReplicas)
} }
rescale = shouldScale(hpa, currentReplicas, desiredReplicas, timestamp) rescale = a.shouldScale(hpa, currentReplicas, desiredReplicas, timestamp)
} }
if rescale { if rescale {
@ -455,7 +460,7 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
return a.updateStatus(hpa, currentReplicas, desiredReplicas, metricStatuses, rescale) return a.updateStatus(hpa, currentReplicas, desiredReplicas, metricStatuses, rescale)
} }
func shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, timestamp time.Time) bool { func (a *HorizontalController) shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, timestamp time.Time) bool {
if desiredReplicas == currentReplicas { if desiredReplicas == currentReplicas {
return false return false
} }
@ -466,13 +471,13 @@ func shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, de
// Going down only if the usageRatio dropped significantly below the target // Going down only if the usageRatio dropped significantly below the target
// and there was no rescaling in the last downscaleForbiddenWindow. // and there was no rescaling in the last downscaleForbiddenWindow.
if desiredReplicas < currentReplicas && hpa.Status.LastScaleTime.Add(downscaleForbiddenWindow).Before(timestamp) { if desiredReplicas < currentReplicas && hpa.Status.LastScaleTime.Add(a.downscaleForbiddenWindow).Before(timestamp) {
return true return true
} }
// Going up only if the usage ratio increased significantly above the target // Going up only if the usage ratio increased significantly above the target
// and there was no rescaling in the last upscaleForbiddenWindow. // and there was no rescaling in the last upscaleForbiddenWindow.
if desiredReplicas > currentReplicas && hpa.Status.LastScaleTime.Add(upscaleForbiddenWindow).Before(timestamp) { if desiredReplicas > currentReplicas && hpa.Status.LastScaleTime.Add(a.upscaleForbiddenWindow).Before(timestamp) {
return true return true
} }

4
pkg/controller/podautoscaler/horizontal_test.go
View File

@ -518,6 +518,8 @@ func (tc *testCase) runTest(t *testing.T) {
} }
informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc()) informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
defaultUpscaleForbiddenWindow := 3 * time.Minute
defaultDownscaleForbiddenWindow := 5 * time.Minute
hpaController := NewHorizontalController( hpaController := NewHorizontalController(
eventClient.Core(), eventClient.Core(),
@ -526,6 +528,8 @@ func (tc *testCase) runTest(t *testing.T) {
replicaCalc, replicaCalc,
informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(), informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
controller.NoResyncPeriodFunc(), controller.NoResyncPeriodFunc(),
defaultUpscaleForbiddenWindow,
defaultDownscaleForbiddenWindow,
) )
hpaController.hpaListerSynced = alwaysReady hpaController.hpaListerSynced = alwaysReady

4
pkg/controller/podautoscaler/legacy_horizontal_test.go
View File

@ -489,6 +489,8 @@ func (tc *legacyTestCase) runTest(t *testing.T) {
} }
informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc()) informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
defaultUpscaleForbiddenWindow := 3 * time.Minute
defaultDownscaleForbiddenWindow := 5 * time.Minute
hpaController := NewHorizontalController( hpaController := NewHorizontalController(
eventClient.Core(), eventClient.Core(),
@ -497,6 +499,8 @@ func (tc *legacyTestCase) runTest(t *testing.T) {
replicaCalc, replicaCalc,
informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(), informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
controller.NoResyncPeriodFunc(), controller.NoResyncPeriodFunc(),
defaultUpscaleForbiddenWindow,
defaultDownscaleForbiddenWindow,
) )
hpaController.hpaListerSynced = alwaysReady hpaController.hpaListerSynced = alwaysReady