mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 20:24:09 +00:00
make hpa upscale and downscale delay window configurable
This commit is contained in:
parent
92f8d9be38
commit
965dab366b
@ -76,6 +76,8 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me
|
|||||||
replicaCalc,
|
replicaCalc,
|
||||||
ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
|
ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
|
||||||
ctx.Options.HorizontalPodAutoscalerSyncPeriod.Duration,
|
ctx.Options.HorizontalPodAutoscalerSyncPeriod.Duration,
|
||||||
|
ctx.Options.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration,
|
||||||
|
ctx.Options.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration,
|
||||||
).Run(ctx.Stop)
|
).Run(ctx.Stop)
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
@ -49,39 +49,41 @@ type CMServer struct {
|
|||||||
func NewCMServer() *CMServer {
|
func NewCMServer() *CMServer {
|
||||||
s := CMServer{
|
s := CMServer{
|
||||||
KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
|
KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
|
||||||
Controllers: []string{"*"},
|
Controllers: []string{"*"},
|
||||||
Port: ports.ControllerManagerPort,
|
Port: ports.ControllerManagerPort,
|
||||||
Address: "0.0.0.0",
|
Address: "0.0.0.0",
|
||||||
ConcurrentEndpointSyncs: 5,
|
ConcurrentEndpointSyncs: 5,
|
||||||
ConcurrentServiceSyncs: 1,
|
ConcurrentServiceSyncs: 1,
|
||||||
ConcurrentRCSyncs: 5,
|
ConcurrentRCSyncs: 5,
|
||||||
ConcurrentRSSyncs: 5,
|
ConcurrentRSSyncs: 5,
|
||||||
ConcurrentDaemonSetSyncs: 2,
|
ConcurrentDaemonSetSyncs: 2,
|
||||||
ConcurrentJobSyncs: 5,
|
ConcurrentJobSyncs: 5,
|
||||||
ConcurrentResourceQuotaSyncs: 5,
|
ConcurrentResourceQuotaSyncs: 5,
|
||||||
ConcurrentDeploymentSyncs: 5,
|
ConcurrentDeploymentSyncs: 5,
|
||||||
ConcurrentNamespaceSyncs: 2,
|
ConcurrentNamespaceSyncs: 2,
|
||||||
ConcurrentSATokenSyncs: 5,
|
ConcurrentSATokenSyncs: 5,
|
||||||
LookupCacheSizeForRC: 4096,
|
LookupCacheSizeForRC: 4096,
|
||||||
LookupCacheSizeForRS: 4096,
|
LookupCacheSizeForRS: 4096,
|
||||||
LookupCacheSizeForDaemonSet: 1024,
|
LookupCacheSizeForDaemonSet: 1024,
|
||||||
ServiceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
|
ServiceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
|
||||||
RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second},
|
RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second},
|
||||||
ResourceQuotaSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
|
ResourceQuotaSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
|
||||||
NamespaceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
|
NamespaceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
|
||||||
PVClaimBinderSyncPeriod: metav1.Duration{Duration: 15 * time.Second},
|
PVClaimBinderSyncPeriod: metav1.Duration{Duration: 15 * time.Second},
|
||||||
HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 30 * time.Second},
|
HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 30 * time.Second},
|
||||||
DeploymentControllerSyncPeriod: metav1.Duration{Duration: 30 * time.Second},
|
HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 3 * time.Minute},
|
||||||
MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour},
|
HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute},
|
||||||
RegisterRetryCount: 10,
|
DeploymentControllerSyncPeriod: metav1.Duration{Duration: 30 * time.Second},
|
||||||
PodEvictionTimeout: metav1.Duration{Duration: 5 * time.Minute},
|
MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour},
|
||||||
NodeMonitorGracePeriod: metav1.Duration{Duration: 40 * time.Second},
|
RegisterRetryCount: 10,
|
||||||
NodeStartupGracePeriod: metav1.Duration{Duration: 60 * time.Second},
|
PodEvictionTimeout: metav1.Duration{Duration: 5 * time.Minute},
|
||||||
NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second},
|
NodeMonitorGracePeriod: metav1.Duration{Duration: 40 * time.Second},
|
||||||
ClusterName: "kubernetes",
|
NodeStartupGracePeriod: metav1.Duration{Duration: 60 * time.Second},
|
||||||
NodeCIDRMaskSize: 24,
|
NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second},
|
||||||
ConfigureCloudRoutes: true,
|
ClusterName: "kubernetes",
|
||||||
TerminatedPodGCThreshold: 12500,
|
NodeCIDRMaskSize: 24,
|
||||||
|
ConfigureCloudRoutes: true,
|
||||||
|
TerminatedPodGCThreshold: 12500,
|
||||||
VolumeConfiguration: componentconfig.VolumeConfiguration{
|
VolumeConfiguration: componentconfig.VolumeConfiguration{
|
||||||
EnableHostPathProvisioning: false,
|
EnableHostPathProvisioning: false,
|
||||||
EnableDynamicProvisioning: true,
|
EnableDynamicProvisioning: true,
|
||||||
@ -161,6 +163,8 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabled
|
|||||||
fs.StringVar(&s.VolumeConfiguration.FlexVolumePluginDir, "flex-volume-plugin-dir", s.VolumeConfiguration.FlexVolumePluginDir, "Full path of the directory in which the flex volume plugin should search for additional third party volume plugins.")
|
fs.StringVar(&s.VolumeConfiguration.FlexVolumePluginDir, "flex-volume-plugin-dir", s.VolumeConfiguration.FlexVolumePluginDir, "Full path of the directory in which the flex volume plugin should search for additional third party volume plugins.")
|
||||||
fs.Int32Var(&s.TerminatedPodGCThreshold, "terminated-pod-gc-threshold", s.TerminatedPodGCThreshold, "Number of terminated pods that can exist before the terminated pod garbage collector starts deleting terminated pods. If <= 0, the terminated pod garbage collector is disabled.")
|
fs.Int32Var(&s.TerminatedPodGCThreshold, "terminated-pod-gc-threshold", s.TerminatedPodGCThreshold, "Number of terminated pods that can exist before the terminated pod garbage collector starts deleting terminated pods. If <= 0, the terminated pod garbage collector is disabled.")
|
||||||
fs.DurationVar(&s.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", s.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.")
|
fs.DurationVar(&s.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", s.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.")
|
||||||
|
fs.DurationVar(&s.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-upscale-delay", s.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "The period since last upscale, before another upscale can be performed in horizontal pod autoscaler.")
|
||||||
|
fs.DurationVar(&s.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-downscale-delay", s.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "The period since last downscale, before another downscale can be performed in horizontal pod autoscaler.")
|
||||||
fs.DurationVar(&s.DeploymentControllerSyncPeriod.Duration, "deployment-controller-sync-period", s.DeploymentControllerSyncPeriod.Duration, "Period for syncing the deployments.")
|
fs.DurationVar(&s.DeploymentControllerSyncPeriod.Duration, "deployment-controller-sync-period", s.DeploymentControllerSyncPeriod.Duration, "Period for syncing the deployments.")
|
||||||
fs.DurationVar(&s.PodEvictionTimeout.Duration, "pod-eviction-timeout", s.PodEvictionTimeout.Duration, "The grace period for deleting pods on failed nodes.")
|
fs.DurationVar(&s.PodEvictionTimeout.Duration, "pod-eviction-timeout", s.PodEvictionTimeout.Duration, "The grace period for deleting pods on failed nodes.")
|
||||||
fs.Float32Var(&s.DeletingPodsQps, "deleting-pods-qps", 0.1, "Number of nodes per second on which pods are deleted in case of node failure.")
|
fs.Float32Var(&s.DeletingPodsQps, "deleting-pods-qps", 0.1, "Number of nodes per second on which pods are deleted in case of node failure.")
|
||||||
|
@ -300,6 +300,8 @@ heapster-port
|
|||||||
heapster-scheme
|
heapster-scheme
|
||||||
heapster-service
|
heapster-service
|
||||||
horizontal-pod-autoscaler-sync-period
|
horizontal-pod-autoscaler-sync-period
|
||||||
|
horizontal-pod-autoscaler-upscale-delay
|
||||||
|
horizontal-pod-autoscaler-downscale-delay
|
||||||
host-cluster-context
|
host-cluster-context
|
||||||
host-ipc-sources
|
host-ipc-sources
|
||||||
hostname-override
|
hostname-override
|
||||||
|
@ -725,6 +725,10 @@ type KubeControllerManagerConfiguration struct {
|
|||||||
// horizontalPodAutoscalerSyncPeriod is the period for syncing the number of
|
// horizontalPodAutoscalerSyncPeriod is the period for syncing the number of
|
||||||
// pods in horizontal pod autoscaler.
|
// pods in horizontal pod autoscaler.
|
||||||
HorizontalPodAutoscalerSyncPeriod metav1.Duration
|
HorizontalPodAutoscalerSyncPeriod metav1.Duration
|
||||||
|
// horizontalPodAutoscalerUpscaleForbiddenWindow is a period after which next upscale allowed.
|
||||||
|
HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
|
||||||
|
// horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed.
|
||||||
|
HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
|
||||||
// deploymentControllerSyncPeriod is the period for syncing the deployments.
|
// deploymentControllerSyncPeriod is the period for syncing the deployments.
|
||||||
DeploymentControllerSyncPeriod metav1.Duration
|
DeploymentControllerSyncPeriod metav1.Duration
|
||||||
// podEvictionTimeout is the grace period for deleting pods on failed nodes.
|
// podEvictionTimeout is the grace period for deleting pods on failed nodes.
|
||||||
|
@ -85,6 +85,9 @@ type HorizontalController struct {
|
|||||||
replicaCalc *ReplicaCalculator
|
replicaCalc *ReplicaCalculator
|
||||||
eventRecorder record.EventRecorder
|
eventRecorder record.EventRecorder
|
||||||
|
|
||||||
|
upscaleForbiddenWindow time.Duration
|
||||||
|
downscaleForbiddenWindow time.Duration
|
||||||
|
|
||||||
// hpaLister is able to list/get HPAs from the shared cache from the informer passed in to
|
// hpaLister is able to list/get HPAs from the shared cache from the informer passed in to
|
||||||
// NewHorizontalController.
|
// NewHorizontalController.
|
||||||
hpaLister autoscalinglisters.HorizontalPodAutoscalerLister
|
hpaLister autoscalinglisters.HorizontalPodAutoscalerLister
|
||||||
@ -94,9 +97,6 @@ type HorizontalController struct {
|
|||||||
queue workqueue.RateLimitingInterface
|
queue workqueue.RateLimitingInterface
|
||||||
}
|
}
|
||||||
|
|
||||||
var downscaleForbiddenWindow = 5 * time.Minute
|
|
||||||
var upscaleForbiddenWindow = 3 * time.Minute
|
|
||||||
|
|
||||||
func NewHorizontalController(
|
func NewHorizontalController(
|
||||||
evtNamespacer v1core.EventsGetter,
|
evtNamespacer v1core.EventsGetter,
|
||||||
scaleNamespacer extensionsclient.ScalesGetter,
|
scaleNamespacer extensionsclient.ScalesGetter,
|
||||||
@ -104,6 +104,9 @@ func NewHorizontalController(
|
|||||||
replicaCalc *ReplicaCalculator,
|
replicaCalc *ReplicaCalculator,
|
||||||
hpaInformer autoscalinginformers.HorizontalPodAutoscalerInformer,
|
hpaInformer autoscalinginformers.HorizontalPodAutoscalerInformer,
|
||||||
resyncPeriod time.Duration,
|
resyncPeriod time.Duration,
|
||||||
|
upscaleForbiddenWindow time.Duration,
|
||||||
|
downscaleForbiddenWindow time.Duration,
|
||||||
|
|
||||||
) *HorizontalController {
|
) *HorizontalController {
|
||||||
broadcaster := record.NewBroadcaster()
|
broadcaster := record.NewBroadcaster()
|
||||||
// TODO: remove the wrapper when every clients have moved to use the clientset.
|
// TODO: remove the wrapper when every clients have moved to use the clientset.
|
||||||
@ -111,11 +114,13 @@ func NewHorizontalController(
|
|||||||
recorder := broadcaster.NewRecorder(api.Scheme, clientv1.EventSource{Component: "horizontal-pod-autoscaler"})
|
recorder := broadcaster.NewRecorder(api.Scheme, clientv1.EventSource{Component: "horizontal-pod-autoscaler"})
|
||||||
|
|
||||||
hpaController := &HorizontalController{
|
hpaController := &HorizontalController{
|
||||||
replicaCalc: replicaCalc,
|
replicaCalc: replicaCalc,
|
||||||
eventRecorder: recorder,
|
eventRecorder: recorder,
|
||||||
scaleNamespacer: scaleNamespacer,
|
scaleNamespacer: scaleNamespacer,
|
||||||
hpaNamespacer: hpaNamespacer,
|
hpaNamespacer: hpaNamespacer,
|
||||||
queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"),
|
upscaleForbiddenWindow: upscaleForbiddenWindow,
|
||||||
|
downscaleForbiddenWindow: downscaleForbiddenWindow,
|
||||||
|
queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"),
|
||||||
}
|
}
|
||||||
|
|
||||||
hpaInformer.Informer().AddEventHandlerWithResyncPeriod(
|
hpaInformer.Informer().AddEventHandlerWithResyncPeriod(
|
||||||
@ -434,7 +439,7 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
|
|||||||
desiredReplicas = calculateScaleUpLimit(currentReplicas)
|
desiredReplicas = calculateScaleUpLimit(currentReplicas)
|
||||||
}
|
}
|
||||||
|
|
||||||
rescale = shouldScale(hpa, currentReplicas, desiredReplicas, timestamp)
|
rescale = a.shouldScale(hpa, currentReplicas, desiredReplicas, timestamp)
|
||||||
}
|
}
|
||||||
|
|
||||||
if rescale {
|
if rescale {
|
||||||
@ -455,7 +460,7 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
|
|||||||
return a.updateStatus(hpa, currentReplicas, desiredReplicas, metricStatuses, rescale)
|
return a.updateStatus(hpa, currentReplicas, desiredReplicas, metricStatuses, rescale)
|
||||||
}
|
}
|
||||||
|
|
||||||
func shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, timestamp time.Time) bool {
|
func (a *HorizontalController) shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, timestamp time.Time) bool {
|
||||||
if desiredReplicas == currentReplicas {
|
if desiredReplicas == currentReplicas {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
@ -466,13 +471,13 @@ func shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, de
|
|||||||
|
|
||||||
// Going down only if the usageRatio dropped significantly below the target
|
// Going down only if the usageRatio dropped significantly below the target
|
||||||
// and there was no rescaling in the last downscaleForbiddenWindow.
|
// and there was no rescaling in the last downscaleForbiddenWindow.
|
||||||
if desiredReplicas < currentReplicas && hpa.Status.LastScaleTime.Add(downscaleForbiddenWindow).Before(timestamp) {
|
if desiredReplicas < currentReplicas && hpa.Status.LastScaleTime.Add(a.downscaleForbiddenWindow).Before(timestamp) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// Going up only if the usage ratio increased significantly above the target
|
// Going up only if the usage ratio increased significantly above the target
|
||||||
// and there was no rescaling in the last upscaleForbiddenWindow.
|
// and there was no rescaling in the last upscaleForbiddenWindow.
|
||||||
if desiredReplicas > currentReplicas && hpa.Status.LastScaleTime.Add(upscaleForbiddenWindow).Before(timestamp) {
|
if desiredReplicas > currentReplicas && hpa.Status.LastScaleTime.Add(a.upscaleForbiddenWindow).Before(timestamp) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -518,6 +518,8 @@ func (tc *testCase) runTest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
|
informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
|
||||||
|
defaultUpscaleForbiddenWindow := 3 * time.Minute
|
||||||
|
defaultDownscaleForbiddenWindow := 5 * time.Minute
|
||||||
|
|
||||||
hpaController := NewHorizontalController(
|
hpaController := NewHorizontalController(
|
||||||
eventClient.Core(),
|
eventClient.Core(),
|
||||||
@ -526,6 +528,8 @@ func (tc *testCase) runTest(t *testing.T) {
|
|||||||
replicaCalc,
|
replicaCalc,
|
||||||
informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
|
informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
|
||||||
controller.NoResyncPeriodFunc(),
|
controller.NoResyncPeriodFunc(),
|
||||||
|
defaultUpscaleForbiddenWindow,
|
||||||
|
defaultDownscaleForbiddenWindow,
|
||||||
)
|
)
|
||||||
hpaController.hpaListerSynced = alwaysReady
|
hpaController.hpaListerSynced = alwaysReady
|
||||||
|
|
||||||
|
@ -489,6 +489,8 @@ func (tc *legacyTestCase) runTest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
|
informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
|
||||||
|
defaultUpscaleForbiddenWindow := 3 * time.Minute
|
||||||
|
defaultDownscaleForbiddenWindow := 5 * time.Minute
|
||||||
|
|
||||||
hpaController := NewHorizontalController(
|
hpaController := NewHorizontalController(
|
||||||
eventClient.Core(),
|
eventClient.Core(),
|
||||||
@ -497,6 +499,8 @@ func (tc *legacyTestCase) runTest(t *testing.T) {
|
|||||||
replicaCalc,
|
replicaCalc,
|
||||||
informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
|
informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
|
||||||
controller.NoResyncPeriodFunc(),
|
controller.NoResyncPeriodFunc(),
|
||||||
|
defaultUpscaleForbiddenWindow,
|
||||||
|
defaultDownscaleForbiddenWindow,
|
||||||
)
|
)
|
||||||
hpaController.hpaListerSynced = alwaysReady
|
hpaController.hpaListerSynced = alwaysReady
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user