Replace scale down forbidden window

Replacement is scale down stabilization window. HPA will scale down only
    to max of recommendations it made during that window. More details in

    https://docs.google.com/document/d/1IdG3sqgCEaRV3urPLA29IDudCufD89RYCohfBPNeWIM
This commit is contained in:
Krzysztof Jastrzebski 2018-08-31 09:32:01 +02:00
parent 2548fb08cd
commit 958cba1c82
12 changed files with 200 additions and 92 deletions

View File

@ -90,6 +90,7 @@ API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alp
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,GroupResource,Resource API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,GroupResource,Resource
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerSyncPeriod API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerSyncPeriod
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerUpscaleForbiddenWindow API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerUpscaleForbiddenWindow
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerDownscaleStabilizationWindow
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerDownscaleForbiddenWindow API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerDownscaleForbiddenWindow
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerTolerance API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerTolerance
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerUseRESTClients API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerUseRESTClients

View File

@ -95,7 +95,7 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me
replicaCalc, replicaCalc,
ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(), ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod.Duration, ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod.Duration,
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration,
).Run(ctx.Stop) ).Run(ctx.Stop)
return nil, true, nil return nil, true, nil
} }

View File

@ -25,13 +25,14 @@ import (
// HPAControllerOptions holds the HPAController options. // HPAControllerOptions holds the HPAController options.
type HPAControllerOptions struct { type HPAControllerOptions struct {
HorizontalPodAutoscalerUseRESTClients bool HorizontalPodAutoscalerUseRESTClients bool
HorizontalPodAutoscalerTolerance float64 HorizontalPodAutoscalerTolerance float64
HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration HorizontalPodAutoscalerDownscaleStabilizationWindow metav1.Duration
HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
HorizontalPodAutoscalerSyncPeriod metav1.Duration HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
HorizontalPodAutoscalerCPUInitializationPeriod metav1.Duration HorizontalPodAutoscalerSyncPeriod metav1.Duration
HorizontalPodAutoscalerInitialReadinessDelay metav1.Duration HorizontalPodAutoscalerCPUInitializationPeriod metav1.Duration
HorizontalPodAutoscalerInitialReadinessDelay metav1.Duration
} }
// AddFlags adds flags related to HPAController for controller manager to the specified FlagSet. // AddFlags adds flags related to HPAController for controller manager to the specified FlagSet.
@ -43,7 +44,9 @@ func (o *HPAControllerOptions) AddFlags(fs *pflag.FlagSet) {
fs.DurationVar(&o.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", o.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.") fs.DurationVar(&o.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", o.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.")
fs.DurationVar(&o.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-upscale-delay", o.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "The period since last upscale, before another upscale can be performed in horizontal pod autoscaler.") fs.DurationVar(&o.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-upscale-delay", o.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "The period since last upscale, before another upscale can be performed in horizontal pod autoscaler.")
fs.MarkDeprecated("horizontal-pod-autoscaler-upscale-delay", "This flag is currently no-op and will be deleted.") fs.MarkDeprecated("horizontal-pod-autoscaler-upscale-delay", "This flag is currently no-op and will be deleted.")
fs.DurationVar(&o.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration, "horizontal-pod-autoscaler-downscale-stabilization", o.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration, "The period for which autoscaler will look backwards and not scale down below any recommendation it made during that period.")
fs.DurationVar(&o.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-downscale-delay", o.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "The period since last downscale, before another downscale can be performed in horizontal pod autoscaler.") fs.DurationVar(&o.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-downscale-delay", o.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "The period since last downscale, before another downscale can be performed in horizontal pod autoscaler.")
fs.MarkDeprecated("horizontal-pod-autoscaler-downscale-delay", "This flag is currently no-op and will be deleted.")
fs.Float64Var(&o.HorizontalPodAutoscalerTolerance, "horizontal-pod-autoscaler-tolerance", o.HorizontalPodAutoscalerTolerance, "The minimum change (from 1.0) in the desired-to-actual metrics ratio for the horizontal pod autoscaler to consider scaling.") fs.Float64Var(&o.HorizontalPodAutoscalerTolerance, "horizontal-pod-autoscaler-tolerance", o.HorizontalPodAutoscalerTolerance, "The minimum change (from 1.0) in the desired-to-actual metrics ratio for the horizontal pod autoscaler to consider scaling.")
fs.BoolVar(&o.HorizontalPodAutoscalerUseRESTClients, "horizontal-pod-autoscaler-use-rest-clients", o.HorizontalPodAutoscalerUseRESTClients, "If set to true, causes the horizontal pod autoscaler controller to use REST clients through the kube-aggregator, instead of using the legacy metrics client through the API server proxy. This is required for custom metrics support in the horizontal pod autoscaler.") fs.BoolVar(&o.HorizontalPodAutoscalerUseRESTClients, "horizontal-pod-autoscaler-use-rest-clients", o.HorizontalPodAutoscalerUseRESTClients, "If set to true, causes the horizontal pod autoscaler controller to use REST clients through the kube-aggregator, instead of using the legacy metrics client through the API server proxy. This is required for custom metrics support in the horizontal pod autoscaler.")
fs.DurationVar(&o.HorizontalPodAutoscalerCPUInitializationPeriod.Duration, "horizontal-pod-autoscaler-cpu-initialization-period", o.HorizontalPodAutoscalerCPUInitializationPeriod.Duration, "The period after pod start when CPU samples might be skipped.") fs.DurationVar(&o.HorizontalPodAutoscalerCPUInitializationPeriod.Duration, "horizontal-pod-autoscaler-cpu-initialization-period", o.HorizontalPodAutoscalerCPUInitializationPeriod.Duration, "The period after pod start when CPU samples might be skipped.")
@ -57,7 +60,7 @@ func (o *HPAControllerOptions) ApplyTo(cfg *componentconfig.HPAControllerConfigu
} }
cfg.HorizontalPodAutoscalerSyncPeriod = o.HorizontalPodAutoscalerSyncPeriod cfg.HorizontalPodAutoscalerSyncPeriod = o.HorizontalPodAutoscalerSyncPeriod
cfg.HorizontalPodAutoscalerDownscaleForbiddenWindow = o.HorizontalPodAutoscalerDownscaleForbiddenWindow cfg.HorizontalPodAutoscalerDownscaleStabilizationWindow = o.HorizontalPodAutoscalerDownscaleStabilizationWindow
cfg.HorizontalPodAutoscalerTolerance = o.HorizontalPodAutoscalerTolerance cfg.HorizontalPodAutoscalerTolerance = o.HorizontalPodAutoscalerTolerance
cfg.HorizontalPodAutoscalerUseRESTClients = o.HorizontalPodAutoscalerUseRESTClients cfg.HorizontalPodAutoscalerUseRESTClients = o.HorizontalPodAutoscalerUseRESTClients
cfg.HorizontalPodAutoscalerCPUInitializationPeriod = o.HorizontalPodAutoscalerCPUInitializationPeriod cfg.HorizontalPodAutoscalerCPUInitializationPeriod = o.HorizontalPodAutoscalerCPUInitializationPeriod

View File

@ -132,13 +132,14 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) {
EnableGarbageCollector: componentConfig.GarbageCollectorController.EnableGarbageCollector, EnableGarbageCollector: componentConfig.GarbageCollectorController.EnableGarbageCollector,
}, },
HPAController: &HPAControllerOptions{ HPAController: &HPAControllerOptions{
HorizontalPodAutoscalerSyncPeriod: componentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod, HorizontalPodAutoscalerSyncPeriod: componentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod,
HorizontalPodAutoscalerUpscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow, HorizontalPodAutoscalerUpscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow,
HorizontalPodAutoscalerDownscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow, HorizontalPodAutoscalerDownscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow,
HorizontalPodAutoscalerCPUInitializationPeriod: componentConfig.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod, HorizontalPodAutoscalerDownscaleStabilizationWindow: componentConfig.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow,
HorizontalPodAutoscalerInitialReadinessDelay: componentConfig.HPAController.HorizontalPodAutoscalerInitialReadinessDelay, HorizontalPodAutoscalerCPUInitializationPeriod: componentConfig.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod,
HorizontalPodAutoscalerTolerance: componentConfig.HPAController.HorizontalPodAutoscalerTolerance, HorizontalPodAutoscalerInitialReadinessDelay: componentConfig.HPAController.HorizontalPodAutoscalerInitialReadinessDelay,
HorizontalPodAutoscalerUseRESTClients: componentConfig.HPAController.HorizontalPodAutoscalerUseRESTClients, HorizontalPodAutoscalerTolerance: componentConfig.HPAController.HorizontalPodAutoscalerTolerance,
HorizontalPodAutoscalerUseRESTClients: componentConfig.HPAController.HorizontalPodAutoscalerUseRESTClients,
}, },
JobController: &JobControllerOptions{ JobController: &JobControllerOptions{
ConcurrentJobSyncs: componentConfig.JobController.ConcurrentJobSyncs, ConcurrentJobSyncs: componentConfig.JobController.ConcurrentJobSyncs,

View File

@ -75,6 +75,7 @@ func TestAddFlags(t *testing.T) {
"--horizontal-pod-autoscaler-downscale-delay=2m", "--horizontal-pod-autoscaler-downscale-delay=2m",
"--horizontal-pod-autoscaler-sync-period=45s", "--horizontal-pod-autoscaler-sync-period=45s",
"--horizontal-pod-autoscaler-upscale-delay=1m", "--horizontal-pod-autoscaler-upscale-delay=1m",
"--horizontal-pod-autoscaler-downscale-stabilization=3m",
"--horizontal-pod-autoscaler-cpu-initialization-period=90s", "--horizontal-pod-autoscaler-cpu-initialization-period=90s",
"--horizontal-pod-autoscaler-initial-readiness-delay=50s", "--horizontal-pod-autoscaler-initial-readiness-delay=50s",
"--http2-max-streams-per-connection=47", "--http2-max-streams-per-connection=47",
@ -186,13 +187,14 @@ func TestAddFlags(t *testing.T) {
EnableGarbageCollector: false, EnableGarbageCollector: false,
}, },
HPAController: &HPAControllerOptions{ HPAController: &HPAControllerOptions{
HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 45 * time.Second},
HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 1 * time.Minute}, HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 1 * time.Minute},
HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute}, HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute},
HorizontalPodAutoscalerCPUInitializationPeriod: metav1.Duration{Duration: 90 * time.Second}, HorizontalPodAutoscalerDownscaleStabilizationWindow: metav1.Duration{Duration: 3 * time.Minute},
HorizontalPodAutoscalerInitialReadinessDelay: metav1.Duration{Duration: 50 * time.Second}, HorizontalPodAutoscalerCPUInitializationPeriod: metav1.Duration{Duration: 90 * time.Second},
HorizontalPodAutoscalerTolerance: 0.1, HorizontalPodAutoscalerInitialReadinessDelay: metav1.Duration{Duration: 50 * time.Second},
HorizontalPodAutoscalerUseRESTClients: true, HorizontalPodAutoscalerTolerance: 0.1,
HorizontalPodAutoscalerUseRESTClients: true,
}, },
JobController: &JobControllerOptions{ JobController: &JobControllerOptions{
ConcurrentJobSyncs: 5, ConcurrentJobSyncs: 5,

View File

@ -264,6 +264,9 @@ type HPAControllerConfiguration struct {
HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
// horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed. // horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed.
HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
// HorizontalPodAutoscalerDowncaleStabilizationWindow is a period for which autoscaler will look
// backwards and not scale down below any recommendation it made during that period.
HorizontalPodAutoscalerDownscaleStabilizationWindow metav1.Duration
// horizontalPodAutoscalerTolerance is the tolerance for when // horizontalPodAutoscalerTolerance is the tolerance for when
// resource usage suggests upscaling/downscaling // resource usage suggests upscaling/downscaling
HorizontalPodAutoscalerTolerance float64 HorizontalPodAutoscalerTolerance float64

View File

@ -89,6 +89,9 @@ func SetDefaults_KubeControllerManagerConfiguration(obj *KubeControllerManagerCo
if obj.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow == zero { if obj.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow == zero {
obj.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow = metav1.Duration{Duration: 3 * time.Minute} obj.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow = metav1.Duration{Duration: 3 * time.Minute}
} }
if obj.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow == zero {
obj.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow = metav1.Duration{Duration: 5 * time.Minute}
}
if obj.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod == zero { if obj.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod == zero {
obj.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod = metav1.Duration{Duration: 5 * time.Minute} obj.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod = metav1.Duration{Duration: 5 * time.Minute}
} }

View File

@ -306,14 +306,17 @@ type GarbageCollectorControllerConfiguration struct {
} }
type HPAControllerConfiguration struct { type HPAControllerConfiguration struct {
// horizontalPodAutoscalerSyncPeriod is the period for syncing the number of // HorizontalPodAutoscalerSyncPeriod is the period for syncing the number of
// pods in horizontal pod autoscaler. // pods in horizontal pod autoscaler.
HorizontalPodAutoscalerSyncPeriod metav1.Duration HorizontalPodAutoscalerSyncPeriod metav1.Duration
// horizontalPodAutoscalerUpscaleForbiddenWindow is a period after which next upscale allowed. // HorizontalPodAutoscalerUpscaleForbiddenWindow is a period after which next upscale allowed.
HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
// horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed. // HorizontalPodAutoscalerDowncaleStabilizationWindow is a period for which autoscaler will look
// backwards and not scale down below any recommendation it made during that period.
HorizontalPodAutoscalerDownscaleStabilizationWindow metav1.Duration
// HorizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed.
HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
// horizontalPodAutoscalerTolerance is the tolerance for when // HorizontalPodAutoscalerTolerance is the tolerance for when
// resource usage suggests upscaling/downscaling // resource usage suggests upscaling/downscaling
HorizontalPodAutoscalerTolerance float64 HorizontalPodAutoscalerTolerance float64
// HorizontalPodAutoscalerUseRESTClients causes the HPA controller to use REST clients // HorizontalPodAutoscalerUseRESTClients causes the HPA controller to use REST clients

View File

@ -83,6 +83,7 @@ go_test(
"//staging/src/k8s.io/metrics/pkg/client/clientset/versioned/fake:go_default_library", "//staging/src/k8s.io/metrics/pkg/client/clientset/versioned/fake:go_default_library",
"//staging/src/k8s.io/metrics/pkg/client/custom_metrics/fake:go_default_library", "//staging/src/k8s.io/metrics/pkg/client/custom_metrics/fake:go_default_library",
"//staging/src/k8s.io/metrics/pkg/client/external_metrics/fake:go_default_library", "//staging/src/k8s.io/metrics/pkg/client/external_metrics/fake:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library",
"//vendor/github.com/stretchr/testify/require:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library",
"//vendor/k8s.io/heapster/metrics/api/v1/types:go_default_library", "//vendor/k8s.io/heapster/metrics/api/v1/types:go_default_library",

View File

@ -53,6 +53,11 @@ var (
scaleUpLimitMinimum = 4.0 scaleUpLimitMinimum = 4.0
) )
type timestampedRecommendation struct {
recommendation int32
timestamp time.Time
}
// HorizontalController is responsible for the synchronizing HPA objects stored // HorizontalController is responsible for the synchronizing HPA objects stored
// in the system with the actual deployments/replication controllers they // in the system with the actual deployments/replication controllers they
// control. // control.
@ -64,7 +69,7 @@ type HorizontalController struct {
replicaCalc *ReplicaCalculator replicaCalc *ReplicaCalculator
eventRecorder record.EventRecorder eventRecorder record.EventRecorder
downscaleForbiddenWindow time.Duration downscaleStabilisationWindow time.Duration
// hpaLister is able to list/get HPAs from the shared cache from the informer passed in to // hpaLister is able to list/get HPAs from the shared cache from the informer passed in to
// NewHorizontalController. // NewHorizontalController.
@ -73,6 +78,9 @@ type HorizontalController struct {
// Controllers that need to be synced // Controllers that need to be synced
queue workqueue.RateLimitingInterface queue workqueue.RateLimitingInterface
// Latest unstabilized recommendations for each autoscaler.
recommendations map[string][]timestampedRecommendation
} }
// NewHorizontalController creates a new HorizontalController. // NewHorizontalController creates a new HorizontalController.
@ -84,7 +92,7 @@ func NewHorizontalController(
replicaCalc *ReplicaCalculator, replicaCalc *ReplicaCalculator,
hpaInformer autoscalinginformers.HorizontalPodAutoscalerInformer, hpaInformer autoscalinginformers.HorizontalPodAutoscalerInformer,
resyncPeriod time.Duration, resyncPeriod time.Duration,
downscaleForbiddenWindow time.Duration, downscaleStabilisationWindow time.Duration,
) *HorizontalController { ) *HorizontalController {
broadcaster := record.NewBroadcaster() broadcaster := record.NewBroadcaster()
@ -93,13 +101,14 @@ func NewHorizontalController(
recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "horizontal-pod-autoscaler"}) recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "horizontal-pod-autoscaler"})
hpaController := &HorizontalController{ hpaController := &HorizontalController{
replicaCalc: replicaCalc, replicaCalc: replicaCalc,
eventRecorder: recorder, eventRecorder: recorder,
scaleNamespacer: scaleNamespacer, scaleNamespacer: scaleNamespacer,
hpaNamespacer: hpaNamespacer, hpaNamespacer: hpaNamespacer,
downscaleForbiddenWindow: downscaleForbiddenWindow, downscaleStabilisationWindow: downscaleStabilisationWindow,
queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"), queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"),
mapper: mapper, mapper: mapper,
recommendations: map[string][]timestampedRecommendation{},
} }
hpaInformer.Informer().AddEventHandlerWithResyncPeriod( hpaInformer.Informer().AddEventHandlerWithResyncPeriod(
@ -275,10 +284,11 @@ func (a *HorizontalController) reconcileKey(key string) error {
hpa, err := a.hpaLister.HorizontalPodAutoscalers(namespace).Get(name) hpa, err := a.hpaLister.HorizontalPodAutoscalers(namespace).Get(name)
if errors.IsNotFound(err) { if errors.IsNotFound(err) {
glog.Infof("Horizontal Pod Autoscaler %s has been deleted in %s", name, namespace) glog.Infof("Horizontal Pod Autoscaler %s has been deleted in %s", name, namespace)
delete(a.recommendations, key)
return nil return nil
} }
return a.reconcileAutoscaler(hpa) return a.reconcileAutoscaler(hpa, key)
} }
// computeStatusForObjectMetric computes the desired number of replicas for the specified metric of type ObjectMetricSourceType. // computeStatusForObjectMetric computes the desired number of replicas for the specified metric of type ObjectMetricSourceType.
@ -431,7 +441,7 @@ func (a *HorizontalController) computeStatusForExternalMetric(currentReplicas in
return 0, time.Time{}, "", fmt.Errorf(errMsg) return 0, time.Time{}, "", fmt.Errorf(errMsg)
} }
func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.HorizontalPodAutoscaler) error { func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.HorizontalPodAutoscaler, key string) error {
// make a copy so that we never mutate the shared informer cache (conversion can mutate the object) // make a copy so that we never mutate the shared informer cache (conversion can mutate the object)
hpav1 := hpav1Shared.DeepCopy() hpav1 := hpav1Shared.DeepCopy()
// then, convert to autoscaling/v2, which makes our lives easier when calculating metrics // then, convert to autoscaling/v2, which makes our lives easier when calculating metrics
@ -527,24 +537,8 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
if desiredReplicas < currentReplicas { if desiredReplicas < currentReplicas {
rescaleReason = "All metrics below target" rescaleReason = "All metrics below target"
} }
desiredReplicas = a.normalizeDesiredReplicas(hpa, key, currentReplicas, desiredReplicas)
desiredReplicas = a.normalizeDesiredReplicas(hpa, currentReplicas, desiredReplicas) rescale = desiredReplicas != currentReplicas
rescale = a.shouldScale(hpa, currentReplicas, desiredReplicas, timestamp)
backoffDown := false
backoffUp := false
if hpa.Status.LastScaleTime != nil {
if !hpa.Status.LastScaleTime.Add(a.downscaleForbiddenWindow).Before(timestamp) {
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionFalse, "BackoffDownscale", "the time since the previous scale is still within the downscale forbidden window")
backoffDown = true
}
}
if !backoffDown && !backoffUp {
// mark that we're not backing off
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ReadyForNewScale", "the last scale time was sufficiently old as to warrant a new scale")
}
} }
if rescale { if rescale {
@ -572,9 +566,39 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
return a.updateStatusIfNeeded(hpaStatusOriginal, hpa) return a.updateStatusIfNeeded(hpaStatusOriginal, hpa)
} }
// stabilizeRecommendation:
// - replaces old recommendation with the newest recommendation,
// - returns max of recommendations that are not older than downscaleStabilisationWindow.
func (a *HorizontalController) stabilizeRecommendation(key string, prenormalizedDesiredReplicas int32) int32 {
maxRecommendation := prenormalizedDesiredReplicas
foundOldSample := false
oldSampleIndex := 0
cutoff := time.Now().Add(-a.downscaleStabilisationWindow)
for i, rec := range a.recommendations[key] {
if rec.timestamp.Before(cutoff) {
foundOldSample = true
oldSampleIndex = i
} else if rec.recommendation > maxRecommendation {
maxRecommendation = rec.recommendation
}
}
if foundOldSample {
a.recommendations[key][oldSampleIndex] = timestampedRecommendation{prenormalizedDesiredReplicas, time.Now()}
} else {
a.recommendations[key] = append(a.recommendations[key], timestampedRecommendation{prenormalizedDesiredReplicas, time.Now()})
}
return maxRecommendation
}
// normalizeDesiredReplicas takes the metrics desired replicas value and normalizes it based on the appropriate conditions (i.e. < maxReplicas, > // normalizeDesiredReplicas takes the metrics desired replicas value and normalizes it based on the appropriate conditions (i.e. < maxReplicas, >
// minReplicas, etc...) // minReplicas, etc...)
func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas int32, prenormalizedDesiredReplicas int32) int32 { func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.HorizontalPodAutoscaler, key string, currentReplicas int32, prenormalizedDesiredReplicas int32) int32 {
stabilizedRecommendation := a.stabilizeRecommendation(key, prenormalizedDesiredReplicas)
if stabilizedRecommendation != prenormalizedDesiredReplicas {
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ScaleDownStabilized", "recent recommendations were higher than current one, applying the highest recent recommendation")
} else {
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ReadyForNewScale", "recommended size matches current size")
}
var minReplicas int32 var minReplicas int32
if hpa.Spec.MinReplicas != nil { if hpa.Spec.MinReplicas != nil {
minReplicas = *hpa.Spec.MinReplicas minReplicas = *hpa.Spec.MinReplicas
@ -582,9 +606,9 @@ func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.Horiz
minReplicas = 0 minReplicas = 0
} }
desiredReplicas, condition, reason := convertDesiredReplicasWithRules(currentReplicas, prenormalizedDesiredReplicas, minReplicas, hpa.Spec.MaxReplicas) desiredReplicas, condition, reason := convertDesiredReplicasWithRules(currentReplicas, stabilizedRecommendation, minReplicas, hpa.Spec.MaxReplicas)
if desiredReplicas == prenormalizedDesiredReplicas { if desiredReplicas == stabilizedRecommendation {
setCondition(hpa, autoscalingv2.ScalingLimited, v1.ConditionFalse, condition, reason) setCondition(hpa, autoscalingv2.ScalingLimited, v1.ConditionFalse, condition, reason)
} else { } else {
setCondition(hpa, autoscalingv2.ScalingLimited, v1.ConditionTrue, condition, reason) setCondition(hpa, autoscalingv2.ScalingLimited, v1.ConditionTrue, condition, reason)
@ -641,29 +665,6 @@ func calculateScaleUpLimit(currentReplicas int32) int32 {
return int32(math.Max(scaleUpLimitFactor*float64(currentReplicas), scaleUpLimitMinimum)) return int32(math.Max(scaleUpLimitFactor*float64(currentReplicas), scaleUpLimitMinimum))
} }
func (a *HorizontalController) shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, timestamp time.Time) bool {
if desiredReplicas == currentReplicas {
return false
}
if hpa.Status.LastScaleTime == nil {
return true
}
// Going down only if the usageRatio dropped significantly below the target
// and there was no rescaling in the last downscaleForbiddenWindow.
if desiredReplicas < currentReplicas && hpa.Status.LastScaleTime.Add(a.downscaleForbiddenWindow).Before(timestamp) {
return true
}
// Going up only if the usage ratio increased significantly above the target.
if desiredReplicas > currentReplicas {
return true
}
return false
}
// scaleForResourceMappings attempts to fetch the scale for the // scaleForResourceMappings attempts to fetch the scale for the
// resource with the given name and namespace, trying each RESTMapping // resource with the given name and namespace, trying each RESTMapping
// in turn until a working one is found. If none work, the first error // in turn until a working one is found. If none work, the first error

View File

@ -51,6 +51,7 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/golang/glog"
_ "k8s.io/kubernetes/pkg/apis/autoscaling/install" _ "k8s.io/kubernetes/pkg/apis/autoscaling/install"
_ "k8s.io/kubernetes/pkg/apis/extensions/install" _ "k8s.io/kubernetes/pkg/apis/extensions/install"
) )
@ -129,6 +130,8 @@ type testCase struct {
testCMClient *cmfake.FakeCustomMetricsClient testCMClient *cmfake.FakeCustomMetricsClient
testEMClient *emfake.FakeExternalMetricsClient testEMClient *emfake.FakeExternalMetricsClient
testScaleClient *scalefake.FakeScaleClient testScaleClient *scalefake.FakeScaleClient
recommendations []timestampedRecommendation
} }
// Needs to be called under a lock. // Needs to be called under a lock.
@ -662,7 +665,7 @@ func (tc *testCase) setupController(t *testing.T) (*HorizontalController, inform
replicaCalc := NewReplicaCalculator(metricsClient, testClient.Core(), defaultTestingTolerance, defaultTestingCpuInitializationPeriod, defaultTestingDelayOfInitialReadinessStatus) replicaCalc := NewReplicaCalculator(metricsClient, testClient.Core(), defaultTestingTolerance, defaultTestingCpuInitializationPeriod, defaultTestingDelayOfInitialReadinessStatus)
informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc()) informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
defaultDownscaleForbiddenWindow := 5 * time.Minute defaultDownscalestabilizationWindow := 5 * time.Minute
hpaController := NewHorizontalController( hpaController := NewHorizontalController(
eventClient.Core(), eventClient.Core(),
@ -672,9 +675,12 @@ func (tc *testCase) setupController(t *testing.T) (*HorizontalController, inform
replicaCalc, replicaCalc,
informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(), informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
controller.NoResyncPeriodFunc(), controller.NoResyncPeriodFunc(),
defaultDownscaleForbiddenWindow, defaultDownscalestabilizationWindow,
) )
hpaController.hpaListerSynced = alwaysReady hpaController.hpaListerSynced = alwaysReady
if tc.recommendations != nil {
hpaController.recommendations["test-namespace/test-hpa"] = tc.recommendations
}
return hpaController, informerFactory return hpaController, informerFactory
} }
@ -709,6 +715,7 @@ func (tc *testCase) runTestWithController(t *testing.T, hpaController *Horizonta
func (tc *testCase) runTest(t *testing.T) { func (tc *testCase) runTest(t *testing.T) {
hpaController, informerFactory := tc.setupController(t) hpaController, informerFactory := tc.setupController(t)
tc.runTestWithController(t, hpaController, informerFactory) tc.runTestWithController(t, hpaController, informerFactory)
glog.Errorf("recommendations: %+v", hpaController.recommendations)
} }
func TestScaleUp(t *testing.T) { func TestScaleUp(t *testing.T) {
@ -2080,8 +2087,7 @@ func TestNoBackoffUpscaleCMNoBackoffCpu(t *testing.T) {
tc.runTest(t) tc.runTest(t)
} }
func TestBackoffDownscale(t *testing.T) { func TestStabilizeDownscale(t *testing.T) {
time := metav1.Time{Time: time.Now().Add(-4 * time.Minute)}
tc := testCase{ tc := testCase{
minReplicas: 1, minReplicas: 1,
maxReplicas: 5, maxReplicas: 5,
@ -2091,16 +2097,19 @@ func TestBackoffDownscale(t *testing.T) {
reportedLevels: []uint64{50, 50, 50}, reportedLevels: []uint64{50, 50, 50},
reportedCPURequests: []resource.Quantity{resource.MustParse("0.1"), resource.MustParse("0.1"), resource.MustParse("0.1")}, reportedCPURequests: []resource.Quantity{resource.MustParse("0.1"), resource.MustParse("0.1"), resource.MustParse("0.1")},
useMetricsAPI: true, useMetricsAPI: true,
lastScaleTime: &time,
expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{ expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
Type: autoscalingv2.AbleToScale, Type: autoscalingv2.AbleToScale,
Status: v1.ConditionTrue, Status: v1.ConditionTrue,
Reason: "ReadyForNewScale", Reason: "ReadyForNewScale",
}, autoscalingv2.HorizontalPodAutoscalerCondition{ }, autoscalingv2.HorizontalPodAutoscalerCondition{
Type: autoscalingv2.AbleToScale, Type: autoscalingv2.AbleToScale,
Status: v1.ConditionFalse, Status: v1.ConditionTrue,
Reason: "BackoffDownscale", Reason: "ScaleDownStabilized",
}), }),
recommendations: []timestampedRecommendation{
{10, time.Now().Add(-10 * time.Minute)},
{4, time.Now().Add(-1 * time.Minute)},
},
} }
tc.runTest(t) tc.runTest(t)
} }
@ -2278,7 +2287,7 @@ func TestAvoidUncessaryUpdates(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
if err := controller.reconcileAutoscaler(&initialHPAs.Items[0]); err != nil { if err := controller.reconcileAutoscaler(&initialHPAs.Items[0], ""); err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
@ -2353,4 +2362,85 @@ func TestConvertDesiredReplicasWithRules(t *testing.T) {
} }
} }
func TestNormalizeDesiredReplicas(t *testing.T) {
tests := []struct {
name string
key string
recommendations []timestampedRecommendation
prenormalizedDesiredReplicas int32
expectedStabilizedReplicas int32
expectedLogLength int
}{
{
"empty log",
"",
[]timestampedRecommendation{},
5,
5,
1,
},
{
"stabilize",
"",
[]timestampedRecommendation{
{4, time.Now().Add(-2 * time.Minute)},
{5, time.Now().Add(-1 * time.Minute)},
},
3,
5,
3,
},
{
"no stabilize",
"",
[]timestampedRecommendation{
{1, time.Now().Add(-2 * time.Minute)},
{2, time.Now().Add(-1 * time.Minute)},
},
3,
3,
3,
},
{
"no stabilize - old recommendations",
"",
[]timestampedRecommendation{
{10, time.Now().Add(-10 * time.Minute)},
{9, time.Now().Add(-9 * time.Minute)},
},
3,
3,
2,
},
{
"stabilize - old recommendations",
"",
[]timestampedRecommendation{
{10, time.Now().Add(-10 * time.Minute)},
{4, time.Now().Add(-1 * time.Minute)},
{5, time.Now().Add(-2 * time.Minute)},
{9, time.Now().Add(-9 * time.Minute)},
},
3,
5,
4,
},
}
for _, tc := range tests {
hc := HorizontalController{
downscaleStabilisationWindow: 5 * time.Minute,
recommendations: map[string][]timestampedRecommendation{
tc.key: tc.recommendations,
},
}
r := hc.stabilizeRecommendation(tc.key, tc.prenormalizedDesiredReplicas)
if r != tc.expectedStabilizedReplicas {
t.Errorf("[%s] got %d stabilized replicas, expected %d", tc.name, r, tc.expectedStabilizedReplicas)
}
if len(hc.recommendations[tc.key]) != tc.expectedLogLength {
t.Errorf("[%s] after stabilization recommendations log has %d entries, expected %d", tc.name, len(hc.recommendations[tc.key]), tc.expectedLogLength)
}
}
}
// TODO: add more tests // TODO: add more tests

View File

@ -488,7 +488,7 @@ func (tc *legacyTestCase) runTest(t *testing.T) {
replicaCalc := NewReplicaCalculator(metricsClient, testClient.Core(), defaultTestingTolerance, defaultTestingCpuInitializationPeriod, defaultTestingDelayOfInitialReadinessStatus) replicaCalc := NewReplicaCalculator(metricsClient, testClient.Core(), defaultTestingTolerance, defaultTestingCpuInitializationPeriod, defaultTestingDelayOfInitialReadinessStatus)
informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc()) informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
defaultDownscaleForbiddenWindow := 5 * time.Minute defaultDownscaleStabilisationWindow := 5 * time.Minute
hpaController := NewHorizontalController( hpaController := NewHorizontalController(
eventClient.Core(), eventClient.Core(),
@ -498,7 +498,7 @@ func (tc *legacyTestCase) runTest(t *testing.T) {
replicaCalc, replicaCalc,
informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(), informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
controller.NoResyncPeriodFunc(), controller.NoResyncPeriodFunc(),
defaultDownscaleForbiddenWindow, defaultDownscaleStabilisationWindow,
) )
hpaController.hpaListerSynced = alwaysReady hpaController.hpaListerSynced = alwaysReady