mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-22 19:31:44 +00:00
Replace scale down forbidden window
Replacement is scale down stabilization window. HPA will scale down only
to max of recommendations it made during that window. More details in
https://docs.google.com/document/d/1IdG3sqgCEaRV3urPLA29IDudCufD89RYCohfBPNeWIM
This commit is contained in:
Krzysztof Jastrzebski
parent
2548fb08cd
commit
958cba1c82
@ -90,6 +90,7 @@ API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alp
|
||||
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,GroupResource,Resource
|
||||
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerSyncPeriod
|
||||
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerUpscaleForbiddenWindow
|
||||
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerDownscaleStabilizationWindow
|
||||
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerDownscaleForbiddenWindow
|
||||
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerTolerance
|
||||
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerUseRESTClients
|
||||
|
||||
@ -95,7 +95,7 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me
|
||||
replicaCalc,
|
||||
ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
|
||||
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod.Duration,
|
||||
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration,
|
||||
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration,
|
||||
).Run(ctx.Stop)
|
||||
return nil, true, nil
|
||||
}
|
||||
|
||||
@ -25,13 +25,14 @@ import (
|
||||
|
||||
// HPAControllerOptions holds the HPAController options.
|
||||
type HPAControllerOptions struct {
|
||||
HorizontalPodAutoscalerUseRESTClients bool
|
||||
HorizontalPodAutoscalerTolerance float64
|
||||
HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
|
||||
HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
|
||||
HorizontalPodAutoscalerSyncPeriod metav1.Duration
|
||||
HorizontalPodAutoscalerCPUInitializationPeriod metav1.Duration
|
||||
HorizontalPodAutoscalerInitialReadinessDelay metav1.Duration
|
||||
HorizontalPodAutoscalerUseRESTClients bool
|
||||
HorizontalPodAutoscalerTolerance float64
|
||||
HorizontalPodAutoscalerDownscaleStabilizationWindow metav1.Duration
|
||||
HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
|
||||
HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
|
||||
HorizontalPodAutoscalerSyncPeriod metav1.Duration
|
||||
HorizontalPodAutoscalerCPUInitializationPeriod metav1.Duration
|
||||
HorizontalPodAutoscalerInitialReadinessDelay metav1.Duration
|
||||
}
|
||||
|
||||
// AddFlags adds flags related to HPAController for controller manager to the specified FlagSet.
|
||||
@ -43,7 +44,9 @@ func (o *HPAControllerOptions) AddFlags(fs *pflag.FlagSet) {
|
||||
fs.DurationVar(&o.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", o.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.")
|
||||
fs.DurationVar(&o.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-upscale-delay", o.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "The period since last upscale, before another upscale can be performed in horizontal pod autoscaler.")
|
||||
fs.MarkDeprecated("horizontal-pod-autoscaler-upscale-delay", "This flag is currently no-op and will be deleted.")
|
||||
fs.DurationVar(&o.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration, "horizontal-pod-autoscaler-downscale-stabilization", o.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration, "The period for which autoscaler will look backwards and not scale down below any recommendation it made during that period.")
|
||||
fs.DurationVar(&o.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-downscale-delay", o.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "The period since last downscale, before another downscale can be performed in horizontal pod autoscaler.")
|
||||
fs.MarkDeprecated("horizontal-pod-autoscaler-downscale-delay", "This flag is currently no-op and will be deleted.")
|
||||
fs.Float64Var(&o.HorizontalPodAutoscalerTolerance, "horizontal-pod-autoscaler-tolerance", o.HorizontalPodAutoscalerTolerance, "The minimum change (from 1.0) in the desired-to-actual metrics ratio for the horizontal pod autoscaler to consider scaling.")
|
||||
fs.BoolVar(&o.HorizontalPodAutoscalerUseRESTClients, "horizontal-pod-autoscaler-use-rest-clients", o.HorizontalPodAutoscalerUseRESTClients, "If set to true, causes the horizontal pod autoscaler controller to use REST clients through the kube-aggregator, instead of using the legacy metrics client through the API server proxy. This is required for custom metrics support in the horizontal pod autoscaler.")
|
||||
fs.DurationVar(&o.HorizontalPodAutoscalerCPUInitializationPeriod.Duration, "horizontal-pod-autoscaler-cpu-initialization-period", o.HorizontalPodAutoscalerCPUInitializationPeriod.Duration, "The period after pod start when CPU samples might be skipped.")
|
||||
@ -57,7 +60,7 @@ func (o *HPAControllerOptions) ApplyTo(cfg *componentconfig.HPAControllerConfigu
|
||||
}
|
||||
|
||||
cfg.HorizontalPodAutoscalerSyncPeriod = o.HorizontalPodAutoscalerSyncPeriod
|
||||
cfg.HorizontalPodAutoscalerDownscaleForbiddenWindow = o.HorizontalPodAutoscalerDownscaleForbiddenWindow
|
||||
cfg.HorizontalPodAutoscalerDownscaleStabilizationWindow = o.HorizontalPodAutoscalerDownscaleStabilizationWindow
|
||||
cfg.HorizontalPodAutoscalerTolerance = o.HorizontalPodAutoscalerTolerance
|
||||
cfg.HorizontalPodAutoscalerUseRESTClients = o.HorizontalPodAutoscalerUseRESTClients
|
||||
cfg.HorizontalPodAutoscalerCPUInitializationPeriod = o.HorizontalPodAutoscalerCPUInitializationPeriod
|
||||
|
||||
@ -132,13 +132,14 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) {
|
||||
EnableGarbageCollector: componentConfig.GarbageCollectorController.EnableGarbageCollector,
|
||||
},
|
||||
HPAController: &HPAControllerOptions{
|
||||
HorizontalPodAutoscalerSyncPeriod: componentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod,
|
||||
HorizontalPodAutoscalerUpscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow,
|
||||
HorizontalPodAutoscalerDownscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow,
|
||||
HorizontalPodAutoscalerCPUInitializationPeriod: componentConfig.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod,
|
||||
HorizontalPodAutoscalerInitialReadinessDelay: componentConfig.HPAController.HorizontalPodAutoscalerInitialReadinessDelay,
|
||||
HorizontalPodAutoscalerTolerance: componentConfig.HPAController.HorizontalPodAutoscalerTolerance,
|
||||
HorizontalPodAutoscalerUseRESTClients: componentConfig.HPAController.HorizontalPodAutoscalerUseRESTClients,
|
||||
HorizontalPodAutoscalerSyncPeriod: componentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod,
|
||||
HorizontalPodAutoscalerUpscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow,
|
||||
HorizontalPodAutoscalerDownscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow,
|
||||
HorizontalPodAutoscalerDownscaleStabilizationWindow: componentConfig.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow,
|
||||
HorizontalPodAutoscalerCPUInitializationPeriod: componentConfig.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod,
|
||||
HorizontalPodAutoscalerInitialReadinessDelay: componentConfig.HPAController.HorizontalPodAutoscalerInitialReadinessDelay,
|
||||
HorizontalPodAutoscalerTolerance: componentConfig.HPAController.HorizontalPodAutoscalerTolerance,
|
||||
HorizontalPodAutoscalerUseRESTClients: componentConfig.HPAController.HorizontalPodAutoscalerUseRESTClients,
|
||||
},
|
||||
JobController: &JobControllerOptions{
|
||||
ConcurrentJobSyncs: componentConfig.JobController.ConcurrentJobSyncs,
|
||||
|
||||
@ -75,6 +75,7 @@ func TestAddFlags(t *testing.T) {
|
||||
"--horizontal-pod-autoscaler-downscale-delay=2m",
|
||||
"--horizontal-pod-autoscaler-sync-period=45s",
|
||||
"--horizontal-pod-autoscaler-upscale-delay=1m",
|
||||
"--horizontal-pod-autoscaler-downscale-stabilization=3m",
|
||||
"--horizontal-pod-autoscaler-cpu-initialization-period=90s",
|
||||
"--horizontal-pod-autoscaler-initial-readiness-delay=50s",
|
||||
"--http2-max-streams-per-connection=47",
|
||||
@ -186,13 +187,14 @@ func TestAddFlags(t *testing.T) {
|
||||
EnableGarbageCollector: false,
|
||||
},
|
||||
HPAController: &HPAControllerOptions{
|
||||
HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 45 * time.Second},
|
||||
HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 1 * time.Minute},
|
||||
HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute},
|
||||
HorizontalPodAutoscalerCPUInitializationPeriod: metav1.Duration{Duration: 90 * time.Second},
|
||||
HorizontalPodAutoscalerInitialReadinessDelay: metav1.Duration{Duration: 50 * time.Second},
|
||||
HorizontalPodAutoscalerTolerance: 0.1,
|
||||
HorizontalPodAutoscalerUseRESTClients: true,
|
||||
HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 45 * time.Second},
|
||||
HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 1 * time.Minute},
|
||||
HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute},
|
||||
HorizontalPodAutoscalerDownscaleStabilizationWindow: metav1.Duration{Duration: 3 * time.Minute},
|
||||
HorizontalPodAutoscalerCPUInitializationPeriod: metav1.Duration{Duration: 90 * time.Second},
|
||||
HorizontalPodAutoscalerInitialReadinessDelay: metav1.Duration{Duration: 50 * time.Second},
|
||||
HorizontalPodAutoscalerTolerance: 0.1,
|
||||
HorizontalPodAutoscalerUseRESTClients: true,
|
||||
},
|
||||
JobController: &JobControllerOptions{
|
||||
ConcurrentJobSyncs: 5,
|
||||
|
||||
@ -264,6 +264,9 @@ type HPAControllerConfiguration struct {
|
||||
HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
|
||||
// horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed.
|
||||
HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
|
||||
// HorizontalPodAutoscalerDowncaleStabilizationWindow is a period for which autoscaler will look
|
||||
// backwards and not scale down below any recommendation it made during that period.
|
||||
HorizontalPodAutoscalerDownscaleStabilizationWindow metav1.Duration
|
||||
// horizontalPodAutoscalerTolerance is the tolerance for when
|
||||
// resource usage suggests upscaling/downscaling
|
||||
HorizontalPodAutoscalerTolerance float64
|
||||
|
||||
@ -89,6 +89,9 @@ func SetDefaults_KubeControllerManagerConfiguration(obj *KubeControllerManagerCo
|
||||
if obj.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow == zero {
|
||||
obj.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow = metav1.Duration{Duration: 3 * time.Minute}
|
||||
}
|
||||
if obj.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow == zero {
|
||||
obj.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow = metav1.Duration{Duration: 5 * time.Minute}
|
||||
}
|
||||
if obj.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod == zero {
|
||||
obj.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod = metav1.Duration{Duration: 5 * time.Minute}
|
||||
}
|
||||
|
||||
@ -306,14 +306,17 @@ type GarbageCollectorControllerConfiguration struct {
|
||||
}
|
||||
|
||||
type HPAControllerConfiguration struct {
|
||||
// horizontalPodAutoscalerSyncPeriod is the period for syncing the number of
|
||||
// HorizontalPodAutoscalerSyncPeriod is the period for syncing the number of
|
||||
// pods in horizontal pod autoscaler.
|
||||
HorizontalPodAutoscalerSyncPeriod metav1.Duration
|
||||
// horizontalPodAutoscalerUpscaleForbiddenWindow is a period after which next upscale allowed.
|
||||
// HorizontalPodAutoscalerUpscaleForbiddenWindow is a period after which next upscale allowed.
|
||||
HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
|
||||
// horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed.
|
||||
// HorizontalPodAutoscalerDowncaleStabilizationWindow is a period for which autoscaler will look
|
||||
// backwards and not scale down below any recommendation it made during that period.
|
||||
HorizontalPodAutoscalerDownscaleStabilizationWindow metav1.Duration
|
||||
// HorizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed.
|
||||
HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
|
||||
// horizontalPodAutoscalerTolerance is the tolerance for when
|
||||
// HorizontalPodAutoscalerTolerance is the tolerance for when
|
||||
// resource usage suggests upscaling/downscaling
|
||||
HorizontalPodAutoscalerTolerance float64
|
||||
// HorizontalPodAutoscalerUseRESTClients causes the HPA controller to use REST clients
|
||||
|
||||
@ -83,6 +83,7 @@ go_test(
|
||||
"//staging/src/k8s.io/metrics/pkg/client/clientset/versioned/fake:go_default_library",
|
||||
"//staging/src/k8s.io/metrics/pkg/client/custom_metrics/fake:go_default_library",
|
||||
"//staging/src/k8s.io/metrics/pkg/client/external_metrics/fake:go_default_library",
|
||||
"//vendor/github.com/golang/glog:go_default_library",
|
||||
"//vendor/github.com/stretchr/testify/assert:go_default_library",
|
||||
"//vendor/github.com/stretchr/testify/require:go_default_library",
|
||||
"//vendor/k8s.io/heapster/metrics/api/v1/types:go_default_library",
|
||||
|
||||
@ -53,6 +53,11 @@ var (
|
||||
scaleUpLimitMinimum = 4.0
|
||||
)
|
||||
|
||||
type timestampedRecommendation struct {
|
||||
recommendation int32
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
// HorizontalController is responsible for the synchronizing HPA objects stored
|
||||
// in the system with the actual deployments/replication controllers they
|
||||
// control.
|
||||
@ -64,7 +69,7 @@ type HorizontalController struct {
|
||||
replicaCalc *ReplicaCalculator
|
||||
eventRecorder record.EventRecorder
|
||||
|
||||
downscaleForbiddenWindow time.Duration
|
||||
downscaleStabilisationWindow time.Duration
|
||||
|
||||
// hpaLister is able to list/get HPAs from the shared cache from the informer passed in to
|
||||
// NewHorizontalController.
|
||||
@ -73,6 +78,9 @@ type HorizontalController struct {
|
||||
|
||||
// Controllers that need to be synced
|
||||
queue workqueue.RateLimitingInterface
|
||||
|
||||
// Latest unstabilized recommendations for each autoscaler.
|
||||
recommendations map[string][]timestampedRecommendation
|
||||
}
|
||||
|
||||
// NewHorizontalController creates a new HorizontalController.
|
||||
@ -84,7 +92,7 @@ func NewHorizontalController(
|
||||
replicaCalc *ReplicaCalculator,
|
||||
hpaInformer autoscalinginformers.HorizontalPodAutoscalerInformer,
|
||||
resyncPeriod time.Duration,
|
||||
downscaleForbiddenWindow time.Duration,
|
||||
downscaleStabilisationWindow time.Duration,
|
||||
|
||||
) *HorizontalController {
|
||||
broadcaster := record.NewBroadcaster()
|
||||
@ -93,13 +101,14 @@ func NewHorizontalController(
|
||||
recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "horizontal-pod-autoscaler"})
|
||||
|
||||
hpaController := &HorizontalController{
|
||||
replicaCalc: replicaCalc,
|
||||
eventRecorder: recorder,
|
||||
scaleNamespacer: scaleNamespacer,
|
||||
hpaNamespacer: hpaNamespacer,
|
||||
downscaleForbiddenWindow: downscaleForbiddenWindow,
|
||||
queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"),
|
||||
mapper: mapper,
|
||||
replicaCalc: replicaCalc,
|
||||
eventRecorder: recorder,
|
||||
scaleNamespacer: scaleNamespacer,
|
||||
hpaNamespacer: hpaNamespacer,
|
||||
downscaleStabilisationWindow: downscaleStabilisationWindow,
|
||||
queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"),
|
||||
mapper: mapper,
|
||||
recommendations: map[string][]timestampedRecommendation{},
|
||||
}
|
||||
|
||||
hpaInformer.Informer().AddEventHandlerWithResyncPeriod(
|
||||
@ -275,10 +284,11 @@ func (a *HorizontalController) reconcileKey(key string) error {
|
||||
hpa, err := a.hpaLister.HorizontalPodAutoscalers(namespace).Get(name)
|
||||
if errors.IsNotFound(err) {
|
||||
glog.Infof("Horizontal Pod Autoscaler %s has been deleted in %s", name, namespace)
|
||||
delete(a.recommendations, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
return a.reconcileAutoscaler(hpa)
|
||||
return a.reconcileAutoscaler(hpa, key)
|
||||
}
|
||||
|
||||
// computeStatusForObjectMetric computes the desired number of replicas for the specified metric of type ObjectMetricSourceType.
|
||||
@ -431,7 +441,7 @@ func (a *HorizontalController) computeStatusForExternalMetric(currentReplicas in
|
||||
return 0, time.Time{}, "", fmt.Errorf(errMsg)
|
||||
}
|
||||
|
||||
func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.HorizontalPodAutoscaler) error {
|
||||
func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.HorizontalPodAutoscaler, key string) error {
|
||||
// make a copy so that we never mutate the shared informer cache (conversion can mutate the object)
|
||||
hpav1 := hpav1Shared.DeepCopy()
|
||||
// then, convert to autoscaling/v2, which makes our lives easier when calculating metrics
|
||||
@ -527,24 +537,8 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
|
||||
if desiredReplicas < currentReplicas {
|
||||
rescaleReason = "All metrics below target"
|
||||
}
|
||||
|
||||
desiredReplicas = a.normalizeDesiredReplicas(hpa, currentReplicas, desiredReplicas)
|
||||
|
||||
rescale = a.shouldScale(hpa, currentReplicas, desiredReplicas, timestamp)
|
||||
backoffDown := false
|
||||
backoffUp := false
|
||||
if hpa.Status.LastScaleTime != nil {
|
||||
if !hpa.Status.LastScaleTime.Add(a.downscaleForbiddenWindow).Before(timestamp) {
|
||||
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionFalse, "BackoffDownscale", "the time since the previous scale is still within the downscale forbidden window")
|
||||
backoffDown = true
|
||||
}
|
||||
}
|
||||
|
||||
if !backoffDown && !backoffUp {
|
||||
// mark that we're not backing off
|
||||
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ReadyForNewScale", "the last scale time was sufficiently old as to warrant a new scale")
|
||||
}
|
||||
|
||||
desiredReplicas = a.normalizeDesiredReplicas(hpa, key, currentReplicas, desiredReplicas)
|
||||
rescale = desiredReplicas != currentReplicas
|
||||
}
|
||||
|
||||
if rescale {
|
||||
@ -572,9 +566,39 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
|
||||
return a.updateStatusIfNeeded(hpaStatusOriginal, hpa)
|
||||
}
|
||||
|
||||
// stabilizeRecommendation:
|
||||
// - replaces old recommendation with the newest recommendation,
|
||||
// - returns max of recommendations that are not older than downscaleStabilisationWindow.
|
||||
func (a *HorizontalController) stabilizeRecommendation(key string, prenormalizedDesiredReplicas int32) int32 {
|
||||
maxRecommendation := prenormalizedDesiredReplicas
|
||||
foundOldSample := false
|
||||
oldSampleIndex := 0
|
||||
cutoff := time.Now().Add(-a.downscaleStabilisationWindow)
|
||||
for i, rec := range a.recommendations[key] {
|
||||
if rec.timestamp.Before(cutoff) {
|
||||
foundOldSample = true
|
||||
oldSampleIndex = i
|
||||
} else if rec.recommendation > maxRecommendation {
|
||||
maxRecommendation = rec.recommendation
|
||||
}
|
||||
}
|
||||
if foundOldSample {
|
||||
a.recommendations[key][oldSampleIndex] = timestampedRecommendation{prenormalizedDesiredReplicas, time.Now()}
|
||||
} else {
|
||||
a.recommendations[key] = append(a.recommendations[key], timestampedRecommendation{prenormalizedDesiredReplicas, time.Now()})
|
||||
}
|
||||
return maxRecommendation
|
||||
}
|
||||
|
||||
// normalizeDesiredReplicas takes the metrics desired replicas value and normalizes it based on the appropriate conditions (i.e. < maxReplicas, >
|
||||
// minReplicas, etc...)
|
||||
func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas int32, prenormalizedDesiredReplicas int32) int32 {
|
||||
func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.HorizontalPodAutoscaler, key string, currentReplicas int32, prenormalizedDesiredReplicas int32) int32 {
|
||||
stabilizedRecommendation := a.stabilizeRecommendation(key, prenormalizedDesiredReplicas)
|
||||
if stabilizedRecommendation != prenormalizedDesiredReplicas {
|
||||
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ScaleDownStabilized", "recent recommendations were higher than current one, applying the highest recent recommendation")
|
||||
} else {
|
||||
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ReadyForNewScale", "recommended size matches current size")
|
||||
}
|
||||
var minReplicas int32
|
||||
if hpa.Spec.MinReplicas != nil {
|
||||
minReplicas = *hpa.Spec.MinReplicas
|
||||
@ -582,9 +606,9 @@ func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.Horiz
|
||||
minReplicas = 0
|
||||
}
|
||||
|
||||
desiredReplicas, condition, reason := convertDesiredReplicasWithRules(currentReplicas, prenormalizedDesiredReplicas, minReplicas, hpa.Spec.MaxReplicas)
|
||||
desiredReplicas, condition, reason := convertDesiredReplicasWithRules(currentReplicas, stabilizedRecommendation, minReplicas, hpa.Spec.MaxReplicas)
|
||||
|
||||
if desiredReplicas == prenormalizedDesiredReplicas {
|
||||
if desiredReplicas == stabilizedRecommendation {
|
||||
setCondition(hpa, autoscalingv2.ScalingLimited, v1.ConditionFalse, condition, reason)
|
||||
} else {
|
||||
setCondition(hpa, autoscalingv2.ScalingLimited, v1.ConditionTrue, condition, reason)
|
||||
@ -641,29 +665,6 @@ func calculateScaleUpLimit(currentReplicas int32) int32 {
|
||||
return int32(math.Max(scaleUpLimitFactor*float64(currentReplicas), scaleUpLimitMinimum))
|
||||
}
|
||||
|
||||
func (a *HorizontalController) shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, timestamp time.Time) bool {
|
||||
if desiredReplicas == currentReplicas {
|
||||
return false
|
||||
}
|
||||
|
||||
if hpa.Status.LastScaleTime == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
// Going down only if the usageRatio dropped significantly below the target
|
||||
// and there was no rescaling in the last downscaleForbiddenWindow.
|
||||
if desiredReplicas < currentReplicas && hpa.Status.LastScaleTime.Add(a.downscaleForbiddenWindow).Before(timestamp) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Going up only if the usage ratio increased significantly above the target.
|
||||
if desiredReplicas > currentReplicas {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// scaleForResourceMappings attempts to fetch the scale for the
|
||||
// resource with the given name and namespace, trying each RESTMapping
|
||||
// in turn until a working one is found. If none work, the first error
|
||||
|
||||
@ -51,6 +51,7 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/golang/glog"
|
||||
_ "k8s.io/kubernetes/pkg/apis/autoscaling/install"
|
||||
_ "k8s.io/kubernetes/pkg/apis/extensions/install"
|
||||
)
|
||||
@ -129,6 +130,8 @@ type testCase struct {
|
||||
testCMClient *cmfake.FakeCustomMetricsClient
|
||||
testEMClient *emfake.FakeExternalMetricsClient
|
||||
testScaleClient *scalefake.FakeScaleClient
|
||||
|
||||
recommendations []timestampedRecommendation
|
||||
}
|
||||
|
||||
// Needs to be called under a lock.
|
||||
@ -662,7 +665,7 @@ func (tc *testCase) setupController(t *testing.T) (*HorizontalController, inform
|
||||
replicaCalc := NewReplicaCalculator(metricsClient, testClient.Core(), defaultTestingTolerance, defaultTestingCpuInitializationPeriod, defaultTestingDelayOfInitialReadinessStatus)
|
||||
|
||||
informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
|
||||
defaultDownscaleForbiddenWindow := 5 * time.Minute
|
||||
defaultDownscalestabilizationWindow := 5 * time.Minute
|
||||
|
||||
hpaController := NewHorizontalController(
|
||||
eventClient.Core(),
|
||||
@ -672,9 +675,12 @@ func (tc *testCase) setupController(t *testing.T) (*HorizontalController, inform
|
||||
replicaCalc,
|
||||
informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
|
||||
controller.NoResyncPeriodFunc(),
|
||||
defaultDownscaleForbiddenWindow,
|
||||
defaultDownscalestabilizationWindow,
|
||||
)
|
||||
hpaController.hpaListerSynced = alwaysReady
|
||||
if tc.recommendations != nil {
|
||||
hpaController.recommendations["test-namespace/test-hpa"] = tc.recommendations
|
||||
}
|
||||
|
||||
return hpaController, informerFactory
|
||||
}
|
||||
@ -709,6 +715,7 @@ func (tc *testCase) runTestWithController(t *testing.T, hpaController *Horizonta
|
||||
func (tc *testCase) runTest(t *testing.T) {
|
||||
hpaController, informerFactory := tc.setupController(t)
|
||||
tc.runTestWithController(t, hpaController, informerFactory)
|
||||
glog.Errorf("recommendations: %+v", hpaController.recommendations)
|
||||
}
|
||||
|
||||
func TestScaleUp(t *testing.T) {
|
||||
@ -2080,8 +2087,7 @@ func TestNoBackoffUpscaleCMNoBackoffCpu(t *testing.T) {
|
||||
tc.runTest(t)
|
||||
}
|
||||
|
||||
func TestBackoffDownscale(t *testing.T) {
|
||||
time := metav1.Time{Time: time.Now().Add(-4 * time.Minute)}
|
||||
func TestStabilizeDownscale(t *testing.T) {
|
||||
tc := testCase{
|
||||
minReplicas: 1,
|
||||
maxReplicas: 5,
|
||||
@ -2091,16 +2097,19 @@ func TestBackoffDownscale(t *testing.T) {
|
||||
reportedLevels: []uint64{50, 50, 50},
|
||||
reportedCPURequests: []resource.Quantity{resource.MustParse("0.1"), resource.MustParse("0.1"), resource.MustParse("0.1")},
|
||||
useMetricsAPI: true,
|
||||
lastScaleTime: &time,
|
||||
expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
|
||||
Type: autoscalingv2.AbleToScale,
|
||||
Status: v1.ConditionTrue,
|
||||
Reason: "ReadyForNewScale",
|
||||
}, autoscalingv2.HorizontalPodAutoscalerCondition{
|
||||
Type: autoscalingv2.AbleToScale,
|
||||
Status: v1.ConditionFalse,
|
||||
Reason: "BackoffDownscale",
|
||||
Status: v1.ConditionTrue,
|
||||
Reason: "ScaleDownStabilized",
|
||||
}),
|
||||
recommendations: []timestampedRecommendation{
|
||||
{10, time.Now().Add(-10 * time.Minute)},
|
||||
{4, time.Now().Add(-1 * time.Minute)},
|
||||
},
|
||||
}
|
||||
tc.runTest(t)
|
||||
}
|
||||
@ -2278,7 +2287,7 @@ func TestAvoidUncessaryUpdates(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if err := controller.reconcileAutoscaler(&initialHPAs.Items[0]); err != nil {
|
||||
if err := controller.reconcileAutoscaler(&initialHPAs.Items[0], ""); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
@ -2353,4 +2362,85 @@ func TestConvertDesiredReplicasWithRules(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeDesiredReplicas(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
recommendations []timestampedRecommendation
|
||||
prenormalizedDesiredReplicas int32
|
||||
expectedStabilizedReplicas int32
|
||||
expectedLogLength int
|
||||
}{
|
||||
{
|
||||
"empty log",
|
||||
"",
|
||||
[]timestampedRecommendation{},
|
||||
5,
|
||||
5,
|
||||
1,
|
||||
},
|
||||
{
|
||||
"stabilize",
|
||||
"",
|
||||
[]timestampedRecommendation{
|
||||
{4, time.Now().Add(-2 * time.Minute)},
|
||||
{5, time.Now().Add(-1 * time.Minute)},
|
||||
},
|
||||
3,
|
||||
5,
|
||||
3,
|
||||
},
|
||||
{
|
||||
"no stabilize",
|
||||
"",
|
||||
[]timestampedRecommendation{
|
||||
{1, time.Now().Add(-2 * time.Minute)},
|
||||
{2, time.Now().Add(-1 * time.Minute)},
|
||||
},
|
||||
3,
|
||||
3,
|
||||
3,
|
||||
},
|
||||
{
|
||||
"no stabilize - old recommendations",
|
||||
"",
|
||||
[]timestampedRecommendation{
|
||||
{10, time.Now().Add(-10 * time.Minute)},
|
||||
{9, time.Now().Add(-9 * time.Minute)},
|
||||
},
|
||||
3,
|
||||
3,
|
||||
2,
|
||||
},
|
||||
{
|
||||
"stabilize - old recommendations",
|
||||
"",
|
||||
[]timestampedRecommendation{
|
||||
{10, time.Now().Add(-10 * time.Minute)},
|
||||
{4, time.Now().Add(-1 * time.Minute)},
|
||||
{5, time.Now().Add(-2 * time.Minute)},
|
||||
{9, time.Now().Add(-9 * time.Minute)},
|
||||
},
|
||||
3,
|
||||
5,
|
||||
4,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
hc := HorizontalController{
|
||||
downscaleStabilisationWindow: 5 * time.Minute,
|
||||
recommendations: map[string][]timestampedRecommendation{
|
||||
tc.key: tc.recommendations,
|
||||
},
|
||||
}
|
||||
r := hc.stabilizeRecommendation(tc.key, tc.prenormalizedDesiredReplicas)
|
||||
if r != tc.expectedStabilizedReplicas {
|
||||
t.Errorf("[%s] got %d stabilized replicas, expected %d", tc.name, r, tc.expectedStabilizedReplicas)
|
||||
}
|
||||
if len(hc.recommendations[tc.key]) != tc.expectedLogLength {
|
||||
t.Errorf("[%s] after stabilization recommendations log has %d entries, expected %d", tc.name, len(hc.recommendations[tc.key]), tc.expectedLogLength)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: add more tests
|
||||
|
||||
@ -488,7 +488,7 @@ func (tc *legacyTestCase) runTest(t *testing.T) {
|
||||
replicaCalc := NewReplicaCalculator(metricsClient, testClient.Core(), defaultTestingTolerance, defaultTestingCpuInitializationPeriod, defaultTestingDelayOfInitialReadinessStatus)
|
||||
|
||||
informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
|
||||
defaultDownscaleForbiddenWindow := 5 * time.Minute
|
||||
defaultDownscaleStabilisationWindow := 5 * time.Minute
|
||||
|
||||
hpaController := NewHorizontalController(
|
||||
eventClient.Core(),
|
||||
@ -498,7 +498,7 @@ func (tc *legacyTestCase) runTest(t *testing.T) {
|
||||
replicaCalc,
|
||||
informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
|
||||
controller.NoResyncPeriodFunc(),
|
||||
defaultDownscaleForbiddenWindow,
|
||||
defaultDownscaleStabilisationWindow,
|
||||
)
|
||||
hpaController.hpaListerSynced = alwaysReady
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user