From 97689c326da2a9170e25ed5e1bdb8f3e2bc50533 Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Thu, 21 Jan 2016 19:31:21 -0800 Subject: [PATCH] Reduce healthy threshold and check interval for Amazon ELBs According to AWS, the ELB healthy threshold is "Number of consecutive health check successes before declaring an EC2 instance healthy." It has an unusual interaction with Kubernetes, since all nodes will enter either an unhealthy state or a healthy state together depending on the service's healthiness as a whole. We have observed that if our service goes down for the unhealthy threshold (which is 2 checks at 30 second intervals = 60 seconds), then the ELB will stop serving traffic to all nodes in the cluster, and will wait for the healthy threshold (currently 10 * 30 = 300 seconds) AFTER the service is restored to add back the cluster nodes, meaning it remains unreachable for an extra 300 seconds. With the new settings, the ELB will continue to timeout dead nodes after 60 seconds, but will restore healthy nodes after 20 seconds. The minimum value for healthyThreshold is 2, and the minimum value for interval is 5 seconds. I went for 10 seconds instead of the minimum sort of arbitrarily because I was not sure how much this value may affect the scalability of clusters in EC2, as it does put some extra load on the kube-proxy. --- pkg/cloudprovider/providers/aws/aws_loadbalancer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/cloudprovider/providers/aws/aws_loadbalancer.go b/pkg/cloudprovider/providers/aws/aws_loadbalancer.go index ae83dc97337..7f7b5dd3e22 100644 --- a/pkg/cloudprovider/providers/aws/aws_loadbalancer.go +++ b/pkg/cloudprovider/providers/aws/aws_loadbalancer.go @@ -195,10 +195,10 @@ func (s *AWSCloud) ensureLoadBalancerHealthCheck(loadBalancer *elb.LoadBalancerD actual := loadBalancer.HealthCheck // Default AWS settings - expectedHealthyThreshold := int64(10) - expectedUnhealthyThreshold := int64(2) + expectedHealthyThreshold := int64(2) + expectedUnhealthyThreshold := int64(6) expectedTimeout := int64(5) - expectedInterval := int64(30) + expectedInterval := int64(10) // We only a TCP health-check on the first port expectedTarget := ""