diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go index 4bd6922147f..698596eef34 100644 --- a/pkg/cloudprovider/providers/aws/aws.go +++ b/pkg/cloudprovider/providers/aws/aws.go @@ -84,6 +84,38 @@ const ServiceAnnotationLoadBalancerInternal = "service.beta.kubernetes.io/aws-lo // certain backends. const ServiceAnnotationLoadBalancerProxyProtocol = "service.beta.kubernetes.io/aws-load-balancer-proxy-protocol" +// ServiceAnnotationLoadBalancerAccessLogEmitInterval is the annotation used to +// specify access log emit interval. +const ServiceAnnotationLoadBalancerAccessLogEmitInterval = "service.beta.kubernetes.io/aws-load-balancer-access-log-emit-interval" + +// ServiceAnnotationLoadBalancerAccessLogEnabled is the annotation used on the +// service to enable or disable access logs. +const ServiceAnnotationLoadBalancerAccessLogEnabled = "service.beta.kubernetes.io/aws-load-balancer-access-log-enabled" + +// ServiceAnnotationLoadBalancerAccessLogS3BucketName is the annotation used to +// specify access log s3 bucket name. +const ServiceAnnotationLoadBalancerAccessLogS3BucketName = "service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-name" + +// ServiceAnnotationLoadBalancerAccessLogS3BucketPrefix is the annotation used +// to specify access log s3 bucket prefix. +const ServiceAnnotationLoadBalancerAccessLogS3BucketPrefix = "service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-prefix" + +// ServiceAnnotationLoadBalancerConnectionDrainingEnabled is the annnotation +// used on the service to enable or disable connection draining. +const ServiceAnnotationLoadBalancerConnectionDrainingEnabled = "service.beta.kubernetes.io/aws-load-balancer-connection-draining-enabled" + +// ServiceAnnotationLoadBalancerConnectionDrainingTimeout is the annotation +// used on the service to specify a connection draining timeout. +const ServiceAnnotationLoadBalancerConnectionDrainingTimeout = "service.beta.kubernetes.io/aws-load-balancer-connection-draining-timeout" + +// ServiceAnnotationLoadBalancerConnectionIdleTimeout is the annotation used +// on the service to specify the idle connection timeout. +const ServiceAnnotationLoadBalancerConnectionIdleTimeout = "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout" + +// ServiceAnnotationLoadBalancerCrossZoneLoadBalancingEnabled is the annotation +// used on the service to enable or disable cross-zone load balancing. +const ServiceAnnotationLoadBalancerCrossZoneLoadBalancingEnabled = "service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled" + // ServiceAnnotationLoadBalancerCertificate is the annotation used on the // service to request a secure listener. Value is a valid certificate ARN. // For more, see http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/elb-listener-config.html @@ -195,6 +227,9 @@ type ELB interface { ApplySecurityGroupsToLoadBalancer(*elb.ApplySecurityGroupsToLoadBalancerInput) (*elb.ApplySecurityGroupsToLoadBalancerOutput, error) ConfigureHealthCheck(*elb.ConfigureHealthCheckInput) (*elb.ConfigureHealthCheckOutput, error) + + DescribeLoadBalancerAttributes(*elb.DescribeLoadBalancerAttributesInput) (*elb.DescribeLoadBalancerAttributesOutput, error) + ModifyLoadBalancerAttributes(*elb.ModifyLoadBalancerAttributesInput) (*elb.ModifyLoadBalancerAttributesOutput, error) } // ASG is a simple pass-through of the Autoscaling client interface, which @@ -2435,6 +2470,104 @@ func (c *Cloud) EnsureLoadBalancer(clusterName string, apiService *api.Service, proxyProtocol = true } + // Some load balancer attributes are required, so defaults are set. These can be overridden by annotations. + loadBalancerAttributes := &elb.LoadBalancerAttributes{ + AccessLog: &elb.AccessLog{Enabled: aws.Bool(false)}, + ConnectionDraining: &elb.ConnectionDraining{Enabled: aws.Bool(false)}, + ConnectionSettings: &elb.ConnectionSettings{IdleTimeout: aws.Int64(60)}, + CrossZoneLoadBalancing: &elb.CrossZoneLoadBalancing{Enabled: aws.Bool(false)}, + } + + // Determine if an access log emit interval has been specified + accessLogEmitIntervalAnnotation := annotations[ServiceAnnotationLoadBalancerAccessLogEmitInterval] + if accessLogEmitIntervalAnnotation != "" { + accessLogEmitInterval, err := strconv.ParseInt(accessLogEmitIntervalAnnotation, 10, 64) + if err != nil { + return nil, fmt.Errorf("error parsing service annotation: %s=%s", + ServiceAnnotationLoadBalancerAccessLogEmitInterval, + accessLogEmitIntervalAnnotation, + ) + } + loadBalancerAttributes.AccessLog.EmitInterval = &accessLogEmitInterval + } + + // Determine if access log enabled/disabled has been specified + accessLogEnabledAnnotation := annotations[ServiceAnnotationLoadBalancerAccessLogEnabled] + if accessLogEnabledAnnotation != "" { + accessLogEnabled, err := strconv.ParseBool(accessLogEnabledAnnotation) + if err != nil { + return nil, fmt.Errorf("error parsing service annotation: %s=%s", + ServiceAnnotationLoadBalancerAccessLogEnabled, + accessLogEnabledAnnotation, + ) + } + loadBalancerAttributes.AccessLog.Enabled = &accessLogEnabled + } + + // Determine if access log s3 bucket name has been specified + accessLogS3BucketNameAnnotation := annotations[ServiceAnnotationLoadBalancerAccessLogS3BucketName] + if accessLogS3BucketNameAnnotation != "" { + loadBalancerAttributes.AccessLog.S3BucketName = &accessLogS3BucketNameAnnotation + } + + // Determine if access log s3 bucket prefix has been specified + accessLogS3BucketPrefixAnnotation := annotations[ServiceAnnotationLoadBalancerAccessLogS3BucketPrefix] + if accessLogS3BucketPrefixAnnotation != "" { + loadBalancerAttributes.AccessLog.S3BucketPrefix = &accessLogS3BucketPrefixAnnotation + } + + // Determine if connection draining enabled/disabled has been specified + connectionDrainingEnabledAnnotation := annotations[ServiceAnnotationLoadBalancerConnectionDrainingEnabled] + if connectionDrainingEnabledAnnotation != "" { + connectionDrainingEnabled, err := strconv.ParseBool(connectionDrainingEnabledAnnotation) + if err != nil { + return nil, fmt.Errorf("error parsing service annotation: %s=%s", + ServiceAnnotationLoadBalancerConnectionDrainingEnabled, + connectionDrainingEnabledAnnotation, + ) + } + loadBalancerAttributes.ConnectionDraining.Enabled = &connectionDrainingEnabled + } + + // Determine if connection draining timeout has been specified + connectionDrainingTimeoutAnnotation := annotations[ServiceAnnotationLoadBalancerConnectionDrainingTimeout] + if connectionDrainingTimeoutAnnotation != "" { + connectionDrainingTimeout, err := strconv.ParseInt(connectionDrainingTimeoutAnnotation, 10, 64) + if err != nil { + return nil, fmt.Errorf("error parsing service annotation: %s=%s", + ServiceAnnotationLoadBalancerConnectionDrainingTimeout, + connectionDrainingTimeoutAnnotation, + ) + } + loadBalancerAttributes.ConnectionDraining.Timeout = &connectionDrainingTimeout + } + + // Determine if connection idle timeout has been specified + connectionIdleTimeoutAnnotation := annotations[ServiceAnnotationLoadBalancerConnectionIdleTimeout] + if connectionIdleTimeoutAnnotation != "" { + connectionIdleTimeout, err := strconv.ParseInt(connectionIdleTimeoutAnnotation, 10, 64) + if err != nil { + return nil, fmt.Errorf("error parsing service annotation: %s=%s", + ServiceAnnotationLoadBalancerConnectionIdleTimeout, + connectionIdleTimeoutAnnotation, + ) + } + loadBalancerAttributes.ConnectionSettings.IdleTimeout = &connectionIdleTimeout + } + + // Determine if cross zone load balancing enabled/disabled has been specified + crossZoneLoadBalancingEnabledAnnotation := annotations[ServiceAnnotationLoadBalancerCrossZoneLoadBalancingEnabled] + if crossZoneLoadBalancingEnabledAnnotation != "" { + crossZoneLoadBalancingEnabled, err := strconv.ParseBool(crossZoneLoadBalancingEnabledAnnotation) + if err != nil { + return nil, fmt.Errorf("error parsing service annotation: %s=%s", + ServiceAnnotationLoadBalancerCrossZoneLoadBalancingEnabled, + crossZoneLoadBalancingEnabledAnnotation, + ) + } + loadBalancerAttributes.CrossZoneLoadBalancing.Enabled = &crossZoneLoadBalancingEnabled + } + // Find the subnets that the ELB will live in subnetIDs, err := c.findELBSubnets(internalELB) if err != nil { @@ -2508,6 +2641,7 @@ func (c *Cloud) EnsureLoadBalancer(clusterName string, apiService *api.Service, securityGroupIDs, internalELB, proxyProtocol, + loadBalancerAttributes, ) if err != nil { return nil, err diff --git a/pkg/cloudprovider/providers/aws/aws_loadbalancer.go b/pkg/cloudprovider/providers/aws/aws_loadbalancer.go index 7b02650427c..e3e5c773620 100644 --- a/pkg/cloudprovider/providers/aws/aws_loadbalancer.go +++ b/pkg/cloudprovider/providers/aws/aws_loadbalancer.go @@ -18,6 +18,7 @@ package aws import ( "fmt" + "reflect" "strconv" "github.com/aws/aws-sdk-go/aws" @@ -30,7 +31,7 @@ import ( const ProxyProtocolPolicyName = "k8s-proxyprotocol-enabled" -func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBalancerName string, listeners []*elb.Listener, subnetIDs []string, securityGroupIDs []string, internalELB, proxyProtocol bool) (*elb.LoadBalancerDescription, error) { +func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBalancerName string, listeners []*elb.Listener, subnetIDs []string, securityGroupIDs []string, internalELB, proxyProtocol bool, loadBalancerAttributes *elb.LoadBalancerAttributes) (*elb.LoadBalancerDescription, error) { loadBalancer, err := c.describeLoadBalancer(loadBalancerName) if err != nil { return nil, err @@ -276,6 +277,33 @@ func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBala } } + // Whether the ELB was new or existing, sync attributes regardless. This accounts for things + // that cannot be specified at the time of creation and can only be modified after the fact, + // e.g. idle connection timeout. + { + describeAttributesRequest := &elb.DescribeLoadBalancerAttributesInput{} + describeAttributesRequest.LoadBalancerName = aws.String(loadBalancerName) + describeAttributesOutput, err := c.elb.DescribeLoadBalancerAttributes(describeAttributesRequest) + if err != nil { + glog.Warning("Unable to retrieve load balancer attributes during attribute sync") + return nil, err + } + + foundAttributes := &describeAttributesOutput.LoadBalancerAttributes + + // Update attributes if they're dirty + if !reflect.DeepEqual(loadBalancerAttributes, foundAttributes) { + modifyAttributesRequest := &elb.ModifyLoadBalancerAttributesInput{} + modifyAttributesRequest.LoadBalancerName = aws.String(loadBalancerName) + modifyAttributesRequest.LoadBalancerAttributes = loadBalancerAttributes + _, err = c.elb.ModifyLoadBalancerAttributes(modifyAttributesRequest) + if err != nil { + return nil, fmt.Errorf("Unable to update load balancer attributes during attribute sync: %v", err) + } + dirty = true + } + } + if dirty { loadBalancer, err = c.describeLoadBalancer(loadBalancerName) if err != nil { diff --git a/pkg/cloudprovider/providers/aws/aws_test.go b/pkg/cloudprovider/providers/aws/aws_test.go index 91d5fb267bd..b9cd7b8600a 100644 --- a/pkg/cloudprovider/providers/aws/aws_test.go +++ b/pkg/cloudprovider/providers/aws/aws_test.go @@ -497,6 +497,14 @@ func (elb *FakeELB) SetLoadBalancerPoliciesForBackendServer(*elb.SetLoadBalancer panic("Not implemented") } +func (elb *FakeELB) DescribeLoadBalancerAttributes(*elb.DescribeLoadBalancerAttributesInput) (*elb.DescribeLoadBalancerAttributesOutput, error) { + panic("Not implemented") +} + +func (elb *FakeELB) ModifyLoadBalancerAttributes(*elb.ModifyLoadBalancerAttributesInput) (*elb.ModifyLoadBalancerAttributesOutput, error) { + panic("Not implemented") +} + type FakeASG struct { aws *FakeAWSServices }