github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-30 15:05:27 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #30695 from krancour/manage-elb-attributes

Browse Source 
Automatic merge from submit-queue

AWS: More ELB attributes via service annotations

Replaces #25015 and addresses all of @justinsb's feedback therein. This is a new PR because I was unable to reopen #25015 to amend it.

I noticed recently that there is existing (but undocumented) precedent for the AWS cloud provider to manage ELB-specifc load balancer configuration based on service annotations.  In particular, one can _already_ designate an ELB as "internal" or enable PROXY protocol.

This PR extends this capability to the management of ELB attributes, which includes the following items:
* Access logs:
    * Enabled / disabled
    * Emit interval
    * S3 bucket name
    * S3 bucket prefix
* Connection draining:
    * Enabled / disabled
    * Timeout
* Connection:
    * Idle timeout
* Cross-zone load balancing:
    * Enabled / disabled

Some of these are possibly more useful than others.  Use cases that immediately come to mind:

* Enabling cross-zone load balancing is potentially useful for "Ubernetes Light," or anyone otherwise attempting to spread worker nodes around multiple AZs.
* Increasing idle timeout is useful for the benefit of anyone dealing with long-running requests. An example I personally care about would be git pushes to Deis' builder component.
This commit is contained in:
Kubernetes Submit Queue 2016-08-22 10:24:12 -07:00 committed by GitHub
commit bfafb6f425
3 changed files with 171 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
pkg/cloudprovider/providers/aws/aws.go
View File

@ -84,6 +84,38 @@ const ServiceAnnotationLoadBalancerInternal = "service.beta.kubernetes.io/aws-lo
// certain backends.
const ServiceAnnotationLoadBalancerProxyProtocol = "service.beta.kubernetes.io/aws-load-balancer-proxy-protocol"
// ServiceAnnotationLoadBalancerAccessLogEmitInterval is the annotation used to
// specify access log emit interval.
const ServiceAnnotationLoadBalancerAccessLogEmitInterval = "service.beta.kubernetes.io/aws-load-balancer-access-log-emit-interval"
// ServiceAnnotationLoadBalancerAccessLogEnabled is the annotation used on the
// service to enable or disable access logs.
const ServiceAnnotationLoadBalancerAccessLogEnabled = "service.beta.kubernetes.io/aws-load-balancer-access-log-enabled"
// ServiceAnnotationLoadBalancerAccessLogS3BucketName is the annotation used to
// specify access log s3 bucket name.
const ServiceAnnotationLoadBalancerAccessLogS3BucketName = "service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-name"
// ServiceAnnotationLoadBalancerAccessLogS3BucketPrefix is the annotation used
// to specify access log s3 bucket prefix.
const ServiceAnnotationLoadBalancerAccessLogS3BucketPrefix = "service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-prefix"
// ServiceAnnotationLoadBalancerConnectionDrainingEnabled is the annnotation
// used on the service to enable or disable connection draining.
const ServiceAnnotationLoadBalancerConnectionDrainingEnabled = "service.beta.kubernetes.io/aws-load-balancer-connection-draining-enabled"
// ServiceAnnotationLoadBalancerConnectionDrainingTimeout is the annotation
// used on the service to specify a connection draining timeout.
const ServiceAnnotationLoadBalancerConnectionDrainingTimeout = "service.beta.kubernetes.io/aws-load-balancer-connection-draining-timeout"
// ServiceAnnotationLoadBalancerConnectionIdleTimeout is the annotation used
// on the service to specify the idle connection timeout.
const ServiceAnnotationLoadBalancerConnectionIdleTimeout = "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout"
// ServiceAnnotationLoadBalancerCrossZoneLoadBalancingEnabled is the annotation
// used on the service to enable or disable cross-zone load balancing.
const ServiceAnnotationLoadBalancerCrossZoneLoadBalancingEnabled = "service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled"
// ServiceAnnotationLoadBalancerCertificate is the annotation used on the
// service to request a secure listener. Value is a valid certificate ARN.
// For more, see http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/elb-listener-config.html
@ -195,6 +227,9 @@ type ELB interface {
ApplySecurityGroupsToLoadBalancer(*elb.ApplySecurityGroupsToLoadBalancerInput) (*elb.ApplySecurityGroupsToLoadBalancerOutput, error)
ConfigureHealthCheck(*elb.ConfigureHealthCheckInput) (*elb.ConfigureHealthCheckOutput, error)
DescribeLoadBalancerAttributes(*elb.DescribeLoadBalancerAttributesInput) (*elb.DescribeLoadBalancerAttributesOutput, error)
ModifyLoadBalancerAttributes(*elb.ModifyLoadBalancerAttributesInput) (*elb.ModifyLoadBalancerAttributesOutput, error)
}
// ASG is a simple pass-through of the Autoscaling client interface, which
@ -2435,6 +2470,104 @@ func (c *Cloud) EnsureLoadBalancer(clusterName string, apiService *api.Service,
proxyProtocol = true
}
// Some load balancer attributes are required, so defaults are set. These can be overridden by annotations.
loadBalancerAttributes := &elb.LoadBalancerAttributes{
AccessLog: &elb.AccessLog{Enabled: aws.Bool(false)},
ConnectionDraining: &elb.ConnectionDraining{Enabled: aws.Bool(false)},
ConnectionSettings: &elb.ConnectionSettings{IdleTimeout: aws.Int64(60)},
CrossZoneLoadBalancing: &elb.CrossZoneLoadBalancing{Enabled: aws.Bool(false)},
}
// Determine if an access log emit interval has been specified
accessLogEmitIntervalAnnotation := annotations[ServiceAnnotationLoadBalancerAccessLogEmitInterval]
if accessLogEmitIntervalAnnotation != "" {
accessLogEmitInterval, err := strconv.ParseInt(accessLogEmitIntervalAnnotation, 10, 64)
if err != nil {
return nil, fmt.Errorf("error parsing service annotation: %s=%s",
ServiceAnnotationLoadBalancerAccessLogEmitInterval,
accessLogEmitIntervalAnnotation,
)
}
loadBalancerAttributes.AccessLog.EmitInterval = &accessLogEmitInterval
}
// Determine if access log enabled/disabled has been specified
accessLogEnabledAnnotation := annotations[ServiceAnnotationLoadBalancerAccessLogEnabled]
if accessLogEnabledAnnotation != "" {
accessLogEnabled, err := strconv.ParseBool(accessLogEnabledAnnotation)
if err != nil {
return nil, fmt.Errorf("error parsing service annotation: %s=%s",
ServiceAnnotationLoadBalancerAccessLogEnabled,
accessLogEnabledAnnotation,
)
}
loadBalancerAttributes.AccessLog.Enabled = &accessLogEnabled
}
// Determine if access log s3 bucket name has been specified
accessLogS3BucketNameAnnotation := annotations[ServiceAnnotationLoadBalancerAccessLogS3BucketName]
if accessLogS3BucketNameAnnotation != "" {
loadBalancerAttributes.AccessLog.S3BucketName = &accessLogS3BucketNameAnnotation
}
// Determine if access log s3 bucket prefix has been specified
accessLogS3BucketPrefixAnnotation := annotations[ServiceAnnotationLoadBalancerAccessLogS3BucketPrefix]
if accessLogS3BucketPrefixAnnotation != "" {
loadBalancerAttributes.AccessLog.S3BucketPrefix = &accessLogS3BucketPrefixAnnotation
}
// Determine if connection draining enabled/disabled has been specified
connectionDrainingEnabledAnnotation := annotations[ServiceAnnotationLoadBalancerConnectionDrainingEnabled]
if connectionDrainingEnabledAnnotation != "" {
connectionDrainingEnabled, err := strconv.ParseBool(connectionDrainingEnabledAnnotation)
if err != nil {
return nil, fmt.Errorf("error parsing service annotation: %s=%s",
ServiceAnnotationLoadBalancerConnectionDrainingEnabled,
connectionDrainingEnabledAnnotation,
)
}
loadBalancerAttributes.ConnectionDraining.Enabled = &connectionDrainingEnabled
}
// Determine if connection draining timeout has been specified
connectionDrainingTimeoutAnnotation := annotations[ServiceAnnotationLoadBalancerConnectionDrainingTimeout]
if connectionDrainingTimeoutAnnotation != "" {
connectionDrainingTimeout, err := strconv.ParseInt(connectionDrainingTimeoutAnnotation, 10, 64)
if err != nil {
return nil, fmt.Errorf("error parsing service annotation: %s=%s",
ServiceAnnotationLoadBalancerConnectionDrainingTimeout,
connectionDrainingTimeoutAnnotation,
)
}
loadBalancerAttributes.ConnectionDraining.Timeout = &connectionDrainingTimeout
}
// Determine if connection idle timeout has been specified
connectionIdleTimeoutAnnotation := annotations[ServiceAnnotationLoadBalancerConnectionIdleTimeout]
if connectionIdleTimeoutAnnotation != "" {
connectionIdleTimeout, err := strconv.ParseInt(connectionIdleTimeoutAnnotation, 10, 64)
if err != nil {
return nil, fmt.Errorf("error parsing service annotation: %s=%s",
ServiceAnnotationLoadBalancerConnectionIdleTimeout,
connectionIdleTimeoutAnnotation,
)
}
loadBalancerAttributes.ConnectionSettings.IdleTimeout = &connectionIdleTimeout
}
// Determine if cross zone load balancing enabled/disabled has been specified
crossZoneLoadBalancingEnabledAnnotation := annotations[ServiceAnnotationLoadBalancerCrossZoneLoadBalancingEnabled]
if crossZoneLoadBalancingEnabledAnnotation != "" {
crossZoneLoadBalancingEnabled, err := strconv.ParseBool(crossZoneLoadBalancingEnabledAnnotation)
if err != nil {
return nil, fmt.Errorf("error parsing service annotation: %s=%s",
ServiceAnnotationLoadBalancerCrossZoneLoadBalancingEnabled,
crossZoneLoadBalancingEnabledAnnotation,
)
}
loadBalancerAttributes.CrossZoneLoadBalancing.Enabled = &crossZoneLoadBalancingEnabled
}
// Find the subnets that the ELB will live in
subnetIDs, err := c.findELBSubnets(internalELB)
if err != nil {
@ -2508,6 +2641,7 @@ func (c *Cloud) EnsureLoadBalancer(clusterName string, apiService *api.Service,
securityGroupIDs,
internalELB,
proxyProtocol,
loadBalancerAttributes,
)
if err != nil {
return nil, err

30
pkg/cloudprovider/providers/aws/aws_loadbalancer.go
View File

@ -18,6 +18,7 @@ package aws
import (
"fmt"
"reflect"
"strconv"
"github.com/aws/aws-sdk-go/aws"
@ -30,7 +31,7 @@ import (
const ProxyProtocolPolicyName = "k8s-proxyprotocol-enabled"
func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBalancerName string, listeners []*elb.Listener, subnetIDs []string, securityGroupIDs []string, internalELB, proxyProtocol bool) (*elb.LoadBalancerDescription, error) {
func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBalancerName string, listeners []*elb.Listener, subnetIDs []string, securityGroupIDs []string, internalELB, proxyProtocol bool, loadBalancerAttributes *elb.LoadBalancerAttributes) (*elb.LoadBalancerDescription, error) {
loadBalancer, err := c.describeLoadBalancer(loadBalancerName)
if err != nil {
return nil, err
@ -276,6 +277,33 @@ func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBala
}
}
// Whether the ELB was new or existing, sync attributes regardless. This accounts for things
// that cannot be specified at the time of creation and can only be modified after the fact,
// e.g. idle connection timeout.
{
describeAttributesRequest := &elb.DescribeLoadBalancerAttributesInput{}
describeAttributesRequest.LoadBalancerName = aws.String(loadBalancerName)
describeAttributesOutput, err := c.elb.DescribeLoadBalancerAttributes(describeAttributesRequest)
if err != nil {
glog.Warning("Unable to retrieve load balancer attributes during attribute sync")
return nil, err
}
foundAttributes := &describeAttributesOutput.LoadBalancerAttributes
// Update attributes if they're dirty
if !reflect.DeepEqual(loadBalancerAttributes, foundAttributes) {
modifyAttributesRequest := &elb.ModifyLoadBalancerAttributesInput{}
modifyAttributesRequest.LoadBalancerName = aws.String(loadBalancerName)
modifyAttributesRequest.LoadBalancerAttributes = loadBalancerAttributes
_, err = c.elb.ModifyLoadBalancerAttributes(modifyAttributesRequest)
if err != nil {
return nil, fmt.Errorf("Unable to update load balancer attributes during attribute sync: %v", err)
}
dirty = true
}
}
if dirty {
loadBalancer, err = c.describeLoadBalancer(loadBalancerName)
if err != nil {

8
pkg/cloudprovider/providers/aws/aws_test.go
View File

@ -497,6 +497,14 @@ func (elb *FakeELB) SetLoadBalancerPoliciesForBackendServer(*elb.SetLoadBalancer
panic("Not implemented")
}
func (elb *FakeELB) DescribeLoadBalancerAttributes(*elb.DescribeLoadBalancerAttributesInput) (*elb.DescribeLoadBalancerAttributesOutput, error) {
panic("Not implemented")
}
func (elb *FakeELB) ModifyLoadBalancerAttributes(*elb.ModifyLoadBalancerAttributesInput) (*elb.ModifyLoadBalancerAttributesOutput, error) {
panic("Not implemented")
}
type FakeASG struct {
aws *FakeAWSServices
}