Merge pull request #23495 from Clarifai/elb

Automatic merge from submit-queue

AWS: SSL support for ELB listeners through annotations

In the API, ports have only either TCP or UDP as their protocols, but ELB distinguishes HTTPS->HTTP[S]? from SSL->(SSL|TCP). 

Per #24978, this is implemented through two separate annotations:

`service.beta.kubernetes.io/aws-load-balancer-ssl-cert=arn:aws:acm:us-east-1:123456789012:certificate/12345678-1234-1234-1234-123456789012`
`service.beta.kubernetes.io/aws-load-balancer-backend-protocol=(https|http|ssl|tcp)`

Mixing plain-text and encrypted listeners will be in a separate PR, implementing #24978's `aws-load-balancer-ssl-ports=LIST`

pkg/cloudprovider/providers/aws/aws.go

@@ -69,6 +69,25 @@ const TagNameSubnetPublicELB = "kubernetes.io/role/elb"
 // This lets us define more advanced semantics in future.
 const ServiceAnnotationLoadBalancerInternal = "service.beta.kubernetes.io/aws-load-balancer-internal"
 
+// Service annotation requesting a secure listener. Value is a valid certificate ARN.
+// For more, see http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/elb-listener-config.html
+// CertARN is an IAM or CM certificate ARN, e.g. arn:aws:acm:us-east-1:123456789012:certificate/12345678-1234-1234-1234-123456789012
+const ServiceAnnotationLoadBalancerCertificate = "service.beta.kubernetes.io/aws-load-balancer-ssl-cert"
+
+// Service annotation specifying the protocol spoken by the backend (pod) behind a secure listener.
+// Only inspected when `aws-load-balancer-ssl-cert` is used.
+// If `http` (default) or `https`, an HTTPS listener that terminates the connection and parses headers is created.
+// If set to `ssl` or `tcp`, a "raw" SSL listener is used.
+const ServiceAnnotationLoadBalancerBEProtocol = "service.beta.kubernetes.io/aws-load-balancer-backend-protocol"
+
+// Maps from backend protocol to ELB protocol
+var backendProtocolMapping = map[string]string{
+	"https": "https",
+	"http":  "https",
+	"ssl":   "ssl",
+	"tcp":   "ssl",
+}
+
 // We sometimes read to see if something exists; then try to create it if we didn't find it
 // This can fail once in a consistent system if done in parallel
 // In an eventually consistent system, it could fail unboundedly
@@ -2099,6 +2118,37 @@ func isSubnetPublic(rt []*ec2.RouteTable, subnetID string) (bool, error) {
 	return false, nil
 }
 
+// buildListener creates a new listener from the given port, adding an SSL certificate
+// if indicated by the appropriate annotations.
+func buildListener(port api.ServicePort, annotations map[string]string) (*elb.Listener, error) {
+	loadBalancerPort := int64(port.Port)
+	instancePort := int64(port.NodePort)
+	protocol := strings.ToLower(string(port.Protocol))
+	instanceProtocol := protocol
+
+	listener := &elb.Listener{}
+	listener.InstancePort = &instancePort
+	listener.LoadBalancerPort = &loadBalancerPort
+	certID := annotations[ServiceAnnotationLoadBalancerCertificate]
+	if certID != "" {
+		instanceProtocol = annotations[ServiceAnnotationLoadBalancerBEProtocol]
+		if instanceProtocol == "" {
+			protocol = "ssl"
+			instanceProtocol = "tcp"
+		} else {
+			protocol = backendProtocolMapping[instanceProtocol]
+			if protocol == "" {
+				return nil, fmt.Errorf("Invalid backend protocol %s for %s in %s", instanceProtocol, certID, ServiceAnnotationLoadBalancerBEProtocol)
+			}
+		}
+		listener.SSLCertificateId = &certID
+	}
+	listener.Protocol = &protocol
+	listener.InstanceProtocol = &instanceProtocol
+
+	return listener, nil
+}
+
 // EnsureLoadBalancer implements LoadBalancer.EnsureLoadBalancer
 func (s *AWSCloud) EnsureLoadBalancer(apiService *api.Service, hosts []string, annotations map[string]string) (*api.LoadBalancerStatus, error) {
 	glog.V(2).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)",
@@ -2113,10 +2163,21 @@ func (s *AWSCloud) EnsureLoadBalancer(apiService *api.Service, hosts []string, a
 		return nil, fmt.Errorf("requested load balancer with no ports")
 	}
 
+	// Figure out what mappings we want on the load balancer
+	listeners := []*elb.Listener{}
 	for _, port := range apiService.Spec.Ports {
 		if port.Protocol != api.ProtocolTCP {
 			return nil, fmt.Errorf("Only TCP LoadBalancer is supported for AWS ELB")
 		}
+		if port.NodePort == 0 {
+			glog.Errorf("Ignoring port without NodePort defined: %v", port)
+			continue
+		}
+		listener, err := buildListener(port, annotations)
+		if err != nil {
+			return nil, err
+		}
+		listeners = append(listeners, listener)
 	}
 
 	if apiService.Spec.LoadBalancerIP != "" {
@@ -2198,26 +2259,6 @@ func (s *AWSCloud) EnsureLoadBalancer(apiService *api.Service, hosts []string, a
 	}
 	securityGroupIDs := []string{securityGroupID}
 
-	// Figure out what mappings we want on the load balancer
-	listeners := []*elb.Listener{}
-	for _, port := range apiService.Spec.Ports {
-		if port.NodePort == 0 {
-			glog.Errorf("Ignoring port without NodePort defined: %v", port)
-			continue
-		}
-		instancePort := int64(port.NodePort)
-		loadBalancerPort := int64(port.Port)
-		protocol := strings.ToLower(string(port.Protocol))
-
-		listener := &elb.Listener{}
-		listener.InstancePort = &instancePort
-		listener.LoadBalancerPort = &loadBalancerPort
-		listener.Protocol = &protocol
-		listener.InstanceProtocol = &protocol
-
-		listeners = append(listeners, listener)
-	}
-
 	// Build the load balancer itself
 	loadBalancer, err := s.ensureLoadBalancer(serviceName, loadBalancerName, listeners, subnetIDs, securityGroupIDs, internalELB)
 	if err != nil {

pkg/cloudprovider/providers/aws/aws_test.go

@@ -1199,3 +1199,106 @@ func TestDescribeLoadBalancerOnEnsure(t *testing.T) {
 
 	c.EnsureLoadBalancer(&api.Service{ObjectMeta: api.ObjectMeta{Name: "myservice", UID: "id"}}, []string{}, map[string]string{})
 }
+
+func TestBuildListener(t *testing.T) {
+	tests := []struct {
+		name string
+
+		lbPort                    int64
+		instancePort              int64
+		backendProtocolAnnotation string
+		certAnnotation            string
+
+		expectError      bool
+		lbProtocol       string
+		instanceProtocol string
+		certID           string
+	}{
+		{
+			"No cert or BE protocol annotation, passthrough",
+			80, 7999, "", "",
+			false, "tcp", "tcp", "",
+		},
+		{
+			"Cert annotation without BE protocol specified, SSL->TCP",
+			80, 8000, "", "cert",
+			false, "ssl", "tcp", "cert",
+		},
+		{
+			"BE protocol without cert annotation, passthrough",
+			443, 8001, "https", "",
+			false, "tcp", "tcp", "",
+		},
+		{
+			"Invalid cert annotation, bogus backend protocol",
+			443, 8002, "bacon", "foo",
+			true, "tcp", "tcp", "cert",
+		},
+		{
+			"Invalid cert annotation, protocol followed by equal sign",
+			443, 8003, "http=", "=",
+			true, "tcp", "tcp", "cert",
+		},
+		{
+			"HTTPS->HTTPS",
+			443, 8004, "https", "cert",
+			false, "https", "https", "cert",
+		},
+		{
+			"HTTPS->HTTP",
+			443, 8005, "http", "cert",
+			false, "https", "http", "cert",
+		},
+		{
+			"SSL->SSL",
+			443, 8006, "ssl", "cert",
+			false, "ssl", "ssl", "cert",
+		},
+		{
+			"SSL->TCP",
+			443, 8007, "tcp", "cert",
+			false, "ssl", "tcp", "cert",
+		},
+	}
+
+	for _, test := range tests {
+		t.Logf("Running test case %s", test.name)
+		annotations := make(map[string]string)
+		if test.backendProtocolAnnotation != "" {
+			annotations[ServiceAnnotationLoadBalancerBEProtocol] = test.backendProtocolAnnotation
+		}
+		if test.certAnnotation != "" {
+			annotations[ServiceAnnotationLoadBalancerCertificate] = test.certAnnotation
+		}
+		l, err := buildListener(api.ServicePort{
+			NodePort: int32(test.instancePort),
+			Port:     int32(test.lbPort),
+			Protocol: api.Protocol("tcp"),
+		}, annotations)
+		if test.expectError {
+			if err == nil {
+				t.Errorf("Should error for case %s", test.name)
+			}
+		} else {
+			if err != nil {
+				t.Errorf("Should succeed for case: %s, got %v", test.name, err)
+			} else {
+				var cert *string
+				if test.certID != "" {
+					cert = &test.certID
+				}
+				expected := &elb.Listener{
+					InstancePort:     &test.instancePort,
+					InstanceProtocol: &test.instanceProtocol,
+					LoadBalancerPort: &test.lbPort,
+					Protocol:         &test.lbProtocol,
+					SSLCertificateId: cert,
+				}
+				if !reflect.DeepEqual(l, expected) {
+					t.Errorf("Incorrect listener (%v vs expected %v) for case: %s",
+						l, expected, test.name)
+				}
+			}
+		}
+	}
+}