Merge pull request #22458 from bprashanth/ing_tls_e2e

Auto commit by PR queue bot

cluster/addons/cluster-loadbalancing/glbc/glbc-controller.yaml

@@ -1,11 +1,11 @@
 apiVersion: v1
 kind: ReplicationController
 metadata:
-  name: l7-lb-controller-v0.5.2
+  name: l7-lb-controller-v0.6.0
   namespace: kube-system
   labels:
     k8s-app: glbc
-    version: v0.5.2
+    version: v0.6.0
     kubernetes.io/cluster-service: "true"
     kubernetes.io/name: "GLBC"
 spec:
@@ -13,12 +13,12 @@ spec:
   replicas: 1
   selector:
     k8s-app: glbc
-    version: v0.5.2
+    version: v0.6.0
   template:
     metadata:
       labels:
         k8s-app: glbc
-        version: v0.5.2
+        version: v0.6.0
         name: glbc
         kubernetes.io/cluster-service: "true"
     spec:
@@ -45,7 +45,7 @@ spec:
           requests:
             cpu: 10m
             memory: 20Mi
-      - image: gcr.io/google_containers/glbc:0.5.2
+      - image: gcr.io/google_containers/glbc:0.6.0
         livenessProbe:
           httpGet:
             path: /healthz

test/e2e/ingress.go

@@ -146,10 +146,11 @@ func ruleByIndex(i int) extensions.IngressRule {
 //	 foo1.bar.com: /foo1
 //	 foo2.bar.com: /foo2
 // }
-func createIngress(c *client.Client, ns string, start, num int) extensions.Ingress {
+func generateIngressSpec(start, num int, ns string) *extensions.Ingress {
+	name := fmt.Sprintf("%v%d", appPrefix, start)
 	ing := extensions.Ingress{
 		ObjectMeta: api.ObjectMeta{
-			Name:      fmt.Sprintf("%v%d", appPrefix, start),
+			Name:      name,
 			Namespace: ns,
 		},
 		Spec: extensions.IngressSpec{
@@ -163,10 +164,15 @@ func createIngress(c *client.Client, ns string, start, num int) extensions.Ingre
 	for i := start; i < start+num; i++ {
 		ing.Spec.Rules = append(ing.Spec.Rules, ruleByIndex(i))
 	}
-	Logf("Creating ingress %v", start)
-	_, err := c.Extensions().Ingress(ns).Create(&ing)
-	Expect(err).NotTo(HaveOccurred())
-	return ing
+	// Create the host for the cert by appending all hosts mentioned in rules.
+	hosts := []string{}
+	for _, rules := range ing.Spec.Rules {
+		hosts = append(hosts, rules.Host)
+	}
+	ing.Spec.TLS = []extensions.IngressTLS{
+		{Hosts: hosts, SecretName: name},
+	}
+	return &ing
 }
 
 // createApp will create a single RC and Svc. The Svc will match pods of the
@@ -323,21 +329,36 @@ func (cont *IngressController) Cleanup(del bool) error {
 	// Ordering is important here because we cannot delete resources that other
 	// resources hold references to.
 	fwList := []compute.ForwardingRule{}
-	gcloudUnmarshal("forwarding-rules", fmt.Sprintf("k8s-fw-.*--%v", cont.UID), cont.Project, &fwList)
-	if len(fwList) != 0 {
-		msg := ""
-		for _, f := range fwList {
-			msg += fmt.Sprintf("%v\n", f.Name)
-			if del {
-				Logf("Deleting forwarding-rule: %v", f.Name)
-				output, err := exec.Command("gcloud", "compute", "forwarding-rules", "delete",
-					f.Name, fmt.Sprintf("--project=%v", cont.Project), "-q", "--global").CombinedOutput()
-				if err != nil {
-					Logf("Error deleting forwarding rules, output: %v\nerror:%v", string(output), err)
+	for _, regex := range []string{fmt.Sprintf("k8s-fw-.*--%v", cont.UID), fmt.Sprintf("k8s-fws-.*--%v", cont.UID)} {
+		gcloudUnmarshal("forwarding-rules", regex, cont.Project, &fwList)
+		if len(fwList) != 0 {
+			msg := ""
+			for _, f := range fwList {
+				msg += fmt.Sprintf("%v\n", f.Name)
+				if del {
+					Logf("Deleting forwarding-rule: %v", f.Name)
+					output, err := exec.Command("gcloud", "compute", "forwarding-rules", "delete",
+						f.Name, fmt.Sprintf("--project=%v", cont.Project), "-q", "--global").CombinedOutput()
+					if err != nil {
+						Logf("Error deleting forwarding rules, output: %v\nerror:%v", string(output), err)
+					}
 				}
 			}
+			errMsg += fmt.Sprintf("\nFound forwarding rules:\n%v", msg)
 		}
-		errMsg += fmt.Sprintf("\nFound forwarding rules:\n%v", msg)
+	}
+	// Static IPs are named after forwarding rules.
+	ipList := []compute.Address{}
+	gcloudUnmarshal("addresses", fmt.Sprintf("k8s-fw-.*--%v", cont.UID), cont.Project, &ipList)
+	if len(ipList) != 0 {
+		msg := ""
+		for _, ip := range ipList {
+			msg += fmt.Sprintf("%v\n", ip.Name)
+			if del {
+				gcloudDelete("addresses", ip.Name, cont.Project)
+			}
+		}
+		errMsg += fmt.Sprintf("Found health check:\n%v", msg)
 	}
 
 	tpList := []compute.TargetHttpProxy{}
@@ -352,6 +373,19 @@ func (cont *IngressController) Cleanup(del bool) error {
 		}
 		errMsg += fmt.Sprintf("Found target proxies:\n%v", msg)
 	}
+	tpsList := []compute.TargetHttpsProxy{}
+	gcloudUnmarshal("target-https-proxies", fmt.Sprintf("k8s-tps-.*--%v", cont.UID), cont.Project, &tpsList)
+	if len(tpsList) != 0 {
+		msg := ""
+		for _, t := range tpsList {
+			msg += fmt.Sprintf("%v\n", t.Name)
+			if del {
+				gcloudDelete("target-http-proxies", t.Name, cont.Project)
+			}
+		}
+		errMsg += fmt.Sprintf("Found target HTTPS proxies:\n%v", msg)
+	}
+	// TODO: Check for leaked ssl certs.
 
 	umList := []compute.UrlMap{}
 	gcloudUnmarshal("url-maps", fmt.Sprintf("k8s-um-.*--%v", cont.UID), cont.Project, &umList)
@@ -439,6 +473,7 @@ var _ = Describe("GCE L7 LoadBalancer Controller [Feature:Ingress]", func() {
 			c:              client,
 		}
 		ingController.create()
+		Logf("Finished creating ingress controller")
 		// If we somehow get the same namespace uid as someone else in this
 		// gce project, just back off.
 		Expect(ingController.Cleanup(false)).NotTo(HaveOccurred())
@@ -503,19 +538,34 @@ var _ = Describe("GCE L7 LoadBalancer Controller [Feature:Ingress]", func() {
 		if numApps < numIng {
 			Failf("Need more apps than Ingress")
 		}
+		Logf("Starting ingress test")
 		appsPerIngress := numApps / numIng
 		By(fmt.Sprintf("Creating %d rcs + svc, and %d apps per Ingress", numApps, appsPerIngress))
+
+		ingCAs := map[string][]byte{}
 		for appID := 0; appID < numApps; appID = appID + appsPerIngress {
 			// Creates appsPerIngress apps, then creates one Ingress with paths to all the apps.
 			for j := appID; j < appID+appsPerIngress; j++ {
 				createApp(client, ns, j)
 			}
-			createIngress(client, ns, appID, appsPerIngress)
+			var err error
+			ing := generateIngressSpec(appID, appsPerIngress, ns)
+
+			// Secrets must be created before the Ingress. The cert of each
+			// Ingress contains all the hostnames of that Ingress as the subject
+			// name field in the cert.
+			By(fmt.Sprintf("Creating secret for ingress %v/%v", ing.Namespace, ing.Name))
+			_, rootCA, _, err := createSecret(client, ing)
+			Expect(err).NotTo(HaveOccurred())
+			ingCAs[ing.Name] = rootCA
+
+			By(fmt.Sprintf("Creating ingress %v/%v", ing.Namespace, ing.Name))
+			ing, err = client.Extensions().Ingress(ing.Namespace).Create(ing)
+			Expect(err).NotTo(HaveOccurred())
 		}
 
 		ings, err := client.Extensions().Ingress(ns).List(api.ListOptions{})
 		Expect(err).NotTo(HaveOccurred())
-
 		for _, ing := range ings.Items {
 			// Wait for the loadbalancer IP.
 			start := time.Now()
@@ -538,10 +588,14 @@ var _ = Describe("GCE L7 LoadBalancer Controller [Feature:Ingress]", func() {
 				if rules.IngressRuleValue.HTTP == nil {
 					continue
 				}
-				for _, p := range rules.IngressRuleValue.HTTP.Paths {
-					route := fmt.Sprintf("http://%v%v", address, p.Path)
-					Logf("Testing route %v host %v with simple GET", route, rules.Host)
+				timeoutClient.Transport, err = buildTransport(rules.Host, ingCAs[ing.Name])
 
+				for _, p := range rules.IngressRuleValue.HTTP.Paths {
+					route := fmt.Sprintf("https://%v%v", address, p.Path)
+					Logf("Testing route %v host %v with simple GET", route, rules.Host)
+					if err != nil {
+						Failf("Unable to create transport: %v", err)
+					}
 					// Make sure the service node port is reachable
 					Expect(curlServiceNodePort(client, ns, p.Backend.ServiceName, int(p.Backend.ServicePort.IntVal))).NotTo(HaveOccurred())
 
@@ -597,9 +651,18 @@ func curlServiceNodePort(client *client.Client, ns, name string, port int) error
 	if err != nil {
 		return err
 	}
-	svcCurlBody, err := simpleGET(timeoutClient, u, "")
-	if err != nil {
-		return fmt.Errorf("Failed to curl service node port, body: %v\nerror %v", svcCurlBody, err)
+	var svcCurlBody string
+	timeout := 30 * time.Second
+	pollErr := wait.Poll(10*time.Second, timeout, func() (bool, error) {
+		svcCurlBody, err = simpleGET(timeoutClient, u, "")
+		if err != nil {
+			Logf("Failed to curl service node port, body: %v\nerror %v", svcCurlBody, err)
+			return false, nil
+		}
+		return true, nil
+	})
+	if pollErr != nil {
+		return fmt.Errorf("Failed to curl service node port in %v, body: %v\nerror %v", timeout, svcCurlBody, err)
 	}
 	Logf("Successfully curled service node port, body: %v", svcCurlBody)
 	return nil

test/e2e/ingress_utils.go

@@ -0,0 +1,146 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"io"
+	"math/big"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/apis/extensions"
+	client "k8s.io/kubernetes/pkg/client/unversioned"
+)
+
+const (
+	rsaBits  = 2048
+	validFor = 365 * 24 * time.Hour
+)
+
+// generateRSACerts generates a basic self signed certificate using a key length
+// of rsaBits, valid for validFor time.
+func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error {
+	if len(host) == 0 {
+		return fmt.Errorf("Require a non-empty host for client hello")
+	}
+	priv, err := rsa.GenerateKey(rand.Reader, rsaBits)
+	if err != nil {
+		return fmt.Errorf("Failed to generate key: %v", err)
+	}
+	notBefore := time.Now()
+	notAfter := notBefore.Add(validFor)
+
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+
+	if err != nil {
+		return fmt.Errorf("failed to generate serial number: %s", err)
+	}
+	template := x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			CommonName:   "default",
+			Organization: []string{"Acme Co"},
+		},
+		NotBefore: notBefore,
+		NotAfter:  notAfter,
+
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+
+	hosts := strings.Split(host, ",")
+	for _, h := range hosts {
+		if ip := net.ParseIP(h); ip != nil {
+			template.IPAddresses = append(template.IPAddresses, ip)
+		} else {
+			template.DNSNames = append(template.DNSNames, h)
+		}
+	}
+
+	if isCA {
+		template.IsCA = true
+		template.KeyUsage |= x509.KeyUsageCertSign
+	}
+
+	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
+	if err != nil {
+		return fmt.Errorf("Failed to create certificate: %s", err)
+	}
+	if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
+		return fmt.Errorf("Failed creating cert: %v", err)
+	}
+	if err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
+		return fmt.Errorf("Failed creating keay: %v", err)
+	}
+	return nil
+}
+
+// buildTransport creates a transport for use in executing HTTPS requests with
+// the given certs. Note that the given rootCA must be configured with isCA=true.
+func buildTransport(serverName string, rootCA []byte) (*http.Transport, error) {
+	pool := x509.NewCertPool()
+	ok := pool.AppendCertsFromPEM(rootCA)
+	if !ok {
+		return nil, fmt.Errorf("Unable to load serverCA.")
+	}
+	return &http.Transport{
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: false,
+			ServerName:         serverName,
+			RootCAs:            pool,
+		},
+	}, nil
+}
+
+// createSecret creates a secret containing TLS certificates for the given Ingress.
+func createSecret(kubeClient *client.Client, ing *extensions.Ingress) (host string, rootCA, privKey []byte, err error) {
+	var k, c bytes.Buffer
+	tls := ing.Spec.TLS[0]
+	host = strings.Join(tls.Hosts, ",")
+	Logf("Generating RSA cert for host %v", host)
+
+	if err = generateRSACerts(host, true, &k, &c); err != nil {
+		return
+	}
+	cert := c.Bytes()
+	key := k.Bytes()
+	secret := &api.Secret{
+		ObjectMeta: api.ObjectMeta{
+			Name: tls.SecretName,
+		},
+		Data: map[string][]byte{
+			api.TLSCertKey:       cert,
+			api.TLSPrivateKeyKey: key,
+		},
+	}
+	Logf("Creating secret %v in ns %v with hosts %v for ingress %v", secret.Name, secret.Namespace, host, ing.Name)
+	_, err = kubeClient.Secrets(ing.Namespace).Create(secret)
+	return host, cert, key, err
+}