From 10cf2d4e4efb7929e9dd4bd2f658c84bde131109 Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Fri, 5 Feb 2016 11:29:06 -0800 Subject: [PATCH 1/2] Ingress https E2E --- test/e2e/ingress.go | 115 +++++++++++++++++++++++------- test/e2e/ingress_utils.go | 146 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 235 insertions(+), 26 deletions(-) create mode 100644 test/e2e/ingress_utils.go diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 50cef37a020..befccdb5ef6 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -146,10 +146,11 @@ func ruleByIndex(i int) extensions.IngressRule { // foo1.bar.com: /foo1 // foo2.bar.com: /foo2 // } -func createIngress(c *client.Client, ns string, start, num int) extensions.Ingress { +func generateIngressSpec(start, num int, ns string) *extensions.Ingress { + name := fmt.Sprintf("%v%d", appPrefix, start) ing := extensions.Ingress{ ObjectMeta: api.ObjectMeta{ - Name: fmt.Sprintf("%v%d", appPrefix, start), + Name: name, Namespace: ns, }, Spec: extensions.IngressSpec{ @@ -163,10 +164,15 @@ func createIngress(c *client.Client, ns string, start, num int) extensions.Ingre for i := start; i < start+num; i++ { ing.Spec.Rules = append(ing.Spec.Rules, ruleByIndex(i)) } - Logf("Creating ingress %v", start) - _, err := c.Extensions().Ingress(ns).Create(&ing) - Expect(err).NotTo(HaveOccurred()) - return ing + // Create the host for the cert by appending all hosts mentioned in rules. + hosts := []string{} + for _, rules := range ing.Spec.Rules { + hosts = append(hosts, rules.Host) + } + ing.Spec.TLS = []extensions.IngressTLS{ + {Hosts: hosts, SecretName: name}, + } + return &ing } // createApp will create a single RC and Svc. The Svc will match pods of the @@ -323,21 +329,36 @@ func (cont *IngressController) Cleanup(del bool) error { // Ordering is important here because we cannot delete resources that other // resources hold references to. fwList := []compute.ForwardingRule{} - gcloudUnmarshal("forwarding-rules", fmt.Sprintf("k8s-fw-.*--%v", cont.UID), cont.Project, &fwList) - if len(fwList) != 0 { - msg := "" - for _, f := range fwList { - msg += fmt.Sprintf("%v\n", f.Name) - if del { - Logf("Deleting forwarding-rule: %v", f.Name) - output, err := exec.Command("gcloud", "compute", "forwarding-rules", "delete", - f.Name, fmt.Sprintf("--project=%v", cont.Project), "-q", "--global").CombinedOutput() - if err != nil { - Logf("Error deleting forwarding rules, output: %v\nerror:%v", string(output), err) + for _, regex := range []string{fmt.Sprintf("k8s-fw-.*--%v", cont.UID), fmt.Sprintf("k8s-fws-.*--%v", cont.UID)} { + gcloudUnmarshal("forwarding-rules", regex, cont.Project, &fwList) + if len(fwList) != 0 { + msg := "" + for _, f := range fwList { + msg += fmt.Sprintf("%v\n", f.Name) + if del { + Logf("Deleting forwarding-rule: %v", f.Name) + output, err := exec.Command("gcloud", "compute", "forwarding-rules", "delete", + f.Name, fmt.Sprintf("--project=%v", cont.Project), "-q", "--global").CombinedOutput() + if err != nil { + Logf("Error deleting forwarding rules, output: %v\nerror:%v", string(output), err) + } } } + errMsg += fmt.Sprintf("\nFound forwarding rules:\n%v", msg) } - errMsg += fmt.Sprintf("\nFound forwarding rules:\n%v", msg) + } + // Static IPs are named after forwarding rules. + ipList := []compute.Address{} + gcloudUnmarshal("addresses", fmt.Sprintf("k8s-fw-.*--%v", cont.UID), cont.Project, &ipList) + if len(ipList) != 0 { + msg := "" + for _, ip := range ipList { + msg += fmt.Sprintf("%v\n", ip.Name) + if del { + gcloudDelete("addresses", ip.Name, cont.Project) + } + } + errMsg += fmt.Sprintf("Found health check:\n%v", msg) } tpList := []compute.TargetHttpProxy{} @@ -352,6 +373,19 @@ func (cont *IngressController) Cleanup(del bool) error { } errMsg += fmt.Sprintf("Found target proxies:\n%v", msg) } + tpsList := []compute.TargetHttpsProxy{} + gcloudUnmarshal("target-https-proxies", fmt.Sprintf("k8s-tps-.*--%v", cont.UID), cont.Project, &tpsList) + if len(tpsList) != 0 { + msg := "" + for _, t := range tpsList { + msg += fmt.Sprintf("%v\n", t.Name) + if del { + gcloudDelete("target-http-proxies", t.Name, cont.Project) + } + } + errMsg += fmt.Sprintf("Found target HTTPS proxies:\n%v", msg) + } + // TODO: Check for leaked ssl certs. umList := []compute.UrlMap{} gcloudUnmarshal("url-maps", fmt.Sprintf("k8s-um-.*--%v", cont.UID), cont.Project, &umList) @@ -439,6 +473,7 @@ var _ = Describe("GCE L7 LoadBalancer Controller [Feature:Ingress]", func() { c: client, } ingController.create() + Logf("Finished creating ingress controller") // If we somehow get the same namespace uid as someone else in this // gce project, just back off. Expect(ingController.Cleanup(false)).NotTo(HaveOccurred()) @@ -503,19 +538,34 @@ var _ = Describe("GCE L7 LoadBalancer Controller [Feature:Ingress]", func() { if numApps < numIng { Failf("Need more apps than Ingress") } + Logf("Starting ingress test") appsPerIngress := numApps / numIng By(fmt.Sprintf("Creating %d rcs + svc, and %d apps per Ingress", numApps, appsPerIngress)) + + ingCAs := map[string][]byte{} for appID := 0; appID < numApps; appID = appID + appsPerIngress { // Creates appsPerIngress apps, then creates one Ingress with paths to all the apps. for j := appID; j < appID+appsPerIngress; j++ { createApp(client, ns, j) } - createIngress(client, ns, appID, appsPerIngress) + var err error + ing := generateIngressSpec(appID, appsPerIngress, ns) + + // Secrets must be created before the Ingress. The cert of each + // Ingress contains all the hostnames of that Ingress as the subject + // name field in the cert. + By(fmt.Sprintf("Creating secret for ingress %v/%v", ing.Namespace, ing.Name)) + _, rootCA, _, err := createSecret(client, ing) + Expect(err).NotTo(HaveOccurred()) + ingCAs[ing.Name] = rootCA + + By(fmt.Sprintf("Creating ingress %v/%v", ing.Namespace, ing.Name)) + ing, err = client.Extensions().Ingress(ing.Namespace).Create(ing) + Expect(err).NotTo(HaveOccurred()) } ings, err := client.Extensions().Ingress(ns).List(api.ListOptions{}) Expect(err).NotTo(HaveOccurred()) - for _, ing := range ings.Items { // Wait for the loadbalancer IP. start := time.Now() @@ -538,10 +588,14 @@ var _ = Describe("GCE L7 LoadBalancer Controller [Feature:Ingress]", func() { if rules.IngressRuleValue.HTTP == nil { continue } - for _, p := range rules.IngressRuleValue.HTTP.Paths { - route := fmt.Sprintf("http://%v%v", address, p.Path) - Logf("Testing route %v host %v with simple GET", route, rules.Host) + timeoutClient.Transport, err = buildTransport(rules.Host, ingCAs[ing.Name]) + for _, p := range rules.IngressRuleValue.HTTP.Paths { + route := fmt.Sprintf("https://%v%v", address, p.Path) + Logf("Testing route %v host %v with simple GET", route, rules.Host) + if err != nil { + Failf("Unable to create transport: %v", err) + } // Make sure the service node port is reachable Expect(curlServiceNodePort(client, ns, p.Backend.ServiceName, int(p.Backend.ServicePort.IntVal))).NotTo(HaveOccurred()) @@ -597,9 +651,18 @@ func curlServiceNodePort(client *client.Client, ns, name string, port int) error if err != nil { return err } - svcCurlBody, err := simpleGET(timeoutClient, u, "") - if err != nil { - return fmt.Errorf("Failed to curl service node port, body: %v\nerror %v", svcCurlBody, err) + var svcCurlBody string + timeout := 30 * time.Second + pollErr := wait.Poll(10*time.Second, timeout, func() (bool, error) { + svcCurlBody, err = simpleGET(timeoutClient, u, "") + if err != nil { + Logf("Failed to curl service node port, body: %v\nerror %v", svcCurlBody, err) + return false, nil + } + return true, nil + }) + if pollErr != nil { + return fmt.Errorf("Failed to curl service node port in %v, body: %v\nerror %v", timeout, svcCurlBody, err) } Logf("Successfully curled service node port, body: %v", svcCurlBody) return nil diff --git a/test/e2e/ingress_utils.go b/test/e2e/ingress_utils.go new file mode 100644 index 00000000000..41952dfbc87 --- /dev/null +++ b/test/e2e/ingress_utils.go @@ -0,0 +1,146 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "io" + "math/big" + "net" + "net/http" + "strings" + "time" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/apis/extensions" + client "k8s.io/kubernetes/pkg/client/unversioned" +) + +const ( + rsaBits = 2048 + validFor = 365 * 24 * time.Hour +) + +// generateRSACerts generates a basic self signed certificate using a key length +// of rsaBits, valid for validFor time. +func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error { + if len(host) == 0 { + return fmt.Errorf("Require a non-empty host for client hello") + } + priv, err := rsa.GenerateKey(rand.Reader, rsaBits) + if err != nil { + return fmt.Errorf("Failed to generate key: %v", err) + } + notBefore := time.Now() + notAfter := notBefore.Add(validFor) + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + + if err != nil { + return fmt.Errorf("failed to generate serial number: %s", err) + } + template := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + CommonName: "default", + Organization: []string{"Acme Co"}, + }, + NotBefore: notBefore, + NotAfter: notAfter, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + hosts := strings.Split(host, ",") + for _, h := range hosts { + if ip := net.ParseIP(h); ip != nil { + template.IPAddresses = append(template.IPAddresses, ip) + } else { + template.DNSNames = append(template.DNSNames, h) + } + } + + if isCA { + template.IsCA = true + template.KeyUsage |= x509.KeyUsageCertSign + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + return fmt.Errorf("Failed to create certificate: %s", err) + } + if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { + return fmt.Errorf("Failed creating cert: %v", err) + } + if err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil { + return fmt.Errorf("Failed creating keay: %v", err) + } + return nil +} + +// buildTransport creates a transport for use in executing HTTPS requests with +// the given certs. Note that the given rootCA must be configured with isCA=true. +func buildTransport(serverName string, rootCA []byte) (*http.Transport, error) { + pool := x509.NewCertPool() + ok := pool.AppendCertsFromPEM(rootCA) + if !ok { + return nil, fmt.Errorf("Unable to load serverCA.") + } + return &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: false, + ServerName: serverName, + RootCAs: pool, + }, + }, nil +} + +// createSecret creates a secret containing TLS certificates for the given Ingress. +func createSecret(kubeClient *client.Client, ing *extensions.Ingress) (host string, rootCA, privKey []byte, err error) { + var k, c bytes.Buffer + tls := ing.Spec.TLS[0] + host = strings.Join(tls.Hosts, ",") + Logf("Generating RSA cert for host %v", host) + + if err = generateRSACerts(host, true, &k, &c); err != nil { + return + } + cert := c.Bytes() + key := k.Bytes() + secret := &api.Secret{ + ObjectMeta: api.ObjectMeta{ + Name: tls.SecretName, + }, + Data: map[string][]byte{ + api.TLSCertKey: cert, + api.TLSPrivateKeyKey: key, + }, + } + Logf("Creating secret %v in ns %v with hosts %v for ingress %v", secret.Name, secret.Namespace, host, ing.Name) + _, err = kubeClient.Secrets(ing.Namespace).Create(secret) + return host, cert, key, err +} From c76f1ab766cf42f95fde85beba8cb336e2e61fab Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Sun, 6 Mar 2016 17:12:42 -0800 Subject: [PATCH 2/2] Bump glbc version. --- .../cluster-loadbalancing/glbc/glbc-controller.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cluster/addons/cluster-loadbalancing/glbc/glbc-controller.yaml b/cluster/addons/cluster-loadbalancing/glbc/glbc-controller.yaml index e7bdb7e7c27..406458dfd6b 100644 --- a/cluster/addons/cluster-loadbalancing/glbc/glbc-controller.yaml +++ b/cluster/addons/cluster-loadbalancing/glbc/glbc-controller.yaml @@ -1,11 +1,11 @@ apiVersion: v1 kind: ReplicationController metadata: - name: l7-lb-controller-v0.5.2 + name: l7-lb-controller-v0.6.0 namespace: kube-system labels: k8s-app: glbc - version: v0.5.2 + version: v0.6.0 kubernetes.io/cluster-service: "true" kubernetes.io/name: "GLBC" spec: @@ -13,12 +13,12 @@ spec: replicas: 1 selector: k8s-app: glbc - version: v0.5.2 + version: v0.6.0 template: metadata: labels: k8s-app: glbc - version: v0.5.2 + version: v0.6.0 name: glbc kubernetes.io/cluster-service: "true" spec: @@ -45,7 +45,7 @@ spec: requests: cpu: 10m memory: 20Mi - - image: gcr.io/google_containers/glbc:0.5.2 + - image: gcr.io/google_containers/glbc:0.6.0 livenessProbe: httpGet: path: /healthz