From 38290535d696f9dd6811044d77be6e2f38efe47f Mon Sep 17 00:00:00 2001 From: Zihong Zheng Date: Fri, 26 Jan 2018 16:35:05 -0800 Subject: [PATCH] [GCE Ingress e2e] Add test for pre-shared certificate --- test/e2e/framework/ingress_utils.go | 112 ++++++++++++------ test/e2e/network/ingress.go | 53 ++++++++- .../ingress/pre-shared-cert/ing.yaml | 16 +++ .../ingress/pre-shared-cert/rc.yaml | 16 +++ .../ingress/pre-shared-cert/svc.yaml | 15 +++ test/e2e/upgrades/ingress.go | 4 +- 6 files changed, 171 insertions(+), 45 deletions(-) create mode 100644 test/e2e/testing-manifests/ingress/pre-shared-cert/ing.yaml create mode 100644 test/e2e/testing-manifests/ingress/pre-shared-cert/rc.yaml create mode 100644 test/e2e/testing-manifests/ingress/pre-shared-cert/svc.yaml diff --git a/test/e2e/framework/ingress_utils.go b/test/e2e/framework/ingress_utils.go index 6f236cabcd4..5a9456e6ca6 100644 --- a/test/e2e/framework/ingress_utils.go +++ b/test/e2e/framework/ingress_utils.go @@ -26,7 +26,6 @@ import ( "encoding/json" "encoding/pem" "fmt" - "io" "math/big" "net" "net/http" @@ -64,11 +63,20 @@ const ( // Ingress class annotation defined in ingress repository. // TODO: All these annotations should be reused from // ingress-gce/pkg/annotations instead of duplicating them here. - IngressClass = "kubernetes.io/ingress.class" + IngressClassKey = "kubernetes.io/ingress.class" // Ingress class annotation value for multi cluster ingress. MulticlusterIngressClassValue = "gce-multi-cluster" + // Static IP annotation defined in ingress repository. + IngressStaticIPKey = "kubernetes.io/ingress.global-static-ip-name" + + // Allow HTTP annotation defined in ingress repository. + IngressAllowHTTPKey = "kubernetes.io/ingress.allow-http" + + // Pre-shared-cert annotation defined in ingress repository. + IngressPreSharedCertKey = "ingress.gcp.kubernetes.io/pre-shared-cert" + // all cloud resources created by the ingress controller start with this // prefix. k8sPrefix = "k8s-" @@ -210,15 +218,15 @@ func CreateIngressComformanceTests(jig *IngressTestJig, ns string, annotations m } } -// generateRSACerts generates a basic self signed certificate using a key length +// GenerateRSACerts generates a basic self signed certificate using a key length // of rsaBits, valid for validFor time. -func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error { +func GenerateRSACerts(host string, isCA bool) ([]byte, []byte, error) { if len(host) == 0 { - return fmt.Errorf("Require a non-empty host for client hello") + return nil, nil, fmt.Errorf("Require a non-empty host for client hello") } priv, err := rsa.GenerateKey(rand.Reader, rsaBits) if err != nil { - return fmt.Errorf("Failed to generate key: %v", err) + return nil, nil, fmt.Errorf("Failed to generate key: %v", err) } notBefore := time.Now() notAfter := notBefore.Add(validFor) @@ -227,7 +235,7 @@ func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error { serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) if err != nil { - return fmt.Errorf("failed to generate serial number: %s", err) + return nil, nil, fmt.Errorf("failed to generate serial number: %s", err) } template := x509.Certificate{ SerialNumber: serialNumber, @@ -257,17 +265,18 @@ func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error { template.KeyUsage |= x509.KeyUsageCertSign } + var keyOut, certOut bytes.Buffer derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) if err != nil { - return fmt.Errorf("Failed to create certificate: %s", err) + return nil, nil, fmt.Errorf("Failed to create certificate: %s", err) } - if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { - return fmt.Errorf("Failed creating cert: %v", err) + if err := pem.Encode(&certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { + return nil, nil, fmt.Errorf("Failed creating cert: %v", err) } - if err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil { - return fmt.Errorf("Failed creating keay: %v", err) + if err := pem.Encode(&keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil { + return nil, nil, fmt.Errorf("Failed creating keay: %v", err) } - return nil + return certOut.Bytes(), keyOut.Bytes(), nil } // buildTransportWithCA creates a transport for use in executing HTTPS requests with @@ -297,16 +306,13 @@ func BuildInsecureClient(timeout time.Duration) *http.Client { // If a secret with the same name already pathExists in the namespace of the // Ingress, it's updated. func createIngressTLSSecret(kubeClient clientset.Interface, ing *extensions.Ingress) (host string, rootCA, privKey []byte, err error) { - var k, c bytes.Buffer tls := ing.Spec.TLS[0] host = strings.Join(tls.Hosts, ",") Logf("Generating RSA cert for host %v", host) - - if err = generateRSACerts(host, true, &k, &c); err != nil { + cert, key, err := GenerateRSACerts(host, true) + if err != nil { return } - cert := c.Bytes() - key := k.Bytes() secret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: tls.SecretName, @@ -1046,7 +1052,7 @@ func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations m j.Ingress, err = manifest.IngressFromManifest(filepath.Join(manifestPath, "ing.yaml")) ExpectNoError(err) j.Ingress.Namespace = ns - j.Ingress.Annotations = map[string]string{IngressClass: j.Class} + j.Ingress.Annotations = map[string]string{IngressClassKey: j.Class} for k, v := range ingAnnotations { j.Ingress.Annotations[k] = v } @@ -1145,6 +1151,31 @@ func WaitForIngressAddress(c clientset.Interface, ns, ingName string, timeout ti return address, err } +func (j *IngressTestJig) PollIngressWithCert(waitForNodePort bool, knownHosts []string, cert []byte) { + // Check that all rules respond to a simple GET. + knownHostsSet := sets.NewString(knownHosts...) + for _, rules := range j.Ingress.Spec.Rules { + timeoutClient := &http.Client{Timeout: IngressReqTimeout} + proto := "http" + if knownHostsSet.Has(rules.Host) { + var err error + // Create transport with cert to verify if the server uses the correct one. + timeoutClient.Transport, err = buildTransportWithCA(rules.Host, cert) + ExpectNoError(err) + proto = "https" + } + for _, p := range rules.IngressRuleValue.HTTP.Paths { + if waitForNodePort { + j.pollServiceNodePort(j.Ingress.Namespace, p.Backend.ServiceName, int(p.Backend.ServicePort.IntVal)) + } + route := fmt.Sprintf("%v://%v%v", proto, j.Address, p.Path) + Logf("Testing route %v host %v with simple GET", route, rules.Host) + ExpectNoError(PollURL(route, rules.Host, LoadBalancerPollTimeout, j.PollInterval, timeoutClient, false)) + } + } + Logf("Finished polling on all rules on ingress %q", j.Ingress.Name) +} + // WaitForIngress waits till the ingress acquires an IP, then waits for its // hosts/urls to respond to a protocol check (either http or https). If // waitForNodePort is true, the NodePort of the Service is verified before @@ -1158,28 +1189,31 @@ func (j *IngressTestJig) WaitForIngress(waitForNodePort bool) { } j.Address = address Logf("Found address %v for ingress %v", j.Address, j.Ingress.Name) - timeoutClient := &http.Client{Timeout: IngressReqTimeout} - // Check that all rules respond to a simple GET. - for _, rules := range j.Ingress.Spec.Rules { - proto := "http" - if len(j.Ingress.Spec.TLS) > 0 { - knownHosts := sets.NewString(j.Ingress.Spec.TLS[0].Hosts...) - if knownHosts.Has(rules.Host) { - timeoutClient.Transport, err = buildTransportWithCA(rules.Host, j.GetRootCA(j.Ingress.Spec.TLS[0].SecretName)) - ExpectNoError(err) - proto = "https" - } - } - for _, p := range rules.IngressRuleValue.HTTP.Paths { - if waitForNodePort { - j.pollServiceNodePort(j.Ingress.Namespace, p.Backend.ServiceName, int(p.Backend.ServicePort.IntVal)) - } - route := fmt.Sprintf("%v://%v%v", proto, address, p.Path) - Logf("Testing route %v host %v with simple GET", route, rules.Host) - ExpectNoError(PollURL(route, rules.Host, LoadBalancerPollTimeout, j.PollInterval, timeoutClient, false)) - } + var knownHosts []string + var cert []byte + if len(j.Ingress.Spec.TLS) > 0 { + knownHosts = j.Ingress.Spec.TLS[0].Hosts + cert = j.GetRootCA(j.Ingress.Spec.TLS[0].SecretName) } + j.PollIngressWithCert(waitForNodePort, knownHosts, cert) +} + +// WaitForIngress waits till the ingress acquires an IP, then waits for its +// hosts/urls to respond to a protocol check (either http or https). If +// waitForNodePort is true, the NodePort of the Service is verified before +// verifying the Ingress. NodePort is currently a requirement for cloudprovider +// Ingress. Hostnames and certificate need to be explicitly passed in. +func (j *IngressTestJig) WaitForIngressWithCert(waitForNodePort bool, knownHosts []string, cert []byte) { + // Wait for the loadbalancer IP. + address, err := WaitForIngressAddress(j.Client, j.Ingress.Namespace, j.Ingress.Name, LoadBalancerPollTimeout) + if err != nil { + Failf("Ingress failed to acquire an IP address within %v", LoadBalancerPollTimeout) + } + j.Address = address + Logf("Found address %v for ingress %v", j.Address, j.Ingress.Name) + + j.PollIngressWithCert(waitForNodePort, knownHosts, cert) } // VerifyURL polls for the given iterations, in intervals, and fails if the diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go index 3db1af1a063..a3165e8e5a9 100644 --- a/test/e2e/network/ingress.go +++ b/test/e2e/network/ingress.go @@ -22,8 +22,11 @@ import ( "strings" "time" + compute "google.golang.org/api/compute/v1" + extensions "k8s.io/api/extensions/v1beta1" rbacv1beta1 "k8s.io/api/rbac/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" @@ -34,7 +37,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - compute "google.golang.org/api/compute/v1" ) const ( @@ -123,8 +125,8 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", ns, ip)) jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns, map[string]string{ - "kubernetes.io/ingress.global-static-ip-name": ns, - "kubernetes.io/ingress.allow-http": "false", + framework.IngressStaticIPKey: ns, + framework.IngressAllowHTTPKey: "false", }, map[string]string{}) By("waiting for Ingress to come up with ip: " + ip) @@ -317,10 +319,52 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { Expect(hcAfterSync.HttpHealthCheck.RequestPath).To(Equal(hcToChange.HttpHealthCheck.RequestPath)) }) + It("should create ingress with pre-shared certificate", func() { + preSharedCertName := "test-pre-shared-cert" + By(fmt.Sprintf("Creating ssl certificate %q on GCE", preSharedCertName)) + testHostname := "test.ingress.com" + cert, key, err := framework.GenerateRSACerts(testHostname, true) + Expect(err).NotTo(HaveOccurred()) + gceCloud, err := framework.GetGCECloud() + Expect(err).NotTo(HaveOccurred()) + defer func() { + // We would not be able to delete the cert until ingress controller + // cleans up the target proxy that references it. + By("Deleting ingress before deleting ssl certificate") + jig.TryDeleteIngress() + By(fmt.Sprintf("Deleting ssl certificate %q on GCE", preSharedCertName)) + err := wait.Poll(framework.LoadBalancerPollInterval, framework.LoadBalancerCleanupTimeout, func() (bool, error) { + if err := gceCloud.DeleteSslCertificate(preSharedCertName); err != nil && !errors.IsNotFound(err) { + framework.Logf("Failed to delete ssl certificate %q: %v. Retrying...", preSharedCertName, err) + return false, nil + } + return true, nil + }) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to delete ssl certificate %q: %v", preSharedCertName, err)) + }() + _, err = gceCloud.CreateSslCertificate(&compute.SslCertificate{ + Name: preSharedCertName, + Certificate: string(cert), + PrivateKey: string(key), + Description: "pre-shared cert for ingress testing", + }) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to create ssl certificate %q: %v", preSharedCertName, err)) + + By("Creating an ingress referencing the pre-shared certificate") + // Create an ingress referencing this cert using pre-shared-cert annotation. + jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "pre-shared-cert"), ns, map[string]string{ + framework.IngressPreSharedCertKey: preSharedCertName, + framework.IngressAllowHTTPKey: "false", + }, map[string]string{}) + + By("Test that ingress works with the pre-shared certificate") + jig.WaitForIngressWithCert(true, []string{testHostname}, cert) + }) + It("multicluster ingress should get instance group annotation", func() { name := "echomap" jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "http"), ns, map[string]string{ - framework.IngressClass: framework.MulticlusterIngressClassValue, + framework.IngressClassKey: framework.MulticlusterIngressClassValue, }, map[string]string{}) By(fmt.Sprintf("waiting for Ingress %s to come up", name)) @@ -343,6 +387,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { // TODO: Implement a multizone e2e that verifies traffic reaches each // zone based on pod labels. }) + Describe("GCE [Slow] [Feature:NEG]", func() { var gceController *framework.GCEIngressController diff --git a/test/e2e/testing-manifests/ingress/pre-shared-cert/ing.yaml b/test/e2e/testing-manifests/ingress/pre-shared-cert/ing.yaml new file mode 100644 index 00000000000..1de721a479e --- /dev/null +++ b/test/e2e/testing-manifests/ingress/pre-shared-cert/ing.yaml @@ -0,0 +1,16 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: pre-shared-cert + # Below annotation will be added upon test: + # annotations: + # ingress.gcp.kubernetes.io/pre-shared-cert: "test-pre-shared-cert" +spec: + rules: + - host: test.ingress.com + http: + paths: + - path: /test + backend: + serviceName: echoheaders-https + servicePort: 80 diff --git a/test/e2e/testing-manifests/ingress/pre-shared-cert/rc.yaml b/test/e2e/testing-manifests/ingress/pre-shared-cert/rc.yaml new file mode 100644 index 00000000000..abf9b036edd --- /dev/null +++ b/test/e2e/testing-manifests/ingress/pre-shared-cert/rc.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: echoheaders-https +spec: + replicas: 2 + template: + metadata: + labels: + app: echoheaders-https + spec: + containers: + - name: echoheaders-https + image: gcr.io/google_containers/echoserver:1.6 + ports: + - containerPort: 8080 diff --git a/test/e2e/testing-manifests/ingress/pre-shared-cert/svc.yaml b/test/e2e/testing-manifests/ingress/pre-shared-cert/svc.yaml new file mode 100644 index 00000000000..b022aa17fce --- /dev/null +++ b/test/e2e/testing-manifests/ingress/pre-shared-cert/svc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: echoheaders-https + labels: + app: echoheaders-https +spec: + type: NodePort + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: echoheaders-https diff --git a/test/e2e/upgrades/ingress.go b/test/e2e/upgrades/ingress.go index 010f2c8fbce..e1e95b34382 100644 --- a/test/e2e/upgrades/ingress.go +++ b/test/e2e/upgrades/ingress.go @@ -94,8 +94,8 @@ func (t *IngressUpgradeTest) Setup(f *framework.Framework) { // Create a working basic Ingress By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", t.ipName, t.ip)) jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip-2"), ns.Name, map[string]string{ - "kubernetes.io/ingress.global-static-ip-name": t.ipName, - "kubernetes.io/ingress.allow-http": "false", + framework.IngressStaticIPKey: t.ipName, + framework.IngressAllowHTTPKey: "false", }, map[string]string{}) t.jig.AddHTTPS("tls-secret", "ingress.test.com")