diff --git a/test/e2e/framework/ingress_utils.go b/test/e2e/framework/ingress_utils.go index 9a063ab5fcd..1a544044d40 100644 --- a/test/e2e/framework/ingress_utils.go +++ b/test/e2e/framework/ingress_utils.go @@ -1231,7 +1231,7 @@ func (j *IngressTestJig) TryDeleteIngress() { } func (j *IngressTestJig) TryDeleteGivenIngress(ing *extensions.Ingress) { - if err := j.runDelete(ing, j.Class); err != nil { + if err := j.runDelete(ing); err != nil { j.Logger.Infof("Error while deleting the ingress %v/%v with class %s: %v", ing.Namespace, ing.Name, j.Class, err) } } @@ -1244,7 +1244,7 @@ func (j *IngressTestJig) TryDeleteGivenService(svc *v1.Service) { } // runDelete runs the required command to delete the given ingress. -func (j *IngressTestJig) runDelete(ing *extensions.Ingress, class string) error { +func (j *IngressTestJig) runDelete(ing *extensions.Ingress) error { if j.Class != MulticlusterIngressClassValue { return j.Client.ExtensionsV1beta1().Ingresses(ing.Namespace).Delete(ing.Name, nil) } @@ -1531,14 +1531,16 @@ func (cont *NginxIngressController) Init() { Logf("ingress controller running in pod %v on ip %v", cont.pod.Name, cont.externalIP) } -func GenerateReencryptionIngressSpec() *extensions.Ingress { +func generateBacksideHTTPSIngressSpec(ns string) *extensions.Ingress { return &extensions.Ingress{ ObjectMeta: metav1.ObjectMeta{ - Name: "echoheaders-reencryption", + Name: "echoheaders-https", + Namespace: ns, }, Spec: extensions.IngressSpec{ + // Note kubemci requres a default backend. Backend: &extensions.IngressBackend{ - ServiceName: "echoheaders-reencryption", + ServiceName: "echoheaders-https", ServicePort: intstr.IntOrString{ Type: intstr.Int, IntVal: 443, @@ -1548,10 +1550,10 @@ func GenerateReencryptionIngressSpec() *extensions.Ingress { } } -func GenerateReencryptionServiceSpec() *v1.Service { +func generateBacksideHTTPSServiceSpec() *v1.Service { return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "echoheaders-reencryption", + Name: "echoheaders-https", Annotations: map[string]string{ ServiceApplicationProtocolKey: `{"my-https-port":"HTTPS"}`, }, @@ -1564,32 +1566,32 @@ func GenerateReencryptionServiceSpec() *v1.Service { TargetPort: intstr.FromString("echo-443"), }}, Selector: map[string]string{ - "app": "echoheaders-reencryption", + "app": "echoheaders-https", }, Type: v1.ServiceTypeNodePort, }, } } -func GenerateReencryptionDeploymentSpec() *extensions.Deployment { +func generateBacksideHTTPSDeploymentSpec() *extensions.Deployment { return &extensions.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "echoheaders-reencryption", + Name: "echoheaders-https", }, Spec: extensions.DeploymentSpec{ Selector: &metav1.LabelSelector{MatchLabels: map[string]string{ - "app": "echoheaders-reencryption", + "app": "echoheaders-https", }}, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "app": "echoheaders-reencryption", + "app": "echoheaders-https", }, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "echoheaders-reencryption", + Name: "echoheaders-https", Image: "k8s.gcr.io/echoserver:1.9", Ports: []v1.ContainerPort{{ ContainerPort: 8443, @@ -1603,26 +1605,32 @@ func GenerateReencryptionDeploymentSpec() *extensions.Deployment { } } -func CreateReencryptionIngress(cs clientset.Interface, namespace string) (*extensions.Deployment, *v1.Service, *extensions.Ingress, error) { - deployCreated, err := cs.ExtensionsV1beta1().Deployments(namespace).Create(GenerateReencryptionDeploymentSpec()) +// SetUpBacksideHTTPSIngress sets up deployment, service and ingress with backside HTTPS configured. +func (j *IngressTestJig) SetUpBacksideHTTPSIngress(cs clientset.Interface, namespace string, staticIPName string) (*extensions.Deployment, *v1.Service, *extensions.Ingress, error) { + deployCreated, err := cs.ExtensionsV1beta1().Deployments(namespace).Create(generateBacksideHTTPSDeploymentSpec()) if err != nil { return nil, nil, nil, err } - svcCreated, err := cs.CoreV1().Services(namespace).Create(GenerateReencryptionServiceSpec()) + svcCreated, err := cs.CoreV1().Services(namespace).Create(generateBacksideHTTPSServiceSpec()) if err != nil { return nil, nil, nil, err } - ingCreated, err := cs.ExtensionsV1beta1().Ingresses(namespace).Create(GenerateReencryptionIngressSpec()) + ingToCreate := generateBacksideHTTPSIngressSpec(namespace) + if staticIPName != "" { + ingToCreate.Annotations[IngressStaticIPKey] = staticIPName + } + ingCreated, err := j.runCreate(ingToCreate) if err != nil { return nil, nil, nil, err } return deployCreated, svcCreated, ingCreated, nil } -func CleanupReencryptionIngress(cs clientset.Interface, deploy *extensions.Deployment, svc *v1.Service, ing *extensions.Ingress) []error { +// DeleteTestResource deletes given deployment, service and ingress. +func (j *IngressTestJig) DeleteTestResource(cs clientset.Interface, deploy *extensions.Deployment, svc *v1.Service, ing *extensions.Ingress) []error { var errs []error if ing != nil { - if err := cs.ExtensionsV1beta1().Ingresses(ing.Namespace).Delete(ing.Name, nil); err != nil { + if err := j.runDelete(ing); err != nil { errs = append(errs, fmt.Errorf("error while deleting ingress %s/%s: %v", ing.Namespace, ing.Name, err)) } } diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go index d5a1afa7749..dacb611e7ff 100644 --- a/test/e2e/network/ingress.go +++ b/test/e2e/network/ingress.go @@ -31,6 +31,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/serviceaccount" gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" @@ -320,80 +321,11 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { }) It("should create ingress with pre-shared certificate", func() { - preSharedCertName := "test-pre-shared-cert" - By(fmt.Sprintf("Creating ssl certificate %q on GCE", preSharedCertName)) - testHostname := "test.ingress.com" - cert, key, err := framework.GenerateRSACerts(testHostname, true) - Expect(err).NotTo(HaveOccurred()) - gceCloud, err := framework.GetGCECloud() - Expect(err).NotTo(HaveOccurred()) - defer func() { - // We would not be able to delete the cert until ingress controller - // cleans up the target proxy that references it. - By("Deleting ingress before deleting ssl certificate") - if jig.Ingress != nil { - jig.TryDeleteIngress() - } - By(fmt.Sprintf("Deleting ssl certificate %q on GCE", preSharedCertName)) - err := wait.Poll(framework.LoadBalancerPollInterval, framework.LoadBalancerCleanupTimeout, func() (bool, error) { - if err := gceCloud.DeleteSslCertificate(preSharedCertName); err != nil && !errors.IsNotFound(err) { - framework.Logf("Failed to delete ssl certificate %q: %v. Retrying...", preSharedCertName, err) - return false, nil - } - return true, nil - }) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to delete ssl certificate %q: %v", preSharedCertName, err)) - }() - _, err = gceCloud.CreateSslCertificate(&compute.SslCertificate{ - Name: preSharedCertName, - Certificate: string(cert), - PrivateKey: string(key), - Description: "pre-shared cert for ingress testing", - }) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to create ssl certificate %q: %v", preSharedCertName, err)) - - By("Creating an ingress referencing the pre-shared certificate") - // Create an ingress referencing this cert using pre-shared-cert annotation. - jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "pre-shared-cert"), ns, map[string]string{ - framework.IngressPreSharedCertKey: preSharedCertName, - framework.IngressAllowHTTPKey: "false", - }, map[string]string{}) - - By("Test that ingress works with the pre-shared certificate") - err = jig.WaitForIngressWithCert(true, []string{testHostname}, cert) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unexpected error while waiting for ingress: %v", err)) + executePresharedCertTest(f, jig, "") }) - It("should create ingress with backside re-encryption", func() { - By("Creating a set of ingress, service and deployment that have backside re-encryption configured") - deployCreated, svcCreated, ingCreated, err := framework.CreateReencryptionIngress(f.ClientSet, f.Namespace.Name) - defer func() { - By("Cleaning up re-encryption ingress, service and deployment") - if errs := framework.CleanupReencryptionIngress(f.ClientSet, deployCreated, svcCreated, ingCreated); len(errs) > 0 { - framework.Failf("Failed to cleanup re-encryption ingress: %v", errs) - } - }() - Expect(err).NotTo(HaveOccurred(), "Failed to create re-encryption ingress") - - By(fmt.Sprintf("Waiting for ingress %s to come up", ingCreated.Name)) - ingIP, err := jig.WaitForIngressAddress(f.ClientSet, f.Namespace.Name, ingCreated.Name, framework.LoadBalancerPollTimeout) - Expect(err).NotTo(HaveOccurred(), "Failed to wait for ingress IP") - - By(fmt.Sprintf("Polling on address %s and verify the backend is serving HTTPS", ingIP)) - timeoutClient := &http.Client{Timeout: framework.IngressReqTimeout} - err = wait.PollImmediate(framework.LoadBalancerPollInterval, framework.LoadBalancerPollTimeout, func() (bool, error) { - resp, err := framework.SimpleGET(timeoutClient, fmt.Sprintf("http://%s", ingIP), "") - if err != nil { - framework.Logf("SimpleGET failed: %v", err) - return false, nil - } - if !strings.Contains(resp, "request_scheme=https") { - return false, fmt.Errorf("request wasn't served by HTTPS, response body: %s", resp) - } - framework.Logf("Poll succeeded, request was served by HTTPS") - return true, nil - }) - Expect(err).NotTo(HaveOccurred(), "Failed to verify backside re-encryption ingress") + It("should create ingress with backend HTTPS", func() { + executeBacksideBacksideHTTPSTest(f, jig, "") }) It("multicluster ingress should get instance group annotation", func() { @@ -603,11 +535,13 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { Describe("GCE [Slow] [Feature:kubemci]", func() { var gceController *framework.GCEIngressController + var ipName string // Platform specific setup BeforeEach(func() { framework.SkipUnlessProviderIs("gce", "gke") jig.Class = framework.MulticlusterIngressClassValue + jig.PollInterval = 5 * time.Second By("Initializing gce controller") gceController = &framework.GCEIngressController{ Ns: ns, @@ -616,6 +550,13 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { } err := gceController.Init() Expect(err).NotTo(HaveOccurred()) + + // TODO(https://github.com/GoogleCloudPlatform/k8s-multicluster-ingress/issues/19): + // Kubemci should reserve a static ip if user has not specified one. + ipName = "kubemci-" + string(uuid.NewUUID()) + // ip released when the rest of lb resources are deleted in CleanupGCEIngressController + ipAddress := gceController.CreateStaticIP(ipName) + By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", ipName, ipAddress)) }) // Platform specific cleanup @@ -635,12 +576,6 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { }) It("should conform to Ingress spec", func() { - jig.PollInterval = 5 * time.Second - // Use the randomly generated namespace name as the ip address name. - ipName := ns - // ip released when the rest of lb resources are deleted in CleanupGCEIngressController - ipAddress := gceController.CreateStaticIP(ipName) - By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", ipName, ipAddress)) conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{ framework.IngressStaticIPKey: ipName, }) @@ -651,6 +586,14 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { jig.WaitForIngress(false /*waitForNodePort*/) } }) + + It("should create ingress with pre-shared certificate", func() { + executePresharedCertTest(f, jig, ipName) + }) + + It("should create ingress with backend HTTPS", func() { + executeBacksideBacksideHTTPSTest(f, jig, ipName) + }) }) // Time: borderline 5m, slow by design @@ -704,3 +647,86 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { }) }) }) + +func executePresharedCertTest(f *framework.Framework, jig *framework.IngressTestJig, staticIPName string) { + preSharedCertName := "test-pre-shared-cert" + By(fmt.Sprintf("Creating ssl certificate %q on GCE", preSharedCertName)) + testHostname := "test.ingress.com" + cert, key, err := framework.GenerateRSACerts(testHostname, true) + Expect(err).NotTo(HaveOccurred()) + gceCloud, err := framework.GetGCECloud() + Expect(err).NotTo(HaveOccurred()) + defer func() { + // We would not be able to delete the cert until ingress controller + // cleans up the target proxy that references it. + By("Deleting ingress before deleting ssl certificate") + if jig.Ingress != nil { + jig.TryDeleteIngress() + } + By(fmt.Sprintf("Deleting ssl certificate %q on GCE", preSharedCertName)) + err := wait.Poll(framework.LoadBalancerPollInterval, framework.LoadBalancerCleanupTimeout, func() (bool, error) { + if err := gceCloud.DeleteSslCertificate(preSharedCertName); err != nil && !errors.IsNotFound(err) { + framework.Logf("Failed to delete ssl certificate %q: %v. Retrying...", preSharedCertName, err) + return false, nil + } + return true, nil + }) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to delete ssl certificate %q: %v", preSharedCertName, err)) + }() + _, err = gceCloud.CreateSslCertificate(&compute.SslCertificate{ + Name: preSharedCertName, + Certificate: string(cert), + PrivateKey: string(key), + Description: "pre-shared cert for ingress testing", + }) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to create ssl certificate %q: %v", preSharedCertName, err)) + + By("Creating an ingress referencing the pre-shared certificate") + // Create an ingress referencing this cert using pre-shared-cert annotation. + ingAnnotations := map[string]string{ + framework.IngressPreSharedCertKey: preSharedCertName, + // Disallow HTTP to save resources. This is irrelevant to the + // pre-shared cert test. + framework.IngressAllowHTTPKey: "false", + } + if staticIPName != "" { + ingAnnotations[framework.IngressStaticIPKey] = staticIPName + } + jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "pre-shared-cert"), f.Namespace.Name, ingAnnotations, map[string]string{}) + + By("Test that ingress works with the pre-shared certificate") + err = jig.WaitForIngressWithCert(true, []string{testHostname}, cert) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unexpected error while waiting for ingress: %v", err)) +} + +func executeBacksideBacksideHTTPSTest(f *framework.Framework, jig *framework.IngressTestJig, staticIPName string) { + By("Creating a set of ingress, service and deployment that have backside re-encryption configured") + deployCreated, svcCreated, ingCreated, err := jig.SetUpBacksideHTTPSIngress(f.ClientSet, f.Namespace.Name, staticIPName) + defer func() { + By("Cleaning up re-encryption ingress, service and deployment") + if errs := jig.DeleteTestResource(f.ClientSet, deployCreated, svcCreated, ingCreated); len(errs) > 0 { + framework.Failf("Failed to cleanup re-encryption ingress: %v", errs) + } + }() + Expect(err).NotTo(HaveOccurred(), "Failed to create re-encryption ingress") + + By(fmt.Sprintf("Waiting for ingress %s to come up", ingCreated.Name)) + ingIP, err := jig.WaitForIngressAddress(f.ClientSet, f.Namespace.Name, ingCreated.Name, framework.LoadBalancerPollTimeout) + Expect(err).NotTo(HaveOccurred(), "Failed to wait for ingress IP") + + By(fmt.Sprintf("Polling on address %s and verify the backend is serving HTTPS", ingIP)) + timeoutClient := &http.Client{Timeout: framework.IngressReqTimeout} + err = wait.PollImmediate(framework.LoadBalancerPollInterval, framework.LoadBalancerPollTimeout, func() (bool, error) { + resp, err := framework.SimpleGET(timeoutClient, fmt.Sprintf("http://%s", ingIP), "") + if err != nil { + framework.Logf("SimpleGET failed: %v", err) + return false, nil + } + if !strings.Contains(resp, "request_scheme=https") { + return false, fmt.Errorf("request wasn't served by HTTPS, response body: %s", resp) + } + framework.Logf("Poll succeeded, request was served by HTTPS") + return true, nil + }) + Expect(err).NotTo(HaveOccurred(), "Failed to verify backside re-encryption ingress") +} diff --git a/test/e2e/testing-manifests/ingress/pre-shared-cert/ing.yaml b/test/e2e/testing-manifests/ingress/pre-shared-cert/ing.yaml index 1de721a479e..1c3e30dc5ed 100644 --- a/test/e2e/testing-manifests/ingress/pre-shared-cert/ing.yaml +++ b/test/e2e/testing-manifests/ingress/pre-shared-cert/ing.yaml @@ -6,6 +6,10 @@ metadata: # annotations: # ingress.gcp.kubernetes.io/pre-shared-cert: "test-pre-shared-cert" spec: + # kubemci requires a default backend. + backend: + serviceName: echoheaders-https + servicePort: 80 rules: - host: test.ingress.com http: