From 65d76f507c549514fe992016f3a7492c4da04031 Mon Sep 17 00:00:00 2001 From: bprashanth Date: Mon, 22 Aug 2016 15:15:12 -0700 Subject: [PATCH] Nginx ingress e2es --- test/e2e/ingress.go | 50 ++++++++++++++++++- test/e2e/ingress_utils.go | 48 +++++++++++++++++- .../testing-manifests/ingress/nginx/rc.yaml | 50 +++++++++++++++++++ 3 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 test/e2e/testing-manifests/ingress/nginx/rc.yaml diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 05bf1919a35..045890d9dbc 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -77,7 +77,7 @@ var _ = framework.KubeDescribe("Loadbalancing: L7 [Feature:Ingress]", func() { // // Slow by design ~10m for each "It" block dominated by loadbalancer setup time // TODO: write similar tests for nginx, haproxy and AWS Ingress. - framework.KubeDescribe("GCE [Feature: Ingress]", func() { + framework.KubeDescribe("GCE [Slow] [Feature: Ingress]", func() { var gceController *GCEIngressController // Platform specific setup @@ -144,4 +144,52 @@ var _ = framework.KubeDescribe("Loadbalancing: L7 [Feature:Ingress]", func() { // TODO: Implement a multizone e2e that verifies traffic reaches each // zone based on pod labels. }) + + // Time: borderline 5m, slow by design + framework.KubeDescribe("Nginx [Slow] [Feature: Ingress]", func() { + var nginxController *NginxIngressController + + BeforeEach(func() { + framework.SkipUnlessProviderIs("gce", "gke") + By("Initializing nginx controller") + jig.class = "nginx" + nginxController = &NginxIngressController{ns: ns, c: jig.client} + + // TODO: This test may fail on other platforms. We can simply skip it + // but we want to allow easy testing where a user might've hand + // configured firewalls. + if framework.ProviderIs("gce", "gke") { + ExpectNoError(gcloudCreate("firewall-rules", fmt.Sprintf("ingress-80-443-%v", ns), framework.TestContext.CloudConfig.ProjectID, "--allow", "tcp:80,tcp:443", "--network", "e2e")) + } else { + framework.Logf("WARNING: Not running on GCE/GKE, cannot create firewall rules for :80, :443. Assuming traffic can reach the external ips of all nodes in cluster on those ports.") + } + + nginxController.init() + }) + + AfterEach(func() { + if framework.ProviderIs("gce", "gke") { + ExpectNoError(gcloudDelete("firewall-rules", fmt.Sprintf("ingress-80-443-%v", ns), framework.TestContext.CloudConfig.ProjectID)) + } + if CurrentGinkgoTestDescription().Failed { + describeIng(ns) + } + if jig.ing == nil { + By("No ingress created, no cleanup necessary") + return + } + By("Deleting ingress") + jig.deleteIngress() + }) + + It("should conform to Ingress spec", func() { + conformanceTests = createComformanceTests(jig, ns) + for _, t := range conformanceTests { + By(t.entryLog) + t.execute() + By(t.exitLog) + jig.waitForIngress() + } + }) + }) }) diff --git a/test/e2e/ingress_utils.go b/test/e2e/ingress_utils.go index 09abd4dbb72..4dba6f591c3 100644 --- a/test/e2e/ingress_utils.go +++ b/test/e2e/ingress_utils.go @@ -43,6 +43,7 @@ import ( apierrs "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/apis/extensions" client "k8s.io/kubernetes/pkg/client/unversioned" + "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/runtime" utilexec "k8s.io/kubernetes/pkg/util/exec" utilnet "k8s.io/kubernetes/pkg/util/net" @@ -58,6 +59,9 @@ import ( const ( rsaBits = 2048 validFor = 365 * 24 * time.Hour + + // Ingress class annotation defined in ingress repository. + ingressClass = "kubernetes.io/ingress.class" ) type testJig struct { @@ -65,6 +69,10 @@ type testJig struct { rootCAs map[string][]byte address string ing *extensions.Ingress + // class is the value of the annotation keyed under + // `kubernetes.io/ingress.class`. It's added to all ingresses created by + // this jig. + class string } type conformanceTests struct { @@ -529,6 +537,7 @@ func gcloudDelete(resource, name, project string, args ...string) error { func gcloudCreate(resource, name, project string, args ...string) error { framework.Logf("Creating %v in project %v: %v", resource, project, name) argsList := append([]string{"compute", resource, "create", name, fmt.Sprintf("--project=%v", project)}, args...) + framework.Logf("Running command: gcloud %+v", strings.Join(argsList, " ")) output, err := exec.Command("gcloud", argsList...).CombinedOutput() if err != nil { framework.Logf("Error creating %v, output: %v\nerror: %+v", resource, string(output), err) @@ -557,8 +566,9 @@ func (j *testJig) createIngress(manifestPath, ns string, ingAnnotations map[stri } j.ing = ingFromManifest(mkpath("ing.yaml")) j.ing.Namespace = ns - if len(ingAnnotations) != 0 { - j.ing.Annotations = ingAnnotations + j.ing.Annotations = map[string]string{ingressClass: j.class} + for k, v := range ingAnnotations { + j.ing.Annotations[k] = v } framework.Logf(fmt.Sprintf("creating" + j.ing.Name + " ingress")) var err error @@ -718,3 +728,37 @@ type GCEIngressController struct { func newTestJig(c *client.Client) *testJig { return &testJig{client: c, rootCAs: map[string][]byte{}} } + +// NginxIngressController manages implementation details of Ingress on Nginx. +type NginxIngressController struct { + ns string + rc *api.ReplicationController + pod *api.Pod + c *client.Client + externalIP string +} + +func (cont *NginxIngressController) init() { + mkpath := func(file string) string { + return filepath.Join(framework.TestContext.RepoRoot, ingressManifestPath, "nginx", file) + } + framework.Logf("initializing nginx ingress controller") + framework.RunKubectlOrDie("create", "-f", mkpath("rc.yaml"), fmt.Sprintf("--namespace=%v", cont.ns)) + + rc, err := cont.c.ReplicationControllers(cont.ns).Get("nginx-ingress-controller") + ExpectNoError(err) + cont.rc = rc + + framework.Logf("waiting for pods with label %v", rc.Spec.Selector) + sel := labels.SelectorFromSet(labels.Set(rc.Spec.Selector)) + ExpectNoError(framework.WaitForPodsWithLabelRunning(cont.c, cont.ns, sel)) + pods, err := cont.c.Pods(cont.ns).List(api.ListOptions{LabelSelector: sel}) + ExpectNoError(err) + if len(pods.Items) == 0 { + framework.Failf("Failed to find nginx ingress controller pods with selector %v", sel) + } + cont.pod = &pods.Items[0] + cont.externalIP, err = framework.GetHostExternalAddress(cont.c, cont.pod) + ExpectNoError(err) + framework.Logf("ingress controller running in pod %v on ip %v", cont.pod.Name, cont.externalIP) +} diff --git a/test/e2e/testing-manifests/ingress/nginx/rc.yaml b/test/e2e/testing-manifests/ingress/nginx/rc.yaml new file mode 100644 index 00000000000..dfcd9accdf2 --- /dev/null +++ b/test/e2e/testing-manifests/ingress/nginx/rc.yaml @@ -0,0 +1,50 @@ +# nginx ingress controller RC +apiVersion: v1 +kind: ReplicationController +metadata: + name: nginx-ingress-controller + labels: + k8s-app: nginx-ingress-lb +spec: + replicas: 1 + selector: + k8s-app: nginx-ingress-lb + template: + metadata: + labels: + k8s-app: nginx-ingress-lb + name: nginx-ingress-lb + spec: + terminationGracePeriodSeconds: 0 + containers: + - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3 + livenessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + name: nginx-ingress-lb + # use downward API + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 80 + hostPort: 80 + - containerPort: 443 + hostPort: 443 + # we expose 18080 to access nginx stats in url /nginx-status + # this is optional + - containerPort: 18080 + hostPort: 18080 + args: + - /nginx-ingress-controller + - --default-backend-service=kube-system/default-http-backend