Merge pull request #31172 from bprashanth/nginxe2e

Automatic merge from submit-queue

Nginx ingress e2es

Runs the basic ingress conformance suite: https://github.com/kubernetes/kubernetes/blob/master/test/e2e/ingress_utils.go#L85

test/e2e/ingress.go

@@ -77,7 +77,7 @@ var _ = framework.KubeDescribe("Loadbalancing: L7 [Feature:Ingress]", func() {
 	//
 	// Slow by design ~10m for each "It" block dominated by loadbalancer setup time
 	// TODO: write similar tests for nginx, haproxy and AWS Ingress.
-	framework.KubeDescribe("GCE [Feature: Ingress]", func() {
+	framework.KubeDescribe("GCE [Slow] [Feature: Ingress]", func() {
 		var gceController *GCEIngressController
 
 		// Platform specific setup
@@ -144,4 +144,52 @@ var _ = framework.KubeDescribe("Loadbalancing: L7 [Feature:Ingress]", func() {
 		// TODO: Implement a multizone e2e that verifies traffic reaches each
 		// zone based on pod labels.
 	})
+
+	// Time: borderline 5m, slow by design
+	framework.KubeDescribe("Nginx [Slow] [Feature: Ingress]", func() {
+		var nginxController *NginxIngressController
+
+		BeforeEach(func() {
+			framework.SkipUnlessProviderIs("gce", "gke")
+			By("Initializing nginx controller")
+			jig.class = "nginx"
+			nginxController = &NginxIngressController{ns: ns, c: jig.client}
+
+			// TODO: This test may fail on other platforms. We can simply skip it
+			// but we want to allow easy testing where a user might've hand
+			// configured firewalls.
+			if framework.ProviderIs("gce", "gke") {
+				ExpectNoError(gcloudCreate("firewall-rules", fmt.Sprintf("ingress-80-443-%v", ns), framework.TestContext.CloudConfig.ProjectID, "--allow", "tcp:80,tcp:443", "--network", "e2e"))
+			} else {
+				framework.Logf("WARNING: Not running on GCE/GKE, cannot create firewall rules for :80, :443. Assuming traffic can reach the external ips of all nodes in cluster on those ports.")
+			}
+
+			nginxController.init()
+		})
+
+		AfterEach(func() {
+			if framework.ProviderIs("gce", "gke") {
+				ExpectNoError(gcloudDelete("firewall-rules", fmt.Sprintf("ingress-80-443-%v", ns), framework.TestContext.CloudConfig.ProjectID))
+			}
+			if CurrentGinkgoTestDescription().Failed {
+				describeIng(ns)
+			}
+			if jig.ing == nil {
+				By("No ingress created, no cleanup necessary")
+				return
+			}
+			By("Deleting ingress")
+			jig.deleteIngress()
+		})
+
+		It("should conform to Ingress spec", func() {
+			conformanceTests = createComformanceTests(jig, ns)
+			for _, t := range conformanceTests {
+				By(t.entryLog)
+				t.execute()
+				By(t.exitLog)
+				jig.waitForIngress()
+			}
+		})
+	})
 })

test/e2e/ingress_utils.go

@@ -43,6 +43,7 @@ import (
 	apierrs "k8s.io/kubernetes/pkg/api/errors"
 	"k8s.io/kubernetes/pkg/apis/extensions"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
+	"k8s.io/kubernetes/pkg/labels"
 	"k8s.io/kubernetes/pkg/runtime"
 	utilexec "k8s.io/kubernetes/pkg/util/exec"
 	utilnet "k8s.io/kubernetes/pkg/util/net"
@@ -58,6 +59,9 @@ import (
 const (
 	rsaBits  = 2048
 	validFor = 365 * 24 * time.Hour
+
+	// Ingress class annotation defined in ingress repository.
+	ingressClass = "kubernetes.io/ingress.class"
 )
 
 type testJig struct {
@@ -65,6 +69,10 @@ type testJig struct {
 	rootCAs map[string][]byte
 	address string
 	ing     *extensions.Ingress
+	// class is the value of the annotation keyed under
+	// `kubernetes.io/ingress.class`. It's added to all ingresses created by
+	// this jig.
+	class string
 }
 
 type conformanceTests struct {
@@ -529,6 +537,7 @@ func gcloudDelete(resource, name, project string, args ...string) error {
 func gcloudCreate(resource, name, project string, args ...string) error {
 	framework.Logf("Creating %v in project %v: %v", resource, project, name)
 	argsList := append([]string{"compute", resource, "create", name, fmt.Sprintf("--project=%v", project)}, args...)
+	framework.Logf("Running command: gcloud %+v", strings.Join(argsList, " "))
 	output, err := exec.Command("gcloud", argsList...).CombinedOutput()
 	if err != nil {
 		framework.Logf("Error creating %v, output: %v\nerror: %+v", resource, string(output), err)
@@ -557,8 +566,9 @@ func (j *testJig) createIngress(manifestPath, ns string, ingAnnotations map[stri
 	}
 	j.ing = ingFromManifest(mkpath("ing.yaml"))
 	j.ing.Namespace = ns
-	if len(ingAnnotations) != 0 {
+	j.ing.Annotations = map[string]string{ingressClass: j.class}
-		j.ing.Annotations = ingAnnotations
+	for k, v := range ingAnnotations {
+		j.ing.Annotations[k] = v
 	}
 	framework.Logf(fmt.Sprintf("creating" + j.ing.Name + " ingress"))
 	var err error
@@ -718,3 +728,37 @@ type GCEIngressController struct {
 func newTestJig(c *client.Client) *testJig {
 	return &testJig{client: c, rootCAs: map[string][]byte{}}
 }
+
+// NginxIngressController manages implementation details of Ingress on Nginx.
+type NginxIngressController struct {
+	ns         string
+	rc         *api.ReplicationController
+	pod        *api.Pod
+	c          *client.Client
+	externalIP string
+}
+
+func (cont *NginxIngressController) init() {
+	mkpath := func(file string) string {
+		return filepath.Join(framework.TestContext.RepoRoot, ingressManifestPath, "nginx", file)
+	}
+	framework.Logf("initializing nginx ingress controller")
+	framework.RunKubectlOrDie("create", "-f", mkpath("rc.yaml"), fmt.Sprintf("--namespace=%v", cont.ns))
+
+	rc, err := cont.c.ReplicationControllers(cont.ns).Get("nginx-ingress-controller")
+	ExpectNoError(err)
+	cont.rc = rc
+
+	framework.Logf("waiting for pods with label %v", rc.Spec.Selector)
+	sel := labels.SelectorFromSet(labels.Set(rc.Spec.Selector))
+	ExpectNoError(framework.WaitForPodsWithLabelRunning(cont.c, cont.ns, sel))
+	pods, err := cont.c.Pods(cont.ns).List(api.ListOptions{LabelSelector: sel})
+	ExpectNoError(err)
+	if len(pods.Items) == 0 {
+		framework.Failf("Failed to find nginx ingress controller pods with selector %v", sel)
+	}
+	cont.pod = &pods.Items[0]
+	cont.externalIP, err = framework.GetHostExternalAddress(cont.c, cont.pod)
+	ExpectNoError(err)
+	framework.Logf("ingress controller running in pod %v on ip %v", cont.pod.Name, cont.externalIP)
+}

test/e2e/testing-manifests/ingress/nginx/rc.yaml

@@ -0,0 +1,50 @@
+# nginx ingress controller RC
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: nginx-ingress-controller
+  labels:
+    k8s-app: nginx-ingress-lb
+spec:
+  replicas: 1
+  selector:
+    k8s-app: nginx-ingress-lb
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-ingress-lb
+        name: nginx-ingress-lb
+    spec:
+      terminationGracePeriodSeconds: 0
+      containers:
+      - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        name: nginx-ingress-lb
+        # use downward API
+        env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+        ports:
+        - containerPort: 80
+          hostPort: 80
+        - containerPort: 443
+          hostPort: 443
+        # we expose 18080 to access nginx stats in url /nginx-status
+        # this is optional
+        - containerPort: 18080
+          hostPort: 18080
+        args:
+        - /nginx-ingress-controller
+        - --default-backend-service=kube-system/default-http-backend