From bd0691d15915e35303c867d98d74f3e89aafeb75 Mon Sep 17 00:00:00 2001
From: Matt Matejczyk <mmat@google.com>
Date: Fri, 6 Sep 2019 13:05:01 +0200
Subject: [PATCH] Create LoadBalancer in nginx ingress tests

This should fix the nginx ingress test in private clusters.
See https://k8s-testgrid.appspot.com/sig-scalability-experiments#gce-private-cluster-correctness

Ref. https://github.com/kubernetes/kubernetes/issues/77538
---
 test/e2e/framework/ingress/ingress_utils.go   | 29 ++++++++++++++-----
 .../testing-manifests/ingress/nginx/rc.yaml   |  1 +
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/test/e2e/framework/ingress/ingress_utils.go b/test/e2e/framework/ingress/ingress_utils.go
index 74fd2284fe4..6c5d8c214b6 100644
--- a/test/e2e/framework/ingress/ingress_utils.go
+++ b/test/e2e/framework/ingress/ingress_utils.go
@@ -827,15 +827,30 @@ func (j *TestJig) GetDistinctResponseFromIngress() (sets.String, error) {
 
 // NginxIngressController manages implementation details of Ingress on Nginx.
 type NginxIngressController struct {
-	Ns         string
-	rc         *v1.ReplicationController
-	pod        *v1.Pod
-	Client     clientset.Interface
-	externalIP string
+	Ns     string
+	rc     *v1.ReplicationController
+	pod    *v1.Pod
+	Client clientset.Interface
 }
 
 // Init initializes the NginxIngressController
 func (cont *NginxIngressController) Init() {
+	// Set up a LoadBalancer service in front of nginx ingress controller and pass it via
+	// --publish-service flag (see <IngressManifestPath>/nginx/rc.yaml) to make it work in private
+	// clusters, i.e. clusters where nodes don't have public IPs.
+	framework.Logf("Creating load balancer service for nginx ingress controller")
+	serviceJig := e2eservice.NewTestJig(cont.Client, "nginx-ingress-lb")
+	serviceJig.CreateTCPServiceOrFail(cont.Ns, func(svc *v1.Service) {
+		svc.Spec.Type = v1.ServiceTypeLoadBalancer
+		svc.Spec.Selector = map[string]string{"k8s-app": "nginx-ingress-lb"}
+		svc.Spec.Ports = []v1.ServicePort{
+			{Name: "http", Port: 80},
+			{Name: "https", Port: 443},
+			{Name: "stats", Port: 18080}}
+	})
+	svc := serviceJig.WaitForLoadBalancerOrFail(cont.Ns, "nginx-ingress-lb", e2eservice.LoadBalancerCreateTimeoutDefault)
+	serviceJig.SanityCheckService(svc, v1.ServiceTypeLoadBalancer)
+
 	read := func(file string) string {
 		return string(testfiles.ReadOrDie(filepath.Join(IngressManifestPath, "nginx", file)))
 	}
@@ -855,9 +870,7 @@ func (cont *NginxIngressController) Init() {
 		framework.Failf("Failed to find nginx ingress controller pods with selector %v", sel)
 	}
 	cont.pod = &pods.Items[0]
-	cont.externalIP, err = framework.GetHostExternalAddress(cont.Client, cont.pod)
-	framework.ExpectNoError(err)
-	framework.Logf("ingress controller running in pod %v on ip %v", cont.pod.Name, cont.externalIP)
+	framework.Logf("ingress controller running in pod %v", cont.pod.Name)
 }
 
 func generateBacksideHTTPSIngressSpec(ns string) *networkingv1beta1.Ingress {
diff --git a/test/e2e/testing-manifests/ingress/nginx/rc.yaml b/test/e2e/testing-manifests/ingress/nginx/rc.yaml
index b80b0837a17..302c032a856 100644
--- a/test/e2e/testing-manifests/ingress/nginx/rc.yaml
+++ b/test/e2e/testing-manifests/ingress/nginx/rc.yaml
@@ -48,3 +48,4 @@ spec:
         args:
         - /nginx-ingress-controller
         - --default-backend-service=kube-system/default-http-backend
+        - --publish-service=$(POD_NAMESPACE)/nginx-ingress-lb