add ingress conformance test for NEG

test/e2e/e2e.go

@@ -79,7 +79,7 @@ func setupProviderConfig() error {
 			managedZones = []string{zone}
 		}
 
-		gceAlphaFeatureGate, err := gcecloud.NewAlphaFeatureGate([]string{})
+		gceAlphaFeatureGate, err := gcecloud.NewAlphaFeatureGate([]string{gcecloud.AlphaFeatureNetworkEndpointGroup})
 		if err != nil {
 			glog.Errorf("Encountered error for creating alpha feature gate: %v", err)
 		}

test/e2e/framework/ingress_utils.go

@@ -126,7 +126,7 @@ type IngressConformanceTests struct {
 // CreateIngressComformanceTests generates an slice of sequential test cases:
 // a simple http ingress, ingress with HTTPS, ingress HTTPS with a modified hostname,
 // ingress https with a modified URLMap
-func CreateIngressComformanceTests(jig *IngressTestJig, ns string) []IngressConformanceTests {
+func CreateIngressComformanceTests(jig *IngressTestJig, ns string, annotations map[string]string) []IngressConformanceTests {
 	manifestPath := filepath.Join(IngressManifestPath, "http")
 	// These constants match the manifests used in IngressManifestPath
 	tlsHost := "foo.bar.com"
@@ -138,7 +138,7 @@ func CreateIngressComformanceTests(jig *IngressTestJig, ns string) []IngressConf
 	return []IngressConformanceTests{
 		{
 			fmt.Sprintf("should create a basic HTTP ingress"),
-			func() { jig.CreateIngress(manifestPath, ns, map[string]string{}) },
+			func() { jig.CreateIngress(manifestPath, ns, annotations, annotations) },
 			fmt.Sprintf("waiting for urls on basic HTTP ingress"),
 		},
 		{
@@ -591,6 +591,39 @@ func (cont *GCEIngressController) deleteInstanceGroup(del bool) (msg string) {
 	return msg
 }
 
+func (cont *GCEIngressController) deleteNetworkEndpointGroup(del bool) (msg string) {
+	gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
+	// TODO: E2E cloudprovider has only 1 zone, but the cluster can have many.
+	// We need to poll on all NEGs across all zones.
+	negList, err := gceCloud.ListNetworkEndpointGroup(cont.Cloud.Zone)
+	if err != nil {
+		if cont.isHTTPErrorCode(err, http.StatusNotFound) {
+			return msg
+		}
+		// Do not return error as NEG is still alpha.
+		Logf("Failed to list network endpoint group: %v", err)
+		return msg
+	}
+	if len(negList) == 0 {
+		return msg
+	}
+	for _, neg := range negList {
+		if !cont.canDeleteNEG(neg.Name, neg.CreationTimestamp, del) {
+			continue
+		}
+		if del {
+			Logf("Deleting network-endpoint-group: %s", neg.Name)
+			if err := gceCloud.DeleteNetworkEndpointGroup(neg.Name, cont.Cloud.Zone); err != nil &&
+				!cont.isHTTPErrorCode(err, http.StatusNotFound) {
+				msg += fmt.Sprintf("Failed to delete network endpoint group %v\n", neg.Name)
+			}
+		} else {
+			msg += fmt.Sprintf("%v (network-endpoint-group)\n", neg.Name)
+		}
+	}
+	return msg
+}
+
 // canDelete returns true if either the name ends in a suffix matching this
 // controller's UID, or the creationTimestamp exceeds the maxAge and del is set
 // to true. Always returns false if the name doesn't match that we expect for
@@ -617,6 +650,28 @@ func (cont *GCEIngressController) canDelete(resourceName, creationTimestamp stri
 	if !delOldResources {
 		return false
 	}
+	return canDeleteWithTimestamp(resourceName, creationTimestamp)
+}
+
+// canDeleteNEG returns true if either the name contains this controller's UID,
+// or the creationTimestamp exceeds the maxAge and del is set to true.
+func (cont *GCEIngressController) canDeleteNEG(resourceName, creationTimestamp string, delOldResources bool) bool {
+	if !strings.HasPrefix(resourceName, "k8s") {
+		return false
+	}
+
+	if strings.Contains(resourceName, cont.UID) {
+		return true
+	}
+
+	if !delOldResources {
+		return false
+	}
+
+	return canDeleteWithTimestamp(resourceName, creationTimestamp)
+}
+
+func canDeleteWithTimestamp(resourceName, creationTimestamp string) bool {
 	createdTime, err := time.Parse(time.RFC3339, creationTimestamp)
 	if err != nil {
 		Logf("WARNING: Failed to parse creation timestamp %v for %v: %v", creationTimestamp, resourceName, err)
@@ -667,6 +722,44 @@ func (cont *GCEIngressController) isHTTPErrorCode(err error, code int) bool {
 	return ok && apiErr.Code == code
 }
 
+// BackendServiceUsingNEG returns true only if all global backend service with matching nodeports pointing to NEG as backend
+func (cont *GCEIngressController) BackendServiceUsingNEG(nodeports []string) (bool, error) {
+	return cont.backendMode(nodeports, "networkEndpointGroups")
+}
+
+// BackendServiceUsingIG returns true only if all global backend service with matching nodeports pointing to IG as backend
+func (cont *GCEIngressController) BackendServiceUsingIG(nodeports []string) (bool, error) {
+	return cont.backendMode(nodeports, "instanceGroups")
+}
+
+func (cont *GCEIngressController) backendMode(nodeports []string, keyword string) (bool, error) {
+	gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
+	beList, err := gceCloud.ListGlobalBackendServices()
+	if err != nil {
+		return false, fmt.Errorf("failed to list backend services: %v", err)
+	}
+
+	matchingBackendService := 0
+	for _, bs := range beList.Items {
+		match := false
+		for _, np := range nodeports {
+			// Warning: This assumes backend service naming convention includes nodeport in the name
+			if strings.Contains(bs.Name, np) {
+				match = true
+				matchingBackendService += 1
+			}
+		}
+		if match {
+			for _, be := range bs.Backends {
+				if !strings.Contains(be.Group, keyword) {
+					return false, nil
+				}
+			}
+		}
+	}
+	return matchingBackendService == len(nodeports), nil
+}
+
 // Cleanup cleans up cloud resources.
 // If del is false, it simply reports existing resources without deleting them.
 // If dle is true, it deletes resources it finds acceptable (see canDelete func).
@@ -683,6 +776,7 @@ func (cont *GCEIngressController) Cleanup(del bool) error {
 	errMsg += cont.deleteHTTPHealthCheck(del)
 
 	errMsg += cont.deleteInstanceGroup(del)
+	errMsg += cont.deleteNetworkEndpointGroup(del)
 	errMsg += cont.deleteFirewallRule(del)
 	errMsg += cont.deleteSSLCertificate(del)
 
@@ -812,7 +906,9 @@ func GcloudComputeResourceCreate(resource, name, project string, args ...string)
 // Required: ing.yaml, rc.yaml, svc.yaml must exist in manifestPath
 // Optional: secret.yaml, ingAnnotations
 // If ingAnnotations is specified it will overwrite any annotations in ing.yaml
-func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations map[string]string) {
+// If svcAnnotations is specified it will overwrite any annotations in svc.yaml
+func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations map[string]string, svcAnnotations map[string]string) {
+	var err error
 	mkpath := func(file string) string {
 		return filepath.Join(TestContext.RepoRoot, manifestPath, file)
 	}
@@ -822,13 +918,22 @@ func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations m
 
 	Logf("creating service")
 	RunKubectlOrDie("create", "-f", mkpath("svc.yaml"), fmt.Sprintf("--namespace=%v", ns))
+	if len(svcAnnotations) > 0 {
+		svcList, err := j.Client.CoreV1().Services(ns).List(metav1.ListOptions{})
+		ExpectNoError(err)
+		for _, svc := range svcList.Items {
+			svc.Annotations = svcAnnotations
+			_, err = j.Client.CoreV1().Services(ns).Update(&svc)
+			ExpectNoError(err)
+		}
+	}
 
 	if exists, _ := utilfile.FileExists(mkpath("secret.yaml")); exists {
 		Logf("creating secret")
 		RunKubectlOrDie("create", "-f", mkpath("secret.yaml"), fmt.Sprintf("--namespace=%v", ns))
 	}
 	Logf("Parsing ingress from %v", filepath.Join(manifestPath, "ing.yaml"))
-	var err error
+
 	j.Ingress, err = manifest.IngressFromManifest(filepath.Join(manifestPath, "ing.yaml"))
 	ExpectNoError(err)
 	j.Ingress.Namespace = ns
@@ -954,14 +1059,16 @@ func (j *IngressTestJig) pollServiceNodePort(ns, name string, port int) {
 	ExpectNoError(PollURL(u, "", 30*time.Second, j.PollInterval, &http.Client{Timeout: IngressReqTimeout}, false))
 }
 
-// GetIngressNodePorts returns all related backend services' nodePorts.
+// GetIngressNodePorts returns related backend services' nodePorts.
 // Current GCE ingress controller allows traffic to the default HTTP backend
-// by default, so retrieve its nodePort as well.
+// by default, so retrieve its nodePort if includeDefaultBackend is true.
-func (j *IngressTestJig) GetIngressNodePorts() []string {
+func (j *IngressTestJig) GetIngressNodePorts(includeDefaultBackend bool) []string {
 	nodePorts := []string{}
+	if includeDefaultBackend {
 		defaultSvc, err := j.Client.Core().Services(metav1.NamespaceSystem).Get(defaultBackendName, metav1.GetOptions{})
 		Expect(err).NotTo(HaveOccurred())
 		nodePorts = append(nodePorts, strconv.Itoa(int(defaultSvc.Spec.Ports[0].NodePort)))
+	}
 
 	backendSvcs := []string{}
 	if j.Ingress.Spec.Backend != nil {
@@ -982,7 +1089,7 @@ func (j *IngressTestJig) GetIngressNodePorts() []string {
 
 // ConstructFirewallForIngress returns the expected GCE firewall rule for the ingress resource
 func (j *IngressTestJig) ConstructFirewallForIngress(gceController *GCEIngressController, nodeTags []string) *compute.Firewall {
-	nodePorts := j.GetIngressNodePorts()
+	nodePorts := j.GetIngressNodePorts(true)
 
 	fw := compute.Firewall{}
 	fw.Name = gceController.GetFirewallRuleName()

test/e2e/network/ingress.go

@@ -30,6 +30,10 @@ import (
 	. "github.com/onsi/gomega"
 )
 
+const (
+	NEGAnnotation = "alpha.cloud.google.com/load-balancer-neg"
+)
+
 var _ = SIGDescribe("Loadbalancing: L7", func() {
 	defer GinkgoRecover()
 	var (
@@ -96,7 +100,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 		})
 
 		It("should conform to Ingress spec", func() {
-			conformanceTests = framework.CreateIngressComformanceTests(jig, ns)
+			conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{})
 			for _, t := range conformanceTests {
 				By(t.EntryLog)
 				t.Execute()
@@ -113,7 +117,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 			jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns, map[string]string{
 				"kubernetes.io/ingress.global-static-ip-name": ns,
 				"kubernetes.io/ingress.allow-http":            "false",
-			})
+			}, map[string]string{})
 
 			By("waiting for Ingress to come up with ip: " + ip)
 			httpClient := framework.BuildInsecureClient(framework.IngressReqTimeout)
@@ -149,6 +153,53 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 		// TODO: Implement a multizone e2e that verifies traffic reaches each
 		// zone based on pod labels.
 	})
+	Describe("GCE [Slow] [Feature:NEG]", func() {
+		var gceController *framework.GCEIngressController
+
+		// Platform specific setup
+		BeforeEach(func() {
+			framework.SkipUnlessProviderIs("gce", "gke")
+			By("Initializing gce controller")
+			gceController = &framework.GCEIngressController{
+				Ns:     ns,
+				Client: jig.Client,
+				Cloud:  framework.TestContext.CloudConfig,
+			}
+			gceController.Init()
+		})
+
+		// Platform specific cleanup
+		AfterEach(func() {
+			if CurrentGinkgoTestDescription().Failed {
+				framework.DescribeIng(ns)
+			}
+			if jig.Ingress == nil {
+				By("No ingress created, no cleanup necessary")
+				return
+			}
+			By("Deleting ingress")
+			jig.TryDeleteIngress()
+
+			By("Cleaning up cloud resources")
+			framework.CleanupGCEIngressController(gceController)
+		})
+
+		It("should conform to Ingress spec", func() {
+			jig.PollInterval = 5 * time.Second
+			conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{
+				NEGAnnotation: "true",
+			})
+			for _, t := range conformanceTests {
+				By(t.EntryLog)
+				t.Execute()
+				By(t.ExitLog)
+				jig.WaitForIngress(true)
+				usingNeg, err := gceController.BackendServiceUsingNEG(jig.GetIngressNodePorts(false))
+				Expect(err).NotTo(HaveOccurred())
+				Expect(usingNeg).To(BeTrue())
+			}
+		})
+	})
 
 	// Time: borderline 5m, slow by design
 	Describe("[Slow] Nginx", func() {
@@ -191,7 +242,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 			// Poll more frequently to reduce e2e completion time.
 			// This test runs in presubmit.
 			jig.PollInterval = 5 * time.Second
-			conformanceTests = framework.CreateIngressComformanceTests(jig, ns)
+			conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{})
 			for _, t := range conformanceTests {
 				By(t.EntryLog)
 				t.Execute()

test/e2e/upgrades/ingress.go

@@ -69,7 +69,7 @@ func (t *IngressUpgradeTest) Setup(f *framework.Framework) {
 	jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns.Name, map[string]string{
 		"kubernetes.io/ingress.global-static-ip-name": t.ipName,
 		"kubernetes.io/ingress.allow-http":            "false",
-	})
+	}, map[string]string{})
 
 	By("waiting for Ingress to come up with ip: " + t.ip)
 	framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, jig.PollInterval, t.httpClient, false))