add e2e test for standalone (exposed) NEG annotation

test/e2e/framework/ingress_utils.go

@@ -121,8 +121,9 @@ const (
 	// a single character of padding.
 	nameLenLimit = 62
 
-	NEGAnnotation    = "alpha.cloud.google.com/load-balancer-neg"
+	NEGAnnotation       = "cloud.google.com/neg"
-	NEGUpdateTimeout = 2 * time.Minute
+	NEGStatusAnnotation = "cloud.google.com/neg-status"
+	NEGUpdateTimeout    = 2 * time.Minute
 
 	InstanceGroupAnnotation = "ingress.gcp.kubernetes.io/instance-groups"
 
@@ -164,6 +165,15 @@ type IngressConformanceTests struct {
 	ExitLog  string
 }
 
+// NegStatus contains name and zone of the Network Endpoint Group
+// resources associated with this service
+type NegStatus struct {
+	// NetworkEndpointGroups returns the mapping between service port and NEG
+	// resource. key is service port, value is the name of the NEG resource.
+	NetworkEndpointGroups map[int32]string `json:"network_endpoint_groups,omitempty"`
+	Zones                 []string         `json:"zones,omitempty"`
+}
+
 // CreateIngressComformanceTests generates an slice of sequential test cases:
 // a simple http ingress, ingress with HTTPS, ingress HTTPS with a modified hostname,
 // ingress https with a modified URLMap
@@ -940,7 +950,7 @@ func (cont *GCEIngressController) backendMode(svcPorts map[string]v1.ServicePort
 		bsMatch := &compute.BackendService{}
 		// Non-NEG BackendServices are named with the Nodeport in the name.
 		// NEG BackendServices' names contain the a sha256 hash of a string.
-		negString := strings.Join([]string{uid, cont.Ns, svcName, sp.TargetPort.String()}, ";")
+		negString := strings.Join([]string{uid, cont.Ns, svcName, fmt.Sprintf("%v", sp.Port)}, ";")
 		negHash := fmt.Sprintf("%x", sha256.Sum256([]byte(negString)))[:8]
 		for _, bs := range beList {
 			if strings.Contains(bs.Name, strconv.Itoa(int(sp.NodePort))) ||

test/e2e/network/ingress.go

@@ -17,6 +17,7 @@ limitations under the License.
 package network
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"path/filepath"
@@ -31,6 +32,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authentication/serviceaccount"
@@ -647,6 +649,77 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 				}
 			})
 		})
+
+		It("should sync endpoints for both Ingress-referenced NEG and standalone NEG [Unreleased]", func() {
+			name := "hostname"
+			scaleAndValidateExposedNEG := func(num int) {
+				scale, err := f.ClientSet.ExtensionsV1beta1().Deployments(ns).GetScale(name, metav1.GetOptions{})
+				Expect(err).NotTo(HaveOccurred())
+				if scale.Spec.Replicas != int32(num) {
+					scale.Spec.Replicas = int32(num)
+					_, err = f.ClientSet.ExtensionsV1beta1().Deployments(ns).UpdateScale(name, scale)
+					Expect(err).NotTo(HaveOccurred())
+				}
+				wait.Poll(10*time.Second, framework.NEGUpdateTimeout, func() (bool, error) {
+					svc, err := f.ClientSet.CoreV1().Services(ns).Get(name, metav1.GetOptions{})
+					Expect(err).NotTo(HaveOccurred())
+
+					negs := sets.NewString()
+					var status framework.NegStatus
+					// Wait for NEG sync loop to find NEGs
+					framework.Logf("Waiting for %v, got: %+v", framework.NEGStatusAnnotation, svc.Annotations)
+					v, ok := svc.Annotations[framework.NEGStatusAnnotation]
+					if !ok {
+						return false, nil
+					}
+					err = json.Unmarshal([]byte(v), &status)
+					if err != nil {
+						framework.Logf("Error in parsing Expose NEG annotation: %v", err)
+						return false, nil
+					}
+					framework.Logf("Got %v: %v", framework.NEGStatusAnnotation, v)
+					if len(status.NetworkEndpointGroups) == 0 {
+						return false, nil
+					}
+					for _, neg := range status.NetworkEndpointGroups {
+						negs.Insert(neg)
+					}
+
+					gceCloud := gceController.Cloud.Provider.(*gcecloud.GCECloud)
+					for _, neg := range negs.List() {
+						exposedNegs, err := gceCloud.ListNetworkEndpoints(neg, gceController.Cloud.Zone, false)
+						framework.Logf("ExposedNegs: %v, err: %v", exposedNegs, err)
+						Expect(err).NotTo(HaveOccurred())
+						if len(exposedNegs) != num {
+							return false, nil
+						}
+					}
+
+					return len(negs.List()) > 0, nil
+				})
+			}
+
+			By("Create a basic HTTP ingress using NEG")
+			jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "neg-exposed"), ns, map[string]string{}, map[string]string{})
+			jig.WaitForIngress(true)
+			usingNEG, err := gceController.BackendServiceUsingNEG(jig.GetServicePorts(false))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(usingNEG).To(BeTrue())
+			// initial replicas number is 1
+			scaleAndValidateExposedNEG(1)
+
+			By("Scale up number of backends to 5")
+			scaleAndValidateExposedNEG(5)
+
+			By("Scale down number of backends to 3")
+			scaleAndValidateExposedNEG(3)
+
+			By("Scale up number of backends to 6")
+			scaleAndValidateExposedNEG(6)
+
+			By("Scale down number of backends to 2")
+			scaleAndValidateExposedNEG(3)
+		})
 	})
 
 	Describe("GCE [Slow] [Feature:kubemci]", func() {

test/e2e/testing-manifests/ingress/neg-clusterip/svc.yaml

@@ -3,7 +3,7 @@ kind: Service
 metadata:
   name: hostname
   annotations:
-    alpha.cloud.google.com/load-balancer-neg: "true"
+    cloud.google.com/neg: '{"ingress":true}'
 spec:
   ports:
   - port: 80

test/e2e/testing-manifests/ingress/neg-exposed/ing.yaml

@@ -0,0 +1,8 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: hostname
+spec:
+  backend:
+    serviceName: hostname
+    servicePort: 80

test/e2e/testing-manifests/ingress/neg-exposed/rc.yaml

@@ -0,0 +1,31 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    run: hostname
+  name: hostname
+spec:
+  template:
+    metadata:
+      labels:
+        run: hostname
+    spec:
+      containers:
+      - image: gcr.io/kubernetes-e2e-test-images/serve-hostname-amd64:1.1
+        name: host1
+        command:
+        - /bin/sh
+        - -c
+        - /serve_hostname -http=true -udp=false -port=8000
+        ports:
+        - protocol: TCP
+          containerPort: 8000
+      - image: gcr.io/kubernetes-e2e-test-images/serve-hostname-amd64:1.1
+        name: host2
+        command:
+        - /bin/sh
+        - -c
+        - /serve_hostname -http=true -udp=false -port=8080
+        ports:
+        - protocol: TCP
+          containerPort: 8080

test/e2e/testing-manifests/ingress/neg-exposed/svc.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: hostname
+  annotations:
+    cloud.google.com/neg: '{"ingress":true,"exposed_ports":{"80":{},"443":{}}}'
+spec:
+  ports:
+  - port: 80
+    name: host1
+    protocol: TCP
+    targetPort: 8000
+  - port: 443
+    name: host2
+    protocol: TCP
+    targetPort: 8080
+  selector:
+    run: hostname
+  sessionAffinity: None
+  type: ClusterIP

test/e2e/testing-manifests/ingress/neg/svc.yaml

@@ -3,7 +3,7 @@ kind: Service
 metadata:
   name: hostname
   annotations:
-    alpha.cloud.google.com/load-balancer-neg: "true"
+    cloud.google.com/neg: '{"ingress":true}'
 spec:
   ports:
   - port: 80
@@ -12,4 +12,4 @@ spec:
   selector:
     run: hostname
   sessionAffinity: None
-  type: NodePort
+  type: NodePort