From a3242482877338214f0459ca4bc25b81cb28de87 Mon Sep 17 00:00:00 2001 From: Minhan Xia Date: Tue, 17 Oct 2017 16:57:07 -0700 Subject: [PATCH 1/5] add ingress conformance test for NEG --- test/e2e/e2e.go | 2 +- test/e2e/framework/ingress_utils.go | 129 +++++++++++++++++++++++++--- test/e2e/network/ingress.go | 57 +++++++++++- test/e2e/upgrades/ingress.go | 2 +- 4 files changed, 174 insertions(+), 16 deletions(-) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index b7d937d7b1b..ec305a17e9d 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -79,7 +79,7 @@ func setupProviderConfig() error { managedZones = []string{zone} } - gceAlphaFeatureGate, err := gcecloud.NewAlphaFeatureGate([]string{}) + gceAlphaFeatureGate, err := gcecloud.NewAlphaFeatureGate([]string{gcecloud.AlphaFeatureNetworkEndpointGroup}) if err != nil { glog.Errorf("Encountered error for creating alpha feature gate: %v", err) } diff --git a/test/e2e/framework/ingress_utils.go b/test/e2e/framework/ingress_utils.go index bbb5223ebec..666fafe60b4 100644 --- a/test/e2e/framework/ingress_utils.go +++ b/test/e2e/framework/ingress_utils.go @@ -126,7 +126,7 @@ type IngressConformanceTests struct { // CreateIngressComformanceTests generates an slice of sequential test cases: // a simple http ingress, ingress with HTTPS, ingress HTTPS with a modified hostname, // ingress https with a modified URLMap -func CreateIngressComformanceTests(jig *IngressTestJig, ns string) []IngressConformanceTests { +func CreateIngressComformanceTests(jig *IngressTestJig, ns string, annotations map[string]string) []IngressConformanceTests { manifestPath := filepath.Join(IngressManifestPath, "http") // These constants match the manifests used in IngressManifestPath tlsHost := "foo.bar.com" @@ -138,7 +138,7 @@ func CreateIngressComformanceTests(jig *IngressTestJig, ns string) []IngressConf return []IngressConformanceTests{ { fmt.Sprintf("should create a basic HTTP ingress"), - func() { jig.CreateIngress(manifestPath, ns, map[string]string{}) }, + func() { jig.CreateIngress(manifestPath, ns, annotations, annotations) }, fmt.Sprintf("waiting for urls on basic HTTP ingress"), }, { @@ -591,6 +591,39 @@ func (cont *GCEIngressController) deleteInstanceGroup(del bool) (msg string) { return msg } +func (cont *GCEIngressController) deleteNetworkEndpointGroup(del bool) (msg string) { + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) + // TODO: E2E cloudprovider has only 1 zone, but the cluster can have many. + // We need to poll on all NEGs across all zones. + negList, err := gceCloud.ListNetworkEndpointGroup(cont.Cloud.Zone) + if err != nil { + if cont.isHTTPErrorCode(err, http.StatusNotFound) { + return msg + } + // Do not return error as NEG is still alpha. + Logf("Failed to list network endpoint group: %v", err) + return msg + } + if len(negList) == 0 { + return msg + } + for _, neg := range negList { + if !cont.canDeleteNEG(neg.Name, neg.CreationTimestamp, del) { + continue + } + if del { + Logf("Deleting network-endpoint-group: %s", neg.Name) + if err := gceCloud.DeleteNetworkEndpointGroup(neg.Name, cont.Cloud.Zone); err != nil && + !cont.isHTTPErrorCode(err, http.StatusNotFound) { + msg += fmt.Sprintf("Failed to delete network endpoint group %v\n", neg.Name) + } + } else { + msg += fmt.Sprintf("%v (network-endpoint-group)\n", neg.Name) + } + } + return msg +} + // canDelete returns true if either the name ends in a suffix matching this // controller's UID, or the creationTimestamp exceeds the maxAge and del is set // to true. Always returns false if the name doesn't match that we expect for @@ -617,6 +650,28 @@ func (cont *GCEIngressController) canDelete(resourceName, creationTimestamp stri if !delOldResources { return false } + return canDeleteWithTimestamp(resourceName, creationTimestamp) +} + +// canDeleteNEG returns true if either the name contains this controller's UID, +// or the creationTimestamp exceeds the maxAge and del is set to true. +func (cont *GCEIngressController) canDeleteNEG(resourceName, creationTimestamp string, delOldResources bool) bool { + if !strings.HasPrefix(resourceName, "k8s") { + return false + } + + if strings.Contains(resourceName, cont.UID) { + return true + } + + if !delOldResources { + return false + } + + return canDeleteWithTimestamp(resourceName, creationTimestamp) +} + +func canDeleteWithTimestamp(resourceName, creationTimestamp string) bool { createdTime, err := time.Parse(time.RFC3339, creationTimestamp) if err != nil { Logf("WARNING: Failed to parse creation timestamp %v for %v: %v", creationTimestamp, resourceName, err) @@ -667,6 +722,44 @@ func (cont *GCEIngressController) isHTTPErrorCode(err error, code int) bool { return ok && apiErr.Code == code } +// BackendServiceUsingNEG returns true only if all global backend service with matching nodeports pointing to NEG as backend +func (cont *GCEIngressController) BackendServiceUsingNEG(nodeports []string) (bool, error) { + return cont.backendMode(nodeports, "networkEndpointGroups") +} + +// BackendServiceUsingIG returns true only if all global backend service with matching nodeports pointing to IG as backend +func (cont *GCEIngressController) BackendServiceUsingIG(nodeports []string) (bool, error) { + return cont.backendMode(nodeports, "instanceGroups") +} + +func (cont *GCEIngressController) backendMode(nodeports []string, keyword string) (bool, error) { + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) + beList, err := gceCloud.ListGlobalBackendServices() + if err != nil { + return false, fmt.Errorf("failed to list backend services: %v", err) + } + + matchingBackendService := 0 + for _, bs := range beList.Items { + match := false + for _, np := range nodeports { + // Warning: This assumes backend service naming convention includes nodeport in the name + if strings.Contains(bs.Name, np) { + match = true + matchingBackendService += 1 + } + } + if match { + for _, be := range bs.Backends { + if !strings.Contains(be.Group, keyword) { + return false, nil + } + } + } + } + return matchingBackendService == len(nodeports), nil +} + // Cleanup cleans up cloud resources. // If del is false, it simply reports existing resources without deleting them. // If dle is true, it deletes resources it finds acceptable (see canDelete func). @@ -683,6 +776,7 @@ func (cont *GCEIngressController) Cleanup(del bool) error { errMsg += cont.deleteHTTPHealthCheck(del) errMsg += cont.deleteInstanceGroup(del) + errMsg += cont.deleteNetworkEndpointGroup(del) errMsg += cont.deleteFirewallRule(del) errMsg += cont.deleteSSLCertificate(del) @@ -812,7 +906,9 @@ func GcloudComputeResourceCreate(resource, name, project string, args ...string) // Required: ing.yaml, rc.yaml, svc.yaml must exist in manifestPath // Optional: secret.yaml, ingAnnotations // If ingAnnotations is specified it will overwrite any annotations in ing.yaml -func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations map[string]string) { +// If svcAnnotations is specified it will overwrite any annotations in svc.yaml +func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations map[string]string, svcAnnotations map[string]string) { + var err error mkpath := func(file string) string { return filepath.Join(TestContext.RepoRoot, manifestPath, file) } @@ -822,13 +918,22 @@ func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations m Logf("creating service") RunKubectlOrDie("create", "-f", mkpath("svc.yaml"), fmt.Sprintf("--namespace=%v", ns)) + if len(svcAnnotations) > 0 { + svcList, err := j.Client.CoreV1().Services(ns).List(metav1.ListOptions{}) + ExpectNoError(err) + for _, svc := range svcList.Items { + svc.Annotations = svcAnnotations + _, err = j.Client.CoreV1().Services(ns).Update(&svc) + ExpectNoError(err) + } + } if exists, _ := utilfile.FileExists(mkpath("secret.yaml")); exists { Logf("creating secret") RunKubectlOrDie("create", "-f", mkpath("secret.yaml"), fmt.Sprintf("--namespace=%v", ns)) } Logf("Parsing ingress from %v", filepath.Join(manifestPath, "ing.yaml")) - var err error + j.Ingress, err = manifest.IngressFromManifest(filepath.Join(manifestPath, "ing.yaml")) ExpectNoError(err) j.Ingress.Namespace = ns @@ -954,14 +1059,16 @@ func (j *IngressTestJig) pollServiceNodePort(ns, name string, port int) { ExpectNoError(PollURL(u, "", 30*time.Second, j.PollInterval, &http.Client{Timeout: IngressReqTimeout}, false)) } -// GetIngressNodePorts returns all related backend services' nodePorts. +// GetIngressNodePorts returns related backend services' nodePorts. // Current GCE ingress controller allows traffic to the default HTTP backend -// by default, so retrieve its nodePort as well. -func (j *IngressTestJig) GetIngressNodePorts() []string { +// by default, so retrieve its nodePort if includeDefaultBackend is true. +func (j *IngressTestJig) GetIngressNodePorts(includeDefaultBackend bool) []string { nodePorts := []string{} - defaultSvc, err := j.Client.Core().Services(metav1.NamespaceSystem).Get(defaultBackendName, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - nodePorts = append(nodePorts, strconv.Itoa(int(defaultSvc.Spec.Ports[0].NodePort))) + if includeDefaultBackend { + defaultSvc, err := j.Client.Core().Services(metav1.NamespaceSystem).Get(defaultBackendName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + nodePorts = append(nodePorts, strconv.Itoa(int(defaultSvc.Spec.Ports[0].NodePort))) + } backendSvcs := []string{} if j.Ingress.Spec.Backend != nil { @@ -982,7 +1089,7 @@ func (j *IngressTestJig) GetIngressNodePorts() []string { // ConstructFirewallForIngress returns the expected GCE firewall rule for the ingress resource func (j *IngressTestJig) ConstructFirewallForIngress(gceController *GCEIngressController, nodeTags []string) *compute.Firewall { - nodePorts := j.GetIngressNodePorts() + nodePorts := j.GetIngressNodePorts(true) fw := compute.Firewall{} fw.Name = gceController.GetFirewallRuleName() diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go index de16b0cf6e8..f408b325d74 100644 --- a/test/e2e/network/ingress.go +++ b/test/e2e/network/ingress.go @@ -30,6 +30,10 @@ import ( . "github.com/onsi/gomega" ) +const ( + NEGAnnotation = "alpha.cloud.google.com/load-balancer-neg" +) + var _ = SIGDescribe("Loadbalancing: L7", func() { defer GinkgoRecover() var ( @@ -96,7 +100,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { }) It("should conform to Ingress spec", func() { - conformanceTests = framework.CreateIngressComformanceTests(jig, ns) + conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{}) for _, t := range conformanceTests { By(t.EntryLog) t.Execute() @@ -113,7 +117,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns, map[string]string{ "kubernetes.io/ingress.global-static-ip-name": ns, "kubernetes.io/ingress.allow-http": "false", - }) + }, map[string]string{}) By("waiting for Ingress to come up with ip: " + ip) httpClient := framework.BuildInsecureClient(framework.IngressReqTimeout) @@ -149,6 +153,53 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { // TODO: Implement a multizone e2e that verifies traffic reaches each // zone based on pod labels. }) + Describe("GCE [Slow] [Feature:NEG]", func() { + var gceController *framework.GCEIngressController + + // Platform specific setup + BeforeEach(func() { + framework.SkipUnlessProviderIs("gce", "gke") + By("Initializing gce controller") + gceController = &framework.GCEIngressController{ + Ns: ns, + Client: jig.Client, + Cloud: framework.TestContext.CloudConfig, + } + gceController.Init() + }) + + // Platform specific cleanup + AfterEach(func() { + if CurrentGinkgoTestDescription().Failed { + framework.DescribeIng(ns) + } + if jig.Ingress == nil { + By("No ingress created, no cleanup necessary") + return + } + By("Deleting ingress") + jig.TryDeleteIngress() + + By("Cleaning up cloud resources") + framework.CleanupGCEIngressController(gceController) + }) + + It("should conform to Ingress spec", func() { + jig.PollInterval = 5 * time.Second + conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{ + NEGAnnotation: "true", + }) + for _, t := range conformanceTests { + By(t.EntryLog) + t.Execute() + By(t.ExitLog) + jig.WaitForIngress(true) + usingNeg, err := gceController.BackendServiceUsingNEG(jig.GetIngressNodePorts(false)) + Expect(err).NotTo(HaveOccurred()) + Expect(usingNeg).To(BeTrue()) + } + }) + }) // Time: borderline 5m, slow by design Describe("[Slow] Nginx", func() { @@ -191,7 +242,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { // Poll more frequently to reduce e2e completion time. // This test runs in presubmit. jig.PollInterval = 5 * time.Second - conformanceTests = framework.CreateIngressComformanceTests(jig, ns) + conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{}) for _, t := range conformanceTests { By(t.EntryLog) t.Execute() diff --git a/test/e2e/upgrades/ingress.go b/test/e2e/upgrades/ingress.go index 7ca9e3b7d13..d2d8c71369a 100644 --- a/test/e2e/upgrades/ingress.go +++ b/test/e2e/upgrades/ingress.go @@ -69,7 +69,7 @@ func (t *IngressUpgradeTest) Setup(f *framework.Framework) { jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns.Name, map[string]string{ "kubernetes.io/ingress.global-static-ip-name": t.ipName, "kubernetes.io/ingress.allow-http": "false", - }) + }, map[string]string{}) By("waiting for Ingress to come up with ip: " + t.ip) framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, jig.PollInterval, t.httpClient, false)) From 59e0fca5a5ff9fd286dbca65cc0ad57657471747 Mon Sep 17 00:00:00 2001 From: Minhan Xia Date: Wed, 18 Oct 2017 15:38:05 -0700 Subject: [PATCH 2/5] add e2e test for switching ingress mode between NEG and IG --- test/e2e/network/ingress.go | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go index f408b325d74..b1fb5d80895 100644 --- a/test/e2e/network/ingress.go +++ b/test/e2e/network/ingress.go @@ -22,7 +22,9 @@ import ( "time" rbacv1beta1 "k8s.io/api/rbac/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/serviceaccount" "k8s.io/kubernetes/test/e2e/framework" @@ -199,6 +201,42 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { Expect(usingNeg).To(BeTrue()) } }) + + It("should be able to switch between IG and NEG modes", func() { + var err error + By("Create a basic HTTP ingress using NEG") + jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "neg"), ns, map[string]string{}, map[string]string{}) + jig.WaitForIngress(true) + usingNEG, err := gceController.BackendServiceUsingNEG(jig.GetIngressNodePorts(false)) + Expect(err).NotTo(HaveOccurred()) + Expect(usingNEG).To(BeTrue()) + + By("Switch backend service to use IG") + svcList, err := f.ClientSet.CoreV1().Services(ns).List(metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred()) + for _, svc := range svcList.Items { + svc.Annotations[NEGAnnotation] = "false" + _, err = f.ClientSet.CoreV1().Services(ns).Update(&svc) + Expect(err).NotTo(HaveOccurred()) + } + wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + return gceController.BackendServiceUsingIG(jig.GetIngressNodePorts(true)) + }) + jig.WaitForIngress(true) + + By("Switch backend service to use NEG") + svcList, err = f.ClientSet.CoreV1().Services(ns).List(metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred()) + for _, svc := range svcList.Items { + svc.Annotations[NEGAnnotation] = "true" + _, err = f.ClientSet.CoreV1().Services(ns).Update(&svc) + Expect(err).NotTo(HaveOccurred()) + } + wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + return gceController.BackendServiceUsingNEG(jig.GetIngressNodePorts(false)) + }) + jig.WaitForIngress(true) + }) }) // Time: borderline 5m, slow by design From 23ff1bb7047782ba7e30963b148b28b7266f6db2 Mon Sep 17 00:00:00 2001 From: Minhan Xia Date: Wed, 18 Oct 2017 16:44:48 -0700 Subject: [PATCH 3/5] add e2e test for syncing endpoints to NEG --- test/e2e/framework/ingress_utils.go | 22 +++++++++++++++ test/e2e/network/ingress.go | 44 ++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/test/e2e/framework/ingress_utils.go b/test/e2e/framework/ingress_utils.go index 666fafe60b4..982e244e677 100644 --- a/test/e2e/framework/ingress_utils.go +++ b/test/e2e/framework/ingress_utils.go @@ -1104,6 +1104,28 @@ func (j *IngressTestJig) ConstructFirewallForIngress(gceController *GCEIngressCo return &fw } +// GetDistinctResponseFromIngress tries GET call to the ingress VIP and return all distinct responses. +func (j *IngressTestJig) GetDistinctResponseFromIngress() (sets.String, error) { + // Wait for the loadbalancer IP. + address, err := WaitForIngressAddress(j.Client, j.Ingress.Namespace, j.Ingress.Name, LoadBalancerPollTimeout) + if err != nil { + Failf("Ingress failed to acquire an IP address within %v", LoadBalancerPollTimeout) + } + responses := sets.NewString() + timeoutClient := &http.Client{Timeout: IngressReqTimeout} + + for i := 0; i < 100; i++ { + url := fmt.Sprintf("http://%v", address) + res, err := SimpleGET(timeoutClient, url, "") + if err != nil { + Logf("Failed to GET %q. Got responses: %q: %v", url, res, err) + return responses, err + } + responses.Insert(res) + } + return responses, nil +} + func (cont *GCEIngressController) getL7AddonUID() (string, error) { Logf("Retrieving UID from config map: %v/%v", metav1.NamespaceSystem, uidConfigMap) cm, err := cont.Client.Core().ConfigMaps(metav1.NamespaceSystem).Get(uidConfigMap, metav1.GetOptions{}) diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go index b1fb5d80895..a25ce072197 100644 --- a/test/e2e/network/ingress.go +++ b/test/e2e/network/ingress.go @@ -33,7 +33,8 @@ import ( ) const ( - NEGAnnotation = "alpha.cloud.google.com/load-balancer-neg" + NEGAnnotation = "alpha.cloud.google.com/load-balancer-neg" + NEGUpdateTimeout = 2 * time.Minute ) var _ = SIGDescribe("Loadbalancing: L7", func() { @@ -237,6 +238,47 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { }) jig.WaitForIngress(true) }) + + It("should sync endpoints to NEG", func() { + name := "hostname" + scaleAndValidateNEG := func(num int) { + scale, err := f.ClientSet.ExtensionsV1beta1().Deployments(ns).GetScale(name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + if scale.Spec.Replicas != int32(num) { + scale.Spec.Replicas = int32(num) + _, err = f.ClientSet.ExtensionsV1beta1().Deployments(ns).UpdateScale(name, scale) + Expect(err).NotTo(HaveOccurred()) + } + wait.Poll(5*time.Second, NEGUpdateTimeout, func() (bool, error) { + res, err := jig.GetDistinctResponseFromIngress() + if err != nil { + return false, err + } + return res.Len() == num, err + }) + } + + By("Create a basic HTTP ingress using NEG") + jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "neg"), ns, map[string]string{}, map[string]string{}) + jig.WaitForIngress(true) + usingNEG, err := gceController.BackendServiceUsingNEG(jig.GetIngressNodePorts(false)) + Expect(err).NotTo(HaveOccurred()) + Expect(usingNEG).To(BeTrue()) + // initial replicas number is 1 + scaleAndValidateNEG(1) + + By("Scale up number of backends to 5") + scaleAndValidateNEG(5) + + By("Scale down number of backends to 3") + scaleAndValidateNEG(3) + + By("Scale up number of backends to 6") + scaleAndValidateNEG(6) + + By("Scale down number of backends to 2") + scaleAndValidateNEG(3) + }) }) // Time: borderline 5m, slow by design From 7bdc55a0875e1fd63889be6edbb969e3b1c58bfe Mon Sep 17 00:00:00 2001 From: Minhan Xia Date: Wed, 18 Oct 2017 16:49:33 -0700 Subject: [PATCH 4/5] add manifest for NEG e2e test --- test/e2e/testing-manifests/ingress/neg/ing.yaml | 8 ++++++++ test/e2e/testing-manifests/ingress/neg/rc.yaml | 17 +++++++++++++++++ test/e2e/testing-manifests/ingress/neg/svc.yaml | 15 +++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 test/e2e/testing-manifests/ingress/neg/ing.yaml create mode 100644 test/e2e/testing-manifests/ingress/neg/rc.yaml create mode 100644 test/e2e/testing-manifests/ingress/neg/svc.yaml diff --git a/test/e2e/testing-manifests/ingress/neg/ing.yaml b/test/e2e/testing-manifests/ingress/neg/ing.yaml new file mode 100644 index 00000000000..e266d98516e --- /dev/null +++ b/test/e2e/testing-manifests/ingress/neg/ing.yaml @@ -0,0 +1,8 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: hostname +spec: + backend: + serviceName: hostname + servicePort: 80 \ No newline at end of file diff --git a/test/e2e/testing-manifests/ingress/neg/rc.yaml b/test/e2e/testing-manifests/ingress/neg/rc.yaml new file mode 100644 index 00000000000..717b61b0541 --- /dev/null +++ b/test/e2e/testing-manifests/ingress/neg/rc.yaml @@ -0,0 +1,17 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + run: hostname + name: hostname +spec: + template: + metadata: + labels: + run: hostname + spec: + containers: + - image: gcr.io/kubernetes-e2e-test-images/serve-hostname-amd64:1.1 + imagePullPolicy: IfNotPresent + name: hostname + terminationGracePeriodSeconds: 120 \ No newline at end of file diff --git a/test/e2e/testing-manifests/ingress/neg/svc.yaml b/test/e2e/testing-manifests/ingress/neg/svc.yaml new file mode 100644 index 00000000000..3a0be750a21 --- /dev/null +++ b/test/e2e/testing-manifests/ingress/neg/svc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: hostname + annotations: + alpha.cloud.google.com/load-balancer-neg: "true" +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 9376 + selector: + run: hostname + sessionAffinity: None + type: NodePort \ No newline at end of file From 4a24ebfad59a3d022d40e0ca8b8535b1af82c67f Mon Sep 17 00:00:00 2001 From: Minhan Xia Date: Mon, 23 Oct 2017 12:23:55 -0700 Subject: [PATCH 5/5] add e2e test for rolling update backend with NEG --- test/e2e/network/ingress.go | 52 +++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go index a25ce072197..45577f1131e 100644 --- a/test/e2e/network/ingress.go +++ b/test/e2e/network/ingress.go @@ -279,6 +279,58 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { By("Scale down number of backends to 2") scaleAndValidateNEG(3) }) + + It("rolling update backend pods should not cause service disruption", func() { + name := "hostname" + replicas := 8 + By("Create a basic HTTP ingress using NEG") + jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "neg"), ns, map[string]string{}, map[string]string{}) + jig.WaitForIngress(true) + usingNEG, err := gceController.BackendServiceUsingNEG(jig.GetIngressNodePorts(false)) + Expect(err).NotTo(HaveOccurred()) + Expect(usingNEG).To(BeTrue()) + + By(fmt.Sprintf("Scale backend replicas to %d", replicas)) + scale, err := f.ClientSet.ExtensionsV1beta1().Deployments(ns).GetScale(name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + scale.Spec.Replicas = int32(replicas) + _, err = f.ClientSet.ExtensionsV1beta1().Deployments(ns).UpdateScale(name, scale) + Expect(err).NotTo(HaveOccurred()) + wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + res, err := jig.GetDistinctResponseFromIngress() + if err != nil { + return false, err + } + return res.Len() == replicas, err + }) + + By("Trigger rolling update and observe service disruption") + deploy, err := f.ClientSet.ExtensionsV1beta1().Deployments(ns).Get(name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + // trigger by changing graceful termination period to 60 seconds + gracePeriod := int64(60) + deploy.Spec.Template.Spec.TerminationGracePeriodSeconds = &gracePeriod + _, err = f.ClientSet.ExtensionsV1beta1().Deployments(ns).Update(deploy) + Expect(err).NotTo(HaveOccurred()) + wait.Poll(30*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + res, err := jig.GetDistinctResponseFromIngress() + Expect(err).NotTo(HaveOccurred()) + deploy, err := f.ClientSet.ExtensionsV1beta1().Deployments(ns).Get(name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + if int(deploy.Status.UpdatedReplicas) == replicas { + if res.Len() == replicas { + return true, nil + } else { + framework.Logf("Expecting %d different responses, but got %d.", replicas, res.Len()) + return false, nil + } + + } else { + framework.Logf("Waiting for rolling update to finished. Keep sending traffic.") + return false, nil + } + }) + }) }) // Time: borderline 5m, slow by design