diff --git a/test/e2e/framework/ingress/ingress_utils.go b/test/e2e/framework/ingress/ingress_utils.go index 6292aa9b241..5fa377d3040 100644 --- a/test/e2e/framework/ingress/ingress_utils.go +++ b/test/e2e/framework/ingress/ingress_utils.go @@ -667,12 +667,26 @@ func (j *TestJig) pollIngressWithCert(ing *extensions.Ingress, address string, k } // WaitForIngress waits for the Ingress to get an address. +// WaitForIngress returns when it gets the first 200 response func (j *TestJig) WaitForIngress(waitForNodePort bool) { if err := j.WaitForGivenIngressWithTimeout(j.Ingress, waitForNodePort, framework.LoadBalancerPollTimeout); err != nil { framework.Failf("error in waiting for ingress to get an address: %s", err) } } +// WaitForIngressToStable waits for the LB return 100 consecutive 200 responses. +func (j *TestJig) WaitForIngressToStable() { + if err := wait.Poll(10*time.Second, framework.LoadBalancerCreateTimeoutDefault, func() (bool, error) { + _, err := j.GetDistinctResponseFromIngress() + if err != nil { + return false, nil + } + return true, nil + }); err != nil { + framework.Failf("error in waiting for ingress to stablize: %v", err) + } +} + // WaitForGivenIngressWithTimeout waits till the ingress acquires an IP, // then waits for its hosts/urls to respond to a protocol check (either // http or https). If waitForNodePort is true, the NodePort of the Service diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go index 1fc1180a08b..2b4470f0bbc 100644 --- a/test/e2e/network/ingress.go +++ b/test/e2e/network/ingress.go @@ -529,9 +529,10 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { _, err = f.ClientSet.CoreV1().Services(ns).Update(&svc) Expect(err).NotTo(HaveOccurred()) } - wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { - return gceController.BackendServiceUsingIG(jig.GetServicePorts(true)) + err = wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + return gceController.BackendServiceUsingIG(jig.GetServicePorts(false)) }) + Expect(err).NotTo(HaveOccurred(), "Expect backend service to target IG, but failed to observe") jig.WaitForIngress(true) By("Switch backend service to use NEG") @@ -542,9 +543,10 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { _, err = f.ClientSet.CoreV1().Services(ns).Update(&svc) Expect(err).NotTo(HaveOccurred()) } - wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + err = wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { return gceController.BackendServiceUsingNEG(jig.GetServicePorts(false)) }) + Expect(err).NotTo(HaveOccurred(), "Expect backend service to target NEG, but failed to observe") jig.WaitForIngress(true) }) @@ -556,7 +558,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { svcPorts := jig.GetServicePorts(false) usingNEG, err := gceController.BackendServiceUsingNEG(svcPorts) Expect(err).NotTo(HaveOccurred()) - Expect(usingNEG).To(BeTrue()) + Expect(usingNEG).To(BeTrue(), "Expect backend service to be using NEG. But not.") // ClusterIP ServicePorts have no NodePort for _, sp := range svcPorts { @@ -574,18 +576,21 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { _, err = f.ClientSet.AppsV1().Deployments(ns).UpdateScale(name, scale) Expect(err).NotTo(HaveOccurred()) } - wait.Poll(10*time.Second, negUpdateTimeout, func() (bool, error) { + err = wait.Poll(10*time.Second, negUpdateTimeout, func() (bool, error) { res, err := jig.GetDistinctResponseFromIngress() if err != nil { return false, nil } + framework.Logf("Expecting %d backends, got %d", num, res.Len()) return res.Len() == num, nil }) + Expect(err).NotTo(HaveOccurred()) } By("Create a basic HTTP ingress using NEG") jig.CreateIngress(filepath.Join(ingress.IngressManifestPath, "neg"), ns, map[string]string{}, map[string]string{}) jig.WaitForIngress(true) + jig.WaitForIngressToStable() usingNEG, err := gceController.BackendServiceUsingNEG(jig.GetServicePorts(false)) Expect(err).NotTo(HaveOccurred()) Expect(usingNEG).To(BeTrue()) @@ -611,6 +616,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { By("Create a basic HTTP ingress using NEG") jig.CreateIngress(filepath.Join(ingress.IngressManifestPath, "neg"), ns, map[string]string{}, map[string]string{}) jig.WaitForIngress(true) + jig.WaitForIngressToStable() usingNEG, err := gceController.BackendServiceUsingNEG(jig.GetServicePorts(false)) Expect(err).NotTo(HaveOccurred()) Expect(usingNEG).To(BeTrue()) @@ -621,13 +627,15 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { scale.Spec.Replicas = int32(replicas) _, err = f.ClientSet.AppsV1().Deployments(ns).UpdateScale(name, scale) Expect(err).NotTo(HaveOccurred()) - wait.Poll(10*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + + err = wait.Poll(10*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { res, err := jig.GetDistinctResponseFromIngress() if err != nil { return false, nil } return res.Len() == replicas, nil }) + Expect(err).NotTo(HaveOccurred()) By("Trigger rolling update and observe service disruption") deploy, err := f.ClientSet.AppsV1().Deployments(ns).Get(name, metav1.GetOptions{}) @@ -637,7 +645,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { deploy.Spec.Template.Spec.TerminationGracePeriodSeconds = &gracePeriod _, err = f.ClientSet.AppsV1().Deployments(ns).Update(deploy) Expect(err).NotTo(HaveOccurred()) - wait.Poll(10*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + err = wait.Poll(10*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { res, err := jig.GetDistinctResponseFromIngress() Expect(err).NotTo(HaveOccurred()) deploy, err := f.ClientSet.AppsV1().Deployments(ns).Get(name, metav1.GetOptions{}) @@ -655,6 +663,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { return false, nil } }) + Expect(err).NotTo(HaveOccurred()) }) It("should sync endpoints for both Ingress-referenced NEG and standalone NEG", func() { @@ -669,7 +678,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { _, err = f.ClientSet.AppsV1().Deployments(ns).UpdateScale(name, scale) Expect(err).NotTo(HaveOccurred()) } - wait.Poll(10*time.Second, negUpdateTimeout, func() (bool, error) { + err = wait.Poll(10*time.Second, negUpdateTimeout, func() (bool, error) { svc, err := f.ClientSet.CoreV1().Services(ns).Get(name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -716,6 +725,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { return true, nil }) + Expect(err).NotTo(HaveOccurred()) } By("Create a basic HTTP ingress using NEG") @@ -1108,7 +1118,7 @@ func detectHttpVersionAndSchemeTest(f *framework.Framework, jig *ingress.TestJig } func detectNegAnnotation(f *framework.Framework, jig *ingress.TestJig, gceController *gce.GCEIngressController, ns, name string, negs int) { - wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) { + if err := wait.Poll(5*time.Second, negUpdateTimeout, func() (bool, error) { svc, err := f.ClientSet.CoreV1().Services(ns).Get(name, metav1.GetOptions{}) if err != nil { return false, nil @@ -1150,5 +1160,7 @@ func detectNegAnnotation(f *framework.Framework, jig *ingress.TestJig, gceContro } return gceController.BackendServiceUsingNEG(jig.GetServicePorts(false)) - }) + }); err != nil { + Expect(err).NotTo(HaveOccurred()) + } }