diff --git a/test/e2e/service.go b/test/e2e/service.go index 2a532e2bf73..9aed621f193 100644 --- a/test/e2e/service.go +++ b/test/e2e/service.go @@ -1380,25 +1380,34 @@ func verifyServeHostnameServiceUp(c *client.Client, ns, host string, expectedPod return output }, } - sort.StringSlice(expectedPods).Sort() + + expectedEndpoints := sets.NewString(expectedPods...) By(fmt.Sprintf("verifying service has %d reachable backends", len(expectedPods))) for _, cmdFunc := range commands { passed := false - gotPods := []string{} + gotEndpoints := sets.NewString() + // Retry cmdFunc for a while - for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(5 * time.Second) { - pods := strings.Split(strings.TrimSpace(cmdFunc()), "\n") - // Uniq pods before the sort because inserting them into a set - // (which is implemented using dicts) can re-order them. - gotPods = sets.NewString(pods...).List() - if api.Semantic.DeepEqual(gotPods, expectedPods) { + for start := time.Now(); time.Since(start) < kubeProxyLagTimeout; time.Sleep(5 * time.Second) { + for _, endpoint := range strings.Split(cmdFunc(), "\n") { + trimmedEp := strings.TrimSpace(endpoint) + if trimmedEp != "" { + gotEndpoints.Insert(trimmedEp) + } + } + if expectedEndpoints.Equal(gotEndpoints) { passed = true break } - Logf("Waiting for expected pods for %s: %v, got: %v", serviceIP, expectedPods, gotPods) + Logf("Unable to reach the following endpoints of service %s: %v", serviceIP, expectedEndpoints.Difference(gotEndpoints)) } if !passed { - return fmt.Errorf("service verification failed for: %s, expected %v, got %v", serviceIP, expectedPods, gotPods) + // Sort the lists so they're easier to visually diff. + exp := expectedEndpoints.List() + got := gotEndpoints.List() + sort.StringSlice(exp).Sort() + sort.StringSlice(got).Sort() + return fmt.Errorf("service verification failed for: %s\nexpected %v\nreceived %v", serviceIP, exp, got) } } return nil diff --git a/test/e2e/util.go b/test/e2e/util.go index 05fcb6f0f9d..bd4236e33a3 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -1351,6 +1351,7 @@ func (b kubectlBuilder) withStdinReader(reader io.Reader) *kubectlBuilder { func (b kubectlBuilder) execOrDie() string { str, err := b.exec() + Logf("stdout: %q", str) Expect(err).NotTo(HaveOccurred()) return str } @@ -1377,7 +1378,6 @@ func (b kubectlBuilder) exec() (string, error) { b.cmd.Process.Kill() return "", fmt.Errorf("Timed out waiting for command %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr) } - Logf("stdout: %q", stdout.String()) Logf("stderr: %q", stderr.String()) // TODO: trimspace should be unnecessary after switching to use kubectl binary directly return strings.TrimSpace(stdout.String()), nil