diff --git a/test/e2e/petset.go b/test/e2e/petset.go index 67bca9535d7..713efe9db4a 100644 --- a/test/e2e/petset.go +++ b/test/e2e/petset.go @@ -57,6 +57,9 @@ const ( // Should the test restart petset clusters? // TODO: enable when we've productionzed bringup of pets in this e2e. restartCluster = false + + // Timeout for reads from databases running on pets. + readTimeout = 60 * time.Second ) // Time: 25m, slow by design. @@ -202,8 +205,8 @@ var _ = framework.KubeDescribe("PetSet [Slow] [Feature:PetSet]", func() { } By("Reading value under foo from member with index 2") - if v := pet.read(2, "foo"); v != "bar" { - framework.Failf("Read unexpected value %v, expected bar under key foo", v) + if err := pollReadWithTimeout(pet, 2, "foo", "bar"); err != nil { + framework.Failf("%v", err) } }) @@ -223,8 +226,8 @@ var _ = framework.KubeDescribe("PetSet [Slow] [Feature:PetSet]", func() { } By("Reading value under foo from member with index 2") - if v := pet.read(2, "foo"); v != "bar" { - framework.Failf("Read unexpected value %v, expected bar under key foo", v) + if err := pollReadWithTimeout(pet, 2, "foo", "bar"); err != nil { + framework.Failf("%v", err) } }) @@ -244,8 +247,8 @@ var _ = framework.KubeDescribe("PetSet [Slow] [Feature:PetSet]", func() { } By("Reading value under foo from member with index 2") - if v := pet.read(2, "foo"); v != "bar" { - framework.Failf("Read unexpected value %v, expected bar under key foo", v) + if err := pollReadWithTimeout(pet, 2, "foo", "bar"); err != nil { + framework.Failf("%v", err) } }) }) @@ -804,6 +807,23 @@ func ExpectNoError(err error) { Expect(err).NotTo(HaveOccurred()) } +func pollReadWithTimeout(pet petTester, petNumber int, key, expectedVal string) error { + err := wait.PollImmediate(time.Second, readTimeout, func() (bool, error) { + val := pet.read(petNumber, key) + if val == "" { + return false, nil + } else if val != expectedVal { + return false, fmt.Errorf("expected value %v, found %v", expectedVal, val) + } + return true, nil + }) + + if err == wait.ErrWaitTimeout { + return fmt.Errorf("timed out when trying to read value for key %v from pet %d", key, petNumber) + } + return err +} + func isInitialized(pod api.Pod) bool { initialized, ok := pod.Annotations[petset.PetSetInitAnnotation] if !ok {