diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 8c09775fe30..892703f5713 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -136,6 +136,12 @@ func (f *Framework) WaitForPodRunning(podName string) error { return waitForPodRunningInNamespace(f.Client, podName, f.Namespace.Name) } +// WaitForPodRunningSlow waits for the pod to run in the namespace. +// It has a longer timeout then WaitForPodRunning (util.slowPodStartTimeout). +func (f *Framework) WaitForPodRunningSlow(podName string) error { + return waitForPodRunningInNamespaceSlow(f.Client, podName, f.Namespace.Name) +} + // Runs the given pod and verifies that the output of exact container matches the desired output. func (f *Framework) TestContainerOutput(scenarioName string, pod *api.Pod, containerIndex int, expectedOutput []string) { testContainerOutput(scenarioName, f.Client, pod, containerIndex, expectedOutput, f.Namespace.Name) diff --git a/test/e2e/pd.go b/test/e2e/pd.go index e5a88cae7c3..9270650ae8c 100644 --- a/test/e2e/pd.go +++ b/test/e2e/pd.go @@ -91,7 +91,7 @@ var _ = Describe("Pod Disks", func() { _, err = podClient.Create(host0Pod) expectNoError(err, fmt.Sprintf("Failed to create host0Pod: %v", err)) - expectNoError(framework.WaitForPodRunning(host0Pod.Name)) + expectNoError(framework.WaitForPodRunningSlow(host0Pod.Name)) testFile := "/testpd1/tracker" testFileContents := fmt.Sprintf("%v", mathrand.Int()) @@ -106,7 +106,7 @@ var _ = Describe("Pod Disks", func() { _, err = podClient.Create(host1Pod) expectNoError(err, "Failed to create host1Pod") - expectNoError(framework.WaitForPodRunning(host1Pod.Name)) + expectNoError(framework.WaitForPodRunningSlow(host1Pod.Name)) v, err := framework.ReadFileViaContainer(host1Pod.Name, containerName, testFile) expectNoError(err) @@ -150,7 +150,7 @@ var _ = Describe("Pod Disks", func() { By("submitting rwPod to ensure PD is formatted") _, err = podClient.Create(rwPod) expectNoError(err, "Failed to create rwPod") - expectNoError(framework.WaitForPodRunning(rwPod.Name)) + expectNoError(framework.WaitForPodRunningSlow(rwPod.Name)) expectNoError(podClient.Delete(rwPod.Name, api.NewDeleteOptions(0)), "Failed to delete host0Pod") expectNoError(waitForPDDetach(diskName, host0Name)) @@ -162,9 +162,9 @@ var _ = Describe("Pod Disks", func() { _, err = podClient.Create(host1ROPod) expectNoError(err, "Failed to create host1ROPod") - expectNoError(framework.WaitForPodRunning(host0ROPod.Name)) + expectNoError(framework.WaitForPodRunningSlow(host0ROPod.Name)) - expectNoError(framework.WaitForPodRunning(host1ROPod.Name)) + expectNoError(framework.WaitForPodRunningSlow(host1ROPod.Name)) By("deleting host0ROPod") expectNoError(podClient.Delete(host0ROPod.Name, api.NewDeleteOptions(0)), "Failed to delete host0ROPod") @@ -204,7 +204,7 @@ var _ = Describe("Pod Disks", func() { _, err = podClient.Create(host0Pod) expectNoError(err, fmt.Sprintf("Failed to create host0Pod: %v", err)) - expectNoError(framework.WaitForPodRunning(host0Pod.Name)) + expectNoError(framework.WaitForPodRunningSlow(host0Pod.Name)) // randomly select a container and read/verify pd contents from it containerName := fmt.Sprintf("mycontainer%v", mathrand.Intn(numContainers)+1) @@ -263,7 +263,7 @@ var _ = Describe("Pod Disks", func() { _, err = podClient.Create(host0Pod) expectNoError(err, fmt.Sprintf("Failed to create host0Pod: %v", err)) - expectNoError(framework.WaitForPodRunning(host0Pod.Name)) + expectNoError(framework.WaitForPodRunningSlow(host0Pod.Name)) // Read/verify pd contents for both disks from container verifyPDContentsViaContainer(framework, host0Pod.Name, containerName, fileAndContentToVerify) diff --git a/test/e2e/util.go b/test/e2e/util.go index aace4e3f7dc..afa61cb0b1f 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -64,6 +64,9 @@ const ( // TODO: Make this 30 seconds once #4566 is resolved. podStartTimeout = 5 * time.Minute + // Some pods can take much longer to get ready due to volume attach/detach latency. + slowPodStartTimeout = 15 * time.Minute + // How long to wait for a service endpoint to be resolvable. serviceStartTimeout = 1 * time.Minute @@ -575,8 +578,20 @@ func deleteNS(c *client.Client, namespace string, timeout time.Duration) error { return nil } +// Waits default ammount of time (podStartTimeout) for the specified pod to become running. +// Returns an error if timeout occurs first, or pod goes in to failed state. func waitForPodRunningInNamespace(c *client.Client, podName string, namespace string) error { - return waitForPodCondition(c, namespace, podName, "running", podStartTimeout, func(pod *api.Pod) (bool, error) { + return waitTimeoutForPodRunningInNamespace(c, podName, namespace, podStartTimeout) +} + +// Waits an extended ammount of time (slowPodStartTimeout) for the specified pod to become running. +// Returns an error if timeout occurs first, or pod goes in to failed state. +func waitForPodRunningInNamespaceSlow(c *client.Client, podName string, namespace string) error { + return waitTimeoutForPodRunningInNamespace(c, podName, namespace, slowPodStartTimeout) +} + +func waitTimeoutForPodRunningInNamespace(c *client.Client, podName string, namespace string, timeout time.Duration) error { + return waitForPodCondition(c, namespace, podName, "running", timeout, func(pod *api.Pod) (bool, error) { if pod.Status.Phase == api.PodRunning { Logf("Found pod '%s' on node '%s'", podName, pod.Spec.NodeName) return true, nil