diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 3f9054e11a7..590bcfd12b7 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -3161,7 +3161,10 @@ func DeleteResourceAndWaitForGC(c clientset.Interface, kind schema.GroupKind, ns terminatePodTime := time.Since(startTime) - deleteTime Logf("Terminating %v %s pods took: %v", kind, name, terminatePodTime) - err = waitForPodsGone(ps, interval, 10*time.Minute) + // In gce, at any point, small percentage of nodes can disappear for + // ~10 minutes due to hostError. 20 minutes should be long enough to + // restart VM in that case and delete the pod. + err = waitForPodsGone(ps, interval, 20*time.Minute) if err != nil { return fmt.Errorf("error while waiting for pods gone %s: %v", name, err) } @@ -3191,21 +3194,30 @@ func waitForPodsInactive(ps *testutils.PodStore, interval, timeout time.Duration if err == wait.ErrWaitTimeout { for _, pod := range activePods { - Logf("Pod %q running on %q is still active", pod.ObjectMeta.Name, pod.Spec.NodeName) + Logf("ERROR: Pod %q running on %q is still active", pod.Name, pod.Spec.NodeName) } - return fmt.Errorf("there are %d active pods. E.g. %q on node %q", len(activePods), activePods[0].ObjectMeta.Name, activePods[0].Spec.NodeName) + return fmt.Errorf("there are %d active pods. E.g. %q on node %q", len(activePods), activePods[0].Name, activePods[0].Spec.NodeName) } return err } // waitForPodsGone waits until there are no pods left in the PodStore. func waitForPodsGone(ps *testutils.PodStore, interval, timeout time.Duration) error { - return wait.PollImmediate(interval, timeout, func() (bool, error) { - if pods := ps.List(); len(pods) == 0 { + var pods []*v1.Pod + err := wait.PollImmediate(interval, timeout, func() (bool, error) { + if pods = ps.List(); len(pods) == 0 { return true, nil } return false, nil }) + + if err == wait.ErrWaitTimeout { + for _, pod := range pods { + Logf("ERROR: Pod %q still exists. Node: %q", pod.Name, pod.Spec.NodeName) + } + return fmt.Errorf("there are %d pods left. E.g. %q on node %q", len(pods), pods[0].Name, pods[0].Spec.NodeName) + } + return err } func WaitForPodsReady(c clientset.Interface, ns, name string, minReadySeconds int) error {