diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 18dae6704cf..1c4ced9e6c3 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -505,7 +505,10 @@ func WaitForPodsSuccess(c clientset.Interface, ns string, successPodLabels map[s podList, err := c.Core().Pods(ns).List(metav1.ListOptions{LabelSelector: successPodSelector.String()}) if err != nil { Logf("Error getting pods in namespace %q: %v", ns, err) - return false, nil + if IsRetryableAPIError(err) { + return false, nil + } + return false, err } if len(podList.Items) == 0 { Logf("Waiting for pods to enter Success, but no pods in %q match label %v", ns, successPodLabels) @@ -564,7 +567,10 @@ func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods, allowedN rcList, err := c.Core().ReplicationControllers(ns).List(metav1.ListOptions{}) if err != nil { Logf("Error getting replication controllers in namespace '%s': %v", ns, err) - return false, nil + if IsRetryableAPIError(err) { + return false, nil + } + return false, err } for _, rc := range rcList.Items { replicas += *rc.Spec.Replicas @@ -574,7 +580,10 @@ func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods, allowedN rsList, err := c.Extensions().ReplicaSets(ns).List(metav1.ListOptions{}) if err != nil { Logf("Error getting replication sets in namespace %q: %v", ns, err) - return false, nil + if IsRetryableAPIError(err) { + return false, nil + } + return false, err } for _, rs := range rsList.Items { replicas += *rs.Spec.Replicas @@ -584,7 +593,10 @@ func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods, allowedN podList, err := c.Core().Pods(ns).List(metav1.ListOptions{}) if err != nil { Logf("Error getting pods in namespace '%s': %v", ns, err) - return false, nil + if IsRetryableAPIError(err) { + return false, nil + } + return false, err } nOk := int32(0) notReady := int32(0) @@ -1435,6 +1447,9 @@ func WaitForPodToDisappear(c clientset.Interface, ns, podName string, label labe options := metav1.ListOptions{LabelSelector: label.String()} pods, err := c.Core().Pods(ns).List(options) if err != nil { + if IsRetryableAPIError(err) { + return false, nil + } return false, err } found := false @@ -1464,6 +1479,9 @@ func WaitForService(c clientset.Interface, namespace, name string, exist bool, i case apierrs.IsNotFound(err): Logf("Service %s in namespace %s disappeared.", name, namespace) return !exist, nil + case !IsRetryableAPIError(err): + Logf("Non-retryable failure while getting service.") + return false, err default: Logf("Get service %s in namespace %s failed: %v", name, namespace, err) return false, nil @@ -1488,6 +1506,9 @@ func WaitForServiceWithSelector(c clientset.Interface, namespace string, selecto case len(services.Items) == 0: Logf("Service with %s in namespace %s disappeared.", selector.String(), namespace) return !exist, nil + case !IsRetryableAPIError(err): + Logf("Non-retryable failure while listing service.") + return false, err default: Logf("List service with %s in namespace %s failed: %v", selector.String(), namespace, err) return false, nil @@ -2315,9 +2336,15 @@ func waitListSchedulableNodesOrDie(c clientset.Interface) *v1.NodeList { nodes, err = c.Core().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{ "spec.unschedulable": "false", }.AsSelector().String()}) - return err == nil, nil + if err != nil { + if IsRetryableAPIError(err) { + return false, nil + } + return false, err + } + return true, nil }) != nil { - ExpectNoError(err, "Timed out while listing nodes for e2e cluster.") + ExpectNoError(err, "Non-retryable failure or timed out while listing nodes for e2e cluster.") } return nodes } @@ -2392,8 +2419,10 @@ func WaitForAllNodesSchedulable(c clientset.Interface, timeout time.Duration) er nodes, err := c.Core().Nodes().List(opts) if err != nil { Logf("Unexpected error listing nodes: %v", err) - // Ignore the error here - it will be retried. - return false, nil + if IsRetryableAPIError(err) { + return false, nil + } + return false, err } for i := range nodes.Items { node := &nodes.Items[i] @@ -2538,7 +2567,12 @@ func NodeHasTaint(c clientset.Interface, nodeName string, taint *v1.Taint) (bool func AddOrUpdateAvoidPodOnNode(c clientset.Interface, nodeName string, avoidPods v1.AvoidPods) { err := wait.PollImmediate(Poll, SingleCallTimeout, func() (bool, error) { node, err := c.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) - ExpectNoError(err) + if err != nil { + if IsRetryableAPIError(err) { + return false, nil + } + return false, err + } taintsData, err := json.Marshal(avoidPods) ExpectNoError(err) @@ -2564,7 +2598,12 @@ func AddOrUpdateAvoidPodOnNode(c clientset.Interface, nodeName string, avoidPods func RemoveAvoidPodsOffNode(c clientset.Interface, nodeName string) { err := wait.PollImmediate(Poll, SingleCallTimeout, func() (bool, error) { node, err := c.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) - ExpectNoError(err) + if err != nil { + if IsRetryableAPIError(err) { + return false, nil + } + return false, err + } if node.Annotations == nil { return true, nil @@ -2668,7 +2707,12 @@ func WaitForPodsWithLabel(c clientset.Interface, ns string, label labels.Selecto for t := time.Now(); time.Since(t) < PodListTimeout; time.Sleep(Poll) { options := metav1.ListOptions{LabelSelector: label.String()} pods, err = c.Core().Pods(ns).List(options) - Expect(err).NotTo(HaveOccurred()) + if err != nil { + if IsRetryableAPIError(err) { + continue + } + return + } if len(pods.Items) > 0 { break } @@ -2688,7 +2732,10 @@ func WaitForPodsWithLabelRunningReady(c clientset.Interface, ns string, label la pods, err := WaitForPodsWithLabel(c, ns, label) if err != nil { Logf("Failed to list pods: %v", err) - return false, nil + if IsRetryableAPIError(err) { + return false, nil + } + return false, err } current = 0 for _, pod := range pods.Items { @@ -3012,6 +3059,9 @@ func UpdateDaemonSetWithRetries(c clientset.Interface, namespace, name string, a var updateErr error pollErr := wait.PollImmediate(10*time.Millisecond, 1*time.Minute, func() (bool, error) { if ds, err = daemonsets.Get(name, metav1.GetOptions{}); err != nil { + if IsRetryableAPIError(err) { + return false, nil + } return false, err } // Apply the update, then attempt to push it to the apiserver. @@ -3240,7 +3290,10 @@ func CreateExecPodOrFail(client clientset.Interface, ns, generateName string, tw err = wait.PollImmediate(Poll, 5*time.Minute, func() (bool, error) { retrievedPod, err := client.Core().Pods(execPod.Namespace).Get(created.Name, metav1.GetOptions{}) if err != nil { - return false, nil + if IsRetryableAPIError(err) { + return false, nil + } + return false, err } return retrievedPod.Status.Phase == v1.PodRunning, nil }) @@ -3509,6 +3562,9 @@ func AllNodesReady(c clientset.Interface, timeout time.Duration) error { // It should be OK to list unschedulable Nodes here. nodes, err := c.Core().Nodes().List(metav1.ListOptions{}) if err != nil { + if IsRetryableAPIError(err) { + return false, nil + } return false, err } for i := range nodes.Items { @@ -3555,6 +3611,9 @@ func WaitForAllNodesHealthy(c clientset.Interface, timeout time.Duration) error // It should be OK to list unschedulable Nodes here. nodes, err := c.Core().Nodes().List(metav1.ListOptions{ResourceVersion: "0"}) if err != nil { + if IsRetryableAPIError(err) { + return false, nil + } return false, err } for _, node := range nodes.Items { @@ -3897,7 +3956,10 @@ func WaitForIngressAddress(c clientset.Interface, ns, ingName string, timeout ti ipOrNameList, err := getIngressAddress(c, ns, ingName) if err != nil || len(ipOrNameList) == 0 { Logf("Waiting for Ingress %v to acquire IP, error %v", ingName, err) - return false, nil + if IsRetryableAPIError(err) { + return false, nil + } + return false, err } address = ipOrNameList[0] return true, nil @@ -3974,7 +4036,13 @@ func GetNodePortURL(client clientset.Interface, ns, name string, svcPort int) (s nodes, err = client.Core().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{ "spec.unschedulable": "false", }.AsSelector().String()}) - return err == nil, nil + if err != nil { + if IsRetryableAPIError(err) { + return false, nil + } + return false, err + } + return true, nil }) != nil { return "", err } @@ -4792,3 +4860,7 @@ func DumpDebugInfo(c clientset.Interface, ns string) { Logf("\nLast 100 log lines of %v:\n%v", s.Name, l) } } + +func IsRetryableAPIError(err error) bool { + return apierrs.IsTimeout(err) || apierrs.IsServerTimeout(err) || apierrs.IsTooManyRequests(err) || apierrs.IsInternalError(err) +}