Merge pull request #112503 from pohly/e2e-framework-pod-polling

e2e framework: pod polling

test/e2e/framework/pod/resource.go

@@ -18,6 +18,7 @@ package pod
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -41,7 +42,11 @@ import (
 
 // errPodCompleted is returned by PodRunning or PodContainerRunning to indicate that
 // the pod has already reached completed state.
-var errPodCompleted = fmt.Errorf("pod ran to completion")
+var errPodCompleted = FinalError(errors.New("pod ran to completion successfully"))
+
+// errPodFailed is returned by PodRunning or PodContainerRunning to indicate that
+// the pod has already reached a permanent failue state.
+var errPodFailed = FinalError(errors.New("pod failed permanently"))
 
 // LabelLogOnPodFailure can be used to mark which Pods will have their logs logged in the case of
 // a test failure. By default, if there are no Pods with this label, only the first 5 Pods will

test/e2e/framework/pod/wait.go

@@ -81,6 +81,39 @@ func TimeoutError(msg string, observedObjects ...interface{}) *timeoutError {
 	}
 }
 
+// FinalError constructs an error that indicates to a poll function that
+// polling can be stopped immediately because some permanent error has been
+// encountered that is not going to go away.
+//
+// TODO (@pohly): move this into framework once the refactoring from
+// https://github.com/kubernetes/kubernetes/pull/112043 allows it. Right now it
+// leads to circular dependencies.
+func FinalError(err error) error {
+	return &FinalErr{Err: err}
+}
+
+type FinalErr struct {
+	Err error
+}
+
+func (err *FinalErr) Error() string {
+	if err.Err != nil {
+		return fmt.Sprintf("final error: %s", err.Err.Error())
+	}
+	return "final error, exact problem unknown"
+}
+
+func (err *FinalErr) Unwrap() error {
+	return err.Err
+}
+
+// IsFinal checks whether the error was marked as final by wrapping some error
+// with FinalError.
+func IsFinal(err error) bool {
+	var finalErr *FinalErr
+	return errors.As(err, &finalErr)
+}
+
 // maybeTimeoutError returns a TimeoutError if err is a timeout. Otherwise, wrap err.
 // taskFormat and taskArgs should be the task being performed when the error occurred,
 // e.g. "waiting for pod to be running".
@@ -244,6 +277,8 @@ func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods, allowedN
 }
 
 // WaitForPodCondition waits a pods to be matched to the given condition.
+// If the condition callback returns an error that matches FinalErr (checked with IsFinal),
+// then polling aborts early.
 func WaitForPodCondition(c clientset.Interface, ns, podName, conditionDesc string, timeout time.Duration, condition podCondition) error {
 	e2elog.Logf("Waiting up to %v for pod %q in namespace %q to be %q", timeout, podName, ns, conditionDesc)
 	var (
@@ -268,8 +303,10 @@ func WaitForPodCondition(c clientset.Interface, ns, podName, conditionDesc strin
 			}
 			return true, err
 		} else if err != nil {
-			// TODO(#109732): stop polling and return the error in this case.
 			e2elog.Logf("Error evaluating pod condition %s: %v", conditionDesc, err)
+			if IsFinal(err) {
+				return false, err
+			}
 		}
 		return false, nil
 	})
@@ -403,7 +440,9 @@ func WaitTimeoutForPodRunningInNamespace(c clientset.Interface, podName, namespa
 		switch pod.Status.Phase {
 		case v1.PodRunning:
 			return true, nil
-		case v1.PodFailed, v1.PodSucceeded:
+		case v1.PodFailed:
+			return false, errPodFailed
+		case v1.PodSucceeded:
 			return false, errPodCompleted
 		}
 		return false, nil
@@ -441,7 +480,10 @@ func WaitForPodNoLongerRunningInNamespace(c clientset.Interface, podName, namesp
 func WaitTimeoutForPodReadyInNamespace(c clientset.Interface, podName, namespace string, timeout time.Duration) error {
 	return WaitForPodCondition(c, namespace, podName, "running and ready", timeout, func(pod *v1.Pod) (bool, error) {
 		switch pod.Status.Phase {
-		case v1.PodFailed, v1.PodSucceeded:
+		case v1.PodFailed:
+			e2elog.Logf("The phase of Pod %s is %s which is unexpected, pod status: %#v", pod.Name, pod.Status.Phase, pod.Status)
+			return false, errPodFailed
+		case v1.PodSucceeded:
 			e2elog.Logf("The phase of Pod %s is %s which is unexpected, pod status: %#v", pod.Name, pod.Status.Phase, pod.Status)
 			return false, errPodCompleted
 		case v1.PodRunning: