Fixed reflector not recovering from "Too large resource version" errors with API servers 1.17.0-1.18.5

Kubernetes-commit: e1f4bfe1db8d3975bf626c2c8536dd8076aeb7d4

tools/cache/reflector.go

@@ -570,5 +570,26 @@ func isExpiredError(err error) bool {
 }
 
 func isTooLargeResourceVersionError(err error) bool {
-	return apierrors.HasStatusCause(err, metav1.CauseTypeResourceVersionTooLarge)
+	if apierrors.HasStatusCause(err, metav1.CauseTypeResourceVersionTooLarge) {
+		return true
+	}
+	// In Kubernetes 1.17.0-1.18.5, the api server doesn't set the error status cause to
+	// metav1.CauseTypeResourceVersionTooLarge to indicate that the requested minimum resource
+	// version is larger than the largest currently available resource version. To ensure backward
+	// compatibility with these server versions we also need to detect the error based on the content
+	// of the error message field.
+	if !apierrors.IsTimeout(err) {
+		return false
+	}
+	apierr, ok := err.(apierrors.APIStatus)
+	if !ok || apierr == nil || apierr.Status().Details == nil {
+		return false
+	}
+	for _, cause := range apierr.Status().Details.Causes {
+		// Matches the message returned by api server 1.17.0-1.18.5 for this error condition
+		if cause.Message == "Too large resource version" {
+			return true
+		}
+	}
+	return false
 }

tools/cache/reflector_test.go

@@ -738,9 +738,14 @@ func TestReflectorFullListIfTooLarge(t *testing.T) {
 				err := apierrors.NewTimeoutError("too large resource version", 1)
 				err.ErrStatus.Details.Causes = []metav1.StatusCause{{Type: metav1.CauseTypeResourceVersionTooLarge}}
 				return nil, err
+			// relist after the initial list (covers the error format used in api server 1.17.0-1.18.5)
+			case "30":
+				err := apierrors.NewTimeoutError("too large resource version", 1)
+				err.ErrStatus.Details.Causes = []metav1.StatusCause{{Message: "Too large resource version"}}
+				return nil, err
 			// relist from etcd after "too large" error
 			case "":
-				return &v1.PodList{ListMeta: metav1.ListMeta{ResourceVersion: "10"}}, nil
+				return &v1.PodList{ListMeta: metav1.ListMeta{ResourceVersion: "30"}}, nil
 			default:
 				return nil, fmt.Errorf("unexpected List call: %s", options.ResourceVersion)
 			}
@@ -759,12 +764,15 @@ func TestReflectorFullListIfTooLarge(t *testing.T) {
 	// may be synced to a different version and they will never converge.
 	// TODO: We should use etcd progress-notify feature to avoid this behavior but until this is
 	// done we simply try to relist from now to avoid continuous errors on relists.
+	for i := 1; i <= 2; i++ {
+		// relist twice to cover the two variants of TooLargeResourceVersion api errors
 		stopCh = make(chan struct{})
 		if err := r.ListAndWatch(stopCh); err != nil {
 			t.Fatal(err)
 		}
+	}
 
-	expectedRVs := []string{"0", "20", ""}
+	expectedRVs := []string{"0", "20", "", "30", ""}
 	if !reflect.DeepEqual(listCallRVs, expectedRVs) {
 		t.Errorf("Expected series of list calls with resource version of %#v but got: %#v", expectedRVs, listCallRVs)
 	}