diff --git a/tools/cache/reflector.go b/tools/cache/reflector.go index a92b36f2..e995abe2 100644 --- a/tools/cache/reflector.go +++ b/tools/cache/reflector.go @@ -570,5 +570,26 @@ func isExpiredError(err error) bool { } func isTooLargeResourceVersionError(err error) bool { - return apierrors.HasStatusCause(err, metav1.CauseTypeResourceVersionTooLarge) + if apierrors.HasStatusCause(err, metav1.CauseTypeResourceVersionTooLarge) { + return true + } + // In Kubernetes 1.17.0-1.18.5, the api server doesn't set the error status cause to + // metav1.CauseTypeResourceVersionTooLarge to indicate that the requested minimum resource + // version is larger than the largest currently available resource version. To ensure backward + // compatibility with these server versions we also need to detect the error based on the content + // of the error message field. + if !apierrors.IsTimeout(err) { + return false + } + apierr, ok := err.(apierrors.APIStatus) + if !ok || apierr == nil || apierr.Status().Details == nil { + return false + } + for _, cause := range apierr.Status().Details.Causes { + // Matches the message returned by api server 1.17.0-1.18.5 for this error condition + if cause.Message == "Too large resource version" { + return true + } + } + return false } diff --git a/tools/cache/reflector_test.go b/tools/cache/reflector_test.go index 4a25fcce..5394e668 100644 --- a/tools/cache/reflector_test.go +++ b/tools/cache/reflector_test.go @@ -738,9 +738,14 @@ func TestReflectorFullListIfTooLarge(t *testing.T) { err := apierrors.NewTimeoutError("too large resource version", 1) err.ErrStatus.Details.Causes = []metav1.StatusCause{{Type: metav1.CauseTypeResourceVersionTooLarge}} return nil, err + // relist after the initial list (covers the error format used in api server 1.17.0-1.18.5) + case "30": + err := apierrors.NewTimeoutError("too large resource version", 1) + err.ErrStatus.Details.Causes = []metav1.StatusCause{{Message: "Too large resource version"}} + return nil, err // relist from etcd after "too large" error case "": - return &v1.PodList{ListMeta: metav1.ListMeta{ResourceVersion: "10"}}, nil + return &v1.PodList{ListMeta: metav1.ListMeta{ResourceVersion: "30"}}, nil default: return nil, fmt.Errorf("unexpected List call: %s", options.ResourceVersion) } @@ -759,12 +764,15 @@ func TestReflectorFullListIfTooLarge(t *testing.T) { // may be synced to a different version and they will never converge. // TODO: We should use etcd progress-notify feature to avoid this behavior but until this is // done we simply try to relist from now to avoid continuous errors on relists. - stopCh = make(chan struct{}) - if err := r.ListAndWatch(stopCh); err != nil { - t.Fatal(err) + for i := 1; i <= 2; i++ { + // relist twice to cover the two variants of TooLargeResourceVersion api errors + stopCh = make(chan struct{}) + if err := r.ListAndWatch(stopCh); err != nil { + t.Fatal(err) + } } - expectedRVs := []string{"0", "20", ""} + expectedRVs := []string{"0", "20", "", "30", ""} if !reflect.DeepEqual(listCallRVs, expectedRVs) { t.Errorf("Expected series of list calls with resource version of %#v but got: %#v", expectedRVs, listCallRVs) }