diff --git a/tools/watch/retrywatcher.go b/tools/watch/retrywatcher.go index 7d702907f..01f401d26 100644 --- a/tools/watch/retrywatcher.go +++ b/tools/watch/retrywatcher.go @@ -166,6 +166,13 @@ func (rw *RetryWatcher) doReceive(ctx context.Context) (bool, time.Duration) { return true, 0 } + // Resource expired or 410 Gone errors (e.g. resource version too old) + // are expected and recoverable, so log at a lower verbosity instead of ERROR. + if apierrors.IsResourceExpired(err) || apierrors.IsGone(err) { + klog.FromContext(ctx).V(4).Info(msg, "err", err) + return false, 0 + } + klog.FromContext(ctx).Error(err, msg) // Retry return false, 0 diff --git a/tools/watch/retrywatcher_test.go b/tools/watch/retrywatcher_test.go index 3c2229779..f518a76ae 100644 --- a/tools/watch/retrywatcher_test.go +++ b/tools/watch/retrywatcher_test.go @@ -199,6 +199,29 @@ func TestRetryWatcher(t *testing.T) { makeTestEvent(2), }, }, + { + name: "recovers from 410 Gone error on watch establishment", + initialRV: "1", + watchClient: &cache.ListWatch{ + WatchFunc: func() func(options metav1.ListOptions) (watch.Interface, error) { + firstRun := true + return func(options metav1.ListOptions) (watch.Interface, error) { + if firstRun { + firstRun = false + return nil, apierrors.NewResourceExpired("") + } + + return watch.NewProxyWatcher(arrayToChannel(fromRV(options.ResourceVersion, []watch.Event{ + makeTestEvent(2), + }))), nil + } + }(), + }, + watchCount: 2, + expected: []watch.Event{ + makeTestEvent(2), + }, + }, { name: "recovers if watchClient returns nil watcher", initialRV: "1",