client-go/tools/watch: downgrade 410 Gone error to V(4) debug log in RetryWatcher

Also check apierrors.IsResourceExpired in addition to IsGone.

Kubernetes-commit: 08e07a914281cda80b8541b444b6d6f1f8e754e2

tools/watch/retrywatcher.go

@@ -166,6 +166,13 @@ func (rw *RetryWatcher) doReceive(ctx context.Context) (bool, time.Duration) {
 			return true, 0
 		}
 
+		// Resource expired or 410 Gone errors (e.g. resource version too old)
+		// are expected and recoverable, so log at a lower verbosity instead of ERROR.
+		if apierrors.IsResourceExpired(err) || apierrors.IsGone(err) {
+			klog.FromContext(ctx).V(4).Info(msg, "err", err)
+			return false, 0
+		}
+
 		klog.FromContext(ctx).Error(err, msg)
 		// Retry
 		return false, 0

tools/watch/retrywatcher_test.go

@@ -199,6 +199,29 @@ func TestRetryWatcher(t *testing.T) {
 				makeTestEvent(2),
 			},
 		},
+		{
+			name:      "recovers from 410 Gone error on watch establishment",
+			initialRV: "1",
+			watchClient: &cache.ListWatch{
+				WatchFunc: func() func(options metav1.ListOptions) (watch.Interface, error) {
+					firstRun := true
+					return func(options metav1.ListOptions) (watch.Interface, error) {
+						if firstRun {
+							firstRun = false
+							return nil, apierrors.NewResourceExpired("")
+						}
+
+						return watch.NewProxyWatcher(arrayToChannel(fromRV(options.ResourceVersion, []watch.Event{
+							makeTestEvent(2),
+						}))), nil
+					}
+				}(),
+			},
+			watchCount: 2,
+			expected: []watch.Event{
+				makeTestEvent(2),
+			},
+		},
 		{
 			name:      "recovers if watchClient returns nil watcher",
 			initialRV: "1",