diff --git a/tools/watch/retrywatcher.go b/tools/watch/retrywatcher.go
index 1ed46ccb..e4806d2e 100644
--- a/tools/watch/retrywatcher.go
+++ b/tools/watch/retrywatcher.go
@@ -268,7 +268,13 @@ func (rw *RetryWatcher) receive() {
 			return
 		}
 
-		time.Sleep(retryAfter)
+		timer := time.NewTimer(retryAfter)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return
+		case <-timer.C:
+		}
 
 		klog.V(4).Infof("Restarting RetryWatcher at RV=%q", rw.lastResourceVersion)
 	}, rw.minRestartDelay)