diff --git a/test/e2e/framework/log_size_monitoring.go b/test/e2e/framework/log_size_monitoring.go
index c578b8a3533..2d57a38b694 100644
--- a/test/e2e/framework/log_size_monitoring.go
+++ b/test/e2e/framework/log_size_monitoring.go
@@ -249,9 +249,13 @@ func (g *LogSizeGatherer) Work() bool {
 	)
 	if err != nil {
 		Logf("Error while trying to SSH to %v, skipping probe. Error: %v", workItem.ip, err)
-		if workItem.backoffMultiplier < 128 {
-			workItem.backoffMultiplier *= 2
+		// In case of repeated error give up.
+		if workItem.backoffMultiplier >= 128 {
+			Logf("Failed to ssh to a node %v multiple times in a row. Giving up.", workItem.ip)
+			g.wg.Done()
+			return false
 		}
+		workItem.backoffMultiplier *= 2
 		go g.pushWorkItem(workItem)
 		return true
 	}