Merge pull request #48757 from codablock/fix_flake_46244

Automatic merge from submit-queue (batch tested with PRs 48672, 47140, 48709, 48786, 48757)

Fix flaky test Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteMany

Only relying on the NewAttacher/Detacher call counts is not enough as they
happen in parallel to the testing/verification code and thus the actual
attaching/detaching may not be done yet, resulting in flaky test results.

Fixes #46244

pkg/controller/volume/attachdetach/reconciler/reconciler_test.go

@@ -364,11 +364,7 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteMany(t *testing.
 	waitForTotalAttachCallCount(t, 2 /* expectedAttachCallCount */, fakePlugin)
 	verifyNewDetacherCallCount(t, true /* expectZeroNewDetacherCallCount */, fakePlugin)
 	waitForDetachCallCount(t, 0 /* expectedDetachCallCount */, fakePlugin)
-
-	nodesForVolume := asw.GetNodesForVolume(generatedVolumeName)
-	if len(nodesForVolume) != 2 {
-		t.Fatal("Volume was not attached to both nodes")
-	}
+	waitForAttachedToNodesCount(t, 2 /* expectedNodeCount */, generatedVolumeName, asw)
 
 	// Act
 	dsw.DeletePod(types.UniquePodName(podName1), generatedVolumeName, nodeName1)
@@ -455,13 +451,9 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteOnce(t *testing.
 	waitForTotalAttachCallCount(t, 1 /* expectedAttachCallCount */, fakePlugin)
 	verifyNewDetacherCallCount(t, true /* expectZeroNewDetacherCallCount */, fakePlugin)
 	waitForDetachCallCount(t, 0 /* expectedDetachCallCount */, fakePlugin)
+	waitForAttachedToNodesCount(t, 1 /* expectedNodeCount */, generatedVolumeName, asw)
 
 	nodesForVolume := asw.GetNodesForVolume(generatedVolumeName)
-	if len(nodesForVolume) == 0 {
-		t.Fatal("Volume was not attached to any node")
-	} else if len(nodesForVolume) != 1 {
-		t.Fatal("Volume was attached to multiple nodes")
-	}
 
 	// Act
 	podToDelete := ""
@@ -688,6 +680,39 @@ func waitForTotalDetachCallCount(
 	}
 }
 
+func waitForAttachedToNodesCount(
+	t *testing.T,
+	expectedNodeCount int,
+	volumeName v1.UniqueVolumeName,
+	asw cache.ActualStateOfWorld) {
+
+	err := retryWithExponentialBackOff(
+		time.Duration(5*time.Millisecond),
+		func() (bool, error) {
+			count := len(asw.GetNodesForVolume(volumeName))
+			if count == expectedNodeCount {
+				return true, nil
+			}
+			t.Logf(
+				"Warning: Wrong number of nodes having <%v> attached. Expected: <%v> Actual: <%v>. Will retry.",
+				volumeName,
+				expectedNodeCount,
+				count)
+
+			return false, nil
+		},
+	)
+
+	if err != nil {
+		count := len(asw.GetNodesForVolume(volumeName))
+		t.Fatalf(
+			"Wrong number of nodes having <%v> attached. Expected: <%v> Actual: <%v>",
+			volumeName,
+			expectedNodeCount,
+			count)
+	}
+}
+
 func verifyNewAttacherCallCount(
 	t *testing.T,
 	expectZeroNewAttacherCallCount bool,