From 61275ad8d40df04e0c962d3dd59369def922a556 Mon Sep 17 00:00:00 2001
From: Alexander Block <ablock84@gmail.com>
Date: Tue, 11 Jul 2017 13:04:53 +0200
Subject: [PATCH] Fix flaky test
 Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteMany

Only relying on the NewAttacher/Detacher call counts is not enough as they
happen in parallel to the testing/verification code and thus the actual
attaching/detaching may not be done yet, resulting in flaky test results.

Fixes #46244
---
 .../reconciler/reconciler_test.go             | 45 ++++++++++++++-----
 1 file changed, 35 insertions(+), 10 deletions(-)

diff --git a/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go b/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go
index ba15e8d5261..be158e65c48 100644
--- a/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go
+++ b/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go
@@ -364,11 +364,7 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteMany(t *testing.
 	waitForTotalAttachCallCount(t, 2 /* expectedAttachCallCount */, fakePlugin)
 	verifyNewDetacherCallCount(t, true /* expectZeroNewDetacherCallCount */, fakePlugin)
 	waitForDetachCallCount(t, 0 /* expectedDetachCallCount */, fakePlugin)
-
-	nodesForVolume := asw.GetNodesForVolume(generatedVolumeName)
-	if len(nodesForVolume) != 2 {
-		t.Fatal("Volume was not attached to both nodes")
-	}
+	waitForAttachedToNodesCount(t, 2 /* expectedNodeCount */, generatedVolumeName, asw)
 
 	// Act
 	dsw.DeletePod(types.UniquePodName(podName1), generatedVolumeName, nodeName1)
@@ -455,13 +451,9 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteOnce(t *testing.
 	waitForTotalAttachCallCount(t, 1 /* expectedAttachCallCount */, fakePlugin)
 	verifyNewDetacherCallCount(t, true /* expectZeroNewDetacherCallCount */, fakePlugin)
 	waitForDetachCallCount(t, 0 /* expectedDetachCallCount */, fakePlugin)
+	waitForAttachedToNodesCount(t, 1 /* expectedNodeCount */, generatedVolumeName, asw)
 
 	nodesForVolume := asw.GetNodesForVolume(generatedVolumeName)
-	if len(nodesForVolume) == 0 {
-		t.Fatal("Volume was not attached to any node")
-	} else if len(nodesForVolume) != 1 {
-		t.Fatal("Volume was attached to multiple nodes")
-	}
 
 	// Act
 	podToDelete := ""
@@ -688,6 +680,39 @@ func waitForTotalDetachCallCount(
 	}
 }
 
+func waitForAttachedToNodesCount(
+	t *testing.T,
+	expectedNodeCount int,
+	volumeName v1.UniqueVolumeName,
+	asw cache.ActualStateOfWorld) {
+
+	err := retryWithExponentialBackOff(
+		time.Duration(5*time.Millisecond),
+		func() (bool, error) {
+			count := len(asw.GetNodesForVolume(volumeName))
+			if count == expectedNodeCount {
+				return true, nil
+			}
+			t.Logf(
+				"Warning: Wrong number of nodes having <%v> attached. Expected: <%v> Actual: <%v>. Will retry.",
+				volumeName,
+				expectedNodeCount,
+				count)
+
+			return false, nil
+		},
+	)
+
+	if err != nil {
+		count := len(asw.GetNodesForVolume(volumeName))
+		t.Fatalf(
+			"Wrong number of nodes having <%v> attached. Expected: <%v> Actual: <%v>",
+			volumeName,
+			expectedNodeCount,
+			count)
+	}
+}
+
 func verifyNewAttacherCallCount(
 	t *testing.T,
 	expectZeroNewAttacherCallCount bool,