From 68d417d7d83d9d067b0fee56e673f1f09410acfa Mon Sep 17 00:00:00 2001
From: Hemant Kumar <hekumar@redhat.com>
Date: Thu, 12 Oct 2017 16:27:54 -0400
Subject: [PATCH] Fix possibly flake in multiattach unit test

It is possible that by the time we check for multiattach
error on node, the reconciler loop may not have processed second
volume and hence we are going to retry for multiattach error
on node before giving up and marking the test as failed.
---
 .../reconciler/reconciler_test.go             | 32 ++++++++++++++-----
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go b/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go
index 32a6c974095..79bbc520fe1 100644
--- a/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go
+++ b/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go
@@ -456,15 +456,9 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteOnce(t *testing.
 	nodesForVolume := asw.GetNodesForVolume(generatedVolumeName)
 
 	// check if multiattach is marked
-	// at least one volume should be marked with multiattach error
+	// at least one volume+node should be marked with multiattach error
 	nodeAttachedTo := nodesForVolume[0]
-	for _, volumeToAttach := range dsw.GetVolumesToAttach() {
-		if volumeToAttach.NodeName != nodeAttachedTo {
-			if !volumeToAttach.MultiAttachErrorReported {
-				t.Fatalf("Expected volume %q on node %q to have multiattach error", volumeToAttach.VolumeName, volumeToAttach.NodeName)
-			}
-		}
-	}
+	waitForMultiAttachErrorOnNode(t, nodeAttachedTo, dsw)
 
 	// Act
 	podToDelete := ""
@@ -495,6 +489,28 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteOnce(t *testing.
 	waitForTotalAttachCallCount(t, 2 /* expectedAttachCallCount */, fakePlugin)
 }
 
+func waitForMultiAttachErrorOnNode(
+	t *testing.T,
+	attachedNode k8stypes.NodeName,
+	dsow cache.DesiredStateOfWorld) {
+	multAttachCheckFunc := func() (bool, error) {
+		for _, volumeToAttach := range dsow.GetVolumesToAttach() {
+			if volumeToAttach.NodeName != attachedNode {
+				if volumeToAttach.MultiAttachErrorReported {
+					return true, nil
+				}
+			}
+		}
+		t.Logf("Warning: MultiAttach error not yet set on Node. Will retry.")
+		return false, nil
+	}
+
+	err := retryWithExponentialBackOff(100*time.Millisecond, multAttachCheckFunc)
+	if err != nil {
+		t.Fatalf("Timed out waiting for MultiAttach Error to be set on non-attached node")
+	}
+}
+
 func waitForNewAttacherCallCount(
 	t *testing.T,
 	expectedCallCount int,