Add "multi-sync" tests.

These test will call syncVolume/syncClaim until they reach consistent state.

pkg/controller/persistentvolume/persistentvolume_framework_test.go

@@ -461,3 +461,113 @@ func runSyncTests(t *testing.T, tests []controllerTest) {
 		evaluateTestResults(ctrl, reactor, test, t)
 	}
 }
+
+// Test multiple calls to syncClaim/syncVolume and periodic sync of all
+// volume/claims. For all tests, the test follows this pattern:
+// 0. Load the controller with initial data.
+// 1. Call controllerTest.testCall() once as in TestSync()
+// 2. For all volumes/claims changed by previous syncVolume/syncClaim calls,
+//    call appropriate syncVolume/syncClaim (simulating "volume/claim changed"
+//    events). Go to 2. if these calls change anything.
+// 3. When all changes are processed and no new changes were made, call
+//    syncVolume/syncClaim on all volumes/claims (simulating "periodic sync").
+// 4. If some changes were done by step 3., go to 2. (simulation of
+//    "volume/claim updated" events, eventually performing step 3. again)
+// 5. When 3. does not do any changes, finish the tests and compare final set
+//    of volumes/claims with expected claims/volumes and report differences.
+// Some limit of calls in enforced to prevent endless loops.
+func runMultisyncTests(t *testing.T, tests []controllerTest) {
+	for _, test := range tests {
+		glog.V(4).Infof("starting multisync test %q", test.name)
+
+		// Initialize the controller
+		client := &fake.Clientset{}
+		ctrl := newPersistentVolumeController(client)
+		reactor := newVolumeReactor(client, ctrl, nil, nil)
+		for _, claim := range test.initialClaims {
+			ctrl.claims.Add(claim)
+			reactor.claims[claim.Name] = claim
+		}
+		for _, volume := range test.initialVolumes {
+			ctrl.volumes.store.Add(volume)
+			reactor.volumes[volume.Name] = volume
+		}
+
+		// Run the tested function
+		err := test.test(ctrl, reactor, test)
+		if err != nil {
+			t.Errorf("Test %q failed: %v", test.name, err)
+		}
+
+		// Simulate any "changed" events and "periodical sync" until we reach a
+		// stable state.
+		firstSync := true
+		counter := 0
+		for {
+			counter++
+			glog.V(4).Infof("test %q: iteration %d", test.name, counter)
+
+			if counter > 100 {
+				t.Errorf("Test %q failed: too many iterations", test.name)
+				break
+			}
+
+			obj := reactor.popChange()
+			if obj == nil {
+				// Nothing was changed, should we exit?
+				if firstSync || reactor.changedSinceLastSync > 0 {
+					// There were some changes after the last "periodic sync".
+					// Simulate "periodic sync" of everything (until it produces
+					// no changes).
+					firstSync = false
+					glog.V(4).Infof("test %q: simulating periodical sync of all claims and volumes", test.name)
+					reactor.syncAll()
+				} else {
+					// Last sync did not produce any updates, the test reached
+					// stable state -> finish.
+					break
+				}
+			}
+
+			// There were some changes, process them
+			switch obj.(type) {
+			case *api.PersistentVolumeClaim:
+				claim := obj.(*api.PersistentVolumeClaim)
+				// Simulate "claim updated" event
+				ctrl.claims.Update(claim)
+				err = ctrl.syncClaim(claim)
+				if err != nil {
+					if err == versionConflictError {
+						// Ignore version errors
+						glog.V(4).Infof("test intentionaly ignores version error.")
+					} else {
+						t.Errorf("Error calling syncClaim: %v", err)
+						// Finish the loop on the first error
+						break
+					}
+				}
+				// Process generated changes
+				continue
+			case *api.PersistentVolume:
+				volume := obj.(*api.PersistentVolume)
+				// Simulate "volume updated" event
+				ctrl.volumes.store.Update(volume)
+				err = ctrl.syncVolume(volume)
+				if err != nil {
+					if err == versionConflictError {
+						// Ignore version errors
+						glog.V(4).Infof("test intentionaly ignores version error.")
+					} else {
+						t.Errorf("Error calling syncVolume: %v", err)
+						// Finish the loop on the first error
+						break
+					}
+				}
+				// Process generated changes
+				continue
+			}
+		}
+		evaluateTestResults(ctrl, reactor, test, t)
+		glog.V(4).Infof("test %q finished after %d iterations", test.name, counter)
+	}
+}

pkg/controller/persistentvolume/persistentvolume_sync_test.go

@@ -377,3 +377,50 @@ func TestSync(t *testing.T) {
 	}
 	runSyncTests(t, tests)
 }
+
+// Test multiple calls to syncClaim/syncVolume and periodic sync of all
+// volume/claims. The test follows this pattern:
+// 0. Load the controller with initial data.
+// 1. Call controllerTest.testCall() once as in TestSync()
+// 2. For all volumes/claims changed by previous syncVolume/syncClaim calls,
+//    call appropriate syncVolume/syncClaim (simulating "volume/claim changed"
+//    events). Go to 2. if these calls change anything.
+// 3. When all changes are processed and no new changes were made, call
+//    syncVolume/syncClaim on all volumes/claims (simulating "periodic sync").
+// 4. If some changes were done by step 3., go to 2. (simulation of
+//    "volume/claim updated" events, eventually performing step 3. again)
+// 5. When 3. does not do any changes, finish the tests and compare final set
+//    of volumes/claims with expected claims/volumes and report differences.
+// Some limit of calls in enforced to prevent endless loops.
+func TestMultiSync(t *testing.T) {
+	tests := []controllerTest{
+		// Test simple binding
+		{
+			// syncClaim binds to a matching unbound volume.
+			"10-1 - successful bind",
+			newVolumeArray("volume10-1", "1Gi", "", "", api.VolumePending),
+			newVolumeArray("volume10-1", "1Gi", "uid10-1", "claim10-1", api.VolumeBound, annBoundByController),
+			newClaimArray("claim10-1", "uid10-1", "1Gi", "", api.ClaimPending),
+			newClaimArray("claim10-1", "uid10-1", "1Gi", "volume10-1", api.ClaimBound, annBoundByController, annBindCompleted),
+			testSyncClaim,
+		},
+		{
+			// Two controllers bound two PVs to single claim. Test one of them
+			// wins and the second rolls back.
+			"10-2 - bind PV race",
+			[]*api.PersistentVolume{
+				newVolume("volume10-2-1", "1Gi", "uid10-2", "claim10-2", api.VolumeBound, annBoundByController),
+				newVolume("volume10-2-2", "1Gi", "uid10-2", "claim10-2", api.VolumeBound, annBoundByController),
+			},
+			[]*api.PersistentVolume{
+				newVolume("volume10-2-1", "1Gi", "uid10-2", "claim10-2", api.VolumeBound, annBoundByController),
+				newVolume("volume10-2-2", "1Gi", "", "", api.VolumeAvailable),
+			},
+			newClaimArray("claim10-2", "uid10-2", "1Gi", "volume10-2-1", api.ClaimBound, annBoundByController, annBindCompleted),
+			newClaimArray("claim10-2", "uid10-2", "1Gi", "volume10-2-1", api.ClaimBound, annBoundByController, annBindCompleted),
+			testSyncClaim,
+		},
+	}
+
+	runMultisyncTests(t, tests)
+}