Make volume binder resilient to races: unit tests

pkg/controller/volume/persistentvolume/framework_test.go

@@ -29,11 +29,13 @@ import (
 
 	"k8s.io/klog"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	storage "k8s.io/api/storage/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/diff"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -136,6 +138,7 @@ type volumeReactor struct {
 	fakeClaimWatch       *watch.FakeWatcher
 	lock                 sync.Mutex
 	errors               []reactorError
+	watchers             map[schema.GroupVersionResource]map[string][]*watch.RaceFreeFakeWatcher
 }
 
 // reactorError is an error that is returned by test reactor (=simulated
@@ -189,11 +192,34 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj
 
 		// Store the updated object to appropriate places.
 		r.volumes[volume.Name] = volume
+		for _, w := range r.getWatches(action.GetResource(), action.GetNamespace()) {
+			w.Add(volume)
+		}
 		r.changedObjects = append(r.changedObjects, volume)
 		r.changedSinceLastSync++
 		klog.V(4).Infof("created volume %s", volume.Name)
 		return true, volume, nil
 
+	case action.Matches("create", "persistentvolumeclaims"):
+		obj := action.(core.UpdateAction).GetObject()
+		claim := obj.(*v1.PersistentVolumeClaim)
+
+		// check the claim does not exist
+		_, found := r.claims[claim.Name]
+		if found {
+			return true, nil, fmt.Errorf("Cannot create claim %s: claim already exists", claim.Name)
+		}
+
+		// Store the updated object to appropriate places.
+		r.claims[claim.Name] = claim
+		for _, w := range r.getWatches(action.GetResource(), action.GetNamespace()) {
+			w.Add(claim)
+		}
+		r.changedObjects = append(r.changedObjects, claim)
+		r.changedSinceLastSync++
+		klog.V(4).Infof("created claim %s", claim.Name)
+		return true, claim, nil
+
 	case action.Matches("update", "persistentvolumes"):
 		obj := action.(core.UpdateAction).GetObject()
 		volume := obj.(*v1.PersistentVolume)
@@ -206,6 +232,10 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj
 			if storedVer != requestedVer {
 				return true, obj, versionConflictError
 			}
+			if reflect.DeepEqual(storedVolume, volume) {
+				klog.V(4).Infof("nothing updated volume %s", volume.Name)
+				return true, volume, nil
+			}
 			// Don't modify the existing object
 			volume = volume.DeepCopy()
 			volume.ResourceVersion = strconv.Itoa(storedVer + 1)
@@ -214,6 +244,9 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj
 		}
 
 		// Store the updated object to appropriate places.
+		for _, w := range r.getWatches(action.GetResource(), action.GetNamespace()) {
+			w.Modify(volume)
+		}
 		r.volumes[volume.Name] = volume
 		r.changedObjects = append(r.changedObjects, volume)
 		r.changedSinceLastSync++
@@ -232,6 +265,10 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj
 			if storedVer != requestedVer {
 				return true, obj, versionConflictError
 			}
+			if reflect.DeepEqual(storedClaim, claim) {
+				klog.V(4).Infof("nothing updated claim %s", claim.Name)
+				return true, claim, nil
+			}
 			// Don't modify the existing object
 			claim = claim.DeepCopy()
 			claim.ResourceVersion = strconv.Itoa(storedVer + 1)
@@ -240,6 +277,9 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj
 		}
 
 		// Store the updated object to appropriate places.
+		for _, w := range r.getWatches(action.GetResource(), action.GetNamespace()) {
+			w.Modify(claim)
+		}
 		r.claims[claim.Name] = claim
 		r.changedObjects = append(r.changedObjects, claim)
 		r.changedSinceLastSync++
@@ -251,18 +291,32 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj
 		volume, found := r.volumes[name]
 		if found {
 			klog.V(4).Infof("GetVolume: found %s", volume.Name)
-			return true, volume, nil
+			return true, volume.DeepCopy(), nil
 		} else {
 			klog.V(4).Infof("GetVolume: volume %s not found", name)
 			return true, nil, fmt.Errorf("Cannot find volume %s", name)
 		}
 
+	case action.Matches("get", "persistentvolumeclaims"):
+		name := action.(core.GetAction).GetName()
+		claim, found := r.claims[name]
+		if found {
+			klog.V(4).Infof("GetClaim: found %s", claim.Name)
+			return true, claim.DeepCopy(), nil
+		} else {
+			klog.V(4).Infof("GetClaim: claim %s not found", name)
+			return true, nil, apierrs.NewNotFound(action.GetResource().GroupResource(), name)
+		}
+
 	case action.Matches("delete", "persistentvolumes"):
 		name := action.(core.DeleteAction).GetName()
 		klog.V(4).Infof("deleted volume %s", name)
-		_, found := r.volumes[name]
+		obj, found := r.volumes[name]
 		if found {
 			delete(r.volumes, name)
+			for _, w := range r.getWatches(action.GetResource(), action.GetNamespace()) {
+				w.Delete(obj)
+			}
 			r.changedSinceLastSync++
 			return true, nil, nil
 		} else {
@@ -272,9 +326,12 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj
 	case action.Matches("delete", "persistentvolumeclaims"):
 		name := action.(core.DeleteAction).GetName()
 		klog.V(4).Infof("deleted claim %s", name)
-		_, found := r.volumes[name]
+		obj, found := r.claims[name]
 		if found {
 			delete(r.claims, name)
+			for _, w := range r.getWatches(action.GetResource(), action.GetNamespace()) {
+				w.Delete(obj)
+			}
 			r.changedSinceLastSync++
 			return true, nil, nil
 		} else {
@@ -285,6 +342,36 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj
 	return false, nil, nil
 }
 
+// Watch watches objects from the volumeReactor. Watch returns a channel which
+// will push added / modified / deleted object.
+func (r *volumeReactor) Watch(gvr schema.GroupVersionResource, ns string) (watch.Interface, error) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	fakewatcher := watch.NewRaceFreeFake()
+
+	if _, exists := r.watchers[gvr]; !exists {
+		r.watchers[gvr] = make(map[string][]*watch.RaceFreeFakeWatcher)
+	}
+	r.watchers[gvr][ns] = append(r.watchers[gvr][ns], fakewatcher)
+	return fakewatcher, nil
+}
+
+func (r *volumeReactor) getWatches(gvr schema.GroupVersionResource, ns string) []*watch.RaceFreeFakeWatcher {
+	watches := []*watch.RaceFreeFakeWatcher{}
+	if r.watchers[gvr] != nil {
+		if w := r.watchers[gvr][ns]; w != nil {
+			watches = append(watches, w...)
+		}
+		if ns != metav1.NamespaceAll {
+			if w := r.watchers[gvr][metav1.NamespaceAll]; w != nil {
+				watches = append(watches, w...)
+			}
+		}
+	}
+	return watches
+}
+
 // injectReactError returns an error when the test requested given action to
 // fail. nil is returned otherwise.
 func (r *volumeReactor) injectReactError(action core.Action) error {
@@ -596,11 +683,14 @@ func newVolumeReactor(client *fake.Clientset, ctrl *PersistentVolumeController,
 		fakeVolumeWatch: fakeVolumeWatch,
 		fakeClaimWatch:  fakeClaimWatch,
 		errors:          errors,
+		watchers:        make(map[schema.GroupVersionResource]map[string][]*watch.RaceFreeFakeWatcher),
 	}
 	client.AddReactor("create", "persistentvolumes", reactor.React)
+	client.AddReactor("create", "persistentvolumeclaims", reactor.React)
 	client.AddReactor("update", "persistentvolumes", reactor.React)
 	client.AddReactor("update", "persistentvolumeclaims", reactor.React)
 	client.AddReactor("get", "persistentvolumes", reactor.React)
+	client.AddReactor("get", "persistentvolumeclaims", reactor.React)
 	client.AddReactor("delete", "persistentvolumes", reactor.React)
 	client.AddReactor("delete", "persistentvolumeclaims", reactor.React)
 

pkg/controller/volume/persistentvolume/scheduler_binder_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package persistentvolume
 
 import (
+	"context"
 	"fmt"
 	"reflect"
 	"testing"
@@ -28,10 +29,13 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/diff"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/informers"
 	coreinformers "k8s.io/client-go/informers/core/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/fake"
+	k8stesting "k8s.io/client-go/testing"
 	"k8s.io/klog"
 	"k8s.io/kubernetes/pkg/api/testapi"
 	"k8s.io/kubernetes/pkg/controller"
@@ -75,14 +79,6 @@ var (
 	pvBoundImmediate           = makeTestPV("pv-bound-immediate", "node1", "1G", "1", immediateBoundPVC, immediateClass)
 	pvBoundImmediateNode2      = makeTestPV("pv-bound-immediate", "node2", "1G", "1", immediateBoundPVC, immediateClass)
 
-	// PVC/PV bindings for manual binding
-	binding1a      = makeBinding(unboundPVC, pvNode1a)
-	binding1b      = makeBinding(unboundPVC2, pvNode1b)
-	bindingNoNode  = makeBinding(unboundPVC, pvNoNode)
-	bindingBad     = makeBinding(badPVC, pvNode1b)
-	binding1aBound = makeBinding(unboundPVC, pvNode1aBound)
-	binding1bBound = makeBinding(unboundPVC2, pvNode1bBound)
-
 	// storage class names
 	waitClass                = "waitClass"
 	immediateClass           = "immediateClass"
@@ -109,15 +105,24 @@ type testEnv struct {
 	internalPVCCache     *pvcAssumeCache
 }
 
-func newTestBinder(t *testing.T) *testEnv {
+func newTestBinder(t *testing.T, stopCh <-chan struct{}) *testEnv {
 	client := &fake.Clientset{}
 	reactor := newVolumeReactor(client, nil, nil, nil, nil)
+	// TODO refactor all tests to use real watch mechanism, see #72327
+	client.AddWatchReactor("*", func(action k8stesting.Action) (handled bool, ret watch.Interface, err error) {
+		gvr := action.GetResource()
+		ns := action.GetNamespace()
+		watch, err := reactor.Watch(gvr, ns)
+		if err != nil {
+			return false, nil, err
+		}
+		return true, watch, nil
+	})
 	informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc())
 
 	nodeInformer := informerFactory.Core().V1().Nodes()
 	pvcInformer := informerFactory.Core().V1().PersistentVolumeClaims()
 	classInformer := informerFactory.Storage().V1().StorageClasses()
-
 	binder := NewVolumeBinder(
 		client,
 		nodeInformer,
@@ -126,6 +131,14 @@ func newTestBinder(t *testing.T) *testEnv {
 		classInformer,
 		10*time.Second)
 
+	// Wait for informers cache sync
+	informerFactory.Start(stopCh)
+	for v, synced := range informerFactory.WaitForCacheSync(stopCh) {
+		if !synced {
+			klog.Fatalf("Error syncing informer for %v", v)
+		}
+	}
+
 	// Add storageclasses
 	waitMode := storagev1.VolumeBindingWaitForFirstConsumer
 	immediateMode := storagev1.VolumeBindingImmediate
@@ -247,6 +260,66 @@ func (env *testEnv) initVolumes(cachedPVs []*v1.PersistentVolume, apiPVs []*v1.P
 
 }
 
+func (env *testEnv) updateVolumes(t *testing.T, pvs []*v1.PersistentVolume, waitCache bool) {
+	for _, pv := range pvs {
+		if _, err := env.client.CoreV1().PersistentVolumes().Update(pv); err != nil {
+			t.Fatalf("failed to update PV %q", pv.Name)
+		}
+	}
+	if waitCache {
+		wait.Poll(100*time.Millisecond, 3*time.Second, func() (bool, error) {
+			for _, pv := range pvs {
+				obj, err := env.internalPVCache.GetAPIObj(pv.Name)
+				if obj == nil || err != nil {
+					return false, nil
+				}
+				pvInCache, ok := obj.(*v1.PersistentVolume)
+				if !ok {
+					return false, fmt.Errorf("PV %s invalid object", pvInCache.Name)
+				}
+				return versioner.CompareResourceVersion(pvInCache, pv) == 0, nil
+			}
+			return true, nil
+		})
+	}
+}
+
+func (env *testEnv) updateClaims(t *testing.T, pvcs []*v1.PersistentVolumeClaim, waitCache bool) {
+	for _, pvc := range pvcs {
+		if _, err := env.client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Update(pvc); err != nil {
+			t.Fatalf("failed to update PVC %q", getPVCName(pvc))
+		}
+	}
+	if waitCache {
+		wait.Poll(100*time.Millisecond, 3*time.Second, func() (bool, error) {
+			for _, pvc := range pvcs {
+				obj, err := env.internalPVCCache.GetAPIObj(getPVCName(pvc))
+				if obj == nil || err != nil {
+					return false, nil
+				}
+				pvcInCache, ok := obj.(*v1.PersistentVolumeClaim)
+				if !ok {
+					return false, fmt.Errorf("PVC %s invalid object", pvcInCache.Name)
+				}
+				return versioner.CompareResourceVersion(pvcInCache, pvc) == 0, nil
+			}
+			return true, nil
+		})
+	}
+}
+
+func (env *testEnv) deleteVolumes(pvs []*v1.PersistentVolume) {
+	for _, pv := range pvs {
+		env.internalPVCache.delete(pv)
+	}
+}
+
+func (env *testEnv) deleteClaims(pvcs []*v1.PersistentVolumeClaim) {
+	for _, pvc := range pvcs {
+		env.internalPVCCache.delete(pvc)
+	}
+}
+
 func (env *testEnv) assumeVolumes(t *testing.T, name, node string, pod *v1.Pod, bindings []*bindingInfo, provisionings []*v1.PersistentVolumeClaim) {
 	pvCache := env.internalBinder.pvCache
 	for _, binding := range bindings {
@@ -540,7 +613,7 @@ func makeTestPV(name, node, capacity, version string, boundToPVC *v1.PersistentV
 
 func pvcSetSelectedNode(pvc *v1.PersistentVolumeClaim, node string) *v1.PersistentVolumeClaim {
 	newPVC := pvc.DeepCopy()
-	metav1.SetMetaDataAnnotation(&pvc.ObjectMeta, annSelectedNode, node)
+	metav1.SetMetaDataAnnotation(&newPVC.ObjectMeta, annSelectedNode, node)
 	return newPVC
 }
 
@@ -676,7 +749,7 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) {
 		"unbound-pvc,pv-same-node": {
 			podPVCs:          []*v1.PersistentVolumeClaim{unboundPVC},
 			pvs:              []*v1.PersistentVolume{pvNode2, pvNode1a, pvNode1b},
-			expectedBindings: []*bindingInfo{binding1a},
+			expectedBindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a)},
 			expectedUnbound:  true,
 			expectedBound:    true,
 		},
@@ -689,28 +762,28 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) {
 		"two-unbound-pvcs": {
 			podPVCs:          []*v1.PersistentVolumeClaim{unboundPVC, unboundPVC2},
 			pvs:              []*v1.PersistentVolume{pvNode1a, pvNode1b},
-			expectedBindings: []*bindingInfo{binding1a, binding1b},
+			expectedBindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a), makeBinding(unboundPVC2, pvNode1b)},
 			expectedUnbound:  true,
 			expectedBound:    true,
 		},
 		"two-unbound-pvcs,order-by-size": {
 			podPVCs:          []*v1.PersistentVolumeClaim{unboundPVC2, unboundPVC},
 			pvs:              []*v1.PersistentVolume{pvNode1a, pvNode1b},
-			expectedBindings: []*bindingInfo{binding1a, binding1b},
+			expectedBindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a), makeBinding(unboundPVC2, pvNode1b)},
 			expectedUnbound:  true,
 			expectedBound:    true,
 		},
 		"two-unbound-pvcs,partial-match": {
 			podPVCs:          []*v1.PersistentVolumeClaim{unboundPVC, unboundPVC2},
 			pvs:              []*v1.PersistentVolume{pvNode1a},
-			expectedBindings: []*bindingInfo{binding1a},
+			expectedBindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a)},
 			expectedUnbound:  false,
 			expectedBound:    true,
 		},
 		"one-bound,one-unbound": {
 			podPVCs:          []*v1.PersistentVolumeClaim{unboundPVC, boundPVC},
 			pvs:              []*v1.PersistentVolume{pvBound, pvNode1a},
-			expectedBindings: []*bindingInfo{binding1a},
+			expectedBindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a)},
 			expectedUnbound:  true,
 			expectedBound:    true,
 		},
@@ -767,11 +840,14 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) {
 		},
 	}
 
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	for name, scenario := range scenarios {
 		klog.V(5).Infof("Running test case %q", name)
 
 		// Setup
-		testEnv := newTestBinder(t)
+		testEnv := newTestBinder(t, ctx.Done())
 		testEnv.initVolumes(scenario.pvs, scenario.pvs)
 
 		// a. Init pvc cache
@@ -833,7 +909,7 @@ func TestFindPodVolumesWithProvisioning(t *testing.T) {
 		"two-unbound-pvcs,one-matched,one-provisioned": {
 			podPVCs:            []*v1.PersistentVolumeClaim{unboundPVC, provisionedPVC},
 			pvs:                []*v1.PersistentVolume{pvNode1a},
-			expectedBindings:   []*bindingInfo{binding1a},
+			expectedBindings:   []*bindingInfo{makeBinding(unboundPVC, pvNode1a)},
 			expectedProvisions: []*v1.PersistentVolumeClaim{provisionedPVC},
 			expectedUnbound:    true,
 			expectedBound:      true,
@@ -845,6 +921,13 @@ func TestFindPodVolumesWithProvisioning(t *testing.T) {
 			expectedUnbound:    true,
 			expectedBound:      true,
 		},
+		"one-binding,one-selected-node": {
+			podPVCs:            []*v1.PersistentVolumeClaim{boundPVC, selectedNodePVC},
+			pvs:                []*v1.PersistentVolume{pvBound},
+			expectedProvisions: []*v1.PersistentVolumeClaim{selectedNodePVC},
+			expectedUnbound:    true,
+			expectedBound:      true,
+		},
 		"immediate-unbound-pvc": {
 			podPVCs:         []*v1.PersistentVolumeClaim{immediateUnboundPVC},
 			expectedUnbound: false,
@@ -879,9 +962,12 @@ func TestFindPodVolumesWithProvisioning(t *testing.T) {
 		},
 	}
 
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	for name, scenario := range scenarios {
 		// Setup
-		testEnv := newTestBinder(t)
+		testEnv := newTestBinder(t, ctx.Done())
 		testEnv.initVolumes(scenario.pvs, scenario.pvs)
 
 		// a. Init pvc cache
@@ -937,59 +1023,62 @@ func TestAssumePodVolumes(t *testing.T) {
 		},
 		"one-binding": {
 			podPVCs:               []*v1.PersistentVolumeClaim{unboundPVC},
-			bindings:              []*bindingInfo{binding1a},
+			bindings:              []*bindingInfo{makeBinding(unboundPVC, pvNode1a)},
 			pvs:                   []*v1.PersistentVolume{pvNode1a},
-			expectedBindings:      []*bindingInfo{binding1aBound},
+			expectedBindings:      []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)},
 			expectedProvisionings: []*v1.PersistentVolumeClaim{},
 		},
 		"two-bindings": {
 			podPVCs:               []*v1.PersistentVolumeClaim{unboundPVC, unboundPVC2},
-			bindings:              []*bindingInfo{binding1a, binding1b},
+			bindings:              []*bindingInfo{makeBinding(unboundPVC, pvNode1a), makeBinding(unboundPVC2, pvNode1b)},
 			pvs:                   []*v1.PersistentVolume{pvNode1a, pvNode1b},
-			expectedBindings:      []*bindingInfo{binding1aBound, binding1bBound},
+			expectedBindings:      []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound), makeBinding(unboundPVC2, pvNode1bBound)},
 			expectedProvisionings: []*v1.PersistentVolumeClaim{},
 		},
 		"pv-already-bound": {
 			podPVCs:               []*v1.PersistentVolumeClaim{unboundPVC},
-			bindings:              []*bindingInfo{binding1aBound},
+			bindings:              []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)},
 			pvs:                   []*v1.PersistentVolume{pvNode1aBound},
-			expectedBindings:      []*bindingInfo{binding1aBound},
+			expectedBindings:      []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)},
 			expectedProvisionings: []*v1.PersistentVolumeClaim{},
 		},
 		"claimref-failed": {
 			podPVCs:    []*v1.PersistentVolumeClaim{unboundPVC},
-			bindings:   []*bindingInfo{binding1a, bindingBad},
+			bindings:   []*bindingInfo{makeBinding(unboundPVC, pvNode1a), makeBinding(badPVC, pvNode1b)},
 			pvs:        []*v1.PersistentVolume{pvNode1a, pvNode1b},
 			shouldFail: true,
 		},
 		"tmpupdate-failed": {
 			podPVCs:    []*v1.PersistentVolumeClaim{unboundPVC},
-			bindings:   []*bindingInfo{binding1a, binding1b},
+			bindings:   []*bindingInfo{makeBinding(unboundPVC, pvNode1a), makeBinding(unboundPVC2, pvNode1b)},
 			pvs:        []*v1.PersistentVolume{pvNode1a},
 			shouldFail: true,
 		},
 		"one-binding, one-pvc-provisioned": {
 			podPVCs:               []*v1.PersistentVolumeClaim{unboundPVC, provisionedPVC},
-			bindings:              []*bindingInfo{binding1a},
+			bindings:              []*bindingInfo{makeBinding(unboundPVC, pvNode1a)},
 			pvs:                   []*v1.PersistentVolume{pvNode1a},
 			provisionedPVCs:       []*v1.PersistentVolumeClaim{provisionedPVC},
-			expectedBindings:      []*bindingInfo{binding1aBound},
+			expectedBindings:      []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)},
 			expectedProvisionings: []*v1.PersistentVolumeClaim{selectedNodePVC},
 		},
 		"one-binding, one-provision-tmpupdate-failed": {
 			podPVCs:         []*v1.PersistentVolumeClaim{unboundPVC, provisionedPVCHigherVersion},
-			bindings:        []*bindingInfo{binding1a},
+			bindings:        []*bindingInfo{makeBinding(unboundPVC, pvNode1a)},
 			pvs:             []*v1.PersistentVolume{pvNode1a},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{provisionedPVC2},
 			shouldFail:      true,
 		},
 	}
 
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	for name, scenario := range scenarios {
 		klog.V(5).Infof("Running test case %q", name)
 
 		// Setup
-		testEnv := newTestBinder(t)
+		testEnv := newTestBinder(t, ctx.Done())
 		testEnv.initClaims(scenario.podPVCs, scenario.podPVCs)
 		pod := makePod(scenario.podPVCs)
 		testEnv.initPodCache(pod, "node1", scenario.bindings, scenario.provisionedPVCs)
@@ -1062,25 +1151,25 @@ func TestBindAPIUpdate(t *testing.T) {
 			provisionedPVCs: []*v1.PersistentVolumeClaim{},
 		},
 		"one-binding": {
-			bindings:        []*bindingInfo{binding1aBound},
+			bindings:        []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)},
 			cachedPVs:       []*v1.PersistentVolume{pvNode1a},
 			expectedPVs:     []*v1.PersistentVolume{pvNode1aBound},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{},
 		},
 		"two-bindings": {
-			bindings:        []*bindingInfo{binding1aBound, binding1bBound},
+			bindings:        []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound), makeBinding(unboundPVC2, pvNode1bBound)},
 			cachedPVs:       []*v1.PersistentVolume{pvNode1a, pvNode1b},
 			expectedPVs:     []*v1.PersistentVolume{pvNode1aBound, pvNode1bBound},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{},
 		},
 		"api-already-updated": {
-			bindings:        []*bindingInfo{binding1aBound},
+			bindings:        []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)},
 			cachedPVs:       []*v1.PersistentVolume{pvNode1aBound},
 			expectedPVs:     []*v1.PersistentVolume{pvNode1aBound},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{},
 		},
 		"api-update-failed": {
-			bindings:        []*bindingInfo{binding1aBound, binding1bBound},
+			bindings:        []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound), makeBinding(unboundPVC2, pvNode1bBound)},
 			cachedPVs:       []*v1.PersistentVolume{pvNode1a, pvNode1b},
 			apiPVs:          []*v1.PersistentVolume{pvNode1a, pvNode1bBoundHigherVersion},
 			expectedPVs:     []*v1.PersistentVolume{pvNode1aBound, pvNode1b},
@@ -1104,7 +1193,7 @@ func TestBindAPIUpdate(t *testing.T) {
 			shouldFail:      true,
 		},
 		"binding-succeed, provision-api-update-failed": {
-			bindings:        []*bindingInfo{binding1aBound},
+			bindings:        []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)},
 			cachedPVs:       []*v1.PersistentVolume{pvNode1a},
 			expectedPVs:     []*v1.PersistentVolume{pvNode1aBound},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC), addProvisionAnn(provisionedPVC2)},
@@ -1115,11 +1204,15 @@ func TestBindAPIUpdate(t *testing.T) {
 			shouldFail:      true,
 		},
 	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	for name, scenario := range scenarios {
 		klog.V(4).Infof("Running test case %q", name)
 
 		// Setup
-		testEnv := newTestBinder(t)
+		testEnv := newTestBinder(t, ctx.Done())
 		pod := makePod(nil)
 		if scenario.apiPVs == nil {
 			scenario.apiPVs = scenario.cachedPVs
@@ -1155,11 +1248,19 @@ func TestBindAPIUpdate(t *testing.T) {
 func TestCheckBindings(t *testing.T) {
 	scenarios := map[string]struct {
 		// Inputs
-		bindings  []*bindingInfo
+		initPVs  []*v1.PersistentVolume
-		cachedPVs []*v1.PersistentVolume
+		initPVCs []*v1.PersistentVolumeClaim
 
+		bindings        []*bindingInfo
 		provisionedPVCs []*v1.PersistentVolumeClaim
-		cachedPVCs      []*v1.PersistentVolumeClaim
+
+		// api updates before checking
+		apiPVs  []*v1.PersistentVolume
+		apiPVCs []*v1.PersistentVolumeClaim
+
+		// delete objects before checking
+		deletePVs  bool
+		deletePVCs bool
 
 		// Expected return values
 		shouldFail    bool
@@ -1182,108 +1283,144 @@ func TestCheckBindings(t *testing.T) {
 			expectedBound:   true,
 		},
 		"binding-bound": {
-			bindings:        []*bindingInfo{binding1aBound},
+			bindings:        []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{},
-			cachedPVs:       []*v1.PersistentVolume{pvNode1aBound},
+			initPVs:         []*v1.PersistentVolume{pvNode1aBound},
-			cachedPVCs:      []*v1.PersistentVolumeClaim{boundPVCNode1a},
+			initPVCs:        []*v1.PersistentVolumeClaim{boundPVCNode1a},
 			expectedBound:   true,
 		},
 		"binding-prebound": {
-			bindings:        []*bindingInfo{binding1aBound},
+			bindings:        []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{},
-			cachedPVs:       []*v1.PersistentVolume{pvNode1aBound},
+			initPVs:         []*v1.PersistentVolume{pvNode1aBound},
-			cachedPVCs:      []*v1.PersistentVolumeClaim{preboundPVCNode1a},
+			initPVCs:        []*v1.PersistentVolumeClaim{preboundPVCNode1a},
 		},
 		"binding-unbound": {
-			bindings:        []*bindingInfo{binding1aBound},
+			bindings:        []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{},
-			cachedPVs:       []*v1.PersistentVolume{pvNode1aBound},
+			initPVs:         []*v1.PersistentVolume{pvNode1aBound},
-			cachedPVCs:      []*v1.PersistentVolumeClaim{unboundPVC},
+			initPVCs:        []*v1.PersistentVolumeClaim{unboundPVC},
 		},
 		"binding-pvc-not-exists": {
-			bindings:        []*bindingInfo{binding1aBound},
+			bindings:        []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{},
-			cachedPVs:       []*v1.PersistentVolume{pvNode1aBound},
+			initPVs:         []*v1.PersistentVolume{pvNode1aBound},
 			shouldFail:      true,
 		},
 		"binding-pv-not-exists": {
-			bindings:        []*bindingInfo{binding1aBound},
+			bindings:        []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{},
-			cachedPVCs:      []*v1.PersistentVolumeClaim{boundPVCNode1a},
+			initPVs:         []*v1.PersistentVolume{pvNode1aBound},
+			initPVCs:        []*v1.PersistentVolumeClaim{boundPVCNode1a},
+			deletePVs:       true,
 			shouldFail:      true,
 		},
 		"binding-claimref-nil": {
-			bindings:        []*bindingInfo{binding1aBound},
+			bindings:        []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{},
-			cachedPVs:       []*v1.PersistentVolume{pvNode1a},
+			initPVs:         []*v1.PersistentVolume{pvNode1a},
-			cachedPVCs:      []*v1.PersistentVolumeClaim{boundPVCNode1a},
+			initPVCs:        []*v1.PersistentVolumeClaim{boundPVCNode1a},
+			apiPVs:          []*v1.PersistentVolume{pvNode1a},
+			apiPVCs:         []*v1.PersistentVolumeClaim{boundPVCNode1a},
 			shouldFail:      true,
 		},
 		"binding-claimref-uid-empty": {
-			bindings:        []*bindingInfo{binding1aBound},
+			bindings:        []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{},
-			cachedPVs:       []*v1.PersistentVolume{pvRemoveClaimUID(pvNode1aBound)},
+			initPVs:         []*v1.PersistentVolume{pvNode1aBound},
-			cachedPVCs:      []*v1.PersistentVolumeClaim{boundPVCNode1a},
+			initPVCs:        []*v1.PersistentVolumeClaim{boundPVCNode1a},
+			apiPVs:          []*v1.PersistentVolume{pvRemoveClaimUID(pvNode1aBound)},
+			apiPVCs:         []*v1.PersistentVolumeClaim{boundPVCNode1a},
 			shouldFail:      true,
 		},
 		"binding-one-bound,one-unbound": {
-			bindings:        []*bindingInfo{binding1aBound, binding1bBound},
+			bindings:        []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound), makeBinding(unboundPVC2, pvNode1bBound)},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{},
-			cachedPVs:       []*v1.PersistentVolume{pvNode1aBound, pvNode1bBound},
+			initPVs:         []*v1.PersistentVolume{pvNode1aBound, pvNode1bBound},
-			cachedPVCs:      []*v1.PersistentVolumeClaim{boundPVCNode1a, unboundPVC2},
+			initPVCs:        []*v1.PersistentVolumeClaim{boundPVCNode1a, unboundPVC2},
 		},
 		"provisioning-pvc-bound": {
 			bindings:        []*bindingInfo{},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)},
-			cachedPVs:       []*v1.PersistentVolume{pvBound},
+			initPVs:         []*v1.PersistentVolume{pvBound},
-			cachedPVCs:      []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVCBound)},
+			initPVCs:        []*v1.PersistentVolumeClaim{provisionedPVCBound},
+			apiPVCs:         []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVCBound)},
 			expectedBound:   true,
 		},
 		"provisioning-pvc-unbound": {
 			bindings:        []*bindingInfo{},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)},
-			cachedPVCs:      []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)},
+			initPVCs:        []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)},
 		},
 		"provisioning-pvc-not-exists": {
 			bindings:        []*bindingInfo{},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)},
+			initPVCs:        []*v1.PersistentVolumeClaim{provisionedPVC},
+			deletePVCs:      true,
 			shouldFail:      true,
 		},
 		"provisioning-pvc-annotations-nil": {
 			bindings:        []*bindingInfo{},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)},
-			cachedPVCs:      []*v1.PersistentVolumeClaim{provisionedPVC},
+			initPVCs:        []*v1.PersistentVolumeClaim{provisionedPVC},
+			apiPVCs:         []*v1.PersistentVolumeClaim{provisionedPVC},
 			shouldFail:      true,
 		},
 		"provisioning-pvc-selected-node-dropped": {
 			bindings:        []*bindingInfo{},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)},
-			cachedPVCs:      []*v1.PersistentVolumeClaim{pvcSetEmptyAnnotations(provisionedPVC)},
+			initPVCs:        []*v1.PersistentVolumeClaim{provisionedPVC},
+			apiPVCs:         []*v1.PersistentVolumeClaim{pvcSetEmptyAnnotations(provisionedPVC)},
 			shouldFail:      true,
 		},
 		"provisioning-pvc-selected-node-wrong-node": {
+			initPVCs:        []*v1.PersistentVolumeClaim{provisionedPVC},
 			bindings:        []*bindingInfo{},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)},
-			cachedPVCs:      []*v1.PersistentVolumeClaim{pvcSetSelectedNode(provisionedPVC, "wrong-node")},
+			apiPVCs:         []*v1.PersistentVolumeClaim{pvcSetSelectedNode(provisionedPVC, "wrong-node")},
 			shouldFail:      true,
 		},
 		"binding-bound-provisioning-unbound": {
-			bindings:        []*bindingInfo{binding1aBound},
+			bindings:        []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)},
 			provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)},
-			cachedPVs:       []*v1.PersistentVolume{pvNode1aBound},
+			initPVs:         []*v1.PersistentVolume{pvNode1aBound},
-			cachedPVCs:      []*v1.PersistentVolumeClaim{boundPVCNode1a, addProvisionAnn(provisionedPVC)},
+			initPVCs:        []*v1.PersistentVolumeClaim{boundPVCNode1a, addProvisionAnn(provisionedPVC)},
+		},
+		"tolerate-provisioning-pvc-bound-pv-not-found": {
+			initPVs:         []*v1.PersistentVolume{pvNode1a},
+			initPVCs:        []*v1.PersistentVolumeClaim{provisionedPVC},
+			bindings:        []*bindingInfo{},
+			provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)},
+			apiPVCs:         []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVCBound)},
+			deletePVs:       true,
 		},
 	}
 
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	for name, scenario := range scenarios {
 		klog.V(4).Infof("Running test case %q", name)
 
 		// Setup
 		pod := makePod(nil)
-		testEnv := newTestBinder(t)
+		testEnv := newTestBinder(t, ctx.Done())
 		testEnv.initNodes([]*v1.Node{node1})
-		testEnv.initVolumes(scenario.cachedPVs, nil)
+		testEnv.initVolumes(scenario.initPVs, nil)
-		testEnv.initClaims(scenario.cachedPVCs, nil)
+		testEnv.initClaims(scenario.initPVCs, nil)
+		testEnv.assumeVolumes(t, name, "node1", pod, scenario.bindings, scenario.provisionedPVCs)
+
+		// Before execute
+		if scenario.deletePVs {
+			testEnv.deleteVolumes(scenario.initPVs)
+		} else {
+			testEnv.updateVolumes(t, scenario.apiPVs, true)
+		}
+		if scenario.deletePVCs {
+			testEnv.deleteClaims(scenario.initPVCs)
+		} else {
+			testEnv.updateClaims(t, scenario.apiPVCs, true)
+		}
 
 		// Execute
 		allBound, err := testEnv.internalBinder.checkBindings(pod, scenario.bindings, scenario.provisionedPVCs)
@@ -1302,63 +1439,96 @@ func TestCheckBindings(t *testing.T) {
 }
 
 func TestBindPodVolumes(t *testing.T) {
-	scenarios := map[string]struct {
+	type scenarioType struct {
 		// Inputs
-		// These tests only support a single pv and pvc and static binding
 		bindingsNil bool // Pass in nil bindings slice
-		binding         *bindingInfo
+
-		cachedPVs       []*v1.PersistentVolume
-		cachedPVCs      []*v1.PersistentVolumeClaim
-		provisionedPVCs []*v1.PersistentVolumeClaim
-		apiPVs          []*v1.PersistentVolume
 		nodes []*v1.Node
 
+		// before assume
+		initPVs  []*v1.PersistentVolume
+		initPVCs []*v1.PersistentVolumeClaim
+
+		// assume PV & PVC with these binding results
+		binding          *bindingInfo
+		claimToProvision *v1.PersistentVolumeClaim
+
+		// API updates after assume before bind
+		apiPV  *v1.PersistentVolume
+		apiPVC *v1.PersistentVolumeClaim
+
 		// This function runs with a delay of 5 seconds
-		delayFunc func(*testing.T, *testEnv, *v1.Pod, *bindingInfo, []*v1.PersistentVolumeClaim)
+		delayFunc func(t *testing.T, testEnv *testEnv, pod *v1.Pod, pvs []*v1.PersistentVolume, pvcs []*v1.PersistentVolumeClaim)
 
 		// Expected return values
 		shouldFail bool
-	}{
+	}
+
+	scenarios := map[string]scenarioType{
 		"nothing-to-bind-nil": {
 			bindingsNil: true,
 			shouldFail:  true,
 		},
 		"nothing-to-bind-empty": {},
 		"already-bound": {
-			binding:    binding1aBound,
+			binding:  makeBinding(unboundPVC, pvNode1aBound),
-			cachedPVs:  []*v1.PersistentVolume{pvNode1aBound},
+			initPVs:  []*v1.PersistentVolume{pvNode1aBound},
-			cachedPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a},
+			initPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a},
 		},
-		"binding-succeeds-after-time": {
+		"binding-static-pv-succeeds-after-time": {
-			binding:    binding1aBound,
+			initPVs:    []*v1.PersistentVolume{pvNode1a},
-			cachedPVs:  []*v1.PersistentVolume{pvNode1a},
+			initPVCs:   []*v1.PersistentVolumeClaim{unboundPVC},
-			cachedPVCs: []*v1.PersistentVolumeClaim{unboundPVC},
+			binding:    makeBinding(unboundPVC, pvNode1aBound),
-			delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, binding *bindingInfo, pvcs []*v1.PersistentVolumeClaim) {
+			shouldFail: false, // Will succeed after PVC is fully bound to this PV by pv controller.
+			delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, pvs []*v1.PersistentVolume, pvcs []*v1.PersistentVolumeClaim) {
+				pvc := pvcs[0]
+				pv := pvs[0]
 				// Update PVC to be fully bound to PV
-				newPVC := binding.pvc.DeepCopy()
+				newPVC := pvc.DeepCopy()
-				newPVC.ResourceVersion = "100"
+				newPVC.Spec.VolumeName = pv.Name
-				newPVC.Spec.VolumeName = binding.pv.Name
 				metav1.SetMetaDataAnnotation(&newPVC.ObjectMeta, annBindCompleted, "yes")
-
+				if _, err := testEnv.client.CoreV1().PersistentVolumeClaims(newPVC.Namespace).Update(newPVC); err != nil {
-				// Update pvc cache, fake client doesn't invoke informers
+					t.Errorf("failed to update PVC %q: %v", newPVC.Name, err)
-				internalBinder, ok := testEnv.binder.(*volumeBinder)
-				if !ok {
-					t.Fatalf("Failed to convert to internal binder")
 				}
-
-				pvcCache := internalBinder.pvcCache
-				internalPVCCache, ok := pvcCache.(*pvcAssumeCache)
-				if !ok {
-					t.Fatalf("Failed to convert to internal PVC cache")
-				}
-				internalPVCCache.add(newPVC)
 			},
 		},
+		"binding-dynamic-pv-succeeds-after-time": {
+			claimToProvision: pvcSetSelectedNode(provisionedPVC, "node1"),
+			initPVCs:         []*v1.PersistentVolumeClaim{provisionedPVC},
+			delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, pvs []*v1.PersistentVolume, pvcs []*v1.PersistentVolumeClaim) {
+				pvc := pvcs[0]
+				// Update PVC to be fully bound to PV
+				newPVC, err := testEnv.client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvc.Name, metav1.GetOptions{})
+				if err != nil {
+					t.Errorf("failed to get PVC %q: %v", pvc.Name, err)
+					return
+				}
+				dynamicPV := makeTestPV("dynamic-pv", "node1", "1G", "1", newPVC, waitClass)
+				dynamicPV, err = testEnv.client.CoreV1().PersistentVolumes().Create(dynamicPV)
+				if err != nil {
+					t.Errorf("failed to create PV %q: %v", dynamicPV.Name, err)
+					return
+				}
+				newPVC.Spec.VolumeName = dynamicPV.Name
+				metav1.SetMetaDataAnnotation(&newPVC.ObjectMeta, annBindCompleted, "yes")
+				if _, err := testEnv.client.CoreV1().PersistentVolumeClaims(newPVC.Namespace).Update(newPVC); err != nil {
+					t.Errorf("failed to update PVC %q: %v", newPVC.Name, err)
+				}
+			},
+		},
+		"bound-by-pv-controller-before-bind": {
+			initPVs:    []*v1.PersistentVolume{pvNode1a},
+			initPVCs:   []*v1.PersistentVolumeClaim{unboundPVC},
+			binding:    makeBinding(unboundPVC, pvNode1aBound),
+			apiPV:      pvNode1aBound,
+			apiPVC:     boundPVCNode1a,
+			shouldFail: true, // bindAPIUpdate will fail because API conflict
+		},
 		"pod-deleted-after-time": {
-			binding:    binding1aBound,
+			binding:  makeBinding(unboundPVC, pvNode1aBound),
-			cachedPVs:  []*v1.PersistentVolume{pvNode1a},
+			initPVs:  []*v1.PersistentVolume{pvNode1a},
-			cachedPVCs: []*v1.PersistentVolumeClaim{unboundPVC},
+			initPVCs: []*v1.PersistentVolumeClaim{unboundPVC},
-			delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, binding *bindingInfo, pvcs []*v1.PersistentVolumeClaim) {
+			delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, pvs []*v1.PersistentVolume, pvcs []*v1.PersistentVolumeClaim) {
 				bindingsCache := testEnv.binder.GetBindingsCache()
 				if bindingsCache == nil {
 					t.Fatalf("Failed to get bindings cache")
@@ -1376,107 +1546,103 @@ func TestBindPodVolumes(t *testing.T) {
 			shouldFail: true,
 		},
 		"binding-times-out": {
-			binding:    binding1aBound,
+			binding:    makeBinding(unboundPVC, pvNode1aBound),
-			cachedPVs:  []*v1.PersistentVolume{pvNode1a},
+			initPVs:    []*v1.PersistentVolume{pvNode1a},
-			cachedPVCs: []*v1.PersistentVolumeClaim{unboundPVC},
+			initPVCs:   []*v1.PersistentVolumeClaim{unboundPVC},
 			shouldFail: true,
 		},
 		"binding-fails": {
-			binding:    binding1bBound,
+			binding:    makeBinding(unboundPVC2, pvNode1bBound),
-			cachedPVs:  []*v1.PersistentVolume{pvNode1b},
+			initPVs:    []*v1.PersistentVolume{pvNode1b},
-			apiPVs:     []*v1.PersistentVolume{pvNode1bBoundHigherVersion},
+			initPVCs:   []*v1.PersistentVolumeClaim{unboundPVC2},
-			cachedPVCs: []*v1.PersistentVolumeClaim{unboundPVC2},
 			shouldFail: true,
 		},
 		"check-fails": {
-			binding:    binding1aBound,
+			binding:  makeBinding(unboundPVC, pvNode1aBound),
-			cachedPVs:  []*v1.PersistentVolume{pvNode1a},
+			initPVs:  []*v1.PersistentVolume{pvNode1a},
-			cachedPVCs: []*v1.PersistentVolumeClaim{unboundPVC},
+			initPVCs: []*v1.PersistentVolumeClaim{unboundPVC},
-			delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, binding *bindingInfo, pvcs []*v1.PersistentVolumeClaim) {
+			delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, pvs []*v1.PersistentVolume, pvcs []*v1.PersistentVolumeClaim) {
-				// Delete PVC
+				pvc := pvcs[0]
-				// Update pvc cache, fake client doesn't invoke informers
+				// Delete PVC will fail check
-				internalBinder, ok := testEnv.binder.(*volumeBinder)
+				if err := testEnv.client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, &metav1.DeleteOptions{}); err != nil {
-				if !ok {
+					t.Errorf("failed to delete PVC %q: %v", pvc.Name, err)
-					t.Fatalf("Failed to convert to internal binder")
 				}
-
-				pvcCache := internalBinder.pvcCache
-				internalPVCCache, ok := pvcCache.(*pvcAssumeCache)
-				if !ok {
-					t.Fatalf("Failed to convert to internal PVC cache")
-				}
-				internalPVCCache.delete(binding.pvc)
 			},
 			shouldFail: true,
 		},
 		"node-affinity-fails": {
-			binding:    binding1aBound,
+			binding:    makeBinding(unboundPVC, pvNode1aBound),
-			cachedPVs:  []*v1.PersistentVolume{pvNode1aBound},
+			initPVs:    []*v1.PersistentVolume{pvNode1aBound},
-			cachedPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a},
+			initPVCs:   []*v1.PersistentVolumeClaim{boundPVCNode1a},
 			nodes:      []*v1.Node{node1NoLabels},
 			shouldFail: true,
 		},
 		"node-affinity-fails-dynamic-provisioning": {
-			cachedPVs:       []*v1.PersistentVolume{pvNode1a, pvNode2},
+			initPVs:          []*v1.PersistentVolume{pvNode1a, pvNode2},
-			cachedPVCs:      []*v1.PersistentVolumeClaim{selectedNodePVC},
+			initPVCs:         []*v1.PersistentVolumeClaim{selectedNodePVC},
-			provisionedPVCs: []*v1.PersistentVolumeClaim{selectedNodePVC},
+			claimToProvision: selectedNodePVC,
 			nodes:            []*v1.Node{node1, node2},
-			delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, binding *bindingInfo, pvcs []*v1.PersistentVolumeClaim) {
+			delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, pvs []*v1.PersistentVolume, pvcs []*v1.PersistentVolumeClaim) {
 				// Update PVC to be fully bound to a PV with a different node
 				newPVC := pvcs[0].DeepCopy()
-				newPVC.ResourceVersion = "100"
 				newPVC.Spec.VolumeName = pvNode2.Name
 				metav1.SetMetaDataAnnotation(&newPVC.ObjectMeta, annBindCompleted, "yes")
-
+				if _, err := testEnv.client.CoreV1().PersistentVolumeClaims(newPVC.Namespace).Update(newPVC); err != nil {
-				// Update PVC cache, fake client doesn't invoke informers
+					t.Errorf("failed to update PVC %q: %v", newPVC.Name, err)
-				internalBinder, ok := testEnv.binder.(*volumeBinder)
-				if !ok {
-					t.Fatalf("Failed to convert to internal binder")
 				}
-
-				pvcCache := internalBinder.pvcCache
-				internalPVCCache, ok := pvcCache.(*pvcAssumeCache)
-				if !ok {
-					t.Fatalf("Failed to convert to internal PVC cache")
-				}
-				internalPVCCache.add(newPVC)
 			},
 			shouldFail: true,
 		},
 	}
 
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	for name, scenario := range scenarios {
 		klog.V(4).Infof("Running test case %q", name)
 
 		// Setup
 		pod := makePod(nil)
-		if scenario.apiPVs == nil {
+		testEnv := newTestBinder(t, ctx.Done())
-			scenario.apiPVs = scenario.cachedPVs
-		}
 		if scenario.nodes == nil {
 			scenario.nodes = []*v1.Node{node1}
 		}
-		if scenario.provisionedPVCs == nil {
-			scenario.provisionedPVCs = []*v1.PersistentVolumeClaim{}
-		}
-		testEnv := newTestBinder(t)
 		if !scenario.bindingsNil {
 			bindings := []*bindingInfo{}
 			if scenario.binding != nil {
 				bindings = []*bindingInfo{scenario.binding}
 			}
+			claimsToProvision := []*v1.PersistentVolumeClaim{}
+			if scenario.claimToProvision != nil {
+				claimsToProvision = []*v1.PersistentVolumeClaim{scenario.claimToProvision}
+			}
 			testEnv.initNodes(scenario.nodes)
-			testEnv.initVolumes(scenario.cachedPVs, scenario.apiPVs)
+			testEnv.initVolumes(scenario.initPVs, scenario.initPVs)
-			testEnv.initClaims(scenario.cachedPVCs, nil)
+			testEnv.initClaims(scenario.initPVCs, scenario.initPVCs)
-			testEnv.assumeVolumes(t, name, "node1", pod, bindings, scenario.provisionedPVCs)
+			testEnv.assumeVolumes(t, name, "node1", pod, bindings, claimsToProvision)
+		}
+
+		// Before Execute
+		if scenario.apiPV != nil {
+			_, err := testEnv.client.CoreV1().PersistentVolumes().Update(scenario.apiPV)
+			if err != nil {
+				t.Fatalf("Test %q failed: failed to update PV %q", name, scenario.apiPV.Name)
+			}
+		}
+		if scenario.apiPVC != nil {
+			_, err := testEnv.client.CoreV1().PersistentVolumeClaims(scenario.apiPVC.Namespace).Update(scenario.apiPVC)
+			if err != nil {
+				t.Fatalf("Test %q failed: failed to update PVC %q", name, getPVCName(scenario.apiPVC))
+			}
 		}
 
 		if scenario.delayFunc != nil {
-			go func() {
+			go func(scenario scenarioType) {
 				time.Sleep(5 * time.Second)
+				// Sleep a while to run after bindAPIUpdate in BindPodVolumes
 				klog.V(5).Infof("Running delay function")
-				scenario.delayFunc(t, testEnv, pod, scenario.binding, scenario.provisionedPVCs)
+				scenario.delayFunc(t, testEnv, pod, scenario.initPVs, scenario.initPVCs)
-			}()
+			}(scenario)
 		}
 
 		// Execute
@@ -1498,7 +1664,9 @@ func TestFindAssumeVolumes(t *testing.T) {
 	pvs := []*v1.PersistentVolume{pvNode2, pvNode1a, pvNode1c}
 
 	// Setup
-	testEnv := newTestBinder(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	testEnv := newTestBinder(t, ctx.Done())
 	testEnv.initVolumes(pvs, pvs)
 	testEnv.initClaims(podPVCs, podPVCs)
 	pod := makePod(podPVCs)
@@ -1548,6 +1716,6 @@ func TestFindAssumeVolumes(t *testing.T) {
 		if !unboundSatisfied {
 			t.Errorf("Test failed: couldn't find PVs for all PVCs")
 		}
-		testEnv.validatePodCache(t, "after-assume", testNode.Name, pod, expectedBindings, []*v1.PersistentVolumeClaim{})
+		testEnv.validatePodCache(t, "after-assume", testNode.Name, pod, expectedBindings, nil)
 	}
 }