From 18de83c6417dcc51d53abbb96c0414d05fec3930 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Wed, 2 Nov 2016 14:13:34 +0100 Subject: [PATCH 1/2] Implement external provisioning proposal In other words, add "provisioned-by" annotation to all PVCs that should be provisioned dynamically. --- .../volume/persistentvolume/framework_test.go | 13 ++++ .../volume/persistentvolume/provision_test.go | 32 +++++----- .../volume/persistentvolume/pv_controller.go | 63 ++++++++++++------- .../persistentvolume/pv_controller_base.go | 30 +++++++++ 4 files changed, 100 insertions(+), 38 deletions(-) diff --git a/pkg/controller/volume/persistentvolume/framework_test.go b/pkg/controller/volume/persistentvolume/framework_test.go index 971347d264c..1048e7b8613 100644 --- a/pkg/controller/volume/persistentvolume/framework_test.go +++ b/pkg/controller/volume/persistentvolume/framework_test.go @@ -750,6 +750,8 @@ func newClaim(name, claimUID, capacity, boundToVolume string, phase api.Persiste switch a { case storageutil.StorageClassAnnotation: claim.Annotations[a] = "gold" + case annStorageProvisioner: + claim.Annotations[a] = mockPluginName default: claim.Annotations[a] = "yes" } @@ -786,6 +788,17 @@ func claimWithClass(className string, claims []*api.PersistentVolumeClaim) []*ap return claims } +// claimWithAnnotation saves given annotation into given claims. +// Meant to be used to compose claims specified inline in a test. +func claimWithAnnotation(name, value string, claims []*api.PersistentVolumeClaim) []*api.PersistentVolumeClaim { + if claims[0].Annotations == nil { + claims[0].Annotations = map[string]string{name: value} + } else { + claims[0].Annotations[name] = value + } + return claims +} + func testSyncClaim(ctrl *PersistentVolumeController, reactor *volumeReactor, test controllerTest) error { return ctrl.syncClaim(test.initialClaims[0]) } diff --git a/pkg/controller/volume/persistentvolume/provision_test.go b/pkg/controller/volume/persistentvolume/provision_test.go index 870fd2de9c8..2f253d9e35f 100644 --- a/pkg/controller/volume/persistentvolume/provision_test.go +++ b/pkg/controller/volume/persistentvolume/provision_test.go @@ -79,7 +79,7 @@ var storageClasses = []*storage.StorageClass{ // call to storageClass 1, returning an error var provision1Error = provisionCall{ - ret: errors.New("Moc provisioner error"), + ret: errors.New("Mock provisioner error"), expectedParameters: class1Parameters, } @@ -112,7 +112,7 @@ func TestProvisionSync(t *testing.T) { newVolumeArray("pvc-uid11-1", "1Gi", "uid11-1", "claim11-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, storageutil.StorageClassAnnotation), newClaimArray("claim11-1", "uid11-1", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), // Binding will be completed in the next syncClaim - newClaimArray("claim11-1", "uid11-1", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-1", "uid11-1", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision1Success}, testSyncClaim), }, { @@ -131,7 +131,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-3", "uid11-3", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), - newClaimArray("claim11-3", "uid11-3", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-3", "uid11-3", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), []string{"Warning ProvisioningFailed"}, noerrors, wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), }, @@ -141,7 +141,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-4", "uid11-4", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), - newClaimArray("claim11-4", "uid11-4", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-4", "uid11-4", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), []string{"Warning ProvisioningFailed"}, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision1Error}, testSyncClaim), }, @@ -165,7 +165,7 @@ func TestProvisionSync(t *testing.T) { newVolumeArray("pvc-uid11-7", "1Gi", "uid11-7", "claim11-7", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, storageutil.StorageClassAnnotation), newClaimArray("claim11-7", "uid11-7", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), // The claim would be bound in next syncClaim - newClaimArray("claim11-7", "uid11-7", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-7", "uid11-7", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), noevents, noerrors, wrapTestWithInjectedOperation(wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), func(ctrl *PersistentVolumeController, reactor *volumeReactor) { // Create a volume before provisionClaimOperation starts. @@ -184,7 +184,7 @@ func TestProvisionSync(t *testing.T) { newVolumeArray("pvc-uid11-8", "1Gi", "uid11-8", "claim11-8", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, storageutil.StorageClassAnnotation), newClaimArray("claim11-8", "uid11-8", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), // Binding will be completed in the next syncClaim - newClaimArray("claim11-8", "uid11-8", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-8", "uid11-8", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), noevents, []reactorError{ // Inject error to the first @@ -201,7 +201,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-9", "uid11-9", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), - newClaimArray("claim11-9", "uid11-9", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-9", "uid11-9", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), []string{"Warning ProvisioningFailed"}, []reactorError{ // Inject error to five kubeclient.PersistentVolumes.Create() @@ -226,7 +226,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-10", "uid11-10", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), - newClaimArray("claim11-10", "uid11-10", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-10", "uid11-10", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), []string{"Warning ProvisioningFailed", "Warning ProvisioningCleanupFailed"}, []reactorError{ // Inject error to five kubeclient.PersistentVolumes.Create() @@ -247,7 +247,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-11", "uid11-11", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), - newClaimArray("claim11-11", "uid11-11", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-11", "uid11-11", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), []string{"Warning ProvisioningFailed", "Warning ProvisioningCleanupFailed"}, []reactorError{ // Inject error to five kubeclient.PersistentVolumes.Create() @@ -277,7 +277,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-12", "uid11-12", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), - newClaimArray("claim11-12", "uid11-12", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-12", "uid11-12", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), []string{"Warning ProvisioningFailed"}, []reactorError{ // Inject error to five kubeclient.PersistentVolumes.Create() @@ -305,7 +305,7 @@ func TestProvisionSync(t *testing.T) { volumeWithClass("silver", newVolumeArray("pvc-uid11-13", "1Gi", "uid11-13", "claim11-13", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned)), claimWithClass("silver", newClaimArray("claim11-13", "uid11-13", "1Gi", "", api.ClaimPending)), // Binding will be completed in the next syncClaim - claimWithClass("silver", newClaimArray("claim11-13", "uid11-13", "1Gi", "", api.ClaimPending)), + claimWithClass("silver", newClaimArray("claim11-13", "uid11-13", "1Gi", "", api.ClaimPending, annStorageProvisioner)), noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision2Success}, testSyncClaim), }, { @@ -341,7 +341,8 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, claimWithClass("external", newClaimArray("claim11-17", "uid11-17", "1Gi", "", api.ClaimPending)), - claimWithClass("external", newClaimArray("claim11-17", "uid11-17", "1Gi", "", api.ClaimPending)), + claimWithAnnotation(annStorageProvisioner, "vendor.com/my-volume", + claimWithClass("external", newClaimArray("claim11-17", "uid11-17", "1Gi", "", api.ClaimPending))), []string{"Normal ExternalProvisioning"}, noerrors, wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), }, @@ -368,7 +369,7 @@ func TestAlphaProvisionSync(t *testing.T) { newVolumeArray("pvc-uid14-1", "1Gi", "uid14-1", "claim14-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned), newClaimArray("claim14-1", "uid14-1", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation), // Binding will be completed in the next syncClaim - newClaimArray("claim14-1", "uid14-1", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation), + newClaimArray("claim14-1", "uid14-1", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation, annStorageProvisioner), noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provisionAlphaSuccess}, testSyncClaim), }, { @@ -382,7 +383,7 @@ func TestAlphaProvisionSync(t *testing.T) { }, newClaimArray("claim14-2", "uid14-2", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation), // Binding will be completed in the next syncClaim - newClaimArray("claim14-2", "uid14-2", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation), + newClaimArray("claim14-2", "uid14-2", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation, annStorageProvisioner), noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provisionAlphaSuccess}, testSyncClaim), }, } @@ -411,8 +412,7 @@ func TestProvisionMultiSync(t *testing.T) { novolumes, newVolumeArray("pvc-uid12-1", "1Gi", "uid12-1", "claim12-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, storageutil.StorageClassAnnotation), newClaimArray("claim12-1", "uid12-1", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), - // Binding will be completed in the next syncClaim - newClaimArray("claim12-1", "uid12-1", "1Gi", "pvc-uid12-1", api.ClaimBound, storageutil.StorageClassAnnotation, annBoundByController, annBindCompleted), + newClaimArray("claim12-1", "uid12-1", "1Gi", "pvc-uid12-1", api.ClaimBound, storageutil.StorageClassAnnotation, annBoundByController, annBindCompleted, annStorageProvisioner), noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision1Success}, testSyncClaim), }, } diff --git a/pkg/controller/volume/persistentvolume/pv_controller.go b/pkg/controller/volume/persistentvolume/pv_controller.go index 72f528d3a88..85ebc909de8 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller.go +++ b/pkg/controller/volume/persistentvolume/pv_controller.go @@ -117,6 +117,11 @@ const annBoundByController = "pv.kubernetes.io/bound-by-controller" // recognize dynamically provisioned PVs in its decisions). const annDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by" +// This annotation is added to a PVC that is supposed to be dynamically +// provisioned. Its value is name of volume plugin that is supposed to provision +// a volume for this PVC. +const annStorageProvisioner = "volume.beta.kubernetes.io/storage-provisioner" + // Name of a tag attached to a real volume in cloud (e.g. AWS EBS or GCE PD) // with namespace of a persistent volume claim used to create this volume. const cloudVolumeCreatedForClaimNamespaceTag = "kubernetes.io/created-for/pvc/namespace" @@ -1197,6 +1202,42 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa claimClass := storageutil.GetClaimStorageClass(claim) glog.V(4).Infof("provisionClaimOperation [%s] started, class: %q", claimToClaimKey(claim), claimClass) + plugin, storageClass, err := ctrl.findProvisionablePlugin(claim) + if err != nil { + ctrl.eventRecorder.Event(claim, api.EventTypeWarning, "ProvisioningFailed", err.Error()) + glog.V(2).Infof("error finding provisioning plugin for claim %s: %v", claimToClaimKey(claim), err) + // The controller will retry provisioning the volume in every + // syncVolume() call. + return + } + + if storageClass != nil { + // Add provisioner annotation so external provisioners know when to start + newClaim, err := ctrl.setClaimProvisioner(claim, storageClass) + if err != nil { + // Save failed, the controller will retry in the next sync + glog.V(2).Infof("error saving claim %s: %v", claimToClaimKey(claim), err) + return + } + claim = newClaim + } + + if plugin == nil { + // findProvisionablePlugin returned no error nor plugin. + // This means that an unknown provisioner is requested. Report an event + // and wait for the external provisioner + if storageClass != nil { + msg := fmt.Sprintf("cannot find provisioner %q, expecting that a volume for the claim is provisioned either manually or via external software", storageClass.Provisioner) + ctrl.eventRecorder.Event(claim, api.EventTypeNormal, "ExternalProvisioning", msg) + glog.V(3).Infof("provisioning claim %q: %s", claimToClaimKey(claim), msg) + } else { + glog.V(3).Infof("cannot find storage class for claim %q", claimToClaimKey(claim)) + } + return + } + + // internal provisioning + // A previous doProvisionClaim may just have finished while we were waiting for // the locks. Check that PV (with deterministic name) hasn't been provisioned // yet. @@ -1217,28 +1258,6 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa return } - plugin, storageClass, err := ctrl.findProvisionablePlugin(claim) - if err != nil { - ctrl.eventRecorder.Event(claim, api.EventTypeWarning, "ProvisioningFailed", err.Error()) - glog.V(2).Infof("error finding provisioning plugin for claim %s: %v", claimToClaimKey(claim), err) - // The controller will retry provisioning the volume in every - // syncVolume() call. - return - } - if plugin == nil { - // findProvisionablePlugin returned no error nor plugin. - // This means that an unknown provisioner is requested. Report an event - // and wait for the external provisioner - if storageClass != nil { - msg := fmt.Sprintf("cannot find provisioner %q, expecting that a volume for the claim is provisioned either manually or via external software", storageClass.Provisioner) - ctrl.eventRecorder.Event(claim, api.EventTypeNormal, "ExternalProvisioning", msg) - glog.V(3).Infof("provisioning claim %q: %s", claimToClaimKey(claim), msg) - } else { - glog.V(3).Infof("cannot find storage class for claim %q", claimToClaimKey(claim)) - } - return - } - // Gather provisioning options tags := make(map[string]string) tags[cloudVolumeCreatedForClaimNamespaceTag] = claim.Namespace diff --git a/pkg/controller/volume/persistentvolume/pv_controller_base.go b/pkg/controller/volume/persistentvolume/pv_controller_base.go index d786b67241b..2706da0fe70 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller_base.go +++ b/pkg/controller/volume/persistentvolume/pv_controller_base.go @@ -494,6 +494,36 @@ func (ctrl *PersistentVolumeController) upgradeVolumeFrom1_2(volume *api.Persist return true } +// setClaimProvisioner saves +// claim.Annotations[annStorageProvisioner] = class.Provisioner +func (ctrl *PersistentVolumeController) setClaimProvisioner(claim *api.PersistentVolumeClaim, class *storage.StorageClass) (*api.PersistentVolumeClaim, error) { + if val, ok := claim.Annotations[annDynamicallyProvisioned]; ok && val == class.Provisioner { + // annotation is already set, nothing to do + return claim, nil + } + + // The volume from method args can be pointing to watcher cache. We must not + // modify these, therefore create a copy. + clone, err := conversion.NewCloner().DeepCopy(claim) + if err != nil { + return nil, fmt.Errorf("Error cloning pv: %v", err) + } + claimClone, ok := clone.(*api.PersistentVolumeClaim) + if !ok { + return nil, fmt.Errorf("Unexpected claim cast error : %v", claimClone) + } + api.SetMetaDataAnnotation(&claimClone.ObjectMeta, annStorageProvisioner, class.Provisioner) + newClaim, err := ctrl.kubeClient.Core().PersistentVolumeClaims(claim.Namespace).Update(claimClone) + if err != nil { + return newClaim, err + } + _, err = ctrl.storeClaimUpdate(newClaim) + if err != nil { + return newClaim, err + } + return newClaim, nil +} + // Stateless functions func getClaimStatusForLogging(claim *api.PersistentVolumeClaim) string { From 2224e80dd7ba8ed8238afcefa64777d8c0cf4c84 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Thu, 3 Nov 2016 16:58:25 +0100 Subject: [PATCH 2/2] Fix race when two provisioner create two PVs for a single claim. --- .../volume/persistentvolume/delete_test.go | 43 +++++++++++++++++++ .../volume/persistentvolume/pv_controller.go | 17 ++++++++ 2 files changed, 60 insertions(+) diff --git a/pkg/controller/volume/persistentvolume/delete_test.go b/pkg/controller/volume/persistentvolume/delete_test.go index e77ac6640a0..59a968c98de 100644 --- a/pkg/controller/volume/persistentvolume/delete_test.go +++ b/pkg/controller/volume/persistentvolume/delete_test.go @@ -148,6 +148,49 @@ func TestDeleteSync(t *testing.T) { return testSyncVolume(ctrl, reactor, test) }, }, + { + // delete success - two PVs are provisioned for a single claim. + // One of the PVs is deleted. + "8-11 - two PVs provisioned for a single claim", + []*api.PersistentVolume{ + newVolume("volume8-11-1", "1Gi", "uid8-11", "claim8-11", api.VolumeBound, api.PersistentVolumeReclaimDelete, annDynamicallyProvisioned), + newVolume("volume8-11-2", "1Gi", "uid8-11", "claim8-11", api.VolumeBound, api.PersistentVolumeReclaimDelete, annDynamicallyProvisioned), + }, + []*api.PersistentVolume{ + newVolume("volume8-11-2", "1Gi", "uid8-11", "claim8-11", api.VolumeBound, api.PersistentVolumeReclaimDelete, annDynamicallyProvisioned), + }, + // the claim is bound to volume8-11-2 -> volume8-11-1 has lost the race and will be deleted + newClaimArray("claim8-11", "uid8-11", "10Gi", "volume8-11-2", api.ClaimBound), + newClaimArray("claim8-11", "uid8-11", "10Gi", "volume8-11-2", api.ClaimBound), + noevents, noerrors, + // Inject deleter into the controller and call syncVolume. The + // deleter simulates one delete() call that succeeds. + wrapTestWithReclaimCalls(operationDelete, []error{nil}, testSyncVolume), + }, + { + // delete success - two PVs are externally provisioned for a single + // claim. One of the PVs is marked as Released to be deleted by the + // external provisioner. + "8-12 - two PVs externally provisioned for a single claim", + []*api.PersistentVolume{ + newVolume("volume8-12-1", "1Gi", "uid8-12", "claim8-12", api.VolumeBound, api.PersistentVolumeReclaimDelete, annDynamicallyProvisioned), + newVolume("volume8-12-2", "1Gi", "uid8-12", "claim8-12", api.VolumeBound, api.PersistentVolumeReclaimDelete, annDynamicallyProvisioned), + }, + []*api.PersistentVolume{ + newVolume("volume8-12-1", "1Gi", "uid8-12", "claim8-12", api.VolumeReleased, api.PersistentVolumeReclaimDelete, annDynamicallyProvisioned), + newVolume("volume8-12-2", "1Gi", "uid8-12", "claim8-12", api.VolumeBound, api.PersistentVolumeReclaimDelete, annDynamicallyProvisioned), + }, + // the claim is bound to volume8-12-2 -> volume8-12-1 has lost the race and will be "Released" + newClaimArray("claim8-12", "uid8-12", "10Gi", "volume8-12-2", api.ClaimBound), + newClaimArray("claim8-12", "uid8-12", "10Gi", "volume8-12-2", api.ClaimBound), + noevents, noerrors, + func(ctrl *PersistentVolumeController, reactor *volumeReactor, test controllerTest) error { + // Inject external deleter annotation + test.initialVolumes[0].Annotations[annDynamicallyProvisioned] = "external.io/test" + test.expectedVolumes[0].Annotations[annDynamicallyProvisioned] = "external.io/test" + return testSyncVolume(ctrl, reactor, test) + }, + }, } runSyncTests(t, tests, []*storage.StorageClass{}) } diff --git a/pkg/controller/volume/persistentvolume/pv_controller.go b/pkg/controller/volume/persistentvolume/pv_controller.go index 85ebc909de8..8a555498e2b 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller.go +++ b/pkg/controller/volume/persistentvolume/pv_controller.go @@ -492,6 +492,17 @@ func (ctrl *PersistentVolumeController) syncVolume(volume *api.PersistentVolume) // This volume was dynamically provisioned for this claim. The // claim got bound elsewhere, and thus this volume is not // needed. Delete it. + // Mark the volume as Released for external deleters and to let + // the user know. Don't overwrite existing Failed status! + if volume.Status.Phase != api.VolumeReleased && volume.Status.Phase != api.VolumeFailed { + // Also, log this only once: + glog.V(2).Infof("dynamically volume %q is released and it will be deleted", volume.Name) + if volume, err = ctrl.updateVolumePhase(volume, api.VolumeReleased, ""); err != nil { + // Nothing was saved; we will fall back into the same condition + // in the next call to this method + return err + } + } if err = ctrl.reclaimVolume(volume); err != nil { // Deletion failed, we will fall back into the same condition // in the next call to this method @@ -1131,6 +1142,12 @@ func (ctrl *PersistentVolumeController) isVolumeReleased(volume *api.PersistentV } if claim != nil && claim.UID == volume.Spec.ClaimRef.UID { // the claim still exists and has the right UID + + if len(claim.Spec.VolumeName) > 0 && claim.Spec.VolumeName != volume.Name { + // the claim is bound to another PV, this PV *is* released + return true, nil + } + glog.V(4).Infof("isVolumeReleased[%s]: ClaimRef is still valid, volume is not released", volume.Name) return false, nil }