From 18de83c6417dcc51d53abbb96c0414d05fec3930 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Wed, 2 Nov 2016 14:13:34 +0100 Subject: [PATCH] Implement external provisioning proposal In other words, add "provisioned-by" annotation to all PVCs that should be provisioned dynamically. --- .../volume/persistentvolume/framework_test.go | 13 ++++ .../volume/persistentvolume/provision_test.go | 32 +++++----- .../volume/persistentvolume/pv_controller.go | 63 ++++++++++++------- .../persistentvolume/pv_controller_base.go | 30 +++++++++ 4 files changed, 100 insertions(+), 38 deletions(-) diff --git a/pkg/controller/volume/persistentvolume/framework_test.go b/pkg/controller/volume/persistentvolume/framework_test.go index 971347d264c..1048e7b8613 100644 --- a/pkg/controller/volume/persistentvolume/framework_test.go +++ b/pkg/controller/volume/persistentvolume/framework_test.go @@ -750,6 +750,8 @@ func newClaim(name, claimUID, capacity, boundToVolume string, phase api.Persiste switch a { case storageutil.StorageClassAnnotation: claim.Annotations[a] = "gold" + case annStorageProvisioner: + claim.Annotations[a] = mockPluginName default: claim.Annotations[a] = "yes" } @@ -786,6 +788,17 @@ func claimWithClass(className string, claims []*api.PersistentVolumeClaim) []*ap return claims } +// claimWithAnnotation saves given annotation into given claims. +// Meant to be used to compose claims specified inline in a test. +func claimWithAnnotation(name, value string, claims []*api.PersistentVolumeClaim) []*api.PersistentVolumeClaim { + if claims[0].Annotations == nil { + claims[0].Annotations = map[string]string{name: value} + } else { + claims[0].Annotations[name] = value + } + return claims +} + func testSyncClaim(ctrl *PersistentVolumeController, reactor *volumeReactor, test controllerTest) error { return ctrl.syncClaim(test.initialClaims[0]) } diff --git a/pkg/controller/volume/persistentvolume/provision_test.go b/pkg/controller/volume/persistentvolume/provision_test.go index 870fd2de9c8..2f253d9e35f 100644 --- a/pkg/controller/volume/persistentvolume/provision_test.go +++ b/pkg/controller/volume/persistentvolume/provision_test.go @@ -79,7 +79,7 @@ var storageClasses = []*storage.StorageClass{ // call to storageClass 1, returning an error var provision1Error = provisionCall{ - ret: errors.New("Moc provisioner error"), + ret: errors.New("Mock provisioner error"), expectedParameters: class1Parameters, } @@ -112,7 +112,7 @@ func TestProvisionSync(t *testing.T) { newVolumeArray("pvc-uid11-1", "1Gi", "uid11-1", "claim11-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, storageutil.StorageClassAnnotation), newClaimArray("claim11-1", "uid11-1", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), // Binding will be completed in the next syncClaim - newClaimArray("claim11-1", "uid11-1", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-1", "uid11-1", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision1Success}, testSyncClaim), }, { @@ -131,7 +131,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-3", "uid11-3", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), - newClaimArray("claim11-3", "uid11-3", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-3", "uid11-3", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), []string{"Warning ProvisioningFailed"}, noerrors, wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), }, @@ -141,7 +141,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-4", "uid11-4", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), - newClaimArray("claim11-4", "uid11-4", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-4", "uid11-4", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), []string{"Warning ProvisioningFailed"}, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision1Error}, testSyncClaim), }, @@ -165,7 +165,7 @@ func TestProvisionSync(t *testing.T) { newVolumeArray("pvc-uid11-7", "1Gi", "uid11-7", "claim11-7", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, storageutil.StorageClassAnnotation), newClaimArray("claim11-7", "uid11-7", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), // The claim would be bound in next syncClaim - newClaimArray("claim11-7", "uid11-7", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-7", "uid11-7", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), noevents, noerrors, wrapTestWithInjectedOperation(wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), func(ctrl *PersistentVolumeController, reactor *volumeReactor) { // Create a volume before provisionClaimOperation starts. @@ -184,7 +184,7 @@ func TestProvisionSync(t *testing.T) { newVolumeArray("pvc-uid11-8", "1Gi", "uid11-8", "claim11-8", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, storageutil.StorageClassAnnotation), newClaimArray("claim11-8", "uid11-8", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), // Binding will be completed in the next syncClaim - newClaimArray("claim11-8", "uid11-8", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-8", "uid11-8", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), noevents, []reactorError{ // Inject error to the first @@ -201,7 +201,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-9", "uid11-9", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), - newClaimArray("claim11-9", "uid11-9", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-9", "uid11-9", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), []string{"Warning ProvisioningFailed"}, []reactorError{ // Inject error to five kubeclient.PersistentVolumes.Create() @@ -226,7 +226,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-10", "uid11-10", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), - newClaimArray("claim11-10", "uid11-10", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-10", "uid11-10", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), []string{"Warning ProvisioningFailed", "Warning ProvisioningCleanupFailed"}, []reactorError{ // Inject error to five kubeclient.PersistentVolumes.Create() @@ -247,7 +247,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-11", "uid11-11", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), - newClaimArray("claim11-11", "uid11-11", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-11", "uid11-11", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), []string{"Warning ProvisioningFailed", "Warning ProvisioningCleanupFailed"}, []reactorError{ // Inject error to five kubeclient.PersistentVolumes.Create() @@ -277,7 +277,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-12", "uid11-12", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), - newClaimArray("claim11-12", "uid11-12", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), + newClaimArray("claim11-12", "uid11-12", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner), []string{"Warning ProvisioningFailed"}, []reactorError{ // Inject error to five kubeclient.PersistentVolumes.Create() @@ -305,7 +305,7 @@ func TestProvisionSync(t *testing.T) { volumeWithClass("silver", newVolumeArray("pvc-uid11-13", "1Gi", "uid11-13", "claim11-13", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned)), claimWithClass("silver", newClaimArray("claim11-13", "uid11-13", "1Gi", "", api.ClaimPending)), // Binding will be completed in the next syncClaim - claimWithClass("silver", newClaimArray("claim11-13", "uid11-13", "1Gi", "", api.ClaimPending)), + claimWithClass("silver", newClaimArray("claim11-13", "uid11-13", "1Gi", "", api.ClaimPending, annStorageProvisioner)), noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision2Success}, testSyncClaim), }, { @@ -341,7 +341,8 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, claimWithClass("external", newClaimArray("claim11-17", "uid11-17", "1Gi", "", api.ClaimPending)), - claimWithClass("external", newClaimArray("claim11-17", "uid11-17", "1Gi", "", api.ClaimPending)), + claimWithAnnotation(annStorageProvisioner, "vendor.com/my-volume", + claimWithClass("external", newClaimArray("claim11-17", "uid11-17", "1Gi", "", api.ClaimPending))), []string{"Normal ExternalProvisioning"}, noerrors, wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), }, @@ -368,7 +369,7 @@ func TestAlphaProvisionSync(t *testing.T) { newVolumeArray("pvc-uid14-1", "1Gi", "uid14-1", "claim14-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned), newClaimArray("claim14-1", "uid14-1", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation), // Binding will be completed in the next syncClaim - newClaimArray("claim14-1", "uid14-1", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation), + newClaimArray("claim14-1", "uid14-1", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation, annStorageProvisioner), noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provisionAlphaSuccess}, testSyncClaim), }, { @@ -382,7 +383,7 @@ func TestAlphaProvisionSync(t *testing.T) { }, newClaimArray("claim14-2", "uid14-2", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation), // Binding will be completed in the next syncClaim - newClaimArray("claim14-2", "uid14-2", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation), + newClaimArray("claim14-2", "uid14-2", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation, annStorageProvisioner), noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provisionAlphaSuccess}, testSyncClaim), }, } @@ -411,8 +412,7 @@ func TestProvisionMultiSync(t *testing.T) { novolumes, newVolumeArray("pvc-uid12-1", "1Gi", "uid12-1", "claim12-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, storageutil.StorageClassAnnotation), newClaimArray("claim12-1", "uid12-1", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation), - // Binding will be completed in the next syncClaim - newClaimArray("claim12-1", "uid12-1", "1Gi", "pvc-uid12-1", api.ClaimBound, storageutil.StorageClassAnnotation, annBoundByController, annBindCompleted), + newClaimArray("claim12-1", "uid12-1", "1Gi", "pvc-uid12-1", api.ClaimBound, storageutil.StorageClassAnnotation, annBoundByController, annBindCompleted, annStorageProvisioner), noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision1Success}, testSyncClaim), }, } diff --git a/pkg/controller/volume/persistentvolume/pv_controller.go b/pkg/controller/volume/persistentvolume/pv_controller.go index 72f528d3a88..85ebc909de8 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller.go +++ b/pkg/controller/volume/persistentvolume/pv_controller.go @@ -117,6 +117,11 @@ const annBoundByController = "pv.kubernetes.io/bound-by-controller" // recognize dynamically provisioned PVs in its decisions). const annDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by" +// This annotation is added to a PVC that is supposed to be dynamically +// provisioned. Its value is name of volume plugin that is supposed to provision +// a volume for this PVC. +const annStorageProvisioner = "volume.beta.kubernetes.io/storage-provisioner" + // Name of a tag attached to a real volume in cloud (e.g. AWS EBS or GCE PD) // with namespace of a persistent volume claim used to create this volume. const cloudVolumeCreatedForClaimNamespaceTag = "kubernetes.io/created-for/pvc/namespace" @@ -1197,6 +1202,42 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa claimClass := storageutil.GetClaimStorageClass(claim) glog.V(4).Infof("provisionClaimOperation [%s] started, class: %q", claimToClaimKey(claim), claimClass) + plugin, storageClass, err := ctrl.findProvisionablePlugin(claim) + if err != nil { + ctrl.eventRecorder.Event(claim, api.EventTypeWarning, "ProvisioningFailed", err.Error()) + glog.V(2).Infof("error finding provisioning plugin for claim %s: %v", claimToClaimKey(claim), err) + // The controller will retry provisioning the volume in every + // syncVolume() call. + return + } + + if storageClass != nil { + // Add provisioner annotation so external provisioners know when to start + newClaim, err := ctrl.setClaimProvisioner(claim, storageClass) + if err != nil { + // Save failed, the controller will retry in the next sync + glog.V(2).Infof("error saving claim %s: %v", claimToClaimKey(claim), err) + return + } + claim = newClaim + } + + if plugin == nil { + // findProvisionablePlugin returned no error nor plugin. + // This means that an unknown provisioner is requested. Report an event + // and wait for the external provisioner + if storageClass != nil { + msg := fmt.Sprintf("cannot find provisioner %q, expecting that a volume for the claim is provisioned either manually or via external software", storageClass.Provisioner) + ctrl.eventRecorder.Event(claim, api.EventTypeNormal, "ExternalProvisioning", msg) + glog.V(3).Infof("provisioning claim %q: %s", claimToClaimKey(claim), msg) + } else { + glog.V(3).Infof("cannot find storage class for claim %q", claimToClaimKey(claim)) + } + return + } + + // internal provisioning + // A previous doProvisionClaim may just have finished while we were waiting for // the locks. Check that PV (with deterministic name) hasn't been provisioned // yet. @@ -1217,28 +1258,6 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa return } - plugin, storageClass, err := ctrl.findProvisionablePlugin(claim) - if err != nil { - ctrl.eventRecorder.Event(claim, api.EventTypeWarning, "ProvisioningFailed", err.Error()) - glog.V(2).Infof("error finding provisioning plugin for claim %s: %v", claimToClaimKey(claim), err) - // The controller will retry provisioning the volume in every - // syncVolume() call. - return - } - if plugin == nil { - // findProvisionablePlugin returned no error nor plugin. - // This means that an unknown provisioner is requested. Report an event - // and wait for the external provisioner - if storageClass != nil { - msg := fmt.Sprintf("cannot find provisioner %q, expecting that a volume for the claim is provisioned either manually or via external software", storageClass.Provisioner) - ctrl.eventRecorder.Event(claim, api.EventTypeNormal, "ExternalProvisioning", msg) - glog.V(3).Infof("provisioning claim %q: %s", claimToClaimKey(claim), msg) - } else { - glog.V(3).Infof("cannot find storage class for claim %q", claimToClaimKey(claim)) - } - return - } - // Gather provisioning options tags := make(map[string]string) tags[cloudVolumeCreatedForClaimNamespaceTag] = claim.Namespace diff --git a/pkg/controller/volume/persistentvolume/pv_controller_base.go b/pkg/controller/volume/persistentvolume/pv_controller_base.go index d786b67241b..2706da0fe70 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller_base.go +++ b/pkg/controller/volume/persistentvolume/pv_controller_base.go @@ -494,6 +494,36 @@ func (ctrl *PersistentVolumeController) upgradeVolumeFrom1_2(volume *api.Persist return true } +// setClaimProvisioner saves +// claim.Annotations[annStorageProvisioner] = class.Provisioner +func (ctrl *PersistentVolumeController) setClaimProvisioner(claim *api.PersistentVolumeClaim, class *storage.StorageClass) (*api.PersistentVolumeClaim, error) { + if val, ok := claim.Annotations[annDynamicallyProvisioned]; ok && val == class.Provisioner { + // annotation is already set, nothing to do + return claim, nil + } + + // The volume from method args can be pointing to watcher cache. We must not + // modify these, therefore create a copy. + clone, err := conversion.NewCloner().DeepCopy(claim) + if err != nil { + return nil, fmt.Errorf("Error cloning pv: %v", err) + } + claimClone, ok := clone.(*api.PersistentVolumeClaim) + if !ok { + return nil, fmt.Errorf("Unexpected claim cast error : %v", claimClone) + } + api.SetMetaDataAnnotation(&claimClone.ObjectMeta, annStorageProvisioner, class.Provisioner) + newClaim, err := ctrl.kubeClient.Core().PersistentVolumeClaims(claim.Namespace).Update(claimClone) + if err != nil { + return newClaim, err + } + _, err = ctrl.storeClaimUpdate(newClaim) + if err != nil { + return newClaim, err + } + return newClaim, nil +} + // Stateless functions func getClaimStatusForLogging(claim *api.PersistentVolumeClaim) string {