github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-01-04 23:17:50 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #29006 from jsafrane/dynprov2

Browse Source 
Automatic merge from submit-queue

Implement dynamic provisioning (beta) of PersistentVolumes via StorageClass

Implemented according to PR #26908. There are several patches in this PR with one huge code regen inside.

* Please review the API changes (the first patch) carefully, sometimes I don't know what the code is doing...

* `PV.Spec.Class` and `PVC.Spec.Class` is not implemented, use annotation `volume.alpha.kubernetes.io/storage-class`

* See e2e test and integration test changes - Kubernetes won't provision a thing without explicit configuration of at least one `StorageClass` instance!

* Multiple provisioning volume plugins can coexist together, e.g. HostPath and AWS EBS. This is important for Gluster and RBD provisioners in #25026

* Contradicting the proposal, `claim.Selector` and `volume.alpha.kubernetes.io/storage-class` annotation are **not** mutually exclusive. They're both used for matching existing PVs. However, only `volume.alpha.kubernetes.io/storage-class` is used for provisioning, configuration of provisioning with `Selector` is left for (near) future.

* Documentation is missing. Can please someone write some while I am out?

For now, AWS volume plugin accepts classes with these parameters:

```
kind: StorageClass
metadata:
  name: slow
provisionerType: kubernetes.io/aws-ebs
provisionerParameters:
  type: io1
  zone: us-east-1d
  iopsPerGB: 10
```

* parameters are case-insensitive
* `type`: `io1`, `gp2`, `sc1`, `st1`. See AWS docs for details
* `iopsPerGB`: only for `io1` volumes. I/O operations per second per GiB. AWS volume plugin multiplies this with size of requested volume to compute IOPS of the volume and caps it at 20 000 IOPS (maximum supported by AWS, see AWS docs).
* of course, the plugin will use some defaults when a parameter is omitted in a `StorageClass` instance (`gp2` in the same zone as in 1.3).

GCE:

```
apiVersion: extensions/v1beta1
kind: StorageClass
metadata:
  name: slow
provisionerType: kubernetes.io/gce-pd
provisionerParameters:
  type: pd-standard
  zone: us-central1-a
```

* `type`: `pd-standard` or `pd-ssd`
* `zone`: GCE zone
* of course, the plugin will use some defaults when a parameter is omitted in a `StorageClass` instance (SSD in the same zone as in 1.3 ?).


No OpenStack/Cinder yet

@kubernetes/sig-storage
This commit is contained in:
Kubernetes Submit Queue
2016-08-18 09:56:16 -07:00
committed by GitHub
parent ff58d04a34 bb5d562f37
commit 9d2a5fe5e8
31 changed files with 1180 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
pkg/controller/volume/persistentvolume/binder_test.go
View File

@@ -20,6 +20,7 @@ import (
"testing"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/apis/extensions"
)
// Test single call to syncClaim and syncVolume methods.
@@ -421,8 +422,88 @@ func TestSync(t *testing.T) {
newClaimArray("claim4-8", "uid4-8", "10Gi", "volume4-8-x", api.ClaimBound),
noevents, noerrors, testSyncVolume,
},
// PVC with class
{
// syncVolume binds a claim to requested class even if there is a
// smaller PV available
"13-1 - binding to class",
[]*api.PersistentVolume{
newVolume("volume13-1-1", "1Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain),
newVolume("volume13-1-2", "10Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain, annClass),
},
[]*api.PersistentVolume{
newVolume("volume13-1-1", "1Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain),
newVolume("volume13-1-2", "10Gi", "uid13-1", "claim13-1", api.VolumeBound, api.PersistentVolumeReclaimRetain, annBoundByController, annClass),
},
newClaimArray("claim13-1", "uid13-1", "1Gi", "", api.ClaimPending, annClass),
withExpectedCapacity("10Gi", newClaimArray("claim13-1", "uid13-1", "1Gi", "volume13-1-2", api.ClaimBound, annBoundByController, annClass, annBindCompleted)),
noevents, noerrors, testSyncClaim,
},
{
// syncVolume binds a claim without a class even if there is a
// smaller PV with a class available
"13-2 - binding without a class",
[]*api.PersistentVolume{
newVolume("volume13-2-1", "1Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain, annClass),
newVolume("volume13-2-2", "10Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain),
},
[]*api.PersistentVolume{
newVolume("volume13-2-1", "1Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain, annClass),
newVolume("volume13-2-2", "10Gi", "uid13-2", "claim13-2", api.VolumeBound, api.PersistentVolumeReclaimRetain, annBoundByController),
},
newClaimArray("claim13-2", "uid13-2", "1Gi", "", api.ClaimPending),
withExpectedCapacity("10Gi", newClaimArray("claim13-2", "uid13-2", "1Gi", "volume13-2-2", api.ClaimBound, annBoundByController, annBindCompleted)),
noevents, noerrors, testSyncClaim,
},
{
// syncVolume binds a claim with given class even if there is a
// smaller PV with different class available
"13-3 - binding to specific a class",
volumeWithClass("silver", []*api.PersistentVolume{
newVolume("volume13-3-1", "1Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain),
newVolume("volume13-3-2", "10Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain, annClass),
}),
volumeWithClass("silver", []*api.PersistentVolume{
newVolume("volume13-3-1", "1Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain),
newVolume("volume13-3-2", "10Gi", "uid13-3", "claim13-3", api.VolumeBound, api.PersistentVolumeReclaimRetain, annBoundByController, annClass),
}),
newClaimArray("claim13-3", "uid13-3", "1Gi", "", api.ClaimPending, annClass),
withExpectedCapacity("10Gi", newClaimArray("claim13-3", "uid13-3", "1Gi", "volume13-3-2", api.ClaimBound, annBoundByController, annBindCompleted, annClass)),
noevents, noerrors, testSyncClaim,
},
{
// syncVolume binds claim requesting class "" to claim to PV with
// class=""
"13-4 - empty class",
volumeWithClass("", newVolumeArray("volume13-4", "1Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain)),
volumeWithClass("", newVolumeArray("volume13-4", "1Gi", "uid13-4", "claim13-4", api.VolumeBound, api.PersistentVolumeReclaimRetain, annBoundByController)),
claimWithClass("", newClaimArray("claim13-4", "uid13-4", "1Gi", "", api.ClaimPending)),
claimWithClass("", newClaimArray("claim13-4", "uid13-4", "1Gi", "volume13-4", api.ClaimBound, annBoundByController, annBindCompleted)),
noevents, noerrors, testSyncClaim,
},
{
// syncVolume binds claim requesting class nil to claim to PV with
// class = ""
"13-5 - nil class",
volumeWithClass("", newVolumeArray("volume13-5", "1Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain)),
volumeWithClass("", newVolumeArray("volume13-5", "1Gi", "uid13-5", "claim13-5", api.VolumeBound, api.PersistentVolumeReclaimRetain, annBoundByController)),
newClaimArray("claim13-5", "uid13-5", "1Gi", "", api.ClaimPending),
newClaimArray("claim13-5", "uid13-5", "1Gi", "volume13-5", api.ClaimBound, annBoundByController, annBindCompleted),
noevents, noerrors, testSyncClaim,
},
{
// syncVolume binds claim requesting class "" to claim to PV with
// class=nil
"13-6 - nil class in PV, '' class in claim",
newVolumeArray("volume13-6", "1Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain),
newVolumeArray("volume13-6", "1Gi", "uid13-6", "claim13-6", api.VolumeBound, api.PersistentVolumeReclaimRetain, annBoundByController),
claimWithClass("", newClaimArray("claim13-6", "uid13-6", "1Gi", "", api.ClaimPending)),
claimWithClass("", newClaimArray("claim13-6", "uid13-6", "1Gi", "volume13-6", api.ClaimBound, annBoundByController, annBindCompleted)),
noevents, noerrors, testSyncClaim,
},
}
runSyncTests(t, tests)
runSyncTests(t, tests, []*extensions.StorageClass{})
}
// Test multiple calls to syncClaim/syncVolume and periodic sync of all
@@ -469,5 +550,5 @@ func TestMultiSync(t *testing.T) {
},
}
runMultisyncTests(t, tests)
runMultisyncTests(t, tests, []*extensions.StorageClass{}, "")
}

158
pkg/controller/volume/persistentvolume/controller.go
View File

@@ -22,6 +22,8 @@ import (
"time"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/unversioned"
"k8s.io/kubernetes/pkg/apis/extensions"
"k8s.io/kubernetes/pkg/client/cache"
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
"k8s.io/kubernetes/pkg/client/record"
@@ -108,10 +110,19 @@ const annBindCompleted = "pv.kubernetes.io/bind-completed"
// pre-bound). Value of this annotation does not matter.
const annBoundByController = "pv.kubernetes.io/bound-by-controller"
// annClass annotation represents a new field which instructs dynamic
// provisioning to choose a particular storage class (aka profile).
// Value of this annotation should be empty.
const annClass = "volume.alpha.kubernetes.io/storage-class"
// annClass annotation represents the storage class associated with a resource:
// - in PersistentVolumeClaim it represents required class to match.
// Only PersistentVolumes with the same class (i.e. annotation with the same
// value) can be bound to the claim. In case no such volume exists, the
// controller will provision a new one using StorageClass instance with
// the same name as the annotation value.
// - in PersistentVolume it represents storage class to which the persistent
// volume belongs.
const annClass = "volume.beta.kubernetes.io/storage-class"
// alphaAnnClass annotation represents the previous alpha storage class
// annotation. it's no longer used and held here for posterity.
const annAlphaClass = "volume.alpha.kubernetes.io/storage-class"
// This annotation is added to a PV that has been dynamically provisioned by
// Kubernetes. Its value is name of volume plugin that created the volume.
@@ -148,11 +159,13 @@ type PersistentVolumeController struct {
claimController *framework.Controller
claimControllerStopCh chan struct{}
claimSource cache.ListerWatcher
classReflector *cache.Reflector
classReflectorStopCh chan struct{}
classSource cache.ListerWatcher
kubeClient clientset.Interface
eventRecorder record.EventRecorder
cloud cloudprovider.Interface
recyclePluginMgr vol.VolumePluginMgr
provisioner vol.ProvisionableVolumePlugin
volumePluginMgr vol.VolumePluginMgr
enableDynamicProvisioning bool
clusterName string
@@ -163,6 +176,7 @@ type PersistentVolumeController struct {
// it saves newer version to etcd.
volumes persistentVolumeOrderedIndex
claims cache.Store
classes cache.Store
// Map of scheduled/running operations.
runningOperations goroutinemap.GoRoutineMap
@@ -173,6 +187,10 @@ type PersistentVolumeController struct {
createProvisionedPVRetryCount int
createProvisionedPVInterval time.Duration
// Provisioner for annAlphaClass.
// TODO: remove in 1.5
alphaProvisioner vol.ProvisionableVolumePlugin
}
// syncClaim is the main controller method to decide what to do with a claim.
@@ -198,6 +216,7 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(claim *api.PersistentVo
// OBSERVATION: pvc is "Pending"
if claim.Spec.VolumeName == "" {
// User did not care which PV they get.
// [Unit test set 1]
volume, err := ctrl.volumes.findBestMatchForClaim(claim)
if err != nil {
@@ -208,7 +227,8 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(claim *api.PersistentVo
glog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: no volume found", claimToClaimKey(claim))
// No PV could be found
// OBSERVATION: pvc is "Pending", will retry
if hasAnnotation(claim.ObjectMeta, annClass) {
// TODO: remove Alpha check in 1.5
if getClaimClass(claim) != "" || hasAnnotation(claim.ObjectMeta, annAlphaClass) {
if err = ctrl.provisionClaim(claim); err != nil {
return err
}
@@ -965,7 +985,7 @@ func (ctrl *PersistentVolumeController) recycleVolumeOperation(arg interface{})
// Find a plugin.
spec := vol.NewSpecFromPersistentVolume(volume, false)
plugin, err := ctrl.recyclePluginMgr.FindRecyclablePluginBySpec(spec)
plugin, err := ctrl.volumePluginMgr.FindRecyclablePluginBySpec(spec)
if err != nil {
// No recycler found. Emit an event and mark the volume Failed.
if _, err = ctrl.updateVolumePhaseWithEvent(volume, api.VolumeFailed, api.EventTypeWarning, "VolumeFailedRecycle", "No recycler plugin found for the volume!"); err != nil {
@@ -1119,14 +1139,33 @@ func (ctrl *PersistentVolumeController) isVolumeReleased(volume *api.PersistentV
// (it will be re-used in future provisioner error cases).
func (ctrl *PersistentVolumeController) doDeleteVolume(volume *api.PersistentVolume) error {
glog.V(4).Infof("doDeleteVolume [%s]", volume.Name)
// Find a plugin.
spec := vol.NewSpecFromPersistentVolume(volume, false)
plugin, err := ctrl.recyclePluginMgr.FindDeletablePluginBySpec(spec)
if err != nil {
// No deleter found. Emit an event and mark the volume Failed.
return fmt.Errorf("Error getting deleter volume plugin for volume %q: %v", volume.Name, err)
var err error
// Find a plugin. Try to find the same plugin that provisioned the volume
var plugin vol.DeletableVolumePlugin
if hasAnnotation(volume.ObjectMeta, annDynamicallyProvisioned) {
provisionPluginName := volume.Annotations[annDynamicallyProvisioned]
if provisionPluginName != "" {
plugin, err = ctrl.volumePluginMgr.FindDeletablePluginByName(provisionPluginName)
if err != nil {
glog.V(3).Infof("did not find a deleter plugin %q for volume %q: %v, will try to find a generic one",
provisionPluginName, volume.Name, err)
}
}
}
spec := vol.NewSpecFromPersistentVolume(volume, false)
if plugin == nil {
// The plugin that provisioned the volume was not found or the volume
// was not dynamically provisioned. Try to find a plugin by spec.
plugin, err = ctrl.volumePluginMgr.FindDeletablePluginBySpec(spec)
if err != nil {
// No deleter found. Emit an event and mark the volume Failed.
return fmt.Errorf("Error getting deleter volume plugin for volume %q: %v", volume.Name, err)
}
}
glog.V(5).Infof("found a deleter plugin %q for volume %q", plugin.GetPluginName(), volume.Name)
// Plugin found
deleter, err := plugin.NewDeleter(spec)
if err != nil {
@@ -1143,7 +1182,8 @@ func (ctrl *PersistentVolumeController) doDeleteVolume(volume *api.PersistentVol
return nil
}
// provisionClaim starts new asynchronous operation to provision a claim if provisioning is enabled.
// provisionClaim starts new asynchronous operation to provision a claim if
// provisioning is enabled.
func (ctrl *PersistentVolumeController) provisionClaim(claim *api.PersistentVolumeClaim) error {
if !ctrl.enableDynamicProvisioning {
return nil
@@ -1165,7 +1205,9 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa
glog.Errorf("Cannot convert provisionClaimOperation argument to claim, got %#v", claimObj)
return
}
glog.V(4).Infof("provisionClaimOperation [%s] started", claimToClaimKey(claim))
claimClass := getClaimClass(claim)
glog.V(4).Infof("provisionClaimOperation [%s] started, class: %q", claimToClaimKey(claim), claimClass)
// A previous doProvisionClaim may just have finished while we were waiting for
// the locks. Check that PV (with deterministic name) hasn't been provisioned
@@ -1187,12 +1229,10 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa
return
}
// TODO: find provisionable plugin based on a class/profile
plugin := ctrl.provisioner
if plugin == nil {
// No provisioner found. Emit an event.
ctrl.eventRecorder.Event(claim, api.EventTypeWarning, "ProvisioningFailed", "No provisioner plugin found for the claim!")
glog.V(2).Infof("no provisioner plugin found for claim %s!", claimToClaimKey(claim))
plugin, storageClass, err := ctrl.findProvisionablePlugin(claim)
if err != nil {
ctrl.eventRecorder.Event(claim, api.EventTypeWarning, "ProvisioningFailed", err.Error())
glog.V(2).Infof("error finding provisioning plugin for claim %s: %v", claimToClaimKey(claim), err)
// The controller will retry provisioning the volume in every
// syncVolume() call.
return
@@ -1212,21 +1252,23 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa
ClusterName: ctrl.clusterName,
PVName: pvName,
PVCName: claim.Name,
Parameters: storageClass.Parameters,
Selector: claim.Spec.Selector,
}
// Provision the volume
provisioner, err := plugin.NewProvisioner(options)
if err != nil {
strerr := fmt.Sprintf("Failed to create provisioner: %v", err)
glog.V(2).Infof("failed to create provisioner for claim %q: %v", claimToClaimKey(claim), err)
glog.V(2).Infof("failed to create provisioner for claim %q with StorageClass %q: %v", claimToClaimKey(claim), storageClass.Name, err)
ctrl.eventRecorder.Event(claim, api.EventTypeWarning, "ProvisioningFailed", strerr)
return
}
volume, err = provisioner.Provision()
if err != nil {
strerr := fmt.Sprintf("Failed to provision volume: %v", err)
glog.V(2).Infof("failed to provision volume for claim %q: %v", claimToClaimKey(claim), err)
strerr := fmt.Sprintf("Failed to provision volume with StorageClass %q: %v", storageClass.Name, err)
glog.V(2).Infof("failed to provision volume for claim %q with StorageClass %q: %v", claimToClaimKey(claim), storageClass.Name, err)
ctrl.eventRecorder.Event(claim, api.EventTypeWarning, "ProvisioningFailed", strerr)
return
}
@@ -1242,6 +1284,12 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa
// Add annBoundByController (used in deleting the volume)
setAnnotation(&volume.ObjectMeta, annBoundByController, "yes")
setAnnotation(&volume.ObjectMeta, annDynamicallyProvisioned, plugin.GetPluginName())
// For Alpha provisioning behavior, do not add annClass for volumes created
// by annAlphaClass
// TODO: remove this check in 1.5, annClass will be always non-empty there.
if claimClass != "" {
setAnnotation(&volume.ObjectMeta, annClass, claimClass)
}
// Try to create the PV object several times
for i := 0; i < ctrl.createProvisionedPVRetryCount; i++ {
@@ -1320,3 +1368,63 @@ func (ctrl *PersistentVolumeController) scheduleOperation(operationName string,
}
}
}
func (ctrl *PersistentVolumeController) findProvisionablePlugin(claim *api.PersistentVolumeClaim) (vol.ProvisionableVolumePlugin, *extensions.StorageClass, error) {
// TODO: remove this alpha behavior in 1.5
alpha := hasAnnotation(claim.ObjectMeta, annAlphaClass)
beta := hasAnnotation(claim.ObjectMeta, annClass)
if alpha && beta {
// Both Alpha and Beta annotations are set. Do beta.
alpha = false
msg := fmt.Sprintf("both %q and %q annotations are present, using %q", annAlphaClass, annClass, annClass)
ctrl.eventRecorder.Event(claim, api.EventTypeNormal, "ProvisioningIgnoreAlpha", msg)
}
if alpha {
// Fall back to fixed list of provisioner plugins
return ctrl.findAlphaProvisionablePlugin()
}
// provisionClaim() which leads here is never called with claimClass=="", we
// can save some checks.
claimClass := getClaimClass(claim)
classObj, found, err := ctrl.classes.GetByKey(claimClass)
if err != nil {
return nil, nil, err
}
if !found {
return nil, nil, fmt.Errorf("StorageClass %q not found", claimClass)
}
class, ok := classObj.(*extensions.StorageClass)
if !ok {
return nil, nil, fmt.Errorf("Cannot convert object to StorageClass: %+v", classObj)
}
// Find a plugin for the class
plugin, err := ctrl.volumePluginMgr.FindProvisionablePluginByName(class.Provisioner)
if err != nil {
return nil, nil, err
}
return plugin, class, nil
}
// findAlphaProvisionablePlugin returns a volume plugin compatible with
// Kubernetes 1.3.
// TODO: remove in Kubernetes 1.5
func (ctrl *PersistentVolumeController) findAlphaProvisionablePlugin() (vol.ProvisionableVolumePlugin, *extensions.StorageClass, error) {
if ctrl.alphaProvisioner == nil {
return nil, nil, fmt.Errorf("cannot find volume plugin for alpha provisioning")
}
// Return a dummy StorageClass instance with no parameters
storageClass := &extensions.StorageClass{
TypeMeta: unversioned.TypeMeta{
Kind: "StorageClass",
},
ObjectMeta: api.ObjectMeta{
Name: "",
},
Provisioner: ctrl.alphaProvisioner.GetPluginName(),
}
glog.V(4).Infof("using alpha provisioner %s", ctrl.alphaProvisioner.GetPluginName())
return ctrl.alphaProvisioner, storageClass, nil
}

83
pkg/controller/volume/persistentvolume/controller_base.go
View File

@@ -24,6 +24,7 @@ import (
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/errors"
"k8s.io/kubernetes/pkg/api/meta"
"k8s.io/kubernetes/pkg/apis/extensions"
"k8s.io/kubernetes/pkg/client/cache"
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
unversioned_core "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/unversioned"
@@ -47,11 +48,11 @@ import (
func NewPersistentVolumeController(
kubeClient clientset.Interface,
syncPeriod time.Duration,
provisioner vol.ProvisionableVolumePlugin,
recyclers []vol.VolumePlugin,
alphaProvisioner vol.ProvisionableVolumePlugin,
volumePlugins []vol.VolumePlugin,
cloud cloudprovider.Interface,
clusterName string,
volumeSource, claimSource cache.ListerWatcher,
volumeSource, claimSource, classSource cache.ListerWatcher,
eventRecorder record.EventRecorder,
enableDynamicProvisioning bool,
) *PersistentVolumeController {
@@ -63,23 +64,23 @@ func NewPersistentVolumeController(
}
controller := &PersistentVolumeController{
volumes: newPersistentVolumeOrderedIndex(),
claims: cache.NewStore(framework.DeletionHandlingMetaNamespaceKeyFunc),
kubeClient: kubeClient,
eventRecorder: eventRecorder,
runningOperations: goroutinemap.NewGoRoutineMap(false /* exponentialBackOffOnError */),
cloud: cloud,
provisioner: provisioner,
volumes: newPersistentVolumeOrderedIndex(),
claims: cache.NewStore(framework.DeletionHandlingMetaNamespaceKeyFunc),
kubeClient: kubeClient,
eventRecorder: eventRecorder,
runningOperations: goroutinemap.NewGoRoutineMap(false /* exponentialBackOffOnError */),
cloud: cloud,
enableDynamicProvisioning: enableDynamicProvisioning,
clusterName: clusterName,
createProvisionedPVRetryCount: createProvisionedPVRetryCount,
createProvisionedPVInterval: createProvisionedPVInterval,
alphaProvisioner: alphaProvisioner,
}
controller.recyclePluginMgr.InitPlugins(recyclers, controller)
if controller.provisioner != nil {
if err := controller.provisioner.Init(controller); err != nil {
glog.Errorf("PersistentVolumeController: error initializing provisioner plugin: %v", err)
controller.volumePluginMgr.InitPlugins(volumePlugins, controller)
if controller.alphaProvisioner != nil {
if err := controller.alphaProvisioner.Init(controller); err != nil {
glog.Errorf("PersistentVolumeController: error initializing alpha provisioner plugin: %v", err)
}
}
@@ -107,6 +108,18 @@ func NewPersistentVolumeController(
}
controller.claimSource = claimSource
if classSource == nil {
classSource = &cache.ListWatch{
ListFunc: func(options api.ListOptions) (runtime.Object, error) {
return kubeClient.Extensions().StorageClasses().List(options)
},
WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
return kubeClient.Extensions().StorageClasses().Watch(options)
},
}
}
controller.classSource = classSource
_, controller.volumeController = framework.NewIndexerInformer(
volumeSource,
&api.PersistentVolume{},
@@ -128,6 +141,16 @@ func NewPersistentVolumeController(
DeleteFunc: controller.deleteClaim,
},
)
// This is just a cache of StorageClass instances, no special actions are
// needed when a class is created/deleted/updated.
controller.classes = cache.NewStore(framework.DeletionHandlingMetaNamespaceKeyFunc)
controller.classReflector = cache.NewReflector(
classSource,
&extensions.StorageClass{},
controller.classes,
syncPeriod,
)
return controller
}
@@ -433,6 +456,11 @@ func (ctrl *PersistentVolumeController) Run() {
ctrl.claimControllerStopCh = make(chan struct{})
go ctrl.claimController.Run(ctrl.claimControllerStopCh)
}
if ctrl.classReflectorStopCh == nil {
ctrl.classReflectorStopCh = make(chan struct{})
go ctrl.classReflector.RunUntil(ctrl.classReflectorStopCh)
}
}
// Stop gracefully shuts down this controller
@@ -440,6 +468,7 @@ func (ctrl *PersistentVolumeController) Stop() {
glog.V(4).Infof("stopping PersistentVolumeController")
close(ctrl.volumeControllerStopCh)
close(ctrl.claimControllerStopCh)
close(ctrl.classReflectorStopCh)
}
const (
@@ -578,3 +607,29 @@ func storeObjectUpdate(store cache.Store, obj interface{}, className string) (bo
}
return true, nil
}
// getVolumeClass returns value of annClass annotation or empty string in case
// the annotation does not exist.
// TODO: change to PersistentVolume.Spec.Class value when this attribute is
// introduced.
func getVolumeClass(volume *api.PersistentVolume) string {
if class, found := volume.Annotations[annClass]; found {
return class
}
// 'nil' is interpreted as "", i.e. the volume does not belong to any class.
return ""
}
// getClaimClass returns name of class that is requested by given claim.
// Request for `nil` class is interpreted as request for class "",
// i.e. for a classless PV.
func getClaimClass(claim *api.PersistentVolumeClaim) string {
// TODO: change to PersistentVolumeClaim.Spec.Class value when this
// attribute is introduced.
if class, found := claim.Annotations[annClass]; found {
return class
}
return ""
}

2
pkg/controller/volume/persistentvolume/controller_test.go
View File

@@ -163,7 +163,7 @@ func TestControllerSync(t *testing.T) {
client := &fake.Clientset{}
volumeSource := framework.NewFakePVControllerSource()
claimSource := framework.NewFakePVCControllerSource()
ctrl := newTestController(client, volumeSource, claimSource, true)
ctrl := newTestController(client, volumeSource, claimSource, nil, true)
reactor := newVolumeReactor(client, ctrl, volumeSource, claimSource, test.errors)
for _, claim := range test.initialClaims {
claimSource.Add(claim)

21
pkg/controller/volume/persistentvolume/delete_test.go
View File

@@ -21,6 +21,7 @@ import (
"testing"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/apis/extensions"
)
// Test single call to syncVolume, expecting recycling to happen.
@@ -39,7 +40,7 @@ func TestDeleteSync(t *testing.T) {
noevents, noerrors,
// Inject deleter into the controller and call syncVolume. The
// deleter simulates one delete() call that succeeds.
wrapTestWithControllerConfig(operationDelete, []error{nil}, testSyncVolume),
wrapTestWithReclaimCalls(operationDelete, []error{nil}, testSyncVolume),
},
{
// delete volume bound by user
@@ -51,7 +52,7 @@ func TestDeleteSync(t *testing.T) {
noevents, noerrors,
// Inject deleter into the controller and call syncVolume. The
// deleter simulates one delete() call that succeeds.
wrapTestWithControllerConfig(operationDelete, []error{nil}, testSyncVolume),
wrapTestWithReclaimCalls(operationDelete, []error{nil}, testSyncVolume),
},
{
// delete failure - plugin not found
@@ -70,7 +71,7 @@ func TestDeleteSync(t *testing.T) {
noclaims,
noclaims,
[]string{"Warning VolumeFailedDelete"}, noerrors,
wrapTestWithControllerConfig(operationDelete, []error{}, testSyncVolume),
wrapTestWithReclaimCalls(operationDelete, []error{}, testSyncVolume),
},
{
// delete failure - delete() returns error
@@ -80,7 +81,7 @@ func TestDeleteSync(t *testing.T) {
noclaims,
noclaims,
[]string{"Warning VolumeFailedDelete"}, noerrors,
wrapTestWithControllerConfig(operationDelete, []error{errors.New("Mock delete error")}, testSyncVolume),
wrapTestWithReclaimCalls(operationDelete, []error{errors.New("Mock delete error")}, testSyncVolume),
},
{
// delete success(?) - volume is deleted before doDelete() starts
@@ -90,7 +91,7 @@ func TestDeleteSync(t *testing.T) {
noclaims,
noclaims,
noevents, noerrors,
wrapTestWithInjectedOperation(wrapTestWithControllerConfig(operationDelete, []error{}, testSyncVolume), func(ctrl *PersistentVolumeController, reactor *volumeReactor) {
wrapTestWithInjectedOperation(wrapTestWithReclaimCalls(operationDelete, []error{}, testSyncVolume), func(ctrl *PersistentVolumeController, reactor *volumeReactor) {
// Delete the volume before delete operation starts
reactor.lock.Lock()
delete(reactor.volumes, "volume8-6")
@@ -107,7 +108,7 @@ func TestDeleteSync(t *testing.T) {
noclaims,
newClaimArray("claim8-7", "uid8-7", "10Gi", "volume8-7", api.ClaimBound),
noevents, noerrors,
wrapTestWithInjectedOperation(wrapTestWithControllerConfig(operationDelete, []error{}, testSyncVolume), func(ctrl *PersistentVolumeController, reactor *volumeReactor) {
wrapTestWithInjectedOperation(wrapTestWithReclaimCalls(operationDelete, []error{}, testSyncVolume), func(ctrl *PersistentVolumeController, reactor *volumeReactor) {
reactor.lock.Lock()
defer reactor.lock.Unlock()
// Bind the volume to resurrected claim (this should never
@@ -130,10 +131,10 @@ func TestDeleteSync(t *testing.T) {
noevents, noerrors,
// Inject deleter into the controller and call syncVolume. The
// deleter simulates one delete() call that succeeds.
wrapTestWithControllerConfig(operationDelete, []error{nil}, testSyncVolume),
wrapTestWithReclaimCalls(operationDelete, []error{nil}, testSyncVolume),
},
}
runSyncTests(t, tests)
runSyncTests(t, tests, []*extensions.StorageClass{})
}
// Test multiple calls to syncClaim/syncVolume and periodic sync of all
@@ -161,9 +162,9 @@ func TestDeleteMultiSync(t *testing.T) {
noclaims,
noclaims,
[]string{"Warning VolumeFailedDelete"}, noerrors,
wrapTestWithControllerConfig(operationDelete, []error{errors.New("Mock delete error"), nil}, testSyncVolume),
wrapTestWithReclaimCalls(operationDelete, []error{errors.New("Mock delete error"), nil}, testSyncVolume),
},
}
runMultisyncTests(t, tests)
runMultisyncTests(t, tests, []*extensions.StorageClass{}, "")
}

161
pkg/controller/volume/persistentvolume/framework_test.go
View File

@@ -33,6 +33,7 @@ import (
"k8s.io/kubernetes/pkg/api/resource"
"k8s.io/kubernetes/pkg/api/testapi"
"k8s.io/kubernetes/pkg/api/unversioned"
"k8s.io/kubernetes/pkg/apis/extensions"
"k8s.io/kubernetes/pkg/client/cache"
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake"
@@ -583,22 +584,26 @@ func newVolumeReactor(client *fake.Clientset, ctrl *PersistentVolumeController,
return reactor
}
func newTestController(kubeClient clientset.Interface, volumeSource, claimSource cache.ListerWatcher, enableDynamicProvisioning bool) *PersistentVolumeController {
func newTestController(kubeClient clientset.Interface, volumeSource, claimSource, classSource cache.ListerWatcher, enableDynamicProvisioning bool) *PersistentVolumeController {
if volumeSource == nil {
volumeSource = framework.NewFakePVControllerSource()
}
if claimSource == nil {
claimSource = framework.NewFakePVCControllerSource()
}
if classSource == nil {
classSource = framework.NewFakeControllerSource()
}
ctrl := NewPersistentVolumeController(
kubeClient,
5*time.Second, // sync period
nil, // provisioner
nil, // alpha provisioner
[]vol.VolumePlugin{}, // recyclers
nil, // cloud
"",
volumeSource,
claimSource,
classSource,
record.NewFakeRecorder(1000), // event recorder
enableDynamicProvisioning,
)
@@ -608,27 +613,6 @@ func newTestController(kubeClient clientset.Interface, volumeSource, claimSource
return ctrl
}
func addRecyclePlugin(ctrl *PersistentVolumeController, expectedRecycleCalls []error) {
plugin := &mockVolumePlugin{
recycleCalls: expectedRecycleCalls,
}
ctrl.recyclePluginMgr.InitPlugins([]vol.VolumePlugin{plugin}, ctrl)
}
func addDeletePlugin(ctrl *PersistentVolumeController, expectedDeleteCalls []error) {
plugin := &mockVolumePlugin{
deleteCalls: expectedDeleteCalls,
}
ctrl.recyclePluginMgr.InitPlugins([]vol.VolumePlugin{plugin}, ctrl)
}
func addProvisionPlugin(ctrl *PersistentVolumeController, expectedDeleteCalls []error) {
plugin := &mockVolumePlugin{
provisionCalls: expectedDeleteCalls,
}
ctrl.provisioner = plugin
}
// newVolume returns a new volume with given attributes
func newVolume(name, capacity, boundToClaimUID, boundToClaimName string, phase api.PersistentVolumePhase, reclaimPolicy api.PersistentVolumeReclaimPolicy, annotations ...string) *api.PersistentVolume {
volume := api.PersistentVolume{
@@ -664,10 +648,13 @@ func newVolume(name, capacity, boundToClaimUID, boundToClaimName string, phase a
if len(annotations) > 0 {
volume.Annotations = make(map[string]string)
for _, a := range annotations {
if a != annDynamicallyProvisioned {
volume.Annotations[a] = "yes"
} else {
switch a {
case annDynamicallyProvisioned:
volume.Annotations[a] = mockPluginName
case annClass:
volume.Annotations[a] = "gold"
default:
volume.Annotations[a] = "yes"
}
}
}
@@ -713,6 +700,17 @@ func withMessage(message string, volumes []*api.PersistentVolume) []*api.Persist
return volumes
}
// volumeWithClass saves given class into annClass annotation.
// Meant to be used to compose claims specified inline in a test.
func volumeWithClass(className string, volumes []*api.PersistentVolume) []*api.PersistentVolume {
if volumes[0].Annotations == nil {
volumes[0].Annotations = map[string]string{annClass: className}
} else {
volumes[0].Annotations[annClass] = className
}
return volumes
}
// newVolumeArray returns array with a single volume that would be returned by
// newVolume() with the same parameters.
func newVolumeArray(name, capacity, boundToClaimUID, boundToClaimName string, phase api.PersistentVolumePhase, reclaimPolicy api.PersistentVolumeReclaimPolicy, annotations ...string) []*api.PersistentVolume {
@@ -749,7 +747,12 @@ func newClaim(name, claimUID, capacity, boundToVolume string, phase api.Persiste
if len(annotations) > 0 {
claim.Annotations = make(map[string]string)
for _, a := range annotations {
claim.Annotations[a] = "yes"
switch a {
case annClass:
claim.Annotations[a] = "gold"
default:
claim.Annotations[a] = "yes"
}
}
}
@@ -772,6 +775,17 @@ func newClaimArray(name, claimUID, capacity, boundToVolume string, phase api.Per
}
}
// claimWithClass saves given class into annClass annotation.
// Meant to be used to compose claims specified inline in a test.
func claimWithClass(className string, claims []*api.PersistentVolumeClaim) []*api.PersistentVolumeClaim {
if claims[0].Annotations == nil {
claims[0].Annotations = map[string]string{annClass: className}
} else {
claims[0].Annotations[annClass] = className
}
return claims
}
func testSyncClaim(ctrl *PersistentVolumeController, reactor *volumeReactor, test controllerTest) error {
return ctrl.syncClaim(test.initialClaims[0])
}
@@ -793,29 +807,47 @@ type operationType string
const operationDelete = "Delete"
const operationRecycle = "Recycle"
const operationProvision = "Provision"
// wrapTestWithControllerConfig returns a testCall that:
// - configures controller with recycler, deleter or provisioner which will
// return provided errors when a volume is deleted, recycled or provisioned
// wrapTestWithPluginCalls returns a testCall that:
// - configures controller with a volume plugin that implements recycler,
// deleter and provisioner. The plugin retunrs provided errors when a volume
// is deleted, recycled or provisioned.
// - calls given testCall
func wrapTestWithControllerConfig(operation operationType, expectedOperationCalls []error, toWrap testCall) testCall {
expected := expectedOperationCalls
func wrapTestWithPluginCalls(expectedRecycleCalls, expectedDeleteCalls []error, expectedProvisionCalls []provisionCall, toWrap testCall) testCall {
return func(ctrl *PersistentVolumeController, reactor *volumeReactor, test controllerTest) error {
switch operation {
case operationDelete:
addDeletePlugin(ctrl, expected)
case operationRecycle:
addRecyclePlugin(ctrl, expected)
case operationProvision:
addProvisionPlugin(ctrl, expected)
plugin := &mockVolumePlugin{
recycleCalls: expectedRecycleCalls,
deleteCalls: expectedDeleteCalls,
provisionCalls: expectedProvisionCalls,
}
ctrl.volumePluginMgr.InitPlugins([]vol.VolumePlugin{plugin}, ctrl)
if expectedProvisionCalls != nil {
ctrl.alphaProvisioner = plugin
}
return toWrap(ctrl, reactor, test)
}
}
// wrapTestWithReclaimCalls returns a testCall that:
// - configures controller with recycler or deleter which will return provided
// errors when a volume is deleted or recycled
// - calls given testCall
func wrapTestWithReclaimCalls(operation operationType, expectedOperationCalls []error, toWrap testCall) testCall {
if operation == operationDelete {
return wrapTestWithPluginCalls(nil, expectedOperationCalls, nil, toWrap)
} else {
return wrapTestWithPluginCalls(expectedOperationCalls, nil, nil, toWrap)
}
}
// wrapTestWithProvisionCalls returns a testCall that:
// - configures controller with a provisioner which will return provided errors
// when a claim is provisioned
// - calls given testCall
func wrapTestWithProvisionCalls(expectedProvisionCalls []provisionCall, toWrap testCall) testCall {
return wrapTestWithPluginCalls(nil, nil, expectedProvisionCalls, toWrap)
}
// wrapTestWithInjectedOperation returns a testCall that:
// - starts the controller and lets it run original testCall until
// scheduleOperation() call. It blocks the controller there and calls the
@@ -873,13 +905,13 @@ func evaluateTestResults(ctrl *PersistentVolumeController, reactor *volumeReacto
// 2. Call the tested function (syncClaim/syncVolume) via
// controllerTest.testCall *once*.
// 3. Compare resulting volumes and claims with expected volumes and claims.
func runSyncTests(t *testing.T, tests []controllerTest) {
func runSyncTests(t *testing.T, tests []controllerTest, storageClasses []*extensions.StorageClass) {
for _, test := range tests {
glog.V(4).Infof("starting test %q", test.name)
// Initialize the controller
client := &fake.Clientset{}
ctrl := newTestController(client, nil, nil, true)
ctrl := newTestController(client, nil, nil, nil, true)
reactor := newVolumeReactor(client, ctrl, nil, nil, test.errors)
for _, claim := range test.initialClaims {
ctrl.claims.Add(claim)
@@ -890,6 +922,15 @@ func runSyncTests(t *testing.T, tests []controllerTest) {
reactor.volumes[volume.Name] = volume
}
// Convert classes to []interface{} and forcefully inject them into
// controller.
storageClassPtrs := make([]interface{}, len(storageClasses))
for i, s := range storageClasses {
storageClassPtrs[i] = s
}
// 1 is the resource version
ctrl.classes.Replace(storageClassPtrs, "1")
// Run the tested functions
err := test.test(ctrl, reactor, test)
if err != nil {
@@ -920,13 +961,22 @@ func runSyncTests(t *testing.T, tests []controllerTest) {
// 5. When 3. does not do any changes, finish the tests and compare final set
// of volumes/claims with expected claims/volumes and report differences.
// Some limit of calls in enforced to prevent endless loops.
func runMultisyncTests(t *testing.T, tests []controllerTest) {
func runMultisyncTests(t *testing.T, tests []controllerTest, storageClasses []*extensions.StorageClass, defaultStorageClass string) {
for _, test := range tests {
glog.V(4).Infof("starting multisync test %q", test.name)
// Initialize the controller
client := &fake.Clientset{}
ctrl := newTestController(client, nil, nil, true)
ctrl := newTestController(client, nil, nil, nil, true)
// Convert classes to []interface{} and forcefully inject them into
// controller.
storageClassPtrs := make([]interface{}, len(storageClasses))
for i, s := range storageClasses {
storageClassPtrs[i] = s
}
ctrl.classes.Replace(storageClassPtrs, "1")
reactor := newVolumeReactor(client, ctrl, nil, nil, test.errors)
for _, claim := range test.initialClaims {
ctrl.claims.Add(claim)
@@ -1022,7 +1072,7 @@ func runMultisyncTests(t *testing.T, tests []controllerTest) {
// Dummy volume plugin for provisioning, deletion and recycling. It contains
// lists of expected return values to simulate errors.
type mockVolumePlugin struct {
provisionCalls []error
provisionCalls []provisionCall
provisionCallCounter int
deleteCalls []error
deleteCallCounter int
@@ -1031,6 +1081,11 @@ type mockVolumePlugin struct {
provisionOptions vol.VolumeOptions
}
type provisionCall struct {
expectedParameters map[string]string
ret error
}
var _ vol.VolumePlugin = &mockVolumePlugin{}
var _ vol.RecyclableVolumePlugin = &mockVolumePlugin{}
var _ vol.DeletableVolumePlugin = &mockVolumePlugin{}
@@ -1087,8 +1142,12 @@ func (plugin *mockVolumePlugin) Provision() (*api.PersistentVolume, error) {
}
var pv *api.PersistentVolume
err := plugin.provisionCalls[plugin.provisionCallCounter]
if err == nil {
call := plugin.provisionCalls[plugin.provisionCallCounter]
if !reflect.DeepEqual(call.expectedParameters, plugin.provisionOptions.Parameters) {
glog.Errorf("invalid provisioner call, expected options: %+v, got: %+v", call.expectedParameters, plugin.provisionOptions.Parameters)
return nil, fmt.Errorf("Mock plugin error: invalid provisioner call")
}
if call.ret == nil {
// Create a fake PV with known GCE volume (to match expected volume)
pv = &api.PersistentVolume{
ObjectMeta: api.ObjectMeta{
@@ -1108,8 +1167,8 @@ func (plugin *mockVolumePlugin) Provision() (*api.PersistentVolume, error) {
}
plugin.provisionCallCounter++
glog.V(4).Infof("mock plugin Provision call nr. %d, returning %v: %v", plugin.provisionCallCounter, pv, err)
return pv, err
glog.V(4).Infof("mock plugin Provision call nr. %d, returning %v: %v", plugin.provisionCallCounter, pv, call.ret)
return pv, call.ret
}
// Deleter interfaces

24
pkg/controller/volume/persistentvolume/index.go
View File

@@ -92,6 +92,7 @@ func (pvIndex *persistentVolumeOrderedIndex) findByClaim(claim *api.PersistentVo
var smallestVolumeSize int64
requestedQty := claim.Spec.Resources.Requests[api.ResourceName(api.ResourceStorage)]
requestedSize := requestedQty.Value()
requestedClass := getClaimClass(claim)
var selector labels.Selector
if claim.Spec.Selector != nil {
@@ -123,14 +124,26 @@ func (pvIndex *persistentVolumeOrderedIndex) findByClaim(claim *api.PersistentVo
return volume, nil
}
// In Alpha dynamic provisioning, we do now want not match claims
// with existing PVs, findByClaim must find only PVs that are
// pre-bound to the claim (by dynamic provisioning). TODO: remove in
// 1.5
if hasAnnotation(claim.ObjectMeta, annAlphaClass) {
continue
}
// filter out:
// - volumes bound to another claim
// - volumes whose labels don't match the claim's selector, if specified
// - volumes in Class that is not requested
if volume.Spec.ClaimRef != nil {
continue
} else if selector != nil && !selector.Matches(labels.Set(volume.Labels)) {
continue
}
if getVolumeClass(volume) != requestedClass {
continue
}
volumeQty := volume.Spec.Capacity[api.ResourceStorage]
volumeSize := volumeQty.Value()
@@ -142,17 +155,6 @@ func (pvIndex *persistentVolumeOrderedIndex) findByClaim(claim *api.PersistentVo
}
}
// We want to provision volumes if the annotation is set even if there
// is matching PV. Therefore, do not look for available PV and let
// a new volume to be provisioned.
//
// When provisioner creates a new PV to this claim, an exact match
// pre-bound to the claim will be found by the checks above during
// subsequent claim sync.
if hasAnnotation(claim.ObjectMeta, annClass) {
return nil, nil
}
if smallestVolume != nil {
// Found a matching volume
return smallestVolume, nil

110
pkg/controller/volume/persistentvolume/index_test.go
View File

@@ -164,7 +164,52 @@ func TestMatchVolume(t *testing.T) {
AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce},
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("10000G"),
api.ResourceName(api.ResourceStorage): resource.MustParse("20000G"),
},
},
},
},
},
"successful-match-with-class": {
expectedMatch: "gce-pd-silver1",
claim: &api.PersistentVolumeClaim{
ObjectMeta: api.ObjectMeta{
Name: "claim01",
Namespace: "myns",
Annotations: map[string]string{
annClass: "silver",
},
},
Spec: api.PersistentVolumeClaimSpec{
AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce},
Selector: &unversioned.LabelSelector{
MatchLabels: map[string]string{
"should-exist": "true",
},
},
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("1G"),
},
},
},
},
},
"successful-match-with-class-and-labels": {
expectedMatch: "gce-pd-silver2",
claim: &api.PersistentVolumeClaim{
ObjectMeta: api.ObjectMeta{
Name: "claim01",
Namespace: "myns",
Annotations: map[string]string{
annClass: "silver",
},
},
Spec: api.PersistentVolumeClaimSpec{
AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce},
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("1G"),
},
},
},
@@ -561,6 +606,29 @@ func createTestVolumes() []*api.PersistentVolume {
"should-exist": "true",
},
},
Spec: api.PersistentVolumeSpec{
Capacity: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("20000G"),
},
PersistentVolumeSource: api.PersistentVolumeSource{
GCEPersistentDisk: &api.GCEPersistentDiskVolumeSource{},
},
AccessModes: []api.PersistentVolumeAccessMode{
api.ReadWriteOnce,
},
},
},
{
ObjectMeta: api.ObjectMeta{
UID: "gce-pd-silver1",
Name: "gce0023",
Labels: map[string]string{
"should-exist": "true",
},
Annotations: map[string]string{
annClass: "silver",
},
},
Spec: api.PersistentVolumeSpec{
Capacity: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("10000G"),
@@ -573,6 +641,46 @@ func createTestVolumes() []*api.PersistentVolume {
},
},
},
{
ObjectMeta: api.ObjectMeta{
UID: "gce-pd-silver2",
Name: "gce0024",
Annotations: map[string]string{
annClass: "silver",
},
},
Spec: api.PersistentVolumeSpec{
Capacity: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("100G"),
},
PersistentVolumeSource: api.PersistentVolumeSource{
GCEPersistentDisk: &api.GCEPersistentDiskVolumeSource{},
},
AccessModes: []api.PersistentVolumeAccessMode{
api.ReadWriteOnce,
},
},
},
{
ObjectMeta: api.ObjectMeta{
UID: "gce-pd-gold",
Name: "gce0025",
Annotations: map[string]string{
annClass: "gold",
},
},
Spec: api.PersistentVolumeSpec{
Capacity: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("50G"),
},
PersistentVolumeSource: api.PersistentVolumeSource{
GCEPersistentDisk: &api.GCEPersistentDiskVolumeSource{},
},
AccessModes: []api.PersistentVolumeAccessMode{
api.ReadWriteOnce,
},
},
},
}
}

198
pkg/controller/volume/persistentvolume/provision_test.go
View File

@@ -21,8 +21,63 @@ import (
"testing"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/unversioned"
"k8s.io/kubernetes/pkg/apis/extensions"
)
var class1Parameters = map[string]string{
"param1": "value1",
}
var class2Parameters = map[string]string{
"param2": "value2",
}
var storageClasses = []*extensions.StorageClass{
{
TypeMeta: unversioned.TypeMeta{
Kind: "StorageClass",
},
ObjectMeta: api.ObjectMeta{
Name: "gold",
},
Provisioner: mockPluginName,
Parameters: class1Parameters,
},
{
TypeMeta: unversioned.TypeMeta{
Kind: "StorageClass",
},
ObjectMeta: api.ObjectMeta{
Name: "silver",
},
Provisioner: mockPluginName,
Parameters: class2Parameters,
},
}
// call to storageClass 1, returning an error
var provision1Error = provisionCall{
ret: errors.New("Moc provisioner error"),
expectedParameters: class1Parameters,
}
// call to storageClass 1, returning a valid PV
var provision1Success = provisionCall{
ret: nil,
expectedParameters: class1Parameters,
}
// call to storageClass 2, returning a valid PV
var provision2Success = provisionCall{
ret: nil,
expectedParameters: class2Parameters,
}
var provisionAlphaSuccess = provisionCall{
ret: nil,
}
// Test single call to syncVolume, expecting provisioning to happen.
// 1. Fill in the controller with initial data
// 2. Call the syncVolume *once*.
@@ -30,14 +85,14 @@ import (
func TestProvisionSync(t *testing.T) {
tests := []controllerTest{
{
// Provision a volume
"11-1 - successful provision",
// Provision a volume (with a default class)
"11-1 - successful provision with storage class 1",
novolumes,
newVolumeArray("pvc-uid11-1", "1Gi", "uid11-1", "claim11-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned),
newVolumeArray("pvc-uid11-1", "1Gi", "uid11-1", "claim11-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, annClass),
newClaimArray("claim11-1", "uid11-1", "1Gi", "", api.ClaimPending, annClass),
// Binding will be completed in the next syncClaim
newClaimArray("claim11-1", "uid11-1", "1Gi", "", api.ClaimPending, annClass),
noevents, noerrors, wrapTestWithControllerConfig(operationProvision, []error{nil}, testSyncClaim),
noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision1Success}, testSyncClaim),
},
{
// Provision failure - plugin not found
@@ -57,7 +112,7 @@ func TestProvisionSync(t *testing.T) {
newClaimArray("claim11-3", "uid11-3", "1Gi", "", api.ClaimPending, annClass),
newClaimArray("claim11-3", "uid11-3", "1Gi", "", api.ClaimPending, annClass),
[]string{"Warning ProvisioningFailed"}, noerrors,
wrapTestWithControllerConfig(operationProvision, []error{}, testSyncClaim),
wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim),
},
{
// Provision failure - Provision returns error
@@ -67,40 +122,35 @@ func TestProvisionSync(t *testing.T) {
newClaimArray("claim11-4", "uid11-4", "1Gi", "", api.ClaimPending, annClass),
newClaimArray("claim11-4", "uid11-4", "1Gi", "", api.ClaimPending, annClass),
[]string{"Warning ProvisioningFailed"}, noerrors,
wrapTestWithControllerConfig(operationProvision, []error{errors.New("Moc provisioner error")}, testSyncClaim),
wrapTestWithProvisionCalls([]provisionCall{provision1Error}, testSyncClaim),
},
{
// Provision success - there is already a volume available, still
// we provision a new one when requested.
// No provisioning if there is a matching volume available
"11-6 - provisioning when there is a volume available",
newVolumeArray("volume11-6", "1Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain),
[]*api.PersistentVolume{
newVolume("volume11-6", "1Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain),
newVolume("pvc-uid11-6", "1Gi", "uid11-6", "claim11-6", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned),
},
newClaimArray("claim11-6", "uid11-6", "1Gi", "", api.ClaimPending, annClass),
// Binding will be completed in the next syncClaim
newVolumeArray("volume11-6", "1Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain, annClass),
newVolumeArray("volume11-6", "1Gi", "uid11-6", "claim11-6", api.VolumeBound, api.PersistentVolumeReclaimRetain, annBoundByController, annClass),
newClaimArray("claim11-6", "uid11-6", "1Gi", "", api.ClaimPending, annClass),
newClaimArray("claim11-6", "uid11-6", "1Gi", "volume11-6", api.ClaimBound, annClass, annBoundByController, annBindCompleted),
noevents, noerrors,
// No provisioning plugin confingure - makes the test fail when
// the controller errorneously tries to provision something
wrapTestWithControllerConfig(operationProvision, []error{nil}, testSyncClaim),
wrapTestWithProvisionCalls([]provisionCall{provision1Success}, testSyncClaim),
},
{
// Provision success? - claim is bound before provisioner creates
// a volume.
"11-7 - claim is bound before provisioning",
novolumes,
newVolumeArray("pvc-uid11-7", "1Gi", "uid11-7", "claim11-7", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned),
newVolumeArray("pvc-uid11-7", "1Gi", "uid11-7", "claim11-7", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, annClass),
newClaimArray("claim11-7", "uid11-7", "1Gi", "", api.ClaimPending, annClass),
// The claim would be bound in next syncClaim
newClaimArray("claim11-7", "uid11-7", "1Gi", "", api.ClaimPending, annClass),
noevents, noerrors,
wrapTestWithInjectedOperation(wrapTestWithControllerConfig(operationProvision, []error{}, testSyncClaim), func(ctrl *PersistentVolumeController, reactor *volumeReactor) {
wrapTestWithInjectedOperation(wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), func(ctrl *PersistentVolumeController, reactor *volumeReactor) {
// Create a volume before provisionClaimOperation starts.
// This similates a parallel controller provisioning the volume.
reactor.lock.Lock()
volume := newVolume("pvc-uid11-7", "1Gi", "uid11-7", "claim11-7", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned)
volume := newVolume("pvc-uid11-7", "1Gi", "uid11-7", "claim11-7", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, annClass)
reactor.volumes[volume.Name] = volume
reactor.lock.Unlock()
}),
@@ -110,7 +160,7 @@ func TestProvisionSync(t *testing.T) {
// second retry succeeds
"11-8 - cannot save provisioned volume",
novolumes,
newVolumeArray("pvc-uid11-8", "1Gi", "uid11-8", "claim11-8", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned),
newVolumeArray("pvc-uid11-8", "1Gi", "uid11-8", "claim11-8", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, annClass),
newClaimArray("claim11-8", "uid11-8", "1Gi", "", api.ClaimPending, annClass),
// Binding will be completed in the next syncClaim
newClaimArray("claim11-8", "uid11-8", "1Gi", "", api.ClaimPending, annClass),
@@ -121,7 +171,7 @@ func TestProvisionSync(t *testing.T) {
// will succeed.
{"create", "persistentvolumes", errors.New("Mock creation error")},
},
wrapTestWithControllerConfig(operationProvision, []error{nil}, testSyncClaim),
wrapTestWithProvisionCalls([]provisionCall{provision1Success}, testSyncClaim),
},
{
// Provision success? - cannot save provisioned PV five times,
@@ -141,8 +191,12 @@ func TestProvisionSync(t *testing.T) {
{"create", "persistentvolumes", errors.New("Mock creation error4")},
{"create", "persistentvolumes", errors.New("Mock creation error5")},
},
wrapTestWithControllerConfig(operationDelete, []error{nil},
wrapTestWithControllerConfig(operationProvision, []error{nil}, testSyncClaim)),
wrapTestWithPluginCalls(
nil, // recycle calls
[]error{nil}, // delete calls
[]provisionCall{provision1Success}, // provision calls
testSyncClaim,
),
},
{
// Provision failure - cannot save provisioned PV five times,
@@ -163,7 +217,7 @@ func TestProvisionSync(t *testing.T) {
{"create", "persistentvolumes", errors.New("Mock creation error5")},
},
// No deleteCalls are configured, which results into no deleter plugin available for the volume
wrapTestWithControllerConfig(operationProvision, []error{nil}, testSyncClaim),
wrapTestWithProvisionCalls([]provisionCall{provision1Success}, testSyncClaim),
},
{
// Provision failure - cannot save provisioned PV five times,
@@ -183,16 +237,17 @@ func TestProvisionSync(t *testing.T) {
{"create", "persistentvolumes", errors.New("Mock creation error4")},
{"create", "persistentvolumes", errors.New("Mock creation error5")},
},
wrapTestWithControllerConfig(
operationDelete, []error{
wrapTestWithPluginCalls(
nil, // recycle calls
[]error{ // delete calls
errors.New("Mock deletion error1"),
errors.New("Mock deletion error2"),
errors.New("Mock deletion error3"),
errors.New("Mock deletion error4"),
errors.New("Mock deletion error5"),
},
wrapTestWithControllerConfig(operationProvision, []error{nil}, testSyncClaim),
),
[]provisionCall{provision1Success}, // provision calls
testSyncClaim),
},
{
// Provision failure - cannot save provisioned PV five times,
@@ -212,16 +267,85 @@ func TestProvisionSync(t *testing.T) {
{"create", "persistentvolumes", errors.New("Mock creation error4")},
{"create", "persistentvolumes", errors.New("Mock creation error5")},
},
wrapTestWithControllerConfig(
operationDelete, []error{
wrapTestWithPluginCalls(
nil, // recycle calls
[]error{ // delete calls
errors.New("Mock deletion error1"),
nil,
},
wrapTestWithControllerConfig(operationProvision, []error{nil}, testSyncClaim),
}, // provison calls
[]provisionCall{provision1Success},
testSyncClaim,
),
},
{
// Provision a volume (with non-default class)
"11-13 - successful provision with storage class 2",
novolumes,
volumeWithClass("silver", newVolumeArray("pvc-uid11-13", "1Gi", "uid11-13", "claim11-13", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned)),
claimWithClass("silver", newClaimArray("claim11-13", "uid11-13", "1Gi", "", api.ClaimPending)),
// Binding will be completed in the next syncClaim
claimWithClass("silver", newClaimArray("claim11-13", "uid11-13", "1Gi", "", api.ClaimPending)),
noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision2Success}, testSyncClaim),
},
{
// Provision error - non existing class
"11-14 - fail due to non-existing class",
novolumes,
novolumes,
claimWithClass("non-existing", newClaimArray("claim11-14", "uid11-14", "1Gi", "", api.ClaimPending)),
claimWithClass("non-existing", newClaimArray("claim11-14", "uid11-14", "1Gi", "", api.ClaimPending)),
noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim),
},
{
// No provisioning with class=""
"11-15 - no provisioning with class=''",
novolumes,
novolumes,
claimWithClass("", newClaimArray("claim11-15", "uid11-15", "1Gi", "", api.ClaimPending)),
claimWithClass("", newClaimArray("claim11-15", "uid11-15", "1Gi", "", api.ClaimPending)),
noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim),
},
{
// No provisioning with class=nil
"11-16 - no provisioning with class=nil",
novolumes,
novolumes,
newClaimArray("claim11-15", "uid11-15", "1Gi", "", api.ClaimPending),
newClaimArray("claim11-15", "uid11-15", "1Gi", "", api.ClaimPending),
noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim),
},
}
runSyncTests(t, tests)
runSyncTests(t, tests, storageClasses)
}
func TestAlphaProvisionSync(t *testing.T) {
tests := []controllerTest{
{
// Provision a volume with alpha annotation
"14-1 - successful alpha provisioning",
novolumes,
newVolumeArray("pvc-uid14-1", "1Gi", "uid14-1", "claim14-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned),
newClaimArray("claim14-1", "uid14-1", "1Gi", "", api.ClaimPending, annAlphaClass),
// Binding will be completed in the next syncClaim
newClaimArray("claim14-1", "uid14-1", "1Gi", "", api.ClaimPending, annAlphaClass),
noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provisionAlphaSuccess}, testSyncClaim),
},
{
// Provision success - there is already a volume available, still
// we provision a new one when requested.
"14-2 - no alpha provisioning when there is a volume available",
newVolumeArray("volume14-2", "1Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain),
[]*api.PersistentVolume{
newVolume("volume14-2", "1Gi", "", "", api.VolumePending, api.PersistentVolumeReclaimRetain),
newVolume("pvc-uid14-2", "1Gi", "uid14-2", "claim14-2", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned),
},
newClaimArray("claim14-2", "uid14-2", "1Gi", "", api.ClaimPending, annAlphaClass),
// Binding will be completed in the next syncClaim
newClaimArray("claim14-2", "uid14-2", "1Gi", "", api.ClaimPending, annAlphaClass),
noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provisionAlphaSuccess}, testSyncClaim),
},
}
runSyncTests(t, tests, []*extensions.StorageClass{})
}
// Test multiple calls to syncClaim/syncVolume and periodic sync of all
@@ -244,20 +368,20 @@ func TestProvisionMultiSync(t *testing.T) {
// Provision a volume with binding
"12-1 - successful provision",
novolumes,
newVolumeArray("pvc-uid12-1", "1Gi", "uid12-1", "claim12-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned),
newVolumeArray("pvc-uid12-1", "1Gi", "uid12-1", "claim12-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, annClass),
newClaimArray("claim12-1", "uid12-1", "1Gi", "", api.ClaimPending, annClass),
// Binding will be completed in the next syncClaim
newClaimArray("claim12-1", "uid12-1", "1Gi", "pvc-uid12-1", api.ClaimBound, annClass, annBoundByController, annBindCompleted),
noevents, noerrors, wrapTestWithControllerConfig(operationProvision, []error{nil}, testSyncClaim),
noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision1Success}, testSyncClaim),
},
}
runMultisyncTests(t, tests)
runMultisyncTests(t, tests, storageClasses, storageClasses[0].Name)
}
// When provisioning is disabled, provisioning a claim should instantly return nil
func TestDisablingDynamicProvisioner(t *testing.T) {
ctrl := newTestController(nil, nil, nil, false)
ctrl := newTestController(nil, nil, nil, nil, false)
retVal := ctrl.provisionClaim(nil)
if retVal != nil {
t.Errorf("Expected nil return but got %v", retVal)

23
pkg/controller/volume/persistentvolume/recycle_test.go
View File

@@ -21,6 +21,7 @@ import (
"testing"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/apis/extensions"
)
// Test single call to syncVolume, expecting recycling to happen.
@@ -39,7 +40,7 @@ func TestRecycleSync(t *testing.T) {
noevents, noerrors,
// Inject recycler into the controller and call syncVolume. The
// recycler simulates one recycle() call that succeeds.
wrapTestWithControllerConfig(operationRecycle, []error{nil}, testSyncVolume),
wrapTestWithReclaimCalls(operationRecycle, []error{nil}, testSyncVolume),
},
{
// recycle volume bound by user
@@ -51,7 +52,7 @@ func TestRecycleSync(t *testing.T) {
noevents, noerrors,
// Inject recycler into the controller and call syncVolume. The
// recycler simulates one recycle() call that succeeds.
wrapTestWithControllerConfig(operationRecycle, []error{nil}, testSyncVolume),
wrapTestWithReclaimCalls(operationRecycle, []error{nil}, testSyncVolume),
},
{
// recycle failure - plugin not found
@@ -70,7 +71,7 @@ func TestRecycleSync(t *testing.T) {
noclaims,
noclaims,
[]string{"Warning VolumeFailedRecycle"}, noerrors,
wrapTestWithControllerConfig(operationRecycle, []error{}, testSyncVolume),
wrapTestWithReclaimCalls(operationRecycle, []error{}, testSyncVolume),
},
{
// recycle failure - recycle returns error
@@ -80,7 +81,7 @@ func TestRecycleSync(t *testing.T) {
noclaims,
noclaims,
[]string{"Warning VolumeFailedRecycle"}, noerrors,
wrapTestWithControllerConfig(operationRecycle, []error{errors.New("Mock recycle error")}, testSyncVolume),
wrapTestWithReclaimCalls(operationRecycle, []error{errors.New("Mock recycle error")}, testSyncVolume),
},
{
// recycle success(?) - volume is deleted before doRecycle() starts
@@ -90,7 +91,7 @@ func TestRecycleSync(t *testing.T) {
noclaims,
noclaims,
noevents, noerrors,
wrapTestWithInjectedOperation(wrapTestWithControllerConfig(operationRecycle, []error{}, testSyncVolume), func(ctrl *PersistentVolumeController, reactor *volumeReactor) {
wrapTestWithInjectedOperation(wrapTestWithReclaimCalls(operationRecycle, []error{}, testSyncVolume), func(ctrl *PersistentVolumeController, reactor *volumeReactor) {
// Delete the volume before recycle operation starts
reactor.lock.Lock()
delete(reactor.volumes, "volume6-6")
@@ -107,7 +108,7 @@ func TestRecycleSync(t *testing.T) {
noclaims,
noclaims,
noevents, noerrors,
wrapTestWithInjectedOperation(wrapTestWithControllerConfig(operationRecycle, []error{}, testSyncVolume), func(ctrl *PersistentVolumeController, reactor *volumeReactor) {
wrapTestWithInjectedOperation(wrapTestWithReclaimCalls(operationRecycle, []error{}, testSyncVolume), func(ctrl *PersistentVolumeController, reactor *volumeReactor) {
// Mark the volume as Available before the recycler starts
reactor.lock.Lock()
volume := reactor.volumes["volume6-7"]
@@ -128,7 +129,7 @@ func TestRecycleSync(t *testing.T) {
noclaims,
noclaims,
noevents, noerrors,
wrapTestWithInjectedOperation(wrapTestWithControllerConfig(operationRecycle, []error{}, testSyncVolume), func(ctrl *PersistentVolumeController, reactor *volumeReactor) {
wrapTestWithInjectedOperation(wrapTestWithReclaimCalls(operationRecycle, []error{}, testSyncVolume), func(ctrl *PersistentVolumeController, reactor *volumeReactor) {
// Mark the volume as Available before the recycler starts
reactor.lock.Lock()
volume := reactor.volumes["volume6-8"]
@@ -148,7 +149,7 @@ func TestRecycleSync(t *testing.T) {
noevents, noerrors,
// Inject recycler into the controller and call syncVolume. The
// recycler simulates one recycle() call that succeeds.
wrapTestWithControllerConfig(operationRecycle, []error{nil}, testSyncVolume),
wrapTestWithReclaimCalls(operationRecycle, []error{nil}, testSyncVolume),
},
{
// volume has unknown reclaim policy - failure expected
@@ -160,7 +161,7 @@ func TestRecycleSync(t *testing.T) {
[]string{"Warning VolumeUnknownReclaimPolicy"}, noerrors, testSyncVolume,
},
}
runSyncTests(t, tests)
runSyncTests(t, tests, []*extensions.StorageClass{})
}
// Test multiple calls to syncClaim/syncVolume and periodic sync of all
@@ -188,9 +189,9 @@ func TestRecycleMultiSync(t *testing.T) {
noclaims,
noclaims,
[]string{"Warning VolumeFailedRecycle"}, noerrors,
wrapTestWithControllerConfig(operationRecycle, []error{errors.New("Mock recycle error"), nil}, testSyncVolume),
wrapTestWithReclaimCalls(operationRecycle, []error{errors.New("Mock recycle error"), nil}, testSyncVolume),
},
}
runMultisyncTests(t, tests)
runMultisyncTests(t, tests, []*extensions.StorageClass{}, "")
}