From 1d7cefe9c4b95008487c765107ba8346fb10535e Mon Sep 17 00:00:00 2001 From: Konstantin Misyutin Date: Tue, 22 Jun 2021 01:30:58 +0800 Subject: [PATCH 1/4] Move volume helpers to "k8s.io/component-helpers/storage/volume". This patch aims to simplify decoupling "pkg/scheduler/framework/plugins" from internal "k8s.io/kubernetes" packages. More described in issue #89930 and PR #102953. Some helpers from "k8s.io/kubernetes/pkg/controller/volume/persistentvolume" package moved to "k8s.io/component-helpers/storage/volume" package: - IsDelayBindingMode - GetBindVolumeToClaim - IsVolumeBoundToClaim - FindMatchingVolume - CheckVolumeModeMismatches - CheckAccessModes - GetVolumeNodeAffinity Also "CheckNodeAffinity" from "k8s.io/kubernetes/pkg/volume/util" package moved to "k8s.io/component-helpers/storage/volume" package to prevent diamond dependency conflict. Signed-off-by: Konstantin Misyutin --- .../volume/persistentvolume/binder_test.go | 2 +- .../volume/persistentvolume/delete_test.go | 4 +- .../volume/persistentvolume/framework_test.go | 2 +- .../volume/persistentvolume/index.go | 4 +- .../volume/persistentvolume/index_test.go | 128 +------- .../volume/persistentvolume/provision_test.go | 2 +- .../volume/persistentvolume/pv_controller.go | 17 +- .../persistentvolume/pv_controller_base.go | 2 +- .../persistentvolume/pv_controller_test.go | 85 +---- .../volume/persistentvolume/recycle_test.go | 4 +- .../volumebinding/assume_cache_test.go | 2 +- .../framework/plugins/volumebinding/binder.go | 14 +- .../plugins/volumebinding/binder_test.go | 2 +- .../plugins/volumebinding/test_utils.go | 2 +- pkg/scheduler/scheduler_test.go | 2 +- .../operationexecutor/operation_generator.go | 3 +- pkg/volume/util/util.go | 25 -- pkg/volume/util/util_test.go | 193 ----------- .../persistentvolume/label/admission.go | 2 +- .../persistentvolume/label/admission_test.go | 2 +- staging/src/k8s.io/component-helpers/go.sum | 1 + .../storage/volume/helpers.go | 31 +- .../storage/volume/helpers_test.go | 216 +++++++++++++ .../storage/volume/pv_helpers.go | 14 +- .../storage/volume/pv_helpers_test.go | 300 ++++++++++++++++++ test/integration/util/util.go | 2 +- 26 files changed, 588 insertions(+), 473 deletions(-) create mode 100644 staging/src/k8s.io/component-helpers/storage/volume/helpers_test.go rename pkg/controller/volume/persistentvolume/util/util.go => staging/src/k8s.io/component-helpers/storage/volume/pv_helpers.go (96%) create mode 100644 staging/src/k8s.io/component-helpers/storage/volume/pv_helpers_test.go diff --git a/pkg/controller/volume/persistentvolume/binder_test.go b/pkg/controller/volume/persistentvolume/binder_test.go index 26843c6a9f0..7a01bc8b314 100644 --- a/pkg/controller/volume/persistentvolume/binder_test.go +++ b/pkg/controller/volume/persistentvolume/binder_test.go @@ -22,7 +22,7 @@ import ( v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" + pvutil "k8s.io/component-helpers/storage/volume" ) // Test single call to syncClaim and syncVolume methods. diff --git a/pkg/controller/volume/persistentvolume/delete_test.go b/pkg/controller/volume/persistentvolume/delete_test.go index 50402d7e2a9..e41697824f8 100644 --- a/pkg/controller/volume/persistentvolume/delete_test.go +++ b/pkg/controller/volume/persistentvolume/delete_test.go @@ -20,10 +20,10 @@ import ( "errors" "testing" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" + pvutil "k8s.io/component-helpers/storage/volume" pvtesting "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/testing" - pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" ) // Test single call to syncVolume, expecting recycling to happen. diff --git a/pkg/controller/volume/persistentvolume/framework_test.go b/pkg/controller/volume/persistentvolume/framework_test.go index 4dd786ce160..8de81480108 100644 --- a/pkg/controller/volume/persistentvolume/framework_test.go +++ b/pkg/controller/volume/persistentvolume/framework_test.go @@ -41,9 +41,9 @@ import ( storagelisters "k8s.io/client-go/listers/storage/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" + pvutil "k8s.io/component-helpers/storage/volume" "k8s.io/kubernetes/pkg/controller" pvtesting "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/testing" - pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" "k8s.io/kubernetes/pkg/volume" vol "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util/recyclerclient" diff --git a/pkg/controller/volume/persistentvolume/index.go b/pkg/controller/volume/persistentvolume/index.go index 9652056aa3b..ee4e8c3e37b 100644 --- a/pkg/controller/volume/persistentvolume/index.go +++ b/pkg/controller/volume/persistentvolume/index.go @@ -20,10 +20,10 @@ import ( "fmt" "sort" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" + pvutil "k8s.io/component-helpers/storage/volume" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" - pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" volumeutil "k8s.io/kubernetes/pkg/volume/util" ) diff --git a/pkg/controller/volume/persistentvolume/index_test.go b/pkg/controller/volume/persistentvolume/index_test.go index 7350fe39299..5f913397a88 100644 --- a/pkg/controller/volume/persistentvolume/index_test.go +++ b/pkg/controller/volume/persistentvolume/index_test.go @@ -25,7 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" ref "k8s.io/client-go/tools/reference" - pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" + pvutil "k8s.io/component-helpers/storage/volume" "k8s.io/kubernetes/pkg/volume/util" ) @@ -1377,132 +1377,6 @@ func TestBestMatchDelayed(t *testing.T) { } } -func TestFindMatchVolumeWithNode(t *testing.T) { - volumes := createTestVolumes() - node1 := &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"key1": "value1"}, - }, - } - node2 := &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"key1": "value2"}, - }, - } - node3 := &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"key1": "value3"}, - }, - } - node4 := &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"key1": "value4"}, - }, - } - - scenarios := map[string]struct { - expectedMatch string - claim *v1.PersistentVolumeClaim - node *v1.Node - excludedVolumes map[string]*v1.PersistentVolume - }{ - "success-match": { - expectedMatch: "affinity-pv", - claim: makePVC("100G", func(pvc *v1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce} - pvc.Spec.StorageClassName = &classWait - }), - node: node1, - }, - "success-prebound": { - expectedMatch: "affinity-prebound", - claim: makePVC("100G", func(pvc *v1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce} - pvc.Spec.StorageClassName = &classWait - pvc.Name = "claim02" - }), - node: node1, - }, - "success-exclusion": { - expectedMatch: "affinity-pv2", - claim: makePVC("100G", func(pvc *v1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce} - pvc.Spec.StorageClassName = &classWait - }), - node: node1, - excludedVolumes: map[string]*v1.PersistentVolume{"affinity001": nil}, - }, - "fail-exclusion": { - expectedMatch: "", - claim: makePVC("100G", func(pvc *v1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce} - pvc.Spec.StorageClassName = &classWait - }), - node: node1, - excludedVolumes: map[string]*v1.PersistentVolume{"affinity001": nil, "affinity002": nil}, - }, - "fail-accessmode": { - expectedMatch: "", - claim: makePVC("100G", func(pvc *v1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteMany} - pvc.Spec.StorageClassName = &classWait - }), - node: node1, - }, - "fail-nodeaffinity": { - expectedMatch: "", - claim: makePVC("100G", func(pvc *v1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce} - pvc.Spec.StorageClassName = &classWait - }), - node: node2, - }, - "fail-prebound-node-affinity": { - expectedMatch: "", - claim: makePVC("100G", func(pvc *v1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce} - pvc.Spec.StorageClassName = &classWait - pvc.Name = "claim02" - }), - node: node3, - }, - "fail-nonavaliable": { - expectedMatch: "", - claim: makePVC("100G", func(pvc *v1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce} - pvc.Spec.StorageClassName = &classWait - pvc.Name = "claim04" - }), - node: node4, - }, - "success-bad-and-good-node-affinity": { - expectedMatch: "affinity-pv3", - claim: makePVC("100G", func(pvc *v1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce} - pvc.Spec.StorageClassName = &classWait - pvc.Name = "claim03" - }), - node: node3, - }, - } - - for name, scenario := range scenarios { - volume, err := pvutil.FindMatchingVolume(scenario.claim, volumes, scenario.node, scenario.excludedVolumes, true) - if err != nil { - t.Errorf("Unexpected error matching volume by claim: %v", err) - } - if len(scenario.expectedMatch) != 0 && volume == nil { - t.Errorf("Expected match but received nil volume for scenario: %s", name) - } - if len(scenario.expectedMatch) != 0 && volume != nil && string(volume.UID) != scenario.expectedMatch { - t.Errorf("Expected %s but got volume %s in scenario %s", scenario.expectedMatch, volume.UID, name) - } - if len(scenario.expectedMatch) == 0 && volume != nil { - t.Errorf("Unexpected match for scenario: %s, matched with %s instead", name, volume.UID) - } - } -} - func TestCheckAccessModes(t *testing.T) { volume := &v1.PersistentVolume{ Spec: v1.PersistentVolumeSpec{ diff --git a/pkg/controller/volume/persistentvolume/provision_test.go b/pkg/controller/volume/persistentvolume/provision_test.go index 0434dadca85..7801bb79c82 100644 --- a/pkg/controller/volume/persistentvolume/provision_test.go +++ b/pkg/controller/volume/persistentvolume/provision_test.go @@ -27,9 +27,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" + pvutil "k8s.io/component-helpers/storage/volume" api "k8s.io/kubernetes/pkg/apis/core" pvtesting "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/testing" - pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" ) var class1Parameters = map[string]string{ diff --git a/pkg/controller/volume/persistentvolume/pv_controller.go b/pkg/controller/volume/persistentvolume/pv_controller.go index 61429c316aa..6e40e38acc4 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller.go +++ b/pkg/controller/volume/persistentvolume/pv_controller.go @@ -43,11 +43,10 @@ import ( "k8s.io/client-go/util/workqueue" cloudprovider "k8s.io/cloud-provider" volerr "k8s.io/cloud-provider/volume/errors" - storagehelpers "k8s.io/component-helpers/storage/volume" + pvutil "k8s.io/component-helpers/storage/volume" "k8s.io/kubernetes/pkg/controller/volume/common" "k8s.io/kubernetes/pkg/controller/volume/events" "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/metrics" - pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" proxyutil "k8s.io/kubernetes/pkg/proxy/util" "k8s.io/kubernetes/pkg/util/goroutinemap" "k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff" @@ -287,8 +286,8 @@ func checkVolumeSatisfyClaim(volume *v1.PersistentVolume, claim *v1.PersistentVo return fmt.Errorf("requested PV is too small") } - requestedClass := storagehelpers.GetPersistentVolumeClaimClass(claim) - if storagehelpers.GetPersistentVolumeClass(volume) != requestedClass { + requestedClass := pvutil.GetPersistentVolumeClaimClass(claim) + if pvutil.GetPersistentVolumeClass(volume) != requestedClass { return fmt.Errorf("storageClassName does not match") } @@ -355,7 +354,7 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(ctx context.Context, cl if err = ctrl.emitEventForUnboundDelayBindingClaim(claim); err != nil { return err } - case storagehelpers.GetPersistentVolumeClaimClass(claim) != "": + case pvutil.GetPersistentVolumeClaimClass(claim) != "": if err = ctrl.provisionClaim(ctx, claim); err != nil { return err } @@ -1526,7 +1525,7 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation( claim *v1.PersistentVolumeClaim, plugin vol.ProvisionableVolumePlugin, storageClass *storage.StorageClass) (string, error) { - claimClass := storagehelpers.GetPersistentVolumeClaimClass(claim) + claimClass := pvutil.GetPersistentVolumeClaimClass(claim) klog.V(4).Infof("provisionClaimOperation [%s] started, class: %q", claimToClaimKey(claim), claimClass) // called from provisionClaim(), in this case, plugin MUST NOT be nil @@ -1736,7 +1735,7 @@ func (ctrl *PersistentVolumeController) provisionClaimOperationExternal( ctx context.Context, claim *v1.PersistentVolumeClaim, storageClass *storage.StorageClass) (string, error) { - claimClass := storagehelpers.GetPersistentVolumeClaimClass(claim) + claimClass := pvutil.GetPersistentVolumeClaimClass(claim) klog.V(4).Infof("provisionClaimOperationExternal [%s] started, class: %q", claimToClaimKey(claim), claimClass) // Set provisionerName to external provisioner name by setClaimProvisioner var err error @@ -1834,7 +1833,7 @@ func (ctrl *PersistentVolumeController) newRecyclerEventRecorder(volume *v1.Pers func (ctrl *PersistentVolumeController) findProvisionablePlugin(claim *v1.PersistentVolumeClaim) (vol.ProvisionableVolumePlugin, *storage.StorageClass, error) { // provisionClaim() which leads here is never called with claimClass=="", we // can save some checks. - claimClass := storagehelpers.GetPersistentVolumeClaimClass(claim) + claimClass := pvutil.GetPersistentVolumeClaimClass(claim) class, err := ctrl.classLister.Get(claimClass) if err != nil { return nil, nil, err @@ -1902,7 +1901,7 @@ func (ctrl *PersistentVolumeController) getProvisionerNameFromVolume(volume *v1. // the AnnDynamicallyProvisioned annotation value, use the storageClass's Provisioner // field to avoid explosion of the metric in the cases like local storage provisioner // tagging a volume with arbitrary provisioner names - storageClass := storagehelpers.GetPersistentVolumeClass(volume) + storageClass := pvutil.GetPersistentVolumeClass(volume) class, err := ctrl.classLister.Get(storageClass) if err != nil { return "N/A" diff --git a/pkg/controller/volume/persistentvolume/pv_controller_base.go b/pkg/controller/volume/persistentvolume/pv_controller_base.go index 5f57add8581..7f24d73074e 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller_base.go +++ b/pkg/controller/volume/persistentvolume/pv_controller_base.go @@ -40,11 +40,11 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" cloudprovider "k8s.io/cloud-provider" + pvutil "k8s.io/component-helpers/storage/volume" csitrans "k8s.io/csi-translation-lib" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller/volume/common" "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/metrics" - pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" "k8s.io/kubernetes/pkg/features" proxyutil "k8s.io/kubernetes/pkg/proxy/util" "k8s.io/kubernetes/pkg/util/goroutinemap" diff --git a/pkg/controller/volume/persistentvolume/pv_controller_test.go b/pkg/controller/volume/persistentvolume/pv_controller_test.go index 645fdb08160..c4cb88cf4b6 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller_test.go +++ b/pkg/controller/volume/persistentvolume/pv_controller_test.go @@ -36,23 +36,16 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" + pvutil "k8s.io/component-helpers/storage/volume" csitrans "k8s.io/csi-translation-lib" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/controller" pvtesting "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/testing" - pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume/csimigration" "k8s.io/kubernetes/pkg/volume/util" ) -var ( - classNotHere = "not-here" - classNoMode = "no-mode" - classImmediateMode = "immediate-mode" - classWaitMode = "wait-mode" -) - // Test the real controller methods (add/update/delete claim/volume) with // a fake API server. // There is no controller API to 'initiate syncAll now', therefore these tests @@ -469,19 +462,6 @@ func TestControllerCacheParsingError(t *testing.T) { } } -func makePVCClass(scName *string) *v1.PersistentVolumeClaim { - claim := &v1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{}, - }, - Spec: v1.PersistentVolumeClaimSpec{ - StorageClassName: scName, - }, - } - - return claim -} - func makeStorageClass(scName string, mode *storagev1.VolumeBindingMode) *storagev1.StorageClass { return &storagev1.StorageClass{ ObjectMeta: metav1.ObjectMeta{ @@ -491,69 +471,6 @@ func makeStorageClass(scName string, mode *storagev1.VolumeBindingMode) *storage } } -func TestDelayBindingMode(t *testing.T) { - tests := map[string]struct { - pvc *v1.PersistentVolumeClaim - shouldDelay bool - shouldFail bool - }{ - "nil-class": { - pvc: makePVCClass(nil), - shouldDelay: false, - }, - "class-not-found": { - pvc: makePVCClass(&classNotHere), - shouldDelay: false, - }, - "no-mode-class": { - pvc: makePVCClass(&classNoMode), - shouldDelay: false, - shouldFail: true, - }, - "immediate-mode-class": { - pvc: makePVCClass(&classImmediateMode), - shouldDelay: false, - }, - "wait-mode-class": { - pvc: makePVCClass(&classWaitMode), - shouldDelay: true, - }, - } - - classes := []*storagev1.StorageClass{ - makeStorageClass(classNoMode, nil), - makeStorageClass(classImmediateMode, &modeImmediate), - makeStorageClass(classWaitMode, &modeWait), - } - - client := &fake.Clientset{} - informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) - classInformer := informerFactory.Storage().V1().StorageClasses() - ctrl := &PersistentVolumeController{ - classLister: classInformer.Lister(), - translator: csitrans.New(), - } - - for _, class := range classes { - if err := classInformer.Informer().GetIndexer().Add(class); err != nil { - t.Fatalf("Failed to add storage class %q: %v", class.Name, err) - } - } - - for name, test := range tests { - shouldDelay, err := pvutil.IsDelayBindingMode(test.pvc, ctrl.classLister) - if err != nil && !test.shouldFail { - t.Errorf("Test %q returned error: %v", name, err) - } - if err == nil && test.shouldFail { - t.Errorf("Test %q returned success, expected error", name) - } - if shouldDelay != test.shouldDelay { - t.Errorf("Test %q returned unexpected %v", name, test.shouldDelay) - } - } -} - func TestAnnealMigrationAnnotations(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigration, true)() diff --git a/pkg/controller/volume/persistentvolume/recycle_test.go b/pkg/controller/volume/persistentvolume/recycle_test.go index f788a9bb9bd..d8bfdbc7477 100644 --- a/pkg/controller/volume/persistentvolume/recycle_test.go +++ b/pkg/controller/volume/persistentvolume/recycle_test.go @@ -20,11 +20,11 @@ import ( "errors" "testing" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + pvutil "k8s.io/component-helpers/storage/volume" pvtesting "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/testing" - pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" ) // Test single call to syncVolume, expecting recycling to happen. diff --git a/pkg/scheduler/framework/plugins/volumebinding/assume_cache_test.go b/pkg/scheduler/framework/plugins/volumebinding/assume_cache_test.go index 89dc58e2b26..f7a91235b1b 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/assume_cache_test.go +++ b/pkg/scheduler/framework/plugins/volumebinding/assume_cache_test.go @@ -22,7 +22,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" + pvutil "k8s.io/component-helpers/storage/volume" ) func verifyListPVs(t *testing.T, cache PVAssumeCache, expectedPVs map[string]*v1.PersistentVolume, storageClassName string) { diff --git a/pkg/scheduler/framework/plugins/volumebinding/binder.go b/pkg/scheduler/framework/plugins/volumebinding/binder.go index 7e4405423f3..6dc820e0fcf 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/binder.go +++ b/pkg/scheduler/framework/plugins/volumebinding/binder.go @@ -41,15 +41,13 @@ import ( storagelisters "k8s.io/client-go/listers/storage/v1" storagelistersv1beta1 "k8s.io/client-go/listers/storage/v1beta1" "k8s.io/component-helpers/storage/ephemeral" - storagehelpers "k8s.io/component-helpers/storage/volume" + pvutil "k8s.io/component-helpers/storage/volume" csitrans "k8s.io/csi-translation-lib" csiplugins "k8s.io/csi-translation-lib/plugins" "k8s.io/klog/v2" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" - pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding/metrics" - volumeutil "k8s.io/kubernetes/pkg/volume/util" ) // ConflictReason is used for the special strings which explain why @@ -608,7 +606,7 @@ func (b *volumeBinder) checkBindings(pod *v1.Pod, bindings []*BindingInfo, claim } // Check PV's node affinity (the node might not have the proper label) - if err := volumeutil.CheckNodeAffinity(pv, node.Labels); err != nil { + if err := pvutil.CheckNodeAffinity(pv, node.Labels); err != nil { return false, fmt.Errorf("pv %q node affinity doesn't match node %q: %w", pv.Name, node.Name, err) } @@ -666,7 +664,7 @@ func (b *volumeBinder) checkBindings(pod *v1.Pod, bindings []*BindingInfo, claim return false, err } - if err := volumeutil.CheckNodeAffinity(pv, node.Labels); err != nil { + if err := pvutil.CheckNodeAffinity(pv, node.Labels); err != nil { return false, fmt.Errorf("pv %q node affinity doesn't match node %q: %w", pv.Name, node.Name, err) } } @@ -806,7 +804,7 @@ func (b *volumeBinder) checkBoundClaims(claims []*v1.PersistentVolumeClaim, node return false, true, err } - err = volumeutil.CheckNodeAffinity(pv, node.Labels) + err = pvutil.CheckNodeAffinity(pv, node.Labels) if err != nil { klog.V(4).InfoS("PersistentVolume and node mismatch for pod", "PV", klog.KRef("", pvName), "node", klog.KObj(node), "pod", klog.KObj(pod), "err", err) return false, true, nil @@ -830,7 +828,7 @@ func (b *volumeBinder) findMatchingVolumes(pod *v1.Pod, claimsToBind []*v1.Persi for _, pvc := range claimsToBind { // Get storage class name from each PVC - storageClassName := storagehelpers.GetPersistentVolumeClaimClass(pvc) + storageClassName := pvutil.GetPersistentVolumeClaimClass(pvc) allPVs := b.pvCache.ListPVs(storageClassName) // Find a matching PV @@ -868,7 +866,7 @@ func (b *volumeBinder) checkVolumeProvisions(pod *v1.Pod, claimsToProvision []*v // fails or we encounter an error. for _, claim := range claimsToProvision { pvcName := getPVCName(claim) - className := storagehelpers.GetPersistentVolumeClaimClass(claim) + className := pvutil.GetPersistentVolumeClaimClass(claim) if className == "" { return false, false, nil, fmt.Errorf("no class for claim %q", pvcName) } diff --git a/pkg/scheduler/framework/plugins/volumebinding/binder_test.go b/pkg/scheduler/framework/plugins/volumebinding/binder_test.go index 8d45e646112..bcc47a6027e 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/binder_test.go +++ b/pkg/scheduler/framework/plugins/volumebinding/binder_test.go @@ -43,10 +43,10 @@ import ( "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" featuregatetesting "k8s.io/component-base/featuregate/testing" + pvutil "k8s.io/component-helpers/storage/volume" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/controller" pvtesting "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/testing" - pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" "k8s.io/kubernetes/pkg/features" ) diff --git a/pkg/scheduler/framework/plugins/volumebinding/test_utils.go b/pkg/scheduler/framework/plugins/volumebinding/test_utils.go index ff759d3d8c5..a64f8e0519f 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/test_utils.go +++ b/pkg/scheduler/framework/plugins/volumebinding/test_utils.go @@ -22,7 +22,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" + pvutil "k8s.io/component-helpers/storage/volume" "k8s.io/utils/pointer" ) diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index cb1cd4e1fa8..0e50a071145 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -46,7 +46,7 @@ import ( clienttesting "k8s.io/client-go/testing" clientcache "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/events" - pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" + pvutil "k8s.io/component-helpers/storage/volume" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 265fc0f6b5a..5230af7658b 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -35,6 +35,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/record" volerr "k8s.io/cloud-provider/volume/errors" + pvutil "k8s.io/component-helpers/storage/volume" csitrans "k8s.io/csi-translation-lib" "k8s.io/klog/v2" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" @@ -2263,7 +2264,7 @@ func checkNodeAffinity(og *operationGenerator, volumeToMount VolumeToMount) erro if err != nil { return err } - err = util.CheckNodeAffinity(pv, nodeLabels) + err = pvutil.CheckNodeAffinity(pv, nodeLabels) if err != nil { return err } diff --git a/pkg/volume/util/util.go b/pkg/volume/util/util.go index 702f136e0de..f772bd5869f 100644 --- a/pkg/volume/util/util.go +++ b/pkg/volume/util/util.go @@ -36,7 +36,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" - "k8s.io/component-helpers/scheduling/corev1" storagehelpers "k8s.io/component-helpers/storage/volume" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/api/legacyscheme" @@ -164,30 +163,6 @@ func GetClassForVolume(kubeClient clientset.Interface, pv *v1.PersistentVolume) return class, nil } -// CheckNodeAffinity looks at the PV node affinity, and checks if the node has the same corresponding labels -// This ensures that we don't mount a volume that doesn't belong to this node -func CheckNodeAffinity(pv *v1.PersistentVolume, nodeLabels map[string]string) error { - return checkVolumeNodeAffinity(pv, &v1.Node{ObjectMeta: metav1.ObjectMeta{Labels: nodeLabels}}) -} - -func checkVolumeNodeAffinity(pv *v1.PersistentVolume, node *v1.Node) error { - if pv.Spec.NodeAffinity == nil { - return nil - } - - if pv.Spec.NodeAffinity.Required != nil { - terms := pv.Spec.NodeAffinity.Required - klog.V(10).Infof("Match for Required node selector terms %+v", terms) - if matches, err := corev1.MatchNodeSelectorTerms(node, terms); err != nil { - return err - } else if !matches { - return fmt.Errorf("no matching NodeSelectorTerms") - } - } - - return nil -} - // LoadPodFromFile will read, decode, and return a Pod from a file. func LoadPodFromFile(filePath string) (*v1.Pod, error) { if filePath == "" { diff --git a/pkg/volume/util/util_test.go b/pkg/volume/util/util_test.go index 51aea65cde3..50758c3758d 100644 --- a/pkg/volume/util/util_test.go +++ b/pkg/volume/util/util_test.go @@ -37,199 +37,6 @@ import ( utilptr "k8s.io/utils/pointer" ) -var nodeLabels = map[string]string{ - "test-key1": "test-value1", - "test-key2": "test-value2", -} - -func TestCheckVolumeNodeAffinity(t *testing.T) { - type affinityTest struct { - name string - expectSuccess bool - pv *v1.PersistentVolume - } - - cases := []affinityTest{ - { - name: "valid-nil", - expectSuccess: true, - pv: testVolumeWithNodeAffinity(t, nil), - }, - { - name: "valid-no-constraints", - expectSuccess: true, - pv: testVolumeWithNodeAffinity(t, &v1.VolumeNodeAffinity{}), - }, - { - name: "select-nothing", - expectSuccess: false, - pv: testVolumeWithNodeAffinity(t, &v1.VolumeNodeAffinity{Required: &v1.NodeSelector{}}), - }, - { - name: "select-nothing-empty-terms", - expectSuccess: false, - pv: testVolumeWithNodeAffinity(t, &v1.VolumeNodeAffinity{ - Required: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{}, - }, - }, - }, - }), - }, - { - name: "valid-multiple-terms", - expectSuccess: true, - pv: testVolumeWithNodeAffinity(t, &v1.VolumeNodeAffinity{ - Required: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "test-key3", - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-value1", "test-value3"}, - }, - }, - }, - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "test-key2", - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-value0", "test-value2"}, - }, - }, - }, - }, - }, - }), - }, - { - name: "valid-multiple-match-expressions", - expectSuccess: true, - pv: testVolumeWithNodeAffinity(t, &v1.VolumeNodeAffinity{ - Required: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "test-key1", - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-value1", "test-value3"}, - }, - { - Key: "test-key2", - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-value0", "test-value2"}, - }, - }, - }, - }, - }, - }), - }, - { - name: "invalid-multiple-match-expressions-key", - expectSuccess: false, - pv: testVolumeWithNodeAffinity(t, &v1.VolumeNodeAffinity{ - Required: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "test-key1", - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-value1", "test-value3"}, - }, - { - Key: "test-key3", - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-value0", "test-value2"}, - }, - }, - }, - }, - }, - }), - }, - { - name: "invalid-multiple-match-expressions-values", - expectSuccess: false, - pv: testVolumeWithNodeAffinity(t, &v1.VolumeNodeAffinity{ - Required: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "test-key1", - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-value3", "test-value4"}, - }, - { - Key: "test-key2", - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-value0", "test-value2"}, - }, - }, - }, - }, - }, - }), - }, - { - name: "invalid-multiple-terms", - expectSuccess: false, - pv: testVolumeWithNodeAffinity(t, &v1.VolumeNodeAffinity{ - Required: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "test-key3", - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-value1", "test-value3"}, - }, - }, - }, - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "test-key2", - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-value0", "test-value1"}, - }, - }, - }, - }, - }, - }), - }, - } - - for _, c := range cases { - err := CheckNodeAffinity(c.pv, nodeLabels) - - if err != nil && c.expectSuccess { - t.Errorf("CheckTopology %v returned error: %v", c.name, err) - } - if err == nil && !c.expectSuccess { - t.Errorf("CheckTopology %v returned success, expected error", c.name) - } - } -} - -func testVolumeWithNodeAffinity(t *testing.T, affinity *v1.VolumeNodeAffinity) *v1.PersistentVolume { - objMeta := metav1.ObjectMeta{Name: "test-constraints"} - return &v1.PersistentVolume{ - ObjectMeta: objMeta, - Spec: v1.PersistentVolumeSpec{ - NodeAffinity: affinity, - }, - } -} - func TestLoadPodFromFile(t *testing.T) { tests := []struct { name string diff --git a/plugin/pkg/admission/storage/persistentvolume/label/admission.go b/plugin/pkg/admission/storage/persistentvolume/label/admission.go index 6ef598fb7a6..a7ae3ba5dfd 100644 --- a/plugin/pkg/admission/storage/persistentvolume/label/admission.go +++ b/plugin/pkg/admission/storage/persistentvolume/label/admission.go @@ -30,10 +30,10 @@ import ( cloudprovider "k8s.io/cloud-provider" cloudvolume "k8s.io/cloud-provider/volume" volumehelpers "k8s.io/cloud-provider/volume/helpers" + persistentvolume "k8s.io/component-helpers/storage/volume" "k8s.io/klog/v2" api "k8s.io/kubernetes/pkg/apis/core" k8s_api_v1 "k8s.io/kubernetes/pkg/apis/core/v1" - persistentvolume "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" ) diff --git a/plugin/pkg/admission/storage/persistentvolume/label/admission_test.go b/plugin/pkg/admission/storage/persistentvolume/label/admission_test.go index 708d1c84f79..84694431994 100644 --- a/plugin/pkg/admission/storage/persistentvolume/label/admission_test.go +++ b/plugin/pkg/admission/storage/persistentvolume/label/admission_test.go @@ -30,8 +30,8 @@ import ( "k8s.io/apiserver/pkg/admission" admissiontesting "k8s.io/apiserver/pkg/admission/testing" cloudprovider "k8s.io/cloud-provider" + persistentvolume "k8s.io/component-helpers/storage/volume" api "k8s.io/kubernetes/pkg/apis/core" - persistentvolume "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" ) type mockVolumes struct { diff --git a/staging/src/k8s.io/component-helpers/go.sum b/staging/src/k8s.io/component-helpers/go.sum index feedc13e42a..b6dccfe6d73 100644 --- a/staging/src/k8s.io/component-helpers/go.sum +++ b/staging/src/k8s.io/component-helpers/go.sum @@ -157,6 +157,7 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= diff --git a/staging/src/k8s.io/component-helpers/storage/volume/helpers.go b/staging/src/k8s.io/component-helpers/storage/volume/helpers.go index a76e9334480..2ca4201c236 100644 --- a/staging/src/k8s.io/component-helpers/storage/volume/helpers.go +++ b/staging/src/k8s.io/component-helpers/storage/volume/helpers.go @@ -16,7 +16,13 @@ limitations under the License. package volume -import v1 "k8s.io/api/core/v1" +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/component-helpers/scheduling/corev1" +) // GetPersistentVolumeClaimClass returns StorageClassName. If no storage class was // requested, it returns "". @@ -42,3 +48,26 @@ func GetPersistentVolumeClass(volume *v1.PersistentVolume) string { return volume.Spec.StorageClassName } + +// CheckNodeAffinity looks at the PV node affinity, and checks if the node has the same corresponding labels +// This ensures that we don't mount a volume that doesn't belong to this node +func CheckNodeAffinity(pv *v1.PersistentVolume, nodeLabels map[string]string) error { + return checkVolumeNodeAffinity(pv, &v1.Node{ObjectMeta: metav1.ObjectMeta{Labels: nodeLabels}}) +} + +func checkVolumeNodeAffinity(pv *v1.PersistentVolume, node *v1.Node) error { + if pv.Spec.NodeAffinity == nil { + return nil + } + + if pv.Spec.NodeAffinity.Required != nil { + terms := pv.Spec.NodeAffinity.Required + if matches, err := corev1.MatchNodeSelectorTerms(node, terms); err != nil { + return err + } else if !matches { + return fmt.Errorf("no matching NodeSelectorTerms") + } + } + + return nil +} diff --git a/staging/src/k8s.io/component-helpers/storage/volume/helpers_test.go b/staging/src/k8s.io/component-helpers/storage/volume/helpers_test.go new file mode 100644 index 00000000000..9eb56750ed7 --- /dev/null +++ b/staging/src/k8s.io/component-helpers/storage/volume/helpers_test.go @@ -0,0 +1,216 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volume + +import ( + "testing" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var nodeLabels = map[string]string{ + "test-key1": "test-value1", + "test-key2": "test-value2", +} + +func TestCheckVolumeNodeAffinity(t *testing.T) { + type affinityTest struct { + name string + expectSuccess bool + pv *v1.PersistentVolume + } + + cases := []affinityTest{ + { + name: "valid-nil", + expectSuccess: true, + pv: testVolumeWithNodeAffinity(t, nil), + }, + { + name: "valid-no-constraints", + expectSuccess: true, + pv: testVolumeWithNodeAffinity(t, &v1.VolumeNodeAffinity{}), + }, + { + name: "select-nothing", + expectSuccess: false, + pv: testVolumeWithNodeAffinity(t, &v1.VolumeNodeAffinity{Required: &v1.NodeSelector{}}), + }, + { + name: "select-nothing-empty-terms", + expectSuccess: false, + pv: testVolumeWithNodeAffinity(t, &v1.VolumeNodeAffinity{ + Required: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{}, + }, + }, + }, + }), + }, + { + name: "valid-multiple-terms", + expectSuccess: true, + pv: testVolumeWithNodeAffinity(t, &v1.VolumeNodeAffinity{ + Required: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "test-key3", + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-value1", "test-value3"}, + }, + }, + }, + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "test-key2", + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-value0", "test-value2"}, + }, + }, + }, + }, + }, + }), + }, + { + name: "valid-multiple-match-expressions", + expectSuccess: true, + pv: testVolumeWithNodeAffinity(t, &v1.VolumeNodeAffinity{ + Required: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "test-key1", + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-value1", "test-value3"}, + }, + { + Key: "test-key2", + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-value0", "test-value2"}, + }, + }, + }, + }, + }, + }), + }, + { + name: "invalid-multiple-match-expressions-key", + expectSuccess: false, + pv: testVolumeWithNodeAffinity(t, &v1.VolumeNodeAffinity{ + Required: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "test-key1", + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-value1", "test-value3"}, + }, + { + Key: "test-key3", + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-value0", "test-value2"}, + }, + }, + }, + }, + }, + }), + }, + { + name: "invalid-multiple-match-expressions-values", + expectSuccess: false, + pv: testVolumeWithNodeAffinity(t, &v1.VolumeNodeAffinity{ + Required: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "test-key1", + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-value3", "test-value4"}, + }, + { + Key: "test-key2", + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-value0", "test-value2"}, + }, + }, + }, + }, + }, + }), + }, + { + name: "invalid-multiple-terms", + expectSuccess: false, + pv: testVolumeWithNodeAffinity(t, &v1.VolumeNodeAffinity{ + Required: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "test-key3", + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-value1", "test-value3"}, + }, + }, + }, + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "test-key2", + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-value0", "test-value1"}, + }, + }, + }, + }, + }, + }), + }, + } + + for _, c := range cases { + err := CheckNodeAffinity(c.pv, nodeLabels) + + if err != nil && c.expectSuccess { + t.Errorf("CheckTopology %v returned error: %v", c.name, err) + } + if err == nil && !c.expectSuccess { + t.Errorf("CheckTopology %v returned success, expected error", c.name) + } + } +} + +func testVolumeWithNodeAffinity(t *testing.T, affinity *v1.VolumeNodeAffinity) *v1.PersistentVolume { + return &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "test-constraints"}, + Spec: v1.PersistentVolumeSpec{ + NodeAffinity: affinity, + }, + } +} diff --git a/pkg/controller/volume/persistentvolume/util/util.go b/staging/src/k8s.io/component-helpers/storage/volume/pv_helpers.go similarity index 96% rename from pkg/controller/volume/persistentvolume/util/util.go rename to staging/src/k8s.io/component-helpers/storage/volume/pv_helpers.go index 184f1b24aea..cf18af43a37 100644 --- a/pkg/controller/volume/persistentvolume/util/util.go +++ b/staging/src/k8s.io/component-helpers/storage/volume/pv_helpers.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package persistentvolume +package volume import ( "fmt" @@ -28,8 +28,6 @@ import ( "k8s.io/client-go/kubernetes/scheme" storagelisters "k8s.io/client-go/listers/storage/v1" "k8s.io/client-go/tools/reference" - storagehelpers "k8s.io/component-helpers/storage/volume" - volumeutil "k8s.io/kubernetes/pkg/volume/util" ) const ( @@ -95,7 +93,7 @@ func IsDelayBindingProvisioning(claim *v1.PersistentVolumeClaim) bool { // IsDelayBindingMode checks if claim is in delay binding mode. func IsDelayBindingMode(claim *v1.PersistentVolumeClaim, classLister storagelisters.StorageClassLister) (bool, error) { - className := storagehelpers.GetPersistentVolumeClaimClass(claim) + className := GetPersistentVolumeClaimClass(claim) if className == "" { return false, nil } @@ -194,7 +192,7 @@ func FindMatchingVolume( var smallestVolume *v1.PersistentVolume var smallestVolumeQty resource.Quantity requestedQty := claim.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] - requestedClass := storagehelpers.GetPersistentVolumeClaimClass(claim) + requestedClass := GetPersistentVolumeClaimClass(claim) var selector labels.Selector if claim.Spec.Selector != nil { @@ -238,9 +236,9 @@ func FindMatchingVolume( if node != nil { // Scheduler path, check that the PV NodeAffinity // is satisfied by the node - // volumeutil.CheckNodeAffinity is the most expensive call in this loop. + // CheckNodeAffinity is the most expensive call in this loop. // We should check cheaper conditions first or consider optimizing this function. - err := volumeutil.CheckNodeAffinity(volume, node.Labels) + err := CheckNodeAffinity(volume, node.Labels) if err != nil { nodeAffinityValid = false } @@ -278,7 +276,7 @@ func FindMatchingVolume( } else if selector != nil && !selector.Matches(labels.Set(volume.Labels)) { continue } - if storagehelpers.GetPersistentVolumeClass(volume) != requestedClass { + if GetPersistentVolumeClass(volume) != requestedClass { continue } if !nodeAffinityValid { diff --git a/staging/src/k8s.io/component-helpers/storage/volume/pv_helpers_test.go b/staging/src/k8s.io/component-helpers/storage/volume/pv_helpers_test.go new file mode 100644 index 00000000000..74badfd01ab --- /dev/null +++ b/staging/src/k8s.io/component-helpers/storage/volume/pv_helpers_test.go @@ -0,0 +1,300 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volume + +import ( + "testing" + + v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" +) + +var ( + classNotHere = "not-here" + classNoMode = "no-mode" + classImmediateMode = "immediate-mode" + classWaitMode = "wait-mode" + + modeImmediate = storagev1.VolumeBindingImmediate + modeWait = storagev1.VolumeBindingWaitForFirstConsumer +) + +func makePVCClass(scName *string) *v1.PersistentVolumeClaim { + claim := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + Spec: v1.PersistentVolumeClaimSpec{ + StorageClassName: scName, + }, + } + + return claim +} + +func makeStorageClass(scName string, mode *storagev1.VolumeBindingMode) *storagev1.StorageClass { + return &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: scName, + }, + VolumeBindingMode: mode, + } +} + +func TestDelayBindingMode(t *testing.T) { + tests := map[string]struct { + pvc *v1.PersistentVolumeClaim + shouldDelay bool + shouldFail bool + }{ + "nil-class": { + pvc: makePVCClass(nil), + shouldDelay: false, + }, + "class-not-found": { + pvc: makePVCClass(&classNotHere), + shouldDelay: false, + }, + "no-mode-class": { + pvc: makePVCClass(&classNoMode), + shouldDelay: false, + shouldFail: true, + }, + "immediate-mode-class": { + pvc: makePVCClass(&classImmediateMode), + shouldDelay: false, + }, + "wait-mode-class": { + pvc: makePVCClass(&classWaitMode), + shouldDelay: true, + }, + } + + classes := []*storagev1.StorageClass{ + makeStorageClass(classNoMode, nil), + makeStorageClass(classImmediateMode, &modeImmediate), + makeStorageClass(classWaitMode, &modeWait), + } + + client := &fake.Clientset{} + informerFactory := informers.NewSharedInformerFactory(client, 0) + classInformer := informerFactory.Storage().V1().StorageClasses() + + for _, class := range classes { + if err := classInformer.Informer().GetIndexer().Add(class); err != nil { + t.Fatalf("Failed to add storage class %q: %v", class.Name, err) + } + } + + for name, test := range tests { + shouldDelay, err := IsDelayBindingMode(test.pvc, classInformer.Lister()) + if err != nil && !test.shouldFail { + t.Errorf("Test %q returned error: %v", name, err) + } + if err == nil && test.shouldFail { + t.Errorf("Test %q returned success, expected error", name) + } + if shouldDelay != test.shouldDelay { + t.Errorf("Test %q returned unexpected %v", name, test.shouldDelay) + } + } +} + +func TestFindMatchVolumeWithNode(t *testing.T) { + volumes := []*v1.PersistentVolume{ + makeTestVolume("local-small", "local001", "5G", true, nil), + makeTestVolume("local-pd-very-large", "local002", "200E", true, func(pv *v1.PersistentVolume) { + pv.Spec.StorageClassName = "large" + }), + makeTestVolume("affinity-pv", "affinity001", "100G", true, func(pv *v1.PersistentVolume) { + pv.Spec.StorageClassName = "wait" + pv.Spec.NodeAffinity = GetVolumeNodeAffinity("key1", "value1") + }), + makeTestVolume("affinity-pv2", "affinity002", "150G", true, func(pv *v1.PersistentVolume) { + pv.Spec.StorageClassName = "wait" + pv.Spec.NodeAffinity = GetVolumeNodeAffinity("key1", "value1") + }), + makeTestVolume("affinity-prebound", "affinity003", "100G", true, func(pv *v1.PersistentVolume) { + pv.Spec.StorageClassName = "wait" + pv.Spec.ClaimRef = &v1.ObjectReference{Name: "claim02", Namespace: "myns"} + pv.Spec.NodeAffinity = GetVolumeNodeAffinity("key1", "value1") + }), + makeTestVolume("affinity-pv3", "affinity003", "200G", true, func(pv *v1.PersistentVolume) { + pv.Spec.StorageClassName = "wait" + pv.Spec.NodeAffinity = GetVolumeNodeAffinity("key1", "value3") + }), + makeTestVolume("affinity-pv4", "affinity004", "200G", false, func(pv *v1.PersistentVolume) { + pv.Spec.StorageClassName = "wait" + pv.Spec.NodeAffinity = GetVolumeNodeAffinity("key1", "value4") + }), + } + + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"key1": "value1"}, + }, + } + node2 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"key1": "value2"}, + }, + } + node3 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"key1": "value3"}, + }, + } + node4 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"key1": "value4"}, + }, + } + + scenarios := map[string]struct { + expectedMatch string + claim *v1.PersistentVolumeClaim + node *v1.Node + excludedVolumes map[string]*v1.PersistentVolume + }{ + "success-match": { + expectedMatch: "affinity-pv", + claim: makeTestPersistentVolumeClaim("claim01", "100G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}), + node: node1, + }, + "success-prebound": { + expectedMatch: "affinity-prebound", + claim: makeTestPersistentVolumeClaim("claim02", "100G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}), + node: node1, + }, + "success-exclusion": { + expectedMatch: "affinity-pv2", + claim: makeTestPersistentVolumeClaim("claim01", "100G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}), + node: node1, + excludedVolumes: map[string]*v1.PersistentVolume{"affinity001": nil}, + }, + "fail-exclusion": { + expectedMatch: "", + claim: makeTestPersistentVolumeClaim("claim01", "100G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}), + node: node1, + excludedVolumes: map[string]*v1.PersistentVolume{"affinity001": nil, "affinity002": nil}, + }, + "fail-accessmode": { + expectedMatch: "", + claim: makeTestPersistentVolumeClaim("claim01", "100G", []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}), + node: node1, + }, + "fail-nodeaffinity": { + expectedMatch: "", + claim: makeTestPersistentVolumeClaim("claim01", "100G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}), + node: node2, + }, + "fail-prebound-node-affinity": { + expectedMatch: "", + claim: makeTestPersistentVolumeClaim("claim02", "100G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}), + node: node3, + }, + "fail-nonavaliable": { + expectedMatch: "", + claim: makeTestPersistentVolumeClaim("claim04", "100G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}), + node: node4, + }, + "success-bad-and-good-node-affinity": { + expectedMatch: "affinity-pv3", + claim: makeTestPersistentVolumeClaim("claim03", "100G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}), + node: node3, + }, + } + + for name, scenario := range scenarios { + volume, err := FindMatchingVolume(scenario.claim, volumes, scenario.node, scenario.excludedVolumes, true) + if err != nil { + t.Errorf("Unexpected error matching volume by claim: %v", err) + } + if len(scenario.expectedMatch) != 0 && volume == nil { + t.Errorf("Expected match but received nil volume for scenario: %s", name) + } + if len(scenario.expectedMatch) != 0 && volume != nil && string(volume.UID) != scenario.expectedMatch { + t.Errorf("Expected %s but got volume %s in scenario %s", scenario.expectedMatch, volume.UID, name) + } + if len(scenario.expectedMatch) == 0 && volume != nil { + t.Errorf("Unexpected match for scenario: %s, matched with %s instead", name, volume.UID) + } + } +} + +func makeTestPersistentVolumeClaim(name string, size string, accessMode []v1.PersistentVolumeAccessMode) *v1.PersistentVolumeClaim { + fs := v1.PersistentVolumeFilesystem + sc := "wait" + return &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "myns", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: accessMode, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceStorage): resource.MustParse(size), + }, + }, + StorageClassName: &sc, + VolumeMode: &fs, + }, + } +} + +func makeTestVolume(uid types.UID, name string, capacity string, available bool, modfn func(*v1.PersistentVolume)) *v1.PersistentVolume { + var status v1.PersistentVolumeStatus + if available { + status = v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + } + } + + fs := v1.PersistentVolumeFilesystem + + pv := v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + UID: uid, + Name: name, + }, + Spec: v1.PersistentVolumeSpec{ + Capacity: v1.ResourceList{ + v1.ResourceName(v1.ResourceStorage): resource.MustParse(capacity), + }, + PersistentVolumeSource: v1.PersistentVolumeSource{ + Local: &v1.LocalVolumeSource{}, + }, + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + v1.ReadOnlyMany, + }, + VolumeMode: &fs, + }, + Status: status, + } + + if modfn != nil { + modfn(&pv) + } + return &pv +} diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 82872b99ea8..7b34f2dc341 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -40,8 +40,8 @@ import ( restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/events" + pvutil "k8s.io/component-helpers/storage/volume" "k8s.io/klog/v2" - pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" "k8s.io/kubernetes/pkg/scheduler" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/profile" From 4ba98a861036d891cdc986f324d1b56a2d90155c Mon Sep 17 00:00:00 2001 From: Konstantin Misyutin Date: Wed, 18 Aug 2021 15:12:42 +0800 Subject: [PATCH 2/4] cleanup: remove unnecessary import aliases --- .../volume/persistentvolume/binder_test.go | 188 +++++++++--------- .../volume/persistentvolume/delete_test.go | 38 ++-- .../volume/persistentvolume/framework_test.go | 6 +- .../volume/persistentvolume/index.go | 8 +- .../volume/persistentvolume/index_test.go | 24 +-- .../volume/persistentvolume/provision_test.go | 100 +++++----- .../volume/persistentvolume/pv_controller.go | 78 ++++---- .../persistentvolume/pv_controller_base.go | 38 ++-- .../persistentvolume/pv_controller_test.go | 170 ++++++++-------- .../volume/persistentvolume/recycle_test.go | 18 +- .../volumebinding/assume_cache_test.go | 4 +- .../framework/plugins/volumebinding/binder.go | 28 +-- .../plugins/volumebinding/binder_test.go | 28 +-- .../plugins/volumebinding/test_utils.go | 4 +- pkg/scheduler/scheduler_test.go | 4 +- .../operationexecutor/operation_generator.go | 4 +- 16 files changed, 370 insertions(+), 370 deletions(-) diff --git a/pkg/controller/volume/persistentvolume/binder_test.go b/pkg/controller/volume/persistentvolume/binder_test.go index 7a01bc8b314..7af3617ed34 100644 --- a/pkg/controller/volume/persistentvolume/binder_test.go +++ b/pkg/controller/volume/persistentvolume/binder_test.go @@ -22,7 +22,7 @@ import ( v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - pvutil "k8s.io/component-helpers/storage/volume" + "k8s.io/component-helpers/storage/volume" ) // Test single call to syncClaim and syncVolume methods. @@ -44,9 +44,9 @@ func TestSync(t *testing.T) { // syncClaim binds to a matching unbound volume. "1-1 - successful bind", newVolumeArray("volume1-1", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), - newVolumeArray("volume1-1", "1Gi", "uid1-1", "claim1-1", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume1-1", "1Gi", "uid1-1", "claim1-1", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), newClaimArray("claim1-1", "uid1-1", "1Gi", "", v1.ClaimPending, nil), - newClaimArray("claim1-1", "uid1-1", "1Gi", "volume1-1", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim1-1", "uid1-1", "1Gi", "volume1-1", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted), noevents, noerrors, testSyncClaim, }, { @@ -79,10 +79,10 @@ func TestSync(t *testing.T) { }, []*v1.PersistentVolume{ newVolume("volume1-4_1", "10Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), - newVolume("volume1-4_2", "1Gi", "uid1-4", "claim1-4", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolume("volume1-4_2", "1Gi", "uid1-4", "claim1-4", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), }, newClaimArray("claim1-4", "uid1-4", "1Gi", "", v1.ClaimPending, nil), - newClaimArray("claim1-4", "uid1-4", "1Gi", "volume1-4_2", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim1-4", "uid1-4", "1Gi", "volume1-4_2", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted), noevents, noerrors, testSyncClaim, }, { @@ -98,7 +98,7 @@ func TestSync(t *testing.T) { newVolume("volume1-5_2", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), }, newClaimArray("claim1-5", "uid1-5", "1Gi", "", v1.ClaimPending, nil), - withExpectedCapacity("10Gi", newClaimArray("claim1-5", "uid1-5", "1Gi", "volume1-5_1", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + withExpectedCapacity("10Gi", newClaimArray("claim1-5", "uid1-5", "1Gi", "volume1-5_1", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, { @@ -114,7 +114,7 @@ func TestSync(t *testing.T) { newVolume("volume1-6_2", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), }, newClaimArray("claim1-6", "uid1-6", "1Gi", "", v1.ClaimPending, nil), - withExpectedCapacity("10Gi", newClaimArray("claim1-6", "uid1-6", "1Gi", "volume1-6_1", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + withExpectedCapacity("10Gi", newClaimArray("claim1-6", "uid1-6", "1Gi", "volume1-6_1", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, { @@ -132,39 +132,39 @@ func TestSync(t *testing.T) { // syncClaim completes binding - simulates controller crash after // PV.ClaimRef is saved "1-8 - complete bind after crash - PV bound", - newVolumeArray("volume1-8", "1Gi", "uid1-8", "claim1-8", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), - newVolumeArray("volume1-8", "1Gi", "uid1-8", "claim1-8", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume1-8", "1Gi", "uid1-8", "claim1-8", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), + newVolumeArray("volume1-8", "1Gi", "uid1-8", "claim1-8", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), newClaimArray("claim1-8", "uid1-8", "1Gi", "", v1.ClaimPending, nil), - newClaimArray("claim1-8", "uid1-8", "1Gi", "volume1-8", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim1-8", "uid1-8", "1Gi", "volume1-8", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted), noevents, noerrors, testSyncClaim, }, { // syncClaim completes binding - simulates controller crash after // PV.Status is saved "1-9 - complete bind after crash - PV status saved", - newVolumeArray("volume1-9", "1Gi", "uid1-9", "claim1-9", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), - newVolumeArray("volume1-9", "1Gi", "uid1-9", "claim1-9", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume1-9", "1Gi", "uid1-9", "claim1-9", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), + newVolumeArray("volume1-9", "1Gi", "uid1-9", "claim1-9", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), newClaimArray("claim1-9", "uid1-9", "1Gi", "", v1.ClaimPending, nil), - newClaimArray("claim1-9", "uid1-9", "1Gi", "volume1-9", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim1-9", "uid1-9", "1Gi", "volume1-9", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted), noevents, noerrors, testSyncClaim, }, { // syncClaim completes binding - simulates controller crash after // PVC.VolumeName is saved "1-10 - complete bind after crash - PVC bound", - newVolumeArray("volume1-10", "1Gi", "uid1-10", "claim1-10", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), - newVolumeArray("volume1-10", "1Gi", "uid1-10", "claim1-10", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), - newClaimArray("claim1-10", "uid1-10", "1Gi", "volume1-10", v1.ClaimPending, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), - newClaimArray("claim1-10", "uid1-10", "1Gi", "volume1-10", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newVolumeArray("volume1-10", "1Gi", "uid1-10", "claim1-10", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), + newVolumeArray("volume1-10", "1Gi", "uid1-10", "claim1-10", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), + newClaimArray("claim1-10", "uid1-10", "1Gi", "volume1-10", v1.ClaimPending, nil, volume.AnnBoundByController, volume.AnnBindCompleted), + newClaimArray("claim1-10", "uid1-10", "1Gi", "volume1-10", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted), noevents, noerrors, testSyncClaim, }, { // syncClaim binds a claim only when the label selector matches the volume "1-11 - bind when selector matches", withLabels(labels, newVolumeArray("volume1-1", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty)), - withLabels(labels, newVolumeArray("volume1-1", "1Gi", "uid1-1", "claim1-1", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController)), + withLabels(labels, newVolumeArray("volume1-1", "1Gi", "uid1-1", "claim1-1", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController)), withLabelSelector(labels, newClaimArray("claim1-1", "uid1-1", "1Gi", "", v1.ClaimPending, nil)), - withLabelSelector(labels, newClaimArray("claim1-1", "uid1-1", "1Gi", "volume1-1", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + withLabelSelector(labels, newClaimArray("claim1-1", "uid1-1", "1Gi", "volume1-1", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, { @@ -193,7 +193,7 @@ func TestSync(t *testing.T) { newVolumeArray("volume1-1", "1Gi", "", "claim1-1", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classWait), newVolumeArray("volume1-1", "1Gi", "uid1-1", "claim1-1", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classWait), newClaimArray("claim1-1", "uid1-1", "1Gi", "", v1.ClaimPending, &classWait), - newClaimArray("claim1-1", "uid1-1", "1Gi", "volume1-1", v1.ClaimBound, &classWait, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim1-1", "uid1-1", "1Gi", "volume1-1", v1.ClaimBound, &classWait, volume.AnnBoundByController, volume.AnnBindCompleted), noevents, noerrors, testSyncClaim, }, { @@ -205,11 +205,11 @@ func TestSync(t *testing.T) { newVolume("volume1-15_2", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), }, []*v1.PersistentVolume{ - newVolume("volume1-15_1", "10Gi", "uid1-15", "claim1-15", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolume("volume1-15_1", "10Gi", "uid1-15", "claim1-15", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), newVolume("volume1-15_2", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), }, newClaimArray("claim1-15", "uid1-15", "1Gi", "volume1-15_1", v1.ClaimPending, nil), - withExpectedCapacity("10Gi", newClaimArray("claim1-15", "uid1-15", "1Gi", "volume1-15_1", v1.ClaimBound, nil, pvutil.AnnBindCompleted)), + withExpectedCapacity("10Gi", newClaimArray("claim1-15", "uid1-15", "1Gi", "volume1-15_1", v1.ClaimBound, nil, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, { @@ -250,10 +250,10 @@ func TestSync(t *testing.T) { "1-18 - successful pre-bound PV to PVC provisioning", newVolumeArray("volume1-18", "1Gi", "uid1-18", "claim1-18", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classWait), newVolumeArray("volume1-18", "1Gi", "uid1-18", "claim1-18", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classWait), - claimWithAnnotation(pvutil.AnnSelectedNode, "node1", + claimWithAnnotation(volume.AnnSelectedNode, "node1", newClaimArray("claim1-18", "uid1-18", "1Gi", "", v1.ClaimPending, &classWait)), - claimWithAnnotation(pvutil.AnnSelectedNode, "node1", - newClaimArray("claim1-18", "uid1-18", "1Gi", "volume1-18", v1.ClaimBound, &classWait, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + claimWithAnnotation(volume.AnnSelectedNode, "node1", + newClaimArray("claim1-18", "uid1-18", "1Gi", "volume1-18", v1.ClaimBound, &classWait, volume.AnnBoundByController, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, @@ -281,33 +281,33 @@ func TestSync(t *testing.T) { }, { // syncClaim with claim pre-bound to a PV that exists and is - // unbound. Check it gets bound and no pvutil.AnnBoundByController is set. + // unbound. Check it gets bound and no volume.AnnBoundByController is set. "2-3 - claim prebound to unbound volume", newVolumeArray("volume2-3", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), - newVolumeArray("volume2-3", "1Gi", "uid2-3", "claim2-3", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume2-3", "1Gi", "uid2-3", "claim2-3", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), newClaimArray("claim2-3", "uid2-3", "1Gi", "volume2-3", v1.ClaimPending, nil), - newClaimArray("claim2-3", "uid2-3", "1Gi", "volume2-3", v1.ClaimBound, nil, pvutil.AnnBindCompleted), + newClaimArray("claim2-3", "uid2-3", "1Gi", "volume2-3", v1.ClaimBound, nil, volume.AnnBindCompleted), noevents, noerrors, testSyncClaim, }, { // claim with claim pre-bound to a PV that is pre-bound to the claim - // by name. Check it gets bound and no pvutil.AnnBoundByController is set. + // by name. Check it gets bound and no volume.AnnBoundByController is set. "2-4 - claim prebound to prebound volume by name", newVolumeArray("volume2-4", "1Gi", "", "claim2-4", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), newVolumeArray("volume2-4", "1Gi", "uid2-4", "claim2-4", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty), newClaimArray("claim2-4", "uid2-4", "1Gi", "volume2-4", v1.ClaimPending, nil), - newClaimArray("claim2-4", "uid2-4", "1Gi", "volume2-4", v1.ClaimBound, nil, pvutil.AnnBindCompleted), + newClaimArray("claim2-4", "uid2-4", "1Gi", "volume2-4", v1.ClaimBound, nil, volume.AnnBindCompleted), noevents, noerrors, testSyncClaim, }, { // syncClaim with claim pre-bound to a PV that is pre-bound to the - // claim by UID. Check it gets bound and no pvutil.AnnBoundByController is + // claim by UID. Check it gets bound and no volume.AnnBoundByController is // set. "2-5 - claim prebound to prebound volume by UID", newVolumeArray("volume2-5", "1Gi", "uid2-5", "claim2-5", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), newVolumeArray("volume2-5", "1Gi", "uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty), newClaimArray("claim2-5", "uid2-5", "1Gi", "volume2-5", v1.ClaimPending, nil), - newClaimArray("claim2-5", "uid2-5", "1Gi", "volume2-5", v1.ClaimBound, nil, pvutil.AnnBindCompleted), + newClaimArray("claim2-5", "uid2-5", "1Gi", "volume2-5", v1.ClaimBound, nil, volume.AnnBindCompleted), noevents, noerrors, testSyncClaim, }, { @@ -326,19 +326,19 @@ func TestSync(t *testing.T) { "2-7 - claim bound by controller to already bound volume", newVolumeArray("volume2-7", "1Gi", "uid2-7_1", "claim2-7_1", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty), newVolumeArray("volume2-7", "1Gi", "uid2-7_1", "claim2-7_1", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty), - newClaimArray("claim2-7", "uid2-7", "1Gi", "volume2-7", v1.ClaimBound, nil, pvutil.AnnBoundByController), - newClaimArray("claim2-7", "uid2-7", "1Gi", "volume2-7", v1.ClaimBound, nil, pvutil.AnnBoundByController), + newClaimArray("claim2-7", "uid2-7", "1Gi", "volume2-7", v1.ClaimBound, nil, volume.AnnBoundByController), + newClaimArray("claim2-7", "uid2-7", "1Gi", "volume2-7", v1.ClaimBound, nil, volume.AnnBoundByController), noevents, noerrors, testSyncClaimError, }, { // syncClaim with claim pre-bound to a PV that exists and is // unbound, but does not match the selector. Check it gets bound - // and no pvutil.AnnBoundByController is set. + // and no volume.AnnBoundByController is set. "2-8 - claim prebound to unbound volume that does not match the selector", newVolumeArray("volume2-8", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), - newVolumeArray("volume2-8", "1Gi", "uid2-8", "claim2-8", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume2-8", "1Gi", "uid2-8", "claim2-8", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), withLabelSelector(labels, newClaimArray("claim2-8", "uid2-8", "1Gi", "volume2-8", v1.ClaimPending, nil)), - withLabelSelector(labels, newClaimArray("claim2-8", "uid2-8", "1Gi", "volume2-8", v1.ClaimBound, nil, pvutil.AnnBindCompleted)), + withLabelSelector(labels, newClaimArray("claim2-8", "uid2-8", "1Gi", "volume2-8", v1.ClaimBound, nil, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, { @@ -370,8 +370,8 @@ func TestSync(t *testing.T) { "3-1 - bound claim with missing VolumeName", novolumes, novolumes, - newClaimArray("claim3-1", "uid3-1", "10Gi", "", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), - newClaimArray("claim3-1", "uid3-1", "10Gi", "", v1.ClaimLost, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim3-1", "uid3-1", "10Gi", "", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted), + newClaimArray("claim3-1", "uid3-1", "10Gi", "", v1.ClaimLost, nil, volume.AnnBoundByController, volume.AnnBindCompleted), []string{"Warning ClaimLost"}, noerrors, testSyncClaim, }, { @@ -380,8 +380,8 @@ func TestSync(t *testing.T) { "3-2 - bound claim with missing volume", novolumes, novolumes, - newClaimArray("claim3-2", "uid3-2", "10Gi", "volume3-2", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), - newClaimArray("claim3-2", "uid3-2", "10Gi", "volume3-2", v1.ClaimLost, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim3-2", "uid3-2", "10Gi", "volume3-2", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted), + newClaimArray("claim3-2", "uid3-2", "10Gi", "volume3-2", v1.ClaimLost, nil, volume.AnnBoundByController, volume.AnnBindCompleted), []string{"Warning ClaimLost"}, noerrors, testSyncClaim, }, { @@ -389,9 +389,9 @@ func TestSync(t *testing.T) { // Also check that Pending phase is set to Bound "3-3 - bound claim with unbound volume", newVolumeArray("volume3-3", "10Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), - newVolumeArray("volume3-3", "10Gi", "uid3-3", "claim3-3", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), - newClaimArray("claim3-3", "uid3-3", "10Gi", "volume3-3", v1.ClaimPending, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), - newClaimArray("claim3-3", "uid3-3", "10Gi", "volume3-3", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newVolumeArray("volume3-3", "10Gi", "uid3-3", "claim3-3", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), + newClaimArray("claim3-3", "uid3-3", "10Gi", "volume3-3", v1.ClaimPending, nil, volume.AnnBoundByController, volume.AnnBindCompleted), + newClaimArray("claim3-3", "uid3-3", "10Gi", "volume3-3", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted), noevents, noerrors, testSyncClaim, }, { @@ -400,8 +400,8 @@ func TestSync(t *testing.T) { "3-4 - bound claim with prebound volume", newVolumeArray("volume3-4", "10Gi", "claim3-4-x", "claim3-4", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), newVolumeArray("volume3-4", "10Gi", "claim3-4-x", "claim3-4", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), - newClaimArray("claim3-4", "uid3-4", "10Gi", "volume3-4", v1.ClaimPending, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), - newClaimArray("claim3-4", "uid3-4", "10Gi", "volume3-4", v1.ClaimLost, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim3-4", "uid3-4", "10Gi", "volume3-4", v1.ClaimPending, nil, volume.AnnBoundByController, volume.AnnBindCompleted), + newClaimArray("claim3-4", "uid3-4", "10Gi", "volume3-4", v1.ClaimLost, nil, volume.AnnBoundByController, volume.AnnBindCompleted), []string{"Warning ClaimMisbound"}, noerrors, testSyncClaim, }, { @@ -411,8 +411,8 @@ func TestSync(t *testing.T) { "3-5 - bound claim with bound volume", newVolumeArray("volume3-5", "10Gi", "uid3-5", "claim3-5", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), newVolumeArray("volume3-5", "10Gi", "uid3-5", "claim3-5", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty), - newClaimArray("claim3-5", "uid3-5", "10Gi", "volume3-5", v1.ClaimPending, nil, pvutil.AnnBindCompleted), - newClaimArray("claim3-5", "uid3-5", "10Gi", "volume3-5", v1.ClaimBound, nil, pvutil.AnnBindCompleted), + newClaimArray("claim3-5", "uid3-5", "10Gi", "volume3-5", v1.ClaimPending, nil, volume.AnnBindCompleted), + newClaimArray("claim3-5", "uid3-5", "10Gi", "volume3-5", v1.ClaimBound, nil, volume.AnnBindCompleted), noevents, noerrors, testSyncClaim, }, { @@ -422,8 +422,8 @@ func TestSync(t *testing.T) { "3-6 - bound claim with bound volume", newVolumeArray("volume3-6", "10Gi", "uid3-6-x", "claim3-6-x", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), newVolumeArray("volume3-6", "10Gi", "uid3-6-x", "claim3-6-x", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), - newClaimArray("claim3-6", "uid3-6", "10Gi", "volume3-6", v1.ClaimPending, nil, pvutil.AnnBindCompleted), - newClaimArray("claim3-6", "uid3-6", "10Gi", "volume3-6", v1.ClaimLost, nil, pvutil.AnnBindCompleted), + newClaimArray("claim3-6", "uid3-6", "10Gi", "volume3-6", v1.ClaimPending, nil, volume.AnnBindCompleted), + newClaimArray("claim3-6", "uid3-6", "10Gi", "volume3-6", v1.ClaimLost, nil, volume.AnnBindCompleted), []string{"Warning ClaimMisbound"}, noerrors, testSyncClaim, }, { @@ -432,9 +432,9 @@ func TestSync(t *testing.T) { // check that Pending phase is set to Bound "3-7 - bound claim with unbound volume where selector doesn't match", newVolumeArray("volume3-3", "10Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), - newVolumeArray("volume3-3", "10Gi", "uid3-3", "claim3-3", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), - withLabelSelector(labels, newClaimArray("claim3-3", "uid3-3", "10Gi", "volume3-3", v1.ClaimPending, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), - withLabelSelector(labels, newClaimArray("claim3-3", "uid3-3", "10Gi", "volume3-3", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + newVolumeArray("volume3-3", "10Gi", "uid3-3", "claim3-3", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), + withLabelSelector(labels, newClaimArray("claim3-3", "uid3-3", "10Gi", "volume3-3", v1.ClaimPending, nil, volume.AnnBoundByController, volume.AnnBindCompleted)), + withLabelSelector(labels, newClaimArray("claim3-3", "uid3-3", "10Gi", "volume3-3", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, // [Unit test set 4] All syncVolume tests. @@ -473,16 +473,16 @@ func TestSync(t *testing.T) { "4-4 - volume bound to claim with different UID", newVolumeArray("volume4-4", "10Gi", "uid4-4", "claim4-4", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty), newVolumeArray("volume4-4", "10Gi", "uid4-4", "claim4-4", v1.VolumeReleased, v1.PersistentVolumeReclaimRetain, classEmpty), - newClaimArray("claim4-4", "uid4-4-x", "10Gi", "volume4-4", v1.ClaimBound, nil, pvutil.AnnBindCompleted), - newClaimArray("claim4-4", "uid4-4-x", "10Gi", "volume4-4", v1.ClaimBound, nil, pvutil.AnnBindCompleted), + newClaimArray("claim4-4", "uid4-4-x", "10Gi", "volume4-4", v1.ClaimBound, nil, volume.AnnBindCompleted), + newClaimArray("claim4-4", "uid4-4-x", "10Gi", "volume4-4", v1.ClaimBound, nil, volume.AnnBindCompleted), noevents, noerrors, testSyncVolume, }, { // syncVolume with volume bound by controller to unbound claim. // Check syncVolume does not do anything. "4-5 - volume bound by controller to unbound claim", - newVolumeArray("volume4-5", "10Gi", "uid4-5", "claim4-5", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), - newVolumeArray("volume4-5", "10Gi", "uid4-5", "claim4-5", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume4-5", "10Gi", "uid4-5", "claim4-5", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), + newVolumeArray("volume4-5", "10Gi", "uid4-5", "claim4-5", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), newClaimArray("claim4-5", "uid4-5", "10Gi", "", v1.ClaimPending, nil), newClaimArray("claim4-5", "uid4-5", "10Gi", "", v1.ClaimPending, nil), noevents, noerrors, testSyncVolume, @@ -511,7 +511,7 @@ func TestSync(t *testing.T) { // syncVolume with volume bound by controller to claim bound to // another volume. Check that the volume is rolled back. "4-7 - volume bound by controller to claim bound somewhere else", - newVolumeArray("volume4-7", "10Gi", "uid4-7", "claim4-7", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume4-7", "10Gi", "uid4-7", "claim4-7", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), newVolumeArray("volume4-7", "10Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), newClaimArray("claim4-7", "uid4-7", "10Gi", "volume4-7-x", v1.ClaimBound, nil), newClaimArray("claim4-7", "uid4-7", "10Gi", "volume4-7-x", v1.ClaimBound, nil), @@ -589,10 +589,10 @@ func TestSync(t *testing.T) { }, []*v1.PersistentVolume{ newVolume("volume13-1-1", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), - newVolume("volume13-1-2", "10Gi", "uid13-1", "claim13-1", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classGold, pvutil.AnnBoundByController), + newVolume("volume13-1-2", "10Gi", "uid13-1", "claim13-1", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classGold, volume.AnnBoundByController), }, newClaimArray("claim13-1", "uid13-1", "1Gi", "", v1.ClaimPending, &classGold), - withExpectedCapacity("10Gi", newClaimArray("claim13-1", "uid13-1", "1Gi", "volume13-1-2", v1.ClaimBound, &classGold, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + withExpectedCapacity("10Gi", newClaimArray("claim13-1", "uid13-1", "1Gi", "volume13-1-2", v1.ClaimBound, &classGold, volume.AnnBoundByController, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, { @@ -605,10 +605,10 @@ func TestSync(t *testing.T) { }, []*v1.PersistentVolume{ newVolume("volume13-2-1", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classGold), - newVolume("volume13-2-2", "10Gi", "uid13-2", "claim13-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolume("volume13-2-2", "10Gi", "uid13-2", "claim13-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), }, newClaimArray("claim13-2", "uid13-2", "1Gi", "", v1.ClaimPending, nil), - withExpectedCapacity("10Gi", newClaimArray("claim13-2", "uid13-2", "1Gi", "volume13-2-2", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + withExpectedCapacity("10Gi", newClaimArray("claim13-2", "uid13-2", "1Gi", "volume13-2-2", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, { @@ -621,10 +621,10 @@ func TestSync(t *testing.T) { }, []*v1.PersistentVolume{ newVolume("volume13-3-1", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classSilver), - newVolume("volume13-3-2", "10Gi", "uid13-3", "claim13-3", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classGold, pvutil.AnnBoundByController), + newVolume("volume13-3-2", "10Gi", "uid13-3", "claim13-3", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classGold, volume.AnnBoundByController), }, newClaimArray("claim13-3", "uid13-3", "1Gi", "", v1.ClaimPending, &classGold), - withExpectedCapacity("10Gi", newClaimArray("claim13-3", "uid13-3", "1Gi", "volume13-3-2", v1.ClaimBound, &classGold, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + withExpectedCapacity("10Gi", newClaimArray("claim13-3", "uid13-3", "1Gi", "volume13-3-2", v1.ClaimBound, &classGold, volume.AnnBoundByController, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, { @@ -632,9 +632,9 @@ func TestSync(t *testing.T) { // class="" "13-4 - empty class", newVolumeArray("volume13-4", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), - newVolumeArray("volume13-4", "1Gi", "uid13-4", "claim13-4", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume13-4", "1Gi", "uid13-4", "claim13-4", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), newClaimArray("claim13-4", "uid13-4", "1Gi", "", v1.ClaimPending, &classEmpty), - newClaimArray("claim13-4", "uid13-4", "1Gi", "volume13-4", v1.ClaimBound, &classEmpty, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim13-4", "uid13-4", "1Gi", "volume13-4", v1.ClaimBound, &classEmpty, volume.AnnBoundByController, volume.AnnBindCompleted), noevents, noerrors, testSyncClaim, }, { @@ -642,9 +642,9 @@ func TestSync(t *testing.T) { // class = "" "13-5 - nil class", newVolumeArray("volume13-5", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), - newVolumeArray("volume13-5", "1Gi", "uid13-5", "claim13-5", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume13-5", "1Gi", "uid13-5", "claim13-5", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), newClaimArray("claim13-5", "uid13-5", "1Gi", "", v1.ClaimPending, nil), - newClaimArray("claim13-5", "uid13-5", "1Gi", "volume13-5", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim13-5", "uid13-5", "1Gi", "volume13-5", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted), noevents, noerrors, testSyncClaim, }, } @@ -668,18 +668,18 @@ func TestSyncBlockVolume(t *testing.T) { // syncVolume binds a requested block claim to a block volume "14-1 - binding to volumeMode block", withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-1", "10Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty)), - withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-1", "10Gi", "uid14-1", "claim14-1", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController)), + withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-1", "10Gi", "uid14-1", "claim14-1", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController)), withClaimVolumeMode(&modeBlock, newClaimArray("claim14-1", "uid14-1", "10Gi", "", v1.ClaimPending, nil)), - withClaimVolumeMode(&modeBlock, newClaimArray("claim14-1", "uid14-1", "10Gi", "volume14-1", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + withClaimVolumeMode(&modeBlock, newClaimArray("claim14-1", "uid14-1", "10Gi", "volume14-1", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, { // syncVolume binds a requested filesystem claim to a filesystem volume "14-2 - binding to volumeMode filesystem", withVolumeVolumeMode(&modeFile, newVolumeArray("volume14-2", "10Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty)), - withVolumeVolumeMode(&modeFile, newVolumeArray("volume14-2", "10Gi", "uid14-2", "claim14-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController)), + withVolumeVolumeMode(&modeFile, newVolumeArray("volume14-2", "10Gi", "uid14-2", "claim14-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController)), withClaimVolumeMode(&modeFile, newClaimArray("claim14-2", "uid14-2", "10Gi", "", v1.ClaimPending, nil)), - withClaimVolumeMode(&modeFile, newClaimArray("claim14-2", "uid14-2", "10Gi", "volume14-2", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + withClaimVolumeMode(&modeFile, newClaimArray("claim14-2", "uid14-2", "10Gi", "volume14-2", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, { @@ -738,7 +738,7 @@ func TestSyncBlockVolume(t *testing.T) { withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-7", "10Gi", "", "claim14-7", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty)), withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-7", "10Gi", "uid14-7", "claim14-7", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty)), withClaimVolumeMode(&modeBlock, newClaimArray("claim14-7", "uid14-7", "10Gi", "", v1.ClaimPending, nil)), - withClaimVolumeMode(&modeBlock, newClaimArray("claim14-7", "uid14-7", "10Gi", "volume14-7", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + withClaimVolumeMode(&modeBlock, newClaimArray("claim14-7", "uid14-7", "10Gi", "volume14-7", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, { @@ -765,9 +765,9 @@ func TestSyncBlockVolume(t *testing.T) { // syncVolume binds when pvc is prebound to pv with matching volumeModes block "14-9 - bind when pvc is prebound to pv with matching volumeModes block", withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-9", "10Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty)), - withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-9", "10Gi", "uid14-9", "claim14-9", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController)), + withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-9", "10Gi", "uid14-9", "claim14-9", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController)), withClaimVolumeMode(&modeBlock, newClaimArray("claim14-9", "uid14-9", "10Gi", "volume14-9", v1.ClaimPending, nil)), - withClaimVolumeMode(&modeBlock, newClaimArray("claim14-9", "uid14-9", "10Gi", "volume14-9", v1.ClaimBound, nil, pvutil.AnnBindCompleted)), + withClaimVolumeMode(&modeBlock, newClaimArray("claim14-9", "uid14-9", "10Gi", "volume14-9", v1.ClaimBound, nil, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, { @@ -776,16 +776,16 @@ func TestSyncBlockVolume(t *testing.T) { withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-10", "10Gi", "", "claim14-10", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty)), withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-10", "10Gi", "uid14-10", "claim14-10", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty)), withClaimVolumeMode(&modeBlock, newClaimArray("claim14-10", "uid14-10", "10Gi", "", v1.ClaimPending, nil)), - withClaimVolumeMode(&modeBlock, newClaimArray("claim14-10", "uid14-10", "10Gi", "volume14-10", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + withClaimVolumeMode(&modeBlock, newClaimArray("claim14-10", "uid14-10", "10Gi", "volume14-10", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, { // syncVolume binds when pvc is prebound to pv with matching volumeModes filesystem "14-11 - bind when pvc is prebound to pv with matching volumeModes filesystem", withVolumeVolumeMode(&modeFile, newVolumeArray("volume14-11", "10Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty)), - withVolumeVolumeMode(&modeFile, newVolumeArray("volume14-11", "10Gi", "uid14-11", "claim14-11", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController)), + withVolumeVolumeMode(&modeFile, newVolumeArray("volume14-11", "10Gi", "uid14-11", "claim14-11", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController)), withClaimVolumeMode(&modeFile, newClaimArray("claim14-11", "uid14-11", "10Gi", "volume14-11", v1.ClaimPending, nil)), - withClaimVolumeMode(&modeFile, newClaimArray("claim14-11", "uid14-11", "10Gi", "volume14-11", v1.ClaimBound, nil, pvutil.AnnBindCompleted)), + withClaimVolumeMode(&modeFile, newClaimArray("claim14-11", "uid14-11", "10Gi", "volume14-11", v1.ClaimBound, nil, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, { @@ -794,14 +794,14 @@ func TestSyncBlockVolume(t *testing.T) { withVolumeVolumeMode(&modeFile, newVolumeArray("volume14-12", "10Gi", "", "claim14-12", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty)), withVolumeVolumeMode(&modeFile, newVolumeArray("volume14-12", "10Gi", "uid14-12", "claim14-12", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty)), withClaimVolumeMode(&modeFile, newClaimArray("claim14-12", "uid14-12", "10Gi", "", v1.ClaimPending, nil)), - withClaimVolumeMode(&modeFile, newClaimArray("claim14-12", "uid14-12", "10Gi", "volume14-12", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + withClaimVolumeMode(&modeFile, newClaimArray("claim14-12", "uid14-12", "10Gi", "volume14-12", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted)), noevents, noerrors, testSyncClaim, }, { // syncVolume output warning when pv is prebound to pvc with mismatching volumeMode "14-13 - output warning when pv is prebound to pvc with different volumeModes", - withVolumeVolumeMode(&modeFile, newVolumeArray("volume14-13", "10Gi", "uid14-13", "claim14-13", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController)), - withVolumeVolumeMode(&modeFile, newVolumeArray("volume14-13", "10Gi", "uid14-13", "claim14-13", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController)), + withVolumeVolumeMode(&modeFile, newVolumeArray("volume14-13", "10Gi", "uid14-13", "claim14-13", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController)), + withVolumeVolumeMode(&modeFile, newVolumeArray("volume14-13", "10Gi", "uid14-13", "claim14-13", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController)), withClaimVolumeMode(&modeBlock, newClaimArray("claim14-13", "uid14-13", "10Gi", "", v1.ClaimPending, nil)), withClaimVolumeMode(&modeBlock, newClaimArray("claim14-13", "uid14-13", "10Gi", "", v1.ClaimPending, nil)), []string{"Warning VolumeMismatch"}, @@ -810,8 +810,8 @@ func TestSyncBlockVolume(t *testing.T) { { // syncVolume output warning when pv is prebound to pvc with mismatching volumeMode "14-13-1 - output warning when pv is prebound to pvc with different volumeModes", - withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-13-1", "10Gi", "uid14-13-1", "claim14-13-1", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController)), - withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-13-1", "10Gi", "uid14-13-1", "claim14-13-1", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController)), + withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-13-1", "10Gi", "uid14-13-1", "claim14-13-1", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController)), + withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-13-1", "10Gi", "uid14-13-1", "claim14-13-1", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController)), withClaimVolumeMode(&modeFile, newClaimArray("claim14-13-1", "uid14-13-1", "10Gi", "", v1.ClaimPending, nil)), withClaimVolumeMode(&modeFile, newClaimArray("claim14-13-1", "uid14-13-1", "10Gi", "", v1.ClaimPending, nil)), []string{"Warning VolumeMismatch"}, @@ -820,8 +820,8 @@ func TestSyncBlockVolume(t *testing.T) { { // syncVolume waits for synClaim without warning when pv is prebound to pvc with matching volumeMode block "14-14 - wait for synClaim without warning when pv is prebound to pvc with matching volumeModes block", - withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-14", "10Gi", "uid14-14", "claim14-14", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController)), - withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-14", "10Gi", "uid14-14", "claim14-14", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController)), + withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-14", "10Gi", "uid14-14", "claim14-14", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController)), + withVolumeVolumeMode(&modeBlock, newVolumeArray("volume14-14", "10Gi", "uid14-14", "claim14-14", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController)), withClaimVolumeMode(&modeBlock, newClaimArray("claim14-14", "uid14-14", "10Gi", "", v1.ClaimPending, nil)), withClaimVolumeMode(&modeBlock, newClaimArray("claim14-14", "uid14-14", "10Gi", "", v1.ClaimPending, nil)), noevents, noerrors, testSyncVolume, @@ -829,8 +829,8 @@ func TestSyncBlockVolume(t *testing.T) { { // syncVolume waits for synClaim without warning when pv is prebound to pvc with matching volumeMode file "14-14-1 - wait for synClaim without warning when pv is prebound to pvc with matching volumeModes file", - withVolumeVolumeMode(&modeFile, newVolumeArray("volume14-14-1", "10Gi", "uid14-14-1", "claim14-14-1", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController)), - withVolumeVolumeMode(&modeFile, newVolumeArray("volume14-14-1", "10Gi", "uid14-14-1", "claim14-14-1", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController)), + withVolumeVolumeMode(&modeFile, newVolumeArray("volume14-14-1", "10Gi", "uid14-14-1", "claim14-14-1", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController)), + withVolumeVolumeMode(&modeFile, newVolumeArray("volume14-14-1", "10Gi", "uid14-14-1", "claim14-14-1", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController)), withClaimVolumeMode(&modeFile, newClaimArray("claim14-14-1", "uid14-14-1", "10Gi", "", v1.ClaimPending, nil)), withClaimVolumeMode(&modeFile, newClaimArray("claim14-14-1", "uid14-14-1", "10Gi", "", v1.ClaimPending, nil)), noevents, noerrors, testSyncVolume, @@ -861,9 +861,9 @@ func TestMultiSync(t *testing.T) { // syncClaim binds to a matching unbound volume. "10-1 - successful bind", newVolumeArray("volume10-1", "1Gi", "", "", v1.VolumePending, v1.PersistentVolumeReclaimRetain, classEmpty), - newVolumeArray("volume10-1", "1Gi", "uid10-1", "claim10-1", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume10-1", "1Gi", "uid10-1", "claim10-1", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), newClaimArray("claim10-1", "uid10-1", "1Gi", "", v1.ClaimPending, nil), - newClaimArray("claim10-1", "uid10-1", "1Gi", "volume10-1", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim10-1", "uid10-1", "1Gi", "volume10-1", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted), noevents, noerrors, testSyncClaim, }, { @@ -871,15 +871,15 @@ func TestMultiSync(t *testing.T) { // wins and the second rolls back. "10-2 - bind PV race", []*v1.PersistentVolume{ - newVolume("volume10-2-1", "1Gi", "uid10-2", "claim10-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), - newVolume("volume10-2-2", "1Gi", "uid10-2", "claim10-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolume("volume10-2-1", "1Gi", "uid10-2", "claim10-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), + newVolume("volume10-2-2", "1Gi", "uid10-2", "claim10-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), }, []*v1.PersistentVolume{ - newVolume("volume10-2-1", "1Gi", "uid10-2", "claim10-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolume("volume10-2-1", "1Gi", "uid10-2", "claim10-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), newVolume("volume10-2-2", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), }, - newClaimArray("claim10-2", "uid10-2", "1Gi", "volume10-2-1", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), - newClaimArray("claim10-2", "uid10-2", "1Gi", "volume10-2-1", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim10-2", "uid10-2", "1Gi", "volume10-2-1", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted), + newClaimArray("claim10-2", "uid10-2", "1Gi", "volume10-2-1", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted), noevents, noerrors, testSyncClaim, }, } diff --git a/pkg/controller/volume/persistentvolume/delete_test.go b/pkg/controller/volume/persistentvolume/delete_test.go index e41697824f8..24d83cfefe8 100644 --- a/pkg/controller/volume/persistentvolume/delete_test.go +++ b/pkg/controller/volume/persistentvolume/delete_test.go @@ -22,7 +22,7 @@ import ( v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" - pvutil "k8s.io/component-helpers/storage/volume" + "k8s.io/component-helpers/storage/volume" pvtesting "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/testing" ) @@ -35,7 +35,7 @@ func TestDeleteSync(t *testing.T) { { // delete volume bound by controller "8-1 - successful delete", - newVolumeArray("volume8-1", "1Gi", "uid8-1", "claim8-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume8-1", "1Gi", "uid8-1", "claim8-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnBoundByController), novolumes, noclaims, noclaims, @@ -103,8 +103,8 @@ func TestDeleteSync(t *testing.T) { // starts. This simulates "volume no longer needs recycling, // skipping". "8-7 - volume is bound before deleting", - newVolumeArray("volume8-7", "1Gi", "uid8-7", "claim8-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, pvutil.AnnBoundByController), - newVolumeArray("volume8-7", "1Gi", "uid8-7", "claim8-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume8-7", "1Gi", "uid8-7", "claim8-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnBoundByController), + newVolumeArray("volume8-7", "1Gi", "uid8-7", "claim8-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnBoundByController), noclaims, newClaimArray("claim8-7", "uid8-7", "10Gi", "volume8-7", v1.ClaimBound, nil), noevents, noerrors, @@ -132,15 +132,15 @@ func TestDeleteSync(t *testing.T) { { // PV requires external deleter "8-10 - external deleter", - newVolumeArray("volume8-10", "1Gi", "uid10-1", "claim10-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, pvutil.AnnBoundByController), - newVolumeArray("volume8-10", "1Gi", "uid10-1", "claim10-1", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume8-10", "1Gi", "uid10-1", "claim10-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnBoundByController), + newVolumeArray("volume8-10", "1Gi", "uid10-1", "claim10-1", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnBoundByController), noclaims, noclaims, noevents, noerrors, func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor, test controllerTest) error { // Inject external deleter annotation - test.initialVolumes[0].Annotations[pvutil.AnnDynamicallyProvisioned] = "external.io/test" - test.expectedVolumes[0].Annotations[pvutil.AnnDynamicallyProvisioned] = "external.io/test" + test.initialVolumes[0].Annotations[volume.AnnDynamicallyProvisioned] = "external.io/test" + test.expectedVolumes[0].Annotations[volume.AnnDynamicallyProvisioned] = "external.io/test" return testSyncVolume(ctrl, reactor, test) }, }, @@ -149,11 +149,11 @@ func TestDeleteSync(t *testing.T) { // One of the PVs is deleted. "8-11 - two PVs provisioned for a single claim", []*v1.PersistentVolume{ - newVolume("volume8-11-1", "1Gi", "uid8-11", "claim8-11", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, pvutil.AnnDynamicallyProvisioned), - newVolume("volume8-11-2", "1Gi", "uid8-11", "claim8-11", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, pvutil.AnnDynamicallyProvisioned), + newVolume("volume8-11-1", "1Gi", "uid8-11", "claim8-11", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned), + newVolume("volume8-11-2", "1Gi", "uid8-11", "claim8-11", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned), }, []*v1.PersistentVolume{ - newVolume("volume8-11-2", "1Gi", "uid8-11", "claim8-11", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, pvutil.AnnDynamicallyProvisioned), + newVolume("volume8-11-2", "1Gi", "uid8-11", "claim8-11", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned), }, // the claim is bound to volume8-11-2 -> volume8-11-1 has lost the race and will be deleted newClaimArray("claim8-11", "uid8-11", "10Gi", "volume8-11-2", v1.ClaimBound, nil), @@ -169,12 +169,12 @@ func TestDeleteSync(t *testing.T) { // external provisioner. "8-12 - two PVs externally provisioned for a single claim", []*v1.PersistentVolume{ - newVolume("volume8-12-1", "1Gi", "uid8-12", "claim8-12", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, pvutil.AnnDynamicallyProvisioned), - newVolume("volume8-12-2", "1Gi", "uid8-12", "claim8-12", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, pvutil.AnnDynamicallyProvisioned), + newVolume("volume8-12-1", "1Gi", "uid8-12", "claim8-12", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned), + newVolume("volume8-12-2", "1Gi", "uid8-12", "claim8-12", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned), }, []*v1.PersistentVolume{ - newVolume("volume8-12-1", "1Gi", "uid8-12", "claim8-12", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classEmpty, pvutil.AnnDynamicallyProvisioned), - newVolume("volume8-12-2", "1Gi", "uid8-12", "claim8-12", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, pvutil.AnnDynamicallyProvisioned), + newVolume("volume8-12-1", "1Gi", "uid8-12", "claim8-12", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned), + newVolume("volume8-12-2", "1Gi", "uid8-12", "claim8-12", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned), }, // the claim is bound to volume8-12-2 -> volume8-12-1 has lost the race and will be "Released" newClaimArray("claim8-12", "uid8-12", "10Gi", "volume8-12-2", v1.ClaimBound, nil), @@ -182,8 +182,8 @@ func TestDeleteSync(t *testing.T) { noevents, noerrors, func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor, test controllerTest) error { // Inject external deleter annotation - test.initialVolumes[0].Annotations[pvutil.AnnDynamicallyProvisioned] = "external.io/test" - test.expectedVolumes[0].Annotations[pvutil.AnnDynamicallyProvisioned] = "external.io/test" + test.initialVolumes[0].Annotations[volume.AnnDynamicallyProvisioned] = "external.io/test" + test.expectedVolumes[0].Annotations[volume.AnnDynamicallyProvisioned] = "external.io/test" return testSyncVolume(ctrl, reactor, test) }, }, @@ -191,8 +191,8 @@ func TestDeleteSync(t *testing.T) { // TODO: Change the expectedVolumes to novolumes after HonorPVReclaimPolicy is enabled by default. // delete success - volume has deletion timestamp before doDelete() starts "8-13 - volume has deletion timestamp and processed", - volumesWithFinalizers(withVolumeDeletionTimestamp(newVolumeArray("volume8-13", "1Gi", "uid8-13", "claim8-13", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, pvutil.AnnBoundByController)), []string{pvutil.PVDeletionInTreeProtectionFinalizer}), - volumesWithFinalizers(withVolumeDeletionTimestamp(newVolumeArray("volume8-13", "1Gi", "uid8-13", "claim8-13", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classEmpty, pvutil.AnnBoundByController)), []string{pvutil.PVDeletionInTreeProtectionFinalizer}), + volumesWithFinalizers(withVolumeDeletionTimestamp(newVolumeArray("volume8-13", "1Gi", "uid8-13", "claim8-13", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnBoundByController)), []string{volume.PVDeletionInTreeProtectionFinalizer}), + volumesWithFinalizers(withVolumeDeletionTimestamp(newVolumeArray("volume8-13", "1Gi", "uid8-13", "claim8-13", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnBoundByController)), []string{volume.PVDeletionInTreeProtectionFinalizer}), noclaims, noclaims, noevents, noerrors, diff --git a/pkg/controller/volume/persistentvolume/framework_test.go b/pkg/controller/volume/persistentvolume/framework_test.go index 8de81480108..c566510ab01 100644 --- a/pkg/controller/volume/persistentvolume/framework_test.go +++ b/pkg/controller/volume/persistentvolume/framework_test.go @@ -41,7 +41,7 @@ import ( storagelisters "k8s.io/client-go/listers/storage/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" - pvutil "k8s.io/component-helpers/storage/volume" + storagehelpers "k8s.io/component-helpers/storage/volume" "k8s.io/kubernetes/pkg/controller" pvtesting "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/testing" "k8s.io/kubernetes/pkg/volume" @@ -286,7 +286,7 @@ func newVolume(name, capacity, boundToClaimUID, boundToClaimName string, phase v volume.Annotations = make(map[string]string) for _, a := range annotations { switch a { - case pvutil.AnnDynamicallyProvisioned: + case storagehelpers.AnnDynamicallyProvisioned: volume.Annotations[a] = mockPluginName default: volume.Annotations[a] = "yes" @@ -409,7 +409,7 @@ func newClaim(name, claimUID, capacity, boundToVolume string, phase v1.Persisten claim.Annotations = make(map[string]string) for _, a := range annotations { switch a { - case pvutil.AnnBetaStorageProvisioner, pvutil.AnnStorageProvisioner: + case storagehelpers.AnnBetaStorageProvisioner, storagehelpers.AnnStorageProvisioner: claim.Annotations[a] = mockPluginName default: claim.Annotations[a] = "yes" diff --git a/pkg/controller/volume/persistentvolume/index.go b/pkg/controller/volume/persistentvolume/index.go index ee4e8c3e37b..2c5dc9e593f 100644 --- a/pkg/controller/volume/persistentvolume/index.go +++ b/pkg/controller/volume/persistentvolume/index.go @@ -22,9 +22,9 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" - pvutil "k8s.io/component-helpers/storage/volume" + "k8s.io/component-helpers/storage/volume" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" - volumeutil "k8s.io/kubernetes/pkg/volume/util" + "k8s.io/kubernetes/pkg/volume/util" ) // persistentVolumeOrderedIndex is a cache.Store that keeps persistent volumes @@ -92,7 +92,7 @@ func (pvIndex *persistentVolumeOrderedIndex) findByClaim(claim *v1.PersistentVol return nil, err } - bestVol, err := pvutil.FindMatchingVolume(claim, volumes, nil /* node for topology binding*/, nil /* exclusion map */, delayBinding) + bestVol, err := volume.FindMatchingVolume(claim, volumes, nil /* node for topology binding*/, nil /* exclusion map */, delayBinding) if err != nil { return nil, err } @@ -152,7 +152,7 @@ func (pvIndex *persistentVolumeOrderedIndex) allPossibleMatchingAccessModes(requ keys := pvIndex.store.ListIndexFuncValues("accessmodes") for _, key := range keys { indexedModes := v1helper.GetAccessModesFromString(key) - if volumeutil.ContainsAllAccessModes(indexedModes, requestedModes) { + if util.ContainsAllAccessModes(indexedModes, requestedModes) { matchedModes = append(matchedModes, indexedModes) } } diff --git a/pkg/controller/volume/persistentvolume/index_test.go b/pkg/controller/volume/persistentvolume/index_test.go index 5f913397a88..2d5bb791c20 100644 --- a/pkg/controller/volume/persistentvolume/index_test.go +++ b/pkg/controller/volume/persistentvolume/index_test.go @@ -25,7 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" ref "k8s.io/client-go/tools/reference" - pvutil "k8s.io/component-helpers/storage/volume" + "k8s.io/component-helpers/storage/volume" "k8s.io/kubernetes/pkg/volume/util" ) @@ -770,7 +770,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadOnlyMany, }, StorageClassName: classWait, - NodeAffinity: pvutil.GetVolumeNodeAffinity("key1", "value1"), + NodeAffinity: volume.GetVolumeNodeAffinity("key1", "value1"), VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ @@ -794,7 +794,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadOnlyMany, }, StorageClassName: classWait, - NodeAffinity: pvutil.GetVolumeNodeAffinity("key1", "value1"), + NodeAffinity: volume.GetVolumeNodeAffinity("key1", "value1"), VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ @@ -819,7 +819,7 @@ func createTestVolumes() []*v1.PersistentVolume { }, StorageClassName: classWait, ClaimRef: &v1.ObjectReference{Name: "claim02", Namespace: "myns"}, - NodeAffinity: pvutil.GetVolumeNodeAffinity("key1", "value1"), + NodeAffinity: volume.GetVolumeNodeAffinity("key1", "value1"), VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ @@ -843,7 +843,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadOnlyMany, }, StorageClassName: classWait, - NodeAffinity: pvutil.GetVolumeNodeAffinity("key1", "value3"), + NodeAffinity: volume.GetVolumeNodeAffinity("key1", "value3"), VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ @@ -867,7 +867,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadOnlyMany, }, StorageClassName: classWait, - NodeAffinity: pvutil.GetVolumeNodeAffinity("key1", "value4"), + NodeAffinity: volume.GetVolumeNodeAffinity("key1", "value4"), VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ @@ -891,7 +891,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadOnlyMany, }, StorageClassName: classWait, - NodeAffinity: pvutil.GetVolumeNodeAffinity("key1", "value4"), + NodeAffinity: volume.GetVolumeNodeAffinity("key1", "value4"), VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ @@ -915,7 +915,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadOnlyMany, }, StorageClassName: classWait, - NodeAffinity: pvutil.GetVolumeNodeAffinity("key1", "value4"), + NodeAffinity: volume.GetVolumeNodeAffinity("key1", "value4"), VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ @@ -939,7 +939,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadOnlyMany, }, StorageClassName: classWait, - NodeAffinity: pvutil.GetVolumeNodeAffinity("key1", "value4"), + NodeAffinity: volume.GetVolumeNodeAffinity("key1", "value4"), VolumeMode: &fs, }, }, @@ -1107,7 +1107,7 @@ func TestVolumeModeCheck(t *testing.T) { for name, scenario := range scenarios { t.Run(name, func(t *testing.T) { - expectedMismatch := pvutil.CheckVolumeModeMismatches(&scenario.pvc.Spec, &scenario.vol.Spec) + expectedMismatch := volume.CheckVolumeModeMismatches(&scenario.pvc.Spec, &scenario.vol.Spec) // expected to match but either got an error or no returned pvmatch if expectedMismatch && !scenario.isExpectedMismatch { t.Errorf("Unexpected failure for scenario, expected not to mismatch on modes but did: %s", name) @@ -1378,7 +1378,7 @@ func TestBestMatchDelayed(t *testing.T) { } func TestCheckAccessModes(t *testing.T) { - volume := &v1.PersistentVolume{ + pv := &v1.PersistentVolume{ Spec: v1.PersistentVolumeSpec{ AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce, v1.ReadWriteMany}, }, @@ -1415,7 +1415,7 @@ func TestCheckAccessModes(t *testing.T) { } for name, scenario := range scenarios { - result := pvutil.CheckAccessModes(scenario.claim, volume) + result := volume.CheckAccessModes(scenario.claim, pv) if result != scenario.shouldSucceed { t.Errorf("Test %q failed: Expected %v, got %v", name, scenario.shouldSucceed, result) } diff --git a/pkg/controller/volume/persistentvolume/provision_test.go b/pkg/controller/volume/persistentvolume/provision_test.go index 7801bb79c82..e10d631b68c 100644 --- a/pkg/controller/volume/persistentvolume/provision_test.go +++ b/pkg/controller/volume/persistentvolume/provision_test.go @@ -27,7 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" - pvutil "k8s.io/component-helpers/storage/volume" + "k8s.io/component-helpers/storage/volume" api "k8s.io/kubernetes/pkg/apis/core" pvtesting "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/testing" ) @@ -172,10 +172,10 @@ func TestProvisionSync(t *testing.T) { // Provision a volume (with a default class) "11-1 - successful provision with storage class 1", novolumes, - newVolumeArray("pvc-uid11-1", "1Gi", "uid11-1", "claim11-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classGold, pvutil.AnnBoundByController, pvutil.AnnDynamicallyProvisioned), + newVolumeArray("pvc-uid11-1", "1Gi", "uid11-1", "claim11-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classGold, volume.AnnBoundByController, volume.AnnDynamicallyProvisioned), newClaimArray("claim11-1", "uid11-1", "1Gi", "", v1.ClaimPending, &classGold), // Binding will be completed in the next syncClaim - newClaimArray("claim11-1", "uid11-1", "1Gi", "", v1.ClaimPending, &classGold, pvutil.AnnStorageProvisioner, pvutil.AnnBetaStorageProvisioner), + newClaimArray("claim11-1", "uid11-1", "1Gi", "", v1.ClaimPending, &classGold, volume.AnnStorageProvisioner, volume.AnnBetaStorageProvisioner), []string{"Normal ProvisioningSucceeded"}, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision1Success}, testSyncClaim), }, { @@ -194,7 +194,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-3", "uid11-3", "1Gi", "", v1.ClaimPending, &classGold), - newClaimArray("claim11-3", "uid11-3", "1Gi", "", v1.ClaimPending, &classGold, pvutil.AnnStorageProvisioner, pvutil.AnnBetaStorageProvisioner), + newClaimArray("claim11-3", "uid11-3", "1Gi", "", v1.ClaimPending, &classGold, volume.AnnStorageProvisioner, volume.AnnBetaStorageProvisioner), []string{"Warning ProvisioningFailed"}, noerrors, wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), }, @@ -204,7 +204,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-4", "uid11-4", "1Gi", "", v1.ClaimPending, &classGold), - newClaimArray("claim11-4", "uid11-4", "1Gi", "", v1.ClaimPending, &classGold, pvutil.AnnStorageProvisioner, pvutil.AnnBetaStorageProvisioner), + newClaimArray("claim11-4", "uid11-4", "1Gi", "", v1.ClaimPending, &classGold, volume.AnnStorageProvisioner, volume.AnnBetaStorageProvisioner), []string{"Warning ProvisioningFailed"}, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision1Error}, testSyncClaim), }, @@ -212,9 +212,9 @@ func TestProvisionSync(t *testing.T) { // No provisioning if there is a matching volume available "11-6 - provisioning when there is a volume available", newVolumeArray("volume11-6", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classGold), - newVolumeArray("volume11-6", "1Gi", "uid11-6", "claim11-6", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classGold, pvutil.AnnBoundByController), + newVolumeArray("volume11-6", "1Gi", "uid11-6", "claim11-6", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classGold, volume.AnnBoundByController), newClaimArray("claim11-6", "uid11-6", "1Gi", "", v1.ClaimPending, &classGold), - newClaimArray("claim11-6", "uid11-6", "1Gi", "volume11-6", v1.ClaimBound, &classGold, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim11-6", "uid11-6", "1Gi", "volume11-6", v1.ClaimBound, &classGold, volume.AnnBoundByController, volume.AnnBindCompleted), noevents, noerrors, // No provisioning plugin confingure - makes the test fail when // the controller erroneously tries to provision something @@ -225,15 +225,15 @@ func TestProvisionSync(t *testing.T) { // a volume. "11-7 - claim is bound before provisioning", novolumes, - newVolumeArray("pvc-uid11-7", "1Gi", "uid11-7", "claim11-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classGold, pvutil.AnnBoundByController, pvutil.AnnDynamicallyProvisioned), + newVolumeArray("pvc-uid11-7", "1Gi", "uid11-7", "claim11-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classGold, volume.AnnBoundByController, volume.AnnDynamicallyProvisioned), newClaimArray("claim11-7", "uid11-7", "1Gi", "", v1.ClaimPending, &classGold), // The claim would be bound in next syncClaim - newClaimArray("claim11-7", "uid11-7", "1Gi", "", v1.ClaimPending, &classGold, pvutil.AnnStorageProvisioner, pvutil.AnnBetaStorageProvisioner), + newClaimArray("claim11-7", "uid11-7", "1Gi", "", v1.ClaimPending, &classGold, volume.AnnStorageProvisioner, volume.AnnBetaStorageProvisioner), noevents, noerrors, wrapTestWithInjectedOperation(wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor) { // Create a volume before provisionClaimOperation starts. // This similates a parallel controller provisioning the volume. - volume := newVolume("pvc-uid11-7", "1Gi", "uid11-7", "claim11-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classGold, pvutil.AnnBoundByController, pvutil.AnnDynamicallyProvisioned) + volume := newVolume("pvc-uid11-7", "1Gi", "uid11-7", "claim11-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classGold, volume.AnnBoundByController, volume.AnnDynamicallyProvisioned) reactor.AddVolume(volume) }), }, @@ -242,10 +242,10 @@ func TestProvisionSync(t *testing.T) { // second retry succeeds "11-8 - cannot save provisioned volume", novolumes, - newVolumeArray("pvc-uid11-8", "1Gi", "uid11-8", "claim11-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classGold, pvutil.AnnBoundByController, pvutil.AnnDynamicallyProvisioned), + newVolumeArray("pvc-uid11-8", "1Gi", "uid11-8", "claim11-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classGold, volume.AnnBoundByController, volume.AnnDynamicallyProvisioned), newClaimArray("claim11-8", "uid11-8", "1Gi", "", v1.ClaimPending, &classGold), // Binding will be completed in the next syncClaim - newClaimArray("claim11-8", "uid11-8", "1Gi", "", v1.ClaimPending, &classGold, pvutil.AnnStorageProvisioner, pvutil.AnnBetaStorageProvisioner), + newClaimArray("claim11-8", "uid11-8", "1Gi", "", v1.ClaimPending, &classGold, volume.AnnStorageProvisioner, volume.AnnBetaStorageProvisioner), []string{"Normal ProvisioningSucceeded"}, []pvtesting.ReactorError{ // Inject error to the first @@ -262,7 +262,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-9", "uid11-9", "1Gi", "", v1.ClaimPending, &classGold), - newClaimArray("claim11-9", "uid11-9", "1Gi", "", v1.ClaimPending, &classGold, pvutil.AnnStorageProvisioner, pvutil.AnnBetaStorageProvisioner), + newClaimArray("claim11-9", "uid11-9", "1Gi", "", v1.ClaimPending, &classGold, volume.AnnStorageProvisioner, volume.AnnBetaStorageProvisioner), []string{"Warning ProvisioningFailed"}, []pvtesting.ReactorError{ // Inject error to five kubeclient.PersistentVolumes.Create() @@ -287,7 +287,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-10", "uid11-10", "1Gi", "", v1.ClaimPending, &classGold), - newClaimArray("claim11-10", "uid11-10", "1Gi", "", v1.ClaimPending, &classGold, pvutil.AnnStorageProvisioner, pvutil.AnnBetaStorageProvisioner), + newClaimArray("claim11-10", "uid11-10", "1Gi", "", v1.ClaimPending, &classGold, volume.AnnStorageProvisioner, volume.AnnBetaStorageProvisioner), []string{"Warning ProvisioningFailed", "Warning ProvisioningCleanupFailed"}, []pvtesting.ReactorError{ // Inject error to five kubeclient.PersistentVolumes.Create() @@ -308,7 +308,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-11", "uid11-11", "1Gi", "", v1.ClaimPending, &classGold), - newClaimArray("claim11-11", "uid11-11", "1Gi", "", v1.ClaimPending, &classGold, pvutil.AnnStorageProvisioner, pvutil.AnnBetaStorageProvisioner), + newClaimArray("claim11-11", "uid11-11", "1Gi", "", v1.ClaimPending, &classGold, volume.AnnStorageProvisioner, volume.AnnBetaStorageProvisioner), []string{"Warning ProvisioningFailed", "Warning ProvisioningCleanupFailed"}, []pvtesting.ReactorError{ // Inject error to five kubeclient.PersistentVolumes.Create() @@ -338,7 +338,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-12", "uid11-12", "1Gi", "", v1.ClaimPending, &classGold), - newClaimArray("claim11-12", "uid11-12", "1Gi", "", v1.ClaimPending, &classGold, pvutil.AnnStorageProvisioner, pvutil.AnnBetaStorageProvisioner), + newClaimArray("claim11-12", "uid11-12", "1Gi", "", v1.ClaimPending, &classGold, volume.AnnStorageProvisioner, volume.AnnBetaStorageProvisioner), []string{"Warning ProvisioningFailed"}, []pvtesting.ReactorError{ // Inject error to five kubeclient.PersistentVolumes.Create() @@ -363,10 +363,10 @@ func TestProvisionSync(t *testing.T) { // Provision a volume (with non-default class) "11-13 - successful provision with storage class 2", novolumes, - newVolumeArray("pvc-uid11-13", "1Gi", "uid11-13", "claim11-13", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classSilver, pvutil.AnnBoundByController, pvutil.AnnDynamicallyProvisioned), + newVolumeArray("pvc-uid11-13", "1Gi", "uid11-13", "claim11-13", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classSilver, volume.AnnBoundByController, volume.AnnDynamicallyProvisioned), newClaimArray("claim11-13", "uid11-13", "1Gi", "", v1.ClaimPending, &classSilver), // Binding will be completed in the next syncClaim - newClaimArray("claim11-13", "uid11-13", "1Gi", "", v1.ClaimPending, &classSilver, pvutil.AnnStorageProvisioner, pvutil.AnnBetaStorageProvisioner), + newClaimArray("claim11-13", "uid11-13", "1Gi", "", v1.ClaimPending, &classSilver, volume.AnnStorageProvisioner, volume.AnnBetaStorageProvisioner), []string{"Normal ProvisioningSucceeded"}, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision2Success}, testSyncClaim), }, { @@ -402,8 +402,8 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-17", "uid11-17", "1Gi", "", v1.ClaimPending, &classExternal), - claimWithAnnotation(pvutil.AnnBetaStorageProvisioner, "vendor.com/my-volume", - claimWithAnnotation(pvutil.AnnStorageProvisioner, "vendor.com/my-volume", + claimWithAnnotation(volume.AnnBetaStorageProvisioner, "vendor.com/my-volume", + claimWithAnnotation(volume.AnnStorageProvisioner, "vendor.com/my-volume", newClaimArray("claim11-17", "uid11-17", "1Gi", "", v1.ClaimPending, &classExternal))), []string{"Normal ExternalProvisioning"}, noerrors, wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), @@ -433,7 +433,7 @@ func TestProvisionSync(t *testing.T) { // end of the test is empty. novolumes, newClaimArray("claim11-19", "uid11-19", "1Gi", "", v1.ClaimPending, &classGold), - newClaimArray("claim11-19", "uid11-19", "1Gi", "", v1.ClaimPending, &classGold, pvutil.AnnStorageProvisioner, pvutil.AnnBetaStorageProvisioner), + newClaimArray("claim11-19", "uid11-19", "1Gi", "", v1.ClaimPending, &classGold, volume.AnnStorageProvisioner, volume.AnnBetaStorageProvisioner), noevents, []pvtesting.ReactorError{ // Inject errors to simulate crashed API server during @@ -454,7 +454,7 @@ func TestProvisionSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim11-20", "uid11-20", "1Gi", "", v1.ClaimPending, &classUnsupportedMountOptions), - newClaimArray("claim11-20", "uid11-20", "1Gi", "", v1.ClaimPending, &classUnsupportedMountOptions, pvutil.AnnStorageProvisioner, pvutil.AnnBetaStorageProvisioner), + newClaimArray("claim11-20", "uid11-20", "1Gi", "", v1.ClaimPending, &classUnsupportedMountOptions, volume.AnnStorageProvisioner, volume.AnnBetaStorageProvisioner), // Expect event to be prefixed with "Mount options" because saving PV will fail anyway []string{"Warning ProvisioningFailed Mount options"}, noerrors, wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), @@ -469,9 +469,9 @@ func TestProvisionSync(t *testing.T) { annotateClaim( newClaim("claim11-21", "uid11-21", "1Gi", "", v1.ClaimPending, &classGold), map[string]string{ - pvutil.AnnStorageProvisioner: "vendor.com/MockCSIDriver", - pvutil.AnnBetaStorageProvisioner: "vendor.com/MockCSIDriver", - pvutil.AnnMigratedTo: "vendor.com/MockCSIDriver", + volume.AnnStorageProvisioner: "vendor.com/MockCSIDriver", + volume.AnnBetaStorageProvisioner: "vendor.com/MockCSIDriver", + volume.AnnMigratedTo: "vendor.com/MockCSIDriver", }), }, []string{"Normal ExternalProvisioning"}, @@ -482,9 +482,9 @@ func TestProvisionSync(t *testing.T) { // in this case, NO normal event with external provisioner should be issued "11-22 - external provisioner with volume available", newVolumeArray("volume11-22", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classExternal), - newVolumeArray("volume11-22", "1Gi", "uid11-22", "claim11-22", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classExternal, pvutil.AnnBoundByController), + newVolumeArray("volume11-22", "1Gi", "uid11-22", "claim11-22", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classExternal, volume.AnnBoundByController), newClaimArray("claim11-22", "uid11-22", "1Gi", "", v1.ClaimPending, &classExternal), - newClaimArray("claim11-22", "uid11-22", "1Gi", "volume11-22", v1.ClaimBound, &classExternal, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim11-22", "uid11-22", "1Gi", "volume11-22", v1.ClaimBound, &classExternal, volume.AnnBoundByController, volume.AnnBindCompleted), noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), }, @@ -494,12 +494,12 @@ func TestProvisionSync(t *testing.T) { newVolumeArray("volume11-23", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimDelete, classCopper), []*v1.PersistentVolume{ newVolume("volume11-23", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimDelete, classCopper), - newVolume("pvc-uid11-23", "1Gi", "uid11-23", "claim11-23", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classCopper, pvutil.AnnDynamicallyProvisioned, pvutil.AnnBoundByController), + newVolume("pvc-uid11-23", "1Gi", "uid11-23", "claim11-23", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classCopper, volume.AnnDynamicallyProvisioned, volume.AnnBoundByController), }, - claimWithAnnotation(pvutil.AnnSelectedNode, "node1", + claimWithAnnotation(volume.AnnSelectedNode, "node1", newClaimArray("claim11-23", "uid11-23", "1Gi", "", v1.ClaimPending, &classCopper)), - claimWithAnnotation(pvutil.AnnSelectedNode, "node1", - newClaimArray("claim11-23", "uid11-23", "1Gi", "", v1.ClaimPending, &classCopper, pvutil.AnnStorageProvisioner, pvutil.AnnBetaStorageProvisioner)), + claimWithAnnotation(volume.AnnSelectedNode, "node1", + newClaimArray("claim11-23", "uid11-23", "1Gi", "", v1.ClaimPending, &classCopper, volume.AnnStorageProvisioner, volume.AnnBetaStorageProvisioner)), []string{"Normal ProvisioningSucceeded"}, noerrors, wrapTestWithInjectedOperation(wrapTestWithProvisionCalls([]provisionCall{provision1Success}, testSyncClaim), @@ -515,11 +515,11 @@ func TestProvisionSync(t *testing.T) { "11-24 - skip finding PV and wait external provisioner for PVC annotated with AnnSelectedNode", newVolumeArray("volume11-24", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimDelete, classExternalWait), newVolumeArray("volume11-24", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimDelete, classExternalWait), - claimWithAnnotation(pvutil.AnnSelectedNode, "node1", + claimWithAnnotation(volume.AnnSelectedNode, "node1", newClaimArray("claim11-24", "uid11-24", "1Gi", "", v1.ClaimPending, &classExternalWait)), - claimWithAnnotation(pvutil.AnnBetaStorageProvisioner, "vendor.com/my-volume-wait", - claimWithAnnotation(pvutil.AnnStorageProvisioner, "vendor.com/my-volume-wait", - claimWithAnnotation(pvutil.AnnSelectedNode, "node1", + claimWithAnnotation(volume.AnnBetaStorageProvisioner, "vendor.com/my-volume-wait", + claimWithAnnotation(volume.AnnStorageProvisioner, "vendor.com/my-volume-wait", + claimWithAnnotation(volume.AnnSelectedNode, "node1", newClaimArray("claim11-24", "uid11-24", "1Gi", "", v1.ClaimPending, &classExternalWait)))), []string{"Normal ExternalProvisioning"}, noerrors, testSyncClaim, @@ -541,9 +541,9 @@ func TestProvisionSync(t *testing.T) { "11-26 - csi with data source", novolumes, novolumes, - claimWithAnnotation(pvutil.AnnStorageProvisioner, "mydriver.csi.k8s.io", + claimWithAnnotation(volume.AnnStorageProvisioner, "mydriver.csi.k8s.io", claimWithDataSource("test-snap", "VolumeSnapshot", "snapshot.storage.k8s.io", newClaimArray("claim11-26", "uid11-26", "1Gi", "", v1.ClaimPending, &classCSI))), - claimWithAnnotation(pvutil.AnnStorageProvisioner, "mydriver.csi.k8s.io", + claimWithAnnotation(volume.AnnStorageProvisioner, "mydriver.csi.k8s.io", claimWithDataSource("test-snap", "VolumeSnapshot", "snapshot.storage.k8s.io", newClaimArray("claim11-26", "uid11-26", "1Gi", "", v1.ClaimPending, &classCSI))), []string{"Normal ExternalProvisioning"}, noerrors, wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), @@ -572,20 +572,20 @@ func TestProvisionMultiSync(t *testing.T) { // Provision a volume with binding "12-1 - successful provision", novolumes, - newVolumeArray("pvc-uid12-1", "1Gi", "uid12-1", "claim12-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classGold, pvutil.AnnBoundByController, pvutil.AnnDynamicallyProvisioned), + newVolumeArray("pvc-uid12-1", "1Gi", "uid12-1", "claim12-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classGold, volume.AnnBoundByController, volume.AnnDynamicallyProvisioned), newClaimArray("claim12-1", "uid12-1", "1Gi", "", v1.ClaimPending, &classGold), - newClaimArray("claim12-1", "uid12-1", "1Gi", "pvc-uid12-1", v1.ClaimBound, &classGold, pvutil.AnnBoundByController, pvutil.AnnBindCompleted, pvutil.AnnStorageProvisioner, pvutil.AnnBetaStorageProvisioner), + newClaimArray("claim12-1", "uid12-1", "1Gi", "pvc-uid12-1", v1.ClaimBound, &classGold, volume.AnnBoundByController, volume.AnnBindCompleted, volume.AnnStorageProvisioner, volume.AnnBetaStorageProvisioner), noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision1Success}, testSyncClaim), }, { // provision a volume (external provisioner) and binding + normal event with external provisioner "12-2 - external provisioner with volume provisioned success", novolumes, - newVolumeArray("pvc-uid12-2", "1Gi", "uid12-2", "claim12-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classExternal, pvutil.AnnBoundByController), + newVolumeArray("pvc-uid12-2", "1Gi", "uid12-2", "claim12-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classExternal, volume.AnnBoundByController), newClaimArray("claim12-2", "uid12-2", "1Gi", "", v1.ClaimPending, &classExternal), - claimWithAnnotation(pvutil.AnnBetaStorageProvisioner, "vendor.com/my-volume", - claimWithAnnotation(pvutil.AnnStorageProvisioner, "vendor.com/my-volume", - newClaimArray("claim12-2", "uid12-2", "1Gi", "pvc-uid12-2", v1.ClaimBound, &classExternal, pvutil.AnnBoundByController, pvutil.AnnBindCompleted))), + claimWithAnnotation(volume.AnnBetaStorageProvisioner, "vendor.com/my-volume", + claimWithAnnotation(volume.AnnStorageProvisioner, "vendor.com/my-volume", + newClaimArray("claim12-2", "uid12-2", "1Gi", "pvc-uid12-2", v1.ClaimBound, &classExternal, volume.AnnBoundByController, volume.AnnBindCompleted))), []string{"Normal ExternalProvisioning"}, noerrors, wrapTestWithInjectedOperation(wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor) { @@ -609,8 +609,8 @@ func TestProvisionMultiSync(t *testing.T) { novolumes, novolumes, newClaimArray("claim12-3", "uid12-3", "1Gi", "", v1.ClaimPending, &classExternal), - claimWithAnnotation(pvutil.AnnBetaStorageProvisioner, "vendor.com/my-volume", - claimWithAnnotation(pvutil.AnnStorageProvisioner, "vendor.com/my-volume", + claimWithAnnotation(volume.AnnBetaStorageProvisioner, "vendor.com/my-volume", + claimWithAnnotation(volume.AnnStorageProvisioner, "vendor.com/my-volume", newClaimArray("claim12-3", "uid12-3", "1Gi", "", v1.ClaimPending, &classExternal))), []string{"Normal ExternalProvisioning"}, noerrors, @@ -620,11 +620,11 @@ func TestProvisionMultiSync(t *testing.T) { // provision a volume (external provisioner) and binding + normal event with external provisioner "12-4 - external provisioner with volume provisioned/bound success", novolumes, - newVolumeArray("pvc-uid12-4", "1Gi", "uid12-4", "claim12-4", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classExternal, pvutil.AnnBoundByController), + newVolumeArray("pvc-uid12-4", "1Gi", "uid12-4", "claim12-4", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classExternal, volume.AnnBoundByController), newClaimArray("claim12-4", "uid12-4", "1Gi", "", v1.ClaimPending, &classExternal), - claimWithAnnotation(pvutil.AnnBetaStorageProvisioner, "vendor.com/my-volume", - claimWithAnnotation(pvutil.AnnStorageProvisioner, "vendor.com/my-volume", - newClaimArray("claim12-4", "uid12-4", "1Gi", "pvc-uid12-4", v1.ClaimBound, &classExternal, pvutil.AnnBoundByController, pvutil.AnnBindCompleted))), + claimWithAnnotation(volume.AnnBetaStorageProvisioner, "vendor.com/my-volume", + claimWithAnnotation(volume.AnnStorageProvisioner, "vendor.com/my-volume", + newClaimArray("claim12-4", "uid12-4", "1Gi", "pvc-uid12-4", v1.ClaimBound, &classExternal, volume.AnnBoundByController, volume.AnnBindCompleted))), []string{"Normal ExternalProvisioning"}, noerrors, wrapTestWithInjectedOperation(wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor) { @@ -636,7 +636,7 @@ func TestProvisionMultiSync(t *testing.T) { // is working on provisioning the PV, also add the operation start timestamp into local cache // operationTimestamps. Rely on the existences of the start time stamp to create a PV for binding if ctrl.operationTimestamps.Has("default/claim12-4") { - volume := newVolume("pvc-uid12-4", "1Gi", "uid12-4", "claim12-4", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classExternal, pvutil.AnnBoundByController) + volume := newVolume("pvc-uid12-4", "1Gi", "uid12-4", "claim12-4", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classExternal, volume.AnnBoundByController) ctrl.volumes.store.Add(volume) // add the volume to controller reactor.AddVolume(volume) } diff --git a/pkg/controller/volume/persistentvolume/pv_controller.go b/pkg/controller/volume/persistentvolume/pv_controller.go index 6e40e38acc4..69599f8b207 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller.go +++ b/pkg/controller/volume/persistentvolume/pv_controller.go @@ -43,7 +43,7 @@ import ( "k8s.io/client-go/util/workqueue" cloudprovider "k8s.io/cloud-provider" volerr "k8s.io/cloud-provider/volume/errors" - pvutil "k8s.io/component-helpers/storage/volume" + storagehelpers "k8s.io/component-helpers/storage/volume" "k8s.io/kubernetes/pkg/controller/volume/common" "k8s.io/kubernetes/pkg/controller/volume/events" "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/metrics" @@ -263,7 +263,7 @@ func (ctrl *PersistentVolumeController) syncClaim(ctx context.Context, claim *v1 } claim = newClaim - if !metav1.HasAnnotation(claim.ObjectMeta, pvutil.AnnBindCompleted) { + if !metav1.HasAnnotation(claim.ObjectMeta, storagehelpers.AnnBindCompleted) { return ctrl.syncUnboundClaim(ctx, claim) } else { return ctrl.syncBoundClaim(claim) @@ -286,16 +286,16 @@ func checkVolumeSatisfyClaim(volume *v1.PersistentVolume, claim *v1.PersistentVo return fmt.Errorf("requested PV is too small") } - requestedClass := pvutil.GetPersistentVolumeClaimClass(claim) - if pvutil.GetPersistentVolumeClass(volume) != requestedClass { + requestedClass := storagehelpers.GetPersistentVolumeClaimClass(claim) + if storagehelpers.GetPersistentVolumeClass(volume) != requestedClass { return fmt.Errorf("storageClassName does not match") } - if pvutil.CheckVolumeModeMismatches(&claim.Spec, &volume.Spec) { + if storagehelpers.CheckVolumeModeMismatches(&claim.Spec, &volume.Spec) { return fmt.Errorf("incompatible volumeMode") } - if !pvutil.CheckAccessModes(claim, volume) { + if !storagehelpers.CheckAccessModes(claim, volume) { return fmt.Errorf("incompatible accessMode") } @@ -334,7 +334,7 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(ctx context.Context, cl // OBSERVATION: pvc is "Pending" if claim.Spec.VolumeName == "" { // User did not care which PV they get. - delayBinding, err := pvutil.IsDelayBindingMode(claim, ctrl.classLister) + delayBinding, err := storagehelpers.IsDelayBindingMode(claim, ctrl.classLister) if err != nil { return err } @@ -350,11 +350,11 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(ctx context.Context, cl // No PV could be found // OBSERVATION: pvc is "Pending", will retry switch { - case delayBinding && !pvutil.IsDelayBindingProvisioning(claim): + case delayBinding && !storagehelpers.IsDelayBindingProvisioning(claim): if err = ctrl.emitEventForUnboundDelayBindingClaim(claim); err != nil { return err } - case pvutil.GetPersistentVolumeClaimClass(claim) != "": + case storagehelpers.GetPersistentVolumeClaimClass(claim) != "": if err = ctrl.provisionClaim(ctx, claim); err != nil { return err } @@ -432,7 +432,7 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(ctx context.Context, cl } // OBSERVATION: pvc is "Bound", pv is "Bound" return nil - } else if pvutil.IsVolumeBoundToClaim(volume, claim) { + } else if storagehelpers.IsVolumeBoundToClaim(volume, claim) { // User asked for a PV that is claimed by this PVC // OBSERVATION: pvc is "Pending", pv is "Bound" klog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume already bound, finishing the binding", claimToClaimKey(claim)) @@ -446,7 +446,7 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(ctx context.Context, cl } else { // User asked for a PV that is claimed by someone else // OBSERVATION: pvc is "Pending", pv is "Bound" - if !metav1.HasAnnotation(claim.ObjectMeta, pvutil.AnnBoundByController) { + if !metav1.HasAnnotation(claim.ObjectMeta, storagehelpers.AnnBoundByController) { klog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume already bound to different claim by user, will retry later", claimToClaimKey(claim)) claimMsg := fmt.Sprintf("volume %q already bound to a different claim.", volume.Name) ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, events.FailedBinding, claimMsg) @@ -472,7 +472,7 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(ctx context.Context, cl // syncBoundClaim is the main controller method to decide what to do with a // bound claim. func (ctrl *PersistentVolumeController) syncBoundClaim(claim *v1.PersistentVolumeClaim) error { - // HasAnnotation(pvc, pvutil.AnnBindCompleted) + // HasAnnotation(pvc, storagehelpers.AnnBindCompleted) // This PVC has previously been bound // OBSERVATION: pvc is not "Pending" // [Unit test set 3] @@ -666,7 +666,7 @@ func (ctrl *PersistentVolumeController) syncVolume(ctx context.Context, volume * } return nil } else if claim.Spec.VolumeName == "" { - if pvutil.CheckVolumeModeMismatches(&claim.Spec, &volume.Spec) { + if storagehelpers.CheckVolumeModeMismatches(&claim.Spec, &volume.Spec) { // Binding for the volume won't be called in syncUnboundClaim, // because findBestMatchForClaim won't return the volume due to volumeMode mismatch. volumeMsg := fmt.Sprintf("Cannot bind PersistentVolume to requested PersistentVolumeClaim %q due to incompatible volumeMode.", claim.Name) @@ -677,7 +677,7 @@ func (ctrl *PersistentVolumeController) syncVolume(ctx context.Context, volume * return nil } - if metav1.HasAnnotation(volume.ObjectMeta, pvutil.AnnBoundByController) { + if metav1.HasAnnotation(volume.ObjectMeta, storagehelpers.AnnBoundByController) { // The binding is not completed; let PVC sync handle it klog.V(4).Infof("synchronizing PersistentVolume[%s]: volume not bound yet, waiting for syncClaim to fix it", volume.Name) } else { @@ -704,7 +704,7 @@ func (ctrl *PersistentVolumeController) syncVolume(ctx context.Context, volume * return nil } else { // Volume is bound to a claim, but the claim is bound elsewhere - if metav1.HasAnnotation(volume.ObjectMeta, pvutil.AnnDynamicallyProvisioned) && volume.Spec.PersistentVolumeReclaimPolicy == v1.PersistentVolumeReclaimDelete { + if metav1.HasAnnotation(volume.ObjectMeta, storagehelpers.AnnDynamicallyProvisioned) && volume.Spec.PersistentVolumeReclaimPolicy == v1.PersistentVolumeReclaimDelete { // This volume was dynamically provisioned for this claim. The // claim got bound elsewhere, and thus this volume is not // needed. Delete it. @@ -728,7 +728,7 @@ func (ctrl *PersistentVolumeController) syncVolume(ctx context.Context, volume * } else { // Volume is bound to a claim, but the claim is bound elsewhere // and it's not dynamically provisioned. - if metav1.HasAnnotation(volume.ObjectMeta, pvutil.AnnBoundByController) { + if metav1.HasAnnotation(volume.ObjectMeta, storagehelpers.AnnBoundByController) { // This is part of the normal operation of the controller; the // controller tried to use this volume for a claim but the claim // was fulfilled by another volume. We did this; fix it. @@ -920,7 +920,7 @@ func (ctrl *PersistentVolumeController) updateVolumePhaseWithEvent(volume *v1.Pe func (ctrl *PersistentVolumeController) bindVolumeToClaim(volume *v1.PersistentVolume, claim *v1.PersistentVolumeClaim) (*v1.PersistentVolume, error) { klog.V(4).Infof("updating PersistentVolume[%s]: binding to %q", volume.Name, claimToClaimKey(claim)) - volumeClone, dirty, err := pvutil.GetBindVolumeToClaim(volume, claim) + volumeClone, dirty, err := storagehelpers.GetBindVolumeToClaim(volume, claim) if err != nil { return nil, err } @@ -978,14 +978,14 @@ func (ctrl *PersistentVolumeController) bindClaimToVolume(claim *v1.PersistentVo claimClone.Spec.VolumeName = volume.Name // Set AnnBoundByController if it is not set yet - if !metav1.HasAnnotation(claimClone.ObjectMeta, pvutil.AnnBoundByController) { - metav1.SetMetaDataAnnotation(&claimClone.ObjectMeta, pvutil.AnnBoundByController, "yes") + if !metav1.HasAnnotation(claimClone.ObjectMeta, storagehelpers.AnnBoundByController) { + metav1.SetMetaDataAnnotation(&claimClone.ObjectMeta, storagehelpers.AnnBoundByController, "yes") } } // Set AnnBindCompleted if it is not set yet - if !metav1.HasAnnotation(claimClone.ObjectMeta, pvutil.AnnBindCompleted) { - metav1.SetMetaDataAnnotation(&claimClone.ObjectMeta, pvutil.AnnBindCompleted, "yes") + if !metav1.HasAnnotation(claimClone.ObjectMeta, storagehelpers.AnnBindCompleted) { + metav1.SetMetaDataAnnotation(&claimClone.ObjectMeta, storagehelpers.AnnBindCompleted, "yes") dirty = true } @@ -1064,10 +1064,10 @@ func (ctrl *PersistentVolumeController) unbindVolume(volume *v1.PersistentVolume // Save the PV only when any modification is necessary. volumeClone := volume.DeepCopy() - if metav1.HasAnnotation(volume.ObjectMeta, pvutil.AnnBoundByController) { + if metav1.HasAnnotation(volume.ObjectMeta, storagehelpers.AnnBoundByController) { // The volume was bound by the controller. volumeClone.Spec.ClaimRef = nil - delete(volumeClone.Annotations, pvutil.AnnBoundByController) + delete(volumeClone.Annotations, storagehelpers.AnnBoundByController) if len(volumeClone.Annotations) == 0 { // No annotations look better than empty annotation map (and it's easier // to test). @@ -1098,7 +1098,7 @@ func (ctrl *PersistentVolumeController) unbindVolume(volume *v1.PersistentVolume // reclaimVolume implements volume.Spec.PersistentVolumeReclaimPolicy and // starts appropriate reclaim action. func (ctrl *PersistentVolumeController) reclaimVolume(volume *v1.PersistentVolume) error { - if migrated := volume.Annotations[pvutil.AnnMigratedTo]; len(migrated) > 0 { + if migrated := volume.Annotations[storagehelpers.AnnMigratedTo]; len(migrated) > 0 { // PV is Migrated. The PV controller should stand down and the external // provisioner will handle this PV return nil @@ -1460,9 +1460,9 @@ func (ctrl *PersistentVolumeController) removeDeletionProtectionFinalizer(ctx co pvUpdateNeeded := false volumeClone := volume.DeepCopy() pvFinalizers := volumeClone.Finalizers - if pvFinalizers != nil && slice.ContainsString(pvFinalizers, pvutil.PVDeletionInTreeProtectionFinalizer, nil) { + if pvFinalizers != nil && slice.ContainsString(pvFinalizers, storagehelpers.PVDeletionInTreeProtectionFinalizer, nil) { pvUpdateNeeded = true - pvFinalizers = slice.RemoveString(pvFinalizers, pvutil.PVDeletionInTreeProtectionFinalizer, nil) + pvFinalizers = slice.RemoveString(pvFinalizers, storagehelpers.PVDeletionInTreeProtectionFinalizer, nil) } if pvUpdateNeeded { volumeClone.SetFinalizers(pvFinalizers) @@ -1525,7 +1525,7 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation( claim *v1.PersistentVolumeClaim, plugin vol.ProvisionableVolumePlugin, storageClass *storage.StorageClass) (string, error) { - claimClass := pvutil.GetPersistentVolumeClaimClass(claim) + claimClass := storagehelpers.GetPersistentVolumeClaimClass(claim) klog.V(4).Infof("provisionClaimOperation [%s] started, class: %q", claimToClaimKey(claim), claimClass) // called from provisionClaim(), in this case, plugin MUST NOT be nil @@ -1613,7 +1613,7 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation( } var selectedNode *v1.Node = nil - if nodeName, ok := claim.Annotations[pvutil.AnnSelectedNode]; ok { + if nodeName, ok := claim.Annotations[storagehelpers.AnnSelectedNode]; ok { selectedNode, err = ctrl.NodeLister.Get(nodeName) if err != nil { strerr := fmt.Sprintf("Failed to get target node: %v", err) @@ -1651,12 +1651,12 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation( volume.Spec.StorageClassName = claimClass // Add AnnBoundByController (used in deleting the volume) - metav1.SetMetaDataAnnotation(&volume.ObjectMeta, pvutil.AnnBoundByController, "yes") - metav1.SetMetaDataAnnotation(&volume.ObjectMeta, pvutil.AnnDynamicallyProvisioned, plugin.GetPluginName()) + metav1.SetMetaDataAnnotation(&volume.ObjectMeta, storagehelpers.AnnBoundByController, "yes") + metav1.SetMetaDataAnnotation(&volume.ObjectMeta, storagehelpers.AnnDynamicallyProvisioned, plugin.GetPluginName()) if utilfeature.DefaultFeatureGate.Enabled(features.HonorPVReclaimPolicy) { // Add finalizer here - volume.SetFinalizers([]string{pvutil.PVDeletionInTreeProtectionFinalizer}) + volume.SetFinalizers([]string{storagehelpers.PVDeletionInTreeProtectionFinalizer}) } // Try to create the PV object several times @@ -1735,7 +1735,7 @@ func (ctrl *PersistentVolumeController) provisionClaimOperationExternal( ctx context.Context, claim *v1.PersistentVolumeClaim, storageClass *storage.StorageClass) (string, error) { - claimClass := pvutil.GetPersistentVolumeClaimClass(claim) + claimClass := storagehelpers.GetPersistentVolumeClaimClass(claim) klog.V(4).Infof("provisionClaimOperationExternal [%s] started, class: %q", claimToClaimKey(claim), claimClass) // Set provisionerName to external provisioner name by setClaimProvisioner var err error @@ -1770,7 +1770,7 @@ func (ctrl *PersistentVolumeController) provisionClaimOperationExternal( // rescheduleProvisioning signal back to the scheduler to retry dynamic provisioning // by removing the AnnSelectedNode annotation func (ctrl *PersistentVolumeController) rescheduleProvisioning(claim *v1.PersistentVolumeClaim) { - if _, ok := claim.Annotations[pvutil.AnnSelectedNode]; !ok { + if _, ok := claim.Annotations[storagehelpers.AnnSelectedNode]; !ok { // Provisioning not triggered by the scheduler, skip return } @@ -1778,10 +1778,10 @@ func (ctrl *PersistentVolumeController) rescheduleProvisioning(claim *v1.Persist // The claim from method args can be pointing to watcher cache. We must not // modify these, therefore create a copy. newClaim := claim.DeepCopy() - delete(newClaim.Annotations, pvutil.AnnSelectedNode) + delete(newClaim.Annotations, storagehelpers.AnnSelectedNode) // Try to update the PVC object if _, err := ctrl.kubeClient.CoreV1().PersistentVolumeClaims(newClaim.Namespace).Update(context.TODO(), newClaim, metav1.UpdateOptions{}); err != nil { - klog.V(4).Infof("Failed to delete annotation 'pvutil.AnnSelectedNode' for PersistentVolumeClaim %q: %v", claimToClaimKey(newClaim), err) + klog.V(4).Infof("Failed to delete annotation 'storagehelpers.AnnSelectedNode' for PersistentVolumeClaim %q: %v", claimToClaimKey(newClaim), err) return } if _, err := ctrl.storeClaimUpdate(newClaim); err != nil { @@ -1833,7 +1833,7 @@ func (ctrl *PersistentVolumeController) newRecyclerEventRecorder(volume *v1.Pers func (ctrl *PersistentVolumeController) findProvisionablePlugin(claim *v1.PersistentVolumeClaim) (vol.ProvisionableVolumePlugin, *storage.StorageClass, error) { // provisionClaim() which leads here is never called with claimClass=="", we // can save some checks. - claimClass := pvutil.GetPersistentVolumeClaimClass(claim) + claimClass := storagehelpers.GetPersistentVolumeClaimClass(claim) class, err := ctrl.classLister.Get(claimClass) if err != nil { return nil, nil, err @@ -1860,8 +1860,8 @@ func (ctrl *PersistentVolumeController) findProvisionablePlugin(claim *v1.Persis func (ctrl *PersistentVolumeController) findDeletablePlugin(volume *v1.PersistentVolume) (vol.DeletableVolumePlugin, error) { // Find a plugin. Try to find the same plugin that provisioned the volume var plugin vol.DeletableVolumePlugin - if metav1.HasAnnotation(volume.ObjectMeta, pvutil.AnnDynamicallyProvisioned) { - provisionPluginName := volume.Annotations[pvutil.AnnDynamicallyProvisioned] + if metav1.HasAnnotation(volume.ObjectMeta, storagehelpers.AnnDynamicallyProvisioned) { + provisionPluginName := volume.Annotations[storagehelpers.AnnDynamicallyProvisioned] if provisionPluginName != "" { plugin, err := ctrl.volumePluginMgr.FindDeletablePluginByName(provisionPluginName) if err != nil { @@ -1901,7 +1901,7 @@ func (ctrl *PersistentVolumeController) getProvisionerNameFromVolume(volume *v1. // the AnnDynamicallyProvisioned annotation value, use the storageClass's Provisioner // field to avoid explosion of the metric in the cases like local storage provisioner // tagging a volume with arbitrary provisioner names - storageClass := pvutil.GetPersistentVolumeClass(volume) + storageClass := storagehelpers.GetPersistentVolumeClass(volume) class, err := ctrl.classLister.Get(storageClass) if err != nil { return "N/A" diff --git a/pkg/controller/volume/persistentvolume/pv_controller_base.go b/pkg/controller/volume/persistentvolume/pv_controller_base.go index 7f24d73074e..6fbfa0bc9ba 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller_base.go +++ b/pkg/controller/volume/persistentvolume/pv_controller_base.go @@ -40,7 +40,7 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" cloudprovider "k8s.io/cloud-provider" - pvutil "k8s.io/component-helpers/storage/volume" + storagehelpers "k8s.io/component-helpers/storage/volume" csitrans "k8s.io/csi-translation-lib" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller/volume/common" @@ -389,15 +389,15 @@ func updateMigrationAnnotationsAndFinalizers(cmpm CSIMigratedPluginManager, tran } var provisionerKey string if claim { - provisionerKey = pvutil.AnnStorageProvisioner + provisionerKey = storagehelpers.AnnStorageProvisioner } else { - provisionerKey = pvutil.AnnDynamicallyProvisioned + provisionerKey = storagehelpers.AnnDynamicallyProvisioned } provisioner, ok := ann[provisionerKey] if !ok { if claim { // Also check beta AnnStorageProvisioner annontation to make sure - provisioner, ok = ann[pvutil.AnnBetaStorageProvisioner] + provisioner, ok = ann[storagehelpers.AnnBetaStorageProvisioner] if !ok { return false } @@ -407,7 +407,7 @@ func updateMigrationAnnotationsAndFinalizers(cmpm CSIMigratedPluginManager, tran } } - migratedToDriver := ann[pvutil.AnnMigratedTo] + migratedToDriver := ann[storagehelpers.AnnMigratedTo] if cmpm.IsMigrationEnabledForPlugin(provisioner) { modified := false csiDriverName, err = translator.GetCSINameFromInTreeName(provisioner) @@ -416,13 +416,13 @@ func updateMigrationAnnotationsAndFinalizers(cmpm CSIMigratedPluginManager, tran return false } if migratedToDriver != csiDriverName { - ann[pvutil.AnnMigratedTo] = csiDriverName + ann[storagehelpers.AnnMigratedTo] = csiDriverName modified = true } // Remove in-tree delete finalizer on the PV as migration is enabled. if !claim && utilfeature.DefaultFeatureGate.Enabled(features.HonorPVReclaimPolicy) { - if finalizers != nil && slice.ContainsString(*finalizers, pvutil.PVDeletionInTreeProtectionFinalizer, nil) { - *finalizers = slice.RemoveString(*finalizers, pvutil.PVDeletionInTreeProtectionFinalizer, nil) + if finalizers != nil && slice.ContainsString(*finalizers, storagehelpers.PVDeletionInTreeProtectionFinalizer, nil) { + *finalizers = slice.RemoveString(*finalizers, storagehelpers.PVDeletionInTreeProtectionFinalizer, nil) modified = true } } @@ -430,20 +430,20 @@ func updateMigrationAnnotationsAndFinalizers(cmpm CSIMigratedPluginManager, tran } else { if migratedToDriver != "" { // Migration annotation exists but the driver isn't migrated currently - delete(ann, pvutil.AnnMigratedTo) + delete(ann, storagehelpers.AnnMigratedTo) if !claim && utilfeature.DefaultFeatureGate.Enabled(features.HonorPVReclaimPolicy) { modified := false if finalizers == nil { *finalizers = []string{} } // Add back the in-tree PV deletion protection finalizer if does not already exists - if !slice.ContainsString(*finalizers, pvutil.PVDeletionInTreeProtectionFinalizer, nil) { - *finalizers = append(*finalizers, pvutil.PVDeletionInTreeProtectionFinalizer) + if !slice.ContainsString(*finalizers, storagehelpers.PVDeletionInTreeProtectionFinalizer, nil) { + *finalizers = append(*finalizers, storagehelpers.PVDeletionInTreeProtectionFinalizer) modified = true } // Remove the external PV deletion protection finalizer - if slice.ContainsString(*finalizers, pvutil.PVDeletionProtectionFinalizer, nil) { - *finalizers = slice.RemoveString(*finalizers, pvutil.PVDeletionProtectionFinalizer, nil) + if slice.ContainsString(*finalizers, storagehelpers.PVDeletionProtectionFinalizer, nil) { + *finalizers = slice.RemoveString(*finalizers, storagehelpers.PVDeletionProtectionFinalizer, nil) modified = true } return modified @@ -596,7 +596,7 @@ func (ctrl *PersistentVolumeController) resync() { // setClaimProvisioner saves // claim.Annotations["volume.kubernetes.io/storage-provisioner"] = class.Provisioner func (ctrl *PersistentVolumeController) setClaimProvisioner(ctx context.Context, claim *v1.PersistentVolumeClaim, provisionerName string) (*v1.PersistentVolumeClaim, error) { - if val, ok := claim.Annotations[pvutil.AnnStorageProvisioner]; ok && val == provisionerName { + if val, ok := claim.Annotations[storagehelpers.AnnStorageProvisioner]; ok && val == provisionerName { // annotation is already set, nothing to do return claim, nil } @@ -605,8 +605,8 @@ func (ctrl *PersistentVolumeController) setClaimProvisioner(ctx context.Context, // modify these, therefore create a copy. claimClone := claim.DeepCopy() // TODO: remove the beta storage provisioner anno after the deprecation period - metav1.SetMetaDataAnnotation(&claimClone.ObjectMeta, pvutil.AnnBetaStorageProvisioner, provisionerName) - metav1.SetMetaDataAnnotation(&claimClone.ObjectMeta, pvutil.AnnStorageProvisioner, provisionerName) + metav1.SetMetaDataAnnotation(&claimClone.ObjectMeta, storagehelpers.AnnBetaStorageProvisioner, provisionerName) + metav1.SetMetaDataAnnotation(&claimClone.ObjectMeta, storagehelpers.AnnStorageProvisioner, provisionerName) updateMigrationAnnotationsAndFinalizers(ctrl.csiMigratedPluginManager, ctrl.translator, claimClone.Annotations, nil, true) newClaim, err := ctrl.kubeClient.CoreV1().PersistentVolumeClaims(claim.Namespace).Update(context.TODO(), claimClone, metav1.UpdateOptions{}) if err != nil { @@ -622,14 +622,14 @@ func (ctrl *PersistentVolumeController) setClaimProvisioner(ctx context.Context, // Stateless functions func getClaimStatusForLogging(claim *v1.PersistentVolumeClaim) string { - bound := metav1.HasAnnotation(claim.ObjectMeta, pvutil.AnnBindCompleted) - boundByController := metav1.HasAnnotation(claim.ObjectMeta, pvutil.AnnBoundByController) + bound := metav1.HasAnnotation(claim.ObjectMeta, storagehelpers.AnnBindCompleted) + boundByController := metav1.HasAnnotation(claim.ObjectMeta, storagehelpers.AnnBoundByController) return fmt.Sprintf("phase: %s, bound to: %q, bindCompleted: %v, boundByController: %v", claim.Status.Phase, claim.Spec.VolumeName, bound, boundByController) } func getVolumeStatusForLogging(volume *v1.PersistentVolume) string { - boundByController := metav1.HasAnnotation(volume.ObjectMeta, pvutil.AnnBoundByController) + boundByController := metav1.HasAnnotation(volume.ObjectMeta, storagehelpers.AnnBoundByController) claimName := "" if volume.Spec.ClaimRef != nil { claimName = fmt.Sprintf("%s/%s (uid: %s)", volume.Spec.ClaimRef.Namespace, volume.Spec.ClaimRef.Name, volume.Spec.ClaimRef.UID) diff --git a/pkg/controller/volume/persistentvolume/pv_controller_test.go b/pkg/controller/volume/persistentvolume/pv_controller_test.go index c4cb88cf4b6..da3650b3bdc 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller_test.go +++ b/pkg/controller/volume/persistentvolume/pv_controller_test.go @@ -36,7 +36,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" - pvutil "k8s.io/component-helpers/storage/volume" + "k8s.io/component-helpers/storage/volume" csitrans "k8s.io/csi-translation-lib" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/controller" @@ -63,9 +63,9 @@ func TestControllerSync(t *testing.T) { // addClaim gets a new claim. Check it's bound to a volume. "5-2 - complete bind", newVolumeArray("volume5-2", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), - newVolumeArray("volume5-2", "1Gi", "uid5-2", "claim5-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume5-2", "1Gi", "uid5-2", "claim5-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), noclaims, /* added in testAddClaim5_2 */ - newClaimArray("claim5-2", "uid5-2", "1Gi", "volume5-2", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim5-2", "uid5-2", "1Gi", "volume5-2", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted), noevents, noerrors, // Custom test function that generates an add event func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor, test controllerTest) error { @@ -77,9 +77,9 @@ func TestControllerSync(t *testing.T) { { "5-2-2 - complete bind when PV and PVC both exist", newVolumeArray("volume5-2", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty), - newVolumeArray("volume5-2", "1Gi", "uid5-2", "claim5-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume5-2", "1Gi", "uid5-2", "claim5-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), newClaimArray("claim5-2", "uid5-2", "1Gi", "", v1.ClaimPending, nil), - newClaimArray("claim5-2", "uid5-2", "1Gi", "volume5-2", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim5-2", "uid5-2", "1Gi", "volume5-2", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted), noevents, noerrors, func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor, test controllerTest) error { return nil @@ -87,10 +87,10 @@ func TestControllerSync(t *testing.T) { }, { "5-2-3 - complete bind when PV and PVC both exist and PV has AnnPreResizeCapacity annotation", - volumesWithAnnotation(util.AnnPreResizeCapacity, "1Gi", newVolumeArray("volume5-2", "2Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController)), - volumesWithAnnotation(util.AnnPreResizeCapacity, "1Gi", newVolumeArray("volume5-2", "2Gi", "uid5-2", "claim5-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController)), + volumesWithAnnotation(util.AnnPreResizeCapacity, "1Gi", newVolumeArray("volume5-2", "2Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController)), + volumesWithAnnotation(util.AnnPreResizeCapacity, "1Gi", newVolumeArray("volume5-2", "2Gi", "uid5-2", "claim5-2", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController)), withExpectedCapacity("2Gi", newClaimArray("claim5-2", "uid5-2", "2Gi", "", v1.ClaimPending, nil)), - withExpectedCapacity("1Gi", newClaimArray("claim5-2", "uid5-2", "2Gi", "volume5-2", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + withExpectedCapacity("1Gi", newClaimArray("claim5-2", "uid5-2", "2Gi", "volume5-2", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted)), noevents, noerrors, func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor, test controllerTest) error { return nil @@ -99,9 +99,9 @@ func TestControllerSync(t *testing.T) { { // deleteClaim with a bound claim makes bound volume released. "5-3 - delete claim", - newVolumeArray("volume5-3", "10Gi", "uid5-3", "claim5-3", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), - newVolumeArray("volume5-3", "10Gi", "uid5-3", "claim5-3", v1.VolumeReleased, v1.PersistentVolumeReclaimRetain, classEmpty, pvutil.AnnBoundByController), - newClaimArray("claim5-3", "uid5-3", "1Gi", "volume5-3", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newVolumeArray("volume5-3", "10Gi", "uid5-3", "claim5-3", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), + newVolumeArray("volume5-3", "10Gi", "uid5-3", "claim5-3", v1.VolumeReleased, v1.PersistentVolumeReclaimRetain, classEmpty, volume.AnnBoundByController), + newClaimArray("claim5-3", "uid5-3", "1Gi", "volume5-3", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted), noclaims, noevents, noerrors, // Custom test function that generates a delete event @@ -117,8 +117,8 @@ func TestControllerSync(t *testing.T) { "5-4 - delete volume", newVolumeArray("volume5-4", "1Gi", "uid5-4", "claim5-4", v1.VolumeBound, v1.PersistentVolumeReclaimRetain, classEmpty), novolumes, - newClaimArray("claim5-4", "uid5-4", "1Gi", "volume5-4", v1.ClaimBound, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), - newClaimArray("claim5-4", "uid5-4", "1Gi", "volume5-4", v1.ClaimLost, nil, pvutil.AnnBoundByController, pvutil.AnnBindCompleted), + newClaimArray("claim5-4", "uid5-4", "1Gi", "volume5-4", v1.ClaimBound, nil, volume.AnnBoundByController, volume.AnnBindCompleted), + newClaimArray("claim5-4", "uid5-4", "1Gi", "volume5-4", v1.ClaimLost, nil, volume.AnnBoundByController, volume.AnnBindCompleted), []string{"Warning ClaimLost"}, noerrors, // Custom test function that generates a delete event func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor, test controllerTest) error { @@ -132,18 +132,18 @@ func TestControllerSync(t *testing.T) { // deleteClaim with a bound claim makes bound volume released with external deleter. // delete the corresponding volume from apiserver, and report latency metric "5-5 - delete claim and delete volume report metric", - volumesWithAnnotation(pvutil.AnnDynamicallyProvisioned, "gcr.io/vendor-csi", - newVolumeArray("volume5-5", "10Gi", "uid5-5", "claim5-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classExternal, pvutil.AnnBoundByController)), + volumesWithAnnotation(volume.AnnDynamicallyProvisioned, "gcr.io/vendor-csi", + newVolumeArray("volume5-5", "10Gi", "uid5-5", "claim5-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classExternal, volume.AnnBoundByController)), novolumes, - claimWithAnnotation(pvutil.AnnStorageProvisioner, "gcr.io/vendor-csi", - newClaimArray("claim5-5", "uid5-5", "1Gi", "volume5-5", v1.ClaimBound, &classExternal, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + claimWithAnnotation(volume.AnnStorageProvisioner, "gcr.io/vendor-csi", + newClaimArray("claim5-5", "uid5-5", "1Gi", "volume5-5", v1.ClaimBound, &classExternal, volume.AnnBoundByController, volume.AnnBindCompleted)), noclaims, noevents, noerrors, // Custom test function that generates a delete claim event which should have been caught by // "deleteClaim" to remove the claim from controller's cache, after that, a volume deleted // event will be generated to trigger "deleteVolume" call for metric reporting func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor, test controllerTest) error { - test.initialVolumes[0].Annotations[pvutil.AnnDynamicallyProvisioned] = "gcr.io/vendor-csi" + test.initialVolumes[0].Annotations[volume.AnnDynamicallyProvisioned] = "gcr.io/vendor-csi" obj := ctrl.claims.List()[0] claim := obj.(*v1.PersistentVolumeClaim) reactor.DeleteClaimEvent(claim) @@ -163,12 +163,12 @@ func TestControllerSync(t *testing.T) { // deleteClaim with a bound claim makes bound volume released with external deleter pending // there should be an entry in operation timestamps cache in controller "5-6 - delete claim and waiting for external volume deletion", - volumesWithAnnotation(pvutil.AnnDynamicallyProvisioned, "gcr.io/vendor-csi", - newVolumeArray("volume5-6", "10Gi", "uid5-6", "claim5-6", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classExternal, pvutil.AnnBoundByController)), - volumesWithAnnotation(pvutil.AnnDynamicallyProvisioned, "gcr.io/vendor-csi", - newVolumeArray("volume5-6", "10Gi", "uid5-6", "claim5-6", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classExternal, pvutil.AnnBoundByController)), - claimWithAnnotation(pvutil.AnnStorageProvisioner, "gcr.io/vendor-csi", - newClaimArray("claim5-6", "uid5-6", "1Gi", "volume5-6", v1.ClaimBound, &classExternal, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + volumesWithAnnotation(volume.AnnDynamicallyProvisioned, "gcr.io/vendor-csi", + newVolumeArray("volume5-6", "10Gi", "uid5-6", "claim5-6", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classExternal, volume.AnnBoundByController)), + volumesWithAnnotation(volume.AnnDynamicallyProvisioned, "gcr.io/vendor-csi", + newVolumeArray("volume5-6", "10Gi", "uid5-6", "claim5-6", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classExternal, volume.AnnBoundByController)), + claimWithAnnotation(volume.AnnStorageProvisioner, "gcr.io/vendor-csi", + newClaimArray("claim5-6", "uid5-6", "1Gi", "volume5-6", v1.ClaimBound, &classExternal, volume.AnnBoundByController, volume.AnnBindCompleted)), noclaims, noevents, noerrors, // Custom test function that generates a delete claim event which should have been caught by @@ -200,10 +200,10 @@ func TestControllerSync(t *testing.T) { // deleteVolume event issued before deleteClaim, no metric should have been reported // and no delete operation start timestamp should be inserted into controller.operationTimestamps cache "5-7 - delete volume event makes claim lost, delete claim event will not report metric", - newVolumeArray("volume5-7", "10Gi", "uid5-7", "claim5-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classExternal, pvutil.AnnBoundByController, pvutil.AnnDynamicallyProvisioned), + newVolumeArray("volume5-7", "10Gi", "uid5-7", "claim5-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classExternal, volume.AnnBoundByController, volume.AnnDynamicallyProvisioned), novolumes, - claimWithAnnotation(pvutil.AnnStorageProvisioner, "gcr.io/vendor-csi", - newClaimArray("claim5-7", "uid5-7", "1Gi", "volume5-7", v1.ClaimBound, &classExternal, pvutil.AnnBoundByController, pvutil.AnnBindCompleted)), + claimWithAnnotation(volume.AnnStorageProvisioner, "gcr.io/vendor-csi", + newClaimArray("claim5-7", "uid5-7", "1Gi", "volume5-7", v1.ClaimBound, &classExternal, volume.AnnBoundByController, volume.AnnBindCompleted)), noclaims, []string{"Warning ClaimLost"}, noerrors, @@ -253,7 +253,7 @@ func TestControllerSync(t *testing.T) { "5-8 - delete claim cleans up operation timestamp cache for provision", novolumes, novolumes, - claimWithAnnotation(pvutil.AnnStorageProvisioner, "gcr.io/vendor-csi", + claimWithAnnotation(volume.AnnStorageProvisioner, "gcr.io/vendor-csi", newClaimArray("claim5-8", "uid5-8", "1Gi", "", v1.ClaimPending, &classExternal)), noclaims, []string{"Normal ExternalProvisioning"}, @@ -291,11 +291,11 @@ func TestControllerSync(t *testing.T) { // back on the PV since migration is disabled. "5-9 - volume has its PV deletion protection finalizer removed as CSI migration is disabled", volumesWithFinalizers( - volumesWithAnnotation(pvutil.AnnMigratedTo, "pd.csi.storage.gke.io", - newVolumeArray("volume-5-9", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimDelete, classEmpty, pvutil.AnnDynamicallyProvisioned)), - []string{pvutil.PVDeletionProtectionFinalizer}, + volumesWithAnnotation(volume.AnnMigratedTo, "pd.csi.storage.gke.io", + newVolumeArray("volume-5-9", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned)), + []string{volume.PVDeletionProtectionFinalizer}, ), - volumesWithFinalizers(newVolumeArray("volume-5-9", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimDelete, classEmpty, pvutil.AnnDynamicallyProvisioned), []string{pvutil.PVDeletionInTreeProtectionFinalizer}), + volumesWithFinalizers(newVolumeArray("volume-5-9", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned), []string{volume.PVDeletionInTreeProtectionFinalizer}), noclaims, noclaims, noevents, @@ -488,55 +488,55 @@ func TestAnnealMigrationAnnotations(t *testing.T) { }{ { name: "migration on for GCE", - volumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin}, - expVolumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin, pvutil.AnnMigratedTo: gceDriver}, - claimAnnotations: map[string]string{pvutil.AnnStorageProvisioner: gcePlugin}, - expClaimAnnotations: map[string]string{pvutil.AnnStorageProvisioner: gcePlugin, pvutil.AnnMigratedTo: gceDriver}, + volumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin}, + expVolumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin, volume.AnnMigratedTo: gceDriver}, + claimAnnotations: map[string]string{volume.AnnStorageProvisioner: gcePlugin}, + expClaimAnnotations: map[string]string{volume.AnnStorageProvisioner: gcePlugin, volume.AnnMigratedTo: gceDriver}, migratedDriverGates: []featuregate.Feature{features.CSIMigrationGCE}, disabledDriverGates: []featuregate.Feature{}, }, { name: "migration on for GCE with Beta storage provisioner annontation", - volumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin}, - expVolumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin, pvutil.AnnMigratedTo: gceDriver}, - claimAnnotations: map[string]string{pvutil.AnnBetaStorageProvisioner: gcePlugin}, - expClaimAnnotations: map[string]string{pvutil.AnnBetaStorageProvisioner: gcePlugin, pvutil.AnnMigratedTo: gceDriver}, + volumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin}, + expVolumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin, volume.AnnMigratedTo: gceDriver}, + claimAnnotations: map[string]string{volume.AnnBetaStorageProvisioner: gcePlugin}, + expClaimAnnotations: map[string]string{volume.AnnBetaStorageProvisioner: gcePlugin, volume.AnnMigratedTo: gceDriver}, migratedDriverGates: []featuregate.Feature{features.CSIMigrationGCE}, disabledDriverGates: []featuregate.Feature{}, }, { name: "migration off for GCE", - volumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin}, - expVolumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin}, - claimAnnotations: map[string]string{pvutil.AnnStorageProvisioner: gcePlugin}, - expClaimAnnotations: map[string]string{pvutil.AnnStorageProvisioner: gcePlugin}, + volumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin}, + expVolumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin}, + claimAnnotations: map[string]string{volume.AnnStorageProvisioner: gcePlugin}, + expClaimAnnotations: map[string]string{volume.AnnStorageProvisioner: gcePlugin}, migratedDriverGates: []featuregate.Feature{}, disabledDriverGates: []featuregate.Feature{features.CSIMigrationGCE}, }, { name: "migration off for GCE removes migrated to (rollback)", - volumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin, pvutil.AnnMigratedTo: gceDriver}, - expVolumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin}, - claimAnnotations: map[string]string{pvutil.AnnStorageProvisioner: gcePlugin, pvutil.AnnMigratedTo: gceDriver}, - expClaimAnnotations: map[string]string{pvutil.AnnStorageProvisioner: gcePlugin}, + volumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin, volume.AnnMigratedTo: gceDriver}, + expVolumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin}, + claimAnnotations: map[string]string{volume.AnnStorageProvisioner: gcePlugin, volume.AnnMigratedTo: gceDriver}, + expClaimAnnotations: map[string]string{volume.AnnStorageProvisioner: gcePlugin}, migratedDriverGates: []featuregate.Feature{}, disabledDriverGates: []featuregate.Feature{features.CSIMigrationGCE}, }, { name: "migration off for GCE removes migrated to (rollback) with Beta storage provisioner annontation", - volumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin, pvutil.AnnMigratedTo: gceDriver}, - expVolumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin}, - claimAnnotations: map[string]string{pvutil.AnnBetaStorageProvisioner: gcePlugin, pvutil.AnnMigratedTo: gceDriver}, - expClaimAnnotations: map[string]string{pvutil.AnnBetaStorageProvisioner: gcePlugin}, + volumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin, volume.AnnMigratedTo: gceDriver}, + expVolumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin}, + claimAnnotations: map[string]string{volume.AnnBetaStorageProvisioner: gcePlugin, volume.AnnMigratedTo: gceDriver}, + expClaimAnnotations: map[string]string{volume.AnnBetaStorageProvisioner: gcePlugin}, migratedDriverGates: []featuregate.Feature{}, disabledDriverGates: []featuregate.Feature{features.CSIMigrationGCE}, }, { name: "migration on for GCE other plugin not affected", - volumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: testPlugin}, - expVolumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: testPlugin}, - claimAnnotations: map[string]string{pvutil.AnnStorageProvisioner: testPlugin}, - expClaimAnnotations: map[string]string{pvutil.AnnStorageProvisioner: testPlugin}, + volumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: testPlugin}, + expVolumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: testPlugin}, + claimAnnotations: map[string]string{volume.AnnStorageProvisioner: testPlugin}, + expClaimAnnotations: map[string]string{volume.AnnStorageProvisioner: testPlugin}, migratedDriverGates: []featuregate.Feature{features.CSIMigrationGCE}, disabledDriverGates: []featuregate.Feature{}, }, @@ -627,9 +627,9 @@ func TestUpdateFinalizer(t *testing.T) { { // Represents a volume provisioned through external-provisioner name: "13-1 migration was never enabled, volume has the finalizer", - volumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gceDriver}, - volumeFinalizers: []string{pvutil.PVDeletionProtectionFinalizer}, - expVolumeFinalizers: []string{pvutil.PVDeletionProtectionFinalizer}, + volumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gceDriver}, + volumeFinalizers: []string{volume.PVDeletionProtectionFinalizer}, + expVolumeFinalizers: []string{volume.PVDeletionProtectionFinalizer}, expModified: false, migratedDriverGates: []featuregate.Feature{}, }, @@ -637,7 +637,7 @@ func TestUpdateFinalizer(t *testing.T) { // Represents a volume provisioned through external-provisioner but the external-provisioner has // yet to sync the volume to add the new finalizer name: "13-2 migration was never enabled, volume does not have the finalizer", - volumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gceDriver}, + volumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gceDriver}, volumeFinalizers: nil, expVolumeFinalizers: nil, expModified: false, @@ -650,17 +650,17 @@ func TestUpdateFinalizer(t *testing.T) { // the migrated-to annotation will be removed shortly when updateVolumeMigrationAnnotationsAndFinalizers // is called followed by adding back the in-tree pv protection finalizer. name: "13-3 migration was disabled but still has migrated-to annotation, volume does not have pv deletion protection finalizer", - volumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin, pvutil.AnnMigratedTo: gceDriver}, + volumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin, volume.AnnMigratedTo: gceDriver}, volumeFinalizers: []string{customFinalizer}, - expVolumeFinalizers: []string{customFinalizer, pvutil.PVDeletionInTreeProtectionFinalizer}, + expVolumeFinalizers: []string{customFinalizer, volume.PVDeletionInTreeProtectionFinalizer}, expModified: true, migratedDriverGates: []featuregate.Feature{}, }, { name: "13-4 migration was disabled but still has migrated-to annotation, volume has no finalizers", - volumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin, pvutil.AnnMigratedTo: gceDriver}, + volumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin, volume.AnnMigratedTo: gceDriver}, volumeFinalizers: nil, - expVolumeFinalizers: []string{pvutil.PVDeletionInTreeProtectionFinalizer}, + expVolumeFinalizers: []string{volume.PVDeletionInTreeProtectionFinalizer}, expModified: true, migratedDriverGates: []featuregate.Feature{}, }, @@ -669,9 +669,9 @@ func TestUpdateFinalizer(t *testing.T) { // finalizer and later the csi migration was disabled. The pv deletion protection finalizer added through // external-provisioner will be removed and the in-tree pv deletion protection finalizer will be added. name: "13-5 migration was disabled as it has the migrated-to annotation, volume has the finalizer", - volumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin, pvutil.AnnMigratedTo: gceDriver}, - volumeFinalizers: []string{pvutil.PVDeletionProtectionFinalizer}, - expVolumeFinalizers: []string{pvutil.PVDeletionInTreeProtectionFinalizer}, + volumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin, volume.AnnMigratedTo: gceDriver}, + volumeFinalizers: []string{volume.PVDeletionProtectionFinalizer}, + expVolumeFinalizers: []string{volume.PVDeletionInTreeProtectionFinalizer}, expModified: true, migratedDriverGates: []featuregate.Feature{}, }, @@ -680,9 +680,9 @@ func TestUpdateFinalizer(t *testing.T) { // protection finalizer added by external-provisioner will be removed and the in-tree pv deletion protection // finalizer will be added. name: "13-6 migration was disabled as it has the migrated-to annotation, volume has multiple finalizers", - volumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin, pvutil.AnnMigratedTo: gceDriver}, - volumeFinalizers: []string{pvutil.PVDeletionProtectionFinalizer, customFinalizer}, - expVolumeFinalizers: []string{customFinalizer, pvutil.PVDeletionInTreeProtectionFinalizer}, + volumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin, volume.AnnMigratedTo: gceDriver}, + volumeFinalizers: []string{volume.PVDeletionProtectionFinalizer, customFinalizer}, + expVolumeFinalizers: []string{customFinalizer, volume.PVDeletionInTreeProtectionFinalizer}, expModified: true, migratedDriverGates: []featuregate.Feature{}, }, @@ -690,9 +690,9 @@ func TestUpdateFinalizer(t *testing.T) { // csi migration is enabled, the pv controller should not delete the finalizer added by the // external-provisioner and the in-tree finalizer should be deleted. name: "13-7 migration is enabled, has the migrated-to annotation, volume has the finalizer", - volumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin, pvutil.AnnMigratedTo: gceDriver}, - volumeFinalizers: []string{pvutil.PVDeletionProtectionFinalizer, pvutil.PVDeletionInTreeProtectionFinalizer}, - expVolumeFinalizers: []string{pvutil.PVDeletionProtectionFinalizer}, + volumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin, volume.AnnMigratedTo: gceDriver}, + volumeFinalizers: []string{volume.PVDeletionProtectionFinalizer, volume.PVDeletionInTreeProtectionFinalizer}, + expVolumeFinalizers: []string{volume.PVDeletionProtectionFinalizer}, expModified: true, migratedDriverGates: []featuregate.Feature{features.CSIMigration, features.CSIMigrationGCE}, }, @@ -700,9 +700,9 @@ func TestUpdateFinalizer(t *testing.T) { // csi-migration is not completely enabled as the specific plugin feature is not present. This is equivalent // of disabled csi-migration. name: "13-8 migration is enabled but plugin migration feature is disabled, has the migrated-to annotation, volume has the finalizer", - volumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin, pvutil.AnnMigratedTo: gceDriver}, - volumeFinalizers: []string{pvutil.PVDeletionProtectionFinalizer}, - expVolumeFinalizers: []string{pvutil.PVDeletionInTreeProtectionFinalizer}, + volumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin, volume.AnnMigratedTo: gceDriver}, + volumeFinalizers: []string{volume.PVDeletionProtectionFinalizer}, + expVolumeFinalizers: []string{volume.PVDeletionInTreeProtectionFinalizer}, expModified: true, migratedDriverGates: []featuregate.Feature{features.CSIMigration}, }, @@ -710,9 +710,9 @@ func TestUpdateFinalizer(t *testing.T) { // same as 13-8 but multiple finalizers exists, only the pv deletion protection finalizer needs to be // removed and the in-tree pv deletion protection finalizer needs to be added. name: "13-9 migration is enabled but plugin migration feature is disabled, has the migrated-to annotation, volume has multiple finalizers", - volumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin, pvutil.AnnMigratedTo: gceDriver}, - volumeFinalizers: []string{pvutil.PVDeletionProtectionFinalizer, customFinalizer}, - expVolumeFinalizers: []string{customFinalizer, pvutil.PVDeletionInTreeProtectionFinalizer}, + volumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin, volume.AnnMigratedTo: gceDriver}, + volumeFinalizers: []string{volume.PVDeletionProtectionFinalizer, customFinalizer}, + expVolumeFinalizers: []string{customFinalizer, volume.PVDeletionInTreeProtectionFinalizer}, expModified: true, migratedDriverGates: []featuregate.Feature{features.CSIMigration}, }, @@ -720,8 +720,8 @@ func TestUpdateFinalizer(t *testing.T) { // corner error case. name: "13-10 missing annotations but finalizers exist", volumeAnnotations: nil, - volumeFinalizers: []string{pvutil.PVDeletionProtectionFinalizer}, - expVolumeFinalizers: []string{pvutil.PVDeletionProtectionFinalizer}, + volumeFinalizers: []string{volume.PVDeletionProtectionFinalizer}, + expVolumeFinalizers: []string{volume.PVDeletionProtectionFinalizer}, expModified: false, migratedDriverGates: []featuregate.Feature{}, }, @@ -737,16 +737,16 @@ func TestUpdateFinalizer(t *testing.T) { // corner error case name: "13-12 missing provisioned-by annotation, existing finalizers", volumeAnnotations: map[string]string{"fake": gcePlugin}, - volumeFinalizers: []string{pvutil.PVDeletionProtectionFinalizer}, - expVolumeFinalizers: []string{pvutil.PVDeletionProtectionFinalizer}, + volumeFinalizers: []string{volume.PVDeletionProtectionFinalizer}, + expVolumeFinalizers: []string{volume.PVDeletionProtectionFinalizer}, expModified: false, migratedDriverGates: []featuregate.Feature{}, }, { // csi migration is enabled, the pv controller should delete the in-tree finalizer name: "13-13 migration is enabled, has the migrated-to annotation, volume has the in-tree finalizer", - volumeAnnotations: map[string]string{pvutil.AnnDynamicallyProvisioned: gcePlugin, pvutil.AnnMigratedTo: gceDriver}, - volumeFinalizers: []string{pvutil.PVDeletionInTreeProtectionFinalizer}, + volumeAnnotations: map[string]string{volume.AnnDynamicallyProvisioned: gcePlugin, volume.AnnMigratedTo: gceDriver}, + volumeFinalizers: []string{volume.PVDeletionInTreeProtectionFinalizer}, expVolumeFinalizers: nil, expModified: true, migratedDriverGates: []featuregate.Feature{features.CSIMigration, features.CSIMigrationGCE}, diff --git a/pkg/controller/volume/persistentvolume/recycle_test.go b/pkg/controller/volume/persistentvolume/recycle_test.go index d8bfdbc7477..84793fe8ed9 100644 --- a/pkg/controller/volume/persistentvolume/recycle_test.go +++ b/pkg/controller/volume/persistentvolume/recycle_test.go @@ -23,7 +23,7 @@ import ( v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - pvutil "k8s.io/component-helpers/storage/volume" + "k8s.io/component-helpers/storage/volume" pvtesting "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/testing" ) @@ -74,7 +74,7 @@ func TestRecycleSync(t *testing.T) { { // recycle volume bound by controller "6-1 - successful recycle", - newVolumeArray("volume6-1", "1Gi", "uid6-1", "claim6-1", v1.VolumeBound, v1.PersistentVolumeReclaimRecycle, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume6-1", "1Gi", "uid6-1", "claim6-1", v1.VolumeBound, v1.PersistentVolumeReclaimRecycle, classEmpty, volume.AnnBoundByController), newVolumeArray("volume6-1", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRecycle, classEmpty), noclaims, noclaims, @@ -142,7 +142,7 @@ func TestRecycleSync(t *testing.T) { // at the time new doRecycle() starts. This simulates "volume no // longer needs recycling, skipping". "6-7 - volume is deleted before recycling", - newVolumeArray("volume6-7", "1Gi", "uid6-7", "claim6-7", v1.VolumeBound, v1.PersistentVolumeReclaimRecycle, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume6-7", "1Gi", "uid6-7", "claim6-7", v1.VolumeBound, v1.PersistentVolumeReclaimRecycle, classEmpty, volume.AnnBoundByController), newVolumeArray("volume6-7", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRecycle, classEmpty), noclaims, noclaims, @@ -193,8 +193,8 @@ func TestRecycleSync(t *testing.T) { { // volume is used by a running pod - failure expected "6-11 - used by running pod", - newVolumeArray("volume6-11", "1Gi", "uid6-11", "runningClaim", v1.VolumeBound, v1.PersistentVolumeReclaimRecycle, classEmpty, pvutil.AnnBoundByController), - newVolumeArray("volume6-11", "1Gi", "uid6-11", "runningClaim", v1.VolumeReleased, v1.PersistentVolumeReclaimRecycle, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume6-11", "1Gi", "uid6-11", "runningClaim", v1.VolumeBound, v1.PersistentVolumeReclaimRecycle, classEmpty, volume.AnnBoundByController), + newVolumeArray("volume6-11", "1Gi", "uid6-11", "runningClaim", v1.VolumeReleased, v1.PersistentVolumeReclaimRecycle, classEmpty, volume.AnnBoundByController), noclaims, noclaims, []string{"Normal VolumeFailedRecycle"}, noerrors, testSyncVolume, @@ -202,8 +202,8 @@ func TestRecycleSync(t *testing.T) { { // volume is used by a pending pod - failure expected "6-12 - used by pending pod", - newVolumeArray("volume6-12", "1Gi", "uid6-12", "pendingClaim", v1.VolumeBound, v1.PersistentVolumeReclaimRecycle, classEmpty, pvutil.AnnBoundByController), - newVolumeArray("volume6-12", "1Gi", "uid6-12", "pendingClaim", v1.VolumeReleased, v1.PersistentVolumeReclaimRecycle, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume6-12", "1Gi", "uid6-12", "pendingClaim", v1.VolumeBound, v1.PersistentVolumeReclaimRecycle, classEmpty, volume.AnnBoundByController), + newVolumeArray("volume6-12", "1Gi", "uid6-12", "pendingClaim", v1.VolumeReleased, v1.PersistentVolumeReclaimRecycle, classEmpty, volume.AnnBoundByController), noclaims, noclaims, []string{"Normal VolumeFailedRecycle"}, noerrors, testSyncVolume, @@ -211,7 +211,7 @@ func TestRecycleSync(t *testing.T) { { // volume is used by a completed pod - recycle succeeds "6-13 - used by completed pod", - newVolumeArray("volume6-13", "1Gi", "uid6-13", "completedClaim", v1.VolumeBound, v1.PersistentVolumeReclaimRecycle, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume6-13", "1Gi", "uid6-13", "completedClaim", v1.VolumeBound, v1.PersistentVolumeReclaimRecycle, classEmpty, volume.AnnBoundByController), newVolumeArray("volume6-13", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRecycle, classEmpty), noclaims, noclaims, @@ -223,7 +223,7 @@ func TestRecycleSync(t *testing.T) { { // volume is used by a completed pod, pod using claim with the same name bound to different pv is running, should recycle "6-14 - seemingly used by running pod", - newVolumeArray("volume6-14", "1Gi", "uid6-14", "completedClaim", v1.VolumeBound, v1.PersistentVolumeReclaimRecycle, classEmpty, pvutil.AnnBoundByController), + newVolumeArray("volume6-14", "1Gi", "uid6-14", "completedClaim", v1.VolumeBound, v1.PersistentVolumeReclaimRecycle, classEmpty, volume.AnnBoundByController), newVolumeArray("volume6-14", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRecycle, classEmpty), newClaimArray("completedClaim", "uid6-14-x", "10Gi", "", v1.ClaimBound, nil), newClaimArray("completedClaim", "uid6-14-x", "10Gi", "", v1.ClaimBound, nil), diff --git a/pkg/scheduler/framework/plugins/volumebinding/assume_cache_test.go b/pkg/scheduler/framework/plugins/volumebinding/assume_cache_test.go index f7a91235b1b..49fcb1cee85 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/assume_cache_test.go +++ b/pkg/scheduler/framework/plugins/volumebinding/assume_cache_test.go @@ -22,7 +22,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - pvutil "k8s.io/component-helpers/storage/volume" + "k8s.io/component-helpers/storage/volume" ) func verifyListPVs(t *testing.T, cache PVAssumeCache, expectedPVs map[string]*v1.PersistentVolume, storageClassName string) { @@ -445,7 +445,7 @@ func TestAssumeUpdatePVCCache(t *testing.T) { // Assume PVC newPVC := pvc.DeepCopy() - newPVC.Annotations[pvutil.AnnSelectedNode] = "test-node" + newPVC.Annotations[volume.AnnSelectedNode] = "test-node" if err := cache.Assume(newPVC); err != nil { t.Fatalf("failed to assume PVC: %v", err) } diff --git a/pkg/scheduler/framework/plugins/volumebinding/binder.go b/pkg/scheduler/framework/plugins/volumebinding/binder.go index 6dc820e0fcf..6c38b519cff 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/binder.go +++ b/pkg/scheduler/framework/plugins/volumebinding/binder.go @@ -41,7 +41,7 @@ import ( storagelisters "k8s.io/client-go/listers/storage/v1" storagelistersv1beta1 "k8s.io/client-go/listers/storage/v1beta1" "k8s.io/component-helpers/storage/ephemeral" - pvutil "k8s.io/component-helpers/storage/volume" + "k8s.io/component-helpers/storage/volume" csitrans "k8s.io/csi-translation-lib" csiplugins "k8s.io/csi-translation-lib/plugins" "k8s.io/klog/v2" @@ -328,7 +328,7 @@ func (b *volumeBinder) FindPodVolumes(pod *v1.Pod, boundClaims, claimsToBind []* // Filter out claims to provision for _, claim := range claimsToBind { - if selectedNode, ok := claim.Annotations[pvutil.AnnSelectedNode]; ok { + if selectedNode, ok := claim.Annotations[volume.AnnSelectedNode]; ok { if selectedNode != node.Name { // Fast path, skip unmatched node. unboundVolumesSatisfied = false @@ -384,7 +384,7 @@ func (b *volumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string, pod // Assume PV newBindings := []*BindingInfo{} for _, binding := range podVolumes.StaticBindings { - newPV, dirty, err := pvutil.GetBindVolumeToClaim(binding.pv, binding.pvc) + newPV, dirty, err := volume.GetBindVolumeToClaim(binding.pv, binding.pvc) klog.V(5).InfoS("AssumePodVolumes: GetBindVolumeToClaim", "pod", klog.KObj(assumedPod), "PV", klog.KObj(binding.pv), @@ -414,7 +414,7 @@ func (b *volumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string, pod // The claims from method args can be pointing to watcher cache. We must not // modify these, therefore create a copy. claimClone := claim.DeepCopy() - metav1.SetMetaDataAnnotation(&claimClone.ObjectMeta, pvutil.AnnSelectedNode, nodeName) + metav1.SetMetaDataAnnotation(&claimClone.ObjectMeta, volume.AnnSelectedNode, nodeName) err = b.pvcCache.Assume(claimClone) if err != nil { b.revertAssumedPVs(newBindings) @@ -606,7 +606,7 @@ func (b *volumeBinder) checkBindings(pod *v1.Pod, bindings []*BindingInfo, claim } // Check PV's node affinity (the node might not have the proper label) - if err := pvutil.CheckNodeAffinity(pv, node.Labels); err != nil { + if err := volume.CheckNodeAffinity(pv, node.Labels); err != nil { return false, fmt.Errorf("pv %q node affinity doesn't match node %q: %w", pv.Name, node.Name, err) } @@ -637,7 +637,7 @@ func (b *volumeBinder) checkBindings(pod *v1.Pod, bindings []*BindingInfo, claim if pvc.Annotations == nil { return false, fmt.Errorf("selectedNode annotation reset for PVC %q", pvc.Name) } - selectedNode := pvc.Annotations[pvutil.AnnSelectedNode] + selectedNode := pvc.Annotations[volume.AnnSelectedNode] if selectedNode != pod.Spec.NodeName { // If provisioner fails to provision a volume, selectedNode // annotation will be removed to signal back to the scheduler to @@ -664,7 +664,7 @@ func (b *volumeBinder) checkBindings(pod *v1.Pod, bindings []*BindingInfo, claim return false, err } - if err := pvutil.CheckNodeAffinity(pv, node.Labels); err != nil { + if err := volume.CheckNodeAffinity(pv, node.Labels); err != nil { return false, fmt.Errorf("pv %q node affinity doesn't match node %q: %w", pv.Name, node.Name, err) } } @@ -732,7 +732,7 @@ func (b *volumeBinder) isPVCBound(namespace, pvcName string) (bool, *v1.Persiste } func (b *volumeBinder) isPVCFullyBound(pvc *v1.PersistentVolumeClaim) bool { - return pvc.Spec.VolumeName != "" && metav1.HasAnnotation(pvc.ObjectMeta, pvutil.AnnBindCompleted) + return pvc.Spec.VolumeName != "" && metav1.HasAnnotation(pvc.ObjectMeta, volume.AnnBindCompleted) } // arePodVolumesBound returns true if all volumes are fully bound @@ -764,7 +764,7 @@ func (b *volumeBinder) GetPodVolumes(pod *v1.Pod) (boundClaims []*v1.PersistentV if volumeBound { boundClaims = append(boundClaims, pvc) } else { - delayBindingMode, err := pvutil.IsDelayBindingMode(pvc, b.classLister) + delayBindingMode, err := volume.IsDelayBindingMode(pvc, b.classLister) if err != nil { return nil, nil, nil, err } @@ -804,7 +804,7 @@ func (b *volumeBinder) checkBoundClaims(claims []*v1.PersistentVolumeClaim, node return false, true, err } - err = pvutil.CheckNodeAffinity(pv, node.Labels) + err = volume.CheckNodeAffinity(pv, node.Labels) if err != nil { klog.V(4).InfoS("PersistentVolume and node mismatch for pod", "PV", klog.KRef("", pvName), "node", klog.KObj(node), "pod", klog.KObj(pod), "err", err) return false, true, nil @@ -828,11 +828,11 @@ func (b *volumeBinder) findMatchingVolumes(pod *v1.Pod, claimsToBind []*v1.Persi for _, pvc := range claimsToBind { // Get storage class name from each PVC - storageClassName := pvutil.GetPersistentVolumeClaimClass(pvc) + storageClassName := volume.GetPersistentVolumeClaimClass(pvc) allPVs := b.pvCache.ListPVs(storageClassName) // Find a matching PV - pv, err := pvutil.FindMatchingVolume(pvc, allPVs, node, chosenPVs, true) + pv, err := volume.FindMatchingVolume(pvc, allPVs, node, chosenPVs, true) if err != nil { return false, nil, nil, err } @@ -866,7 +866,7 @@ func (b *volumeBinder) checkVolumeProvisions(pod *v1.Pod, claimsToProvision []*v // fails or we encounter an error. for _, claim := range claimsToProvision { pvcName := getPVCName(claim) - className := pvutil.GetPersistentVolumeClaimClass(claim) + className := volume.GetPersistentVolumeClaimClass(claim) if className == "" { return false, false, nil, fmt.Errorf("no class for claim %q", pvcName) } @@ -876,7 +876,7 @@ func (b *volumeBinder) checkVolumeProvisions(pod *v1.Pod, claimsToProvision []*v return false, false, nil, fmt.Errorf("failed to find storage class %q", className) } provisioner := class.Provisioner - if provisioner == "" || provisioner == pvutil.NotSupportedProvisioner { + if provisioner == "" || provisioner == volume.NotSupportedProvisioner { klog.V(4).InfoS("Storage class of claim does not support dynamic provisioning", "storageClassName", className, "PVC", klog.KObj(claim)) return false, true, nil, nil } diff --git a/pkg/scheduler/framework/plugins/volumebinding/binder_test.go b/pkg/scheduler/framework/plugins/volumebinding/binder_test.go index bcc47a6027e..707f26ba06b 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/binder_test.go +++ b/pkg/scheduler/framework/plugins/volumebinding/binder_test.go @@ -43,7 +43,7 @@ import ( "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" featuregatetesting "k8s.io/component-base/featuregate/testing" - pvutil "k8s.io/component-helpers/storage/volume" + "k8s.io/component-helpers/storage/volume" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/controller" pvtesting "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/testing" @@ -496,8 +496,8 @@ func (env *testEnv) validateAssume(t *testing.T, pod *v1.Pod, bindings []*Bindin t.Errorf("GetPVC %q returned error: %v", pvcKey, err) continue } - if pvc.Annotations[pvutil.AnnSelectedNode] != nodeLabelValue { - t.Errorf("expected pvutil.AnnSelectedNode of pvc %q to be %q, but got %q", pvcKey, nodeLabelValue, pvc.Annotations[pvutil.AnnSelectedNode]) + if pvc.Annotations[volume.AnnSelectedNode] != nodeLabelValue { + t.Errorf("expected volume.AnnSelectedNode of pvc %q to be %q, but got %q", pvcKey, nodeLabelValue, pvc.Annotations[volume.AnnSelectedNode]) } } } @@ -523,8 +523,8 @@ func (env *testEnv) validateCacheRestored(t *testing.T, pod *v1.Pod, bindings [] t.Errorf("GetPVC %q returned error: %v", pvcKey, err) continue } - if pvc.Annotations[pvutil.AnnSelectedNode] != "" { - t.Errorf("expected pvutil.AnnSelectedNode of pvc %q empty, but got %q", pvcKey, pvc.Annotations[pvutil.AnnSelectedNode]) + if pvc.Annotations[volume.AnnSelectedNode] != "" { + t.Errorf("expected volume.AnnSelectedNode of pvc %q empty, but got %q", pvcKey, pvc.Annotations[volume.AnnSelectedNode]) } } } @@ -636,10 +636,10 @@ func makeTestPVC(name, size, node string, pvcBoundState int, pvName, resourceVer switch pvcBoundState { case pvcSelectedNode: - metav1.SetMetaDataAnnotation(&pvc.ObjectMeta, pvutil.AnnSelectedNode, node) + metav1.SetMetaDataAnnotation(&pvc.ObjectMeta, volume.AnnSelectedNode, node) // don't fallthrough case pvcBound: - metav1.SetMetaDataAnnotation(&pvc.ObjectMeta, pvutil.AnnBindCompleted, "yes") + metav1.SetMetaDataAnnotation(&pvc.ObjectMeta, volume.AnnBindCompleted, "yes") fallthrough case pvcPrebound: pvc.Spec.VolumeName = pvName @@ -666,7 +666,7 @@ func makeTestPV(name, node, capacity, version string, boundToPVC *v1.PersistentV }, } if node != "" { - pv.Spec.NodeAffinity = pvutil.GetVolumeNodeAffinity(nodeLabelKey, node) + pv.Spec.NodeAffinity = volume.GetVolumeNodeAffinity(nodeLabelKey, node) } if boundToPVC != nil { @@ -678,7 +678,7 @@ func makeTestPV(name, node, capacity, version string, boundToPVC *v1.PersistentV Namespace: boundToPVC.Namespace, UID: boundToPVC.UID, } - metav1.SetMetaDataAnnotation(&pv.ObjectMeta, pvutil.AnnBoundByController, "yes") + metav1.SetMetaDataAnnotation(&pv.ObjectMeta, volume.AnnBoundByController, "yes") } return pv @@ -701,7 +701,7 @@ func makeTestPVForCSIMigration(labels map[string]string, pvc *v1.PersistentVolum func pvcSetSelectedNode(pvc *v1.PersistentVolumeClaim, node string) *v1.PersistentVolumeClaim { newPVC := pvc.DeepCopy() - metav1.SetMetaDataAnnotation(&newPVC.ObjectMeta, pvutil.AnnSelectedNode, node) + metav1.SetMetaDataAnnotation(&newPVC.ObjectMeta, volume.AnnSelectedNode, node) return newPVC } @@ -768,7 +768,7 @@ func makeBinding(pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) *Bindin func addProvisionAnn(pvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { res := pvc.DeepCopy() // Add provision related annotations - metav1.SetMetaDataAnnotation(&res.ObjectMeta, pvutil.AnnSelectedNode, nodeLabelValue) + metav1.SetMetaDataAnnotation(&res.ObjectMeta, volume.AnnSelectedNode, nodeLabelValue) return res } @@ -1927,7 +1927,7 @@ func TestBindPodVolumes(t *testing.T) { // Update PVC to be fully bound to PV newPVC := pvc.DeepCopy() newPVC.Spec.VolumeName = pv.Name - metav1.SetMetaDataAnnotation(&newPVC.ObjectMeta, pvutil.AnnBindCompleted, "yes") + metav1.SetMetaDataAnnotation(&newPVC.ObjectMeta, volume.AnnBindCompleted, "yes") if _, err := testEnv.client.CoreV1().PersistentVolumeClaims(newPVC.Namespace).Update(context.TODO(), newPVC, metav1.UpdateOptions{}); err != nil { t.Errorf("failed to update PVC %q: %v", newPVC.Name, err) } @@ -1951,7 +1951,7 @@ func TestBindPodVolumes(t *testing.T) { return } newPVC.Spec.VolumeName = dynamicPV.Name - metav1.SetMetaDataAnnotation(&newPVC.ObjectMeta, pvutil.AnnBindCompleted, "yes") + metav1.SetMetaDataAnnotation(&newPVC.ObjectMeta, volume.AnnBindCompleted, "yes") if _, err := testEnv.client.CoreV1().PersistentVolumeClaims(newPVC.Namespace).Update(context.TODO(), newPVC, metav1.UpdateOptions{}); err != nil { t.Errorf("failed to update PVC %q: %v", newPVC.Name, err) } @@ -2015,7 +2015,7 @@ func TestBindPodVolumes(t *testing.T) { // Update PVC to be fully bound to a PV with a different node newPVC := pvcs[0].DeepCopy() newPVC.Spec.VolumeName = pvNode2.Name - metav1.SetMetaDataAnnotation(&newPVC.ObjectMeta, pvutil.AnnBindCompleted, "yes") + metav1.SetMetaDataAnnotation(&newPVC.ObjectMeta, volume.AnnBindCompleted, "yes") if _, err := testEnv.client.CoreV1().PersistentVolumeClaims(newPVC.Namespace).Update(context.TODO(), newPVC, metav1.UpdateOptions{}); err != nil { t.Errorf("failed to update PVC %q: %v", newPVC.Name, err) } diff --git a/pkg/scheduler/framework/plugins/volumebinding/test_utils.go b/pkg/scheduler/framework/plugins/volumebinding/test_utils.go index a64f8e0519f..44841094481 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/test_utils.go +++ b/pkg/scheduler/framework/plugins/volumebinding/test_utils.go @@ -22,7 +22,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - pvutil "k8s.io/component-helpers/storage/volume" + "k8s.io/component-helpers/storage/volume" "k8s.io/utils/pointer" ) @@ -122,7 +122,7 @@ func makePVC(name string, storageClassName string) pvcBuilder { func (pvcb pvcBuilder) withBoundPV(pvName string) pvcBuilder { pvcb.PersistentVolumeClaim.Spec.VolumeName = pvName - metav1.SetMetaDataAnnotation(&pvcb.PersistentVolumeClaim.ObjectMeta, pvutil.AnnBindCompleted, "true") + metav1.SetMetaDataAnnotation(&pvcb.PersistentVolumeClaim.ObjectMeta, volume.AnnBindCompleted, "true") return pvcb } diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 0e50a071145..b1133a50faf 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -46,7 +46,7 @@ import ( clienttesting "k8s.io/client-go/testing" clientcache "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/events" - pvutil "k8s.io/component-helpers/storage/volume" + "k8s.io/component-helpers/storage/volume" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" @@ -2414,7 +2414,7 @@ func TestSchedulerSchedulePod(t *testing.T) { cs := clientsetfake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(cs, 0) for _, pvc := range test.pvcs { - metav1.SetMetaDataAnnotation(&pvc.ObjectMeta, pvutil.AnnBindCompleted, "true") + metav1.SetMetaDataAnnotation(&pvc.ObjectMeta, volume.AnnBindCompleted, "true") cs.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(ctx, &pvc, metav1.CreateOptions{}) if pvName := pvc.Spec.VolumeName; pvName != "" { pv := v1.PersistentVolume{ObjectMeta: metav1.ObjectMeta{Name: pvName}} diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 5230af7658b..51f74c3acdd 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -35,7 +35,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/record" volerr "k8s.io/cloud-provider/volume/errors" - pvutil "k8s.io/component-helpers/storage/volume" + storagehelpers "k8s.io/component-helpers/storage/volume" csitrans "k8s.io/csi-translation-lib" "k8s.io/klog/v2" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" @@ -2264,7 +2264,7 @@ func checkNodeAffinity(og *operationGenerator, volumeToMount VolumeToMount) erro if err != nil { return err } - err = pvutil.CheckNodeAffinity(pv, nodeLabels) + err = storagehelpers.CheckNodeAffinity(pv, nodeLabels) if err != nil { return err } From 0b5b04842c0d16b9500aca09ff30be33b04c79a1 Mon Sep 17 00:00:00 2001 From: Konstantin Misyutin Date: Wed, 10 Nov 2021 21:50:19 +0800 Subject: [PATCH 3/4] Cleanup: merge checkVolumeNodeAffinity with CheckVolumeNodeAffinity Signed-off-by: Konstantin Misyutin --- .../src/k8s.io/component-helpers/storage/volume/helpers.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/staging/src/k8s.io/component-helpers/storage/volume/helpers.go b/staging/src/k8s.io/component-helpers/storage/volume/helpers.go index 2ca4201c236..7ec376f34a0 100644 --- a/staging/src/k8s.io/component-helpers/storage/volume/helpers.go +++ b/staging/src/k8s.io/component-helpers/storage/volume/helpers.go @@ -52,15 +52,12 @@ func GetPersistentVolumeClass(volume *v1.PersistentVolume) string { // CheckNodeAffinity looks at the PV node affinity, and checks if the node has the same corresponding labels // This ensures that we don't mount a volume that doesn't belong to this node func CheckNodeAffinity(pv *v1.PersistentVolume, nodeLabels map[string]string) error { - return checkVolumeNodeAffinity(pv, &v1.Node{ObjectMeta: metav1.ObjectMeta{Labels: nodeLabels}}) -} - -func checkVolumeNodeAffinity(pv *v1.PersistentVolume, node *v1.Node) error { if pv.Spec.NodeAffinity == nil { return nil } if pv.Spec.NodeAffinity.Required != nil { + node := &v1.Node{ObjectMeta: metav1.ObjectMeta{Labels: nodeLabels}} terms := pv.Spec.NodeAffinity.Required if matches, err := corev1.MatchNodeSelectorTerms(node, terms); err != nil { return err From 438d224f0e846339c38347f29a83d230777cba23 Mon Sep 17 00:00:00 2001 From: Konstantin Misyutin Date: Thu, 11 Nov 2021 15:47:53 +0800 Subject: [PATCH 4/4] Cleanup k8s.io/component-helpers/storage/volume package Signed-off-by: Konstantin Misyutin --- .../volume/persistentvolume/index_test.go | 35 ++++++++++++++----- .../plugins/volumebinding/binder_test.go | 16 ++++++++- .../storage/volume/pv_helpers.go | 19 ---------- .../storage/volume/pv_helpers_test.go | 29 ++++++++++++--- 4 files changed, 66 insertions(+), 33 deletions(-) diff --git a/pkg/controller/volume/persistentvolume/index_test.go b/pkg/controller/volume/persistentvolume/index_test.go index 2d5bb791c20..a4b72acee36 100644 --- a/pkg/controller/volume/persistentvolume/index_test.go +++ b/pkg/controller/volume/persistentvolume/index_test.go @@ -447,6 +447,25 @@ func TestFindingVolumeWithDifferentAccessModes(t *testing.T) { } } +// createVolumeNodeAffinity returns a VolumeNodeAffinity for given key and value. +func createNodeAffinity(key string, value string) *v1.VolumeNodeAffinity { + return &v1.VolumeNodeAffinity{ + Required: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: key, + Operator: v1.NodeSelectorOpIn, + Values: []string{value}, + }, + }, + }, + }, + }, + } +} + func createTestVolumes() []*v1.PersistentVolume { fs := v1.PersistentVolumeFilesystem // these volumes are deliberately out-of-order to test indexing and sorting @@ -770,7 +789,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadOnlyMany, }, StorageClassName: classWait, - NodeAffinity: volume.GetVolumeNodeAffinity("key1", "value1"), + NodeAffinity: createNodeAffinity("key1", "value1"), VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ @@ -794,7 +813,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadOnlyMany, }, StorageClassName: classWait, - NodeAffinity: volume.GetVolumeNodeAffinity("key1", "value1"), + NodeAffinity: createNodeAffinity("key1", "value1"), VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ @@ -819,7 +838,7 @@ func createTestVolumes() []*v1.PersistentVolume { }, StorageClassName: classWait, ClaimRef: &v1.ObjectReference{Name: "claim02", Namespace: "myns"}, - NodeAffinity: volume.GetVolumeNodeAffinity("key1", "value1"), + NodeAffinity: createNodeAffinity("key1", "value1"), VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ @@ -843,7 +862,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadOnlyMany, }, StorageClassName: classWait, - NodeAffinity: volume.GetVolumeNodeAffinity("key1", "value3"), + NodeAffinity: createNodeAffinity("key1", "value3"), VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ @@ -867,7 +886,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadOnlyMany, }, StorageClassName: classWait, - NodeAffinity: volume.GetVolumeNodeAffinity("key1", "value4"), + NodeAffinity: createNodeAffinity("key1", "value4"), VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ @@ -891,7 +910,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadOnlyMany, }, StorageClassName: classWait, - NodeAffinity: volume.GetVolumeNodeAffinity("key1", "value4"), + NodeAffinity: createNodeAffinity("key1", "value4"), VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ @@ -915,7 +934,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadOnlyMany, }, StorageClassName: classWait, - NodeAffinity: volume.GetVolumeNodeAffinity("key1", "value4"), + NodeAffinity: createNodeAffinity("key1", "value4"), VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ @@ -939,7 +958,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadOnlyMany, }, StorageClassName: classWait, - NodeAffinity: volume.GetVolumeNodeAffinity("key1", "value4"), + NodeAffinity: createNodeAffinity("key1", "value4"), VolumeMode: &fs, }, }, diff --git a/pkg/scheduler/framework/plugins/volumebinding/binder_test.go b/pkg/scheduler/framework/plugins/volumebinding/binder_test.go index 707f26ba06b..cedc27d6f31 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/binder_test.go +++ b/pkg/scheduler/framework/plugins/volumebinding/binder_test.go @@ -666,7 +666,21 @@ func makeTestPV(name, node, capacity, version string, boundToPVC *v1.PersistentV }, } if node != "" { - pv.Spec.NodeAffinity = volume.GetVolumeNodeAffinity(nodeLabelKey, node) + pv.Spec.NodeAffinity = &v1.VolumeNodeAffinity{ + Required: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: nodeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{node}, + }, + }, + }, + }, + }, + } } if boundToPVC != nil { diff --git a/staging/src/k8s.io/component-helpers/storage/volume/pv_helpers.go b/staging/src/k8s.io/component-helpers/storage/volume/pv_helpers.go index cf18af43a37..f927b72314a 100644 --- a/staging/src/k8s.io/component-helpers/storage/volume/pv_helpers.go +++ b/staging/src/k8s.io/component-helpers/storage/volume/pv_helpers.go @@ -340,22 +340,3 @@ func CheckAccessModes(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolu func claimToClaimKey(claim *v1.PersistentVolumeClaim) string { return fmt.Sprintf("%s/%s", claim.Namespace, claim.Name) } - -// GetVolumeNodeAffinity returns a VolumeNodeAffinity for given key and value. -func GetVolumeNodeAffinity(key string, value string) *v1.VolumeNodeAffinity { - return &v1.VolumeNodeAffinity{ - Required: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: key, - Operator: v1.NodeSelectorOpIn, - Values: []string{value}, - }, - }, - }, - }, - }, - } -} diff --git a/staging/src/k8s.io/component-helpers/storage/volume/pv_helpers_test.go b/staging/src/k8s.io/component-helpers/storage/volume/pv_helpers_test.go index 74badfd01ab..6482ecfa6fc 100644 --- a/staging/src/k8s.io/component-helpers/storage/volume/pv_helpers_test.go +++ b/staging/src/k8s.io/component-helpers/storage/volume/pv_helpers_test.go @@ -119,6 +119,25 @@ func TestDelayBindingMode(t *testing.T) { } } +// makeVolumeNodeAffinity returns a VolumeNodeAffinity for given key and value. +func makeNodeAffinity(key string, value string) *v1.VolumeNodeAffinity { + return &v1.VolumeNodeAffinity{ + Required: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: key, + Operator: v1.NodeSelectorOpIn, + Values: []string{value}, + }, + }, + }, + }, + }, + } +} + func TestFindMatchVolumeWithNode(t *testing.T) { volumes := []*v1.PersistentVolume{ makeTestVolume("local-small", "local001", "5G", true, nil), @@ -127,24 +146,24 @@ func TestFindMatchVolumeWithNode(t *testing.T) { }), makeTestVolume("affinity-pv", "affinity001", "100G", true, func(pv *v1.PersistentVolume) { pv.Spec.StorageClassName = "wait" - pv.Spec.NodeAffinity = GetVolumeNodeAffinity("key1", "value1") + pv.Spec.NodeAffinity = makeNodeAffinity("key1", "value1") }), makeTestVolume("affinity-pv2", "affinity002", "150G", true, func(pv *v1.PersistentVolume) { pv.Spec.StorageClassName = "wait" - pv.Spec.NodeAffinity = GetVolumeNodeAffinity("key1", "value1") + pv.Spec.NodeAffinity = makeNodeAffinity("key1", "value1") }), makeTestVolume("affinity-prebound", "affinity003", "100G", true, func(pv *v1.PersistentVolume) { pv.Spec.StorageClassName = "wait" pv.Spec.ClaimRef = &v1.ObjectReference{Name: "claim02", Namespace: "myns"} - pv.Spec.NodeAffinity = GetVolumeNodeAffinity("key1", "value1") + pv.Spec.NodeAffinity = makeNodeAffinity("key1", "value1") }), makeTestVolume("affinity-pv3", "affinity003", "200G", true, func(pv *v1.PersistentVolume) { pv.Spec.StorageClassName = "wait" - pv.Spec.NodeAffinity = GetVolumeNodeAffinity("key1", "value3") + pv.Spec.NodeAffinity = makeNodeAffinity("key1", "value3") }), makeTestVolume("affinity-pv4", "affinity004", "200G", false, func(pv *v1.PersistentVolume) { pv.Spec.StorageClassName = "wait" - pv.Spec.NodeAffinity = GetVolumeNodeAffinity("key1", "value4") + pv.Spec.NodeAffinity = makeNodeAffinity("key1", "value4") }), }