From affcd0128b5e47688a45596de59352af1fcdddb8 Mon Sep 17 00:00:00 2001 From: Fabio Bertinatto Date: Wed, 16 Oct 2019 12:58:28 +0200 Subject: [PATCH 1/4] Promote volume limits to GA --- pkg/features/kube_features.go | 4 +- pkg/kubelet/kubelet_node_status.go | 8 +-- pkg/registry/storage/csinode/BUILD | 5 -- pkg/registry/storage/csinode/strategy.go | 33 ---------- pkg/registry/storage/csinode/strategy_test.go | 64 ++----------------- .../predicates/csi_volume_predicate.go | 6 -- .../predicates/csi_volume_predicate_test.go | 1 - .../max_attachable_volume_predicate_test.go | 29 --------- .../algorithm/predicates/predicates.go | 8 +-- .../apis/config/testing/compatibility_test.go | 24 ------- .../plugins/nodevolumelimits/azure_test.go | 5 -- .../plugins/nodevolumelimits/cinder_test.go | 5 -- .../plugins/nodevolumelimits/csi_test.go | 1 - .../plugins/nodevolumelimits/ebs_test.go | 5 -- .../plugins/nodevolumelimits/gce_test.go | 5 -- .../csi/nodeinfomanager/nodeinfomanager.go | 18 +++--- .../nodeinfomanager/nodeinfomanager_test.go | 1 - .../authorizer/rbac/bootstrappolicy/policy.go | 6 +- 18 files changed, 23 insertions(+), 205 deletions(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 3158286c859..105dc0795cc 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -282,6 +282,8 @@ const ( // owner: @gnufied // beta : v1.12 + // GA : v1.17 + // // Add support for volume plugins to report node specific // volume limits @@ -523,7 +525,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS ExpandPersistentVolumes: {Default: true, PreRelease: featuregate.Beta}, ExpandInUsePersistentVolumes: {Default: true, PreRelease: featuregate.Beta}, ExpandCSIVolumes: {Default: true, PreRelease: featuregate.Beta}, - AttachVolumeLimit: {Default: true, PreRelease: featuregate.Beta}, + AttachVolumeLimit: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.19 CPUManager: {Default: true, PreRelease: featuregate.Beta}, CPUCFSQuotaPeriod: {Default: false, PreRelease: featuregate.Alpha}, TopologyManager: {Default: false, PreRelease: featuregate.Alpha}, diff --git a/pkg/kubelet/kubelet_node_status.go b/pkg/kubelet/kubelet_node_status.go index af41b4856a7..f56817ebcd9 100644 --- a/pkg/kubelet/kubelet_node_status.go +++ b/pkg/kubelet/kubelet_node_status.go @@ -31,12 +31,10 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - utilfeature "k8s.io/apiserver/pkg/util/feature" cloudprovider "k8s.io/cloud-provider" "k8s.io/klog" k8s_api_v1 "k8s.io/kubernetes/pkg/apis/core/v1" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" - "k8s.io/kubernetes/pkg/features" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/kubelet/nodestatus" @@ -548,9 +546,9 @@ func (kl *Kubelet) defaultNodeStatusFuncs() []func(*v1.Node) error { nodestatus.Images(kl.nodeStatusMaxImages, kl.imageManager.GetImageList), nodestatus.GoRuntime(), ) - if utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { - setters = append(setters, nodestatus.VolumeLimits(kl.volumePluginMgr.ListVolumePluginWithLimits)) - } + // Volume limits + setters = append(setters, nodestatus.VolumeLimits(kl.volumePluginMgr.ListVolumePluginWithLimits)) + setters = append(setters, nodestatus.MemoryPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderMemoryPressure, kl.recordNodeStatusEvent), nodestatus.DiskPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderDiskPressure, kl.recordNodeStatusEvent), diff --git a/pkg/registry/storage/csinode/BUILD b/pkg/registry/storage/csinode/BUILD index 1c90ff26a87..f14adabc140 100644 --- a/pkg/registry/storage/csinode/BUILD +++ b/pkg/registry/storage/csinode/BUILD @@ -12,11 +12,9 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/storage:go_default_library", "//pkg/apis/storage/validation:go_default_library", - "//pkg/features:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", ], ) @@ -43,12 +41,9 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/apis/storage:go_default_library", - "//pkg/features:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", "//vendor/k8s.io/utils/pointer:go_default_library", ], ) diff --git a/pkg/registry/storage/csinode/strategy.go b/pkg/registry/storage/csinode/strategy.go index 160f6d5f566..d7ac8871e4f 100644 --- a/pkg/registry/storage/csinode/strategy.go +++ b/pkg/registry/storage/csinode/strategy.go @@ -22,11 +22,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/storage/names" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/storage" "k8s.io/kubernetes/pkg/apis/storage/validation" - "k8s.io/kubernetes/pkg/features" ) // csiNodeStrategy implements behavior for CSINode objects @@ -45,12 +43,6 @@ func (csiNodeStrategy) NamespaceScoped() bool { // PrepareForCreate clears fields that are not allowed to be set on creation. func (csiNodeStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) { - csiNode := obj.(*storage.CSINode) - if !utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { - for i := range csiNode.Spec.Drivers { - csiNode.Spec.Drivers[i].Allocatable = nil - } - } } func (csiNodeStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList { @@ -72,31 +64,6 @@ func (csiNodeStrategy) AllowCreateOnUpdate() bool { // PrepareForUpdate sets the driver's Allocatable fields that are not allowed to be set by an end user updating a CSINode. func (csiNodeStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) { - newCSINode := obj.(*storage.CSINode) - oldCSINode := old.(*storage.CSINode) - - inUse := getAllocatablesInUse(oldCSINode) - - if !utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { - for i := range newCSINode.Spec.Drivers { - if !inUse[newCSINode.Spec.Drivers[i].Name] { - newCSINode.Spec.Drivers[i].Allocatable = nil - } - } - } -} - -func getAllocatablesInUse(obj *storage.CSINode) map[string]bool { - inUse := make(map[string]bool) - if obj == nil { - return inUse - } - for i := range obj.Spec.Drivers { - if obj.Spec.Drivers[i].Allocatable != nil { - inUse[obj.Spec.Drivers[i].Name] = true - } - } - return inUse } func (csiNodeStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList { diff --git a/pkg/registry/storage/csinode/strategy_test.go b/pkg/registry/storage/csinode/strategy_test.go index 07dee6b73b0..e228a63a8d5 100644 --- a/pkg/registry/storage/csinode/strategy_test.go +++ b/pkg/registry/storage/csinode/strategy_test.go @@ -23,10 +23,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/apis/storage" - "k8s.io/kubernetes/pkg/features" utilpointer "k8s.io/utils/pointer" ) @@ -47,7 +44,7 @@ func TestPrepareForCreate(t *testing.T) { }, } - volumeLimitsEnabledCases := []struct { + volumeLimitsCases := []struct { name string obj *storage.CSINode expected *storage.CSINode @@ -64,32 +61,7 @@ func TestPrepareForCreate(t *testing.T) { }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() - for _, test := range volumeLimitsEnabledCases { - t.Run(test.name, func(t *testing.T) { - testPrepareForCreate(t, test.obj, test.expected) - }) - } - - volumeLimitsDisabledCases := []struct { - name string - obj *storage.CSINode - expected *storage.CSINode - }{ - { - "empty allocatable", - emptyAllocatable, - emptyAllocatable, - }, - { - "drop allocatable", - valid, - emptyAllocatable, - }, - } - - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, false)() - for _, test := range volumeLimitsDisabledCases { + for _, test := range volumeLimitsCases { t.Run(test.name, func(t *testing.T) { testPrepareForCreate(t, test.obj, test.expected) }) @@ -140,7 +112,7 @@ func TestPrepareForUpdate(t *testing.T) { }, } - volumeLimitsEnabledCases := []struct { + volumeLimitsCases := []struct { name string old *storage.CSINode new *storage.CSINode @@ -166,35 +138,7 @@ func TestPrepareForUpdate(t *testing.T) { }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() - for _, test := range volumeLimitsEnabledCases { - t.Run(test.name, func(t *testing.T) { - testPrepareForUpdate(t, test.new, test.old, test.expected) - }) - } - - volumeLimitsDisabledCases := []struct { - name string - old *storage.CSINode - new *storage.CSINode - expected *storage.CSINode - }{ - { - "allow empty allocatable when it's not set", - emptyAllocatable, - emptyAllocatable, - emptyAllocatable, - }, - { - "drop allocatable when it's not set", - emptyAllocatable, - valid, - emptyAllocatable, - }, - } - - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, false)() - for _, test := range volumeLimitsDisabledCases { + for _, test := range volumeLimitsCases { t.Run(test.name, func(t *testing.T) { testPrepareForUpdate(t, test.new, test.old, test.expected) }) diff --git a/pkg/scheduler/algorithm/predicates/csi_volume_predicate.go b/pkg/scheduler/algorithm/predicates/csi_volume_predicate.go index 7d6236f72ab..46809a3bf1f 100644 --- a/pkg/scheduler/algorithm/predicates/csi_volume_predicate.go +++ b/pkg/scheduler/algorithm/predicates/csi_volume_predicate.go @@ -22,14 +22,12 @@ import ( v1 "k8s.io/api/core/v1" storagev1beta1 "k8s.io/api/storage/v1beta1" "k8s.io/apimachinery/pkg/util/rand" - utilfeature "k8s.io/apiserver/pkg/util/feature" corelisters "k8s.io/client-go/listers/core/v1" storagelisters "k8s.io/client-go/listers/storage/v1" v1beta1storagelisters "k8s.io/client-go/listers/storage/v1beta1" csitrans "k8s.io/csi-translation-lib" "k8s.io/klog" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" - "k8s.io/kubernetes/pkg/features" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" volumeutil "k8s.io/kubernetes/pkg/volume/util" ) @@ -93,10 +91,6 @@ func (c *CSIMaxVolumeLimitChecker) attachableLimitPredicate( return true, nil, nil } - if !utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { - return true, nil, nil - } - node := nodeInfo.Node() if node == nil { return false, nil, fmt.Errorf("node not found") diff --git a/pkg/scheduler/algorithm/predicates/csi_volume_predicate_test.go b/pkg/scheduler/algorithm/predicates/csi_volume_predicate_test.go index cee8bf8c861..9ec8c0d11fb 100644 --- a/pkg/scheduler/algorithm/predicates/csi_volume_predicate_test.go +++ b/pkg/scheduler/algorithm/predicates/csi_volume_predicate_test.go @@ -434,7 +434,6 @@ func TestCSIVolumeCountPredicate(t *testing.T) { }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() // running attachable predicate tests with feature gate and limit present on nodes for _, test := range tests { t.Run(test.test, func(t *testing.T) { diff --git a/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go b/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go index 6f69a4b5c9f..373e979c464 100644 --- a/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go +++ b/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go @@ -19,7 +19,6 @@ package predicates import ( "os" "reflect" - "strconv" "strings" "testing" @@ -27,10 +26,7 @@ import ( "k8s.io/api/storage/v1beta1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" csilibplugins "k8s.io/csi-translation-lib/plugins" - "k8s.io/kubernetes/pkg/features" fakelisters "k8s.io/kubernetes/pkg/scheduler/listers/fake" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" volumeutil "k8s.io/kubernetes/pkg/volume/util" @@ -848,31 +844,6 @@ func TestVolumeCountConflicts(t *testing.T) { expectedFailureReasons := []PredicateFailureReason{ErrMaxVolumeCountExceeded} - // running attachable predicate tests without feature gate and no limit present on nodes - for _, test := range tests { - os.Setenv(KubeMaxPDVols, strconv.Itoa(test.maxVols)) - node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) - pred := NewMaxPDVolumeCountPredicate(test.filterName, - getFakeCSINodeLister(csiNode), - getFakeStorageClassLister(test.filterName), - getFakePVLister(test.filterName), - getFakePVCLister(test.filterName)) - - factory := &MetadataProducerFactory{} - fits, reasons, err := pred(test.newPod, factory.GetPredicateMetadata(test.newPod, nil), node) - if err != nil { - t.Errorf("[%s]%s: unexpected error: %v", test.filterName, test.test, err) - } - if !fits && !reflect.DeepEqual(reasons, expectedFailureReasons) { - t.Errorf("[%s]%s: unexpected failure reasons: %v, want: %v", test.filterName, test.test, reasons, expectedFailureReasons) - } - if fits != test.fits { - t.Errorf("[%s]%s: expected %v, got %v", test.filterName, test.test, test.fits, fits) - } - } - - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() - // running attachable predicate tests with feature gate and limit present on nodes for _, test := range tests { node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) diff --git a/pkg/scheduler/algorithm/predicates/predicates.go b/pkg/scheduler/algorithm/predicates/predicates.go index 0354d08c9c0..c3d961ddbb9 100644 --- a/pkg/scheduler/algorithm/predicates/predicates.go +++ b/pkg/scheduler/algorithm/predicates/predicates.go @@ -477,11 +477,9 @@ func (c *MaxPDVolumeCountChecker) predicate(pod *v1.Pod, meta Metadata, nodeInfo numNewVolumes := len(newVolumes) maxAttachLimit := c.maxVolumeFunc(node) - if utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { - volumeLimits := nodeInfo.VolumeLimits() - if maxAttachLimitFromAllocatable, ok := volumeLimits[c.volumeLimitKey]; ok { - maxAttachLimit = int(maxAttachLimitFromAllocatable) - } + volumeLimits := nodeInfo.VolumeLimits() + if maxAttachLimitFromAllocatable, ok := volumeLimits[c.volumeLimitKey]; ok { + maxAttachLimit = int(maxAttachLimitFromAllocatable) } if numExistingVolumes+numNewVolumes > maxAttachLimit { diff --git a/pkg/scheduler/apis/config/testing/compatibility_test.go b/pkg/scheduler/apis/config/testing/compatibility_test.go index 5fc6e1a39e7..53d5e4a8e9c 100644 --- a/pkg/scheduler/apis/config/testing/compatibility_test.go +++ b/pkg/scheduler/apis/config/testing/compatibility_test.go @@ -1169,30 +1169,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, }, }, - { - name: "disable beta feature AttachVolumeLimit", - JSON: `{ - "kind": "Policy", - "apiVersion": "v1", - "predicates": [ - {"name": "MaxCSIVolumeCountPred"}, - {"name": "CheckVolumeBinding"} - ], - "priorities": [ - ] - }`, - featureGates: map[featuregate.Feature]bool{ - features.AttachVolumeLimit: false, - }, - wantPlugins: map[string][]config.Plugin{ - "FilterPlugin": { - {Name: "NodeUnschedulable"}, - {Name: "TaintToleration"}, - {Name: "NodeVolumeLimits"}, - {Name: "VolumeBinding"}, - }, - }, - }, } registeredPredicates := sets.NewString(scheduler.ListRegisteredFitPredicates()...) registeredPriorities := sets.NewString(scheduler.ListRegisteredPriorityFunctions()...) diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/azure_test.go b/pkg/scheduler/framework/plugins/nodevolumelimits/azure_test.go index 8c97ea24374..575b9d07f35 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/azure_test.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/azure_test.go @@ -22,9 +22,6 @@ import ( "testing" v1 "k8s.io/api/core/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" ) @@ -355,8 +352,6 @@ func TestAzureDiskLimits(t *testing.T) { }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() - for _, test := range tests { t.Run(test.test, func(t *testing.T) { node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/cinder_test.go b/pkg/scheduler/framework/plugins/nodevolumelimits/cinder_test.go index 08d197d6ba6..20ebb14de82 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/cinder_test.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/cinder_test.go @@ -22,9 +22,6 @@ import ( "testing" v1 "k8s.io/api/core/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" ) @@ -84,8 +81,6 @@ func TestCinderLimits(t *testing.T) { }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() - for _, test := range tests { t.Run(test.test, func(t *testing.T) { node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go b/pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go index 2b9479f46d6..aa975f40f39 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go @@ -447,7 +447,6 @@ func TestCSILimits(t *testing.T) { }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() // running attachable predicate tests with feature gate and limit present on nodes for _, test := range tests { t.Run(test.test, func(t *testing.T) { diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/ebs_test.go b/pkg/scheduler/framework/plugins/nodevolumelimits/ebs_test.go index 39c1b1dd39d..c9aa0d89557 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/ebs_test.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/ebs_test.go @@ -24,10 +24,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" csilibplugins "k8s.io/csi-translation-lib/plugins" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" fakelisters "k8s.io/kubernetes/pkg/scheduler/listers/fake" @@ -469,8 +466,6 @@ func TestEBSLimits(t *testing.T) { }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() - for _, test := range tests { t.Run(test.test, func(t *testing.T) { node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/gce_test.go b/pkg/scheduler/framework/plugins/nodevolumelimits/gce_test.go index 414426561ec..957d3cfd485 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/gce_test.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/gce_test.go @@ -22,9 +22,6 @@ import ( "testing" v1 "k8s.io/api/core/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" ) @@ -355,8 +352,6 @@ func TestGCEPDLimits(t *testing.T) { }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() - for _, test := range tests { t.Run(test.test, func(t *testing.T) { node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) diff --git a/pkg/volume/csi/nodeinfomanager/nodeinfomanager.go b/pkg/volume/csi/nodeinfomanager/nodeinfomanager.go index b6f15bdd12a..b6920ba078d 100644 --- a/pkg/volume/csi/nodeinfomanager/nodeinfomanager.go +++ b/pkg/volume/csi/nodeinfomanager/nodeinfomanager.go @@ -556,17 +556,15 @@ func (nim *nodeInfoManager) installDriverToCSINode( TopologyKeys: topologyKeys.List(), } - if utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { - if maxAttachLimit > 0 { - if maxAttachLimit > math.MaxInt32 { - klog.Warningf("Exceeded max supported attach limit value, truncating it to %d", math.MaxInt32) - maxAttachLimit = math.MaxInt32 - } - m := int32(maxAttachLimit) - driverSpec.Allocatable = &storagev1.VolumeNodeResources{Count: &m} - } else { - klog.Errorf("Invalid attach limit value %d cannot be added to CSINode object for %q", maxAttachLimit, driverName) + if maxAttachLimit > 0 { + if maxAttachLimit > math.MaxInt32 { + klog.Warningf("Exceeded max supported attach limit value, truncating it to %d", math.MaxInt32) + maxAttachLimit = math.MaxInt32 } + m := int32(maxAttachLimit) + driverSpec.Allocatable = &storagev1.VolumeNodeResources{Count: &m} + } else { + klog.Errorf("Invalid attach limit value %d cannot be added to CSINode object for %q", maxAttachLimit, driverName) } newDriverSpecs = append(newDriverSpecs, driverSpec) diff --git a/pkg/volume/csi/nodeinfomanager/nodeinfomanager_test.go b/pkg/volume/csi/nodeinfomanager/nodeinfomanager_test.go index 22ba95e1326..95bff7507b4 100644 --- a/pkg/volume/csi/nodeinfomanager/nodeinfomanager_test.go +++ b/pkg/volume/csi/nodeinfomanager/nodeinfomanager_test.go @@ -1011,7 +1011,6 @@ func getClientSet(existingNode *v1.Node, existingCSINode *storage.CSINode) *fake func test(t *testing.T, addNodeInfo bool, csiNodeInfoEnabled bool, testcases []testcase) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSINodeInfo, csiNodeInfoEnabled)() - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() for _, tc := range testcases { t.Logf("test case: %q", tc.name) diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go index 2b6a0e42f47..10ba64be035 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go @@ -498,10 +498,8 @@ func ClusterRoles() []rbacv1.ClusterRole { // Needed to check API access. These creates are non-mutating rbacv1helpers.NewRule("create").Groups(authenticationGroup).Resources("tokenreviews").RuleOrDie(), rbacv1helpers.NewRule("create").Groups(authorizationGroup).Resources("subjectaccessreviews").RuleOrDie(), - } - if utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) && - utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { - kubeSchedulerRules = append(kubeSchedulerRules, rbacv1helpers.NewRule(Read...).Groups(storageGroup).Resources("csinodes").RuleOrDie()) + // Needed for volume limits + rbacv1helpers.NewRule(Read...).Groups(storageGroup).Resources("csinodes").RuleOrDie(), } roles = append(roles, rbacv1.ClusterRole{ // a role to use for the kube-scheduler From f6de8ab7273a1305bf4b03050d36fe34b3caf7f4 Mon Sep 17 00:00:00 2001 From: Fabio Bertinatto Date: Tue, 29 Oct 2019 13:08:24 +0100 Subject: [PATCH 2/4] Enable volume limits e2e tests for CSI hostpath driver --- test/e2e/storage/drivers/csi.go | 1 + test/e2e/storage/testsuites/volumelimits.go | 26 ++++++++++++--------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 853e9091719..836df3c3028 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -106,6 +106,7 @@ func InitHostPathCSIDriver() testsuites.TestDriver { testsuites.CapPVCDataSource: true, testsuites.CapControllerExpansion: true, testsuites.CapSingleNodeVolume: true, + testsuites.CapVolumeLimits: true, } return initHostPathCSIDriver("csi-hostpath", capabilities, diff --git a/test/e2e/storage/testsuites/volumelimits.go b/test/e2e/storage/testsuites/volumelimits.go index 3f7e3b9ec5c..0e67e95f5da 100644 --- a/test/e2e/storage/testsuites/volumelimits.go +++ b/test/e2e/storage/testsuites/volumelimits.go @@ -109,7 +109,7 @@ func (t *volumeLimitsTestSuite) defineTests(driver TestDriver, pattern testpatte // And one extra pod with a CSI volume should get Pending with a condition // that says it's unschedulable because of volume limit. // BEWARE: the test may create lot of volumes and it's really slow. - ginkgo.It("should support volume limits [Slow][Serial]", func() { + ginkgo.It("should support volume limits [Serial]", func() { driverInfo := driver.GetDriverInfo() if !driverInfo.Capabilities[CapVolumeLimits] { ginkgo.Skip(fmt.Sprintf("driver %s does not support volume limits", driverInfo.Name)) @@ -124,15 +124,19 @@ func (t *volumeLimitsTestSuite) defineTests(driver TestDriver, pattern testpatte l.config, l.testCleanup = driver.PrepareTest(f) defer l.testCleanup() - ginkgo.By("Picking a random node") - var nodeName string - node, err := e2enode.GetRandomReadySchedulableNode(f.ClientSet) - framework.ExpectNoError(err) - nodeName = node.Name + ginkgo.By("Picking a node") + // Some CSI drivers are deployed to a single node (e.g csi-hostpath), + // so we use that node instead of picking a random one. + nodeName := l.config.ClientNodeName + if nodeName == "" { + node, err := e2enode.GetRandomReadySchedulableNode(f.ClientSet) + framework.ExpectNoError(err) + nodeName = node.Name + } framework.Logf("Selected node %s", nodeName) ginkgo.By("Checking node limits") - limit, err := getNodeLimits(l.cs, nodeName, driverInfo) + limit, err := getNodeLimits(l.cs, l.config, nodeName, driverInfo) framework.ExpectNoError(err) framework.Logf("Node %s can handle %d volumes of driver %s", nodeName, limit, driverInfo.Name) @@ -283,9 +287,9 @@ func waitForAllPVCsPhase(cs clientset.Interface, timeout time.Duration, pvcs []* return pvNames, err } -func getNodeLimits(cs clientset.Interface, nodeName string, driverInfo *DriverInfo) (int, error) { +func getNodeLimits(cs clientset.Interface, config *PerTestConfig, nodeName string, driverInfo *DriverInfo) (int, error) { if len(driverInfo.InTreePluginName) == 0 { - return getCSINodeLimits(cs, nodeName, driverInfo) + return getCSINodeLimits(cs, config, nodeName, driverInfo) } return getInTreeNodeLimits(cs, nodeName, driverInfo) } @@ -317,7 +321,7 @@ func getInTreeNodeLimits(cs clientset.Interface, nodeName string, driverInfo *Dr return int(limit.Value()), nil } -func getCSINodeLimits(cs clientset.Interface, nodeName string, driverInfo *DriverInfo) (int, error) { +func getCSINodeLimits(cs clientset.Interface, config *PerTestConfig, nodeName string, driverInfo *DriverInfo) (int, error) { // Wait in a loop, the driver might just have been installed and kubelet takes a while to publish everything. var limit int err := wait.PollImmediate(2*time.Second, csiNodeInfoTimeout, func() (bool, error) { @@ -328,7 +332,7 @@ func getCSINodeLimits(cs clientset.Interface, nodeName string, driverInfo *Drive } var csiDriver *storagev1.CSINodeDriver for _, c := range csiNode.Spec.Drivers { - if c.Name == driverInfo.Name { + if c.Name == driverInfo.Name || c.Name == config.GetUniqueDriverName() { csiDriver = &c break } From bf0182b9d32e7a0293f7a526a58344c2d054ba75 Mon Sep 17 00:00:00 2001 From: Fabio Bertinatto Date: Mon, 4 Nov 2019 15:08:47 +0100 Subject: [PATCH 3/4] Update hostpath CSI driver manifest This is required for promoting volume limits to GA. The new version of the driver reports the max number of volumes it supports. Such number should be specified as a CLI argument when starting the driver. --- .../storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml index 2cac6097136..dcec409787a 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml @@ -62,12 +62,13 @@ spec: name: csi-data-dir - name: hostpath - image: quay.io/k8scsi/hostpathplugin:v1.2.0 + image: quay.io/k8scsi/hostpathplugin:v1.3.0-rc1 args: - "--drivername=hostpath.csi.k8s.io" - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" + - "--maxvolumespernode=10" env: - name: CSI_ENDPOINT value: unix:///csi/csi.sock From a59abc3cc381f563743d60ac54ddc922b4e007b3 Mon Sep 17 00:00:00 2001 From: Fabio Bertinatto Date: Mon, 11 Nov 2019 10:36:33 +0100 Subject: [PATCH 4/4] Convert scheduler to use CSINode GA --- pkg/scheduler/BUILD | 2 -- pkg/scheduler/algorithm/predicates/BUILD | 3 --- .../predicates/csi_volume_predicate.go | 15 +++++++------ .../predicates/csi_volume_predicate_test.go | 6 +++--- .../max_attachable_volume_predicate_test.go | 16 +++++++------- .../algorithm/predicates/predicates.go | 18 +++++++--------- pkg/scheduler/algorithm/predicates/utils.go | 6 +++--- pkg/scheduler/algorithm_factory.go | 3 +-- pkg/scheduler/factory.go | 8 +++---- pkg/scheduler/factory_test.go | 2 +- .../framework/plugins/nodevolumelimits/BUILD | 4 ++-- .../plugins/nodevolumelimits/csi_test.go | 21 +++++++++---------- .../nodevolumelimits/csinode_helper.go | 6 +++--- pkg/scheduler/listers/fake/BUILD | 2 -- pkg/scheduler/listers/fake/listers.go | 14 ++++++------- pkg/scheduler/scheduler.go | 6 +++--- 16 files changed, 58 insertions(+), 74 deletions(-) diff --git a/pkg/scheduler/BUILD b/pkg/scheduler/BUILD index 91f86930bb6..8055bc6f4da 100644 --- a/pkg/scheduler/BUILD +++ b/pkg/scheduler/BUILD @@ -48,13 +48,11 @@ go_library( "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/informers/policy/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/informers/storage/v1:go_default_library", - "//staging/src/k8s.io/client-go/informers/storage/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/listers/apps/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/policy/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/listers/storage/v1:go_default_library", - "//staging/src/k8s.io/client-go/listers/storage/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/events:go_default_library", "//vendor/k8s.io/klog:go_default_library", diff --git a/pkg/scheduler/algorithm/predicates/BUILD b/pkg/scheduler/algorithm/predicates/BUILD index f457e2daed8..8b8d1a9a685 100644 --- a/pkg/scheduler/algorithm/predicates/BUILD +++ b/pkg/scheduler/algorithm/predicates/BUILD @@ -28,7 +28,6 @@ go_library( "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", - "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", @@ -37,7 +36,6 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/storage/v1:go_default_library", - "//staging/src/k8s.io/client-go/listers/storage/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", "//staging/src/k8s.io/cloud-provider/volume/helpers:go_default_library", "//staging/src/k8s.io/csi-translation-lib:go_default_library", @@ -67,7 +65,6 @@ go_test( "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", - "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", diff --git a/pkg/scheduler/algorithm/predicates/csi_volume_predicate.go b/pkg/scheduler/algorithm/predicates/csi_volume_predicate.go index 46809a3bf1f..d853aa14dc7 100644 --- a/pkg/scheduler/algorithm/predicates/csi_volume_predicate.go +++ b/pkg/scheduler/algorithm/predicates/csi_volume_predicate.go @@ -20,11 +20,10 @@ import ( "fmt" v1 "k8s.io/api/core/v1" - storagev1beta1 "k8s.io/api/storage/v1beta1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/util/rand" corelisters "k8s.io/client-go/listers/core/v1" storagelisters "k8s.io/client-go/listers/storage/v1" - v1beta1storagelisters "k8s.io/client-go/listers/storage/v1beta1" csitrans "k8s.io/csi-translation-lib" "k8s.io/klog" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" @@ -44,7 +43,7 @@ type InTreeToCSITranslator interface { // CSIMaxVolumeLimitChecker defines predicate needed for counting CSI volumes type CSIMaxVolumeLimitChecker struct { - csiNodeLister v1beta1storagelisters.CSINodeLister + csiNodeLister storagelisters.CSINodeLister pvLister corelisters.PersistentVolumeLister pvcLister corelisters.PersistentVolumeClaimLister scLister storagelisters.StorageClassLister @@ -56,7 +55,7 @@ type CSIMaxVolumeLimitChecker struct { // NewCSIMaxVolumeLimitPredicate returns a predicate for counting CSI volumes func NewCSIMaxVolumeLimitPredicate( - csiNodeLister v1beta1storagelisters.CSINodeLister, pvLister corelisters.PersistentVolumeLister, pvcLister corelisters.PersistentVolumeClaimLister, scLister storagelisters.StorageClassLister) FitPredicate { + csiNodeLister storagelisters.CSINodeLister, pvLister corelisters.PersistentVolumeLister, pvcLister corelisters.PersistentVolumeClaimLister, scLister storagelisters.StorageClassLister) FitPredicate { c := &CSIMaxVolumeLimitChecker{ csiNodeLister: csiNodeLister, pvLister: pvLister, @@ -68,7 +67,7 @@ func NewCSIMaxVolumeLimitPredicate( return c.attachableLimitPredicate } -func getVolumeLimits(nodeInfo *schedulernodeinfo.NodeInfo, csiNode *storagev1beta1.CSINode) map[v1.ResourceName]int64 { +func getVolumeLimits(nodeInfo *schedulernodeinfo.NodeInfo, csiNode *storagev1.CSINode) map[v1.ResourceName]int64 { // TODO: stop getting values from Node object in v1.18 nodeVolumeLimits := nodeInfo.VolumeLimits() if csiNode != nil { @@ -154,7 +153,7 @@ func (c *CSIMaxVolumeLimitChecker) attachableLimitPredicate( } func (c *CSIMaxVolumeLimitChecker) filterAttachableVolumes( - csiNode *storagev1beta1.CSINode, volumes []v1.Volume, namespace string, result map[string]string) error { + csiNode *storagev1.CSINode, volumes []v1.Volume, namespace string, result map[string]string) error { for _, vol := range volumes { // CSI volumes can only be used as persistent volumes if vol.PersistentVolumeClaim == nil { @@ -189,7 +188,7 @@ func (c *CSIMaxVolumeLimitChecker) filterAttachableVolumes( // getCSIDriverInfo returns the CSI driver name and volume ID of a given PVC. // If the PVC is from a migrated in-tree plugin, this function will return // the information of the CSI driver that the plugin has been migrated to. -func (c *CSIMaxVolumeLimitChecker) getCSIDriverInfo(csiNode *storagev1beta1.CSINode, pvc *v1.PersistentVolumeClaim) (string, string) { +func (c *CSIMaxVolumeLimitChecker) getCSIDriverInfo(csiNode *storagev1.CSINode, pvc *v1.PersistentVolumeClaim) (string, string) { pvName := pvc.Spec.VolumeName namespace := pvc.Namespace pvcName := pvc.Name @@ -244,7 +243,7 @@ func (c *CSIMaxVolumeLimitChecker) getCSIDriverInfo(csiNode *storagev1beta1.CSIN } // getCSIDriverInfoFromSC returns the CSI driver name and a random volume ID of a given PVC's StorageClass. -func (c *CSIMaxVolumeLimitChecker) getCSIDriverInfoFromSC(csiNode *storagev1beta1.CSINode, pvc *v1.PersistentVolumeClaim) (string, string) { +func (c *CSIMaxVolumeLimitChecker) getCSIDriverInfoFromSC(csiNode *storagev1.CSINode, pvc *v1.PersistentVolumeClaim) (string, string) { namespace := pvc.Namespace pvcName := pvc.Name scName := v1helper.GetPersistentVolumeClaimClass(pvc) diff --git a/pkg/scheduler/algorithm/predicates/csi_volume_predicate_test.go b/pkg/scheduler/algorithm/predicates/csi_volume_predicate_test.go index 9ec8c0d11fb..0fbfbe0685e 100644 --- a/pkg/scheduler/algorithm/predicates/csi_volume_predicate_test.go +++ b/pkg/scheduler/algorithm/predicates/csi_volume_predicate_test.go @@ -23,7 +23,7 @@ import ( "testing" v1 "k8s.io/api/core/v1" - storagev1beta1 "k8s.io/api/storage/v1beta1" + storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -546,7 +546,7 @@ func getFakeCSIPVCLister(volumeName, scName string, driverNames ...string) fakel return pvcLister } -func enableMigrationOnNode(csiNode *storagev1beta1.CSINode, pluginName string) { +func enableMigrationOnNode(csiNode *storagev1.CSINode, pluginName string) { nodeInfoAnnotations := csiNode.GetAnnotations() if nodeInfoAnnotations == nil { nodeInfoAnnotations = map[string]string{} @@ -569,7 +569,7 @@ func getFakeCSIStorageClassLister(scName, provisionerName string) fakelisters.St } } -func getFakeCSINodeLister(csiNode *storagev1beta1.CSINode) fakelisters.CSINodeLister { +func getFakeCSINodeLister(csiNode *storagev1.CSINode) fakelisters.CSINodeLister { if csiNode != nil { return fakelisters.CSINodeLister(*csiNode) } diff --git a/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go b/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go index 373e979c464..b8f35919cb9 100644 --- a/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go +++ b/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go @@ -23,7 +23,7 @@ import ( "testing" v1 "k8s.io/api/core/v1" - "k8s.io/api/storage/v1beta1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" csilibplugins "k8s.io/csi-translation-lib/plugins" @@ -1075,7 +1075,7 @@ func TestMaxVolumeFuncM4WithBothBetaAndStableLabels(t *testing.T) { } } -func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int64, driverNames ...string) (*schedulernodeinfo.NodeInfo, *v1beta1.CSINode) { +func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int64, driverNames ...string) (*schedulernodeinfo.NodeInfo, *storagev1.CSINode) { nodeInfo := schedulernodeinfo.NewNodeInfo(pods...) node := &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: "node-for-max-pd-test-1"}, @@ -1083,7 +1083,7 @@ func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int Allocatable: v1.ResourceList{}, }, } - var csiNode *v1beta1.CSINode + var csiNode *storagev1.CSINode addLimitToNode := func() { for _, driver := range driverNames { @@ -1092,10 +1092,10 @@ func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int } initCSINode := func() { - csiNode = &v1beta1.CSINode{ + csiNode = &storagev1.CSINode{ ObjectMeta: metav1.ObjectMeta{Name: "csi-node-for-max-pd-test-1"}, - Spec: v1beta1.CSINodeSpec{ - Drivers: []v1beta1.CSINodeDriver{}, + Spec: storagev1.CSINodeSpec{ + Drivers: []storagev1.CSINodeDriver{}, }, } } @@ -1103,12 +1103,12 @@ func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int addDriversCSINode := func(addLimits bool) { initCSINode() for _, driver := range driverNames { - driver := v1beta1.CSINodeDriver{ + driver := storagev1.CSINodeDriver{ Name: driver, NodeID: "node-for-max-pd-test-1", } if addLimits { - driver.Allocatable = &v1beta1.VolumeNodeResources{ + driver.Allocatable = &storagev1.VolumeNodeResources{ Count: utilpointer.Int32Ptr(int32(limit)), } } diff --git a/pkg/scheduler/algorithm/predicates/predicates.go b/pkg/scheduler/algorithm/predicates/predicates.go index c3d961ddbb9..210d14277c2 100644 --- a/pkg/scheduler/algorithm/predicates/predicates.go +++ b/pkg/scheduler/algorithm/predicates/predicates.go @@ -27,7 +27,6 @@ import ( v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" - v1beta1storage "k8s.io/api/storage/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" @@ -36,7 +35,6 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" corelisters "k8s.io/client-go/listers/core/v1" storagelisters "k8s.io/client-go/listers/storage/v1" - v1beta1storagelisters "k8s.io/client-go/listers/storage/v1beta1" volumehelpers "k8s.io/cloud-provider/volume/helpers" csilibplugins "k8s.io/csi-translation-lib/plugins" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" @@ -225,7 +223,7 @@ type MaxPDVolumeCountChecker struct { filter VolumeFilter volumeLimitKey v1.ResourceName maxVolumeFunc func(node *v1.Node) int - csiNodeLister v1beta1storagelisters.CSINodeLister + csiNodeLister storagelisters.CSINodeLister pvLister corelisters.PersistentVolumeLister pvcLister corelisters.PersistentVolumeClaimLister scLister storagelisters.StorageClassLister @@ -244,7 +242,7 @@ type VolumeFilter struct { // MatchProvisioner evaluates if the StorageClass provisioner matches the running predicate MatchProvisioner func(sc *storage.StorageClass) (relevant bool) // IsMigrated returns a boolean specifying whether the plugin is migrated to a CSI driver - IsMigrated func(csiNode *v1beta1storage.CSINode) bool + IsMigrated func(csiNode *storage.CSINode) bool } // NewMaxPDVolumeCountPredicate creates a predicate which evaluates whether a pod can fit based on the @@ -257,7 +255,7 @@ type VolumeFilter struct { // The predicate looks for both volumes used directly, as well as PVC volumes that are backed by relevant volume // types, counts the number of unique volumes, and rejects the new pod if it would place the total count over // the maximum. -func NewMaxPDVolumeCountPredicate(filterName string, csiNodeLister v1beta1storagelisters.CSINodeLister, scLister storagelisters.StorageClassLister, +func NewMaxPDVolumeCountPredicate(filterName string, csiNodeLister storagelisters.CSINodeLister, scLister storagelisters.StorageClassLister, pvLister corelisters.PersistentVolumeLister, pvcLister corelisters.PersistentVolumeClaimLister) FitPredicate { var filter VolumeFilter var volumeLimitKey v1.ResourceName @@ -441,7 +439,7 @@ func (c *MaxPDVolumeCountChecker) predicate(pod *v1.Pod, meta Metadata, nodeInfo } var ( - csiNode *v1beta1storage.CSINode + csiNode *storage.CSINode err error ) if c.csiNodeLister != nil { @@ -518,7 +516,7 @@ var EBSVolumeFilter = VolumeFilter{ return false }, - IsMigrated: func(csiNode *v1beta1storage.CSINode) bool { + IsMigrated: func(csiNode *storage.CSINode) bool { return isCSIMigrationOn(csiNode, csilibplugins.AWSEBSInTreePluginName) }, } @@ -546,7 +544,7 @@ var GCEPDVolumeFilter = VolumeFilter{ return false }, - IsMigrated: func(csiNode *v1beta1storage.CSINode) bool { + IsMigrated: func(csiNode *storage.CSINode) bool { return isCSIMigrationOn(csiNode, csilibplugins.GCEPDInTreePluginName) }, } @@ -574,7 +572,7 @@ var AzureDiskVolumeFilter = VolumeFilter{ return false }, - IsMigrated: func(csiNode *v1beta1storage.CSINode) bool { + IsMigrated: func(csiNode *storage.CSINode) bool { return isCSIMigrationOn(csiNode, csilibplugins.AzureDiskInTreePluginName) }, } @@ -603,7 +601,7 @@ var CinderVolumeFilter = VolumeFilter{ return false }, - IsMigrated: func(csiNode *v1beta1storage.CSINode) bool { + IsMigrated: func(csiNode *storage.CSINode) bool { return isCSIMigrationOn(csiNode, csilibplugins.CinderInTreePluginName) }, } diff --git a/pkg/scheduler/algorithm/predicates/utils.go b/pkg/scheduler/algorithm/predicates/utils.go index 3115b4e8391..7a76a0cf10c 100644 --- a/pkg/scheduler/algorithm/predicates/utils.go +++ b/pkg/scheduler/algorithm/predicates/utils.go @@ -19,8 +19,8 @@ package predicates import ( "strings" - "k8s.io/api/core/v1" - storagev1beta1 "k8s.io/api/storage/v1beta1" + v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -97,7 +97,7 @@ func SetPredicatesOrderingDuringTest(value []string) func() { // isCSIMigrationOn returns a boolean value indicating whether // the CSI migration has been enabled for a particular storage plugin. -func isCSIMigrationOn(csiNode *storagev1beta1.CSINode, pluginName string) bool { +func isCSIMigrationOn(csiNode *storagev1.CSINode, pluginName string) bool { if csiNode == nil || len(pluginName) == 0 { return false } diff --git a/pkg/scheduler/algorithm_factory.go b/pkg/scheduler/algorithm_factory.go index 16743df9398..87b14f84494 100644 --- a/pkg/scheduler/algorithm_factory.go +++ b/pkg/scheduler/algorithm_factory.go @@ -29,7 +29,6 @@ import ( corelisters "k8s.io/client-go/listers/core/v1" policylisters "k8s.io/client-go/listers/policy/v1beta1" storagelisters "k8s.io/client-go/listers/storage/v1" - v1beta1storagelisters "k8s.io/client-go/listers/storage/v1beta1" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" @@ -53,7 +52,7 @@ type PluginFactoryArgs struct { ReplicaSetLister appslisters.ReplicaSetLister StatefulSetLister appslisters.StatefulSetLister PDBLister policylisters.PodDisruptionBudgetLister - CSINodeLister v1beta1storagelisters.CSINodeLister + CSINodeLister storagelisters.CSINodeLister PVLister corelisters.PersistentVolumeLister PVCLister corelisters.PersistentVolumeClaimLister StorageClassLister storagelisters.StorageClassLister diff --git a/pkg/scheduler/factory.go b/pkg/scheduler/factory.go index f3261372db0..945e9eb383e 100644 --- a/pkg/scheduler/factory.go +++ b/pkg/scheduler/factory.go @@ -34,13 +34,11 @@ import ( coreinformers "k8s.io/client-go/informers/core/v1" policyinformers "k8s.io/client-go/informers/policy/v1beta1" storageinformersv1 "k8s.io/client-go/informers/storage/v1" - storageinformersv1beta1 "k8s.io/client-go/informers/storage/v1beta1" clientset "k8s.io/client-go/kubernetes" appslisters "k8s.io/client-go/listers/apps/v1" corelisters "k8s.io/client-go/listers/core/v1" policylisters "k8s.io/client-go/listers/policy/v1beta1" storagelistersv1 "k8s.io/client-go/listers/storage/v1" - storagelistersv1beta1 "k8s.io/client-go/listers/storage/v1beta1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/events" "k8s.io/klog" @@ -134,7 +132,7 @@ type Configurator struct { // a means to list all StorageClasses storageClassLister storagelistersv1.StorageClassLister // a means to list all CSINodes - csiNodeLister storagelistersv1beta1.CSINodeLister + csiNodeLister storagelistersv1.CSINodeLister // a means to list all Nodes nodeLister corelisters.NodeLister // a means to list all Pods @@ -195,7 +193,7 @@ type ConfigFactoryArgs struct { ServiceInformer coreinformers.ServiceInformer PdbInformer policyinformers.PodDisruptionBudgetInformer StorageClassInformer storageinformersv1.StorageClassInformer - CSINodeInformer storageinformersv1beta1.CSINodeInformer + CSINodeInformer storageinformersv1.CSINodeInformer VolumeBinder *volumebinder.VolumeBinder SchedulerCache internalcache.Cache HardPodAffinitySymmetricWeight int32 @@ -225,7 +223,7 @@ func NewConfigFactory(args *ConfigFactoryArgs) *Configurator { storageClassLister = args.StorageClassInformer.Lister() } - var csiNodeLister storagelistersv1beta1.CSINodeLister + var csiNodeLister storagelistersv1.CSINodeLister if args.CSINodeInformer != nil { csiNodeLister = args.CSINodeInformer.Lister() } diff --git a/pkg/scheduler/factory_test.go b/pkg/scheduler/factory_test.go index a32afb2617f..97643d9c9de 100644 --- a/pkg/scheduler/factory_test.go +++ b/pkg/scheduler/factory_test.go @@ -552,7 +552,7 @@ func newConfigFactoryWithFrameworkRegistry( ServiceInformer: informerFactory.Core().V1().Services(), PdbInformer: informerFactory.Policy().V1beta1().PodDisruptionBudgets(), StorageClassInformer: informerFactory.Storage().V1().StorageClasses(), - CSINodeInformer: informerFactory.Storage().V1beta1().CSINodes(), + CSINodeInformer: informerFactory.Storage().V1().CSINodes(), HardPodAffinitySymmetricWeight: hardPodAffinitySymmetricWeight, DisablePreemption: disablePodPreemption, PercentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore, diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/BUILD b/pkg/scheduler/framework/plugins/nodevolumelimits/BUILD index 444a2bb985f..671040e97c2 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/BUILD +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/BUILD @@ -22,7 +22,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", - "//staging/src/k8s.io/client-go/listers/storage/v1beta1:go_default_library", + "//staging/src/k8s.io/client-go/listers/storage/v1:go_default_library", ], ) @@ -44,7 +44,7 @@ go_test( "//pkg/scheduler/nodeinfo:go_default_library", "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", + "//staging/src/k8s.io/api/storage/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go b/pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go index aa975f40f39..51149c7122a 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go @@ -24,8 +24,7 @@ import ( "testing" v1 "k8s.io/api/core/v1" - "k8s.io/api/storage/v1beta1" - storagev1beta1 "k8s.io/api/storage/v1beta1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -545,7 +544,7 @@ func getFakeCSIPVCLister(volumeName, scName string, driverNames ...string) fakel return pvcLister } -func enableMigrationOnNode(csiNode *storagev1beta1.CSINode, pluginName string) { +func enableMigrationOnNode(csiNode *storagev1.CSINode, pluginName string) { nodeInfoAnnotations := csiNode.GetAnnotations() if nodeInfoAnnotations == nil { nodeInfoAnnotations = map[string]string{} @@ -568,14 +567,14 @@ func getFakeCSIStorageClassLister(scName, provisionerName string) fakelisters.St } } -func getFakeCSINodeLister(csiNode *storagev1beta1.CSINode) fakelisters.CSINodeLister { +func getFakeCSINodeLister(csiNode *storagev1.CSINode) fakelisters.CSINodeLister { if csiNode != nil { return fakelisters.CSINodeLister(*csiNode) } return fakelisters.CSINodeLister{} } -func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int64, driverNames ...string) (*schedulernodeinfo.NodeInfo, *v1beta1.CSINode) { +func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int64, driverNames ...string) (*schedulernodeinfo.NodeInfo, *storagev1.CSINode) { nodeInfo := schedulernodeinfo.NewNodeInfo(pods...) node := &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: "node-for-max-pd-test-1"}, @@ -583,7 +582,7 @@ func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int Allocatable: v1.ResourceList{}, }, } - var csiNode *v1beta1.CSINode + var csiNode *storagev1.CSINode addLimitToNode := func() { for _, driver := range driverNames { @@ -592,10 +591,10 @@ func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int } initCSINode := func() { - csiNode = &v1beta1.CSINode{ + csiNode = &storagev1.CSINode{ ObjectMeta: metav1.ObjectMeta{Name: "csi-node-for-max-pd-test-1"}, - Spec: v1beta1.CSINodeSpec{ - Drivers: []v1beta1.CSINodeDriver{}, + Spec: storagev1.CSINodeSpec{ + Drivers: []storagev1.CSINodeDriver{}, }, } } @@ -603,12 +602,12 @@ func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int addDriversCSINode := func(addLimits bool) { initCSINode() for _, driver := range driverNames { - driver := v1beta1.CSINodeDriver{ + driver := storagev1.CSINodeDriver{ Name: driver, NodeID: "node-for-max-pd-test-1", } if addLimits { - driver.Allocatable = &v1beta1.VolumeNodeResources{ + driver.Allocatable = &storagev1.VolumeNodeResources{ Count: utilpointer.Int32Ptr(int32(limit)), } } diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/csinode_helper.go b/pkg/scheduler/framework/plugins/nodevolumelimits/csinode_helper.go index ef35ba9b8cd..196aa3db34d 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/csinode_helper.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/csinode_helper.go @@ -19,14 +19,14 @@ package nodevolumelimits import ( utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" - v1beta1 "k8s.io/client-go/listers/storage/v1beta1" + storagelisters "k8s.io/client-go/listers/storage/v1" kubefeatures "k8s.io/kubernetes/pkg/features" ) // getCSINodeListerIfEnabled returns the CSINode lister or nil if the feature is disabled -func getCSINodeListerIfEnabled(factory informers.SharedInformerFactory) v1beta1.CSINodeLister { +func getCSINodeListerIfEnabled(factory informers.SharedInformerFactory) storagelisters.CSINodeLister { if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CSINodeInfo) { return nil } - return factory.Storage().V1beta1().CSINodes().Lister() + return factory.Storage().V1().CSINodes().Lister() } diff --git a/pkg/scheduler/listers/fake/BUILD b/pkg/scheduler/listers/fake/BUILD index 57efb259970..b826290bdb4 100644 --- a/pkg/scheduler/listers/fake/BUILD +++ b/pkg/scheduler/listers/fake/BUILD @@ -11,13 +11,11 @@ go_library( "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", - "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/client-go/listers/apps/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/storage/v1:go_default_library", - "//staging/src/k8s.io/client-go/listers/storage/v1beta1:go_default_library", ], ) diff --git a/pkg/scheduler/listers/fake/listers.go b/pkg/scheduler/listers/fake/listers.go index 172870b3b97..2856f6c8bd7 100644 --- a/pkg/scheduler/listers/fake/listers.go +++ b/pkg/scheduler/listers/fake/listers.go @@ -22,13 +22,11 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" - storagev1beta1 "k8s.io/api/storage/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" appslisters "k8s.io/client-go/listers/apps/v1" corelisters "k8s.io/client-go/listers/core/v1" storagelisters "k8s.io/client-go/listers/storage/v1" - v1beta1storagelisters "k8s.io/client-go/listers/storage/v1beta1" schedulerlisters "k8s.io/kubernetes/pkg/scheduler/listers" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" ) @@ -280,19 +278,19 @@ func NewNodeInfoLister(nodes []*v1.Node) schedulerlisters.NodeInfoLister { return NodeInfoLister(nodeInfoList) } -var _ v1beta1storagelisters.CSINodeLister = CSINodeLister{} +var _ storagelisters.CSINodeLister = CSINodeLister{} -// CSINodeLister declares a storagev1beta1.CSINode type for testing. -type CSINodeLister storagev1beta1.CSINode +// CSINodeLister declares a storagev1.CSINode type for testing. +type CSINodeLister storagev1.CSINode // Get returns a fake CSINode object. -func (n CSINodeLister) Get(name string) (*storagev1beta1.CSINode, error) { - csiNode := storagev1beta1.CSINode(n) +func (n CSINodeLister) Get(name string) (*storagev1.CSINode, error) { + csiNode := storagev1.CSINode(n) return &csiNode, nil } // List lists all CSINodes in the indexer. -func (n CSINodeLister) List(selector labels.Selector) (ret []*storagev1beta1.CSINode, err error) { +func (n CSINodeLister) List(selector labels.Selector) (ret []*storagev1.CSINode, err error) { return nil, fmt.Errorf("not implemented") } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 6fe15338bc0..788091f15ba 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -33,7 +33,7 @@ import ( "k8s.io/client-go/informers" coreinformers "k8s.io/client-go/informers/core/v1" policyv1beta1informers "k8s.io/client-go/informers/policy/v1beta1" - storagev1beta1informers "k8s.io/client-go/informers/storage/v1beta1" + storageinformers "k8s.io/client-go/informers/storage/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/events" @@ -288,9 +288,9 @@ func New(client clientset.Interface, pdbInformer = informerFactory.Policy().V1beta1().PodDisruptionBudgets() } - var csiNodeInformer storagev1beta1informers.CSINodeInformer + var csiNodeInformer storageinformers.CSINodeInformer if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CSINodeInfo) { - csiNodeInformer = informerFactory.Storage().V1beta1().CSINodes() + csiNodeInformer = informerFactory.Storage().V1().CSINodes() } // Set up the configurator which can create schedulers from configs.