diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 3158286c859..105dc0795cc 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -282,6 +282,8 @@ const ( // owner: @gnufied // beta : v1.12 + // GA : v1.17 + // // Add support for volume plugins to report node specific // volume limits @@ -523,7 +525,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS ExpandPersistentVolumes: {Default: true, PreRelease: featuregate.Beta}, ExpandInUsePersistentVolumes: {Default: true, PreRelease: featuregate.Beta}, ExpandCSIVolumes: {Default: true, PreRelease: featuregate.Beta}, - AttachVolumeLimit: {Default: true, PreRelease: featuregate.Beta}, + AttachVolumeLimit: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.19 CPUManager: {Default: true, PreRelease: featuregate.Beta}, CPUCFSQuotaPeriod: {Default: false, PreRelease: featuregate.Alpha}, TopologyManager: {Default: false, PreRelease: featuregate.Alpha}, diff --git a/pkg/kubelet/kubelet_node_status.go b/pkg/kubelet/kubelet_node_status.go index af41b4856a7..f56817ebcd9 100644 --- a/pkg/kubelet/kubelet_node_status.go +++ b/pkg/kubelet/kubelet_node_status.go @@ -31,12 +31,10 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - utilfeature "k8s.io/apiserver/pkg/util/feature" cloudprovider "k8s.io/cloud-provider" "k8s.io/klog" k8s_api_v1 "k8s.io/kubernetes/pkg/apis/core/v1" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" - "k8s.io/kubernetes/pkg/features" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/kubelet/nodestatus" @@ -548,9 +546,9 @@ func (kl *Kubelet) defaultNodeStatusFuncs() []func(*v1.Node) error { nodestatus.Images(kl.nodeStatusMaxImages, kl.imageManager.GetImageList), nodestatus.GoRuntime(), ) - if utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { - setters = append(setters, nodestatus.VolumeLimits(kl.volumePluginMgr.ListVolumePluginWithLimits)) - } + // Volume limits + setters = append(setters, nodestatus.VolumeLimits(kl.volumePluginMgr.ListVolumePluginWithLimits)) + setters = append(setters, nodestatus.MemoryPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderMemoryPressure, kl.recordNodeStatusEvent), nodestatus.DiskPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderDiskPressure, kl.recordNodeStatusEvent), diff --git a/pkg/registry/storage/csinode/BUILD b/pkg/registry/storage/csinode/BUILD index 1c90ff26a87..f14adabc140 100644 --- a/pkg/registry/storage/csinode/BUILD +++ b/pkg/registry/storage/csinode/BUILD @@ -12,11 +12,9 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/storage:go_default_library", "//pkg/apis/storage/validation:go_default_library", - "//pkg/features:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", ], ) @@ -43,12 +41,9 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/apis/storage:go_default_library", - "//pkg/features:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", "//vendor/k8s.io/utils/pointer:go_default_library", ], ) diff --git a/pkg/registry/storage/csinode/strategy.go b/pkg/registry/storage/csinode/strategy.go index 160f6d5f566..d7ac8871e4f 100644 --- a/pkg/registry/storage/csinode/strategy.go +++ b/pkg/registry/storage/csinode/strategy.go @@ -22,11 +22,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/storage/names" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/storage" "k8s.io/kubernetes/pkg/apis/storage/validation" - "k8s.io/kubernetes/pkg/features" ) // csiNodeStrategy implements behavior for CSINode objects @@ -45,12 +43,6 @@ func (csiNodeStrategy) NamespaceScoped() bool { // PrepareForCreate clears fields that are not allowed to be set on creation. func (csiNodeStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) { - csiNode := obj.(*storage.CSINode) - if !utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { - for i := range csiNode.Spec.Drivers { - csiNode.Spec.Drivers[i].Allocatable = nil - } - } } func (csiNodeStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList { @@ -72,31 +64,6 @@ func (csiNodeStrategy) AllowCreateOnUpdate() bool { // PrepareForUpdate sets the driver's Allocatable fields that are not allowed to be set by an end user updating a CSINode. func (csiNodeStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) { - newCSINode := obj.(*storage.CSINode) - oldCSINode := old.(*storage.CSINode) - - inUse := getAllocatablesInUse(oldCSINode) - - if !utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { - for i := range newCSINode.Spec.Drivers { - if !inUse[newCSINode.Spec.Drivers[i].Name] { - newCSINode.Spec.Drivers[i].Allocatable = nil - } - } - } -} - -func getAllocatablesInUse(obj *storage.CSINode) map[string]bool { - inUse := make(map[string]bool) - if obj == nil { - return inUse - } - for i := range obj.Spec.Drivers { - if obj.Spec.Drivers[i].Allocatable != nil { - inUse[obj.Spec.Drivers[i].Name] = true - } - } - return inUse } func (csiNodeStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList { diff --git a/pkg/registry/storage/csinode/strategy_test.go b/pkg/registry/storage/csinode/strategy_test.go index 07dee6b73b0..e228a63a8d5 100644 --- a/pkg/registry/storage/csinode/strategy_test.go +++ b/pkg/registry/storage/csinode/strategy_test.go @@ -23,10 +23,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/apis/storage" - "k8s.io/kubernetes/pkg/features" utilpointer "k8s.io/utils/pointer" ) @@ -47,7 +44,7 @@ func TestPrepareForCreate(t *testing.T) { }, } - volumeLimitsEnabledCases := []struct { + volumeLimitsCases := []struct { name string obj *storage.CSINode expected *storage.CSINode @@ -64,32 +61,7 @@ func TestPrepareForCreate(t *testing.T) { }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() - for _, test := range volumeLimitsEnabledCases { - t.Run(test.name, func(t *testing.T) { - testPrepareForCreate(t, test.obj, test.expected) - }) - } - - volumeLimitsDisabledCases := []struct { - name string - obj *storage.CSINode - expected *storage.CSINode - }{ - { - "empty allocatable", - emptyAllocatable, - emptyAllocatable, - }, - { - "drop allocatable", - valid, - emptyAllocatable, - }, - } - - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, false)() - for _, test := range volumeLimitsDisabledCases { + for _, test := range volumeLimitsCases { t.Run(test.name, func(t *testing.T) { testPrepareForCreate(t, test.obj, test.expected) }) @@ -140,7 +112,7 @@ func TestPrepareForUpdate(t *testing.T) { }, } - volumeLimitsEnabledCases := []struct { + volumeLimitsCases := []struct { name string old *storage.CSINode new *storage.CSINode @@ -166,35 +138,7 @@ func TestPrepareForUpdate(t *testing.T) { }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() - for _, test := range volumeLimitsEnabledCases { - t.Run(test.name, func(t *testing.T) { - testPrepareForUpdate(t, test.new, test.old, test.expected) - }) - } - - volumeLimitsDisabledCases := []struct { - name string - old *storage.CSINode - new *storage.CSINode - expected *storage.CSINode - }{ - { - "allow empty allocatable when it's not set", - emptyAllocatable, - emptyAllocatable, - emptyAllocatable, - }, - { - "drop allocatable when it's not set", - emptyAllocatable, - valid, - emptyAllocatable, - }, - } - - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, false)() - for _, test := range volumeLimitsDisabledCases { + for _, test := range volumeLimitsCases { t.Run(test.name, func(t *testing.T) { testPrepareForUpdate(t, test.new, test.old, test.expected) }) diff --git a/pkg/scheduler/BUILD b/pkg/scheduler/BUILD index 91f86930bb6..8055bc6f4da 100644 --- a/pkg/scheduler/BUILD +++ b/pkg/scheduler/BUILD @@ -48,13 +48,11 @@ go_library( "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/informers/policy/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/informers/storage/v1:go_default_library", - "//staging/src/k8s.io/client-go/informers/storage/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/listers/apps/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/policy/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/listers/storage/v1:go_default_library", - "//staging/src/k8s.io/client-go/listers/storage/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/events:go_default_library", "//vendor/k8s.io/klog:go_default_library", diff --git a/pkg/scheduler/algorithm/predicates/BUILD b/pkg/scheduler/algorithm/predicates/BUILD index f457e2daed8..8b8d1a9a685 100644 --- a/pkg/scheduler/algorithm/predicates/BUILD +++ b/pkg/scheduler/algorithm/predicates/BUILD @@ -28,7 +28,6 @@ go_library( "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", - "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", @@ -37,7 +36,6 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/storage/v1:go_default_library", - "//staging/src/k8s.io/client-go/listers/storage/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", "//staging/src/k8s.io/cloud-provider/volume/helpers:go_default_library", "//staging/src/k8s.io/csi-translation-lib:go_default_library", @@ -67,7 +65,6 @@ go_test( "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", - "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", diff --git a/pkg/scheduler/algorithm/predicates/csi_volume_predicate.go b/pkg/scheduler/algorithm/predicates/csi_volume_predicate.go index 7d6236f72ab..d853aa14dc7 100644 --- a/pkg/scheduler/algorithm/predicates/csi_volume_predicate.go +++ b/pkg/scheduler/algorithm/predicates/csi_volume_predicate.go @@ -20,16 +20,13 @@ import ( "fmt" v1 "k8s.io/api/core/v1" - storagev1beta1 "k8s.io/api/storage/v1beta1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/util/rand" - utilfeature "k8s.io/apiserver/pkg/util/feature" corelisters "k8s.io/client-go/listers/core/v1" storagelisters "k8s.io/client-go/listers/storage/v1" - v1beta1storagelisters "k8s.io/client-go/listers/storage/v1beta1" csitrans "k8s.io/csi-translation-lib" "k8s.io/klog" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" - "k8s.io/kubernetes/pkg/features" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" volumeutil "k8s.io/kubernetes/pkg/volume/util" ) @@ -46,7 +43,7 @@ type InTreeToCSITranslator interface { // CSIMaxVolumeLimitChecker defines predicate needed for counting CSI volumes type CSIMaxVolumeLimitChecker struct { - csiNodeLister v1beta1storagelisters.CSINodeLister + csiNodeLister storagelisters.CSINodeLister pvLister corelisters.PersistentVolumeLister pvcLister corelisters.PersistentVolumeClaimLister scLister storagelisters.StorageClassLister @@ -58,7 +55,7 @@ type CSIMaxVolumeLimitChecker struct { // NewCSIMaxVolumeLimitPredicate returns a predicate for counting CSI volumes func NewCSIMaxVolumeLimitPredicate( - csiNodeLister v1beta1storagelisters.CSINodeLister, pvLister corelisters.PersistentVolumeLister, pvcLister corelisters.PersistentVolumeClaimLister, scLister storagelisters.StorageClassLister) FitPredicate { + csiNodeLister storagelisters.CSINodeLister, pvLister corelisters.PersistentVolumeLister, pvcLister corelisters.PersistentVolumeClaimLister, scLister storagelisters.StorageClassLister) FitPredicate { c := &CSIMaxVolumeLimitChecker{ csiNodeLister: csiNodeLister, pvLister: pvLister, @@ -70,7 +67,7 @@ func NewCSIMaxVolumeLimitPredicate( return c.attachableLimitPredicate } -func getVolumeLimits(nodeInfo *schedulernodeinfo.NodeInfo, csiNode *storagev1beta1.CSINode) map[v1.ResourceName]int64 { +func getVolumeLimits(nodeInfo *schedulernodeinfo.NodeInfo, csiNode *storagev1.CSINode) map[v1.ResourceName]int64 { // TODO: stop getting values from Node object in v1.18 nodeVolumeLimits := nodeInfo.VolumeLimits() if csiNode != nil { @@ -93,10 +90,6 @@ func (c *CSIMaxVolumeLimitChecker) attachableLimitPredicate( return true, nil, nil } - if !utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { - return true, nil, nil - } - node := nodeInfo.Node() if node == nil { return false, nil, fmt.Errorf("node not found") @@ -160,7 +153,7 @@ func (c *CSIMaxVolumeLimitChecker) attachableLimitPredicate( } func (c *CSIMaxVolumeLimitChecker) filterAttachableVolumes( - csiNode *storagev1beta1.CSINode, volumes []v1.Volume, namespace string, result map[string]string) error { + csiNode *storagev1.CSINode, volumes []v1.Volume, namespace string, result map[string]string) error { for _, vol := range volumes { // CSI volumes can only be used as persistent volumes if vol.PersistentVolumeClaim == nil { @@ -195,7 +188,7 @@ func (c *CSIMaxVolumeLimitChecker) filterAttachableVolumes( // getCSIDriverInfo returns the CSI driver name and volume ID of a given PVC. // If the PVC is from a migrated in-tree plugin, this function will return // the information of the CSI driver that the plugin has been migrated to. -func (c *CSIMaxVolumeLimitChecker) getCSIDriverInfo(csiNode *storagev1beta1.CSINode, pvc *v1.PersistentVolumeClaim) (string, string) { +func (c *CSIMaxVolumeLimitChecker) getCSIDriverInfo(csiNode *storagev1.CSINode, pvc *v1.PersistentVolumeClaim) (string, string) { pvName := pvc.Spec.VolumeName namespace := pvc.Namespace pvcName := pvc.Name @@ -250,7 +243,7 @@ func (c *CSIMaxVolumeLimitChecker) getCSIDriverInfo(csiNode *storagev1beta1.CSIN } // getCSIDriverInfoFromSC returns the CSI driver name and a random volume ID of a given PVC's StorageClass. -func (c *CSIMaxVolumeLimitChecker) getCSIDriverInfoFromSC(csiNode *storagev1beta1.CSINode, pvc *v1.PersistentVolumeClaim) (string, string) { +func (c *CSIMaxVolumeLimitChecker) getCSIDriverInfoFromSC(csiNode *storagev1.CSINode, pvc *v1.PersistentVolumeClaim) (string, string) { namespace := pvc.Namespace pvcName := pvc.Name scName := v1helper.GetPersistentVolumeClaimClass(pvc) diff --git a/pkg/scheduler/algorithm/predicates/csi_volume_predicate_test.go b/pkg/scheduler/algorithm/predicates/csi_volume_predicate_test.go index cee8bf8c861..0fbfbe0685e 100644 --- a/pkg/scheduler/algorithm/predicates/csi_volume_predicate_test.go +++ b/pkg/scheduler/algorithm/predicates/csi_volume_predicate_test.go @@ -23,7 +23,7 @@ import ( "testing" v1 "k8s.io/api/core/v1" - storagev1beta1 "k8s.io/api/storage/v1beta1" + storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -434,7 +434,6 @@ func TestCSIVolumeCountPredicate(t *testing.T) { }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() // running attachable predicate tests with feature gate and limit present on nodes for _, test := range tests { t.Run(test.test, func(t *testing.T) { @@ -547,7 +546,7 @@ func getFakeCSIPVCLister(volumeName, scName string, driverNames ...string) fakel return pvcLister } -func enableMigrationOnNode(csiNode *storagev1beta1.CSINode, pluginName string) { +func enableMigrationOnNode(csiNode *storagev1.CSINode, pluginName string) { nodeInfoAnnotations := csiNode.GetAnnotations() if nodeInfoAnnotations == nil { nodeInfoAnnotations = map[string]string{} @@ -570,7 +569,7 @@ func getFakeCSIStorageClassLister(scName, provisionerName string) fakelisters.St } } -func getFakeCSINodeLister(csiNode *storagev1beta1.CSINode) fakelisters.CSINodeLister { +func getFakeCSINodeLister(csiNode *storagev1.CSINode) fakelisters.CSINodeLister { if csiNode != nil { return fakelisters.CSINodeLister(*csiNode) } diff --git a/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go b/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go index 6f69a4b5c9f..b8f35919cb9 100644 --- a/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go +++ b/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go @@ -19,18 +19,14 @@ package predicates import ( "os" "reflect" - "strconv" "strings" "testing" v1 "k8s.io/api/core/v1" - "k8s.io/api/storage/v1beta1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" csilibplugins "k8s.io/csi-translation-lib/plugins" - "k8s.io/kubernetes/pkg/features" fakelisters "k8s.io/kubernetes/pkg/scheduler/listers/fake" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" volumeutil "k8s.io/kubernetes/pkg/volume/util" @@ -848,31 +844,6 @@ func TestVolumeCountConflicts(t *testing.T) { expectedFailureReasons := []PredicateFailureReason{ErrMaxVolumeCountExceeded} - // running attachable predicate tests without feature gate and no limit present on nodes - for _, test := range tests { - os.Setenv(KubeMaxPDVols, strconv.Itoa(test.maxVols)) - node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) - pred := NewMaxPDVolumeCountPredicate(test.filterName, - getFakeCSINodeLister(csiNode), - getFakeStorageClassLister(test.filterName), - getFakePVLister(test.filterName), - getFakePVCLister(test.filterName)) - - factory := &MetadataProducerFactory{} - fits, reasons, err := pred(test.newPod, factory.GetPredicateMetadata(test.newPod, nil), node) - if err != nil { - t.Errorf("[%s]%s: unexpected error: %v", test.filterName, test.test, err) - } - if !fits && !reflect.DeepEqual(reasons, expectedFailureReasons) { - t.Errorf("[%s]%s: unexpected failure reasons: %v, want: %v", test.filterName, test.test, reasons, expectedFailureReasons) - } - if fits != test.fits { - t.Errorf("[%s]%s: expected %v, got %v", test.filterName, test.test, test.fits, fits) - } - } - - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() - // running attachable predicate tests with feature gate and limit present on nodes for _, test := range tests { node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) @@ -1104,7 +1075,7 @@ func TestMaxVolumeFuncM4WithBothBetaAndStableLabels(t *testing.T) { } } -func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int64, driverNames ...string) (*schedulernodeinfo.NodeInfo, *v1beta1.CSINode) { +func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int64, driverNames ...string) (*schedulernodeinfo.NodeInfo, *storagev1.CSINode) { nodeInfo := schedulernodeinfo.NewNodeInfo(pods...) node := &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: "node-for-max-pd-test-1"}, @@ -1112,7 +1083,7 @@ func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int Allocatable: v1.ResourceList{}, }, } - var csiNode *v1beta1.CSINode + var csiNode *storagev1.CSINode addLimitToNode := func() { for _, driver := range driverNames { @@ -1121,10 +1092,10 @@ func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int } initCSINode := func() { - csiNode = &v1beta1.CSINode{ + csiNode = &storagev1.CSINode{ ObjectMeta: metav1.ObjectMeta{Name: "csi-node-for-max-pd-test-1"}, - Spec: v1beta1.CSINodeSpec{ - Drivers: []v1beta1.CSINodeDriver{}, + Spec: storagev1.CSINodeSpec{ + Drivers: []storagev1.CSINodeDriver{}, }, } } @@ -1132,12 +1103,12 @@ func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int addDriversCSINode := func(addLimits bool) { initCSINode() for _, driver := range driverNames { - driver := v1beta1.CSINodeDriver{ + driver := storagev1.CSINodeDriver{ Name: driver, NodeID: "node-for-max-pd-test-1", } if addLimits { - driver.Allocatable = &v1beta1.VolumeNodeResources{ + driver.Allocatable = &storagev1.VolumeNodeResources{ Count: utilpointer.Int32Ptr(int32(limit)), } } diff --git a/pkg/scheduler/algorithm/predicates/predicates.go b/pkg/scheduler/algorithm/predicates/predicates.go index 0354d08c9c0..210d14277c2 100644 --- a/pkg/scheduler/algorithm/predicates/predicates.go +++ b/pkg/scheduler/algorithm/predicates/predicates.go @@ -27,7 +27,6 @@ import ( v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" - v1beta1storage "k8s.io/api/storage/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" @@ -36,7 +35,6 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" corelisters "k8s.io/client-go/listers/core/v1" storagelisters "k8s.io/client-go/listers/storage/v1" - v1beta1storagelisters "k8s.io/client-go/listers/storage/v1beta1" volumehelpers "k8s.io/cloud-provider/volume/helpers" csilibplugins "k8s.io/csi-translation-lib/plugins" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" @@ -225,7 +223,7 @@ type MaxPDVolumeCountChecker struct { filter VolumeFilter volumeLimitKey v1.ResourceName maxVolumeFunc func(node *v1.Node) int - csiNodeLister v1beta1storagelisters.CSINodeLister + csiNodeLister storagelisters.CSINodeLister pvLister corelisters.PersistentVolumeLister pvcLister corelisters.PersistentVolumeClaimLister scLister storagelisters.StorageClassLister @@ -244,7 +242,7 @@ type VolumeFilter struct { // MatchProvisioner evaluates if the StorageClass provisioner matches the running predicate MatchProvisioner func(sc *storage.StorageClass) (relevant bool) // IsMigrated returns a boolean specifying whether the plugin is migrated to a CSI driver - IsMigrated func(csiNode *v1beta1storage.CSINode) bool + IsMigrated func(csiNode *storage.CSINode) bool } // NewMaxPDVolumeCountPredicate creates a predicate which evaluates whether a pod can fit based on the @@ -257,7 +255,7 @@ type VolumeFilter struct { // The predicate looks for both volumes used directly, as well as PVC volumes that are backed by relevant volume // types, counts the number of unique volumes, and rejects the new pod if it would place the total count over // the maximum. -func NewMaxPDVolumeCountPredicate(filterName string, csiNodeLister v1beta1storagelisters.CSINodeLister, scLister storagelisters.StorageClassLister, +func NewMaxPDVolumeCountPredicate(filterName string, csiNodeLister storagelisters.CSINodeLister, scLister storagelisters.StorageClassLister, pvLister corelisters.PersistentVolumeLister, pvcLister corelisters.PersistentVolumeClaimLister) FitPredicate { var filter VolumeFilter var volumeLimitKey v1.ResourceName @@ -441,7 +439,7 @@ func (c *MaxPDVolumeCountChecker) predicate(pod *v1.Pod, meta Metadata, nodeInfo } var ( - csiNode *v1beta1storage.CSINode + csiNode *storage.CSINode err error ) if c.csiNodeLister != nil { @@ -477,11 +475,9 @@ func (c *MaxPDVolumeCountChecker) predicate(pod *v1.Pod, meta Metadata, nodeInfo numNewVolumes := len(newVolumes) maxAttachLimit := c.maxVolumeFunc(node) - if utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { - volumeLimits := nodeInfo.VolumeLimits() - if maxAttachLimitFromAllocatable, ok := volumeLimits[c.volumeLimitKey]; ok { - maxAttachLimit = int(maxAttachLimitFromAllocatable) - } + volumeLimits := nodeInfo.VolumeLimits() + if maxAttachLimitFromAllocatable, ok := volumeLimits[c.volumeLimitKey]; ok { + maxAttachLimit = int(maxAttachLimitFromAllocatable) } if numExistingVolumes+numNewVolumes > maxAttachLimit { @@ -520,7 +516,7 @@ var EBSVolumeFilter = VolumeFilter{ return false }, - IsMigrated: func(csiNode *v1beta1storage.CSINode) bool { + IsMigrated: func(csiNode *storage.CSINode) bool { return isCSIMigrationOn(csiNode, csilibplugins.AWSEBSInTreePluginName) }, } @@ -548,7 +544,7 @@ var GCEPDVolumeFilter = VolumeFilter{ return false }, - IsMigrated: func(csiNode *v1beta1storage.CSINode) bool { + IsMigrated: func(csiNode *storage.CSINode) bool { return isCSIMigrationOn(csiNode, csilibplugins.GCEPDInTreePluginName) }, } @@ -576,7 +572,7 @@ var AzureDiskVolumeFilter = VolumeFilter{ return false }, - IsMigrated: func(csiNode *v1beta1storage.CSINode) bool { + IsMigrated: func(csiNode *storage.CSINode) bool { return isCSIMigrationOn(csiNode, csilibplugins.AzureDiskInTreePluginName) }, } @@ -605,7 +601,7 @@ var CinderVolumeFilter = VolumeFilter{ return false }, - IsMigrated: func(csiNode *v1beta1storage.CSINode) bool { + IsMigrated: func(csiNode *storage.CSINode) bool { return isCSIMigrationOn(csiNode, csilibplugins.CinderInTreePluginName) }, } diff --git a/pkg/scheduler/algorithm/predicates/utils.go b/pkg/scheduler/algorithm/predicates/utils.go index 1386c93d54f..d74a4a106d4 100644 --- a/pkg/scheduler/algorithm/predicates/utils.go +++ b/pkg/scheduler/algorithm/predicates/utils.go @@ -19,8 +19,8 @@ package predicates import ( "strings" - "k8s.io/api/core/v1" - storagev1beta1 "k8s.io/api/storage/v1beta1" + v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -97,7 +97,7 @@ func SetPredicatesOrderingDuringTest(value []string) func() { // isCSIMigrationOn returns a boolean value indicating whether // the CSI migration has been enabled for a particular storage plugin. -func isCSIMigrationOn(csiNode *storagev1beta1.CSINode, pluginName string) bool { +func isCSIMigrationOn(csiNode *storagev1.CSINode, pluginName string) bool { if csiNode == nil || len(pluginName) == 0 { return false } diff --git a/pkg/scheduler/algorithm_factory.go b/pkg/scheduler/algorithm_factory.go index 16743df9398..87b14f84494 100644 --- a/pkg/scheduler/algorithm_factory.go +++ b/pkg/scheduler/algorithm_factory.go @@ -29,7 +29,6 @@ import ( corelisters "k8s.io/client-go/listers/core/v1" policylisters "k8s.io/client-go/listers/policy/v1beta1" storagelisters "k8s.io/client-go/listers/storage/v1" - v1beta1storagelisters "k8s.io/client-go/listers/storage/v1beta1" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" @@ -53,7 +52,7 @@ type PluginFactoryArgs struct { ReplicaSetLister appslisters.ReplicaSetLister StatefulSetLister appslisters.StatefulSetLister PDBLister policylisters.PodDisruptionBudgetLister - CSINodeLister v1beta1storagelisters.CSINodeLister + CSINodeLister storagelisters.CSINodeLister PVLister corelisters.PersistentVolumeLister PVCLister corelisters.PersistentVolumeClaimLister StorageClassLister storagelisters.StorageClassLister diff --git a/pkg/scheduler/apis/config/testing/compatibility_test.go b/pkg/scheduler/apis/config/testing/compatibility_test.go index 5fc6e1a39e7..53d5e4a8e9c 100644 --- a/pkg/scheduler/apis/config/testing/compatibility_test.go +++ b/pkg/scheduler/apis/config/testing/compatibility_test.go @@ -1169,30 +1169,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, }, }, - { - name: "disable beta feature AttachVolumeLimit", - JSON: `{ - "kind": "Policy", - "apiVersion": "v1", - "predicates": [ - {"name": "MaxCSIVolumeCountPred"}, - {"name": "CheckVolumeBinding"} - ], - "priorities": [ - ] - }`, - featureGates: map[featuregate.Feature]bool{ - features.AttachVolumeLimit: false, - }, - wantPlugins: map[string][]config.Plugin{ - "FilterPlugin": { - {Name: "NodeUnschedulable"}, - {Name: "TaintToleration"}, - {Name: "NodeVolumeLimits"}, - {Name: "VolumeBinding"}, - }, - }, - }, } registeredPredicates := sets.NewString(scheduler.ListRegisteredFitPredicates()...) registeredPriorities := sets.NewString(scheduler.ListRegisteredPriorityFunctions()...) diff --git a/pkg/scheduler/factory.go b/pkg/scheduler/factory.go index 4eb83c86ba0..33b5d991d02 100644 --- a/pkg/scheduler/factory.go +++ b/pkg/scheduler/factory.go @@ -34,13 +34,11 @@ import ( coreinformers "k8s.io/client-go/informers/core/v1" policyinformers "k8s.io/client-go/informers/policy/v1beta1" storageinformersv1 "k8s.io/client-go/informers/storage/v1" - storageinformersv1beta1 "k8s.io/client-go/informers/storage/v1beta1" clientset "k8s.io/client-go/kubernetes" appslisters "k8s.io/client-go/listers/apps/v1" corelisters "k8s.io/client-go/listers/core/v1" policylisters "k8s.io/client-go/listers/policy/v1beta1" storagelistersv1 "k8s.io/client-go/listers/storage/v1" - storagelistersv1beta1 "k8s.io/client-go/listers/storage/v1beta1" "k8s.io/client-go/tools/cache" "k8s.io/klog" "k8s.io/kubernetes/pkg/features" @@ -93,7 +91,7 @@ type Configurator struct { // a means to list all StorageClasses storageClassLister storagelistersv1.StorageClassLister // a means to list all CSINodes - csiNodeLister storagelistersv1beta1.CSINodeLister + csiNodeLister storagelistersv1.CSINodeLister // a means to list all Nodes nodeLister corelisters.NodeLister // a means to list all Pods @@ -154,7 +152,7 @@ type ConfigFactoryArgs struct { ServiceInformer coreinformers.ServiceInformer PdbInformer policyinformers.PodDisruptionBudgetInformer StorageClassInformer storageinformersv1.StorageClassInformer - CSINodeInformer storageinformersv1beta1.CSINodeInformer + CSINodeInformer storageinformersv1.CSINodeInformer VolumeBinder *volumebinder.VolumeBinder SchedulerCache internalcache.Cache HardPodAffinitySymmetricWeight int32 @@ -184,7 +182,7 @@ func NewConfigFactory(args *ConfigFactoryArgs) *Configurator { storageClassLister = args.StorageClassInformer.Lister() } - var csiNodeLister storagelistersv1beta1.CSINodeLister + var csiNodeLister storagelistersv1.CSINodeLister if args.CSINodeInformer != nil { csiNodeLister = args.CSINodeInformer.Lister() } diff --git a/pkg/scheduler/factory_test.go b/pkg/scheduler/factory_test.go index 980284a4b34..729724273fe 100644 --- a/pkg/scheduler/factory_test.go +++ b/pkg/scheduler/factory_test.go @@ -552,7 +552,7 @@ func newConfigFactoryWithFrameworkRegistry( ServiceInformer: informerFactory.Core().V1().Services(), PdbInformer: informerFactory.Policy().V1beta1().PodDisruptionBudgets(), StorageClassInformer: informerFactory.Storage().V1().StorageClasses(), - CSINodeInformer: informerFactory.Storage().V1beta1().CSINodes(), + CSINodeInformer: informerFactory.Storage().V1().CSINodes(), HardPodAffinitySymmetricWeight: hardPodAffinitySymmetricWeight, DisablePreemption: disablePodPreemption, PercentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore, diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/BUILD b/pkg/scheduler/framework/plugins/nodevolumelimits/BUILD index 444a2bb985f..671040e97c2 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/BUILD +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/BUILD @@ -22,7 +22,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", - "//staging/src/k8s.io/client-go/listers/storage/v1beta1:go_default_library", + "//staging/src/k8s.io/client-go/listers/storage/v1:go_default_library", ], ) @@ -44,7 +44,7 @@ go_test( "//pkg/scheduler/nodeinfo:go_default_library", "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", + "//staging/src/k8s.io/api/storage/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/azure_test.go b/pkg/scheduler/framework/plugins/nodevolumelimits/azure_test.go index 8c97ea24374..575b9d07f35 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/azure_test.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/azure_test.go @@ -22,9 +22,6 @@ import ( "testing" v1 "k8s.io/api/core/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" ) @@ -355,8 +352,6 @@ func TestAzureDiskLimits(t *testing.T) { }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() - for _, test := range tests { t.Run(test.test, func(t *testing.T) { node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/cinder_test.go b/pkg/scheduler/framework/plugins/nodevolumelimits/cinder_test.go index 08d197d6ba6..20ebb14de82 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/cinder_test.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/cinder_test.go @@ -22,9 +22,6 @@ import ( "testing" v1 "k8s.io/api/core/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" ) @@ -84,8 +81,6 @@ func TestCinderLimits(t *testing.T) { }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() - for _, test := range tests { t.Run(test.test, func(t *testing.T) { node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go b/pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go index 2b9479f46d6..51149c7122a 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go @@ -24,8 +24,7 @@ import ( "testing" v1 "k8s.io/api/core/v1" - "k8s.io/api/storage/v1beta1" - storagev1beta1 "k8s.io/api/storage/v1beta1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -447,7 +446,6 @@ func TestCSILimits(t *testing.T) { }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() // running attachable predicate tests with feature gate and limit present on nodes for _, test := range tests { t.Run(test.test, func(t *testing.T) { @@ -546,7 +544,7 @@ func getFakeCSIPVCLister(volumeName, scName string, driverNames ...string) fakel return pvcLister } -func enableMigrationOnNode(csiNode *storagev1beta1.CSINode, pluginName string) { +func enableMigrationOnNode(csiNode *storagev1.CSINode, pluginName string) { nodeInfoAnnotations := csiNode.GetAnnotations() if nodeInfoAnnotations == nil { nodeInfoAnnotations = map[string]string{} @@ -569,14 +567,14 @@ func getFakeCSIStorageClassLister(scName, provisionerName string) fakelisters.St } } -func getFakeCSINodeLister(csiNode *storagev1beta1.CSINode) fakelisters.CSINodeLister { +func getFakeCSINodeLister(csiNode *storagev1.CSINode) fakelisters.CSINodeLister { if csiNode != nil { return fakelisters.CSINodeLister(*csiNode) } return fakelisters.CSINodeLister{} } -func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int64, driverNames ...string) (*schedulernodeinfo.NodeInfo, *v1beta1.CSINode) { +func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int64, driverNames ...string) (*schedulernodeinfo.NodeInfo, *storagev1.CSINode) { nodeInfo := schedulernodeinfo.NewNodeInfo(pods...) node := &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: "node-for-max-pd-test-1"}, @@ -584,7 +582,7 @@ func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int Allocatable: v1.ResourceList{}, }, } - var csiNode *v1beta1.CSINode + var csiNode *storagev1.CSINode addLimitToNode := func() { for _, driver := range driverNames { @@ -593,10 +591,10 @@ func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int } initCSINode := func() { - csiNode = &v1beta1.CSINode{ + csiNode = &storagev1.CSINode{ ObjectMeta: metav1.ObjectMeta{Name: "csi-node-for-max-pd-test-1"}, - Spec: v1beta1.CSINodeSpec{ - Drivers: []v1beta1.CSINodeDriver{}, + Spec: storagev1.CSINodeSpec{ + Drivers: []storagev1.CSINodeDriver{}, }, } } @@ -604,12 +602,12 @@ func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int addDriversCSINode := func(addLimits bool) { initCSINode() for _, driver := range driverNames { - driver := v1beta1.CSINodeDriver{ + driver := storagev1.CSINodeDriver{ Name: driver, NodeID: "node-for-max-pd-test-1", } if addLimits { - driver.Allocatable = &v1beta1.VolumeNodeResources{ + driver.Allocatable = &storagev1.VolumeNodeResources{ Count: utilpointer.Int32Ptr(int32(limit)), } } diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/csinode_helper.go b/pkg/scheduler/framework/plugins/nodevolumelimits/csinode_helper.go index ef35ba9b8cd..196aa3db34d 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/csinode_helper.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/csinode_helper.go @@ -19,14 +19,14 @@ package nodevolumelimits import ( utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" - v1beta1 "k8s.io/client-go/listers/storage/v1beta1" + storagelisters "k8s.io/client-go/listers/storage/v1" kubefeatures "k8s.io/kubernetes/pkg/features" ) // getCSINodeListerIfEnabled returns the CSINode lister or nil if the feature is disabled -func getCSINodeListerIfEnabled(factory informers.SharedInformerFactory) v1beta1.CSINodeLister { +func getCSINodeListerIfEnabled(factory informers.SharedInformerFactory) storagelisters.CSINodeLister { if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CSINodeInfo) { return nil } - return factory.Storage().V1beta1().CSINodes().Lister() + return factory.Storage().V1().CSINodes().Lister() } diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/ebs_test.go b/pkg/scheduler/framework/plugins/nodevolumelimits/ebs_test.go index 39c1b1dd39d..c9aa0d89557 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/ebs_test.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/ebs_test.go @@ -24,10 +24,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" csilibplugins "k8s.io/csi-translation-lib/plugins" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" fakelisters "k8s.io/kubernetes/pkg/scheduler/listers/fake" @@ -469,8 +466,6 @@ func TestEBSLimits(t *testing.T) { }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() - for _, test := range tests { t.Run(test.test, func(t *testing.T) { node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/gce_test.go b/pkg/scheduler/framework/plugins/nodevolumelimits/gce_test.go index 414426561ec..957d3cfd485 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/gce_test.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/gce_test.go @@ -22,9 +22,6 @@ import ( "testing" v1 "k8s.io/api/core/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" ) @@ -355,8 +352,6 @@ func TestGCEPDLimits(t *testing.T) { }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() - for _, test := range tests { t.Run(test.test, func(t *testing.T) { node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) diff --git a/pkg/scheduler/listers/fake/BUILD b/pkg/scheduler/listers/fake/BUILD index 57efb259970..b826290bdb4 100644 --- a/pkg/scheduler/listers/fake/BUILD +++ b/pkg/scheduler/listers/fake/BUILD @@ -11,13 +11,11 @@ go_library( "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", - "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/client-go/listers/apps/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/storage/v1:go_default_library", - "//staging/src/k8s.io/client-go/listers/storage/v1beta1:go_default_library", ], ) diff --git a/pkg/scheduler/listers/fake/listers.go b/pkg/scheduler/listers/fake/listers.go index 172870b3b97..2856f6c8bd7 100644 --- a/pkg/scheduler/listers/fake/listers.go +++ b/pkg/scheduler/listers/fake/listers.go @@ -22,13 +22,11 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" - storagev1beta1 "k8s.io/api/storage/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" appslisters "k8s.io/client-go/listers/apps/v1" corelisters "k8s.io/client-go/listers/core/v1" storagelisters "k8s.io/client-go/listers/storage/v1" - v1beta1storagelisters "k8s.io/client-go/listers/storage/v1beta1" schedulerlisters "k8s.io/kubernetes/pkg/scheduler/listers" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" ) @@ -280,19 +278,19 @@ func NewNodeInfoLister(nodes []*v1.Node) schedulerlisters.NodeInfoLister { return NodeInfoLister(nodeInfoList) } -var _ v1beta1storagelisters.CSINodeLister = CSINodeLister{} +var _ storagelisters.CSINodeLister = CSINodeLister{} -// CSINodeLister declares a storagev1beta1.CSINode type for testing. -type CSINodeLister storagev1beta1.CSINode +// CSINodeLister declares a storagev1.CSINode type for testing. +type CSINodeLister storagev1.CSINode // Get returns a fake CSINode object. -func (n CSINodeLister) Get(name string) (*storagev1beta1.CSINode, error) { - csiNode := storagev1beta1.CSINode(n) +func (n CSINodeLister) Get(name string) (*storagev1.CSINode, error) { + csiNode := storagev1.CSINode(n) return &csiNode, nil } // List lists all CSINodes in the indexer. -func (n CSINodeLister) List(selector labels.Selector) (ret []*storagev1beta1.CSINode, err error) { +func (n CSINodeLister) List(selector labels.Selector) (ret []*storagev1.CSINode, err error) { return nil, fmt.Errorf("not implemented") } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index b2afd8d5154..ebc2833bcf5 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -33,7 +33,7 @@ import ( "k8s.io/client-go/informers" coreinformers "k8s.io/client-go/informers/core/v1" policyv1beta1informers "k8s.io/client-go/informers/policy/v1beta1" - storagev1beta1informers "k8s.io/client-go/informers/storage/v1beta1" + storageinformers "k8s.io/client-go/informers/storage/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/events" @@ -292,9 +292,9 @@ func New(client clientset.Interface, pdbInformer = informerFactory.Policy().V1beta1().PodDisruptionBudgets() } - var csiNodeInformer storagev1beta1informers.CSINodeInformer + var csiNodeInformer storageinformers.CSINodeInformer if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CSINodeInfo) { - csiNodeInformer = informerFactory.Storage().V1beta1().CSINodes() + csiNodeInformer = informerFactory.Storage().V1().CSINodes() } // Set up the configurator which can create schedulers from configs. diff --git a/pkg/volume/csi/nodeinfomanager/nodeinfomanager.go b/pkg/volume/csi/nodeinfomanager/nodeinfomanager.go index b6f15bdd12a..b6920ba078d 100644 --- a/pkg/volume/csi/nodeinfomanager/nodeinfomanager.go +++ b/pkg/volume/csi/nodeinfomanager/nodeinfomanager.go @@ -556,17 +556,15 @@ func (nim *nodeInfoManager) installDriverToCSINode( TopologyKeys: topologyKeys.List(), } - if utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { - if maxAttachLimit > 0 { - if maxAttachLimit > math.MaxInt32 { - klog.Warningf("Exceeded max supported attach limit value, truncating it to %d", math.MaxInt32) - maxAttachLimit = math.MaxInt32 - } - m := int32(maxAttachLimit) - driverSpec.Allocatable = &storagev1.VolumeNodeResources{Count: &m} - } else { - klog.Errorf("Invalid attach limit value %d cannot be added to CSINode object for %q", maxAttachLimit, driverName) + if maxAttachLimit > 0 { + if maxAttachLimit > math.MaxInt32 { + klog.Warningf("Exceeded max supported attach limit value, truncating it to %d", math.MaxInt32) + maxAttachLimit = math.MaxInt32 } + m := int32(maxAttachLimit) + driverSpec.Allocatable = &storagev1.VolumeNodeResources{Count: &m} + } else { + klog.Errorf("Invalid attach limit value %d cannot be added to CSINode object for %q", maxAttachLimit, driverName) } newDriverSpecs = append(newDriverSpecs, driverSpec) diff --git a/pkg/volume/csi/nodeinfomanager/nodeinfomanager_test.go b/pkg/volume/csi/nodeinfomanager/nodeinfomanager_test.go index 22ba95e1326..95bff7507b4 100644 --- a/pkg/volume/csi/nodeinfomanager/nodeinfomanager_test.go +++ b/pkg/volume/csi/nodeinfomanager/nodeinfomanager_test.go @@ -1011,7 +1011,6 @@ func getClientSet(existingNode *v1.Node, existingCSINode *storage.CSINode) *fake func test(t *testing.T, addNodeInfo bool, csiNodeInfoEnabled bool, testcases []testcase) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSINodeInfo, csiNodeInfoEnabled)() - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() for _, tc := range testcases { t.Logf("test case: %q", tc.name) diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go index 2b6a0e42f47..10ba64be035 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go @@ -498,10 +498,8 @@ func ClusterRoles() []rbacv1.ClusterRole { // Needed to check API access. These creates are non-mutating rbacv1helpers.NewRule("create").Groups(authenticationGroup).Resources("tokenreviews").RuleOrDie(), rbacv1helpers.NewRule("create").Groups(authorizationGroup).Resources("subjectaccessreviews").RuleOrDie(), - } - if utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) && - utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { - kubeSchedulerRules = append(kubeSchedulerRules, rbacv1helpers.NewRule(Read...).Groups(storageGroup).Resources("csinodes").RuleOrDie()) + // Needed for volume limits + rbacv1helpers.NewRule(Read...).Groups(storageGroup).Resources("csinodes").RuleOrDie(), } roles = append(roles, rbacv1.ClusterRole{ // a role to use for the kube-scheduler diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 044152b302c..e146813bb12 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -106,6 +106,7 @@ func InitHostPathCSIDriver() testsuites.TestDriver { testsuites.CapPVCDataSource: true, testsuites.CapControllerExpansion: true, testsuites.CapSingleNodeVolume: true, + testsuites.CapVolumeLimits: true, } return initHostPathCSIDriver("csi-hostpath", capabilities, diff --git a/test/e2e/storage/testsuites/volumelimits.go b/test/e2e/storage/testsuites/volumelimits.go index 3f7e3b9ec5c..0e67e95f5da 100644 --- a/test/e2e/storage/testsuites/volumelimits.go +++ b/test/e2e/storage/testsuites/volumelimits.go @@ -109,7 +109,7 @@ func (t *volumeLimitsTestSuite) defineTests(driver TestDriver, pattern testpatte // And one extra pod with a CSI volume should get Pending with a condition // that says it's unschedulable because of volume limit. // BEWARE: the test may create lot of volumes and it's really slow. - ginkgo.It("should support volume limits [Slow][Serial]", func() { + ginkgo.It("should support volume limits [Serial]", func() { driverInfo := driver.GetDriverInfo() if !driverInfo.Capabilities[CapVolumeLimits] { ginkgo.Skip(fmt.Sprintf("driver %s does not support volume limits", driverInfo.Name)) @@ -124,15 +124,19 @@ func (t *volumeLimitsTestSuite) defineTests(driver TestDriver, pattern testpatte l.config, l.testCleanup = driver.PrepareTest(f) defer l.testCleanup() - ginkgo.By("Picking a random node") - var nodeName string - node, err := e2enode.GetRandomReadySchedulableNode(f.ClientSet) - framework.ExpectNoError(err) - nodeName = node.Name + ginkgo.By("Picking a node") + // Some CSI drivers are deployed to a single node (e.g csi-hostpath), + // so we use that node instead of picking a random one. + nodeName := l.config.ClientNodeName + if nodeName == "" { + node, err := e2enode.GetRandomReadySchedulableNode(f.ClientSet) + framework.ExpectNoError(err) + nodeName = node.Name + } framework.Logf("Selected node %s", nodeName) ginkgo.By("Checking node limits") - limit, err := getNodeLimits(l.cs, nodeName, driverInfo) + limit, err := getNodeLimits(l.cs, l.config, nodeName, driverInfo) framework.ExpectNoError(err) framework.Logf("Node %s can handle %d volumes of driver %s", nodeName, limit, driverInfo.Name) @@ -283,9 +287,9 @@ func waitForAllPVCsPhase(cs clientset.Interface, timeout time.Duration, pvcs []* return pvNames, err } -func getNodeLimits(cs clientset.Interface, nodeName string, driverInfo *DriverInfo) (int, error) { +func getNodeLimits(cs clientset.Interface, config *PerTestConfig, nodeName string, driverInfo *DriverInfo) (int, error) { if len(driverInfo.InTreePluginName) == 0 { - return getCSINodeLimits(cs, nodeName, driverInfo) + return getCSINodeLimits(cs, config, nodeName, driverInfo) } return getInTreeNodeLimits(cs, nodeName, driverInfo) } @@ -317,7 +321,7 @@ func getInTreeNodeLimits(cs clientset.Interface, nodeName string, driverInfo *Dr return int(limit.Value()), nil } -func getCSINodeLimits(cs clientset.Interface, nodeName string, driverInfo *DriverInfo) (int, error) { +func getCSINodeLimits(cs clientset.Interface, config *PerTestConfig, nodeName string, driverInfo *DriverInfo) (int, error) { // Wait in a loop, the driver might just have been installed and kubelet takes a while to publish everything. var limit int err := wait.PollImmediate(2*time.Second, csiNodeInfoTimeout, func() (bool, error) { @@ -328,7 +332,7 @@ func getCSINodeLimits(cs clientset.Interface, nodeName string, driverInfo *Drive } var csiDriver *storagev1.CSINodeDriver for _, c := range csiNode.Spec.Drivers { - if c.Name == driverInfo.Name { + if c.Name == driverInfo.Name || c.Name == config.GetUniqueDriverName() { csiDriver = &c break } diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml index 2cac6097136..dcec409787a 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml @@ -62,12 +62,13 @@ spec: name: csi-data-dir - name: hostpath - image: quay.io/k8scsi/hostpathplugin:v1.2.0 + image: quay.io/k8scsi/hostpathplugin:v1.3.0-rc1 args: - "--drivername=hostpath.csi.k8s.io" - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" + - "--maxvolumespernode=10" env: - name: CSI_ENDPOINT value: unix:///csi/csi.sock