From a94c14106043ecf25a622ba1d67b4af444cdc4d8 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 3 Mar 2021 12:42:21 +0100 Subject: [PATCH] CSIStorageCapacity: promote API to beta The v1alpha1 API is left in place for now to ease the migration. --- api/api-rules/violation_exceptions.list | 1 + pkg/apis/storage/types.go | 6 +- .../storageversionhashdata/data.go | 1 + pkg/features/kube_features.go | 3 +- .../default_storage_factory_builder.go | 2 +- pkg/registry/storage/rest/storage_storage.go | 9 ++ staging/src/k8s.io/api/storage/v1/types.go | 2 +- .../src/k8s.io/api/storage/v1alpha1/types.go | 4 +- .../k8s.io/api/storage/v1beta1/register.go | 3 + .../src/k8s.io/api/storage/v1beta1/types.go | 88 ++++++++++++++++++- 10 files changed, 112 insertions(+), 7 deletions(-) diff --git a/api/api-rules/violation_exceptions.list b/api/api-rules/violation_exceptions.list index 6c5503e71cf..398e9f1a685 100644 --- a/api/api-rules/violation_exceptions.list +++ b/api/api-rules/violation_exceptions.list @@ -262,6 +262,7 @@ API rule violation: list_type_missing,k8s.io/api/storage/v1alpha1,CSIStorageCapa API rule violation: list_type_missing,k8s.io/api/storage/v1beta1,CSIDriverSpec,VolumeLifecycleModes API rule violation: list_type_missing,k8s.io/api/storage/v1beta1,CSINodeDriver,TopologyKeys API rule violation: list_type_missing,k8s.io/api/storage/v1beta1,CSINodeSpec,Drivers +API rule violation: list_type_missing,k8s.io/api/storage/v1beta1,CSIStorageCapacityList,Items API rule violation: list_type_missing,k8s.io/api/storage/v1beta1,StorageClass,AllowedTopologies API rule violation: list_type_missing,k8s.io/api/storage/v1beta1,StorageClass,MountOptions API rule violation: list_type_missing,k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1,ConversionRequest,Objects diff --git a/pkg/apis/storage/types.go b/pkg/apis/storage/types.go index a1afc02b07b..05597e2d6fb 100644 --- a/pkg/apis/storage/types.go +++ b/pkg/apis/storage/types.go @@ -341,7 +341,7 @@ type CSIDriverSpec struct { // unset or false and it can be flipped later when storage // capacity information has been published. // - // This is an alpha field and only available when the CSIStorageCapacity + // This is a beta field and only available when the CSIStorageCapacity // feature is enabled. The default is false. // // +optional @@ -564,7 +564,9 @@ type CSINodeList struct { // // The producer of these objects can decide which approach is more suitable. // -// This is an alpha feature and only available when the CSIStorageCapacity feature is enabled. +// They are consumed by the kube-scheduler if the CSIStorageCapacity beta feature gate +// is enabled there and a CSI driver opts into capacity-aware scheduling with +// CSIDriver.StorageCapacity. type CSIStorageCapacity struct { metav1.TypeMeta // Standard object's metadata. The name has no particular meaning. It must be diff --git a/pkg/controlplane/storageversionhashdata/data.go b/pkg/controlplane/storageversionhashdata/data.go index 6aa9e3eb7e3..cfe1f14cf0b 100644 --- a/pkg/controlplane/storageversionhashdata/data.go +++ b/pkg/controlplane/storageversionhashdata/data.go @@ -95,6 +95,7 @@ var GVRToStorageVersionHash = map[string]string{ "storage.k8s.io/v1/volumeattachments": "vQAqD28V4AY=", "storage.k8s.io/v1beta1/csidrivers": "hL6j/rwBV5w=", "storage.k8s.io/v1beta1/csinodes": "Pe62DkZtjuo=", + "storage.k8s.io/v1beta1/csistoragecapacities": "xeVl+2Ly1kE=", "storage.k8s.io/v1beta1/storageclasses": "K+m6uJwbjGY=", "storage.k8s.io/v1beta1/volumeattachments": "vQAqD28V4AY=", "apps/v1/controllerrevisions": "85nkx63pcBU=", diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 4873732bcdf..fd6695e15c5 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -233,6 +233,7 @@ const ( // owner: @pohly // alpha: v1.19 + // beta: v1.21 // // Enables tracking of available storage capacity that CSI drivers provide. CSIStorageCapacity featuregate.Feature = "CSIStorageCapacity" @@ -767,7 +768,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS ConfigurableFSGroupPolicy: {Default: true, PreRelease: featuregate.Beta}, BalanceAttachedNodeVolumes: {Default: false, PreRelease: featuregate.Alpha}, CSIInlineVolume: {Default: true, PreRelease: featuregate.Beta}, - CSIStorageCapacity: {Default: false, PreRelease: featuregate.Alpha}, + CSIStorageCapacity: {Default: true, PreRelease: featuregate.Beta}, CSIServiceAccountToken: {Default: false, PreRelease: featuregate.Alpha}, GenericEphemeralVolume: {Default: false, PreRelease: featuregate.Alpha}, CSIVolumeFSGroupPolicy: {Default: true, PreRelease: featuregate.Beta}, diff --git a/pkg/kubeapiserver/default_storage_factory_builder.go b/pkg/kubeapiserver/default_storage_factory_builder.go index b3b4dbd9310..d19f6578953 100644 --- a/pkg/kubeapiserver/default_storage_factory_builder.go +++ b/pkg/kubeapiserver/default_storage_factory_builder.go @@ -63,7 +63,7 @@ func NewStorageFactoryConfig() *StorageFactoryConfig { resources := []schema.GroupVersionResource{ batch.Resource("cronjobs").WithVersion("v1beta1"), - apisstorage.Resource("csistoragecapacities").WithVersion("v1alpha1"), + apisstorage.Resource("csistoragecapacities").WithVersion("v1beta1"), } return &StorageFactoryConfig{ diff --git a/pkg/registry/storage/rest/storage_storage.go b/pkg/registry/storage/rest/storage_storage.go index 283fec61190..199967f0d5c 100644 --- a/pkg/registry/storage/rest/storage_storage.go +++ b/pkg/registry/storage/rest/storage_storage.go @@ -119,6 +119,15 @@ func (p RESTStorageProvider) v1beta1Storage(apiResourceConfigSource serverstorag } storage["csidrivers"] = csiDriverStorage.CSIDriver + // register csistoragecapacity if CSIStorageCapacity feature gate is enabled + if utilfeature.DefaultFeatureGate.Enabled(features.CSIStorageCapacity) { + csiStorageStorage, err := csistoragecapacitystore.NewStorage(restOptionsGetter) + if err != nil { + return storage, err + } + storage["csistoragecapacities"] = csiStorageStorage.CSIStorageCapacity + } + return storage, nil } diff --git a/staging/src/k8s.io/api/storage/v1/types.go b/staging/src/k8s.io/api/storage/v1/types.go index 1daff2df4c5..9fa40eebe33 100644 --- a/staging/src/k8s.io/api/storage/v1/types.go +++ b/staging/src/k8s.io/api/storage/v1/types.go @@ -331,7 +331,7 @@ type CSIDriverSpec struct { // unset or false and it can be flipped later when storage // capacity information has been published. // - // This is an alpha field and only available when the CSIStorageCapacity + // This is a beta field and only available when the CSIStorageCapacity // feature is enabled. The default is false. // // +optional diff --git a/staging/src/k8s.io/api/storage/v1alpha1/types.go b/staging/src/k8s.io/api/storage/v1alpha1/types.go index 5e65bcebcf9..364e797177e 100644 --- a/staging/src/k8s.io/api/storage/v1alpha1/types.go +++ b/staging/src/k8s.io/api/storage/v1alpha1/types.go @@ -156,7 +156,9 @@ type VolumeError struct { // // The producer of these objects can decide which approach is more suitable. // -// This is an alpha feature and only available when the CSIStorageCapacity feature is enabled. +// They are consumed by the kube-scheduler if the CSIStorageCapacity beta feature gate +// is enabled there and a CSI driver opts into capacity-aware scheduling with +// CSIDriver.StorageCapacity. type CSIStorageCapacity struct { metav1.TypeMeta `json:",inline"` // Standard object's metadata. The name has no particular meaning. It must be diff --git a/staging/src/k8s.io/api/storage/v1beta1/register.go b/staging/src/k8s.io/api/storage/v1beta1/register.go index c270ace57d6..a281d0f26ec 100644 --- a/staging/src/k8s.io/api/storage/v1beta1/register.go +++ b/staging/src/k8s.io/api/storage/v1beta1/register.go @@ -55,6 +55,9 @@ func addKnownTypes(scheme *runtime.Scheme) error { &CSINode{}, &CSINodeList{}, + + &CSIStorageCapacity{}, + &CSIStorageCapacityList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) diff --git a/staging/src/k8s.io/api/storage/v1beta1/types.go b/staging/src/k8s.io/api/storage/v1beta1/types.go index 81c6be9bd12..eaabf67be96 100644 --- a/staging/src/k8s.io/api/storage/v1beta1/types.go +++ b/staging/src/k8s.io/api/storage/v1beta1/types.go @@ -18,6 +18,7 @@ package v1beta1 import ( v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -351,7 +352,7 @@ type CSIDriverSpec struct { // unset or false and it can be flipped later when storage // capacity information has been published. // - // This is an alpha field and only available when the CSIStorageCapacity + // This is a beta field and only available when the CSIStorageCapacity // feature is enabled. The default is false. // // +optional @@ -569,3 +570,88 @@ type CSINodeList struct { // items is the list of CSINode Items []CSINode `json:"items" protobuf:"bytes,2,rep,name=items"` } + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:prerelease-lifecycle-gen:introduced=1.21 + +// CSIStorageCapacity stores the result of one CSI GetCapacity call. +// For a given StorageClass, this describes the available capacity in a +// particular topology segment. This can be used when considering where to +// instantiate new PersistentVolumes. +// +// For example this can express things like: +// - StorageClass "standard" has "1234 GiB" available in "topology.kubernetes.io/zone=us-east1" +// - StorageClass "localssd" has "10 GiB" available in "kubernetes.io/hostname=knode-abc123" +// +// The following three cases all imply that no capacity is available for +// a certain combination: +// - no object exists with suitable topology and storage class name +// - such an object exists, but the capacity is unset +// - such an object exists, but the capacity is zero +// +// The producer of these objects can decide which approach is more suitable. +// +// They are consumed by the kube-scheduler if the CSIStorageCapacity beta feature gate +// is enabled there and a CSI driver opts into capacity-aware scheduling with +// CSIDriver.StorageCapacity. +type CSIStorageCapacity struct { + metav1.TypeMeta `json:",inline"` + // Standard object's metadata. The name has no particular meaning. It must be + // be a DNS subdomain (dots allowed, 253 characters). To ensure that + // there are no conflicts with other CSI drivers on the cluster, the recommendation + // is to use csisc-, a generated name, or a reverse-domain name which ends + // with the unique CSI driver name. + // + // Objects are namespaced. + // + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // NodeTopology defines which nodes have access to the storage + // for which capacity was reported. If not set, the storage is + // not accessible from any node in the cluster. If empty, the + // storage is accessible from all nodes. This field is + // immutable. + // + // +optional + NodeTopology *metav1.LabelSelector `json:"nodeTopology,omitempty" protobuf:"bytes,2,opt,name=nodeTopology"` + + // The name of the StorageClass that the reported capacity applies to. + // It must meet the same requirements as the name of a StorageClass + // object (non-empty, DNS subdomain). If that object no longer exists, + // the CSIStorageCapacity object is obsolete and should be removed by its + // creator. + // This field is immutable. + StorageClassName string `json:"storageClassName" protobuf:"bytes,3,name=storageClassName"` + + // Capacity is the value reported by the CSI driver in its GetCapacityResponse + // for a GetCapacityRequest with topology and parameters that match the + // previous fields. + // + // The semantic is currently (CSI spec 1.2) defined as: + // The available capacity, in bytes, of the storage that can be used + // to provision volumes. If not set, that information is currently + // unavailable and treated like zero capacity. + // + // +optional + Capacity *resource.Quantity `json:"capacity,omitempty" protobuf:"bytes,4,opt,name=capacity"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:prerelease-lifecycle-gen:introduced=1.21 + +// CSIStorageCapacityList is a collection of CSIStorageCapacity objects. +type CSIStorageCapacityList struct { + metav1.TypeMeta `json:",inline"` + // Standard list metadata + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // Items is the list of CSIStorageCapacity objects. + // +listType=map + // +listMapKey=name + Items []CSIStorageCapacity `json:"items" protobuf:"bytes,2,rep,name=items"` +}