diff --git a/api/api-rules/violation_exceptions.list b/api/api-rules/violation_exceptions.list index bbb45f218ab..e1a3ce9833a 100644 --- a/api/api-rules/violation_exceptions.list +++ b/api/api-rules/violation_exceptions.list @@ -253,6 +253,7 @@ API rule violation: list_type_missing,k8s.io/api/rbac/v1beta1,Role,Rules API rule violation: list_type_missing,k8s.io/api/rbac/v1beta1,RoleBinding,Subjects API rule violation: list_type_missing,k8s.io/api/storage/v1,CSINodeDriver,TopologyKeys API rule violation: list_type_missing,k8s.io/api/storage/v1,CSINodeSpec,Drivers +API rule violation: list_type_missing,k8s.io/api/storage/v1,CSIStorageCapacityList,Items API rule violation: list_type_missing,k8s.io/api/storage/v1,StorageClass,MountOptions API rule violation: list_type_missing,k8s.io/api/storage/v1alpha1,CSIStorageCapacityList,Items API rule violation: list_type_missing,k8s.io/api/storage/v1beta1,CSIDriverSpec,VolumeLifecycleModes diff --git a/pkg/apis/storage/types.go b/pkg/apis/storage/types.go index a0abcb5e99e..68f344aef2f 100644 --- a/pkg/apis/storage/types.go +++ b/pkg/apis/storage/types.go @@ -572,9 +572,8 @@ type CSINodeList struct { // // The producer of these objects can decide which approach is more suitable. // -// They are consumed by the kube-scheduler if the CSIStorageCapacity beta feature gate -// is enabled there and a CSI driver opts into capacity-aware scheduling with -// CSIDriver.StorageCapacity. +// They are consumed by the kube-scheduler if a CSI driver opts into +// capacity-aware scheduling with CSIDriverSpec.StorageCapacity. type CSIStorageCapacity struct { metav1.TypeMeta // Standard object's metadata. The name has no particular meaning. It must be diff --git a/pkg/controlplane/storageversionhashdata/data.go b/pkg/controlplane/storageversionhashdata/data.go index 3b02e2878ec..d5b4e7616ff 100644 --- a/pkg/controlplane/storageversionhashdata/data.go +++ b/pkg/controlplane/storageversionhashdata/data.go @@ -78,6 +78,7 @@ var GVRToStorageVersionHash = map[string]string{ "storage.k8s.io/v1/csidrivers": "hL6j/rwBV5w=", "storage.k8s.io/v1/csinodes": "Pe62DkZtjuo=", "storage.k8s.io/v1/storageclasses": "K+m6uJwbjGY=", + "storage.k8s.io/v1/csistoragecapacities": "xeVl+2Ly1kE=", "storage.k8s.io/v1/volumeattachments": "vQAqD28V4AY=", "storage.k8s.io/v1beta1/csistoragecapacities": "xeVl+2Ly1kE=", "apps/v1/controllerrevisions": "85nkx63pcBU=", diff --git a/pkg/kubeapiserver/default_storage_factory_builder.go b/pkg/kubeapiserver/default_storage_factory_builder.go index 31479d77646..a398d316411 100644 --- a/pkg/kubeapiserver/default_storage_factory_builder.go +++ b/pkg/kubeapiserver/default_storage_factory_builder.go @@ -61,6 +61,8 @@ func DefaultWatchCacheSizes() map[schema.GroupResource]int { func NewStorageFactoryConfig() *StorageFactoryConfig { resources := []schema.GroupVersionResource{ + // TODO (https://github.com/kubernetes/kubernetes/issues/108451): remove the override in + // 1.25. apisstorage.Resource("csistoragecapacities").WithVersion("v1beta1"), } diff --git a/pkg/registry/storage/rest/storage_storage.go b/pkg/registry/storage/rest/storage_storage.go index b4d73aafe1d..207c6f1b320 100644 --- a/pkg/registry/storage/rest/storage_storage.go +++ b/pkg/registry/storage/rest/storage_storage.go @@ -131,6 +131,15 @@ func (p RESTStorageProvider) v1Storage(apiResourceConfigSource serverstorage.API storage[resource] = csiDriverStorage.CSIDriver } + // register csistoragecapacities + if resource := "csistoragecapacities"; apiResourceConfigSource.ResourceEnabled(storageapiv1.SchemeGroupVersion.WithResource(resource)) { + csiStorageStorage, err := csistoragecapacitystore.NewStorage(restOptionsGetter) + if err != nil { + return storage, err + } + storage[resource] = csiStorageStorage.CSIStorageCapacity + } + return storage, nil } diff --git a/staging/src/k8s.io/api/storage/v1/register.go b/staging/src/k8s.io/api/storage/v1/register.go index 1a2f83d1b86..094fa282173 100644 --- a/staging/src/k8s.io/api/storage/v1/register.go +++ b/staging/src/k8s.io/api/storage/v1/register.go @@ -55,6 +55,9 @@ func addKnownTypes(scheme *runtime.Scheme) error { &CSIDriver{}, &CSIDriverList{}, + + &CSIStorageCapacity{}, + &CSIStorageCapacityList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) diff --git a/staging/src/k8s.io/api/storage/v1/types.go b/staging/src/k8s.io/api/storage/v1/types.go index 6da0657ecc2..c054464acf7 100644 --- a/staging/src/k8s.io/api/storage/v1/types.go +++ b/staging/src/k8s.io/api/storage/v1/types.go @@ -18,6 +18,7 @@ package v1 import ( v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -344,9 +345,6 @@ type CSIDriverSpec struct { // // This field was immutable in Kubernetes <= 1.22 and now is mutable. // - // This is a beta field and only available when the CSIStorageCapacity - // feature is enabled. The default is false. - // // +optional // +featureGate=CSIStorageCapacity StorageCapacity *bool `json:"storageCapacity,omitempty" protobuf:"bytes,4,opt,name=storageCapacity"` @@ -560,3 +558,99 @@ type CSINodeList struct { // items is the list of CSINode Items []CSINode `json:"items" protobuf:"bytes,2,rep,name=items"` } + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// CSIStorageCapacity stores the result of one CSI GetCapacity call. +// For a given StorageClass, this describes the available capacity in a +// particular topology segment. This can be used when considering where to +// instantiate new PersistentVolumes. +// +// For example this can express things like: +// - StorageClass "standard" has "1234 GiB" available in "topology.kubernetes.io/zone=us-east1" +// - StorageClass "localssd" has "10 GiB" available in "kubernetes.io/hostname=knode-abc123" +// +// The following three cases all imply that no capacity is available for +// a certain combination: +// - no object exists with suitable topology and storage class name +// - such an object exists, but the capacity is unset +// - such an object exists, but the capacity is zero +// +// The producer of these objects can decide which approach is more suitable. +// +// They are consumed by the kube-scheduler if a CSI driver opts into +// capacity-aware scheduling with CSIDriverSpec.StorageCapacity. +type CSIStorageCapacity struct { + metav1.TypeMeta `json:",inline"` + // Standard object's metadata. The name has no particular meaning. It must be + // be a DNS subdomain (dots allowed, 253 characters). To ensure that + // there are no conflicts with other CSI drivers on the cluster, the recommendation + // is to use csisc-, a generated name, or a reverse-domain name which ends + // with the unique CSI driver name. + // + // Objects are namespaced. + // + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // NodeTopology defines which nodes have access to the storage + // for which capacity was reported. If not set, the storage is + // not accessible from any node in the cluster. If empty, the + // storage is accessible from all nodes. This field is + // immutable. + // + // +optional + NodeTopology *metav1.LabelSelector `json:"nodeTopology,omitempty" protobuf:"bytes,2,opt,name=nodeTopology"` + + // The name of the StorageClass that the reported capacity applies to. + // It must meet the same requirements as the name of a StorageClass + // object (non-empty, DNS subdomain). If that object no longer exists, + // the CSIStorageCapacity object is obsolete and should be removed by its + // creator. + // This field is immutable. + StorageClassName string `json:"storageClassName" protobuf:"bytes,3,name=storageClassName"` + + // Capacity is the value reported by the CSI driver in its GetCapacityResponse + // for a GetCapacityRequest with topology and parameters that match the + // previous fields. + // + // The semantic is currently (CSI spec 1.2) defined as: + // The available capacity, in bytes, of the storage that can be used + // to provision volumes. If not set, that information is currently + // unavailable and treated like zero capacity. + // + // +optional + Capacity *resource.Quantity `json:"capacity,omitempty" protobuf:"bytes,4,opt,name=capacity"` + + // MaximumVolumeSize is the value reported by the CSI driver in its GetCapacityResponse + // for a GetCapacityRequest with topology and parameters that match the + // previous fields. + // + // This is defined since CSI spec 1.4.0 as the largest size + // that may be used in a + // CreateVolumeRequest.capacity_range.required_bytes field to + // create a volume with the same parameters as those in + // GetCapacityRequest. The corresponding value in the Kubernetes + // API is ResourceRequirements.Requests in a volume claim. + // + // +optional + MaximumVolumeSize *resource.Quantity `json:"maximumVolumeSize,omitempty" protobuf:"bytes,5,opt,name=maximumVolumeSize"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// CSIStorageCapacityList is a collection of CSIStorageCapacity objects. +type CSIStorageCapacityList struct { + metav1.TypeMeta `json:",inline"` + // Standard list metadata + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // Items is the list of CSIStorageCapacity objects. + // +listType=map + // +listMapKey=name + Items []CSIStorageCapacity `json:"items" protobuf:"bytes,2,rep,name=items"` +} diff --git a/staging/src/k8s.io/api/storage/v1alpha1/types.go b/staging/src/k8s.io/api/storage/v1alpha1/types.go index afb0495db54..e819ce29d1d 100644 --- a/staging/src/k8s.io/api/storage/v1alpha1/types.go +++ b/staging/src/k8s.io/api/storage/v1alpha1/types.go @@ -17,7 +17,7 @@ limitations under the License. package v1alpha1 import ( - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -165,9 +165,8 @@ type VolumeError struct { // // The producer of these objects can decide which approach is more suitable. // -// They are consumed by the kube-scheduler if the CSIStorageCapacity beta feature gate -// is enabled there and a CSI driver opts into capacity-aware scheduling with -// CSIDriver.StorageCapacity. +// They are consumed by the kube-scheduler if a CSI driver opts into +// capacity-aware scheduling with CSIDriverSpec.StorageCapacity. type CSIStorageCapacity struct { metav1.TypeMeta `json:",inline"` // Standard object's metadata. The name has no particular meaning. It must be diff --git a/staging/src/k8s.io/api/storage/v1beta1/types.go b/staging/src/k8s.io/api/storage/v1beta1/types.go index b9a26521f84..1c4ba7f5f34 100644 --- a/staging/src/k8s.io/api/storage/v1beta1/types.go +++ b/staging/src/k8s.io/api/storage/v1beta1/types.go @@ -584,6 +584,8 @@ type CSINodeList struct { // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +k8s:prerelease-lifecycle-gen:introduced=1.21 +// +k8s:prerelease-lifecycle-gen:deprecated=1.24 +// +k8s:prerelease-lifecycle-gen:replacement=storage.k8s.io,v1,CSIStorageCapacity // CSIStorageCapacity stores the result of one CSI GetCapacity call. // For a given StorageClass, this describes the available capacity in a @@ -602,9 +604,8 @@ type CSINodeList struct { // // The producer of these objects can decide which approach is more suitable. // -// They are consumed by the kube-scheduler if the CSIStorageCapacity beta feature gate -// is enabled there and a CSI driver opts into capacity-aware scheduling with -// CSIDriver.StorageCapacity. +// They are consumed by the kube-scheduler if a CSI driver opts into +// capacity-aware scheduling with CSIDriverSpec.StorageCapacity. type CSIStorageCapacity struct { metav1.TypeMeta `json:",inline"` // Standard object's metadata. The name has no particular meaning. It must be @@ -665,6 +666,8 @@ type CSIStorageCapacity struct { // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +k8s:prerelease-lifecycle-gen:introduced=1.21 +// +k8s:prerelease-lifecycle-gen:deprecated=1.24 +// +k8s:prerelease-lifecycle-gen:replacement=storage.k8s.io,v1,CSIStorageCapacityList // CSIStorageCapacityList is a collection of CSIStorageCapacity objects. type CSIStorageCapacityList struct { diff --git a/test/integration/etcd/data.go b/test/integration/etcd/data.go index 8381ac30ea0..eeb5e99c025 100644 --- a/test/integration/etcd/data.go +++ b/test/integration/etcd/data.go @@ -314,7 +314,6 @@ func GetEtcdStorageDataForNamespace(namespace string) map[schema.GroupVersionRes }, // -- - // k8s.io/kubernetes/pkg/apis/storage/v1beta1 // k8s.io/kubernetes/pkg/apis/storage/v1beta1 gvr("storage.k8s.io", "v1beta1", "csistoragecapacities"): { Stub: `{"metadata": {"name": "csc-12345-2"}, "storageClassName": "sc1"}`, @@ -327,6 +326,11 @@ func GetEtcdStorageDataForNamespace(namespace string) map[schema.GroupVersionRes Stub: `{"metadata": {"name": "sc2"}, "provisioner": "aws"}`, ExpectedEtcdPath: "/registry/storageclasses/sc2", }, + gvr("storage.k8s.io", "v1", "csistoragecapacities"): { + Stub: `{"metadata": {"name": "csc-12345-3"}, "storageClassName": "sc1"}`, + ExpectedEtcdPath: "/registry/csistoragecapacities/" + namespace + "/csc-12345-3", + ExpectedGVK: gvkP("storage.k8s.io", "v1beta1", "CSIStorageCapacity"), + }, // -- // k8s.io/kubernetes/pkg/apis/rbac/v1