From 22aeb81e840184a1436e1cbd9fe1814b3906ade6 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 4 Jun 2020 20:49:25 +0200 Subject: [PATCH] CSIStorageCapacity: CSIStorageCapacity API This adds the CSIStorageCapacity API change for https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/1472-storage-capacity-tracking --- api/api-rules/violation_exceptions.list | 1 + pkg/apis/storage/register.go | 2 + pkg/apis/storage/types.go | 79 ++++++++ pkg/apis/storage/validation/validation.go | 34 ++++ .../storage/validation/validation_test.go | 101 ++++++++++ pkg/features/kube_features.go | 2 +- .../default_storage_factory_builder.go | 1 + pkg/printers/internalversion/printers.go | 35 ++++ .../storage/csistoragecapacity/doc.go | 19 ++ .../csistoragecapacity/storage/storage.go | 59 ++++++ .../storage/storage_test.go | 153 +++++++++++++++ .../storage/csistoragecapacity/strategy.go | 78 ++++++++ .../csistoragecapacity/strategy_test.go | 182 ++++++++++++++++++ pkg/registry/storage/rest/storage_storage.go | 10 + .../k8s.io/api/storage/v1alpha1/register.go | 2 + .../src/k8s.io/api/storage/v1alpha1/types.go | 82 ++++++++ test/integration/apiserver/print_test.go | 5 + 17 files changed, 844 insertions(+), 1 deletion(-) create mode 100644 pkg/registry/storage/csistoragecapacity/doc.go create mode 100644 pkg/registry/storage/csistoragecapacity/storage/storage.go create mode 100644 pkg/registry/storage/csistoragecapacity/storage/storage_test.go create mode 100644 pkg/registry/storage/csistoragecapacity/strategy.go create mode 100644 pkg/registry/storage/csistoragecapacity/strategy_test.go diff --git a/api/api-rules/violation_exceptions.list b/api/api-rules/violation_exceptions.list index 599e55546cb..63f329b8fa7 100644 --- a/api/api-rules/violation_exceptions.list +++ b/api/api-rules/violation_exceptions.list @@ -265,6 +265,7 @@ API rule violation: list_type_missing,k8s.io/api/storage/v1,CSINodeDriver,Topolo API rule violation: list_type_missing,k8s.io/api/storage/v1,CSINodeSpec,Drivers API rule violation: list_type_missing,k8s.io/api/storage/v1,StorageClass,AllowedTopologies API rule violation: list_type_missing,k8s.io/api/storage/v1,StorageClass,MountOptions +API rule violation: list_type_missing,k8s.io/api/storage/v1alpha1,CSIStorageCapacityList,Items API rule violation: list_type_missing,k8s.io/api/storage/v1beta1,CSIDriverSpec,VolumeLifecycleModes API rule violation: list_type_missing,k8s.io/api/storage/v1beta1,CSINodeDriver,TopologyKeys API rule violation: list_type_missing,k8s.io/api/storage/v1beta1,CSINodeSpec,Drivers diff --git a/pkg/apis/storage/register.go b/pkg/apis/storage/register.go index fffba5fc5a2..bf59e88cd1e 100644 --- a/pkg/apis/storage/register.go +++ b/pkg/apis/storage/register.go @@ -52,6 +52,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &CSINodeList{}, &CSIDriver{}, &CSIDriverList{}, + &CSIStorageCapacity{}, + &CSIStorageCapacityList{}, ) return nil } diff --git a/pkg/apis/storage/types.go b/pkg/apis/storage/types.go index 78185dfdbdb..5c741a17405 100644 --- a/pkg/apis/storage/types.go +++ b/pkg/apis/storage/types.go @@ -17,6 +17,7 @@ limitations under the License. package storage import ( + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" api "k8s.io/kubernetes/pkg/apis/core" ) @@ -424,3 +425,81 @@ type CSINodeList struct { // items is the list of CSINode Items []CSINode } + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// CSIStorageCapacity stores the result of one CSI GetCapacity call. +// For a given StorageClass, this describes the available capacity in a +// particular topology segment. This can be used when considering where to +// instantiate new PersistentVolumes. +// +// For example this can express things like: +// - StorageClass "standard" has "1234 GiB" available in "topology.kubernetes.io/zone=us-east1" +// - StorageClass "localssd" has "10 GiB" available in "kubernetes.io/hostname=knode-abc123" +// +// The following three cases all imply that no capacity is available for +// a certain combination: +// - no object exists with suitable topology and storage class name +// - such an object exists, but the capacity is unset +// - such an object exists, but the capacity is zero +// +// The producer of these objects can decide which approach is more suitable. +// +// This is an alpha feature and only available when the CSIStorageCapacity feature is enabled. +type CSIStorageCapacity struct { + metav1.TypeMeta + // Standard object's metadata. The name has no particular meaning. It must be + // be a DNS subdomain (dots allowed, 253 characters). To ensure that + // there are no conflicts with other CSI drivers on the cluster, the recommendation + // is to use csisc-, a generated name, or a reverse-domain name which ends + // with the unique CSI driver name. + // + // Objects are namespaced. + // + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + metav1.ObjectMeta + + // NodeTopology defines which nodes have access to the storage + // for which capacity was reported. If not set, the storage is + // not accessible from any node in the cluster. If empty, the + // storage is accessible from all nodes. This field is + // immutable. + // + // +optional + NodeTopology *metav1.LabelSelector + + // The name of the StorageClass that the reported capacity applies to. + // It must meet the same requirements as the name of a StorageClass + // object (non-empty, DNS subdomain). If that object no longer exists, + // the CSIStorageCapacity object is obsolete and should be removed by its + // creator. + // This field is immutable. + StorageClassName string + + // Capacity is the value reported by the CSI driver in its GetCapacityResponse + // for a GetCapacityRequest with topology and parameters that match the + // previous fields. + // + // The semantic is currently (CSI spec 1.2) defined as: + // The available capacity, in bytes, of the storage that can be used + // to provision volumes. If not set, that information is currently + // unavailable and treated like zero capacity. + // + // +optional + Capacity *resource.Quantity +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// CSIStorageCapacityList is a collection of CSIStorageCapacity objects. +type CSIStorageCapacityList struct { + metav1.TypeMeta + // Standard list metadata + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + metav1.ListMeta + + // Items is the list of CSIStorageCapacity objects. + Items []CSIStorageCapacity +} diff --git a/pkg/apis/storage/validation/validation.go b/pkg/apis/storage/validation/validation.go index b45fdea6c35..b299f5a03ef 100644 --- a/pkg/apis/storage/validation/validation.go +++ b/pkg/apis/storage/validation/validation.go @@ -22,6 +22,7 @@ import ( "strings" apiequality "k8s.io/apimachinery/pkg/api/equality" + apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" @@ -459,3 +460,36 @@ func validateVolumeLifecycleModes(modes []storage.VolumeLifecycleMode, fldPath * return allErrs } + +// ValidateStorageCapacityName checks that a name is appropriate for a +// CSIStorageCapacity object. +var ValidateStorageCapacityName = apimachineryvalidation.NameIsDNSSubdomain + +// ValidateCSIStorageCapacity validates a CSIStorageCapacity. +func ValidateCSIStorageCapacity(capacity *storage.CSIStorageCapacity) field.ErrorList { + allErrs := apivalidation.ValidateObjectMeta(&capacity.ObjectMeta, true, ValidateStorageCapacityName, field.NewPath("metadata")) + allErrs = append(allErrs, metav1validation.ValidateLabelSelector(capacity.NodeTopology, field.NewPath("nodeTopology"))...) + for _, msg := range apivalidation.ValidateClassName(capacity.StorageClassName, false) { + allErrs = append(allErrs, field.Invalid(field.NewPath("storageClassName"), capacity.StorageClassName, msg)) + } + if capacity.Capacity != nil { + allErrs = append(allErrs, apivalidation.ValidateNonnegativeQuantity(*capacity.Capacity, field.NewPath("capacity"))...) + } + return allErrs +} + +// ValidateCSIStorageCapacityUpdate tests if an update to CSIStorageCapacity is valid. +func ValidateCSIStorageCapacityUpdate(capacity, oldCapacity *storage.CSIStorageCapacity) field.ErrorList { + allErrs := apivalidation.ValidateObjectMetaUpdate(&capacity.ObjectMeta, &oldCapacity.ObjectMeta, field.NewPath("metadata")) + + // Input fields for CSI GetCapacity are immutable. + // If this ever relaxes in the future, make sure to increment the Generation number in PrepareForUpdate + if !apiequality.Semantic.DeepEqual(capacity.NodeTopology, oldCapacity.NodeTopology) { + allErrs = append(allErrs, field.Invalid(field.NewPath("nodeTopology"), capacity.NodeTopology, "field is immutable")) + } + if capacity.StorageClassName != oldCapacity.StorageClassName { + allErrs = append(allErrs, field.Invalid(field.NewPath("storageClassName"), capacity.StorageClassName, "field is immutable")) + } + + return allErrs +} diff --git a/pkg/apis/storage/validation/validation_test.go b/pkg/apis/storage/validation/validation_test.go index 375627f43cc..d335ce89a1a 100644 --- a/pkg/apis/storage/validation/validation_test.go +++ b/pkg/apis/storage/validation/validation_test.go @@ -1939,3 +1939,104 @@ func TestCSIDriverValidationUpdate(t *testing.T) { }) } } + +func TestValidateCSIStorageCapacity(t *testing.T) { + storageClassName := "test-sc" + invalidName := "-invalid-@#$%^&*()-" + + goodCapacity := storage.CSIStorageCapacity{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csc-329803da-fdd2-42e4-af6f-7b07e7ccc305", + Namespace: metav1.NamespaceDefault, + }, + StorageClassName: storageClassName, + } + goodTopology := metav1.LabelSelector{ + MatchLabels: map[string]string{"foo": "bar"}, + } + + scenarios := map[string]struct { + isExpectedFailure bool + capacity *storage.CSIStorageCapacity + }{ + "good-capacity": { + capacity: &goodCapacity, + }, + "missing-storage-class-name": { + isExpectedFailure: true, + capacity: func() *storage.CSIStorageCapacity { + capacity := goodCapacity + capacity.StorageClassName = "" + return &capacity + }(), + }, + "bad-storage-class-name": { + isExpectedFailure: true, + capacity: func() *storage.CSIStorageCapacity { + capacity := goodCapacity + capacity.StorageClassName = invalidName + return &capacity + }(), + }, + "good-capacity-value": { + capacity: func() *storage.CSIStorageCapacity { + capacity := goodCapacity + capacity.Capacity = resource.NewQuantity(1, resource.BinarySI) + return &capacity + }(), + }, + "bad-capacity-value": { + isExpectedFailure: true, + capacity: func() *storage.CSIStorageCapacity { + capacity := goodCapacity + capacity.Capacity = resource.NewQuantity(-11, resource.BinarySI) + return &capacity + }(), + }, + "good-topology": { + capacity: func() *storage.CSIStorageCapacity { + capacity := goodCapacity + capacity.NodeTopology = &goodTopology + return &capacity + }(), + }, + "empty-topology": { + capacity: func() *storage.CSIStorageCapacity { + capacity := goodCapacity + capacity.NodeTopology = &metav1.LabelSelector{} + return &capacity + }(), + }, + "bad-topology-fields": { + isExpectedFailure: true, + capacity: func() *storage.CSIStorageCapacity { + capacity := goodCapacity + capacity.NodeTopology = &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "foo", + Operator: metav1.LabelSelectorOperator("no-such-operator"), + Values: []string{ + "bar", + }, + }, + }, + } + return &capacity + }(), + }, + } + + for name, scenario := range scenarios { + t.Run(name, func(t *testing.T) { + errs := ValidateCSIStorageCapacity(scenario.capacity) + if len(errs) == 0 && scenario.isExpectedFailure { + t.Errorf("Unexpected success") + } + if len(errs) > 0 && !scenario.isExpectedFailure { + t.Errorf("Unexpected failure: %+v", errs) + } + }) + } + +} diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 793fd3b9972..563789ece2e 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -294,7 +294,7 @@ const ( CSIInlineVolume featuregate.Feature = "CSIInlineVolume" // owner: @pohly - // alpha: v1.18 + // alpha: v1.19 // // Enables tracking of available storage capacity that CSI drivers provide. CSIStorageCapacity featuregate.Feature = "CSIStorageCapacity" diff --git a/pkg/kubeapiserver/default_storage_factory_builder.go b/pkg/kubeapiserver/default_storage_factory_builder.go index 35e8ef16207..eaf38a5fe38 100644 --- a/pkg/kubeapiserver/default_storage_factory_builder.go +++ b/pkg/kubeapiserver/default_storage_factory_builder.go @@ -57,6 +57,7 @@ func NewStorageFactoryConfig() *StorageFactoryConfig { networking.Resource("ingresses").WithVersion("v1beta1"), networking.Resource("ingressclasses").WithVersion("v1beta1"), apisstorage.Resource("csidrivers").WithVersion("v1beta1"), + apisstorage.Resource("csistoragecapacities").WithVersion("v1alpha1"), } return &StorageFactoryConfig{ diff --git a/pkg/printers/internalversion/printers.go b/pkg/printers/internalversion/printers.go index ac58d853bbb..ff47448d2ba 100644 --- a/pkg/printers/internalversion/printers.go +++ b/pkg/printers/internalversion/printers.go @@ -39,6 +39,7 @@ import ( rbacv1beta1 "k8s.io/api/rbac/v1beta1" schedulingv1 "k8s.io/api/scheduling/v1" storagev1 "k8s.io/api/storage/v1" + storagev1alpha1 "k8s.io/api/storage/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -515,6 +516,14 @@ func AddHandlers(h printers.PrintHandler) { h.TableHandler(csiDriverColumnDefinitions, printCSIDriver) h.TableHandler(csiDriverColumnDefinitions, printCSIDriverList) + csiStorageCapacityColumnDefinitions := []metav1.TableColumnDefinition{ + {Name: "Name", Type: "string", Format: "name", Description: metav1.ObjectMeta{}.SwaggerDoc()["name"]}, + {Name: "StorageClassName", Type: "string", Description: storagev1alpha1.CSIStorageCapacity{}.SwaggerDoc()["storageClassName"]}, + {Name: "Capacity", Type: "string", Description: storagev1alpha1.CSIStorageCapacity{}.SwaggerDoc()["capacity"]}, + } + h.TableHandler(csiStorageCapacityColumnDefinitions, printCSIStorageCapacity) + h.TableHandler(csiStorageCapacityColumnDefinitions, printCSIStorageCapacityList) + mutatingWebhookColumnDefinitions := []metav1.TableColumnDefinition{ {Name: "Name", Type: "string", Format: "name", Description: metav1.ObjectMeta{}.SwaggerDoc()["name"]}, {Name: "Webhooks", Type: "integer", Description: "Webhooks indicates the number of webhooks registered in this configuration"}, @@ -1373,6 +1382,32 @@ func printCSIDriverList(list *storage.CSIDriverList, options printers.GenerateOp return rows, nil } +func printCSIStorageCapacity(obj *storage.CSIStorageCapacity, options printers.GenerateOptions) ([]metav1.TableRow, error) { + row := metav1.TableRow{ + Object: runtime.RawExtension{Object: obj}, + } + + capacity := "" + if obj.Capacity != nil { + capacity = obj.Capacity.String() + } + + row.Cells = append(row.Cells, obj.Name, obj.StorageClassName, capacity) + return []metav1.TableRow{row}, nil +} + +func printCSIStorageCapacityList(list *storage.CSIStorageCapacityList, options printers.GenerateOptions) ([]metav1.TableRow, error) { + rows := make([]metav1.TableRow, 0, len(list.Items)) + for i := range list.Items { + r, err := printCSIStorageCapacity(&list.Items[i], options) + if err != nil { + return nil, err + } + rows = append(rows, r...) + } + return rows, nil +} + func printMutatingWebhook(obj *admissionregistration.MutatingWebhookConfiguration, options printers.GenerateOptions) ([]metav1.TableRow, error) { row := metav1.TableRow{ Object: runtime.RawExtension{Object: obj}, diff --git a/pkg/registry/storage/csistoragecapacity/doc.go b/pkg/registry/storage/csistoragecapacity/doc.go new file mode 100644 index 00000000000..3b99e2cf6bf --- /dev/null +++ b/pkg/registry/storage/csistoragecapacity/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package csistoragecapacity provides Registry interface and its REST +// implementation for storing csistoragecapacity api objects. +package csistoragecapacity diff --git a/pkg/registry/storage/csistoragecapacity/storage/storage.go b/pkg/registry/storage/csistoragecapacity/storage/storage.go new file mode 100644 index 00000000000..b638d96b0f0 --- /dev/null +++ b/pkg/registry/storage/csistoragecapacity/storage/storage.go @@ -0,0 +1,59 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/registry/generic" + genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" + "k8s.io/apiserver/pkg/registry/rest" + storageapi "k8s.io/kubernetes/pkg/apis/storage" + "k8s.io/kubernetes/pkg/registry/storage/csistoragecapacity" +) + +// CSIStorageCapacityStorage includes storage for CSIStorageCapacity and all subresources +type CSIStorageCapacityStorage struct { + CSIStorageCapacity *REST +} + +// REST object that will work for CSIStorageCapacity +type REST struct { + *genericregistry.Store +} + +// NewStorage returns a RESTStorage object that will work against CSIStorageCapacity +func NewStorage(optsGetter generic.RESTOptionsGetter) (*CSIStorageCapacityStorage, error) { + store := &genericregistry.Store{ + NewFunc: func() runtime.Object { return &storageapi.CSIStorageCapacity{} }, + NewListFunc: func() runtime.Object { return &storageapi.CSIStorageCapacityList{} }, + DefaultQualifiedResource: storageapi.Resource("csistoragecapacities"), + + TableConvertor: rest.NewDefaultTableConvertor(storageapi.Resource("csistoragecapacities")), + + CreateStrategy: csistoragecapacity.Strategy, + UpdateStrategy: csistoragecapacity.Strategy, + DeleteStrategy: csistoragecapacity.Strategy, + } + options := &generic.StoreOptions{RESTOptions: optsGetter} + if err := store.CompleteWithOptions(options); err != nil { + return nil, err + } + + return &CSIStorageCapacityStorage{ + CSIStorageCapacity: &REST{store}, + }, nil +} diff --git a/pkg/registry/storage/csistoragecapacity/storage/storage_test.go b/pkg/registry/storage/csistoragecapacity/storage/storage_test.go new file mode 100644 index 00000000000..4e0f536c0d4 --- /dev/null +++ b/pkg/registry/storage/csistoragecapacity/storage/storage_test.go @@ -0,0 +1,153 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "testing" + + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/registry/generic" + genericregistrytest "k8s.io/apiserver/pkg/registry/generic/testing" + etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing" + storageapi "k8s.io/kubernetes/pkg/apis/storage" + _ "k8s.io/kubernetes/pkg/apis/storage/install" + "k8s.io/kubernetes/pkg/registry/registrytest" +) + +func newStorage(t *testing.T) (*REST, *etcd3testing.EtcdTestServer) { + etcdStorage, server := registrytest.NewEtcdStorageForResource(t, storageapi.SchemeGroupVersion.WithResource("csistoragecapacities").GroupResource()) + restOptions := generic.RESTOptions{ + StorageConfig: etcdStorage, + Decorator: generic.UndecoratedStorage, + DeleteCollectionWorkers: 1, + ResourcePrefix: "csistoragecapacities", + } + csiStorageCapacityStorage, err := NewStorage(restOptions) + if err != nil { + t.Fatalf("unexpected error from REST storage: %v", err) + } + return csiStorageCapacityStorage.CSIStorageCapacity, server +} + +func validNewCSIStorageCapacity(name string) *storageapi.CSIStorageCapacity { + selector := metav1.LabelSelector{ + MatchLabels: map[string]string{"kubernetes.io/hostname": "node-a"}, + } + capacity := resource.MustParse("1Gi") + return &storageapi.CSIStorageCapacity{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: metav1.NamespaceDefault, + }, + NodeTopology: &selector, + StorageClassName: "some-storage-class", + Capacity: &capacity, + } +} + +func TestCreate(t *testing.T) { + storage, server := newStorage(t) + defer server.Terminate(t) + defer storage.Store.DestroyFunc() + test := genericregistrytest.New(t, storage.Store) + csiStorageCapacity := validNewCSIStorageCapacity("foo") + csiStorageCapacity.ObjectMeta = metav1.ObjectMeta{GenerateName: "foo-"} + test.TestCreate( + // valid + csiStorageCapacity, + // invalid + &storageapi.CSIStorageCapacity{ + ObjectMeta: metav1.ObjectMeta{Name: "*BadName!"}, + }, + ) +} + +func TestUpdate(t *testing.T) { + storage, server := newStorage(t) + defer server.Terminate(t) + defer storage.Store.DestroyFunc() + test := genericregistrytest.New(t, storage.Store) + + test.TestUpdate( + // valid + validNewCSIStorageCapacity("foo"), + // updateFunc + func(obj runtime.Object) runtime.Object { + object := obj.(*storageapi.CSIStorageCapacity) + object.Labels = map[string]string{"a": "b"} + return object + }, + //invalid update + func(obj runtime.Object) runtime.Object { + object := obj.(*storageapi.CSIStorageCapacity) + object.Name = "!@#$%" + return object + }, + ) +} + +func TestDelete(t *testing.T) { + storage, server := newStorage(t) + defer server.Terminate(t) + defer storage.Store.DestroyFunc() + test := genericregistrytest.New(t, storage.Store).ReturnDeletedObject() + test.TestDelete(validNewCSIStorageCapacity("foo")) +} + +func TestGet(t *testing.T) { + storage, server := newStorage(t) + defer server.Terminate(t) + defer storage.Store.DestroyFunc() + test := genericregistrytest.New(t, storage.Store) + test.TestGet(validNewCSIStorageCapacity("foo")) +} + +func TestList(t *testing.T) { + storage, server := newStorage(t) + defer server.Terminate(t) + defer storage.Store.DestroyFunc() + test := genericregistrytest.New(t, storage.Store) + test.TestList(validNewCSIStorageCapacity("foo")) +} + +func TestWatch(t *testing.T) { + storage, server := newStorage(t) + defer server.Terminate(t) + defer storage.Store.DestroyFunc() + test := genericregistrytest.New(t, storage.Store) + test.TestWatch( + validNewCSIStorageCapacity("foo"), + // matching labels + []labels.Set{}, + // not matching labels + []labels.Set{ + {"foo": "bar"}, + }, + // matching fields + []fields.Set{ + {"metadata.name": "foo"}, + }, + // not matching fields + []fields.Set{ + {"metadata.name": "bar"}, + }, + ) +} diff --git a/pkg/registry/storage/csistoragecapacity/strategy.go b/pkg/registry/storage/csistoragecapacity/strategy.go new file mode 100644 index 00000000000..35e1df57be7 --- /dev/null +++ b/pkg/registry/storage/csistoragecapacity/strategy.go @@ -0,0 +1,78 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csistoragecapacity + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apiserver/pkg/storage/names" + "k8s.io/kubernetes/pkg/api/legacyscheme" + "k8s.io/kubernetes/pkg/apis/storage" + "k8s.io/kubernetes/pkg/apis/storage/validation" +) + +// csiStorageCapacityStrategy implements behavior for CSIStorageCapacity objects +type csiStorageCapacityStrategy struct { + runtime.ObjectTyper + names.NameGenerator +} + +// Strategy is the default logic that applies when creating and updating +// CSIStorageCapacity objects via the REST API. +var Strategy = csiStorageCapacityStrategy{legacyscheme.Scheme, names.SimpleNameGenerator} + +func (csiStorageCapacityStrategy) NamespaceScoped() bool { + return true +} + +// PrepareForCreate is currently a NOP. +func (csiStorageCapacityStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) { +} + +func (csiStorageCapacityStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList { + csiStorageCapacity := obj.(*storage.CSIStorageCapacity) + + errs := validation.ValidateCSIStorageCapacity(csiStorageCapacity) + errs = append(errs, validation.ValidateCSIStorageCapacity(csiStorageCapacity)...) + + return errs +} + +// Canonicalize normalizes the object after validation. +func (csiStorageCapacityStrategy) Canonicalize(obj runtime.Object) { +} + +func (csiStorageCapacityStrategy) AllowCreateOnUpdate() bool { + return false +} + +// PrepareForUpdate is currently a NOP. +func (csiStorageCapacityStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) { +} + +func (csiStorageCapacityStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList { + newCSIStorageCapacityObj := obj.(*storage.CSIStorageCapacity) + oldCSIStorageCapacityObj := old.(*storage.CSIStorageCapacity) + errorList := validation.ValidateCSIStorageCapacity(newCSIStorageCapacityObj) + return append(errorList, validation.ValidateCSIStorageCapacityUpdate(newCSIStorageCapacityObj, oldCSIStorageCapacityObj)...) +} + +func (csiStorageCapacityStrategy) AllowUnconditionalUpdate() bool { + return false +} diff --git a/pkg/registry/storage/csistoragecapacity/strategy_test.go b/pkg/registry/storage/csistoragecapacity/strategy_test.go new file mode 100644 index 00000000000..048503274fc --- /dev/null +++ b/pkg/registry/storage/csistoragecapacity/strategy_test.go @@ -0,0 +1,182 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csistoragecapacity + +import ( + "testing" + + apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/diff" + genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/apis/storage" + "k8s.io/kubernetes/pkg/features" +) + +// getValidCSIStorageCapacity returns a fully-populated CSIStorageCapacity. +func getValidCSIStorageCapacity(name string, capacityStr string) *storage.CSIStorageCapacity { + mib := resource.MustParse("1Mi") + c := &storage.CSIStorageCapacity{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: metav1.NamespaceDefault, + ResourceVersion: "1", + }, + StorageClassName: "bar", + NodeTopology: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "node", + Operator: metav1.LabelSelectorOpIn, + Values: []string{ + "node1", + }, + }, + }, + }, + Capacity: &mib, + } + if capacityStr != "" { + capacityQuantity := resource.MustParse(capacityStr) + c.Capacity = &capacityQuantity + } + return c +} + +func TestCSIStorageCapacityStrategy(t *testing.T) { + ctx := genericapirequest.WithRequestInfo(genericapirequest.NewContext(), &genericapirequest.RequestInfo{ + APIGroup: "storage.k8s.io", + APIVersion: "v1alphav1", + Resource: "csistoragecapacities", + }) + if !Strategy.NamespaceScoped() { + t.Errorf("CSIStorageCapacity must be namespace scoped") + } + if Strategy.AllowCreateOnUpdate() { + t.Errorf("CSIStorageCapacity should not allow create on update") + } + + capacity := getValidCSIStorageCapacity("valid", "") + original := capacity.DeepCopy() + Strategy.PrepareForCreate(ctx, capacity) + errs := Strategy.Validate(ctx, capacity) + if len(errs) != 0 { + t.Errorf("unexpected error validating %v", errs) + } + + // Create with status should have kept status and all other fields. + if !apiequality.Semantic.DeepEqual(capacity, original) { + t.Errorf("unexpected objects difference after creation: %v", diff.ObjectDiff(original, capacity)) + } + + // Update of immutable fields is disallowed + fields := []struct { + name string + update func(capacity *storage.CSIStorageCapacity) + }{ + { + name: "Topology", + update: func(capacity *storage.CSIStorageCapacity) { + capacity.NodeTopology.MatchLabels = map[string]string{"some-label": "some-value"} + }, + }, + { + name: "StorageClass", + update: func(capacity *storage.CSIStorageCapacity) { + capacity.StorageClassName += "-suffix" + }, + }, + } + for _, field := range fields { + t.Run(field.name, func(t *testing.T) { + newCapacity := capacity.DeepCopy() + field.update(newCapacity) + Strategy.PrepareForUpdate(ctx, newCapacity, capacity) + errs = Strategy.ValidateUpdate(ctx, newCapacity, capacity) + if len(errs) == 0 { + t.Errorf("Expected a validation error") + } + }) + } +} + +func TestCSIStorageCapacityValidation(t *testing.T) { + ctx := genericapirequest.WithRequestInfo(genericapirequest.NewContext(), &genericapirequest.RequestInfo{ + APIGroup: "storage.k8s.io", + APIVersion: "v1alphav1", + Resource: "csistoragecapacities", + }) + + tests := []struct { + name string + expectError bool + old, update *storage.CSIStorageCapacity + }{ + { + name: "before: no capacity, update: 1Gi capacity", + old: getValidCSIStorageCapacity("test", ""), + update: getValidCSIStorageCapacity("test", "1Gi"), + }, + { + name: "before: 1Gi capacity, update: no capacity", + old: getValidCSIStorageCapacity("test", "1Gi"), + update: getValidCSIStorageCapacity("test", ""), + }, + { + name: "name change", + expectError: true, + old: getValidCSIStorageCapacity("a", ""), + update: getValidCSIStorageCapacity("b", ""), + }, + { + name: "storage class name change", + expectError: true, + old: getValidCSIStorageCapacity("test", ""), + update: func() *storage.CSIStorageCapacity { + capacity := getValidCSIStorageCapacity("test", "") + capacity.StorageClassName += "-update" + return capacity + }(), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIStorageCapacity, true)() + + oldCapacity := test.old.DeepCopy() + Strategy.PrepareForCreate(ctx, oldCapacity) + errs := Strategy.Validate(ctx, oldCapacity) + if len(errs) != 0 { + t.Errorf("unexpected validating errors for create: %v", errs) + } + + newCapacity := test.update.DeepCopy() + Strategy.PrepareForUpdate(ctx, newCapacity, test.old) + errs = Strategy.ValidateUpdate(ctx, newCapacity, oldCapacity) + if len(errs) > 0 && !test.expectError { + t.Errorf("unexpected validation failure: %+v", errs) + } + if len(errs) == 0 && test.expectError { + t.Errorf("validation unexpectedly succeeded") + } + }) + } +} diff --git a/pkg/registry/storage/rest/storage_storage.go b/pkg/registry/storage/rest/storage_storage.go index cb86ab2d3b5..9deb79e0f43 100644 --- a/pkg/registry/storage/rest/storage_storage.go +++ b/pkg/registry/storage/rest/storage_storage.go @@ -30,6 +30,7 @@ import ( "k8s.io/kubernetes/pkg/features" csidriverstore "k8s.io/kubernetes/pkg/registry/storage/csidriver/storage" csinodestore "k8s.io/kubernetes/pkg/registry/storage/csinode/storage" + csistoragecapacitystore "k8s.io/kubernetes/pkg/registry/storage/csistoragecapacity/storage" storageclassstore "k8s.io/kubernetes/pkg/registry/storage/storageclass/storage" volumeattachmentstore "k8s.io/kubernetes/pkg/registry/storage/volumeattachment/storage" ) @@ -76,6 +77,15 @@ func (p RESTStorageProvider) v1alpha1Storage(apiResourceConfigSource serverstora } storage["volumeattachments"] = volumeAttachmentStorage.VolumeAttachment + // register csistoragecapacity if CSIStorageCapacity feature gate is enabled + if utilfeature.DefaultFeatureGate.Enabled(features.CSIStorageCapacity) { + csiStorageStorage, err := csistoragecapacitystore.NewStorage(restOptionsGetter) + if err != nil { + return storage, err + } + storage["csistoragecapacities"] = csiStorageStorage.CSIStorageCapacity + } + return storage, nil } diff --git a/staging/src/k8s.io/api/storage/v1alpha1/register.go b/staging/src/k8s.io/api/storage/v1alpha1/register.go index 7b81ee49c2b..779c858028c 100644 --- a/staging/src/k8s.io/api/storage/v1alpha1/register.go +++ b/staging/src/k8s.io/api/storage/v1alpha1/register.go @@ -43,6 +43,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &VolumeAttachment{}, &VolumeAttachmentList{}, + &CSIStorageCapacity{}, + &CSIStorageCapacityList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) diff --git a/staging/src/k8s.io/api/storage/v1alpha1/types.go b/staging/src/k8s.io/api/storage/v1alpha1/types.go index 39408857c26..5e65bcebcf9 100644 --- a/staging/src/k8s.io/api/storage/v1alpha1/types.go +++ b/staging/src/k8s.io/api/storage/v1alpha1/types.go @@ -18,6 +18,7 @@ package v1alpha1 import ( "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -134,3 +135,84 @@ type VolumeError struct { // +optional Message string `json:"message,omitempty" protobuf:"bytes,2,opt,name=message"` } + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// CSIStorageCapacity stores the result of one CSI GetCapacity call. +// For a given StorageClass, this describes the available capacity in a +// particular topology segment. This can be used when considering where to +// instantiate new PersistentVolumes. +// +// For example this can express things like: +// - StorageClass "standard" has "1234 GiB" available in "topology.kubernetes.io/zone=us-east1" +// - StorageClass "localssd" has "10 GiB" available in "kubernetes.io/hostname=knode-abc123" +// +// The following three cases all imply that no capacity is available for +// a certain combination: +// - no object exists with suitable topology and storage class name +// - such an object exists, but the capacity is unset +// - such an object exists, but the capacity is zero +// +// The producer of these objects can decide which approach is more suitable. +// +// This is an alpha feature and only available when the CSIStorageCapacity feature is enabled. +type CSIStorageCapacity struct { + metav1.TypeMeta `json:",inline"` + // Standard object's metadata. The name has no particular meaning. It must be + // be a DNS subdomain (dots allowed, 253 characters). To ensure that + // there are no conflicts with other CSI drivers on the cluster, the recommendation + // is to use csisc-, a generated name, or a reverse-domain name which ends + // with the unique CSI driver name. + // + // Objects are namespaced. + // + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // NodeTopology defines which nodes have access to the storage + // for which capacity was reported. If not set, the storage is + // not accessible from any node in the cluster. If empty, the + // storage is accessible from all nodes. This field is + // immutable. + // + // +optional + NodeTopology *metav1.LabelSelector `json:"nodeTopology,omitempty" protobuf:"bytes,2,opt,name=nodeTopology"` + + // The name of the StorageClass that the reported capacity applies to. + // It must meet the same requirements as the name of a StorageClass + // object (non-empty, DNS subdomain). If that object no longer exists, + // the CSIStorageCapacity object is obsolete and should be removed by its + // creator. + // This field is immutable. + StorageClassName string `json:"storageClassName" protobuf:"bytes,3,name=storageClassName"` + + // Capacity is the value reported by the CSI driver in its GetCapacityResponse + // for a GetCapacityRequest with topology and parameters that match the + // previous fields. + // + // The semantic is currently (CSI spec 1.2) defined as: + // The available capacity, in bytes, of the storage that can be used + // to provision volumes. If not set, that information is currently + // unavailable and treated like zero capacity. + // + // +optional + Capacity *resource.Quantity `json:"capacity,omitempty" protobuf:"bytes,4,opt,name=capacity"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// CSIStorageCapacityList is a collection of CSIStorageCapacity objects. +type CSIStorageCapacityList struct { + metav1.TypeMeta `json:",inline"` + // Standard list metadata + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // Items is the list of CSIStorageCapacity objects. + // +listType=map + // +listMapKey=name + Items []CSIStorageCapacity `json:"items" protobuf:"bytes,2,rep,name=items"` +} diff --git a/test/integration/apiserver/print_test.go b/test/integration/apiserver/print_test.go index 3078f98db60..53f59dcaa1d 100644 --- a/test/integration/apiserver/print_test.go +++ b/test/integration/apiserver/print_test.go @@ -41,13 +41,16 @@ import ( metav1beta1 "k8s.io/apimachinery/pkg/apis/meta/v1beta1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/cli-runtime/pkg/genericclioptions" diskcached "k8s.io/client-go/discovery/cached/disk" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/gengo/examples/set-gen/sets" "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubernetes/pkg/api/legacyscheme" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/printers" printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" "k8s.io/kubernetes/test/integration/framework" @@ -154,6 +157,8 @@ var unservedTypes = map[schema.GroupVersionKind]bool{ } func TestServerSidePrint(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIStorageCapacity, true)() + s, _, closeFn := setupWithResources(t, // additional groupversions needed for the test to run []schema.GroupVersion{