From eaa1cad7faf4cd8f701bc36381d36172e44edf10 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 12 Sep 2023 15:32:09 +0200 Subject: [PATCH] resource quota: clone PVC quota evaluator for DRA --- .../v1/evaluator/core/resource_claims.go | 237 ++++++++++++++++++ .../v1/evaluator/core/resource_claims_test.go | 225 +++++++++++++++++ 2 files changed, 462 insertions(+) create mode 100644 pkg/quota/v1/evaluator/core/resource_claims.go create mode 100644 pkg/quota/v1/evaluator/core/resource_claims_test.go diff --git a/pkg/quota/v1/evaluator/core/resource_claims.go b/pkg/quota/v1/evaluator/core/resource_claims.go new file mode 100644 index 00000000000..059dd6d1544 --- /dev/null +++ b/pkg/quota/v1/evaluator/core/resource_claims.go @@ -0,0 +1,237 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/admission" + quota "k8s.io/apiserver/pkg/quota/v1" + "k8s.io/apiserver/pkg/quota/v1/generic" + utilfeature "k8s.io/apiserver/pkg/util/feature" + storagehelpers "k8s.io/component-helpers/storage/volume" + api "k8s.io/kubernetes/pkg/apis/core" + k8s_api_v1 "k8s.io/kubernetes/pkg/apis/core/v1" + k8sfeatures "k8s.io/kubernetes/pkg/features" +) + +// the name used for object count quota +var pvcObjectCountName = generic.ObjectCountQuotaResourceNameFor(corev1.SchemeGroupVersion.WithResource("persistentvolumeclaims").GroupResource()) + +// pvcResources are the set of static resources managed by quota associated with pvcs. +// for each resource in this list, it may be refined dynamically based on storage class. +var pvcResources = []corev1.ResourceName{ + corev1.ResourcePersistentVolumeClaims, + corev1.ResourceRequestsStorage, +} + +// storageClassSuffix is the suffix to the qualified portion of storage class resource name. +// For example, if you want to quota storage by storage class, you would have a declaration +// that follows .storageclass.storage.k8s.io/. +// For example: +// * gold.storageclass.storage.k8s.io/: 500Gi +// * bronze.storageclass.storage.k8s.io/requests.storage: 500Gi +const storageClassSuffix string = ".storageclass.storage.k8s.io/" + +/* TODO: prune? +// ResourceByStorageClass returns a quota resource name by storage class. +func ResourceByStorageClass(storageClass string, resourceName corev1.ResourceName) corev1.ResourceName { + return corev1.ResourceName(string(storageClass + storageClassSuffix + string(resourceName))) +} +*/ + +// V1ResourceByStorageClass returns a quota resource name by storage class. +func V1ResourceByStorageClass(storageClass string, resourceName corev1.ResourceName) corev1.ResourceName { + return corev1.ResourceName(string(storageClass + storageClassSuffix + string(resourceName))) +} + +// NewPersistentVolumeClaimEvaluator returns an evaluator that can evaluate persistent volume claims +func NewPersistentVolumeClaimEvaluator(f quota.ListerForResourceFunc) quota.Evaluator { + listFuncByNamespace := generic.ListResourceUsingListerFunc(f, corev1.SchemeGroupVersion.WithResource("persistentvolumeclaims")) + pvcEvaluator := &pvcEvaluator{listFuncByNamespace: listFuncByNamespace} + return pvcEvaluator +} + +// pvcEvaluator knows how to evaluate quota usage for persistent volume claims +type pvcEvaluator struct { + // listFuncByNamespace knows how to list pvc claims + listFuncByNamespace generic.ListFuncByNamespace +} + +// Constraints verifies that all required resources are present on the item. +func (p *pvcEvaluator) Constraints(required []corev1.ResourceName, item runtime.Object) error { + // no-op for persistent volume claims + return nil +} + +// GroupResource that this evaluator tracks +func (p *pvcEvaluator) GroupResource() schema.GroupResource { + return corev1.SchemeGroupVersion.WithResource("persistentvolumeclaims").GroupResource() +} + +// Handles returns true if the evaluator should handle the specified operation. +func (p *pvcEvaluator) Handles(a admission.Attributes) bool { + op := a.GetOperation() + if op == admission.Create { + return true + } + if op == admission.Update { + return true + } + return false +} + +// Matches returns true if the evaluator matches the specified quota with the provided input item +func (p *pvcEvaluator) Matches(resourceQuota *corev1.ResourceQuota, item runtime.Object) (bool, error) { + return generic.Matches(resourceQuota, item, p.MatchingResources, generic.MatchesNoScopeFunc) +} + +// MatchingScopes takes the input specified list of scopes and input object. Returns the set of scopes resource matches. +func (p *pvcEvaluator) MatchingScopes(item runtime.Object, scopes []corev1.ScopedResourceSelectorRequirement) ([]corev1.ScopedResourceSelectorRequirement, error) { + return []corev1.ScopedResourceSelectorRequirement{}, nil +} + +// UncoveredQuotaScopes takes the input matched scopes which are limited by configuration and the matched quota scopes. +// It returns the scopes which are in limited scopes but don't have a corresponding covering quota scope +func (p *pvcEvaluator) UncoveredQuotaScopes(limitedScopes []corev1.ScopedResourceSelectorRequirement, matchedQuotaScopes []corev1.ScopedResourceSelectorRequirement) ([]corev1.ScopedResourceSelectorRequirement, error) { + return []corev1.ScopedResourceSelectorRequirement{}, nil +} + +// MatchingResources takes the input specified list of resources and returns the set of resources it matches. +func (p *pvcEvaluator) MatchingResources(items []corev1.ResourceName) []corev1.ResourceName { + result := []corev1.ResourceName{} + for _, item := range items { + // match object count quota fields + if quota.Contains([]corev1.ResourceName{pvcObjectCountName}, item) { + result = append(result, item) + continue + } + // match pvc resources + if quota.Contains(pvcResources, item) { + result = append(result, item) + continue + } + // match pvc resources scoped by storage class (.storageclass.storage.k8s.io/) + for _, resource := range pvcResources { + byStorageClass := storageClassSuffix + string(resource) + if strings.HasSuffix(string(item), byStorageClass) { + result = append(result, item) + break + } + } + } + return result +} + +// Usage knows how to measure usage associated with item. +func (p *pvcEvaluator) Usage(item runtime.Object) (corev1.ResourceList, error) { + result := corev1.ResourceList{} + pvc, err := toExternalPersistentVolumeClaimOrError(item) + if err != nil { + return result, err + } + + // charge for claim + result[corev1.ResourcePersistentVolumeClaims] = *(resource.NewQuantity(1, resource.DecimalSI)) + result[pvcObjectCountName] = *(resource.NewQuantity(1, resource.DecimalSI)) + storageClassRef := storagehelpers.GetPersistentVolumeClaimClass(pvc) + if len(storageClassRef) > 0 { + storageClassClaim := corev1.ResourceName(storageClassRef + storageClassSuffix + string(corev1.ResourcePersistentVolumeClaims)) + result[storageClassClaim] = *(resource.NewQuantity(1, resource.DecimalSI)) + } + + requestedStorage := p.getStorageUsage(pvc) + if requestedStorage != nil { + result[corev1.ResourceRequestsStorage] = *requestedStorage + // charge usage to the storage class (if present) + if len(storageClassRef) > 0 { + storageClassStorage := corev1.ResourceName(storageClassRef + storageClassSuffix + string(corev1.ResourceRequestsStorage)) + result[storageClassStorage] = *requestedStorage + } + } + + return result, nil +} + +func (p *pvcEvaluator) getStorageUsage(pvc *corev1.PersistentVolumeClaim) *resource.Quantity { + var result *resource.Quantity + roundUpFunc := func(i *resource.Quantity) *resource.Quantity { + roundedRequest := i.DeepCopy() + if !roundedRequest.RoundUp(0) { + // Ensure storage requests are counted as whole byte values, to pass resourcequota validation. + // See https://issue.k8s.io/94313 + return &roundedRequest + } + return i + } + + if userRequest, ok := pvc.Spec.Resources.Requests[corev1.ResourceStorage]; ok { + result = roundUpFunc(&userRequest) + } + + if utilfeature.DefaultFeatureGate.Enabled(k8sfeatures.RecoverVolumeExpansionFailure) && result != nil { + if len(pvc.Status.AllocatedResources) == 0 { + return result + } + + // if AllocatedResources is set and is greater than user request, we should use it. + if allocatedRequest, ok := pvc.Status.AllocatedResources[corev1.ResourceStorage]; ok { + if allocatedRequest.Cmp(*result) > 0 { + result = roundUpFunc(&allocatedRequest) + } + } + } + return result +} + +// UsageStats calculates aggregate usage for the object. +func (p *pvcEvaluator) UsageStats(options quota.UsageStatsOptions) (quota.UsageStats, error) { + return generic.CalculateUsageStats(options, p.listFuncByNamespace, generic.MatchesNoScopeFunc, p.Usage) +} + +// ensure we implement required interface +var _ quota.Evaluator = &pvcEvaluator{} + +func toExternalPersistentVolumeClaimOrError(obj runtime.Object) (*corev1.PersistentVolumeClaim, error) { + pvc := &corev1.PersistentVolumeClaim{} + switch t := obj.(type) { + case *corev1.PersistentVolumeClaim: + pvc = t + case *api.PersistentVolumeClaim: + if err := k8s_api_v1.Convert_core_PersistentVolumeClaim_To_v1_PersistentVolumeClaim(t, pvc, nil); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("expect *api.PersistentVolumeClaim or *v1.PersistentVolumeClaim, got %v", t) + } + return pvc, nil +} + +// RequiresQuotaReplenish enables quota monitoring for PVCs. +func RequiresQuotaReplenish(pvc, oldPVC *corev1.PersistentVolumeClaim) bool { + if utilfeature.DefaultFeatureGate.Enabled(k8sfeatures.RecoverVolumeExpansionFailure) { + if oldPVC.Status.AllocatedResources.Storage() != pvc.Status.AllocatedResources.Storage() { + return true + } + } + return false +} diff --git a/pkg/quota/v1/evaluator/core/resource_claims_test.go b/pkg/quota/v1/evaluator/core/resource_claims_test.go new file mode 100644 index 00000000000..187c1c78660 --- /dev/null +++ b/pkg/quota/v1/evaluator/core/resource_claims_test.go @@ -0,0 +1,225 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + quota "k8s.io/apiserver/pkg/quota/v1" + "k8s.io/apiserver/pkg/quota/v1/generic" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/features" +) + +func testVolumeClaim(name string, namespace string, spec core.PersistentVolumeClaimSpec) *core.PersistentVolumeClaim { + return &core.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: spec, + } +} + +func TestPersistentVolumeClaimEvaluatorUsage(t *testing.T) { + classGold := "gold" + validClaim := testVolumeClaim("foo", "ns", core.PersistentVolumeClaimSpec{ + Selector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "key2", + Operator: "Exists", + }, + }, + }, + AccessModes: []core.PersistentVolumeAccessMode{ + core.ReadWriteOnce, + core.ReadOnlyMany, + }, + Resources: core.VolumeResourceRequirements{ + Requests: core.ResourceList{ + core.ResourceName(core.ResourceStorage): resource.MustParse("10Gi"), + }, + }, + }) + validClaimByStorageClass := testVolumeClaim("foo", "ns", core.PersistentVolumeClaimSpec{ + Selector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "key2", + Operator: "Exists", + }, + }, + }, + AccessModes: []core.PersistentVolumeAccessMode{ + core.ReadWriteOnce, + core.ReadOnlyMany, + }, + Resources: core.VolumeResourceRequirements{ + Requests: core.ResourceList{ + core.ResourceName(core.ResourceStorage): resource.MustParse("10Gi"), + }, + }, + StorageClassName: &classGold, + }) + + validClaimWithNonIntegerStorage := validClaim.DeepCopy() + validClaimWithNonIntegerStorage.Spec.Resources.Requests[core.ResourceName(core.ResourceStorage)] = resource.MustParse("1001m") + + validClaimByStorageClassWithNonIntegerStorage := validClaimByStorageClass.DeepCopy() + validClaimByStorageClassWithNonIntegerStorage.Spec.Resources.Requests[core.ResourceName(core.ResourceStorage)] = resource.MustParse("1001m") + + evaluator := NewPersistentVolumeClaimEvaluator(nil) + testCases := map[string]struct { + pvc *core.PersistentVolumeClaim + usage corev1.ResourceList + enableRecoverFromExpansion bool + }{ + "pvc-usage": { + pvc: validClaim, + usage: corev1.ResourceList{ + corev1.ResourceRequestsStorage: resource.MustParse("10Gi"), + corev1.ResourcePersistentVolumeClaims: resource.MustParse("1"), + generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "persistentvolumeclaims"}): resource.MustParse("1"), + }, + enableRecoverFromExpansion: true, + }, + "pvc-usage-by-class": { + pvc: validClaimByStorageClass, + usage: corev1.ResourceList{ + corev1.ResourceRequestsStorage: resource.MustParse("10Gi"), + corev1.ResourcePersistentVolumeClaims: resource.MustParse("1"), + V1ResourceByStorageClass(classGold, corev1.ResourceRequestsStorage): resource.MustParse("10Gi"), + V1ResourceByStorageClass(classGold, corev1.ResourcePersistentVolumeClaims): resource.MustParse("1"), + generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "persistentvolumeclaims"}): resource.MustParse("1"), + }, + enableRecoverFromExpansion: true, + }, + + "pvc-usage-rounded": { + pvc: validClaimWithNonIntegerStorage, + usage: corev1.ResourceList{ + corev1.ResourceRequestsStorage: resource.MustParse("2"), // 1001m -> 2 + corev1.ResourcePersistentVolumeClaims: resource.MustParse("1"), + generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "persistentvolumeclaims"}): resource.MustParse("1"), + }, + enableRecoverFromExpansion: true, + }, + "pvc-usage-by-class-rounded": { + pvc: validClaimByStorageClassWithNonIntegerStorage, + usage: corev1.ResourceList{ + corev1.ResourceRequestsStorage: resource.MustParse("2"), // 1001m -> 2 + corev1.ResourcePersistentVolumeClaims: resource.MustParse("1"), + V1ResourceByStorageClass(classGold, corev1.ResourceRequestsStorage): resource.MustParse("2"), // 1001m -> 2 + V1ResourceByStorageClass(classGold, corev1.ResourcePersistentVolumeClaims): resource.MustParse("1"), + generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "persistentvolumeclaims"}): resource.MustParse("1"), + }, + enableRecoverFromExpansion: true, + }, + "pvc-usage-higher-allocated-resource": { + pvc: getPVCWithAllocatedResource("5G", "10G"), + usage: corev1.ResourceList{ + corev1.ResourceRequestsStorage: resource.MustParse("10G"), + corev1.ResourcePersistentVolumeClaims: resource.MustParse("1"), + generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "persistentvolumeclaims"}): resource.MustParse("1"), + }, + enableRecoverFromExpansion: true, + }, + "pvc-usage-lower-allocated-resource": { + pvc: getPVCWithAllocatedResource("10G", "5G"), + usage: corev1.ResourceList{ + corev1.ResourceRequestsStorage: resource.MustParse("10G"), + corev1.ResourcePersistentVolumeClaims: resource.MustParse("1"), + generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "persistentvolumeclaims"}): resource.MustParse("1"), + }, + enableRecoverFromExpansion: true, + }, + } + for testName, testCase := range testCases { + t.Run(testName, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RecoverVolumeExpansionFailure, testCase.enableRecoverFromExpansion)() + actual, err := evaluator.Usage(testCase.pvc) + if err != nil { + t.Errorf("%s unexpected error: %v", testName, err) + } + if !quota.Equals(testCase.usage, actual) { + t.Errorf("%s expected:\n%v\n, actual:\n%v", testName, testCase.usage, actual) + } + }) + + } +} + +func getPVCWithAllocatedResource(pvcSize, allocatedSize string) *core.PersistentVolumeClaim { + validPVCWithAllocatedResources := testVolumeClaim("foo", "ns", core.PersistentVolumeClaimSpec{ + Resources: core.VolumeResourceRequirements{ + Requests: core.ResourceList{ + core.ResourceStorage: resource.MustParse(pvcSize), + }, + }, + }) + validPVCWithAllocatedResources.Status.AllocatedResources = core.ResourceList{ + core.ResourceName(core.ResourceStorage): resource.MustParse(allocatedSize), + } + return validPVCWithAllocatedResources +} + +func TestPersistentVolumeClaimEvaluatorMatchingResources(t *testing.T) { + evaluator := NewPersistentVolumeClaimEvaluator(nil) + testCases := map[string]struct { + items []corev1.ResourceName + want []corev1.ResourceName + }{ + "supported-resources": { + items: []corev1.ResourceName{ + "count/persistentvolumeclaims", + "requests.storage", + "persistentvolumeclaims", + "gold.storageclass.storage.k8s.io/requests.storage", + "gold.storageclass.storage.k8s.io/persistentvolumeclaims", + }, + + want: []corev1.ResourceName{ + "count/persistentvolumeclaims", + "requests.storage", + "persistentvolumeclaims", + "gold.storageclass.storage.k8s.io/requests.storage", + "gold.storageclass.storage.k8s.io/persistentvolumeclaims", + }, + }, + "unsupported-resources": { + items: []corev1.ResourceName{ + "storage", + "ephemeral-storage", + "bronze.storageclass.storage.k8s.io/storage", + "gold.storage.k8s.io/requests.storage", + }, + want: []corev1.ResourceName{}, + }, + } + for testName, testCase := range testCases { + actual := evaluator.MatchingResources(testCase.items) + + if !reflect.DeepEqual(testCase.want, actual) { + t.Errorf("%s expected:\n%v\n, actual:\n%v", testName, testCase.want, actual) + } + } +}