diff --git a/pkg/apis/core/helper/helpers.go b/pkg/apis/core/helper/helpers.go index 01913653083..43d482f52d1 100644 --- a/pkg/apis/core/helper/helpers.go +++ b/pkg/apis/core/helper/helpers.go @@ -119,6 +119,7 @@ var standardResourceQuotaScopes = sets.New( core.ResourceQuotaScopeBestEffort, core.ResourceQuotaScopeNotBestEffort, core.ResourceQuotaScopePriorityClass, + core.ResourceQuotaScopeVolumeAttributesClass, ) // IsStandardResourceQuotaScope returns true if the scope is a standard value @@ -139,6 +140,14 @@ var podComputeQuotaResources = sets.New( core.ResourceRequestsMemory, ) +var pvcObjectCountQuotaResources = sets.New( + core.ResourcePersistentVolumeClaims, +) + +var pvcStorageQuotaResources = sets.New( + core.ResourceRequestsStorage, +) + // IsResourceQuotaScopeValidForResource returns true if the resource applies to the specified scope func IsResourceQuotaScopeValidForResource(scope core.ResourceQuotaScope, resource core.ResourceName) bool { switch scope { @@ -147,6 +156,8 @@ func IsResourceQuotaScopeValidForResource(scope core.ResourceQuotaScope, resourc return podObjectCountQuotaResources.Has(resource) || podComputeQuotaResources.Has(resource) case core.ResourceQuotaScopeBestEffort: return podObjectCountQuotaResources.Has(resource) + case core.ResourceQuotaScopeVolumeAttributesClass: + return pvcObjectCountQuotaResources.Has(resource) || pvcStorageQuotaResources.Has(resource) default: return true } diff --git a/pkg/apis/core/types.go b/pkg/apis/core/types.go index 693e6d67daf..7e2c2bef74f 100644 --- a/pkg/apis/core/types.go +++ b/pkg/apis/core/types.go @@ -6048,6 +6048,9 @@ const ( ResourceQuotaScopePriorityClass ResourceQuotaScope = "PriorityClass" // Match all pod objects that have cross-namespace pod (anti)affinity mentioned ResourceQuotaScopeCrossNamespacePodAffinity ResourceQuotaScope = "CrossNamespacePodAffinity" + + // Match all pvc objects that have volume attributes class mentioned. + ResourceQuotaScopeVolumeAttributesClass ResourceQuotaScope = "VolumeAttributesClass" ) // ResourceQuotaSpec defines the desired hard limits to enforce for Quota diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 093699b5b08..17c2f38cc7f 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -30302,7 +30302,7 @@ func schema_k8sio_api_core_v1_ResourceQuotaSpec(ref common.ReferenceCallback) co Default: "", Type: []string{"string"}, Format: "", - Enum: []interface{}{"BestEffort", "CrossNamespacePodAffinity", "NotBestEffort", "NotTerminating", "PriorityClass", "Terminating"}, + Enum: []interface{}{"BestEffort", "CrossNamespacePodAffinity", "NotBestEffort", "NotTerminating", "PriorityClass", "Terminating", "VolumeAttributesClass"}, }, }, }, @@ -30743,11 +30743,11 @@ func schema_k8sio_api_core_v1_ScopedResourceSelectorRequirement(ref common.Refer Properties: map[string]spec.Schema{ "scopeName": { SchemaProps: spec.SchemaProps{ - Description: "The name of the scope that the selector applies to.\n\nPossible enum values:\n - `\"BestEffort\"` Match all pod objects that have best effort quality of service\n - `\"CrossNamespacePodAffinity\"` Match all pod objects that have cross-namespace pod (anti)affinity mentioned.\n - `\"NotBestEffort\"` Match all pod objects that do not have best effort quality of service\n - `\"NotTerminating\"` Match all pod objects where spec.activeDeadlineSeconds is nil\n - `\"PriorityClass\"` Match all pod objects that have priority class mentioned\n - `\"Terminating\"` Match all pod objects where spec.activeDeadlineSeconds >=0", + Description: "The name of the scope that the selector applies to.\n\nPossible enum values:\n - `\"BestEffort\"` Match all pod objects that have best effort quality of service\n - `\"CrossNamespacePodAffinity\"` Match all pod objects that have cross-namespace pod (anti)affinity mentioned.\n - `\"NotBestEffort\"` Match all pod objects that do not have best effort quality of service\n - `\"NotTerminating\"` Match all pod objects where spec.activeDeadlineSeconds is nil\n - `\"PriorityClass\"` Match all pod objects that have priority class mentioned\n - `\"Terminating\"` Match all pod objects where spec.activeDeadlineSeconds >=0\n - `\"VolumeAttributesClass\"` Match all pvc objects that have volume attributes class mentioned.", Default: "", Type: []string{"string"}, Format: "", - Enum: []interface{}{"BestEffort", "CrossNamespacePodAffinity", "NotBestEffort", "NotTerminating", "PriorityClass", "Terminating"}, + Enum: []interface{}{"BestEffort", "CrossNamespacePodAffinity", "NotBestEffort", "NotTerminating", "PriorityClass", "Terminating", "VolumeAttributesClass"}, }, }, "operator": { diff --git a/pkg/quota/v1/evaluator/core/persistent_volume_claims.go b/pkg/quota/v1/evaluator/core/persistent_volume_claims.go index a4da87b496b..592f6080d52 100644 --- a/pkg/quota/v1/evaluator/core/persistent_volume_claims.go +++ b/pkg/quota/v1/evaluator/core/persistent_volume_claims.go @@ -22,8 +22,10 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/admission" quota "k8s.io/apiserver/pkg/quota/v1" "k8s.io/apiserver/pkg/quota/v1/generic" @@ -31,7 +33,9 @@ import ( storagehelpers "k8s.io/component-helpers/storage/volume" api "k8s.io/kubernetes/pkg/apis/core" k8s_api_v1 "k8s.io/kubernetes/pkg/apis/core/v1" + "k8s.io/kubernetes/pkg/apis/core/v1/helper" k8sfeatures "k8s.io/kubernetes/pkg/features" + "k8s.io/utils/ptr" ) // the name used for object count quota @@ -90,26 +94,68 @@ func (p *pvcEvaluator) GroupResource() schema.GroupResource { // Handles returns true if the evaluator should handle the specified operation. func (p *pvcEvaluator) Handles(a admission.Attributes) bool { - if a.GetSubresource() != "" { + op := a.GetOperation() + switch a.GetSubresource() { + case "": + return op == admission.Create || op == admission.Update + case "status": + pvc, err1 := toExternalPersistentVolumeClaimOrError(a.GetObject()) + oldPVC, err2 := toExternalPersistentVolumeClaimOrError(a.GetOldObject()) + if err1 != nil || err2 != nil { + return false + } + return RequiresQuotaReplenish(pvc, oldPVC) + default: return false } - op := a.GetOperation() - return admission.Create == op || admission.Update == op } // Matches returns true if the evaluator matches the specified quota with the provided input item func (p *pvcEvaluator) Matches(resourceQuota *corev1.ResourceQuota, item runtime.Object) (bool, error) { + if utilfeature.DefaultFeatureGate.Enabled(k8sfeatures.VolumeAttributesClass) { + return generic.Matches(resourceQuota, item, p.MatchingResources, pvcMatchesScopeFunc) + } return generic.Matches(resourceQuota, item, p.MatchingResources, generic.MatchesNoScopeFunc) } // MatchingScopes takes the input specified list of scopes and input object. Returns the set of scopes resource matches. -func (p *pvcEvaluator) MatchingScopes(item runtime.Object, scopes []corev1.ScopedResourceSelectorRequirement) ([]corev1.ScopedResourceSelectorRequirement, error) { +func (p *pvcEvaluator) MatchingScopes(item runtime.Object, scopeSelectors []corev1.ScopedResourceSelectorRequirement) ([]corev1.ScopedResourceSelectorRequirement, error) { + if utilfeature.DefaultFeatureGate.Enabled(k8sfeatures.VolumeAttributesClass) { + matchedScopes := []corev1.ScopedResourceSelectorRequirement{} + for _, selector := range scopeSelectors { + match, err := pvcMatchesScopeFunc(selector, item) + if err != nil { + return []corev1.ScopedResourceSelectorRequirement{}, fmt.Errorf("error on matching scope %v: %w", selector, err) + } + if match { + matchedScopes = append(matchedScopes, selector) + } + } + return matchedScopes, nil + } return []corev1.ScopedResourceSelectorRequirement{}, nil } // UncoveredQuotaScopes takes the input matched scopes which are limited by configuration and the matched quota scopes. // It returns the scopes which are in limited scopes but don't have a corresponding covering quota scope func (p *pvcEvaluator) UncoveredQuotaScopes(limitedScopes []corev1.ScopedResourceSelectorRequirement, matchedQuotaScopes []corev1.ScopedResourceSelectorRequirement) ([]corev1.ScopedResourceSelectorRequirement, error) { + if utilfeature.DefaultFeatureGate.Enabled(k8sfeatures.VolumeAttributesClass) { + uncoveredScopes := []corev1.ScopedResourceSelectorRequirement{} + for _, selector := range limitedScopes { + isCovered := false + for _, matchedScopeSelector := range matchedQuotaScopes { + if matchedScopeSelector.ScopeName == selector.ScopeName { + isCovered = true + break + } + } + + if !isCovered { + uncoveredScopes = append(uncoveredScopes, selector) + } + } + return uncoveredScopes, nil + } return []corev1.ScopedResourceSelectorRequirement{}, nil } @@ -202,6 +248,9 @@ func (p *pvcEvaluator) getStorageUsage(pvc *corev1.PersistentVolumeClaim) *resou // UsageStats calculates aggregate usage for the object. func (p *pvcEvaluator) UsageStats(options quota.UsageStatsOptions) (quota.UsageStats, error) { + if utilfeature.DefaultFeatureGate.Enabled(k8sfeatures.VolumeAttributesClass) { + return generic.CalculateUsageStats(options, p.listFuncByNamespace, pvcMatchesScopeFunc, p.Usage) + } return generic.CalculateUsageStats(options, p.listFuncByNamespace, generic.MatchesNoScopeFunc, p.Usage) } @@ -230,5 +279,65 @@ func RequiresQuotaReplenish(pvc, oldPVC *corev1.PersistentVolumeClaim) bool { return true } } + if utilfeature.DefaultFeatureGate.Enabled(k8sfeatures.VolumeAttributesClass) { + oldNames := getReferencedVolumeAttributesClassNames(oldPVC) + newNames := getReferencedVolumeAttributesClassNames(pvc) + if !oldNames.Equal(newNames) { + return true + } + } return false } + +// pvcMatchesScopeFunc is a function that knows how to evaluate if a pvc matches a scope +func pvcMatchesScopeFunc(selector corev1.ScopedResourceSelectorRequirement, object runtime.Object) (bool, error) { + pvc, err := toExternalPersistentVolumeClaimOrError(object) + if err != nil { + return false, err + } + + if selector.ScopeName == corev1.ResourceQuotaScopeVolumeAttributesClass { + if selector.Operator == corev1.ScopeSelectorOpExists { + // This is just checking for existence of a volumeAttributesClass on the pvc, + // no need to take the overhead of selector parsing/evaluation. + vacNames := getReferencedVolumeAttributesClassNames(pvc) + return len(vacNames) != 0, nil + } + return pvcMatchesSelector(pvc, selector) + } + return false, nil +} + +func pvcMatchesSelector(pvc *corev1.PersistentVolumeClaim, selector corev1.ScopedResourceSelectorRequirement) (bool, error) { + labelSelector, err := helper.ScopedResourceSelectorRequirementsAsSelector(selector) + if err != nil { + return false, fmt.Errorf("failed to parse and convert selector: %w", err) + } + + vacNames := getReferencedVolumeAttributesClassNames(pvc) + if len(vacNames) == 0 { + return labelSelector.Matches(labels.Set{}), nil + } + for vacName := range vacNames { + m := labels.Set{string(corev1.ResourceQuotaScopeVolumeAttributesClass): vacName} + if labelSelector.Matches(m) { + return true, nil + } + } + return false, nil +} + +func getReferencedVolumeAttributesClassNames(pvc *corev1.PersistentVolumeClaim) sets.Set[string] { + vacNames := sets.New[string]() + if len(ptr.Deref(pvc.Spec.VolumeAttributesClassName, "")) != 0 { + vacNames.Insert(*pvc.Spec.VolumeAttributesClassName) + } + if len(ptr.Deref(pvc.Status.CurrentVolumeAttributesClassName, "")) != 0 { + vacNames.Insert(*pvc.Status.CurrentVolumeAttributesClassName) + } + modifyStatus := pvc.Status.ModifyVolumeStatus + if modifyStatus != nil && len(modifyStatus.TargetVolumeAttributesClassName) != 0 { + vacNames.Insert(modifyStatus.TargetVolumeAttributesClassName) + } + return vacNames +} diff --git a/pkg/quota/v1/evaluator/core/persistent_volume_claims_test.go b/pkg/quota/v1/evaluator/core/persistent_volume_claims_test.go index 697f679e6ab..5cd9ad48aea 100644 --- a/pkg/quota/v1/evaluator/core/persistent_volume_claims_test.go +++ b/pkg/quota/v1/evaluator/core/persistent_volume_claims_test.go @@ -20,6 +20,7 @@ import ( "reflect" "testing" + "github.com/google/go-cmp/cmp" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,6 +32,7 @@ import ( featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/features" + "k8s.io/utils/ptr" ) func testVolumeClaim(name string, namespace string, spec core.PersistentVolumeClaimSpec) *core.PersistentVolumeClaim { @@ -40,6 +42,122 @@ func testVolumeClaim(name string, namespace string, spec core.PersistentVolumeCl } } +func TestPersistentVolumeClaimEvaluatorMatchingScopes(t *testing.T) { + evaluator := NewPersistentVolumeClaimEvaluator(nil) + testCases := map[string]struct { + claim *core.PersistentVolumeClaim + selectors []corev1.ScopedResourceSelectorRequirement + wantSelectors []corev1.ScopedResourceSelectorRequirement + }{ + "EmptyPVC": { + claim: &core.PersistentVolumeClaim{}, + selectors: []corev1.ScopedResourceSelectorRequirement{ + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpDoesNotExist}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpExists}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpNotIn, Values: []string{"class4"}}, + }, + wantSelectors: []corev1.ScopedResourceSelectorRequirement{ + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpDoesNotExist}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpNotIn, Values: []string{"class4"}}, + }, + }, + "VolumeAttributesClass": { + claim: &core.PersistentVolumeClaim{ + Spec: core.PersistentVolumeClaimSpec{ + VolumeAttributesClassName: ptr.To("class1"), + }, + }, + selectors: []corev1.ScopedResourceSelectorRequirement{ + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpDoesNotExist}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpExists}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class4"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpNotIn, Values: []string{"class4"}}, + }, + wantSelectors: []corev1.ScopedResourceSelectorRequirement{ + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpExists}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpNotIn, Values: []string{"class4"}}, + }, + }, + "VolumeAttributesClassWithTarget": { + claim: &core.PersistentVolumeClaim{ + Spec: core.PersistentVolumeClaimSpec{ + VolumeAttributesClassName: ptr.To("class1"), + }, + Status: core.PersistentVolumeClaimStatus{ + CurrentVolumeAttributesClassName: ptr.To("class2"), + }, + }, + selectors: []corev1.ScopedResourceSelectorRequirement{ + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpDoesNotExist}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpExists}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class2"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1", "class2"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1", "class2", "class4"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class4"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpNotIn, Values: []string{"class4"}}, + }, + wantSelectors: []corev1.ScopedResourceSelectorRequirement{ + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpExists}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class2"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1", "class2"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1", "class2", "class4"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpNotIn, Values: []string{"class4"}}, + }, + }, + "VolumeAttributesClassWithModityStatus": { + claim: &core.PersistentVolumeClaim{ + Spec: core.PersistentVolumeClaimSpec{ + VolumeAttributesClassName: ptr.To("class1"), + }, + Status: core.PersistentVolumeClaimStatus{ + CurrentVolumeAttributesClassName: ptr.To("class2"), + ModifyVolumeStatus: &core.ModifyVolumeStatus{ + TargetVolumeAttributesClassName: "class3", + }, + }, + }, + selectors: []corev1.ScopedResourceSelectorRequirement{ + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpDoesNotExist}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpExists}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class2"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class3"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1", "class2", "class3"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1", "class2", "class3", "class4"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class4"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpNotIn, Values: []string{"class4"}}, + }, + wantSelectors: []corev1.ScopedResourceSelectorRequirement{ + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpExists}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class2"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class3"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1", "class2", "class3"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1", "class2", "class3", "class4"}}, + {ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpNotIn, Values: []string{"class4"}}, + }, + }, + } + + for testName, testCase := range testCases { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeAttributesClass, true) + t.Run(testName, func(t *testing.T) { + gotSelectors, err := evaluator.MatchingScopes(testCase.claim, testCase.selectors) + if err != nil { + t.Error(err) + } + if diff := cmp.Diff(testCase.wantSelectors, gotSelectors); diff != "" { + t.Errorf("%v: unexpected diff (-want, +got):\n%s", testName, diff) + } + }) + } +} + func TestPersistentVolumeClaimEvaluatorUsage(t *testing.T) { classGold := "gold" validClaim := testVolumeClaim("foo", "ns", core.PersistentVolumeClaimSpec{ diff --git a/plugin/pkg/admission/resourcequota/admission_test.go b/plugin/pkg/admission/resourcequota/admission_test.go index 080f52ec5f3..74223fae59d 100644 --- a/plugin/pkg/admission/resourcequota/admission_test.go +++ b/plugin/pkg/admission/resourcequota/admission_test.go @@ -26,18 +26,22 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/admission" genericadmissioninitializer "k8s.io/apiserver/pkg/admission/initializer" "k8s.io/apiserver/pkg/admission/plugin/resourcequota" resourcequotaapi "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" testcore "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" + featuregatetesting "k8s.io/component-base/featuregate/testing" api "k8s.io/kubernetes/pkg/apis/core" controlplaneadmission "k8s.io/kubernetes/pkg/controlplane/apiserver/admission" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/quota/v1/install" ) @@ -982,6 +986,509 @@ func TestAdmitBelowTerminatingQuotaLimitWhenPodScopeUpdated(t *testing.T) { } } +// TestAdmitBelowVolumeAttributesClassQuotaLimit ensures that pvcs with a given vac are charged to the right quota. +// It creates a glod and silver quota, and creates a pvc with the glod class. +// It ensures that the glod quota is incremented, and the silver quota is not. +func TestAdmitBelowVolumeAttributesClassQuotaLimit(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeAttributesClass, true) + + classGold := "gold" + classSilver := "silver" + + resourceQuotaGold := &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "quota-gold", Namespace: "test", ResourceVersion: "124"}, + Spec: corev1.ResourceQuotaSpec{ + ScopeSelector: &corev1.ScopeSelector{ + MatchExpressions: []corev1.ScopedResourceSelectorRequirement{ + { + ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, + Operator: corev1.ScopeSelectorOpIn, + Values: []string{classGold}, + }, + }, + }, + }, + Status: corev1.ResourceQuotaStatus{ + Hard: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("3"), + corev1.ResourceRequestsStorage: resource.MustParse("100Gi"), + }, + Used: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("1"), + corev1.ResourceRequestsStorage: resource.MustParse("10Gi"), + }, + }, + } + resourceQuotaSilver := &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "quota-silver", Namespace: "test", ResourceVersion: "124"}, + Spec: corev1.ResourceQuotaSpec{ + ScopeSelector: &corev1.ScopeSelector{ + MatchExpressions: []corev1.ScopedResourceSelectorRequirement{ + { + ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, + Operator: corev1.ScopeSelectorOpIn, + Values: []string{classSilver}, + }, + }, + }, + }, + Status: corev1.ResourceQuotaStatus{ + Hard: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("3"), + corev1.ResourceRequestsStorage: resource.MustParse("100Gi"), + }, + Used: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("1"), + corev1.ResourceRequestsStorage: resource.MustParse("10Gi"), + }, + }, + } + stopCh := make(chan struct{}) + defer close(stopCh) + + kubeClient := fake.NewSimpleClientset(resourceQuotaGold, resourceQuotaSilver) + informerFactory := informers.NewSharedInformerFactory(kubeClient, 0) + + handler, err := createHandler(kubeClient, informerFactory, stopCh) + if err != nil { + t.Errorf("Error occurred while creating admission plugin: %v", err) + } + + err = informerFactory.Core().V1().ResourceQuotas().Informer().GetIndexer().Add(resourceQuotaGold) + if err != nil { + t.Errorf("Error occurred while adding resource quota to the indexer: %v", err) + } + err = informerFactory.Core().V1().ResourceQuotas().Informer().GetIndexer().Add(resourceQuotaSilver) + if err != nil { + t.Errorf("Error occurred while adding resource quota to the indexer: %v", err) + } + + // create a pvc that references the gold class + newPvc := validPersistentVolumeClaim("allowed-pvc", getVolumeResourceRequirements(api.ResourceList{api.ResourceStorage: resource.MustParse("1Gi")}, api.ResourceList{})) + newPvc.Spec.VolumeAttributesClassName = &classGold + err = handler.Validate(context.TODO(), admission.NewAttributesRecord(newPvc, nil, api.Kind("PersistentVolumeClaim").WithVersion("version"), newPvc.Namespace, newPvc.Name, corev1.Resource("persistentvolumeclaims").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, nil), nil) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if len(kubeClient.Actions()) == 0 { + t.Errorf("Expected a client action") + } + + expectedActionSet := sets.NewString( + strings.Join([]string{"update", "resourcequotas", "status"}, "-"), + ) + actionSet := sets.NewString() + for _, action := range kubeClient.Actions() { + actionSet.Insert(strings.Join([]string{action.GetVerb(), action.GetResource().Resource, action.GetSubresource()}, "-")) + } + if !actionSet.HasAll(expectedActionSet.List()...) { + t.Errorf("Expected actions:\n%v\n but got:\n%v\nDifference:\n%v", expectedActionSet, actionSet, expectedActionSet.Difference(actionSet)) + } + + decimatedActions := removeListWatch(kubeClient.Actions()) + lastActionIndex := len(decimatedActions) - 1 + usage := decimatedActions[lastActionIndex].(testcore.UpdateAction).GetObject().(*corev1.ResourceQuota) + + // ensure only the quota-gold was updated + if usage.Name != resourceQuotaGold.Name { + t.Errorf("Incremented the wrong quota, expected %v, actual %v", resourceQuotaGold.Name, usage.Name) + } + + expectedUsage := corev1.ResourceQuota{ + Status: corev1.ResourceQuotaStatus{ + Hard: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("3"), + corev1.ResourceRequestsStorage: resource.MustParse("100Gi"), + }, + Used: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("2"), + corev1.ResourceRequestsStorage: resource.MustParse("11Gi"), + }, + }, + } + for k, v := range expectedUsage.Status.Used { + actual := usage.Status.Used[k] + actualValue := actual.String() + expectedValue := v.String() + if expectedValue != actualValue { + t.Errorf("Usage Used: Key: %v, Expected: %v, Actual: %v", k, expectedValue, actualValue) + } + } +} + +// TestAdmitBelowVolumeAttributesClassQuotaLimitWhenPVCScopeUpdated ensures that pvcs with a given vac are charged to the right quota. +// It creates three quotas, gold, silver, and copper, and changes the pvc to reference the different classes. +// It ensures that the expected quota is incremented, and others are not. +// +// The Quota admission is intended to fail "high" in this case, and depends on a controller reconciling actual persisted +// use to lower / free the reserved quota. We need always overcount in the admission plugin if something later causes +// the request to be rejected, so you can not reduce quota with requests that aren't completed. +func TestAdmitBelowVolumeAttributesClassQuotaLimitWhenPVCScopeUpdated(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeAttributesClass, true) + + classGold := "gold" + classSilver := "silver" + classCopper := "copper" + + resourceQuotaGold := &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "quota-gold", Namespace: "test", ResourceVersion: "124"}, + Spec: corev1.ResourceQuotaSpec{ + ScopeSelector: &corev1.ScopeSelector{ + MatchExpressions: []corev1.ScopedResourceSelectorRequirement{ + { + ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, + Operator: corev1.ScopeSelectorOpIn, + Values: []string{classGold}, + }, + }, + }, + }, + Status: corev1.ResourceQuotaStatus{ + Hard: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("3"), + corev1.ResourceRequestsStorage: resource.MustParse("100Gi"), + }, + Used: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("1"), + corev1.ResourceRequestsStorage: resource.MustParse("10Gi"), + }, + }, + } + resourceQuotaSilver := &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "quota-silver", Namespace: "test", ResourceVersion: "124"}, + Spec: corev1.ResourceQuotaSpec{ + ScopeSelector: &corev1.ScopeSelector{ + MatchExpressions: []corev1.ScopedResourceSelectorRequirement{ + { + ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, + Operator: corev1.ScopeSelectorOpIn, + Values: []string{classSilver}, + }, + }, + }, + }, + Status: corev1.ResourceQuotaStatus{ + Hard: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("3"), + corev1.ResourceRequestsStorage: resource.MustParse("100Gi"), + }, + Used: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("1"), + corev1.ResourceRequestsStorage: resource.MustParse("10Gi"), + }, + }, + } + resourceQuotaCopper := &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "quota-copper", Namespace: "test", ResourceVersion: "124"}, + Spec: corev1.ResourceQuotaSpec{ + ScopeSelector: &corev1.ScopeSelector{ + MatchExpressions: []corev1.ScopedResourceSelectorRequirement{ + { + ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, + Operator: corev1.ScopeSelectorOpIn, + Values: []string{classCopper}, + }, + }, + }, + }, + Status: corev1.ResourceQuotaStatus{ + Hard: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("3"), + corev1.ResourceRequestsStorage: resource.MustParse("100Gi"), + }, + Used: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("1"), + corev1.ResourceRequestsStorage: resource.MustParse("10Gi"), + }, + }, + } + + expectedUsage := corev1.ResourceQuota{ + Status: corev1.ResourceQuotaStatus{ + Hard: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("3"), + corev1.ResourceRequestsStorage: resource.MustParse("100Gi"), + }, + Used: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("2"), + corev1.ResourceRequestsStorage: resource.MustParse("11Gi"), + }, + }, + } + + testCases := []struct { + desc string + existingQuotas []runtime.Object + subresource string + claims func() (*api.PersistentVolumeClaim, *api.PersistentVolumeClaim) + expectedActionSet sets.Set[string] + expectedQuotaName string + expectedUsage corev1.ResourceQuota + }{ + { + desc: "update the desired class of a pvc from nil to gold", + existingQuotas: []runtime.Object{resourceQuotaGold, resourceQuotaSilver, resourceQuotaCopper}, + claims: func() (*api.PersistentVolumeClaim, *api.PersistentVolumeClaim) { + // old pvc didn't reference any class + existingPVC := validPersistentVolumeClaim("allowed-pvc", getVolumeResourceRequirements(api.ResourceList{api.ResourceStorage: resource.MustParse("1Gi")}, api.ResourceList{})) + existingPVC.ResourceVersion = "1" + existingPVC.Status.Phase = api.ClaimBound + + // updated version references a single class: gold. + newPVC := validPersistentVolumeClaim("allowed-pvc", getVolumeResourceRequirements(api.ResourceList{api.ResourceStorage: resource.MustParse("1Gi")}, api.ResourceList{})) + newPVC.Spec.VolumeAttributesClassName = &classGold + newPVC.Status.Phase = api.ClaimBound + return existingPVC, newPVC + }, + expectedActionSet: sets.New( + strings.Join([]string{"update", "resourcequotas", "status"}, "-"), + ), + expectedQuotaName: resourceQuotaGold.Name, + expectedUsage: expectedUsage, + }, + { + desc: "update the target class of a pvc to gold ", + existingQuotas: []runtime.Object{resourceQuotaGold, resourceQuotaSilver, resourceQuotaCopper}, + subresource: "status", + claims: func() (*api.PersistentVolumeClaim, *api.PersistentVolumeClaim) { + // old pvc referenced a single class: gold. + existingPVC := validPersistentVolumeClaim("allowed-pvc", getVolumeResourceRequirements(api.ResourceList{api.ResourceStorage: resource.MustParse("1Gi")}, api.ResourceList{})) + existingPVC.ResourceVersion = "1" + existingPVC.Spec.VolumeAttributesClassName = &classGold + existingPVC.Status.Phase = api.ClaimBound + + // updated version references a single class: gold. same as the old pvc. + newPVC := validPersistentVolumeClaim("allowed-pvc", getVolumeResourceRequirements(api.ResourceList{api.ResourceStorage: resource.MustParse("1Gi")}, api.ResourceList{})) + newPVC.Spec.VolumeAttributesClassName = &classGold + newPVC.Status.Phase = api.ClaimBound + newPVC.Status.ModifyVolumeStatus = &api.ModifyVolumeStatus{TargetVolumeAttributesClassName: classGold} + return existingPVC, newPVC + }, + expectedActionSet: sets.New[string](), + expectedQuotaName: "", + }, + { + desc: "update the current class of a pvc from nil to gold", + existingQuotas: []runtime.Object{resourceQuotaGold, resourceQuotaSilver, resourceQuotaCopper}, + subresource: "status", + claims: func() (*api.PersistentVolumeClaim, *api.PersistentVolumeClaim) { + // old pvc referenced a single class: gold. + existingPVC := validPersistentVolumeClaim("allowed-pvc", getVolumeResourceRequirements(api.ResourceList{api.ResourceStorage: resource.MustParse("1Gi")}, api.ResourceList{})) + existingPVC.ResourceVersion = "1" + existingPVC.Spec.VolumeAttributesClassName = &classGold + existingPVC.Status.Phase = api.ClaimBound + existingPVC.Status.ModifyVolumeStatus = &api.ModifyVolumeStatus{TargetVolumeAttributesClassName: classGold} + + // updated version references a single class: gold. same as the old pvc. + newPVC := validPersistentVolumeClaim("allowed-pvc", getVolumeResourceRequirements(api.ResourceList{api.ResourceStorage: resource.MustParse("1Gi")}, api.ResourceList{})) + newPVC.Spec.VolumeAttributesClassName = &classGold + newPVC.Status.Phase = api.ClaimBound + newPVC.Status.CurrentVolumeAttributesClassName = &classGold + return existingPVC, newPVC + }, + expectedActionSet: sets.New[string](), + expectedQuotaName: "", + }, + { + desc: "update the desired class of a pvc from gold to silver", + existingQuotas: []runtime.Object{resourceQuotaGold, resourceQuotaSilver, resourceQuotaCopper}, + claims: func() (*api.PersistentVolumeClaim, *api.PersistentVolumeClaim) { + // old pvc referenced the gold class + existingPVC := validPersistentVolumeClaim("allowed-pvc", getVolumeResourceRequirements(api.ResourceList{api.ResourceStorage: resource.MustParse("1Gi")}, api.ResourceList{})) + existingPVC.ResourceVersion = "1" + existingPVC.Spec.VolumeAttributesClassName = &classGold + existingPVC.Status.Phase = api.ClaimBound + existingPVC.Status.CurrentVolumeAttributesClassName = &classGold + + // updated version references two different classes: silver and gold. + newPVC := validPersistentVolumeClaim("allowed-pvc", getVolumeResourceRequirements(api.ResourceList{api.ResourceStorage: resource.MustParse("1Gi")}, api.ResourceList{})) + newPVC.Spec.VolumeAttributesClassName = &classSilver + newPVC.Status.Phase = api.ClaimBound + newPVC.Status.CurrentVolumeAttributesClassName = &classGold + return existingPVC, newPVC + }, + expectedActionSet: sets.New( + strings.Join([]string{"update", "resourcequotas", "status"}, "-"), + ), + expectedQuotaName: resourceQuotaSilver.Name, + expectedUsage: expectedUsage, + }, + { + desc: "update the target class of a pvc to silver", + existingQuotas: []runtime.Object{resourceQuotaGold, resourceQuotaSilver, resourceQuotaCopper}, + subresource: "status", + claims: func() (*api.PersistentVolumeClaim, *api.PersistentVolumeClaim) { + // old pvc referenced two different classes: silver and gold. + existingPVC := validPersistentVolumeClaim("allowed-pvc", getVolumeResourceRequirements(api.ResourceList{api.ResourceStorage: resource.MustParse("1Gi")}, api.ResourceList{})) + existingPVC.ResourceVersion = "1" + existingPVC.Spec.VolumeAttributesClassName = &classSilver + existingPVC.Status.Phase = api.ClaimBound + existingPVC.Status.CurrentVolumeAttributesClassName = &classGold + + // updated version references two different classes: silver and gold. same as the old pvc. + newPVC := validPersistentVolumeClaim("allowed-pvc", getVolumeResourceRequirements(api.ResourceList{api.ResourceStorage: resource.MustParse("1Gi")}, api.ResourceList{})) + newPVC.Spec.VolumeAttributesClassName = &classSilver + newPVC.Status.Phase = api.ClaimBound + newPVC.Status.ModifyVolumeStatus = &api.ModifyVolumeStatus{TargetVolumeAttributesClassName: classSilver} + newPVC.Status.CurrentVolumeAttributesClassName = &classGold + return existingPVC, newPVC + }, + expectedActionSet: sets.New[string](), + expectedQuotaName: "", + }, + { + desc: "update the desired class of a pvc from silver to copper", + existingQuotas: []runtime.Object{resourceQuotaGold, resourceQuotaSilver, resourceQuotaCopper}, + claims: func() (*api.PersistentVolumeClaim, *api.PersistentVolumeClaim) { + // old pvc referenced two different classes: silver and gold. + existingPVC := validPersistentVolumeClaim("allowed-pvc", getVolumeResourceRequirements(api.ResourceList{api.ResourceStorage: resource.MustParse("1Gi")}, api.ResourceList{})) + existingPVC.ResourceVersion = "1" + existingPVC.Spec.VolumeAttributesClassName = &classSilver + existingPVC.Status.Phase = api.ClaimBound + existingPVC.Status.ModifyVolumeStatus = &api.ModifyVolumeStatus{TargetVolumeAttributesClassName: classSilver} + existingPVC.Status.CurrentVolumeAttributesClassName = &classGold + + // updated version references three different classes: copper, silver and gold. + newPVC := validPersistentVolumeClaim("allowed-pvc", getVolumeResourceRequirements(api.ResourceList{api.ResourceStorage: resource.MustParse("1Gi")}, api.ResourceList{})) + newPVC.Spec.VolumeAttributesClassName = &classCopper + newPVC.Status.Phase = api.ClaimBound + newPVC.Status.ModifyVolumeStatus = &api.ModifyVolumeStatus{TargetVolumeAttributesClassName: classSilver} + newPVC.Status.CurrentVolumeAttributesClassName = &classGold + return existingPVC, newPVC + }, + expectedActionSet: sets.New( + strings.Join([]string{"update", "resourcequotas", "status"}, "-"), + ), + expectedQuotaName: resourceQuotaCopper.Name, + expectedUsage: expectedUsage, + }, + { + desc: "allow update pvc status when a quota is exceeded", + existingQuotas: []runtime.Object{resourceQuotaGold, &corev1.ResourceQuota{ + ObjectMeta: resourceQuotaSilver.ObjectMeta, + Spec: corev1.ResourceQuotaSpec{ + ScopeSelector: resourceQuotaSilver.Spec.ScopeSelector, + Hard: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("1"), + corev1.ResourceRequestsStorage: resource.MustParse("10Gi"), + }, + }, + Status: corev1.ResourceQuotaStatus{ + Hard: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("1"), + corev1.ResourceRequestsStorage: resource.MustParse("10Gi"), + }, + Used: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("1"), + corev1.ResourceRequestsStorage: resource.MustParse("10Gi"), + }, + }, + }}, + subresource: "status", + claims: func() (*api.PersistentVolumeClaim, *api.PersistentVolumeClaim) { + // old pvc referenced a single class: gold. + existingPVC := validPersistentVolumeClaim("allowed-pvc", getVolumeResourceRequirements(api.ResourceList{api.ResourceStorage: resource.MustParse("1Gi")}, api.ResourceList{})) + existingPVC.ResourceVersion = "1" + existingPVC.Spec.VolumeAttributesClassName = &classGold + existingPVC.Status.Phase = api.ClaimBound + existingPVC.Status.CurrentVolumeAttributesClassName = &classGold + + // updated version references two different classes: silver and gold. + newPVC := validPersistentVolumeClaim("allowed-pvc", getVolumeResourceRequirements(api.ResourceList{api.ResourceStorage: resource.MustParse("1Gi")}, api.ResourceList{})) + newPVC.Spec.VolumeAttributesClassName = &classGold + newPVC.Status.Phase = api.ClaimBound + newPVC.Status.ModifyVolumeStatus = &api.ModifyVolumeStatus{TargetVolumeAttributesClassName: classGold} + newPVC.Status.CurrentVolumeAttributesClassName = &classSilver + return existingPVC, newPVC + }, + expectedActionSet: sets.New( + strings.Join([]string{"update", "resourcequotas", "status"}, "-"), + ), + expectedQuotaName: resourceQuotaSilver.Name, + expectedUsage: corev1.ResourceQuota{ + Status: corev1.ResourceQuotaStatus{ + Hard: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("1"), + corev1.ResourceRequestsStorage: resource.MustParse("10Gi"), + }, + Used: corev1.ResourceList{ + corev1.ResourcePersistentVolumeClaims: resource.MustParse("2"), + corev1.ResourceRequestsStorage: resource.MustParse("11Gi"), + }, + }, + }, + }, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run(testCase.desc, func(t *testing.T) { + stopCh := make(chan struct{}) + defer close(stopCh) + + kubeClient := fake.NewSimpleClientset(testCase.existingQuotas...) + informerFactory := informers.NewSharedInformerFactory(kubeClient, 0) + + handler, err := createHandler(kubeClient, informerFactory, stopCh) + if err != nil { + t.Errorf("Error occurred while creating admission plugin: %v", err) + } + + for _, obj := range testCase.existingQuotas { + err = informerFactory.Core().V1().ResourceQuotas().Informer().GetIndexer().Add(obj) + if err != nil { + t.Errorf("Error occurred while adding resource quota to the indexer: %v", err) + } + } + + existingPVC, newPVC := testCase.claims() + err = handler.Validate(context.TODO(), admission.NewAttributesRecord(newPVC, existingPVC, api.Kind("PersistentVolumeClaim").WithVersion("version"), newPVC.Namespace, newPVC.Name, corev1.Resource("persistentvolumeclaims").WithVersion("version"), testCase.subresource, admission.Update, &metav1.CreateOptions{}, false, nil), nil) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + if len(testCase.expectedActionSet) == 0 || testCase.expectedQuotaName == "" { + decimatedActions := removeListWatch(kubeClient.Actions()) + if len(decimatedActions) != 0 { + t.Errorf("Expected no actions, but got %v", decimatedActions) + } + return + } + + if len(kubeClient.Actions()) == 0 { + t.Errorf("Expected a client action") + } + + actionSet := sets.New[string]() + for _, action := range kubeClient.Actions() { + actionSet.Insert(strings.Join([]string{action.GetVerb(), action.GetResource().Resource, action.GetSubresource()}, "-")) + } + + if !actionSet.HasAll(sets.List(testCase.expectedActionSet)...) { + t.Errorf("Expected actions:\n%v\n but got:\n%v\nDifference:\n%v", testCase.expectedActionSet, actionSet, testCase.expectedActionSet.Difference(actionSet)) + } + + decimatedActions := removeListWatch(kubeClient.Actions()) + lastActionIndex := len(decimatedActions) - 1 + usage := decimatedActions[lastActionIndex].(testcore.UpdateAction).GetObject().(*corev1.ResourceQuota) + + // ensure only the exoected quota was updated + if usage.Name != testCase.expectedQuotaName { + t.Errorf("Incremented the wrong quota, expected %v, actual %v", testCase.expectedQuotaName, usage.Name) + } + + for k, v := range testCase.expectedUsage.Status.Used { + actual := usage.Status.Used[k] + actualValue := actual.String() + expectedValue := v.String() + if expectedValue != actualValue { + t.Errorf("Usage Used: Key: %v, Expected: %v, Actual: %v", k, expectedValue, actualValue) + } + } + }) + } +} + // TestAdmitBelowBestEffortQuotaLimit creates a best effort and non-best effort quota. // It verifies that best effort pods are properly scoped to the best effort quota document. func TestAdmitBelowBestEffortQuotaLimit(t *testing.T) { diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index 92c34d170c2..6e6179e3283 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -7282,6 +7282,9 @@ const ( ResourceQuotaScopePriorityClass ResourceQuotaScope = "PriorityClass" // Match all pod objects that have cross-namespace pod (anti)affinity mentioned. ResourceQuotaScopeCrossNamespacePodAffinity ResourceQuotaScope = "CrossNamespacePodAffinity" + + // Match all pvc objects that have volume attributes class mentioned. + ResourceQuotaScopeVolumeAttributesClass ResourceQuotaScope = "VolumeAttributesClass" ) // ResourceQuotaSpec defines the desired hard limits to enforce for Quota. diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/resourcequota/controller.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/resourcequota/controller.go index a49692a4140..95c9c84f6f2 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/resourcequota/controller.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/resourcequota/controller.go @@ -610,18 +610,21 @@ func CheckRequest(quotas []corev1.ResourceQuota, a admission.Attributes, evaluat hardResources := quota.ResourceNames(resourceQuota.Status.Hard) requestedUsage := quota.Mask(deltaUsage, hardResources) newUsage := quota.Add(resourceQuota.Status.Used, requestedUsage) - maskedNewUsage := quota.Mask(newUsage, quota.ResourceNames(requestedUsage)) - if allowed, exceeded := quota.LessThanOrEqual(maskedNewUsage, resourceQuota.Status.Hard); !allowed { - failedRequestedUsage := quota.Mask(requestedUsage, exceeded) - failedUsed := quota.Mask(resourceQuota.Status.Used, exceeded) - failedHard := quota.Mask(resourceQuota.Status.Hard, exceeded) - return nil, admission.NewForbidden(a, - fmt.Errorf("exceeded quota: %s, requested: %s, used: %s, limited: %s", - resourceQuota.Name, - prettyPrint(failedRequestedUsage), - prettyPrint(failedUsed), - prettyPrint(failedHard))) + if a.GetSubresource() != "status" { + maskedNewUsage := quota.Mask(newUsage, quota.ResourceNames(requestedUsage)) + + if allowed, exceeded := quota.LessThanOrEqual(maskedNewUsage, resourceQuota.Status.Hard); !allowed { + failedRequestedUsage := quota.Mask(requestedUsage, exceeded) + failedUsed := quota.Mask(resourceQuota.Status.Used, exceeded) + failedHard := quota.Mask(resourceQuota.Status.Hard, exceeded) + return nil, admission.NewForbidden(a, + fmt.Errorf("exceeded quota: %s, requested: %s, used: %s, limited: %s", + resourceQuota.Name, + prettyPrint(failedRequestedUsage), + prettyPrint(failedUsed), + prettyPrint(failedHard))) + } } // update to the new usage number diff --git a/test/e2e/apimachinery/resource_quota.go b/test/e2e/apimachinery/resource_quota.go index fcf8a3df77b..b05cb948583 100644 --- a/test/e2e/apimachinery/resource_quota.go +++ b/test/e2e/apimachinery/resource_quota.go @@ -46,13 +46,16 @@ import ( "k8s.io/client-go/tools/cache" watchtools "k8s.io/client-go/tools/watch" "k8s.io/client-go/util/retry" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/quota/v1/evaluator/core" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" + e2epv "k8s.io/kubernetes/test/e2e/framework/pv" "k8s.io/kubernetes/test/utils/crd" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" @@ -65,6 +68,7 @@ const ( ) var classGold = "gold" +var classSilver = "silver" var extendedResourceName = "example.com/dongle" var _ = SIGDescribe("ResourceQuota", func() { @@ -1270,6 +1274,156 @@ var _ = SIGDescribe("ResourceQuota", func() { }) }) +var _ = SIGDescribe("ResourceQuota", feature.VolumeAttributesClass, framework.WithFeatureGate(features.VolumeAttributesClass), func() { + f := framework.NewDefaultFramework("resourcequota-volumeattributesclass") + f.NamespacePodSecurityLevel = admissionapi.LevelBaseline + + ginkgo.It("should verify ResourceQuota's volume attributes class scope (quota set to pvc count: 1) against 2 pvcs with same volume attributes class.", func(ctx context.Context) { + hard := v1.ResourceList{} + hard[v1.ResourceRequestsStorage] = resource.MustParse("5Gi") + hard[v1.ResourcePersistentVolumeClaims] = resource.MustParse("1") + + ginkgo.By("Creating a ResourceQuota with volume attributes class scope") + quota, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScopeForVolumeAttributesClass("quota-volumeattributesclass", hard, v1.ScopeSelectorOpIn, []string{classGold})) + framework.ExpectNoError(err) + + ginkgo.By("Ensuring ResourceQuota status is calculated") + usedResources := v1.ResourceList{} + usedResources[v1.ResourceRequestsStorage] = resource.MustParse("0") + usedResources[v1.ResourcePersistentVolumeClaims] = resource.MustParse("0") + err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quota.Name, usedResources) + framework.ExpectNoError(err) + + ginkgo.By("Creating a pvc with volume attributes class") + pvc1 := newTestPersistentVolumeClaimForQuota("test-claim-1") + pvc1.Spec.StorageClassName = ptr.To("") + pvc1.Spec.VolumeAttributesClassName = &classGold + pvc1, err = f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(ctx, pvc1, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Ensuring resource quota with volume attributes class scope captures the pvc usage") + usedResources[v1.ResourceRequestsStorage] = resource.MustParse("1Gi") + usedResources[v1.ResourcePersistentVolumeClaims] = resource.MustParse("1") + err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quota.Name, usedResources) + framework.ExpectNoError(err) + + ginkgo.By("Creating 2nd pod with priority class should fail") + pvc2 := newTestPersistentVolumeClaimForQuota("test-claim-2") + pvc2.Spec.StorageClassName = ptr.To("") + pvc2.Spec.VolumeAttributesClassName = &classGold + _, err = f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(ctx, pvc2, metav1.CreateOptions{}) + gomega.Expect(err).To(gomega.MatchError(apierrors.IsForbidden, "expect a forbidden error when creating a PVC that exceeds quota")) + + ginkgo.By("Deleting first pvc") + err = f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Delete(ctx, pvc1.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Ensuring resource quota status released the pvc usage") + usedResources[v1.ResourceRequestsStorage] = resource.MustParse("0") + usedResources[v1.ResourcePersistentVolumeClaims] = resource.MustParse("0") + err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quota.Name, usedResources) + framework.ExpectNoError(err) + }) + + ginkgo.It("should verify ResourceQuota's volume attributes class scope (quota set to pvc count: 1) against a pvc with different volume attributes class.", func(ctx context.Context) { + hard := v1.ResourceList{} + hard[v1.ResourceRequestsStorage] = resource.MustParse("5Gi") + hard[v1.ResourcePersistentVolumeClaims] = resource.MustParse("1") + + ginkgo.By("Creating 2 ResourceQuotas with volume attributes class scope") + quotaGold, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScopeForVolumeAttributesClass("quota-volumeattributesclass-gold", hard, v1.ScopeSelectorOpIn, []string{classGold})) + framework.ExpectNoError(err) + quotaSilver, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScopeForVolumeAttributesClass("quota-volumeattributesclass-silver", hard, v1.ScopeSelectorOpIn, []string{classSilver})) + framework.ExpectNoError(err) + + ginkgo.By("Ensuring all ResourceQuotas status is calculated") + usedResources := v1.ResourceList{} + usedResources[v1.ResourceRequestsStorage] = resource.MustParse("0") + usedResources[v1.ResourcePersistentVolumeClaims] = resource.MustParse("0") + err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaGold.Name, usedResources) + framework.ExpectNoError(err) + err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaSilver.Name, usedResources) + framework.ExpectNoError(err) + + ginkgo.By("Creating a pvc with volume attributes class gold") + pvcName := "test-claim" + pvc, pv := newTestPersistentVolumeClaimWithFakeCSIVolumeForQuota(f.Namespace.Name, pvcName, &classGold) + _, err = f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(ctx, pvc, metav1.CreateOptions{}) + framework.ExpectNoError(err) + _, err = f.ClientSet.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}) + framework.ExpectNoError(err) + ginkgo.DeferCleanup(framework.IgnoreNotFound(f.ClientSet.CoreV1().PersistentVolumes().Delete), pv.Name, metav1.DeleteOptions{}) + + ginkgo.By("Waiting for the PVC to be bound") + pvs, err := e2epv.WaitForPVClaimBoundPhase(ctx, f.ClientSet, []*v1.PersistentVolumeClaim{pvc}, framework.ClaimProvisionTimeout) + framework.ExpectNoError(err) + gomega.Expect(pvs).To(gomega.HaveLen(1)) + gomega.Expect(pvs[0].Name).To(gomega.Equal(pv.Name), "Expected PV %q to be bound to PVC %q", pv.Name, pvc.Name) + + ginkgo.By("Ensuring resource quota with volume attributes class scope captures the pvc usage") + usedResources[v1.ResourceRequestsStorage] = resource.MustParse("1Gi") + usedResources[v1.ResourcePersistentVolumeClaims] = resource.MustParse("1") + err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaGold.Name, usedResources) + framework.ExpectNoError(err) + + ginkgo.By("Updating the desired volume attributes class of the pvc to silver") + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + gotPVC, err := f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return err + } + gotPVC.Spec.VolumeAttributesClassName = &classSilver + _, err = f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Update(ctx, gotPVC, metav1.UpdateOptions{}) + return err + }) + framework.ExpectNoError(err) + + ginkgo.By("Ensuring resource quota with volume attributes class scope captures the pvc usage") + // the pvc references two different classes, one is in the spec which represents the desired class + // and another is in the status which represents the current class. so the actual usage is 1 for each quota. + usedResources[v1.ResourceRequestsStorage] = resource.MustParse("1Gi") + usedResources[v1.ResourcePersistentVolumeClaims] = resource.MustParse("1") + err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaSilver.Name, usedResources) + framework.ExpectNoError(err) + err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaGold.Name, usedResources) + framework.ExpectNoError(err) + + ginkgo.By("Update the pv to have the volume attributes class silver") + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + gotPV, err := f.ClientSet.CoreV1().PersistentVolumes().Get(ctx, pv.Name, metav1.GetOptions{}) + if err != nil { + return err + } + gotPV.Spec.VolumeAttributesClassName = &classSilver + _, err = f.ClientSet.CoreV1().PersistentVolumes().Update(ctx, gotPV, metav1.UpdateOptions{}) + return err + }) + framework.ExpectNoError(err) + + ginkgo.By("Update the current volume attributes class of the pvc to silver") + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { // no real driver, so we have to simulate the driver behavior + gotPVC, err := f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return err + } + gotPVC.Status.CurrentVolumeAttributesClassName = &classSilver + _, err = f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).UpdateStatus(ctx, gotPVC, metav1.UpdateOptions{}) + return err + }) + framework.ExpectNoError(err) + + ginkgo.By("Ensuring resource quota with volume attributes class scope captures the pvc usage") + usedResources[v1.ResourceRequestsStorage] = resource.MustParse("0") + usedResources[v1.ResourcePersistentVolumeClaims] = resource.MustParse("0") + err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaGold.Name, usedResources) + framework.ExpectNoError(err) + usedResources[v1.ResourceRequestsStorage] = resource.MustParse("1Gi") + usedResources[v1.ResourcePersistentVolumeClaims] = resource.MustParse("1") + err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaSilver.Name, usedResources) + framework.ExpectNoError(err) + }) +}) + var _ = SIGDescribe("ResourceQuota", func() { f := framework.NewDefaultFramework("scope-selectors") f.NamespacePodSecurityLevel = admissionapi.LevelBaseline @@ -1967,6 +2121,25 @@ func newTestResourceQuotaWithScopeForPriorityClass(name string, hard v1.Resource } } +// newTestResourceQuotaWithScopeForVolumeAttributesClass returns a quota +// that enforces default constraints for testing with ResourceQuotaScopeVolumeAttributesClass scope +func newTestResourceQuotaWithScopeForVolumeAttributesClass(name string, hard v1.ResourceList, op v1.ScopeSelectorOperator, values []string) *v1.ResourceQuota { + return &v1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1.ResourceQuotaSpec{Hard: hard, + ScopeSelector: &v1.ScopeSelector{ + MatchExpressions: []v1.ScopedResourceSelectorRequirement{ + { + ScopeName: v1.ResourceQuotaScopeVolumeAttributesClass, + Operator: op, + Values: values, + }, + }, + }, + }, + } +} + // newTestResourceQuota returns a quota that enforces default constraints for testing func newTestResourceQuota(name string) *v1.ResourceQuota { hard := v1.ResourceList{} @@ -2100,6 +2273,64 @@ func newTestPersistentVolumeClaimForQuota(name string) *v1.PersistentVolumeClaim } } +func newTestPersistentVolumeClaimWithFakeCSIVolumeForQuota(namespace, name string, vacName *string) (*v1.PersistentVolumeClaim, *v1.PersistentVolume) { + volumeName := fmt.Sprintf("quota-fake-volume-%s-%s", namespace, name) + pvc := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: v1.PersistentVolumeClaimSpec{ + StorageClassName: ptr.To(""), // avoid binding to a real storage class + VolumeAttributesClassName: vacName, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + // prevent disruption to other test workloads in parallel test runs by ensuring the quota + // test claims don't get bound to a volume in use by other tests + "x-test.k8s.io/satisfiable": "fake-volume", + }, + }, + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Gi"), + }, + }, + VolumeName: volumeName, + }, + } + pv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: volumeName, + Labels: map[string]string{ + "x-test.k8s.io/satisfiable": "fake-volume", + }, + }, + Spec: v1.PersistentVolumeSpec{ + StorageClassName: "", + VolumeAttributesClassName: vacName, + PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimDelete, + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Capacity: v1.ResourceList{ + v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Gi"), + }, + PersistentVolumeSource: v1.PersistentVolumeSource{ + CSI: &v1.CSIPersistentVolumeSource{ + Driver: "fake.csi.k8s.io", + VolumeHandle: "volume-handle", + }, + }, + ClaimRef: &v1.ObjectReference{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + Namespace: namespace, + Name: name, + }, + }, + } + return pvc, pv +} + // newTestResourceClaimForQuota returns a simple resource claim func newTestResourceClaimForQuota(name string) *resourceapi.ResourceClaim { return &resourceapi.ResourceClaim{