diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/cel/filter_test.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/cel/filter_test.go index a87b24474a0..6f45eb7f57b 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/cel/filter_test.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/cel/filter_test.go @@ -20,12 +20,6 @@ import ( "context" "errors" "fmt" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/selection" - genericfeatures "k8s.io/apiserver/pkg/features" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" "reflect" "strings" "testing" @@ -34,19 +28,25 @@ import ( celtypes "github.com/google/cel-go/common/types" "github.com/stretchr/testify/require" - "k8s.io/utils/pointer" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/admission" celconfig "k8s.io/apiserver/pkg/apis/cel" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" apiservercel "k8s.io/apiserver/pkg/cel" "k8s.io/apiserver/pkg/cel/environment" + genericfeatures "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/utils/pointer" ) type condition struct { @@ -182,6 +182,9 @@ func TestFilter(t *testing.T) { }, } + v130 := version.MajorMinor(1, 30) + v131 := version.MajorMinor(1, 31) + var nilUnstructured *unstructured.Unstructured cases := []struct { name string @@ -195,6 +198,8 @@ func TestFilter(t *testing.T) { namespaceObject *corev1.Namespace strictCost bool enableSelectors bool + + compatibilityVersion *version.Version }{ { name: "valid syntax for object", @@ -494,6 +499,38 @@ func TestFilter(t *testing.T) { APIVersion: "*", }), }, + { + name: "test authorizer error using fieldSelector with 1.30 compatibility", + validations: []ExpressionAccessor{ + &condition{ + Expression: "authorizer.group('apps').resource('deployments').fieldSelector('foo=bar').labelSelector('apple=banana').subresource('status').namespace('test').name('backend').check('create').allowed()", + }, + }, + attributes: newValidAttribute(&podObject, false), + results: []EvaluationResult{ + { + Error: fmt.Errorf("fieldSelector"), + }, + }, + authorizer: newAuthzAllowMatch(authorizer.AttributesRecord{ + ResourceRequest: true, + APIGroup: "apps", + Resource: "deployments", + Subresource: "status", + Namespace: "test", + Name: "backend", + Verb: "create", + APIVersion: "*", + FieldSelectorRequirements: fields.Requirements{ + {Operator: "=", Field: "foo", Value: "bar"}, + }, + LabelSelectorRequirements: labels.Requirements{ + *simpleLabelSelector, + }, + }), + enableSelectors: true, + compatibilityVersion: v130, + }, { name: "test authorizer allow resource check with all fields", validations: []ExpressionAccessor{ @@ -523,7 +560,8 @@ func TestFilter(t *testing.T) { *simpleLabelSelector, }, }), - enableSelectors: true, + enableSelectors: true, + compatibilityVersion: v131, }, { name: "test authorizer allow resource check with parse failures", @@ -550,7 +588,8 @@ func TestFilter(t *testing.T) { FieldSelectorParsingErr: errors.New("invalid selector: 'foo badoperator bar'; can't understand 'foo badoperator bar'"), LabelSelectorParsingErr: errors.New("unable to parse requirement: found 'badoperator', expected: in, notin, =, ==, !=, gt, lt"), }), - enableSelectors: true, + enableSelectors: true, + compatibilityVersion: v131, }, { name: "test authorizer allow resource check with all fields, without gate", @@ -575,6 +614,7 @@ func TestFilter(t *testing.T) { Verb: "create", APIVersion: "*", }), + compatibilityVersion: v131, }, { name: "test authorizer not allowed resource check one incorrect field", @@ -837,9 +877,13 @@ func TestFilter(t *testing.T) { if tc.testPerCallLimit == 0 { tc.testPerCallLimit = celconfig.PerCallLimit } - env, err := environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion(), tc.strictCost).Extend( + compatibilityVersion := tc.compatibilityVersion + if compatibilityVersion == nil { + compatibilityVersion = environment.DefaultCompatibilityVersion() + } + env, err := environment.MustBaseEnvSet(compatibilityVersion, tc.strictCost).Extend( environment.VersionedOptions{ - IntroducedVersion: environment.DefaultCompatibilityVersion(), + IntroducedVersion: compatibilityVersion, ProgramOptions: []celgo.ProgramOption{celgo.CostLimit(tc.testPerCallLimit)}, }, ) @@ -868,12 +912,16 @@ func TestFilter(t *testing.T) { } require.Equal(t, len(evalResults), len(tc.results)) for i, result := range tc.results { - if result.EvalResult != evalResults[i].EvalResult { - t.Errorf("Expected result '%v' but got '%v'", result.EvalResult, evalResults[i].EvalResult) - } if result.Error != nil && !strings.Contains(evalResults[i].Error.Error(), result.Error.Error()) { t.Errorf("Expected result '%v' but got '%v'", result.Error, evalResults[i].Error) } + if result.Error == nil && evalResults[i].Error != nil { + t.Errorf("Expected result '%v' but got error '%v'", result.EvalResult, evalResults[i].Error) + continue + } + if result.EvalResult != evalResults[i].EvalResult { + t.Errorf("Expected result '%v' but got '%v'", result.EvalResult, evalResults[i].EvalResult) + } } }) } diff --git a/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go b/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go index 5e2e995c1d1..563d34e13f7 100644 --- a/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go +++ b/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go @@ -20,6 +20,7 @@ import ( "fmt" "strconv" "sync" + "sync/atomic" "github.com/google/cel-go/cel" "github.com/google/cel-go/checker" @@ -30,6 +31,8 @@ import ( "k8s.io/apimachinery/pkg/util/version" celconfig "k8s.io/apiserver/pkg/apis/cel" "k8s.io/apiserver/pkg/cel/library" + genericfeatures "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" ) @@ -146,6 +149,38 @@ var baseOptsWithoutStrictCost = []VersionedOptions{ library.Format(), }, }, + // Authz selectors + { + IntroducedVersion: version.MajorMinor(1, 31), + FeatureEnabled: func() bool { + enabled := utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AuthorizeWithSelectors) + authzSelectorsLibraryInit.Do(func() { + // Record the first time feature enablement was checked for this library. + // This is checked from integration tests to ensure no cached cel envs + // are constructed before feature enablement is effectively set. + authzSelectorsLibraryEnabled.Store(enabled) + // Uncomment to debug where the first initialization is coming from if needed. + // debug.PrintStack() + }) + return enabled + }, + EnvOptions: []cel.EnvOption{ + library.AuthzSelectors(), + }, + }, +} + +var ( + authzSelectorsLibraryInit sync.Once + authzSelectorsLibraryEnabled atomic.Value +) + +// AuthzSelectorsLibraryEnabled returns whether the AuthzSelectors library was enabled when it was constructed. +// If it has not been contructed yet, this returns `false, false`. +// This is solely for the benefit of the integration tests making sure feature gates get correctly parsed before AuthzSelector ever has to check for enablement. +func AuthzSelectorsLibraryEnabled() (enabled, constructed bool) { + enabled, constructed = authzSelectorsLibraryEnabled.Load().(bool) + return } var StrictCostOpt = VersionedOptions{ diff --git a/staging/src/k8s.io/apiserver/pkg/cel/environment/environment.go b/staging/src/k8s.io/apiserver/pkg/cel/environment/environment.go index b47bc8e984b..8033507eb62 100644 --- a/staging/src/k8s.io/apiserver/pkg/cel/environment/environment.go +++ b/staging/src/k8s.io/apiserver/pkg/cel/environment/environment.go @@ -175,7 +175,15 @@ type VersionedOptions struct { // // Optional. RemovedVersion *version.Version - + // FeatureEnabled returns true if these options are enabled by feature gates, + // and returns false if these options are not enabled due to feature gates. + // + // This takes priority over IntroducedVersion / RemovedVersion for the NewExpressions environment. + // + // The StoredExpressions environment ignores this function. + // + // Optional. + FeatureEnabled func() bool // EnvOptions provides CEL EnvOptions. This may be used to add a cel.Variable, a // cel.Library, or to enable other CEL EnvOptions such as language settings. // @@ -210,7 +218,7 @@ type VersionedOptions struct { // making multiple calls to Extend. func (e *EnvSet) Extend(options ...VersionedOptions) (*EnvSet, error) { if len(options) > 0 { - newExprOpts, err := e.filterAndBuildOpts(e.newExpressions, e.compatibilityVersion, options) + newExprOpts, err := e.filterAndBuildOpts(e.newExpressions, e.compatibilityVersion, true, options) if err != nil { return nil, err } @@ -218,7 +226,7 @@ func (e *EnvSet) Extend(options ...VersionedOptions) (*EnvSet, error) { if err != nil { return nil, err } - storedExprOpt, err := e.filterAndBuildOpts(e.storedExpressions, version.MajorMinor(math.MaxUint, math.MaxUint), options) + storedExprOpt, err := e.filterAndBuildOpts(e.storedExpressions, version.MajorMinor(math.MaxUint, math.MaxUint), false, options) if err != nil { return nil, err } @@ -231,13 +239,26 @@ func (e *EnvSet) Extend(options ...VersionedOptions) (*EnvSet, error) { return e, nil } -func (e *EnvSet) filterAndBuildOpts(base *cel.Env, compatVer *version.Version, opts []VersionedOptions) (cel.EnvOption, error) { +func (e *EnvSet) filterAndBuildOpts(base *cel.Env, compatVer *version.Version, honorFeatureGateEnablement bool, opts []VersionedOptions) (cel.EnvOption, error) { var envOpts []cel.EnvOption var progOpts []cel.ProgramOption var declTypes []*apiservercel.DeclType for _, opt := range opts { + var allowedByFeatureGate, allowedByVersion bool + if opt.FeatureEnabled != nil && honorFeatureGateEnablement { + // Feature-gate-enabled libraries must follow compatible default feature enablement. + // Enabling alpha features in their first release enables libraries the previous API server is unaware of. + allowedByFeatureGate = opt.FeatureEnabled() + if !allowedByFeatureGate { + continue + } + } if compatVer.AtLeast(opt.IntroducedVersion) && (opt.RemovedVersion == nil || compatVer.LessThan(opt.RemovedVersion)) { + allowedByVersion = true + } + + if allowedByFeatureGate || allowedByVersion { envOpts = append(envOpts, opt.EnvOptions...) progOpts = append(progOpts, opt.ProgramOptions...) declTypes = append(declTypes, opt.DeclTypes...) diff --git a/staging/src/k8s.io/apiserver/pkg/cel/library/authz.go b/staging/src/k8s.io/apiserver/pkg/cel/library/authz.go index 63fde879445..1fd489fc916 100644 --- a/staging/src/k8s.io/apiserver/pkg/cel/library/authz.go +++ b/staging/src/k8s.io/apiserver/pkg/cel/library/authz.go @@ -19,12 +19,13 @@ package library import ( "context" "fmt" + "reflect" + "strings" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" genericfeatures "k8s.io/apiserver/pkg/features" utilfeature "k8s.io/apiserver/pkg/util/feature" - "reflect" - "strings" "github.com/google/cel-go/cel" "github.com/google/cel-go/common/types" @@ -198,6 +199,30 @@ import ( // Examples: // // authorizer.group('').resource('pods').namespace('default').check('create').error() +// +// fieldSelector +// +// Takes a string field selector, parses it to field selector requirements, and includes it in the authorization check. +// If the field selector does not parse successfully, no field selector requirements are included in the authorization check. +// Added in Kubernetes 1.31+, Authz library version 1. +// +// .fieldSelector() +// +// Examples: +// +// authorizer.group('').resource('pods').fieldSelector('spec.nodeName=mynode').check('list').allowed() +// +// labelSelector (added in v1, Kubernetes 1.31+) +// +// Takes a string label selector, parses it to label selector requirements, and includes it in the authorization check. +// If the label selector does not parse successfully, no label selector requirements are included in the authorization check. +// Added in Kubernetes 1.31+, Authz library version 1. +// +// .labelSelector() +// +// Examples: +// +// authorizer.group('').resource('pods').labelSelector('app=example').check('list').allowed() func Authz() cel.EnvOption { return cel.Lib(authzLib) } @@ -226,12 +251,6 @@ var authzLibraryDecls = map[string][]cel.FunctionOpt{ "subresource": { cel.MemberOverload("resourcecheck_subresource", []*cel.Type{ResourceCheckType, cel.StringType}, ResourceCheckType, cel.BinaryBinding(resourceCheckSubresource))}, - "fieldSelector": { - cel.MemberOverload("authorizer_fieldselector", []*cel.Type{ResourceCheckType, cel.StringType}, ResourceCheckType, - cel.BinaryBinding(resourceCheckFieldSelector))}, - "labelSelector": { - cel.MemberOverload("authorizer_labelselector", []*cel.Type{ResourceCheckType, cel.StringType}, ResourceCheckType, - cel.BinaryBinding(resourceCheckLabelSelector))}, "namespace": { cel.MemberOverload("resourcecheck_namespace", []*cel.Type{ResourceCheckType, cel.StringType}, ResourceCheckType, cel.BinaryBinding(resourceCheckNamespace))}, @@ -269,6 +288,66 @@ func (*authz) ProgramOptions() []cel.ProgramOption { return []cel.ProgramOption{} } +// AuthzSelectors provides a CEL function library extension for adding fieldSelector and +// labelSelector filters to authorization checks. This requires the Authz library. +// See documentation of the Authz library for use and availability of the authorizer variable. +// +// fieldSelector +// +// Takes a string field selector, parses it to field selector requirements, and includes it in the authorization check. +// If the field selector does not parse successfully, no field selector requirements are included in the authorization check. +// Added in Kubernetes 1.31+. +// +// .fieldSelector() +// +// Examples: +// +// authorizer.group('').resource('pods').fieldSelector('spec.nodeName=mynode').check('list').allowed() +// +// labelSelector +// +// Takes a string label selector, parses it to label selector requirements, and includes it in the authorization check. +// If the label selector does not parse successfully, no label selector requirements are included in the authorization check. +// Added in Kubernetes 1.31+. +// +// .labelSelector() +// +// Examples: +// +// authorizer.group('').resource('pods').labelSelector('app=example').check('list').allowed() +func AuthzSelectors() cel.EnvOption { + return cel.Lib(authzSelectorsLib) +} + +var authzSelectorsLib = &authzSelectors{} + +type authzSelectors struct{} + +func (*authzSelectors) LibraryName() string { + return "k8s.authzSelectors" +} + +var authzSelectorsLibraryDecls = map[string][]cel.FunctionOpt{ + "fieldSelector": { + cel.MemberOverload("authorizer_fieldselector", []*cel.Type{ResourceCheckType, cel.StringType}, ResourceCheckType, + cel.BinaryBinding(resourceCheckFieldSelector))}, + "labelSelector": { + cel.MemberOverload("authorizer_labelselector", []*cel.Type{ResourceCheckType, cel.StringType}, ResourceCheckType, + cel.BinaryBinding(resourceCheckLabelSelector))}, +} + +func (*authzSelectors) CompileOptions() []cel.EnvOption { + options := make([]cel.EnvOption, 0, len(authzSelectorsLibraryDecls)) + for name, overloads := range authzSelectorsLibraryDecls { + options = append(options, cel.Function(name, overloads...)) + } + return options +} + +func (*authzSelectors) ProgramOptions() []cel.ProgramOption { + return []cel.ProgramOption{} +} + func authorizerPath(arg1, arg2 ref.Val) ref.Val { authz, ok := arg1.(authorizerVal) if !ok { diff --git a/staging/src/k8s.io/apiserver/pkg/cel/library/cost.go b/staging/src/k8s.io/apiserver/pkg/cel/library/cost.go index 79980d4bcbb..b71686309ab 100644 --- a/staging/src/k8s.io/apiserver/pkg/cel/library/cost.go +++ b/staging/src/k8s.io/apiserver/pkg/cel/library/cost.go @@ -55,6 +55,25 @@ type CostEstimator struct { SizeEstimator checker.CostEstimator } +const ( + // shortest repeatable selector requirement that allocates a values slice is 2 characters: k, + selectorLengthToRequirementCount = float64(.5) + // the expensive parts to represent each requirement are a struct and a values slice + costPerRequirement = float64(common.ListCreateBaseCost + common.StructCreateBaseCost) +) + +// a selector consists of a list of requirements held in a slice +var baseSelectorCost = checker.CostEstimate{Min: common.ListCreateBaseCost, Max: common.ListCreateBaseCost} + +func selectorCostEstimate(selectorLength checker.SizeEstimate) checker.CostEstimate { + parseCost := selectorLength.MultiplyByCostFactor(common.StringTraversalCostFactor) + + requirementCount := selectorLength.MultiplyByCostFactor(selectorLengthToRequirementCount) + requirementCost := requirementCount.MultiplyByCostFactor(costPerRequirement) + + return baseSelectorCost.Add(parseCost).Add(requirementCost) +} + func (l *CostEstimator) CallCost(function, overloadId string, args []ref.Val, result ref.Val) *uint64 { switch function { case "check": @@ -66,6 +85,13 @@ func (l *CostEstimator) CallCost(function, overloadId string, args []ref.Val, re // All authorization builder and accessor functions have a nominal cost cost := uint64(1) return &cost + case "fieldSelector", "labelSelector": + // field and label selector parse is a string parse into a structured set of requirements + if len(args) >= 2 { + selectorLength := actualSize(args[1]) + cost := selectorCostEstimate(checker.SizeEstimate{Min: selectorLength, Max: selectorLength}) + return &cost.Max + } case "isSorted", "sum", "max", "min", "indexOf", "lastIndexOf": var cost uint64 if len(args) > 0 { @@ -227,6 +253,11 @@ func (l *CostEstimator) EstimateCallCost(function, overloadId string, target *ch case "serviceAccount", "path", "group", "resource", "subresource", "namespace", "name", "allowed", "reason", "error", "errored": // All authorization builder and accessor functions have a nominal cost return &checker.CallEstimate{CostEstimate: checker.CostEstimate{Min: 1, Max: 1}} + case "fieldSelector", "labelSelector": + // field and label selector parse is a string parse into a structured set of requirements + if len(args) == 1 { + return &checker.CallEstimate{CostEstimate: selectorCostEstimate(l.sizeEstimate(args[0]))} + } case "isSorted", "sum", "max", "min", "indexOf", "lastIndexOf": if target != nil { // Charge 1 cost for comparing each element in the list diff --git a/staging/src/k8s.io/apiserver/pkg/cel/library/cost_test.go b/staging/src/k8s.io/apiserver/pkg/cel/library/cost_test.go index b46591cd2d4..260977c2c61 100644 --- a/staging/src/k8s.io/apiserver/pkg/cel/library/cost_test.go +++ b/staging/src/k8s.io/apiserver/pkg/cel/library/cost_test.go @@ -629,6 +629,18 @@ func TestAuthzLibrary(t *testing.T) { expectEstimatedCost: checker.CostEstimate{Min: 6, Max: 6}, expectRuntimeCost: 6, }, + { + name: "fieldSelector", + expr: "authorizer.group('').resource('pods').fieldSelector('spec.nodeName=example-node-name.fully.qualified.domain.name.example.com')", + expectEstimatedCost: checker.CostEstimate{Min: 1821, Max: 1821}, + expectRuntimeCost: 1821, // authorizer(1) + group(1) + resource(1) + fieldSelector(10 + ceil(71/2)*50=1800 + ceil(71*.1)=8) + }, + { + name: "labelSelector", + expr: "authorizer.group('').resource('pods').labelSelector('spec.nodeName=example-node-name.fully.qualified.domain.name.example.com')", + expectEstimatedCost: checker.CostEstimate{Min: 1821, Max: 1821}, + expectRuntimeCost: 1821, // authorizer(1) + group(1) + resource(1) + fieldSelector(10 + ceil(71/2)*50=1800 + ceil(71*.1)=8) + }, { name: "path check allowed", expr: "authorizer.path('/healthz').check('get').allowed()", @@ -1064,6 +1076,7 @@ func testCost(t *testing.T, expr string, expectEsimatedCost checker.CostEstimate Regex(), Lists(), Authz(), + AuthzSelectors(), Quantity(), ext.Sets(), IP(), diff --git a/staging/src/k8s.io/apiserver/pkg/cel/library/library_compatibility_test.go b/staging/src/k8s.io/apiserver/pkg/cel/library/library_compatibility_test.go index d920f9c4ab8..81691d556f0 100644 --- a/staging/src/k8s.io/apiserver/pkg/cel/library/library_compatibility_test.go +++ b/staging/src/k8s.io/apiserver/pkg/cel/library/library_compatibility_test.go @@ -26,7 +26,7 @@ import ( func TestLibraryCompatibility(t *testing.T) { var libs []map[string][]cel.FunctionOpt - libs = append(libs, authzLibraryDecls, listsLibraryDecls, regexLibraryDecls, urlLibraryDecls, quantityLibraryDecls, ipLibraryDecls, cidrLibraryDecls) + libs = append(libs, authzLibraryDecls, listsLibraryDecls, regexLibraryDecls, urlLibraryDecls, quantityLibraryDecls, ipLibraryDecls, cidrLibraryDecls, formatLibraryDecls, authzSelectorsLibraryDecls) functionNames := sets.New[string]() for _, lib := range libs { for name := range lib { @@ -49,6 +49,8 @@ func TestLibraryCompatibility(t *testing.T) { "add", "asApproximateFloat", "asInteger", "compareTo", "isGreaterThan", "isInteger", "isLessThan", "isQuantity", "quantity", "sign", "sub", // Kubernetes <1.30>: "ip", "family", "isUnspecified", "isLoopback", "isLinkLocalMulticast", "isLinkLocalUnicast", "isGlobalUnicast", "ip.isCanonical", "isIP", "cidr", "containsIP", "containsCIDR", "masked", "prefixLength", "isCIDR", "string", + // Kubernetes <1.31>: + "fieldSelector", "labelSelector", "validate", "format.named", // Kubernetes <1.??>: ) diff --git a/test/integration/apiserver/cel/authorizerselector/helper.go b/test/integration/apiserver/cel/authorizerselector/helper.go new file mode 100644 index 00000000000..aeb82897e66 --- /dev/null +++ b/test/integration/apiserver/cel/authorizerselector/helper.go @@ -0,0 +1,292 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package authorizerselector + +import ( + "context" + "fmt" + "regexp" + "strings" + "testing" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + resourcev1alpha2 "k8s.io/api/resource/v1alpha2" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + extclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/cel/environment" + "k8s.io/client-go/kubernetes" + apiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" + "k8s.io/kubernetes/test/integration/framework" + "k8s.io/utils/ptr" +) + +func RunAuthzSelectorsLibraryTests(t *testing.T, featureEnabled bool) { + if _, initialized := environment.AuthzSelectorsLibraryEnabled(); initialized { + // This ensures CEL environments don't get initialized during init(), + // before they can be informed by configured feature gates. + // If this check fails, uncomment the debug.PrintStack() when the authz selectors + // library is first initialized to find the culprit, and modify it to be lazily initialized on first use. + t.Fatalf("authz selector library was initialized before feature gates were finalized (possibly from an init() or package variable)") + } + + // Start the server with the desired feature enablement + server, err := apiservertesting.StartTestServer(t, nil, []string{ + fmt.Sprintf("--feature-gates=AuthorizeWithSelectors=%v", featureEnabled), + "--runtime-config=resource.k8s.io/v1alpha2=true", + }, framework.SharedEtcd()) + if err != nil { + t.Fatal(err) + } + defer server.TearDownFn() + + // Ensure the authz selectors library was initialzed and saw the right feature enablement + if gotEnabled, initialized := environment.AuthzSelectorsLibraryEnabled(); !initialized { + t.Fatalf("authz selector library was not initialized during API server construction") + } else if gotEnabled != featureEnabled { + t.Fatalf("authz selector library enabled=%v, expected %v", gotEnabled, featureEnabled) + } + + // Attempt to create API objects using the fieldSelector and labelSelector authorizer functions, + // and ensure they are only allowed when the feature is enabled. + + c, err := kubernetes.NewForConfig(server.ClientConfig) + if err != nil { + t.Fatalf("Failed to create clientset: %v", err) + } + crdClient, err := extclientset.NewForConfig(server.ClientConfig) + if err != nil { + t.Fatalf("Failed to create clientset: %v", err) + } + + boolFieldSelectorExpression := `type(authorizer.group('').resource('').fieldSelector('')) == string` + stringFieldSelectorExpression := boolFieldSelectorExpression + ` ? 'yes' : 'no'` + fieldSelectorErrorSubstring := `undeclared reference to 'fieldSelector'` + + testcases := []struct { + name string + createObject func() error + expectErrorsWhenEnabled []*regexp.Regexp + expectErrorsWhenDisabled []*regexp.Regexp + }{ + { + name: "ValidatingAdmissionPolicy", + createObject: func() error { + obj := &admissionregistrationv1.ValidatingAdmissionPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "test-with-variables"}, + Spec: admissionregistrationv1.ValidatingAdmissionPolicySpec{ + MatchConstraints: &admissionregistrationv1.MatchResources{ + ResourceRules: []admissionregistrationv1.NamedRuleWithOperations{{ + RuleWithOperations: admissionregistrationv1.RuleWithOperations{ + Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.OperationAll}, + Rule: admissionregistrationv1.Rule{APIGroups: []string{"example.com"}, APIVersions: []string{"*"}, Resources: []string{"*"}}}}}}, + Validations: []admissionregistrationv1.Validation{{ + Expression: boolFieldSelectorExpression, + MessageExpression: stringFieldSelectorExpression}}, + AuditAnnotations: []admissionregistrationv1.AuditAnnotation{{Key: "test", ValueExpression: stringFieldSelectorExpression}}, + MatchConditions: []admissionregistrationv1.MatchCondition{{Name: "test", Expression: boolFieldSelectorExpression}}, + Variables: []admissionregistrationv1.Variable{{Name: "test", Expression: boolFieldSelectorExpression}}}} + _, err := c.AdmissionregistrationV1().ValidatingAdmissionPolicies().Create(context.TODO(), obj, metav1.CreateOptions{}) + return err + }, + expectErrorsWhenEnabled: []*regexp.Regexp{ + // authorizer is not available to messageExpression + regexp.MustCompile(`spec\.validations\[0\]\.messageExpression:.*undeclared reference to 'authorizer'`), + }, + expectErrorsWhenDisabled: []*regexp.Regexp{ + regexp.MustCompile(`spec\.validations\[0\]\.expression:.*` + fieldSelectorErrorSubstring), + // authorizer is not available to messageExpression + regexp.MustCompile(`spec\.validations\[0\]\.messageExpression:.*undeclared reference to 'authorizer'`), + regexp.MustCompile(`spec\.auditAnnotations\[0\]\.valueExpression:.*` + fieldSelectorErrorSubstring), + regexp.MustCompile(`spec\.matchConditions\[0\]\.expression:.*` + fieldSelectorErrorSubstring), + regexp.MustCompile(`spec\.variables\[0\]\.expression:.*` + fieldSelectorErrorSubstring), + }, + }, + { + name: "ValidatingWebhookConfiguration", + createObject: func() error { + obj := &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Webhooks: []admissionregistrationv1.ValidatingWebhook{{ + Name: "test.example.com", + ClientConfig: admissionregistrationv1.WebhookClientConfig{URL: ptr.To("https://127.0.0.1")}, + AdmissionReviewVersions: []string{"v1"}, + SideEffects: ptr.To(admissionregistrationv1.SideEffectClassNone), + Rules: []admissionregistrationv1.RuleWithOperations{{ + Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.OperationAll}, + Rule: admissionregistrationv1.Rule{APIGroups: []string{"example.com"}, APIVersions: []string{"*"}, Resources: []string{"*"}}}}, + MatchConditions: []admissionregistrationv1.MatchCondition{{Name: "test", Expression: boolFieldSelectorExpression}}}}} + _, err := c.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(context.TODO(), obj, metav1.CreateOptions{}) + return err + }, + expectErrorsWhenDisabled: []*regexp.Regexp{ + regexp.MustCompile(`webhooks\[0\]\.matchConditions\[0\]\.expression:.*` + fieldSelectorErrorSubstring), + }, + }, + { + name: "MutatingWebhookConfiguration", + createObject: func() error { + obj := &admissionregistrationv1.MutatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Webhooks: []admissionregistrationv1.MutatingWebhook{{ + Name: "test.example.com", + ClientConfig: admissionregistrationv1.WebhookClientConfig{URL: ptr.To("https://127.0.0.1")}, + AdmissionReviewVersions: []string{"v1"}, + SideEffects: ptr.To(admissionregistrationv1.SideEffectClassNone), + Rules: []admissionregistrationv1.RuleWithOperations{{ + Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.OperationAll}, + Rule: admissionregistrationv1.Rule{APIGroups: []string{"example.com"}, APIVersions: []string{"*"}, Resources: []string{"*"}}}}, + MatchConditions: []admissionregistrationv1.MatchCondition{{Name: "test", Expression: boolFieldSelectorExpression}}}}} + _, err := c.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), obj, metav1.CreateOptions{}) + return err + }, + expectErrorsWhenDisabled: []*regexp.Regexp{ + regexp.MustCompile(`webhooks\[0\]\.matchConditions\[0\]\.expression:.*` + fieldSelectorErrorSubstring), + }, + }, + { + name: "ResourceClaimParameters", + createObject: func() error { + obj := &resourcev1alpha2.ResourceClaimParameters{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + DriverRequests: []resourcev1alpha2.DriverRequests{{ + DriverName: "example.com", + Requests: []resourcev1alpha2.ResourceRequest{{ + ResourceRequestModel: resourcev1alpha2.ResourceRequestModel{ + NamedResources: &resourcev1alpha2.NamedResourcesRequest{Selector: boolFieldSelectorExpression}}}}}}} + _, err := c.ResourceV1alpha2().ResourceClaimParameters("default").Create(context.TODO(), obj, metav1.CreateOptions{}) + return err + }, + // authorizer is not available to resource APIs + expectErrorsWhenEnabled: []*regexp.Regexp{regexp.MustCompile(`driverRequests\[0\]\.requests\[0\]\.namedResources\.selector:.*undeclared reference to 'authorizer'`)}, + expectErrorsWhenDisabled: []*regexp.Regexp{regexp.MustCompile(`driverRequests\[0\]\.requests\[0\]\.namedResources\.selector:.*undeclared reference to 'authorizer'`)}, + }, + { + name: "CustomResourceDefinition - rule", + createObject: func() error { + obj := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "crontabs.apis.example.com"}, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "apis.example.com", + Scope: apiextensionsv1.NamespaceScoped, + Names: apiextensionsv1.CustomResourceDefinitionNames{Plural: "crontabs", Singular: "crontab", Kind: "CronTab", ListKind: "CronTabList"}, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{ + Name: "v1beta1", + Served: true, + Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "spec": { + Type: "object", + XValidations: apiextensionsv1.ValidationRules{{Rule: boolFieldSelectorExpression}}}}}}}}}} + _, err := crdClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), obj, metav1.CreateOptions{}) + return err + }, + // authorizer is not available to CRD validation + expectErrorsWhenEnabled: []*regexp.Regexp{regexp.MustCompile(`x-kubernetes-validations\[0\]\.rule:.*undeclared reference to 'authorizer'`)}, + expectErrorsWhenDisabled: []*regexp.Regexp{regexp.MustCompile(`x-kubernetes-validations\[0\]\.rule:.*undeclared reference to 'authorizer'`)}, + }, + { + name: "CustomResourceDefinition - messageExpression", + createObject: func() error { + obj := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "crontabs.apis.example.com"}, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "apis.example.com", + Scope: apiextensionsv1.NamespaceScoped, + Names: apiextensionsv1.CustomResourceDefinitionNames{Plural: "crontabs", Singular: "crontab", Kind: "CronTab", ListKind: "CronTabList"}, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{ + Name: "v1beta1", + Served: true, + Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "spec": { + Type: "object", + XValidations: apiextensionsv1.ValidationRules{{Rule: `self == oldSelf`, MessageExpression: stringFieldSelectorExpression}}}}}}}}}} + _, err := crdClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), obj, metav1.CreateOptions{}) + return err + }, + // authorizer is not available to CRD validation + expectErrorsWhenEnabled: []*regexp.Regexp{regexp.MustCompile(`x-kubernetes-validations\[0\]\.messageExpression:.*undeclared reference to 'authorizer'`)}, + expectErrorsWhenDisabled: []*regexp.Regexp{regexp.MustCompile(`x-kubernetes-validations\[0\]\.messageExpression:.*undeclared reference to 'authorizer'`)}, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + err := tc.createObject() + + var expectedErrors []*regexp.Regexp + if featureEnabled { + expectedErrors = tc.expectErrorsWhenEnabled + } else { + expectedErrors = tc.expectErrorsWhenDisabled + } + + switch { + case len(expectedErrors) == 0 && err == nil: + // success + case len(expectedErrors) == 0 && err != nil: + t.Fatalf("expected success, got error:\n%s", strings.Join(sets.List(getCauses(t, err)), "\n\n")) + case len(expectedErrors) > 0 && err == nil: + t.Fatalf("expected error, got success") + case len(expectedErrors) > 0 && err != nil: + // make sure errors match expectations + actualCauses := getCauses(t, err) + for _, expectCause := range expectedErrors { + found := false + for _, cause := range actualCauses.UnsortedList() { + if expectCause.MatchString(cause) { + actualCauses.Delete(cause) + found = true + break + } + } + if !found { + t.Errorf("missing error matching %s", expectCause) + } + } + if len(actualCauses) > 0 { + t.Errorf("unexpected errors:\n%s", strings.Join(sets.List(actualCauses), "\n\n")) + } + } + }) + } +} + +func getCauses(t *testing.T, err error) sets.Set[string] { + t.Helper() + status, ok := err.(apierrors.APIStatus) + if !ok { + t.Fatalf("expected API status error, got %#v", err) + } + if len(status.Status().Details.Causes) == 0 { + t.Fatalf("expected API status error with causes, got %#v", err) + } + causes := sets.New[string]() + for _, cause := range status.Status().Details.Causes { + causes.Insert(cause.Field + ": " + cause.Message) + } + return causes +} diff --git a/test/integration/apiserver/cel/authorizerselector/selectordisabled/authzselector_test.go b/test/integration/apiserver/cel/authorizerselector/selectordisabled/authzselector_test.go new file mode 100644 index 00000000000..f49d7be9f1e --- /dev/null +++ b/test/integration/apiserver/cel/authorizerselector/selectordisabled/authzselector_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package selectordisabled + +import ( + "testing" + + "k8s.io/kubernetes/test/integration/apiserver/cel/authorizerselector" +) + +// TestAuthzSelectorsLibraryDisabled ensures that the authzselectors library feature disablement works properly. +// CEL envs and compilers cached per process mean this must be the only test in this package. +func TestAuthzSelectorsLibraryDisabled(t *testing.T) { + authorizerselector.RunAuthzSelectorsLibraryTests(t, false) +} diff --git a/test/integration/apiserver/cel/authorizerselector/selectordisabled/main_test.go b/test/integration/apiserver/cel/authorizerselector/selectordisabled/main_test.go new file mode 100644 index 00000000000..cd04eaf7302 --- /dev/null +++ b/test/integration/apiserver/cel/authorizerselector/selectordisabled/main_test.go @@ -0,0 +1,27 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package selectordisabled + +import ( + "testing" + + "k8s.io/kubernetes/test/integration/framework" +) + +func TestMain(m *testing.M) { + framework.EtcdMain(m.Run) +} diff --git a/test/integration/apiserver/cel/authorizerselector/selectorenabled/authzselector_test.go b/test/integration/apiserver/cel/authorizerselector/selectorenabled/authzselector_test.go new file mode 100644 index 00000000000..0ea69e3a0cb --- /dev/null +++ b/test/integration/apiserver/cel/authorizerselector/selectorenabled/authzselector_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package selectorenabled + +import ( + "testing" + + "k8s.io/kubernetes/test/integration/apiserver/cel/authorizerselector" +) + +// TestAuthzSelectorsLibraryEnabled ensures that the authzselectors library feature enablement works properly. +// CEL envs and compilers cached per process mean this must be the only test in this package. +func TestAuthzSelectorsLibraryEnabled(t *testing.T) { + authorizerselector.RunAuthzSelectorsLibraryTests(t, true) +} diff --git a/test/integration/apiserver/cel/authorizerselector/selectorenabled/main_test.go b/test/integration/apiserver/cel/authorizerselector/selectorenabled/main_test.go new file mode 100644 index 00000000000..36f73921c94 --- /dev/null +++ b/test/integration/apiserver/cel/authorizerselector/selectorenabled/main_test.go @@ -0,0 +1,27 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package selectorenabled + +import ( + "testing" + + "k8s.io/kubernetes/test/integration/framework" +) + +func TestMain(m *testing.M) { + framework.EtcdMain(m.Run) +}