diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/admission_test.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/admission_test.go index 8c382be9919..5af72302639 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/admission_test.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/admission_test.go @@ -36,6 +36,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" utiljson "k8s.io/apimachinery/pkg/util/json" @@ -70,6 +71,13 @@ var ( Kind: paramsGVK.Kind + "List", }, &unstructured.UnstructuredList{}) + res.AddKnownTypeWithName(clusterScopedParamsGVK, &unstructured.Unstructured{}) + res.AddKnownTypeWithName(schema.GroupVersionKind{ + Group: clusterScopedParamsGVK.Group, + Version: clusterScopedParamsGVK.Version, + Kind: clusterScopedParamsGVK.Kind + "List", + }, &unstructured.UnstructuredList{}) + if err := v1alpha1.AddToScheme(res); err != nil { panic(err) } @@ -80,6 +88,13 @@ var ( return res }() + + clusterScopedParamsGVK schema.GroupVersionKind = schema.GroupVersionKind{ + Group: "example.com", + Version: "v1", + Kind: "ClusterScopedParamsConfig", + } + paramsGVK schema.GroupVersionKind = schema.GroupVersionKind{ Group: "example.com", Version: "v1", @@ -95,6 +110,7 @@ var ( }) res.Add(paramsGVK, meta.RESTScopeNamespace) + res.Add(clusterScopedParamsGVK, meta.RESTScopeRoot) res.Add(definitionGVK, meta.RESTScopeRoot) res.Add(bindingGVK, meta.RESTScopeRoot) res.Add(v1.SchemeGroupVersion.WithKind("ConfigMap"), meta.RESTScopeNamespace) @@ -133,6 +149,7 @@ var ( "kind": paramsGVK.Kind, "metadata": map[string]interface{}{ "name": "replicas-test.example.com", + "namespace": "default", "resourceVersion": "1", }, "maxReplicas": int64(3), @@ -149,6 +166,8 @@ var ( ParamRef: &v1alpha1.ParamRef{ Name: fakeParams.GetName(), Namespace: fakeParams.GetNamespace(), + // fake object tracker does not populate defaults + ParameterNotFoundAction: ptrTo(v1alpha1.DenyAction), }, ValidationActions: []v1alpha1.ValidationAction{v1alpha1.Deny}, }, @@ -196,6 +215,40 @@ var ( } ) +func newParam(name, namespace string, labels map[string]string) *unstructured.Unstructured { + if len(namespace) == 0 { + namespace = metav1.NamespaceDefault + } + res := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": paramsGVK.GroupVersion().String(), + "kind": paramsGVK.Kind, + "metadata": map[string]interface{}{ + "name": name, + "namespace": namespace, + "resourceVersion": "1", + }, + }, + } + res.SetLabels(labels) + return res +} + +func newClusterScopedParam(name string, labels map[string]string) *unstructured.Unstructured { + res := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": clusterScopedParamsGVK.GroupVersion().String(), + "kind": clusterScopedParamsGVK.Kind, + "metadata": map[string]interface{}{ + "name": name, + "resourceVersion": "1", + }, + }, + } + res.SetLabels(labels) + return res +} + // Interface which has fake compile functionality for use in tests // So that we can test the controller without pulling in any CEL functionality type fakeCompiler struct { @@ -562,7 +615,14 @@ func (c *celAdmissionController) getCurrentObject(obj runtime.Object) (runtime.O } // Param type. Just check informer for its GVK - item, err := paramInformer.Get(accessor.GetName()) + var item runtime.Object + var err error + if namespace := accessor.GetNamespace(); len(namespace) > 0 { + item, err = paramInformer.Namespaced(namespace).Get(accessor.GetName()) + } else { + item, err = paramInformer.Get(accessor.GetName()) + } + if err != nil { if k8serrors.IsNotFound(err) { return nil, nil @@ -940,7 +1000,9 @@ func TestReconfigureBinding(t *testing.T) { "apiVersion": paramsGVK.GroupVersion().String(), "kind": paramsGVK.Kind, "metadata": map[string]interface{}{ - "name": "replicas-test2.example.com", + "name": "replicas-test2.example.com", + // fake object tracker does not populate missing namespace + "namespace": "default", "resourceVersion": "2", }, "maxReplicas": int64(35), @@ -976,8 +1038,9 @@ func TestReconfigureBinding(t *testing.T) { Spec: v1alpha1.ValidatingAdmissionPolicyBindingSpec{ PolicyName: denyPolicy.Name, ParamRef: &v1alpha1.ParamRef{ - Name: fakeParams2.GetName(), - Namespace: fakeParams2.GetNamespace(), + Name: fakeParams2.GetName(), + Namespace: fakeParams2.GetNamespace(), + ParameterNotFoundAction: ptrTo(v1alpha1.DenyAction), }, ValidationActions: []v1alpha1.ValidationAction{v1alpha1.Deny}, }, @@ -1019,7 +1082,7 @@ func TestReconfigureBinding(t *testing.T) { &admission.RuntimeObjectInterfaces{}, ) - require.ErrorContains(t, err, `failed to configure binding: replicas-test2.example.com not found`) + require.ErrorContains(t, err, "no params found for policy binding with `Deny` parameterNotFoundAction") // Add the missing params require.NoError(t, paramTracker.Add(fakeParams2)) @@ -1275,10 +1338,80 @@ func TestInvalidParamSourceInstanceName(t *testing.T) { // expect the specific error to be that the param was not found, not that CRD // is not existing require.ErrorContains(t, err, - `failed to configure binding: replicas-test.example.com not found`) + "no params found for policy binding with `Deny` parameterNotFoundAction") require.Len(t, passedParams, 0) } +// Show that policy still gets evaluated with `nil` param if paramRef & namespaceParamRef +// are both unset +func TestEmptyParamRef(t *testing.T) { + reset() + testContext, testContextCancel := context.WithCancel(context.Background()) + defer testContextCancel() + + compiler := &fakeCompiler{} + validator := &fakeValidator{} + matcher := &fakeMatcher{ + DefaultMatch: true, + } + + handler, _, tracker, controller := setupFakeTest(t, compiler, matcher) + + datalock := sync.Mutex{} + numCompiles := 0 + + compiler.RegisterDefinition(denyPolicy, func([]cel.ExpressionAccessor, cel.OptionalVariableDeclarations) cel.Filter { + datalock.Lock() + numCompiles += 1 + datalock.Unlock() + + return &fakeFilter{ + keyId: denyPolicy.Spec.Validations[0].Expression, + } + }) + + validator.RegisterDefinition(denyPolicy, func(ctx context.Context, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object, namespace *v1.Namespace, runtimeCELCostBudget int64, authz authorizer.Authorizer) ValidateResult { + // Versioned params must be nil to pass the test + if versionedParams != nil { + return ValidateResult{ + Decisions: []PolicyDecision{ + { + Action: ActionAdmit, + }, + }, + } + } + return ValidateResult{ + Decisions: []PolicyDecision{ + { + Action: ActionDeny, + Message: "Denied", + }, + }, + } + }) + + require.NoError(t, tracker.Create(definitionsGVR, denyPolicy, denyPolicy.Namespace)) + require.NoError(t, tracker.Create(bindingsGVR, denyBindingWithNoParamRef, denyBindingWithNoParamRef.Namespace)) + + // Wait for controller to reconcile given objects + require.NoError(t, + waitForReconcile( + testContext, controller, + denyBindingWithNoParamRef, denyPolicy)) + + err := handler.Validate( + testContext, + // Object is irrelevant/unchecked for this test. Just test that + // the evaluator is executed, and returns a denial + attributeRecord(nil, fakeParams, admission.Create), + &admission.RuntimeObjectInterfaces{}, + ) + + require.ErrorContains(t, err, `Denied`) + require.Equal(t, 1, numCompiles) +} + // Shows that a definition with no param source works just fine, and has // nil params passed to its evaluator. // @@ -1574,7 +1707,7 @@ func TestNativeTypeParam(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "replicas-test.example.com", - Namespace: "", + Namespace: "default", ResourceVersion: "1", }, Data: map[string]string{ @@ -1807,6 +1940,615 @@ func TestAllValidationActions(t *testing.T) { require.ErrorContains(t, err, "I'm sorry Dave") } +func TestNamespaceParamRefName(t *testing.T) { + reset() + testContext, testContextCancel := context.WithCancel(context.Background()) + defer testContextCancel() + + compiler := &fakeCompiler{} + validator := &fakeValidator{} + matcher := &fakeMatcher{ + DefaultMatch: true, + } + handler, _, tracker, controller := setupFakeTest(t, compiler, matcher) + + compiles := atomic.Int64{} + evaluations := atomic.Int64{} + + // Use ConfigMap native-typed param + nativeTypeParamPolicy := *denyPolicy + nativeTypeParamPolicy.Spec.ParamKind = &v1alpha1.ParamKind{ + APIVersion: "v1", + Kind: "ConfigMap", + } + + namespaceParamBinding := *denyBinding + namespaceParamBinding.Spec.ParamRef = &v1alpha1.ParamRef{ + Name: "replicas-test.example.com", + } + + compiler.RegisterDefinition(&nativeTypeParamPolicy, func([]cel.ExpressionAccessor, cel.OptionalVariableDeclarations) cel.Filter { + compiles.Add(1) + + return &fakeFilter{ + keyId: nativeTypeParamPolicy.Spec.Validations[0].Expression, + } + }) + + lock := sync.Mutex{} + observedParamNamespaces := []string{} + validator.RegisterDefinition(&nativeTypeParamPolicy, func(ctx context.Context, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object, namespace *v1.Namespace, runtimeCELCostBudget int64, authz authorizer.Authorizer) ValidateResult { + lock.Lock() + defer lock.Unlock() + + evaluations.Add(1) + if p, ok := versionedParams.(*v1.ConfigMap); ok { + observedParamNamespaces = append(observedParamNamespaces, p.Namespace) + return ValidateResult{ + Decisions: []PolicyDecision{ + { + Action: ActionDeny, + Message: "correct type", + }, + }, + } + } + return ValidateResult{ + Decisions: []PolicyDecision{ + { + Action: ActionDeny, + Message: "Incorrect param type", + }, + }, + } + }) + + configMapParam := &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "replicas-test.example.com", + Namespace: "default", + ResourceVersion: "1", + }, + Data: map[string]string{ + "coolkey": "default", + }, + } + configMapParam2 := &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "replicas-test.example.com", + Namespace: "mynamespace", + ResourceVersion: "1", + }, + Data: map[string]string{ + "coolkey": "mynamespace", + }, + } + configMapParam3 := &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "replicas-test.example.com", + Namespace: "othernamespace", + ResourceVersion: "1", + }, + Data: map[string]string{ + "coolkey": "othernamespace", + }, + } + require.NoError(t, tracker.Create(definitionsGVR, &nativeTypeParamPolicy, nativeTypeParamPolicy.Namespace)) + require.NoError(t, tracker.Create(bindingsGVR, &namespaceParamBinding, namespaceParamBinding.Namespace)) + require.NoError(t, tracker.Add(configMapParam)) + require.NoError(t, tracker.Add(configMapParam2)) + require.NoError(t, tracker.Add(configMapParam3)) + + // Wait for controller to reconcile given objects + require.NoError(t, + waitForReconcile( + testContext, controller, + &namespaceParamBinding, &nativeTypeParamPolicy, configMapParam, configMapParam2, configMapParam3)) + + // Object is irrelevant/unchecked for this test. Just test that + // the evaluator is executed with correct namespace, and returns admit + // meaning the params passed was a configmap + err := handler.Validate( + testContext, + attributeRecord(nil, configMapParam, admission.Create), + &admission.RuntimeObjectInterfaces{}, + ) + + func() { + lock.Lock() + defer lock.Unlock() + require.ErrorContains(t, err, "correct type") + require.EqualValues(t, 1, compiles.Load()) + require.EqualValues(t, 1, evaluations.Load()) + }() + + err = handler.Validate( + testContext, + attributeRecord(nil, configMapParam2, admission.Create), + &admission.RuntimeObjectInterfaces{}, + ) + + func() { + lock.Lock() + defer lock.Unlock() + require.ErrorContains(t, err, "correct type") + require.EqualValues(t, 1, compiles.Load()) + require.EqualValues(t, 2, evaluations.Load()) + }() + + err = handler.Validate( + testContext, + attributeRecord(nil, configMapParam3, admission.Create), + &admission.RuntimeObjectInterfaces{}, + ) + + func() { + lock.Lock() + defer lock.Unlock() + require.ErrorContains(t, err, "correct type") + require.EqualValues(t, 1, compiles.Load()) + require.EqualValues(t, 3, evaluations.Load()) + }() + + err = handler.Validate( + testContext, + attributeRecord(nil, configMapParam, admission.Create), + &admission.RuntimeObjectInterfaces{}, + ) + + func() { + lock.Lock() + defer lock.Unlock() + require.ErrorContains(t, err, "correct type") + require.EqualValues(t, []string{"default", "mynamespace", "othernamespace", "default"}, observedParamNamespaces) + require.EqualValues(t, 1, compiles.Load()) + require.EqualValues(t, 4, evaluations.Load()) + }() +} + +func TestParamRef(t *testing.T) { + for _, paramIsClusterScoped := range []bool{false, true} { + for _, nameIsSet := range []bool{false, true} { + for _, namespaceIsSet := range []bool{false, true} { + if paramIsClusterScoped && namespaceIsSet { + // Skip invalid configuration + continue + } + + for _, selectorIsSet := range []bool{false, true} { + if selectorIsSet && nameIsSet { + // SKip invalid configuration + continue + } + + for _, denyNotFound := range []bool{false, true} { + + name := "ParamRef" + + if paramIsClusterScoped { + name = "ClusterScoped" + name + } + + if nameIsSet { + name = name + "WithName" + } else if selectorIsSet { + name = name + "WithLabelSelector" + } else { + name = name + "WithEverythingSelector" + } + + if namespaceIsSet { + name = name + "WithNamespace" + } + + if denyNotFound { + name = name + "DenyNotFound" + } else { + name = name + "AllowNotFound" + } + + t.Run(name, func(t *testing.T) { + // Test creating a policy with a cluster or namesapce-scoped param + // and binding with the provided configuration. Test will ensure + // that the provided configuration is capable of matching + // params as expected, and not matching params when not expected. + // Also ensures the NotFound setting works as expected with this particular + // configuration of ParamRef when all the previously + // matched params are deleted. + testParamRefCase(t, paramIsClusterScoped, nameIsSet, namespaceIsSet, selectorIsSet, denyNotFound) + }) + } + } + } + } + } +} + +// testParamRefCase constructs a ParamRef and policy with appropriate ParamKind +// for the given parameters, then constructs a scenario with several matching/non-matching params +// of varying names, namespaces, labels. +// +// Test then selects subset of params that should match provided configuration +// and ensuers those params are the only ones used. +// +// Also ensures NotFound action is enforced correctly by deleting all found +// params and ensuring the Action is used. +// +// This test is not meant to test every possible scenario of matching/not matching: +// only that each ParamRef CAN be evaluated correctly for both cluster scoped +// and namespace-scoped request kinds, and that the failure action is correctly +// applied. +func testParamRefCase(t *testing.T, paramIsClusterScoped, nameIsSet, namespaceIsSet, selectorIsSet, denyNotFound bool) { + // Create a cluster scoped and a namespace scoped CRD + policy := *denyPolicy + binding := *denyBinding + binding.Spec.ParamRef = &v1alpha1.ParamRef{} + paramRef := binding.Spec.ParamRef + + shouldErrorOnClusterScopedRequests := !namespaceIsSet && !paramIsClusterScoped + + matchingParamName := "replicas-test.example.com" + matchingNamespace := "mynamespace" + nonMatchingNamespace := "othernamespace" + + matchingLabels := labels.Set{"doesitmatch": "yes"} + nonmatchingLabels := labels.Set{"doesitmatch": "no"} + otherNonmatchingLabels := labels.Set{"notaffiliated": "no"} + + if paramIsClusterScoped { + policy.Spec.ParamKind = &v1alpha1.ParamKind{ + APIVersion: clusterScopedParamsGVK.GroupVersion().String(), + Kind: clusterScopedParamsGVK.Kind, + } + } else { + policy.Spec.ParamKind = &v1alpha1.ParamKind{ + APIVersion: paramsGVK.GroupVersion().String(), + Kind: paramsGVK.Kind, + } + } + + if nameIsSet { + paramRef.Name = matchingParamName + } else if selectorIsSet { + paramRef.Selector = metav1.SetAsLabelSelector(matchingLabels) + } else { + paramRef.Selector = &metav1.LabelSelector{} + } + + if namespaceIsSet { + paramRef.Namespace = matchingNamespace + } + + if denyNotFound { + paramRef.ParameterNotFoundAction = ptrTo(v1alpha1.DenyAction) + } else { + paramRef.ParameterNotFoundAction = ptrTo(v1alpha1.AllowAction) + } + + compiler := &fakeCompiler{} + validator := &fakeValidator{} + matcher := &fakeMatcher{ + DefaultMatch: true, + } + + var matchedParams []runtime.Object + paramLock := sync.Mutex{} + observeParam := func(p runtime.Object) { + paramLock.Lock() + defer paramLock.Unlock() + matchedParams = append(matchedParams, p) + } + getAndResetObservedParams := func() []runtime.Object { + paramLock.Lock() + defer paramLock.Unlock() + oldParams := matchedParams + matchedParams = nil + return oldParams + } + + compiler.RegisterDefinition(&policy, func(ea []cel.ExpressionAccessor, ovd cel.OptionalVariableDeclarations) cel.Filter { + return &fakeFilter{ + keyId: policy.Spec.Validations[0].Expression, + } + }) + + validator.RegisterDefinition(&policy, func(ctx context.Context, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object, namespace *v1.Namespace, runtimeCELCostBudget int64, authz authorizer.Authorizer) ValidateResult { + observeParam(versionedParams) + return ValidateResult{ + Decisions: []PolicyDecision{ + { + Action: ActionDeny, + Message: "Denied by policy", + }, + }, + } + }) + + handler, paramTracker, tracker, controller := setupFakeTest(t, compiler, matcher) + + // Create library of params to try to fool the controller + params := []*unstructured.Unstructured{ + newParam(matchingParamName, v1.NamespaceDefault, nonmatchingLabels), + newParam(matchingParamName, matchingNamespace, nonmatchingLabels), + newParam(matchingParamName, nonMatchingNamespace, nonmatchingLabels), + + newParam(matchingParamName+"1", v1.NamespaceDefault, matchingLabels), + newParam(matchingParamName+"1", matchingNamespace, matchingLabels), + newParam(matchingParamName+"1", nonMatchingNamespace, matchingLabels), + + newParam(matchingParamName+"2", v1.NamespaceDefault, otherNonmatchingLabels), + newParam(matchingParamName+"2", matchingNamespace, otherNonmatchingLabels), + newParam(matchingParamName+"2", nonMatchingNamespace, otherNonmatchingLabels), + + newParam(matchingParamName+"3", v1.NamespaceDefault, otherNonmatchingLabels), + newParam(matchingParamName+"3", matchingNamespace, matchingLabels), + newParam(matchingParamName+"3", nonMatchingNamespace, matchingLabels), + + newClusterScopedParam(matchingParamName, matchingLabels), + newClusterScopedParam(matchingParamName+"1", nonmatchingLabels), + newClusterScopedParam(matchingParamName+"2", otherNonmatchingLabels), + newClusterScopedParam(matchingParamName+"3", matchingLabels), + newClusterScopedParam(matchingParamName+"4", nonmatchingLabels), + newClusterScopedParam(matchingParamName+"5", otherNonmatchingLabels), + } + + require.NoError(t, tracker.Create(definitionsGVR, &policy, policy.Namespace)) + require.NoError(t, tracker.Create(bindingsGVR, &binding, binding.Namespace)) + require.NoError(t, waitForReconcile(context.TODO(), controller, &policy, &binding)) + + for _, p := range params { + paramTracker.Add(p) + } + + namespacedRequestObject := newParam("some param", nonMatchingNamespace, nil) + clusterScopedRequestObject := newClusterScopedParam("other param", nil) + + // Validate a namespaced object, and verify that the params being validated + // are the ones we would expect + var expectedParamsForNamespacedRequest []*unstructured.Unstructured + for _, p := range params { + if p.GetAPIVersion() != policy.Spec.ParamKind.APIVersion || p.GetKind() != policy.Spec.ParamKind.Kind { + continue + } else if len(paramRef.Name) > 0 && p.GetName() != paramRef.Name { + continue + } else if len(paramRef.Namespace) > 0 && p.GetNamespace() != paramRef.Namespace { + continue + } + + if !paramIsClusterScoped { + if len(paramRef.Namespace) == 0 && p.GetNamespace() != namespacedRequestObject.GetNamespace() { + continue + } + } + + if paramRef.Selector != nil { + ls := p.GetLabels() + matched := true + + for k, v := range paramRef.Selector.MatchLabels { + if l, hasLabel := ls[k]; !hasLabel { + matched = false + break + } else if l != v { + matched = false + break + } + } + + // Empty selector matches everything + if len(paramRef.Selector.MatchExpressions) == 0 && len(paramRef.Selector.MatchLabels) == 0 { + matched = true + } + + if !matched { + continue + } + } + + expectedParamsForNamespacedRequest = append(expectedParamsForNamespacedRequest, p) + require.NoError(t, waitForReconcile(context.TODO(), controller, p)) + } + require.NotEmpty(t, expectedParamsForNamespacedRequest, "all test cases should match at least one param") + require.ErrorContains(t, handler.Validate(context.TODO(), attributeRecord(nil, namespacedRequestObject, admission.Create), &admission.RuntimeObjectInterfaces{}), "Denied by policy") + require.ElementsMatch(t, expectedParamsForNamespacedRequest, getAndResetObservedParams(), "should exactly match expected params") + + // Validate a cluster-scoped object, and verify that the params being validated + // are the ones we would expect + var expectedParamsForClusterScopedRequest []*unstructured.Unstructured + for _, p := range params { + if shouldErrorOnClusterScopedRequests { + continue + } else if p.GetAPIVersion() != policy.Spec.ParamKind.APIVersion || p.GetKind() != policy.Spec.ParamKind.Kind { + continue + } else if len(paramRef.Name) > 0 && p.GetName() != paramRef.Name { + continue + } else if len(paramRef.Namespace) > 0 && p.GetNamespace() != paramRef.Namespace { + continue + } else if !paramIsClusterScoped && len(paramRef.Namespace) == 0 && p.GetNamespace() != v1.NamespaceDefault { + continue + } + + if paramRef.Selector != nil { + ls := p.GetLabels() + matched := true + for k, v := range paramRef.Selector.MatchLabels { + if l, hasLabel := ls[k]; !hasLabel { + matched = false + break + } else if l != v { + matched = false + break + } + } + + // Empty selector matches everything + if len(paramRef.Selector.MatchExpressions) == 0 && len(paramRef.Selector.MatchLabels) == 0 { + matched = true + } + + if !matched { + continue + } + } + + expectedParamsForClusterScopedRequest = append(expectedParamsForClusterScopedRequest, p) + require.NoError(t, waitForReconcile(context.TODO(), controller, p)) + } + + err := handler.Validate(context.TODO(), attributeRecord(nil, clusterScopedRequestObject, admission.Create), &admission.RuntimeObjectInterfaces{}) + if shouldErrorOnClusterScopedRequests { + // Cannot validate cliuster-scoped resources against a paramRef that sets namespace + require.ErrorContains(t, err, "failed to configure binding: cannot use namespaced paramRef in policy binding that matches cluster-scoped resources") + } else { + require.NotEmpty(t, expectedParamsForClusterScopedRequest, "all test cases should match at least one param") + require.ErrorContains(t, err, "Denied by policy") + } + require.ElementsMatch(t, expectedParamsForClusterScopedRequest, getAndResetObservedParams(), "should exactly match expected params") + + // Remove all params matched by namespaced, and cluster-scoped validation. + // Validate again to make sure NotFoundAction is respected + var deleted []runtime.Object + for _, p := range expectedParamsForNamespacedRequest { + if paramIsClusterScoped { + require.NoError(t, paramTracker.Delete(paramsGVK.GroupVersion().WithResource("clusterscopedparamsconfigs"), p.GetNamespace(), p.GetName())) + } else { + require.NoError(t, paramTracker.Delete(paramsGVK.GroupVersion().WithResource("paramsconfigs"), p.GetNamespace(), p.GetName())) + } + deleted = append(deleted, p) + } + + for _, p := range expectedParamsForClusterScopedRequest { + // Tracker.Delete docs says it wont raise error for not found, but its implmenetation + // pretty plainly does... + rsrsc := "paramsconfigs" + if paramIsClusterScoped { + rsrsc = "clusterscopedparamsconfigs" + } + if err := paramTracker.Delete(paramsGVK.GroupVersion().WithResource(rsrsc), p.GetNamespace(), p.GetName()); err != nil && !k8serrors.IsNotFound(err) { + require.NoError(t, err) + deleted = append(deleted, p) + } + } + require.NoError(t, waitForReconcileDeletion(context.TODO(), controller, deleted...)) + + controller.refreshPolicies() + + // Check that NotFound is working correctly for both namespaeed & non-namespaced + // request object + err = handler.Validate(context.TODO(), attributeRecord(nil, namespacedRequestObject, admission.Create), &admission.RuntimeObjectInterfaces{}) + if denyNotFound { + require.ErrorContains(t, err, "no params found for policy binding with `Deny` parameterNotFoundAction") + } else { + require.NoError(t, err, "Allow not found expects no error when no params found. Policy should have been skipped") + } + require.Empty(t, getAndResetObservedParams(), "policy should not have been evaluated") + + err = handler.Validate(context.TODO(), attributeRecord(nil, clusterScopedRequestObject, admission.Create), &admission.RuntimeObjectInterfaces{}) + if shouldErrorOnClusterScopedRequests { + require.ErrorContains(t, err, "failed to configure binding: cannot use namespaced paramRef in policy binding that matches cluster-scoped resources") + + } else if denyNotFound { + require.ErrorContains(t, err, "no params found for policy binding with `Deny` parameterNotFoundAction") + } else { + require.NoError(t, err, "Allow not found expects no error when no params found. Policy should have been skipped") + } + require.Empty(t, getAndResetObservedParams(), "policy should not have been evaluated") +} + +// If the ParamKind is ClusterScoped, and namespace param is used. +// This is a Configuration Error of the policy +func TestNamespaceParamRefClusterScopedParamError(t *testing.T) { + reset() + testContext, testContextCancel := context.WithCancel(context.Background()) + defer testContextCancel() + + compiler := &fakeCompiler{} + validator := &fakeValidator{} + matcher := &fakeMatcher{ + DefaultMatch: true, + } + handler, _, tracker, controller := setupFakeTest(t, compiler, matcher) + + compiles := atomic.Int64{} + evaluations := atomic.Int64{} + + // Use ValidatingAdmissionPolicy for param type since it is cluster-scoped + nativeTypeParamPolicy := *denyPolicy + nativeTypeParamPolicy.Spec.ParamKind = &v1alpha1.ParamKind{ + APIVersion: "admissionregistration.k8s.io/v1alpha1", + Kind: "ValidatingAdmissionPolicy", + } + + namespaceParamBinding := *denyBinding + namespaceParamBinding.Spec.ParamRef = &v1alpha1.ParamRef{ + Name: "other-param-to-use-with-no-label.example.com", + Namespace: "mynamespace", + } + + compiler.RegisterDefinition(&nativeTypeParamPolicy, func([]cel.ExpressionAccessor, cel.OptionalVariableDeclarations) cel.Filter { + compiles.Add(1) + + return &fakeFilter{ + keyId: nativeTypeParamPolicy.Spec.Validations[0].Expression, + } + }) + + validator.RegisterDefinition(&nativeTypeParamPolicy, func(ctx context.Context, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object, namespace *v1.Namespace, runtimeCELCostBudget int64, authz authorizer.Authorizer) ValidateResult { + evaluations.Add(1) + if _, ok := versionedParams.(*v1alpha1.ValidatingAdmissionPolicy); ok { + return ValidateResult{ + Decisions: []PolicyDecision{ + { + Action: ActionAdmit, + Message: "correct type", + }, + }, + } + } + return ValidateResult{ + Decisions: []PolicyDecision{ + { + Action: ActionDeny, + Message: fmt.Sprintf("Incorrect param type %T", versionedParams), + }, + }, + } + }) + + require.NoError(t, tracker.Create(definitionsGVR, &nativeTypeParamPolicy, nativeTypeParamPolicy.Namespace)) + require.NoError(t, tracker.Create(bindingsGVR, &namespaceParamBinding, namespaceParamBinding.Namespace)) + // Wait for controller to reconcile given objects + require.NoError(t, + waitForReconcile( + testContext, controller, + &namespaceParamBinding, &nativeTypeParamPolicy)) + + // Object is irrelevant/unchecked for this test. Just test that + // the evaluator is executed with correct namespace, and returns admit + // meaning the params passed was a configmap + err := handler.Validate( + testContext, + attributeRecord(nil, fakeParams, admission.Create), + &admission.RuntimeObjectInterfaces{}, + ) + + require.ErrorContains(t, err, "paramRef.namespace must not be provided for a cluster-scoped `paramKind`") + require.EqualValues(t, 1, compiles.Load()) + require.EqualValues(t, 0, evaluations.Load()) +} + func TestAuditAnnotations(t *testing.T) { testContext, testContextCancel := context.WithCancel(context.Background()) defer testContextCancel() diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/controller.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/controller.go index da1940fbcbb..b3c092ab8aa 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/controller.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/controller.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" utiljson "k8s.io/apimachinery/pkg/util/json" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" @@ -72,9 +73,8 @@ type celAdmissionController struct { // against all of its registered bindings. type policyData struct { definitionInfo - paramController generic.Controller[runtime.Object] - paramScope meta.RESTScope - bindings []bindingInfo + paramInfo + bindings []bindingInfo } // contains the cel PolicyDecisions along with the ValidatingAdmissionPolicy and ValidatingAdmissionPolicyBinding @@ -237,6 +237,12 @@ func (c *celAdmissionController) Validate( authz := newCachingAuthorizer(c.authz) for _, definitionInfo := range policyDatas { + // versionedAttributes will be set to non-nil inside of the loop, but + // is scoped outside of the param loop so we only convert once. We defer + // conversion so that it is only performed when we know a policy matches, + // saving the cost of converting non-matching requests. + var versionedAttr *admission.VersionedAttributes + definition := definitionInfo.lastReconciledValue matches, matchKind, err := c.policyController.matcher.DefinitionMatches(a, o, definition) if err != nil { @@ -268,63 +274,23 @@ func (c *celAdmissionController) Validate( continue } - var param runtime.Object - - // versionedAttributes will be set to non-nil inside of the loop, but - // is scoped outside of the param loop so we only convert once. We defer - // conversion so that it is only performed when we know a policy matches, - // saving the cost of converting non-matching requests. - var versionedAttr *admission.VersionedAttributes - - // If definition has paramKind, paramRef is required in binding. - // If definition has no paramKind, paramRef set in binding will be ignored. - paramKind := definition.Spec.ParamKind - paramRef := binding.Spec.ParamRef - if paramKind != nil && paramRef != nil { - paramController := definitionInfo.paramController - if paramController == nil { - addConfigError(fmt.Errorf("paramKind kind `%v` not known", - paramKind.String()), definition, binding) - continue - } - - // If the param informer for this admission policy has not yet - // had time to perform an initial listing, don't attempt to use - // it. - timeoutCtx, cancel := context.WithTimeout(c.policyController.context, 1*time.Second) - defer cancel() - - if !cache.WaitForCacheSync(timeoutCtx.Done(), paramController.HasSynced) { - addConfigError(fmt.Errorf("paramKind kind `%v` not yet synced to use for admission", - paramKind.String()), definition, binding) - continue - } - - if len(paramRef.Namespace) == 0 { - param, err = paramController.Informer().Get(paramRef.Name) - } else { - param, err = paramController.Informer().Namespaced(paramRef.Namespace).Get(paramRef.Name) - } - + params, err := c.collectParams(definition.Spec.ParamKind, definitionInfo.paramInfo, binding.Spec.ParamRef, a.GetNamespace()) + if err != nil { + addConfigError(err, definition, binding) + continue + } else if versionedAttr == nil && len(params) > 0 { + // As optimization versionedAttr creation is deferred until + // first use. Since > 0 params, we will validate + va, err := admission.NewVersionedAttributes(a, matchKind, o) if err != nil { - // Apply failure policy - addConfigError(err, definition, binding) - - if k8serrors.IsInvalid(err) { - // Param mis-configured - // require to set paramRef.namespace for namespaced resource and unset paramRef.namespace for cluster scoped resource - continue - } else if k8serrors.IsNotFound(err) { - // Param not yet available. User may need to wait a bit - // before being able to use it for validation. - continue - } - - // There was a bad internal error - utilruntime.HandleError(err) + wrappedErr := fmt.Errorf("failed to convert object version: %w", err) + addConfigError(wrappedErr, definition, binding) continue } + versionedAttr = va } + + var validationResults []ValidateResult var namespace *v1.Namespace namespaceName := a.GetNamespace() @@ -343,72 +309,79 @@ func (c *celAdmissionController) Validate( } } - if versionedAttr == nil { - va, err := admission.NewVersionedAttributes(a, matchKind, o) - if err != nil { - wrappedErr := fmt.Errorf("failed to convert object version: %w", err) - addConfigError(wrappedErr, definition, binding) - continue - } - versionedAttr = va - } - - validationResult := bindingInfo.validator.Validate(ctx, versionedAttr, param, namespace, celconfig.RuntimeCELCostBudget, authz) - - for i, decision := range validationResult.Decisions { - switch decision.Action { - case ActionAdmit: - if decision.Evaluation == EvalError { - celmetrics.Metrics.ObserveAdmissionWithError(ctx, decision.Elapsed, definition.Name, binding.Name, "active") - } - case ActionDeny: - for _, action := range binding.Spec.ValidationActions { - switch action { - case v1alpha1.Deny: - deniedDecisions = append(deniedDecisions, policyDecisionWithMetadata{ - Definition: definition, - Binding: binding, - PolicyDecision: decision, - }) - celmetrics.Metrics.ObserveRejection(ctx, decision.Elapsed, definition.Name, binding.Name, "active") - case v1alpha1.Audit: - c.publishValidationFailureAnnotation(binding, i, decision, versionedAttr) - celmetrics.Metrics.ObserveAudit(ctx, decision.Elapsed, definition.Name, binding.Name, "active") - case v1alpha1.Warn: - warning.AddWarning(ctx, "", fmt.Sprintf("Validation failed for ValidatingAdmissionPolicy '%s' with binding '%s': %s", definition.Name, binding.Name, decision.Message)) - celmetrics.Metrics.ObserveWarn(ctx, decision.Elapsed, definition.Name, binding.Name, "active") - } - } - default: - return fmt.Errorf("unrecognized evaluation decision '%s' for ValidatingAdmissionPolicyBinding '%s' with ValidatingAdmissionPolicy '%s'", - decision.Action, binding.Name, definition.Name) - } - } - - for _, auditAnnotation := range validationResult.AuditAnnotations { - switch auditAnnotation.Action { - case AuditAnnotationActionPublish: - value := auditAnnotation.Value - if len(auditAnnotation.Value) > maxAuditAnnotationValueLength { - value = value[:maxAuditAnnotationValueLength] - } - auditAnnotationCollector.add(auditAnnotation.Key, value) - case AuditAnnotationActionError: - // When failurePolicy=fail, audit annotation errors result in deny - deniedDecisions = append(deniedDecisions, policyDecisionWithMetadata{ - Definition: definition, - Binding: binding, - PolicyDecision: PolicyDecision{ - Action: ActionDeny, - Evaluation: EvalError, - Message: auditAnnotation.Error, - Elapsed: auditAnnotation.Elapsed, + for _, param := range params { + var p runtime.Object = param + if p != nil && p.GetObjectKind().GroupVersionKind().Empty() { + // Make sure param has TypeMeta populated + // This is a simple hack to make sure typeMeta is + // available to CEL without making copies of objects, etc. + p = &wrappedParam{ + TypeMeta: metav1.TypeMeta{ + APIVersion: definition.Spec.ParamKind.APIVersion, + Kind: definition.Spec.ParamKind.Kind, }, - }) - celmetrics.Metrics.ObserveRejection(ctx, auditAnnotation.Elapsed, definition.Name, binding.Name, "active") - case AuditAnnotationActionExclude: // skip it - default: - return fmt.Errorf("unsupported AuditAnnotation Action: %s", auditAnnotation.Action) + nested: param, + } + } + validationResults = append(validationResults, bindingInfo.validator.Validate(ctx, versionedAttr, p, namespace, celconfig.RuntimeCELCostBudget, authz)) + } + + for _, validationResult := range validationResults { + for i, decision := range validationResult.Decisions { + switch decision.Action { + case ActionAdmit: + if decision.Evaluation == EvalError { + celmetrics.Metrics.ObserveAdmissionWithError(ctx, decision.Elapsed, definition.Name, binding.Name, "active") + } + case ActionDeny: + for _, action := range binding.Spec.ValidationActions { + switch action { + case v1alpha1.Deny: + deniedDecisions = append(deniedDecisions, policyDecisionWithMetadata{ + Definition: definition, + Binding: binding, + PolicyDecision: decision, + }) + celmetrics.Metrics.ObserveRejection(ctx, decision.Elapsed, definition.Name, binding.Name, "active") + case v1alpha1.Audit: + c.publishValidationFailureAnnotation(binding, i, decision, versionedAttr) + celmetrics.Metrics.ObserveAudit(ctx, decision.Elapsed, definition.Name, binding.Name, "active") + case v1alpha1.Warn: + warning.AddWarning(ctx, "", fmt.Sprintf("Validation failed for ValidatingAdmissionPolicy '%s' with binding '%s': %s", definition.Name, binding.Name, decision.Message)) + celmetrics.Metrics.ObserveWarn(ctx, decision.Elapsed, definition.Name, binding.Name, "active") + } + } + default: + return fmt.Errorf("unrecognized evaluation decision '%s' for ValidatingAdmissionPolicyBinding '%s' with ValidatingAdmissionPolicy '%s'", + decision.Action, binding.Name, definition.Name) + } + } + + for _, auditAnnotation := range validationResult.AuditAnnotations { + switch auditAnnotation.Action { + case AuditAnnotationActionPublish: + value := auditAnnotation.Value + if len(auditAnnotation.Value) > maxAuditAnnotationValueLength { + value = value[:maxAuditAnnotationValueLength] + } + auditAnnotationCollector.add(auditAnnotation.Key, value) + case AuditAnnotationActionError: + // When failurePolicy=fail, audit annotation errors result in deny + deniedDecisions = append(deniedDecisions, policyDecisionWithMetadata{ + Definition: definition, + Binding: binding, + PolicyDecision: PolicyDecision{ + Action: ActionDeny, + Evaluation: EvalError, + Message: auditAnnotation.Error, + Elapsed: auditAnnotation.Elapsed, + }, + }) + celmetrics.Metrics.ObserveRejection(ctx, auditAnnotation.Elapsed, definition.Name, binding.Name, "active") + case AuditAnnotationActionExclude: // skip it + default: + return fmt.Errorf("unsupported AuditAnnotation Action: %s", auditAnnotation.Action) + } } } } @@ -437,6 +410,123 @@ func (c *celAdmissionController) Validate( return nil } +// Returns objects to use to evaluate the policy +func (c *celAdmissionController) collectParams( + paramKind *v1alpha1.ParamKind, + info paramInfo, + paramRef *v1alpha1.ParamRef, + namespace string, +) ([]runtime.Object, error) { + // If definition has paramKind, paramRef is required in binding. + // If definition has no paramKind, paramRef set in binding will be ignored. + var params []runtime.Object + var paramStore generic.NamespacedLister[runtime.Object] + + // Make sure the param kind is ready to use + if paramKind != nil && paramRef != nil { + if info.controller == nil { + return nil, fmt.Errorf("paramKind kind `%v` not known", + paramKind.String()) + } + + // Set up cluster-scoped, or namespaced access to the params + // "default" if not provided, and paramKind is namespaced + paramStore = info.controller.Informer() + if info.scope.Name() == meta.RESTScopeNameNamespace { + paramsNamespace := namespace + if len(paramRef.Namespace) > 0 { + paramsNamespace = paramRef.Namespace + } else if len(paramsNamespace) == 0 { + // You must supply namespace if your matcher can possibly + // match a cluster-scoped resource + return nil, fmt.Errorf("cannot use namespaced paramRef in policy binding that matches cluster-scoped resources") + } + + paramStore = info.controller.Informer().Namespaced(paramsNamespace) + } + + // If the param informer for this admission policy has not yet + // had time to perform an initial listing, don't attempt to use + // it. + timeoutCtx, cancel := context.WithTimeout(c.policyController.context, 1*time.Second) + defer cancel() + + if !cache.WaitForCacheSync(timeoutCtx.Done(), info.controller.HasSynced) { + return nil, fmt.Errorf("paramKind kind `%v` not yet synced to use for admission", + paramKind.String()) + } + } + + // Find params to use with policy + switch { + case paramKind == nil: + // ParamKind is unset. Ignore any globalParamRef or namespaceParamRef + // setting. + return []runtime.Object{nil}, nil + case paramRef == nil: + // Policy ParamKind is set, but binding does not use it. + // Validate with nil params + return []runtime.Object{nil}, nil + case len(paramRef.Namespace) > 0 && info.scope.Name() == meta.RESTScopeRoot.Name(): + // Not allowed to set namespace for cluster-scoped param + return nil, fmt.Errorf("paramRef.namespace must not be provided for a cluster-scoped `paramKind`") + + case len(paramRef.Name) > 0: + if paramRef.Selector != nil { + // This should be validated, but just in case. + return nil, fmt.Errorf("paramRef.name and paramRef.selector are mutually exclusive") + } + + switch param, err := paramStore.Get(paramRef.Name); { + case err == nil: + params = []runtime.Object{param} + case k8serrors.IsNotFound(err): + // Param not yet available. User may need to wait a bit + // before being able to use it for validation. + // + // Set params to nil to prepare for not found action + params = nil + case k8serrors.IsInvalid(err): + // Param mis-configured + // require to set namespace for namespaced resource + // and unset namespace for cluster scoped resource + return nil, err + default: + // Internal error + utilruntime.HandleError(err) + return nil, err + } + case paramRef.Selector != nil: + // Select everything by default if empty name and selector + selector, err := metav1.LabelSelectorAsSelector(paramRef.Selector) + if err != nil { + // Cannot parse label selector: configuration error + return nil, err + + } + + paramList, err := paramStore.List(selector) + if err != nil { + // There was a bad internal error + utilruntime.HandleError(err) + return nil, err + } + + // Successfully grabbed params + params = paramList + default: + // Should be unreachable due to validation + return nil, fmt.Errorf("one of name or selector must be provided") + } + + // Apply fail action for params not found case + if len(params) == 0 && paramRef.ParameterNotFoundAction != nil && *paramRef.ParameterNotFoundAction == v1alpha1.DenyAction { + return nil, errors.New("no params found for policy binding with `Deny` parameterNotFoundAction") + } + + return params, nil +} + func (c *celAdmissionController) publishValidationFailureAnnotation(binding *v1alpha1.ValidatingAdmissionPolicyBinding, expressionIndex int, decision PolicyDecision, attributes admission.Attributes) { key := "validation.policy.admission.k8s.io/validation_failure" // Marshal to a list of failures since, in the future, we may need to support multiple failures @@ -512,3 +602,48 @@ func (a auditAnnotationCollector) publish(policyName string, attributes admissio } } } + +// A workaround to fact that native types do not have TypeMeta populated, which +// is needed for CEL expressions to be able to access the value. +type wrappedParam struct { + metav1.TypeMeta + nested runtime.Object +} + +func (w *wrappedParam) MarshalJSON() ([]byte, error) { + return nil, errors.New("MarshalJSON unimplemented for wrappedParam") +} + +func (w *wrappedParam) UnmarshalJSON(data []byte) error { + return errors.New("UnmarshalJSON unimplemented for wrappedParam") +} + +func (w *wrappedParam) ToUnstructured() interface{} { + res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(w.nested) + + if err != nil { + return nil + } + + metaRes, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&w.TypeMeta) + if err != nil { + return nil + } + + for k, v := range metaRes { + res[k] = v + } + + return res +} + +func (w *wrappedParam) DeepCopyObject() runtime.Object { + return &wrappedParam{ + TypeMeta: w.TypeMeta, + nested: w.nested.DeepCopyObject(), + } +} + +func (w *wrappedParam) GetObjectKind() schema.ObjectKind { + return w +} diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/controller_reconcile.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/controller_reconcile.go index aa014881a53..b7cc81349d9 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/controller_reconcile.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/controller_reconcile.go @@ -464,20 +464,17 @@ func (c *policyController) latestPolicyData() []policyData { bindingInfos = append(bindingInfos, *bindingInfo) } - var paramController generic.Controller[runtime.Object] - var paramScope meta.RESTScope + var pInfo paramInfo if paramKind := definitionInfo.lastReconciledValue.Spec.ParamKind; paramKind != nil { if info, ok := c.paramsCRDControllers[*paramKind]; ok { - paramController = info.controller - paramScope = info.scope + pInfo = *info } } res = append(res, policyData{ - definitionInfo: *definitionInfo, - paramController: paramController, - paramScope: paramScope, - bindings: bindingInfos, + definitionInfo: *definitionInfo, + paramInfo: pInfo, + bindings: bindingInfos, }) } diff --git a/test/integration/apiserver/cel/validatingadmissionpolicy_test.go b/test/integration/apiserver/cel/validatingadmissionpolicy_test.go index c0a43f08ece..fffc3df8a4b 100644 --- a/test/integration/apiserver/cel/validatingadmissionpolicy_test.go +++ b/test/integration/apiserver/cel/validatingadmissionpolicy_test.go @@ -2026,7 +2026,7 @@ func Test_ValidatingAdmissionPolicy_ParamResourceDeletedThenRecreated(t *testing return false, nil } - if !strings.Contains(err.Error(), "failed to configure binding: test not found") { + if !strings.Contains(err.Error(), "failed to configure binding: no params found for policy binding with `Deny` parameterNotFoundAction") { return false, err } @@ -2049,7 +2049,7 @@ func Test_ValidatingAdmissionPolicy_ParamResourceDeletedThenRecreated(t *testing _, err = client.CoreV1().Namespaces().Create(context.TODO(), disallowedNamespace, metav1.CreateOptions{}) // cache not synced with new object yet, try again - if strings.Contains(err.Error(), "failed to configure binding: test not found") { + if strings.Contains(err.Error(), "failed to configure binding: no params found for policy binding with `Deny` parameterNotFoundAction") { return false, nil }