diff --git a/test/integration/apiserver/cel/admission_policy_test.go b/test/integration/apiserver/cel/admission_policy_test.go new file mode 100644 index 00000000000..4d90108398d --- /dev/null +++ b/test/integration/apiserver/cel/admission_policy_test.go @@ -0,0 +1,714 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cel + +import ( + "bytes" + "context" + "encoding/csv" + "sort" + "strings" + "sync" + "testing" + "time" + + "k8s.io/api/admission/v1beta1" + corev1 "k8s.io/api/core/v1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + genericfeatures "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + + apiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" + "k8s.io/kubernetes/pkg/apis/admissionregistration" + admissionregistrationv1alpha1apis "k8s.io/kubernetes/pkg/apis/admissionregistration/v1alpha1" + admissionregistrationv1beta1apis "k8s.io/kubernetes/pkg/apis/admissionregistration/v1beta1" + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/test/integration/etcd" + "k8s.io/kubernetes/test/integration/framework" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + clientset "k8s.io/client-go/kubernetes" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + admissionregistrationv1alpha1 "k8s.io/api/admissionregistration/v1alpha1" + admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" +) + +const ( + beginSentinel = "###___BEGIN_SENTINEL___###" + recordSeparator = `###$$$###` +) + +// Policy registration helpers +var testSpec admissionregistration.ValidatingAdmissionPolicy = admissionregistration.ValidatingAdmissionPolicy{ + Spec: admissionregistration.ValidatingAdmissionPolicySpec{ + ParamKind: &admissionregistration.ParamKind{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + Variables: []admissionregistration.Variable{ + { + Name: "shouldFail", + Expression: `true`, + }, + { + Name: "resourceGroup", + Expression: `has(request.resource.group) ? request.resource.group : ""`, + }, + { + Name: "resourceVersion", + Expression: `has(request.resource.version) ? request.resource.version : ""`, + }, + { + Name: "resourceResource", + Expression: `has(request.resource.resource) ? request.resource.resource : ""`, + }, + { + Name: "subresource", + Expression: `has(request.subResource) ? request.subResource : ""`, + }, + { + Name: "operation", + Expression: `has(request.operation) ? request.operation : ""`, + }, + { + Name: "name", + Expression: `has(request.name) ? request.name : ""`, + }, + { + Name: "namespaceName", + Expression: `has(request.namespace) ? request.namespace : ""`, + }, + { + Name: "objectExists", + Expression: `object != null ? "true" : "false"`, + }, + { + Name: "objectAPIVersion", + Expression: `(object != null && has(object.apiVersion)) ? object.apiVersion : ""`, + }, + { + Name: "objectKind", + Expression: `(object != null && has(object.kind)) ? object.kind : ""`, + }, + { + Name: "oldObjectExists", + Expression: `oldObject != null ? "true" : "false"`, + }, + { + Name: "oldObjectAPIVersion", + Expression: `(oldObject != null && has(oldObject.apiVersion)) ? oldObject.apiVersion : ""`, + }, + { + Name: "oldObjectKind", + Expression: `(oldObject != null && has(oldObject.kind)) ? oldObject.kind : ""`, + }, + { + Name: "optionsExists", + Expression: `(has(request.options) && request.options != null) ? "true" : "false"`, + }, + { + Name: "optionsKind", + Expression: `(has(request.options) && has(request.options.kind)) ? request.options.kind : ""`, + }, + { + Name: "optionsAPIVersion", + Expression: `(has(request.options) && has(request.options.apiVersion)) ? request.options.apiVersion : ""`, + }, + { + Name: "paramsPhase", + Expression: `params.data.phase`, + }, + { + Name: "paramsVersion", + Expression: `params.data.version`, + }, + { + Name: "paramsConvert", + Expression: `params.data.convert`, + }, + }, + // Would be nice to use CEL to create a single map + // and stringify it. Unfortunately those library functions + // are not yet available, so we must create a map + // like so + Validations: []admissionregistration.Validation{ + { + // newlines forbidden so use recordSeparator + Expression: "!variables.shouldFail", + MessageExpression: `"` + beginSentinel + `resourceGroup,resourceVersion,resourceResource,subresource,operation,name,namespace,objectExists,objectKind,objectAPIVersion,oldObjectExists,oldObjectKind,oldObjectAPIVersion,optionsExists,optionsKind,optionsAPIVersion,paramsPhase,paramsVersion,paramsConvert` + recordSeparator + `"+variables.resourceGroup + "," + variables.resourceVersion + "," + variables.resourceResource + "," + variables.subresource + "," + variables.operation + "," + variables.name + "," + variables.namespaceName + "," + variables.objectExists + "," + variables.objectKind + "," + variables.objectAPIVersion + "," + variables.oldObjectExists + "," + variables.oldObjectKind + "," + variables.oldObjectAPIVersion + "," + variables.optionsExists + "," + variables.optionsKind + "," + variables.optionsAPIVersion + "," + variables.paramsPhase + "," + variables.paramsVersion + "," + variables.paramsConvert`, + }, + }, + MatchConditions: []admissionregistration.MatchCondition{ + { + Name: "testclient-only", + Expression: `request.userInfo.username == "` + testClientUsername + `"`, + }, + { + Name: "ignore-test-config", + Expression: `object == null || !has(object.metadata) || !has(object.metadata.annotations) || !has(object.metadata.annotations.skipMatch) || object.metadata.annotations.skipMatch != "yes"`, + }, + }, + }, +} + +func createV1beta1ValidatingPolicyAndBinding(client clientset.Interface, convertedRules []admissionregistrationv1beta1.NamedRuleWithOperations) error { + denyAction := admissionregistrationv1beta1.DenyAction + exact := admissionregistrationv1beta1.Exact + equivalent := admissionregistrationv1beta1.Equivalent + + var outSpec admissionregistrationv1beta1.ValidatingAdmissionPolicy + if err := admissionregistrationv1beta1apis.Convert_admissionregistration_ValidatingAdmissionPolicy_To_v1beta1_ValidatingAdmissionPolicy(&testSpec, &outSpec, nil); err != nil { + return err + } + + exactPolicyTemplate := outSpec.DeepCopy() + convertedPolicyTemplate := outSpec.DeepCopy() + + exactPolicyTemplate.SetName("test-policy-v1beta1") + exactPolicyTemplate.Spec.MatchConstraints = &admissionregistrationv1beta1.MatchResources{ + ResourceRules: []admissionregistrationv1beta1.NamedRuleWithOperations{ + { + RuleWithOperations: admissionregistrationv1.RuleWithOperations{ + Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.OperationAll}, + Rule: admissionregistrationv1.Rule{APIGroups: []string{"*"}, APIVersions: []string{"*"}, Resources: []string{"*/*"}}, + }, + }, + }, + MatchPolicy: &exact, + } + + convertedPolicyTemplate.SetName("test-policy-v1beta1-convert") + convertedPolicyTemplate.Spec.MatchConstraints = &admissionregistrationv1beta1.MatchResources{ + ResourceRules: convertedRules, + MatchPolicy: &equivalent, + } + + exactPolicy, err := client.AdmissionregistrationV1beta1().ValidatingAdmissionPolicies().Create(context.TODO(), exactPolicyTemplate, metav1.CreateOptions{}) + if err != nil { + return err + } + + convertPolicy, err := client.AdmissionregistrationV1beta1().ValidatingAdmissionPolicies().Create(context.TODO(), convertedPolicyTemplate, metav1.CreateOptions{}) + if err != nil { + return err + } + + // Create a param that holds the options for this + configuration, err := client.CoreV1().ConfigMaps("default").Create(context.TODO(), &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-policy-v1beta1-param", + Namespace: "default", + Annotations: map[string]string{ + "skipMatch": "yes", + }, + }, + Data: map[string]string{ + "version": "v1beta1", + "phase": validation, + "convert": "false", + }, + }, metav1.CreateOptions{}) + if err != nil { + return err + } + + configurationConvert, err := client.CoreV1().ConfigMaps("default").Create(context.TODO(), &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-policy-v1beta1-convert-param", + Namespace: "default", + Annotations: map[string]string{ + "skipMatch": "yes", + }, + }, + Data: map[string]string{ + "version": "v1beta1", + "phase": validation, + "convert": "true", + }, + }, metav1.CreateOptions{}) + if err != nil { + return err + } + + _, err = client.AdmissionregistrationV1beta1().ValidatingAdmissionPolicyBindings().Create(context.TODO(), &admissionregistrationv1beta1.ValidatingAdmissionPolicyBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-policy-v1beta1-binding", + }, + Spec: admissionregistrationv1beta1.ValidatingAdmissionPolicyBindingSpec{ + PolicyName: exactPolicy.GetName(), + ValidationActions: []admissionregistrationv1beta1.ValidationAction{admissionregistrationv1beta1.Warn}, + ParamRef: &admissionregistrationv1beta1.ParamRef{ + Name: configuration.GetName(), + Namespace: configuration.GetNamespace(), + ParameterNotFoundAction: &denyAction, + }, + }, + }, metav1.CreateOptions{}) + if err != nil { + return err + } + _, err = client.AdmissionregistrationV1beta1().ValidatingAdmissionPolicyBindings().Create(context.TODO(), &admissionregistrationv1beta1.ValidatingAdmissionPolicyBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-policy-v1beta1-convert-binding", + }, + Spec: admissionregistrationv1beta1.ValidatingAdmissionPolicyBindingSpec{ + PolicyName: convertPolicy.GetName(), + ValidationActions: []admissionregistrationv1beta1.ValidationAction{admissionregistrationv1beta1.Warn}, + ParamRef: &admissionregistrationv1beta1.ParamRef{ + Name: configurationConvert.GetName(), + Namespace: configurationConvert.GetNamespace(), + ParameterNotFoundAction: &denyAction, + }, + }, + }, metav1.CreateOptions{}) + if err != nil { + return err + } + + return nil +} + +func createV1alpha1ValidatingPolicyAndBinding(client clientset.Interface, convertedRules []admissionregistrationv1alpha1.NamedRuleWithOperations) error { + exact := admissionregistrationv1alpha1.Exact + equivalent := admissionregistrationv1alpha1.Equivalent + denyAction := admissionregistrationv1alpha1.DenyAction + + var outSpec admissionregistrationv1alpha1.ValidatingAdmissionPolicy + if err := admissionregistrationv1alpha1apis.Convert_admissionregistration_ValidatingAdmissionPolicy_To_v1alpha1_ValidatingAdmissionPolicy(&testSpec, &outSpec, nil); err != nil { + return err + } + + exactPolicyTemplate := outSpec.DeepCopy() + convertedPolicyTemplate := outSpec.DeepCopy() + + exactPolicyTemplate.SetName("test-policy-v1alpha1") + exactPolicyTemplate.Spec.MatchConstraints = &admissionregistrationv1alpha1.MatchResources{ + ResourceRules: []admissionregistrationv1alpha1.NamedRuleWithOperations{ + { + RuleWithOperations: admissionregistrationv1.RuleWithOperations{ + Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.OperationAll}, + Rule: admissionregistrationv1.Rule{APIGroups: []string{"*"}, APIVersions: []string{"*"}, Resources: []string{"*/*"}}, + }, + }, + }, + MatchPolicy: &exact, + } + + convertedPolicyTemplate.SetName("test-policy-v1alpha1-convert") + convertedPolicyTemplate.Spec.MatchConstraints = &admissionregistrationv1alpha1.MatchResources{ + ResourceRules: convertedRules, + MatchPolicy: &equivalent, + } + + exactPolicy, err := client.AdmissionregistrationV1alpha1().ValidatingAdmissionPolicies().Create(context.TODO(), exactPolicyTemplate, metav1.CreateOptions{}) + if err != nil { + return err + } + + convertPolicy, err := client.AdmissionregistrationV1alpha1().ValidatingAdmissionPolicies().Create(context.TODO(), convertedPolicyTemplate, metav1.CreateOptions{}) + if err != nil { + return err + } + + // Create a param that holds the options for this + configuration, err := client.CoreV1().ConfigMaps("default").Create(context.TODO(), &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-policy-v1alpha1-param", + Namespace: "default", + Annotations: map[string]string{ + "skipMatch": "yes", + }, + }, + Data: map[string]string{ + "version": "v1alpha1", + "phase": validation, + "convert": "false", + }, + }, metav1.CreateOptions{}) + if err != nil { + return err + } + + configurationConvert, err := client.CoreV1().ConfigMaps("default").Create(context.TODO(), &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-policy-v1alpha1-convert-param", + Namespace: "default", + Annotations: map[string]string{ + "skipMatch": "yes", + }, + }, + Data: map[string]string{ + "version": "v1alpha1", + "phase": validation, + "convert": "true", + }, + }, metav1.CreateOptions{}) + if err != nil { + return err + } + + _, err = client.AdmissionregistrationV1alpha1().ValidatingAdmissionPolicyBindings().Create(context.TODO(), &admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-policy-v1alpha1-binding", + }, + Spec: admissionregistrationv1alpha1.ValidatingAdmissionPolicyBindingSpec{ + PolicyName: exactPolicy.GetName(), + ValidationActions: []admissionregistrationv1alpha1.ValidationAction{admissionregistrationv1alpha1.Warn}, + ParamRef: &admissionregistrationv1alpha1.ParamRef{ + Name: configuration.GetName(), + Namespace: configuration.GetNamespace(), + ParameterNotFoundAction: &denyAction, + }, + }, + }, metav1.CreateOptions{}) + if err != nil { + return err + } + _, err = client.AdmissionregistrationV1alpha1().ValidatingAdmissionPolicyBindings().Create(context.TODO(), &admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-policy-v1alpha1-convert-binding", + }, + Spec: admissionregistrationv1alpha1.ValidatingAdmissionPolicyBindingSpec{ + PolicyName: convertPolicy.GetName(), + ValidationActions: []admissionregistrationv1alpha1.ValidationAction{admissionregistrationv1alpha1.Warn}, + ParamRef: &admissionregistrationv1alpha1.ParamRef{ + Name: configurationConvert.GetName(), + Namespace: configurationConvert.GetNamespace(), + ParameterNotFoundAction: &denyAction, + }, + }, + }, metav1.CreateOptions{}) + if err != nil { + return err + } + + return nil +} + +// This test shows that policy intercepts all requests for all resources, +// subresources, verbs, and input versions of policy/binding. +// +// This test tries to mirror very closely the same test for webhook admission +// test/integration/apiserver/admissionwebhook/admission_test.go testWebhookAdmission +func TestPolicyAdmission(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.ValidatingAdmissionPolicy, true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APISelfSubjectReview, true)() + + holder := &policyExpectationHolder{ + holder: holder{ + t: t, + gvrToConvertedGVR: map[metav1.GroupVersionResource]metav1.GroupVersionResource{}, + gvrToConvertedGVK: map[metav1.GroupVersionResource]schema.GroupVersionKind{}, + }, + } + + server := apiservertesting.StartTestServerOrDie(t, nil, []string{ + "--enable-admission-plugins", "ValidatingAdmissionPolicy", + // turn off admission plugins that add finalizers + "--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection", + // force enable all resources so we can check storage. + "--runtime-config=api/all=true", + }, framework.SharedEtcd()) + defer server.TearDownFn() + + // Create admission policy & binding that match everything + clientConfig := server.ClientConfig + clientConfig.Impersonate.UserName = testClientUsername + clientConfig.Impersonate.Groups = []string{"system:masters", "system:authenticated"} + clientConfig.WarningHandler = holder + client, err := clientset.NewForConfig(clientConfig) + if err != nil { + t.Fatal(err) + } + + // create CRDs + etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(server.ClientConfig), false, etcd.GetCustomResourceDefinitionData()...) + + if _, err := client.CoreV1().Namespaces().Create(context.TODO(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}}, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + + // gather resources to test + dynamicClient, err := dynamic.NewForConfig(clientConfig) + if err != nil { + t.Fatal(err) + } + + _, resources, err := client.Discovery().ServerGroupsAndResources() + if err != nil { + t.Fatalf("Failed to get ServerGroupsAndResources with error: %+v", err) + } + + gvrsToTest := []schema.GroupVersionResource{} + resourcesByGVR := map[schema.GroupVersionResource]metav1.APIResource{} + + for _, list := range resources { + defaultGroupVersion, err := schema.ParseGroupVersion(list.GroupVersion) + if err != nil { + t.Errorf("Failed to get GroupVersion for: %+v", list) + continue + } + for _, resource := range list.APIResources { + if resource.Group == "" { + resource.Group = defaultGroupVersion.Group + } + if resource.Version == "" { + resource.Version = defaultGroupVersion.Version + } + gvr := defaultGroupVersion.WithResource(resource.Name) + resourcesByGVR[gvr] = resource + if shouldTestResource(gvr, resource) { + gvrsToTest = append(gvrsToTest, gvr) + } + } + } + + sort.SliceStable(gvrsToTest, func(i, j int) bool { + if gvrsToTest[i].Group < gvrsToTest[j].Group { + return true + } + if gvrsToTest[i].Group > gvrsToTest[j].Group { + return false + } + if gvrsToTest[i].Version < gvrsToTest[j].Version { + return true + } + if gvrsToTest[i].Version > gvrsToTest[j].Version { + return false + } + if gvrsToTest[i].Resource < gvrsToTest[j].Resource { + return true + } + if gvrsToTest[i].Resource > gvrsToTest[j].Resource { + return false + } + return true + }) + + // map unqualified resource names to the fully qualified resource we will expect to be converted to + // Note: this only works because there are no overlapping resource names in-process that are not co-located + convertedResources := map[string]schema.GroupVersionResource{} + // build the webhook rules enumerating the specific group/version/resources we want + convertedV1beta1Rules := []admissionregistrationv1beta1.NamedRuleWithOperations{} + convertedV1alpha1Rules := []admissionregistrationv1alpha1.NamedRuleWithOperations{} + for _, gvr := range gvrsToTest { + metaGVR := metav1.GroupVersionResource{Group: gvr.Group, Version: gvr.Version, Resource: gvr.Resource} + + convertedGVR, ok := convertedResources[gvr.Resource] + if !ok { + // this is the first time we've seen this resource + // record the fully qualified resource we expect + convertedGVR = gvr + convertedResources[gvr.Resource] = gvr + // add an admission rule indicating we can receive this version + convertedV1beta1Rules = append(convertedV1beta1Rules, admissionregistrationv1beta1.NamedRuleWithOperations{ + RuleWithOperations: admissionregistrationv1.RuleWithOperations{ + Operations: []admissionregistrationv1beta1.OperationType{admissionregistrationv1beta1.OperationAll}, + Rule: admissionregistrationv1beta1.Rule{APIGroups: []string{gvr.Group}, APIVersions: []string{gvr.Version}, Resources: []string{gvr.Resource}}, + }, + }) + convertedV1alpha1Rules = append(convertedV1alpha1Rules, admissionregistrationv1alpha1.NamedRuleWithOperations{ + RuleWithOperations: admissionregistrationv1.RuleWithOperations{ + Operations: []admissionregistrationv1alpha1.OperationType{admissionregistrationv1alpha1.OperationAll}, + Rule: admissionregistrationv1alpha1.Rule{APIGroups: []string{gvr.Group}, APIVersions: []string{gvr.Version}, Resources: []string{gvr.Resource}}, + }, + }) + } + + // record the expected resource and kind + holder.gvrToConvertedGVR[metaGVR] = metav1.GroupVersionResource{Group: convertedGVR.Group, Version: convertedGVR.Version, Resource: convertedGVR.Resource} + holder.gvrToConvertedGVK[metaGVR] = schema.GroupVersionKind{Group: resourcesByGVR[convertedGVR].Group, Version: resourcesByGVR[convertedGVR].Version, Kind: resourcesByGVR[convertedGVR].Kind} + } + + if err := createV1alpha1ValidatingPolicyAndBinding(client, convertedV1alpha1Rules); err != nil { + t.Fatal(err) + } + + if err := createV1beta1ValidatingPolicyAndBinding(client, convertedV1beta1Rules); err != nil { + t.Fatal(err) + } + + // Allow the policy & binding to establish + time.Sleep(1 * time.Second) + + start := time.Now() + count := 0 + + // Test admission on all resources, subresources, and verbs + for _, gvr := range gvrsToTest { + resource := resourcesByGVR[gvr] + t.Run(gvr.Group+"."+gvr.Version+"."+strings.ReplaceAll(resource.Name, "/", "."), func(t *testing.T) { + for _, verb := range []string{"create", "update", "patch", "connect", "delete", "deletecollection"} { + if shouldTestResourceVerb(gvr, resource, verb) { + t.Run(verb, func(t *testing.T) { + count++ + holder.reset(t) + testFunc := getTestFunc(gvr, verb) + testFunc(&testContext{ + t: t, + admissionHolder: holder, + client: dynamicClient, + clientset: client, + verb: verb, + gvr: gvr, + resource: resource, + resources: resourcesByGVR, + }) + holder.verify(t) + }) + } + } + }) + } + + if count >= 10 { + duration := time.Since(start) + perResourceDuration := time.Duration(int(duration) / count) + if perResourceDuration >= 150*time.Millisecond { + t.Errorf("expected resources to process in < 150ms, average was %v", perResourceDuration) + } + } +} + +// Policy admission holder for test framework + +type policyExpectationHolder struct { + holder + warningLock sync.Mutex + warnings []string +} + +func (p *policyExpectationHolder) reset(t *testing.T) { + p.warningLock.Lock() + defer p.warningLock.Unlock() + p.warnings = nil + + p.holder.reset(t) + +} +func (p *policyExpectationHolder) expect(gvr schema.GroupVersionResource, gvk, optionsGVK schema.GroupVersionKind, operation v1beta1.Operation, name, namespace string, object, oldObject, options bool) { + p.holder.expect(gvr, gvk, optionsGVK, operation, name, namespace, object, oldObject, options) + + p.lock.Lock() + defer p.lock.Unlock() + // Set up the recorded map with nil records for all combinations + p.recorded = map[webhookOptions]*admissionRequest{} + for _, phase := range []string{validation} { + for _, converted := range []bool{true, false} { + for _, version := range []string{"v1alpha1", "v1beta1"} { + p.recorded[webhookOptions{version: version, phase: phase, converted: converted}] = nil + } + } + } +} + +func (p *policyExpectationHolder) verify(t *testing.T) { + p.warningLock.Lock() + defer p.warningLock.Unlock() + + // Process all detected warnings and record in the nested handler + for _, w := range p.warnings { + var currentRequest *admissionRequest + var currentParams webhookOptions + if idx := strings.Index(w, beginSentinel); idx >= 0 { + + csvData := strings.ReplaceAll(w[idx+len(beginSentinel):], recordSeparator, "\n") + + b := bytes.Buffer{} + b.WriteString(csvData) + reader := csv.NewReader(&b) + csvRecords, err := reader.ReadAll() + if err != nil { + t.Fatal(err) + return + } + + mappedCSV := []map[string]string{} + var header []string + for line, record := range csvRecords { + if line == 0 { + header = record + } else { + line := map[string]string{} + for i := 0; i < len(record); i++ { + line[header[i]] = record[i] + } + mappedCSV = append(mappedCSV, line) + } + } + + if len(mappedCSV) != 1 { + t.Fatal("incorrect # CSV elements in parsed warning") + return + } + + data := mappedCSV[0] + currentRequest = &admissionRequest{ + Operation: data["operation"], + Name: data["name"], + Namespace: data["namespace"], + Resource: metav1.GroupVersionResource{ + Group: data["resourceGroup"], + Version: data["resourceVersion"], + Resource: data["resourceResource"], + }, + SubResource: data["subresource"], + } + currentParams = webhookOptions{ + version: data["paramsVersion"], + phase: data["paramsPhase"], + converted: data["paramsConvert"] == "true", + } + + if e, ok := data["objectExists"]; ok && e == "true" { + currentRequest.Object.Object = &unstructured.Unstructured{} + currentRequest.Object.Object.(*unstructured.Unstructured).SetAPIVersion(data["objectAPIVersion"]) + currentRequest.Object.Object.(*unstructured.Unstructured).SetKind(data["objectKind"]) + } + + if e, ok := data["oldObjectExists"]; ok && e == "true" { + currentRequest.OldObject.Object = &unstructured.Unstructured{} + currentRequest.OldObject.Object.(*unstructured.Unstructured).SetAPIVersion(data["oldObjectAPIVersion"]) + currentRequest.OldObject.Object.(*unstructured.Unstructured).SetKind(data["oldObjectKind"]) + } + + if e, ok := data["optionsExists"]; ok && e == "true" { + currentRequest.Options.Object = &unstructured.Unstructured{} + currentRequest.Options.Object.(*unstructured.Unstructured).SetAPIVersion(data["optionsAPIVersion"]) + currentRequest.Options.Object.(*unstructured.Unstructured).SetKind(data["optionsKind"]) + } + + p.holder.record(currentParams.version, currentParams.phase, currentParams.converted, currentRequest) + } + } + + p.holder.verify(t) +} + +func (p *policyExpectationHolder) HandleWarningHeader(code int, agent string, message string) { + if code != 299 || len(message) == 0 { + return + } + p.warningLock.Lock() + defer p.warningLock.Unlock() + p.warnings = append(p.warnings, message) +} diff --git a/test/integration/apiserver/cel/admission_test_util.go b/test/integration/apiserver/cel/admission_test_util.go new file mode 100644 index 00000000000..9d38536731f --- /dev/null +++ b/test/integration/apiserver/cel/admission_test_util.go @@ -0,0 +1,1095 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cel + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "testing" + "time" + + "k8s.io/api/admission/v1beta1" + appsv1beta1 "k8s.io/api/apps/v1beta1" + authenticationv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + extensionsv1beta1 "k8s.io/api/extensions/v1beta1" + policyv1 "k8s.io/api/policy/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" + "k8s.io/kubernetes/test/integration/etcd" +) + +// Admission test framework copied from test/integration/apiserver/admissionwebhook/admission_test.go +// +// All differences between two are minor and called out in comments prefixed with +// "DIFF" + +const ( + testNamespace = "webhook-integration" + testClientUsername = "webhook-integration-client" + + mutation = "mutation" + validation = "validation" +) + +// DIFF: Added interface to replace direct *holder usage in testContext to be +// able to inject a policy-specific holder +type admissionTestExpectationHolder interface { + reset(t *testing.T) + expect(gvr schema.GroupVersionResource, gvk, optionsGVK schema.GroupVersionKind, operation v1beta1.Operation, name, namespace string, object, oldObject, options bool) + verify(t *testing.T) +} + +type testContext struct { + t *testing.T + + // DIFF: Changed from *holder to interface + admissionHolder admissionTestExpectationHolder + + client dynamic.Interface + clientset clientset.Interface + verb string + gvr schema.GroupVersionResource + resource metav1.APIResource + resources map[schema.GroupVersionResource]metav1.APIResource +} + +type testFunc func(*testContext) + +var ( + // defaultResourceFuncs holds the default test functions. + // may be overridden for specific resources by customTestFuncs. + defaultResourceFuncs = map[string]testFunc{ + "create": testResourceCreate, + "update": testResourceUpdate, + "patch": testResourcePatch, + "delete": testResourceDelete, + "deletecollection": testResourceDeletecollection, + } + + // defaultSubresourceFuncs holds default subresource test functions. + // may be overridden for specific resources by customTestFuncs. + defaultSubresourceFuncs = map[string]testFunc{ + "update": testSubresourceUpdate, + "patch": testSubresourcePatch, + } + + // customTestFuncs holds custom test functions by resource and verb. + customTestFuncs = map[schema.GroupVersionResource]map[string]testFunc{ + gvr("", "v1", "namespaces"): {"delete": testNamespaceDelete}, + + gvr("apps", "v1beta1", "deployments/rollback"): {"create": testDeploymentRollback}, + gvr("extensions", "v1beta1", "deployments/rollback"): {"create": testDeploymentRollback}, + + gvr("", "v1", "pods/attach"): {"create": testPodConnectSubresource}, + gvr("", "v1", "pods/exec"): {"create": testPodConnectSubresource}, + gvr("", "v1", "pods/portforward"): {"create": testPodConnectSubresource}, + + gvr("", "v1", "bindings"): {"create": testPodBindingEviction}, + gvr("", "v1", "pods/binding"): {"create": testPodBindingEviction}, + gvr("", "v1", "pods/eviction"): {"create": testPodBindingEviction}, + + gvr("", "v1", "nodes/proxy"): {"*": testSubresourceProxy}, + gvr("", "v1", "pods/proxy"): {"*": testSubresourceProxy}, + gvr("", "v1", "services/proxy"): {"*": testSubresourceProxy}, + + gvr("", "v1", "serviceaccounts/token"): {"create": testTokenCreate}, + + gvr("random.numbers.com", "v1", "integers"): {"create": testPruningRandomNumbers}, + + // DIFF: This test is used for webhook test but disabled here until we have mutating + // admission policy to write to "foo" field + // gvr("custom.fancy.com", "v2", "pants"): {"create": testNoPruningCustomFancy}, + } + + // admissionExemptResources lists objects which are exempt from admission validation/mutation, + // only resources exempted from admission processing by API server should be listed here. + admissionExemptResources = map[schema.GroupVersionResource]bool{ + // DIFF: WebhookConfigurations are exempt for webhook admission but not + // for policy admission. + // gvr("admissionregistration.k8s.io", "v1beta1", "mutatingwebhookconfigurations"): true, + // gvr("admissionregistration.k8s.io", "v1beta1", "validatingwebhookconfigurations"): true, + // gvr("admissionregistration.k8s.io", "v1", "mutatingwebhookconfigurations"): true, + // gvr("admissionregistration.k8s.io", "v1", "validatingwebhookconfigurations"): true, + gvr("admissionregistration.k8s.io", "v1alpha1", "validatingadmissionpolicies"): true, + gvr("admissionregistration.k8s.io", "v1alpha1", "validatingadmissionpolicies/status"): true, + gvr("admissionregistration.k8s.io", "v1alpha1", "validatingadmissionpolicybindings"): true, + gvr("admissionregistration.k8s.io", "v1beta1", "validatingadmissionpolicies"): true, + gvr("admissionregistration.k8s.io", "v1beta1", "validatingadmissionpolicies/status"): true, + gvr("admissionregistration.k8s.io", "v1beta1", "validatingadmissionpolicybindings"): true, + } + + parentResources = map[schema.GroupVersionResource]schema.GroupVersionResource{ + gvr("extensions", "v1beta1", "replicationcontrollers/scale"): gvr("", "v1", "replicationcontrollers"), + } + + // stubDataOverrides holds either non persistent resources' definitions or resources where default stub needs to be overridden. + stubDataOverrides = map[schema.GroupVersionResource]string{ + // Non persistent Reviews resource + gvr("authentication.k8s.io", "v1", "tokenreviews"): `{"metadata": {"name": "tokenreview"}, "spec": {"token": "token", "audience": ["audience1","audience2"]}}`, + gvr("authentication.k8s.io", "v1beta1", "tokenreviews"): `{"metadata": {"name": "tokenreview"}, "spec": {"token": "token", "audience": ["audience1","audience2"]}}`, + gvr("authentication.k8s.io", "v1alpha1", "selfsubjectreviews"): `{"metadata": {"name": "SelfSubjectReview"},"status":{"userInfo":{}}}`, + gvr("authentication.k8s.io", "v1beta1", "selfsubjectreviews"): `{"metadata": {"name": "SelfSubjectReview"},"status":{"userInfo":{}}}`, + gvr("authentication.k8s.io", "v1", "selfsubjectreviews"): `{"metadata": {"name": "SelfSubjectReview"},"status":{"userInfo":{}}}`, + gvr("authorization.k8s.io", "v1", "localsubjectaccessreviews"): `{"metadata": {"name": "", "namespace":"` + testNamespace + `"}, "spec": {"uid": "token", "user": "user1","groups": ["group1","group2"],"resourceAttributes": {"name":"name1","namespace":"` + testNamespace + `"}}}`, + gvr("authorization.k8s.io", "v1", "subjectaccessreviews"): `{"metadata": {"name": "", "namespace":""}, "spec": {"user":"user1","resourceAttributes": {"name":"name1", "namespace":"` + testNamespace + `"}}}`, + gvr("authorization.k8s.io", "v1", "selfsubjectaccessreviews"): `{"metadata": {"name": "", "namespace":""}, "spec": {"resourceAttributes": {"name":"name1", "namespace":""}}}`, + gvr("authorization.k8s.io", "v1", "selfsubjectrulesreviews"): `{"metadata": {"name": "", "namespace":"` + testNamespace + `"}, "spec": {"namespace":"` + testNamespace + `"}}`, + gvr("authorization.k8s.io", "v1beta1", "localsubjectaccessreviews"): `{"metadata": {"name": "", "namespace":"` + testNamespace + `"}, "spec": {"uid": "token", "user": "user1","groups": ["group1","group2"],"resourceAttributes": {"name":"name1","namespace":"` + testNamespace + `"}}}`, + gvr("authorization.k8s.io", "v1beta1", "subjectaccessreviews"): `{"metadata": {"name": "", "namespace":""}, "spec": {"user":"user1","resourceAttributes": {"name":"name1", "namespace":"` + testNamespace + `"}}}`, + gvr("authorization.k8s.io", "v1beta1", "selfsubjectaccessreviews"): `{"metadata": {"name": "", "namespace":""}, "spec": {"resourceAttributes": {"name":"name1", "namespace":""}}}`, + gvr("authorization.k8s.io", "v1beta1", "selfsubjectrulesreviews"): `{"metadata": {"name": "", "namespace":"` + testNamespace + `"}, "spec": {"namespace":"` + testNamespace + `"}}`, + + // Other Non persistent resources + } +) + +type webhookOptions struct { + version string + + // phase indicates whether this is a mutating or validating webhook + phase string + // converted indicates if this webhook makes use of matchPolicy:equivalent and expects conversion. + // if true, recordGVR and expectGVK are mapped through gvrToConvertedGVR/gvrToConvertedGVK. + // if false, recordGVR and expectGVK are compared directly to the admission review. + converted bool +} + +type holder struct { + lock sync.RWMutex + + t *testing.T + + // DIFF: Warning handler removed in policy test. + // warningHandler *warningHandler + + recordGVR metav1.GroupVersionResource + recordOperation string + recordNamespace string + recordName string + + expectGVK schema.GroupVersionKind + expectObject bool + expectOldObject bool + expectOptionsGVK schema.GroupVersionKind + expectOptions bool + + // gvrToConvertedGVR maps the GVR submitted to the API server to the expected GVR when converted to the webhook-recognized resource. + // When a converted request is recorded, gvrToConvertedGVR[recordGVR] is compared to the GVR seen by the webhook. + gvrToConvertedGVR map[metav1.GroupVersionResource]metav1.GroupVersionResource + // gvrToConvertedGVR maps the GVR submitted to the API server to the expected GVK when converted to the webhook-recognized resource. + // When a converted request is recorded, gvrToConvertedGVR[expectGVK] is compared to the GVK seen by the webhook. + gvrToConvertedGVK map[metav1.GroupVersionResource]schema.GroupVersionKind + + recorded map[webhookOptions]*admissionRequest +} + +func (h *holder) reset(t *testing.T) { + h.lock.Lock() + defer h.lock.Unlock() + h.t = t + h.recordGVR = metav1.GroupVersionResource{} + h.expectGVK = schema.GroupVersionKind{} + h.recordOperation = "" + h.recordName = "" + h.recordNamespace = "" + h.expectObject = false + h.expectOldObject = false + h.expectOptionsGVK = schema.GroupVersionKind{} + h.expectOptions = false + // DIFF: Warning handler removed + // h.warningHandler.reset() + + // Set up the recorded map with nil records for all combinations + h.recorded = map[webhookOptions]*admissionRequest{} + for _, phase := range []string{mutation, validation} { + for _, converted := range []bool{true, false} { + for _, version := range []string{"v1", "v1beta1"} { + h.recorded[webhookOptions{version: version, phase: phase, converted: converted}] = nil + } + } + } +} +func (h *holder) expect(gvr schema.GroupVersionResource, gvk, optionsGVK schema.GroupVersionKind, operation v1beta1.Operation, name, namespace string, object, oldObject, options bool) { + // Special-case namespaces, since the object name shows up in request attributes + if len(namespace) == 0 && gvk.Group == "" && gvk.Version == "v1" && gvk.Kind == "Namespace" { + namespace = name + } + + h.lock.Lock() + defer h.lock.Unlock() + h.recordGVR = metav1.GroupVersionResource{Group: gvr.Group, Version: gvr.Version, Resource: gvr.Resource} + h.expectGVK = gvk + h.recordOperation = string(operation) + h.recordName = name + h.recordNamespace = namespace + h.expectObject = object + h.expectOldObject = oldObject + h.expectOptionsGVK = optionsGVK + h.expectOptions = options + // DIFF: Warning handler removed + // h.warningHandler.reset() + + // Set up the recorded map with nil records for all combinations + h.recorded = map[webhookOptions]*admissionRequest{} + for _, phase := range []string{mutation, validation} { + for _, converted := range []bool{true, false} { + for _, version := range []string{"v1", "v1beta1"} { + h.recorded[webhookOptions{version: version, phase: phase, converted: converted}] = nil + } + } + } +} + +type admissionRequest struct { + Operation string + Resource metav1.GroupVersionResource + SubResource string + Namespace string + Name string + Object runtime.RawExtension + OldObject runtime.RawExtension + Options runtime.RawExtension +} + +func (h *holder) record(version string, phase string, converted bool, request *admissionRequest) { + h.lock.Lock() + defer h.lock.Unlock() + + // this is useful to turn on if items aren't getting recorded and you need to figure out why + debug := false + if debug { + h.t.Logf("%s %#v %v", request.Operation, request.Resource, request.SubResource) + } + + resource := request.Resource + if len(request.SubResource) > 0 { + resource.Resource += "/" + request.SubResource + } + + // See if we should record this + gvrToRecord := h.recordGVR + if converted { + // If this is a converted webhook, map to the GVR we expect the webhook to see + gvrToRecord = h.gvrToConvertedGVR[h.recordGVR] + } + if resource != gvrToRecord { + if debug { + h.t.Log(resource, "!=", gvrToRecord) + } + return + } + + if request.Operation != h.recordOperation { + if debug { + h.t.Log(request.Operation, "!=", h.recordOperation) + } + return + } + if request.Namespace != h.recordNamespace { + if debug { + h.t.Log(request.Namespace, "!=", h.recordNamespace) + } + return + } + + name := request.Name + if name != h.recordName { + if debug { + h.t.Log(name, "!=", h.recordName) + } + return + } + + if debug { + h.t.Logf("recording: %#v = %s %#v %v", webhookOptions{version: version, phase: phase, converted: converted}, request.Operation, request.Resource, request.SubResource) + } + h.recorded[webhookOptions{version: version, phase: phase, converted: converted}] = request +} + +func (h *holder) verify(t *testing.T) { + h.lock.Lock() + defer h.lock.Unlock() + + for options, value := range h.recorded { + if err := h.verifyRequest(options, value); err != nil { + t.Errorf("version: %v, phase:%v, converted:%v error: %v", options.version, options.phase, options.converted, err) + } + } +} + +func (h *holder) verifyRequest(webhookOptions webhookOptions, request *admissionRequest) error { + converted := webhookOptions.converted + + // Check if current resource should be exempted from Admission processing + if admissionExemptResources[gvr(h.recordGVR.Group, h.recordGVR.Version, h.recordGVR.Resource)] { + if request == nil { + return nil + } + return fmt.Errorf("admission webhook was called, but not supposed to") + } + + if request == nil { + return fmt.Errorf("no request received") + } + + if h.expectObject { + if err := h.verifyObject(converted, request.Object.Object); err != nil { + return fmt.Errorf("object error: %v", err) + } + } else if request.Object.Object != nil { + return fmt.Errorf("unexpected object: %#v", request.Object.Object) + } + + if h.expectOldObject { + if err := h.verifyObject(converted, request.OldObject.Object); err != nil { + return fmt.Errorf("old object error: %v", err) + } + } else if request.OldObject.Object != nil { + return fmt.Errorf("unexpected old object: %#v", request.OldObject.Object) + } + + if h.expectOptions { + if err := h.verifyOptions(request.Options.Object); err != nil { + return fmt.Errorf("options error: %v", err) + } + } else if request.Options.Object != nil { + return fmt.Errorf("unexpected options: %#v", request.Options.Object) + } + + // DIFF: This check was removed for policy tests since it only applies + // to webhook + // if !h.warningHandler.hasWarning(makeWarning(webhookOptions.version, webhookOptions.phase, webhookOptions.converted)) { + // return fmt.Errorf("no warning received from webhook") + // } + + return nil +} + +func (h *holder) verifyObject(converted bool, obj runtime.Object) error { + if obj == nil { + return fmt.Errorf("no object sent") + } + expectGVK := h.expectGVK + if converted { + expectGVK = h.gvrToConvertedGVK[h.recordGVR] + } + if obj.GetObjectKind().GroupVersionKind() != expectGVK { + return fmt.Errorf("expected %#v, got %#v", expectGVK, obj.GetObjectKind().GroupVersionKind()) + } + return nil +} + +func (h *holder) verifyOptions(options runtime.Object) error { + if options == nil { + return fmt.Errorf("no options sent") + } + if options.GetObjectKind().GroupVersionKind() != h.expectOptionsGVK { + return fmt.Errorf("expected %#v, got %#v", h.expectOptionsGVK, options.GetObjectKind().GroupVersionKind()) + } + return nil +} + +func getTestFunc(gvr schema.GroupVersionResource, verb string) testFunc { + if f, found := customTestFuncs[gvr][verb]; found { + return f + } + if f, found := customTestFuncs[gvr]["*"]; found { + return f + } + if strings.Contains(gvr.Resource, "/") { + if f, found := defaultSubresourceFuncs[verb]; found { + return f + } + return unimplemented + } + if f, found := defaultResourceFuncs[verb]; found { + return f + } + return unimplemented +} + +func getStubObj(gvr schema.GroupVersionResource, resource metav1.APIResource) (*unstructured.Unstructured, error) { + stub := "" + if data, ok := etcd.GetEtcdStorageDataForNamespace(testNamespace)[gvr]; ok { + stub = data.Stub + } + if data, ok := stubDataOverrides[gvr]; ok { + stub = data + } + if len(stub) == 0 { + return nil, fmt.Errorf("no stub data for %#v", gvr) + } + + stubObj := &unstructured.Unstructured{Object: map[string]interface{}{}} + if err := json.Unmarshal([]byte(stub), &stubObj.Object); err != nil { + return nil, fmt.Errorf("error unmarshaling stub for %#v: %v", gvr, err) + } + return stubObj, nil +} + +func createOrGetResource(client dynamic.Interface, gvr schema.GroupVersionResource, resource metav1.APIResource) (*unstructured.Unstructured, error) { + stubObj, err := getStubObj(gvr, resource) + if err != nil { + return nil, err + } + ns := "" + if resource.Namespaced { + ns = testNamespace + } + obj, err := client.Resource(gvr).Namespace(ns).Get(context.TODO(), stubObj.GetName(), metav1.GetOptions{}) + if err == nil { + return obj, nil + } + if !apierrors.IsNotFound(err) { + return nil, err + } + return client.Resource(gvr).Namespace(ns).Create(context.TODO(), stubObj, metav1.CreateOptions{}) +} + +func gvr(group, version, resource string) schema.GroupVersionResource { + return schema.GroupVersionResource{Group: group, Version: version, Resource: resource} +} +func gvk(group, version, kind string) schema.GroupVersionKind { + return schema.GroupVersionKind{Group: group, Version: version, Kind: kind} +} + +var ( + gvkCreateOptions = metav1.SchemeGroupVersion.WithKind("CreateOptions") + gvkUpdateOptions = metav1.SchemeGroupVersion.WithKind("UpdateOptions") + gvkDeleteOptions = metav1.SchemeGroupVersion.WithKind("DeleteOptions") +) + +func shouldTestResource(gvr schema.GroupVersionResource, resource metav1.APIResource) bool { + return sets.NewString(resource.Verbs...).HasAny("create", "update", "patch", "connect", "delete", "deletecollection") +} + +func shouldTestResourceVerb(gvr schema.GroupVersionResource, resource metav1.APIResource, verb string) bool { + return sets.NewString(resource.Verbs...).Has(verb) +} + +// +// generic resource testing +// + +func testResourceCreate(c *testContext) { + stubObj, err := getStubObj(c.gvr, c.resource) + if err != nil { + c.t.Error(err) + return + } + ns := "" + if c.resource.Namespaced { + ns = testNamespace + } + c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkCreateOptions, v1beta1.Create, stubObj.GetName(), ns, true, false, true) + _, err = c.client.Resource(c.gvr).Namespace(ns).Create(context.TODO(), stubObj, metav1.CreateOptions{}) + if err != nil { + c.t.Error(err) + return + } +} + +func testResourceUpdate(c *testContext) { + if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + obj, err := createOrGetResource(c.client, c.gvr, c.resource) + if err != nil { + return err + } + obj.SetAnnotations(map[string]string{"update": "true"}) + c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkUpdateOptions, v1beta1.Update, obj.GetName(), obj.GetNamespace(), true, true, true) + _, err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Update(context.TODO(), obj, metav1.UpdateOptions{}) + return err + }); err != nil { + c.t.Error(err) + return + } +} + +func testResourcePatch(c *testContext) { + obj, err := createOrGetResource(c.client, c.gvr, c.resource) + if err != nil { + c.t.Error(err) + return + } + c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkUpdateOptions, v1beta1.Update, obj.GetName(), obj.GetNamespace(), true, true, true) + _, err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Patch( + context.TODO(), + obj.GetName(), + types.MergePatchType, + []byte(`{"metadata":{"annotations":{"patch":"true"}}}`), + metav1.PatchOptions{}) + if err != nil { + c.t.Error(err) + return + } +} + +func testResourceDelete(c *testContext) { + // Verify that an immediate delete triggers the webhook and populates the admisssionRequest.oldObject. + obj, err := createOrGetResource(c.client, c.gvr, c.resource) + if err != nil { + c.t.Error(err) + return + } + background := metav1.DeletePropagationBackground + zero := int64(0) + c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkDeleteOptions, v1beta1.Delete, obj.GetName(), obj.GetNamespace(), false, true, true) + err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Delete(context.TODO(), obj.GetName(), metav1.DeleteOptions{GracePeriodSeconds: &zero, PropagationPolicy: &background}) + if err != nil { + c.t.Error(err) + return + } + c.admissionHolder.verify(c.t) + + // wait for the item to be gone + err = wait.PollImmediate(100*time.Millisecond, 10*time.Second, func() (bool, error) { + obj, err := c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return true, nil + } + if err == nil { + c.t.Logf("waiting for %#v to be deleted (name: %s, finalizers: %v)...\n", c.gvr, obj.GetName(), obj.GetFinalizers()) + return false, nil + } + return false, err + }) + if err != nil { + c.t.Error(err) + return + } + + // Verify that an update-on-delete triggers the webhook and populates the admisssionRequest.oldObject. + obj, err = createOrGetResource(c.client, c.gvr, c.resource) + if err != nil { + c.t.Error(err) + return + } + // Adding finalizer to the object, then deleting it. + // We don't add finalizers by setting DeleteOptions.PropagationPolicy + // because some resource (e.g., events) do not support garbage + // collector finalizers. + _, err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Patch( + context.TODO(), + obj.GetName(), + types.MergePatchType, + []byte(`{"metadata":{"finalizers":["test/k8s.io"]}}`), + metav1.PatchOptions{}) + if err != nil { + c.t.Error(err) + return + } + c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkDeleteOptions, v1beta1.Delete, obj.GetName(), obj.GetNamespace(), false, true, true) + err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Delete(context.TODO(), obj.GetName(), metav1.DeleteOptions{GracePeriodSeconds: &zero, PropagationPolicy: &background}) + if err != nil { + c.t.Error(err) + return + } + c.admissionHolder.verify(c.t) + + // wait other finalizers (e.g., crd's customresourcecleanup finalizer) to be removed. + err = wait.PollImmediate(100*time.Millisecond, 10*time.Second, func() (bool, error) { + obj, err := c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + if err != nil { + return false, err + } + finalizers := obj.GetFinalizers() + if len(finalizers) != 1 { + c.t.Logf("waiting for other finalizers on %#v %s to be removed, existing finalizers are %v", c.gvr, obj.GetName(), obj.GetFinalizers()) + return false, nil + } + if finalizers[0] != "test/k8s.io" { + return false, fmt.Errorf("expected the single finalizer on %#v %s to be test/k8s.io, got %v", c.gvr, obj.GetName(), obj.GetFinalizers()) + } + return true, nil + }) + if err != nil { + c.t.Error(err) + return + } + + // remove the finalizer + _, err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Patch( + context.TODO(), + obj.GetName(), + types.MergePatchType, + []byte(`{"metadata":{"finalizers":[]}}`), + metav1.PatchOptions{}) + if err != nil { + c.t.Error(err) + return + } + // wait for the item to be gone + err = wait.PollImmediate(100*time.Millisecond, 10*time.Second, func() (bool, error) { + obj, err := c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return true, nil + } + if err == nil { + c.t.Logf("waiting for %#v to be deleted (name: %s, finalizers: %v)...\n", c.gvr, obj.GetName(), obj.GetFinalizers()) + return false, nil + } + return false, err + }) + if err != nil { + c.t.Error(err) + return + } +} + +func testResourceDeletecollection(c *testContext) { + obj, err := createOrGetResource(c.client, c.gvr, c.resource) + if err != nil { + c.t.Error(err) + return + } + background := metav1.DeletePropagationBackground + zero := int64(0) + + // update the object with a label that matches our selector + _, err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Patch( + context.TODO(), + obj.GetName(), + types.MergePatchType, + []byte(`{"metadata":{"labels":{"webhooktest":"true"}}}`), + metav1.PatchOptions{}) + if err != nil { + c.t.Error(err) + return + } + + // set expectations + c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkDeleteOptions, v1beta1.Delete, "", obj.GetNamespace(), false, true, true) + + // delete + err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).DeleteCollection(context.TODO(), metav1.DeleteOptions{GracePeriodSeconds: &zero, PropagationPolicy: &background}, metav1.ListOptions{LabelSelector: "webhooktest=true"}) + if err != nil { + c.t.Error(err) + return + } + + // wait for the item to be gone + err = wait.PollImmediate(100*time.Millisecond, 10*time.Second, func() (bool, error) { + obj, err := c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return true, nil + } + if err == nil { + c.t.Logf("waiting for %#v to be deleted (name: %s, finalizers: %v)...\n", c.gvr, obj.GetName(), obj.GetFinalizers()) + return false, nil + } + return false, err + }) + if err != nil { + c.t.Error(err) + return + } +} + +func getParentGVR(gvr schema.GroupVersionResource) schema.GroupVersionResource { + parentGVR, found := parentResources[gvr] + // if no special override is found, just drop the subresource + if !found { + parentGVR = gvr + parentGVR.Resource = strings.Split(parentGVR.Resource, "/")[0] + } + return parentGVR +} + +func testTokenCreate(c *testContext) { + saGVR := gvr("", "v1", "serviceaccounts") + sa, err := createOrGetResource(c.client, saGVR, c.resources[saGVR]) + if err != nil { + c.t.Error(err) + return + } + + c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkCreateOptions, v1beta1.Create, sa.GetName(), sa.GetNamespace(), true, false, true) + if err = c.clientset.CoreV1().RESTClient().Post().Namespace(sa.GetNamespace()).Resource("serviceaccounts").Name(sa.GetName()).SubResource("token").Body(&authenticationv1.TokenRequest{ + ObjectMeta: metav1.ObjectMeta{Name: sa.GetName()}, + Spec: authenticationv1.TokenRequestSpec{ + Audiences: []string{"api"}, + }, + }).Do(context.TODO()).Error(); err != nil { + c.t.Error(err) + return + } + c.admissionHolder.verify(c.t) +} + +func testSubresourceUpdate(c *testContext) { + if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + parentGVR := getParentGVR(c.gvr) + parentResource := c.resources[parentGVR] + obj, err := createOrGetResource(c.client, parentGVR, parentResource) + if err != nil { + return err + } + + // Save the parent object as what we submit + submitObj := obj + + gvrWithoutSubresources := c.gvr + gvrWithoutSubresources.Resource = strings.Split(gvrWithoutSubresources.Resource, "/")[0] + subresources := strings.Split(c.gvr.Resource, "/")[1:] + + // If the subresource supports get, fetch that as the object to submit (namespaces/finalize, */scale, etc) + if sets.NewString(c.resource.Verbs...).Has("get") { + submitObj, err = c.client.Resource(gvrWithoutSubresources).Namespace(obj.GetNamespace()).Get(context.TODO(), obj.GetName(), metav1.GetOptions{}, subresources...) + if err != nil { + return err + } + } + + // Modify the object + submitObj.SetAnnotations(map[string]string{"subresourceupdate": "true"}) + + // set expectations + c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkUpdateOptions, v1beta1.Update, obj.GetName(), obj.GetNamespace(), true, true, true) + + _, err = c.client.Resource(gvrWithoutSubresources).Namespace(obj.GetNamespace()).Update( + context.TODO(), + submitObj, + metav1.UpdateOptions{}, + subresources..., + ) + return err + }); err != nil { + c.t.Error(err) + } +} + +func testSubresourcePatch(c *testContext) { + parentGVR := getParentGVR(c.gvr) + parentResource := c.resources[parentGVR] + obj, err := createOrGetResource(c.client, parentGVR, parentResource) + if err != nil { + c.t.Error(err) + return + } + + gvrWithoutSubresources := c.gvr + gvrWithoutSubresources.Resource = strings.Split(gvrWithoutSubresources.Resource, "/")[0] + subresources := strings.Split(c.gvr.Resource, "/")[1:] + + // set expectations + c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkUpdateOptions, v1beta1.Update, obj.GetName(), obj.GetNamespace(), true, true, true) + + _, err = c.client.Resource(gvrWithoutSubresources).Namespace(obj.GetNamespace()).Patch( + context.TODO(), + obj.GetName(), + types.MergePatchType, + []byte(`{"metadata":{"annotations":{"subresourcepatch":"true"}}}`), + metav1.PatchOptions{}, + subresources..., + ) + if err != nil { + c.t.Error(err) + return + } +} + +func unimplemented(c *testContext) { + c.t.Errorf("Test function for %+v has not been implemented...", c.gvr) +} + +// +// custom methods +// + +// testNamespaceDelete verifies namespace-specific delete behavior: +// - ensures admission is called on first delete (which only sets deletionTimestamp and terminating state) +// - removes finalizer from namespace +// - ensures admission is called on final delete once finalizers are removed +func testNamespaceDelete(c *testContext) { + obj, err := createOrGetResource(c.client, c.gvr, c.resource) + if err != nil { + c.t.Error(err) + return + } + background := metav1.DeletePropagationBackground + zero := int64(0) + + c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkDeleteOptions, v1beta1.Delete, obj.GetName(), obj.GetNamespace(), false, true, true) + err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Delete(context.TODO(), obj.GetName(), metav1.DeleteOptions{GracePeriodSeconds: &zero, PropagationPolicy: &background}) + if err != nil { + c.t.Error(err) + return + } + c.admissionHolder.verify(c.t) + + // do the finalization so the namespace can be deleted + obj, err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + if err != nil { + c.t.Error(err) + return + } + err = unstructured.SetNestedField(obj.Object, nil, "spec", "finalizers") + if err != nil { + c.t.Error(err) + return + } + _, err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Update(context.TODO(), obj, metav1.UpdateOptions{}, "finalize") + if err != nil { + c.t.Error(err) + return + } + // verify namespace is gone + obj, err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + if err == nil || !apierrors.IsNotFound(err) { + c.t.Errorf("expected namespace to be gone, got %#v, %v", obj, err) + } +} + +// testDeploymentRollback verifies rollback-specific behavior: +// - creates a parent deployment +// - creates a rollback object and posts it +func testDeploymentRollback(c *testContext) { + deploymentGVR := gvr("apps", "v1", "deployments") + obj, err := createOrGetResource(c.client, deploymentGVR, c.resources[deploymentGVR]) + if err != nil { + c.t.Error(err) + return + } + + gvrWithoutSubresources := c.gvr + gvrWithoutSubresources.Resource = strings.Split(gvrWithoutSubresources.Resource, "/")[0] + subresources := strings.Split(c.gvr.Resource, "/")[1:] + + c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkCreateOptions, v1beta1.Create, obj.GetName(), obj.GetNamespace(), true, false, true) + + var rollbackObj runtime.Object + switch c.gvr { + case gvr("apps", "v1beta1", "deployments/rollback"): + rollbackObj = &appsv1beta1.DeploymentRollback{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1beta1", Kind: "DeploymentRollback"}, + Name: obj.GetName(), + RollbackTo: appsv1beta1.RollbackConfig{Revision: 0}, + } + case gvr("extensions", "v1beta1", "deployments/rollback"): + rollbackObj = &extensionsv1beta1.DeploymentRollback{ + TypeMeta: metav1.TypeMeta{APIVersion: "extensions/v1beta1", Kind: "DeploymentRollback"}, + Name: obj.GetName(), + RollbackTo: extensionsv1beta1.RollbackConfig{Revision: 0}, + } + default: + c.t.Errorf("unknown rollback resource %#v", c.gvr) + return + } + + rollbackUnstructuredBody, err := runtime.DefaultUnstructuredConverter.ToUnstructured(rollbackObj) + if err != nil { + c.t.Errorf("ToUnstructured failed: %v", err) + return + } + rollbackUnstructuredObj := &unstructured.Unstructured{Object: rollbackUnstructuredBody} + rollbackUnstructuredObj.SetName(obj.GetName()) + + _, err = c.client.Resource(gvrWithoutSubresources).Namespace(obj.GetNamespace()).Create(context.TODO(), rollbackUnstructuredObj, metav1.CreateOptions{}, subresources...) + if err != nil { + c.t.Error(err) + return + } +} + +// testPodConnectSubresource verifies connect subresources +func testPodConnectSubresource(c *testContext) { + podGVR := gvr("", "v1", "pods") + pod, err := createOrGetResource(c.client, podGVR, c.resources[podGVR]) + if err != nil { + c.t.Error(err) + return + } + + // check all upgradeable verbs + for _, httpMethod := range []string{"GET", "POST"} { + c.t.Logf("verifying %v", httpMethod) + + c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), schema.GroupVersionKind{}, v1beta1.Connect, pod.GetName(), pod.GetNamespace(), true, false, false) + var err error + switch c.gvr { + case gvr("", "v1", "pods/exec"): + err = c.clientset.CoreV1().RESTClient().Verb(httpMethod).Namespace(pod.GetNamespace()).Resource("pods").Name(pod.GetName()).SubResource("exec").Do(context.TODO()).Error() + case gvr("", "v1", "pods/attach"): + err = c.clientset.CoreV1().RESTClient().Verb(httpMethod).Namespace(pod.GetNamespace()).Resource("pods").Name(pod.GetName()).SubResource("attach").Do(context.TODO()).Error() + case gvr("", "v1", "pods/portforward"): + err = c.clientset.CoreV1().RESTClient().Verb(httpMethod).Namespace(pod.GetNamespace()).Resource("pods").Name(pod.GetName()).SubResource("portforward").Do(context.TODO()).Error() + default: + c.t.Errorf("unknown subresource %#v", c.gvr) + return + } + + if err != nil { + c.t.Logf("debug: result of subresource connect: %v", err) + } + c.admissionHolder.verify(c.t) + + } +} + +// testPodBindingEviction verifies pod binding and eviction admission +func testPodBindingEviction(c *testContext) { + podGVR := gvr("", "v1", "pods") + pod, err := createOrGetResource(c.client, podGVR, c.resources[podGVR]) + if err != nil { + c.t.Error(err) + return + } + + background := metav1.DeletePropagationBackground + zero := int64(0) + forceDelete := metav1.DeleteOptions{GracePeriodSeconds: &zero, PropagationPolicy: &background} + defer func() { + err := c.clientset.CoreV1().Pods(pod.GetNamespace()).Delete(context.TODO(), pod.GetName(), forceDelete) + if err != nil && !apierrors.IsNotFound(err) { + c.t.Error(err) + return + } + }() + + c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkCreateOptions, v1beta1.Create, pod.GetName(), pod.GetNamespace(), true, false, true) + + switch c.gvr { + case gvr("", "v1", "bindings"): + err = c.clientset.CoreV1().RESTClient().Post().Namespace(pod.GetNamespace()).Resource("bindings").Body(&corev1.Binding{ + ObjectMeta: metav1.ObjectMeta{Name: pod.GetName()}, + Target: corev1.ObjectReference{Name: "foo", Kind: "Node", APIVersion: "v1"}, + }).Do(context.TODO()).Error() + + case gvr("", "v1", "pods/binding"): + err = c.clientset.CoreV1().RESTClient().Post().Namespace(pod.GetNamespace()).Resource("pods").Name(pod.GetName()).SubResource("binding").Body(&corev1.Binding{ + ObjectMeta: metav1.ObjectMeta{Name: pod.GetName()}, + Target: corev1.ObjectReference{Name: "foo", Kind: "Node", APIVersion: "v1"}, + }).Do(context.TODO()).Error() + + case gvr("", "v1", "pods/eviction"): + err = c.clientset.CoreV1().RESTClient().Post().Namespace(pod.GetNamespace()).Resource("pods").Name(pod.GetName()).SubResource("eviction").Body(&policyv1.Eviction{ + ObjectMeta: metav1.ObjectMeta{Name: pod.GetName()}, + DeleteOptions: &forceDelete, + }).Do(context.TODO()).Error() + + default: + c.t.Errorf("unhandled resource %#v", c.gvr) + return + } + + if err != nil { + c.t.Error(err) + return + } +} + +// testSubresourceProxy verifies proxy subresources +func testSubresourceProxy(c *testContext) { + parentGVR := getParentGVR(c.gvr) + parentResource := c.resources[parentGVR] + obj, err := createOrGetResource(c.client, parentGVR, parentResource) + if err != nil { + c.t.Error(err) + return + } + + gvrWithoutSubresources := c.gvr + gvrWithoutSubresources.Resource = strings.Split(gvrWithoutSubresources.Resource, "/")[0] + subresources := strings.Split(c.gvr.Resource, "/")[1:] + + verbToHTTPMethods := map[string][]string{ + "create": {"POST", "GET", "HEAD", "OPTIONS"}, // also test read-only verbs map to Connect admission + "update": {"PUT"}, + "patch": {"PATCH"}, + "delete": {"DELETE"}, + } + httpMethodsToTest, ok := verbToHTTPMethods[c.verb] + if !ok { + c.t.Errorf("unknown verb %v", c.verb) + return + } + + for _, httpMethod := range httpMethodsToTest { + c.t.Logf("testing %v", httpMethod) + request := c.clientset.CoreV1().RESTClient().Verb(httpMethod) + + // add the namespace if required + if len(obj.GetNamespace()) > 0 { + request = request.Namespace(obj.GetNamespace()) + } + + // set expectations + c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), schema.GroupVersionKind{}, v1beta1.Connect, obj.GetName(), obj.GetNamespace(), true, false, false) + // run the request. we don't actually care if the request is successful, just that admission gets called as expected + err = request.Resource(gvrWithoutSubresources.Resource).Name(obj.GetName()).SubResource(subresources...).Do(context.TODO()).Error() + if err != nil { + c.t.Logf("debug: result of subresource proxy (error expected): %v", err) + } + // verify the result + c.admissionHolder.verify(c.t) + } +} + +func testPruningRandomNumbers(c *testContext) { + testResourceCreate(c) + + cr2pant, err := c.client.Resource(c.gvr).Get(context.TODO(), "fortytwo", metav1.GetOptions{}) + if err != nil { + c.t.Error(err) + return + } + + foo, found, err := unstructured.NestedString(cr2pant.Object, "foo") + if err != nil { + c.t.Error(err) + return + } + if found { + c.t.Errorf("expected .foo to be pruned, but got: %s", foo) + } +} + +// DIFF: Commented out for policy test. To be added back for mutating policy tests. +// This test deoends on "foo" being set to test by admission webhook/policy. +// func testNoPruningCustomFancy(c *testContext) { +// testResourceCreate(c) + +// cr2pant, err := c.client.Resource(c.gvr).Get(context.TODO(), "cr2pant", metav1.GetOptions{}) +// if err != nil { +// c.t.Error(err) +// return +// } + +// foo, _, err := unstructured.NestedString(cr2pant.Object, "foo") +// if err != nil { +// c.t.Error(err) +// return +// } + +// // check that no pruning took place +// if expected, got := "test", foo; expected != got { +// c.t.Errorf("expected /foo to be %q, got: %q", expected, got) +// } +// }