diff --git a/test/e2e/apimachinery/crd_selectable_fields.go b/test/e2e/apimachinery/crd_selectable_fields.go index ecbedfb27be..7330f2dbb73 100644 --- a/test/e2e/apimachinery/crd_selectable_fields.go +++ b/test/e2e/apimachinery/crd_selectable_fields.go @@ -18,12 +18,11 @@ package apimachinery import ( "context" - "encoding/json" + "fmt" "github.com/onsi/gomega" "time" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apiextensions-apiserver/test/integration/fixtures" "k8s.io/apimachinery/pkg/api/meta" @@ -35,7 +34,10 @@ import ( "k8s.io/apiserver/pkg/storage/names" "k8s.io/client-go/dynamic" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/utils/crd" + imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" + "k8s.io/utils/ptr" "github.com/onsi/ginkgo/v2" ) @@ -46,34 +48,67 @@ var _ = SIGDescribe("CustomResourceFieldSelectors [Privileged:ClusterAdmin]", fr f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged ginkgo.Context("CustomResourceFieldSelectors", func() { - var apiExtensionClient *clientset.Clientset - ginkgo.BeforeEach(func() { - var err error - apiExtensionClient, err = clientset.NewForConfig(f.ClientConfig()) - framework.ExpectNoError(err, "initializing apiExtensionClient") - }) - - customResourceClient := func(crd *apiextensionsv1.CustomResourceDefinition) (dynamic.NamespaceableResourceInterface, schema.GroupVersionResource) { + customResourceClient := func(crd *apiextensionsv1.CustomResourceDefinition, version string) (dynamic.NamespaceableResourceInterface, schema.GroupVersionResource) { gvrs := fixtures.GetGroupVersionResourcesOfCustomResource(crd) - if len(gvrs) != 1 { - ginkgo.Fail("Expected one version in custom resource definition") - } - gvr := gvrs[0] - return f.DynamicClient.Resource(gvr), gvr - } - - var schemaWithValidationExpression = unmarshalSchema([]byte(`{ - "type":"object", - "properties":{ - "spec":{ - "type":"object", - "properties":{ - "color":{ "type":"string" }, - "quantity":{ "type":"integer" } - } + for _, gvr := range gvrs { + if gvr.Version == version { + return f.DynamicClient.Resource(gvr), gvr } } - }`)) + ginkgo.Fail(fmt.Sprintf("Expected version '%s' in custom resource definition", version)) + return nil, schema.GroupVersionResource{} + } + + var apiVersions = []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "hostPort": {Type: "string"}, + }, + }, + }, + SelectableFields: []apiextensionsv1.SelectableField{ + {JSONPath: ".hostPort"}, + }, + }, + { + Name: "v2", + Served: true, + Storage: false, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "host": {Type: "string"}, + "port": {Type: "string"}, + }, + }, + }, + SelectableFields: []apiextensionsv1.SelectableField{ + {JSONPath: ".host"}, + {JSONPath: ".port"}, + }, + }, + } + + var certCtx *certContext + servicePort := int32(9443) + containerPort := int32(9444) + + ginkgo.BeforeEach(func(ctx context.Context) { + ginkgo.DeferCleanup(cleanCRDWebhookTest, f.ClientSet, f.Namespace.Name) + + ginkgo.By("Setting up server cert") + certCtx = setupServerCert(f.Namespace.Name, serviceCRDName) + createAuthReaderRoleBindingForCRDConversion(ctx, f, f.Namespace.Name) + + deployCustomResourceWebhookAndService(ctx, f, imageutils.GetE2EImage(imageutils.Agnhost), certCtx, servicePort, containerPort) + }) /* Release: v1.31 @@ -86,46 +121,58 @@ var _ = SIGDescribe("CustomResourceFieldSelectors [Privileged:ClusterAdmin]", fr */ framework.It("MUST list and watch custom resources matching the field selector", func(ctx context.Context) { ginkgo.By("Creating a custom resource definition with selectable fields") - crd := fixtures.NewRandomNameV1CustomResourceDefinitionWithSchema(apiextensionsv1.NamespaceScoped, schemaWithValidationExpression, false) - for i := range crd.Spec.Versions { - crd.Spec.Versions[i].SelectableFields = []apiextensionsv1.SelectableField{ - {JSONPath: ".spec.color"}, - {JSONPath: ".spec.quantity"}, + testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *apiextensionsv1.CustomResourceDefinition) { + crd.Spec.Versions = apiVersions + crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{ + Strategy: apiextensionsv1.WebhookConverter, + Webhook: &apiextensionsv1.WebhookConversion{ + ClientConfig: &apiextensionsv1.WebhookClientConfig{ + CABundle: certCtx.signingCert, + Service: &apiextensionsv1.ServiceReference{ + Namespace: f.Namespace.Name, + Name: serviceCRDName, + Path: ptr.To("/crdconvert"), + Port: ptr.To(servicePort), + }, + }, + ConversionReviewVersions: []string{"v1", "v1beta1"}, + }, } + crd.Spec.PreserveUnknownFields = false + }) + if err != nil { + return } - crd, err := fixtures.CreateNewV1CustomResourceDefinition(crd, apiExtensionClient, f.DynamicClient) - framework.ExpectNoError(err, "creating CustomResourceDefinition") - defer func() { - err = fixtures.DeleteV1CustomResourceDefinition(crd, apiExtensionClient) - framework.ExpectNoError(err, "deleting CustomResourceDefinition") - }() + ginkgo.DeferCleanup(testcrd.CleanUp) + + ginkgo.By("Creating a custom resource conversion webhook") + waitWebhookConversionReady(ctx, f, testcrd.Crd, testcrd.DynamicClients, "v2") + crd := testcrd.Crd ginkgo.By("Watching with field selectors") - crClient, gvr := customResourceClient(crd) - watchSimpleSelector, err := crClient.Namespace(f.Namespace.Name).Watch(ctx, metav1.ListOptions{FieldSelector: "spec.color=blue"}) + v2Client, gvr := customResourceClient(crd, "v2") + hostWatch, err := v2Client.Namespace(f.Namespace.Name).Watch(ctx, metav1.ListOptions{FieldSelector: "host=host1"}) framework.ExpectNoError(err, "watching custom resources with field selector") - defer func() { - watchSimpleSelector.Stop() - }() - watchCompoundSelector, err := crClient.Namespace(f.Namespace.Name).Watch(ctx, metav1.ListOptions{FieldSelector: "spec.color=blue,spec.quantity=2"}) + v2hostPortWatch, err := v2Client.Namespace(f.Namespace.Name).Watch(ctx, metav1.ListOptions{FieldSelector: "host=host1,port=80"}) + framework.ExpectNoError(err, "watching custom resources with field selector") + + v1Client, _ := customResourceClient(crd, "v1") + v1hostPortWatch, err := v1Client.Namespace(f.Namespace.Name).Watch(ctx, metav1.ListOptions{FieldSelector: "hostPort=host1:80"}) framework.ExpectNoError(err, "watching custom resources with field selector") - defer func() { - watchCompoundSelector.Stop() - }() ginkgo.By("Creating custom resources") toCreate := []map[string]any{ { - "color": "blue", - "quantity": int64(2), + "host": "host1", + "port": "80", }, { - "color": "blue", - "quantity": int64(3), + "host": "host1", + "port": "8080", }, { - "color": "green", + "host": "host2", }, } @@ -133,76 +180,91 @@ var _ = SIGDescribe("CustomResourceFieldSelectors [Privileged:ClusterAdmin]", fr for i, spec := range toCreate { name := names.SimpleNameGenerator.GenerateName("selectable-field-cr") crNames[i] = name - _, err = crClient.Namespace(f.Namespace.Name).Create(ctx, &unstructured.Unstructured{Object: map[string]interface{}{ + + obj := map[string]interface{}{ "apiVersion": gvr.Group + "/" + gvr.Version, "kind": crd.Spec.Names.Kind, "metadata": map[string]interface{}{ "name": name, "namespace": f.Namespace.Name, }, - "spec": spec, - }}, metav1.CreateOptions{}) + } + for k, v := range spec { + obj[k] = v + } + _, err = v2Client.Namespace(f.Namespace.Name).Create(ctx, &unstructured.Unstructured{Object: obj}, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating custom resource") } - - ginkgo.By("Listing custom resources with field selector spec.color=blue") - list, err := crClient.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "spec.color=blue"}) + ginkgo.By("Listing v2 custom resources with field selector host=host1") + list, err := v2Client.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "host=host1"}) framework.ExpectNoError(err, "listing custom resources with field selector") gomega.Expect(listResultToNames(list)).To(gomega.Equal(sets.New(crNames[0], crNames[1]))) - ginkgo.By("Listing custom resources with field selector spec.color=blue,spec.quantity=2") - list, err = crClient.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "spec.color=blue,spec.quantity=2"}) + ginkgo.By("Listing v2 custom resources with field selector host=host1,port=80") + list, err = v2Client.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "host=host1,port=80"}) framework.ExpectNoError(err, "listing custom resources with field selector") gomega.Expect(listResultToNames(list)).To(gomega.Equal(sets.New(crNames[0]))) - ginkgo.By("Waiting for watch events to contain custom resources for field selector spec.color=blue") - gomega.Eventually(ctx, watchAccumulator(watchSimpleSelector)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second). + ginkgo.By("Listing v1 custom resources with field selector hostPort=host1:80") + list, err = v1Client.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "hostPort=host1:80"}) + framework.ExpectNoError(err, "listing custom resources with field selector") + gomega.Expect(listResultToNames(list)).To(gomega.Equal(sets.New(crNames[0]))) + + ginkgo.By("Listing v1 custom resources with field selector hostPort=host1:8080") + list, err = v1Client.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "hostPort=host1:8080"}) + framework.ExpectNoError(err, "listing custom resources with field selector") + gomega.Expect(listResultToNames(list)).To(gomega.Equal(sets.New(crNames[1]))) + + ginkgo.By("Waiting for watch events to contain v2 custom resources for field selector host=host1") + gomega.Eventually(ctx, watchAccumulator(hostWatch)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second). Should(gomega.Equal(addedEvents(sets.New(crNames[0], crNames[1])))) - ginkgo.By("Waiting for watch events to contain custom resources for field selector spec.color=blue,spec.quantity=2") - gomega.Eventually(ctx, watchAccumulator(watchCompoundSelector)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second). + ginkgo.By("Waiting for watch events to contain v2 custom resources for field selector host=host1,port=80") + gomega.Eventually(ctx, watchAccumulator(v2hostPortWatch)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second). + Should(gomega.Equal(addedEvents(sets.New(crNames[0])))) + + ginkgo.By("Waiting for watch events to contain v1 custom resources for field selector hostPort=host1:80") + gomega.Eventually(ctx, watchAccumulator(v1hostPortWatch)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second). Should(gomega.Equal(addedEvents(sets.New(crNames[0])))) ginkgo.By("Deleting one custom resources to ensure that deletions are observed") var gracePeriod int64 = 0 - err = crClient.Namespace(f.Namespace.Name).Delete(ctx, crNames[0], metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod}) + err = v2Client.Namespace(f.Namespace.Name).DeleteCollection(ctx, metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod}, metav1.ListOptions{FieldSelector: "host=host1,port=80"}) framework.ExpectNoError(err, "deleting custom resource") ginkgo.By("Updating one custom resources to ensure that deletions are observed") - u, err := crClient.Namespace(f.Namespace.Name).Get(ctx, crNames[1], metav1.GetOptions{}) + u, err := v2Client.Namespace(f.Namespace.Name).Get(ctx, crNames[1], metav1.GetOptions{}) framework.ExpectNoError(err, "getting custom resource") - u.Object["spec"].(map[string]any)["color"] = "green" - _, err = crClient.Namespace(f.Namespace.Name).Update(ctx, u, metav1.UpdateOptions{}) + u.Object["host"] = "host2" + _, err = v2Client.Namespace(f.Namespace.Name).Update(ctx, u, metav1.UpdateOptions{}) framework.ExpectNoError(err, "updating custom resource") - ginkgo.By("Listing custom resources after updates and deletes for field selector spec.color=blue") - list, err = crClient.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "spec.color=blue"}) + ginkgo.By("Listing v2 custom resources after updates and deletes for field selector host=host1") + list, err = v2Client.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "host=host1"}) framework.ExpectNoError(err, "listing custom resources with field selector") gomega.Expect(listResultToNames(list)).To(gomega.Equal(sets.New[string]())) - ginkgo.By("Listing custom resources after updates and deletes for field selector spec.color=blue,spec.quantity=2") - list, err = crClient.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "spec.color=blue,spec.quantity=2"}) + ginkgo.By("Listing v2 custom resources after updates and deletes for field selector host=host1,port=80") + list, err = v2Client.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "host=host1,port=80"}) framework.ExpectNoError(err, "listing custom resources with field selector") gomega.Expect(listResultToNames(list)).To(gomega.Equal(sets.New[string]())) - ginkgo.By("Waiting for watch events after updates and deletes for field selector spec.color=blue") - gomega.Eventually(ctx, watchAccumulator(watchSimpleSelector)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second). + ginkgo.By("Waiting for v2 watch events after updates and deletes for field selector host=host1") + gomega.Eventually(ctx, watchAccumulator(hostWatch)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second). Should(gomega.Equal(deletedEvents(sets.New(crNames[0], crNames[1])))) - ginkgo.By("Waiting for watch events after updates and deletes for field selector spec.color=blue,spec.quantity=2") - gomega.Eventually(ctx, watchAccumulator(watchCompoundSelector)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second). + ginkgo.By("Waiting for v2 watch events after updates and deletes for field selector host=host1,port=80") + gomega.Eventually(ctx, watchAccumulator(v2hostPortWatch)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second). + Should(gomega.Equal(deletedEvents(sets.New(crNames[0])))) + + ginkgo.By("Waiting for v1 watch events after updates and deletes for field selector hostPort=host1:80") + gomega.Eventually(ctx, watchAccumulator(v1hostPortWatch)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second). Should(gomega.Equal(deletedEvents(sets.New(crNames[0])))) }) + }) }) -func unmarshalSchema(schemaJSON []byte) *apiextensionsv1.JSONSchemaProps { - var c apiextensionsv1.JSONSchemaProps - err := json.Unmarshal(schemaJSON, &c) - framework.ExpectNoError(err, "unmarshalling OpenAPIv3 schema") - return &c -} - type accumulatedEvents struct { added, deleted sets.Set[string] }