diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD b/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD index 475230197ee..54bf33f7282 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD @@ -89,6 +89,7 @@ go_library( importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/test/integration", importpath = "k8s.io/apiextensions-apiserver/test/integration", deps = [ + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/BUILD b/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/BUILD index f6f09d2d013..7c451ce7b2d 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/BUILD @@ -32,6 +32,7 @@ go_library( "//staging/src/k8s.io/client-go/restmapper:go_default_library", "//staging/src/k8s.io/client-go/scale:go_default_library", "//vendor/github.com/pborman/uuid:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/resources.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/resources.go index 75dd0fd2b47..3f0c27acb25 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/resources.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/resources.go @@ -20,6 +20,8 @@ import ( "fmt" "time" + "k8s.io/utils/pointer" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" @@ -81,6 +83,36 @@ func NewNoxuCustomResourceDefinition(scope apiextensionsv1beta1.ResourceScope) * } } +// NewNoxuV1CustomResourceDefinition returns a WishIHadChosenNoxu CRD. +func NewNoxuV1CustomResourceDefinition(scope apiextensionsv1.ResourceScope) *apiextensionsv1.CustomResourceDefinition { + return &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"}, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "mygroup.example.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{ + Name: "v1beta1", + Served: true, + Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + XPreserveUnknownFields: pointer.BoolPtr(true), + Type: "object", + }, + }, + }}, + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Plural: "noxus", + Singular: "nonenglishnoxu", + Kind: "WishIHadChosenNoxu", + ShortNames: []string{"foo", "bar", "abc", "def"}, + ListKind: "NoxuItemList", + Categories: []string{"all"}, + }, + Scope: scope, + }, + } +} + // NewVersionedNoxuInstance returns a WishIHadChosenNoxu instance for a given version func NewVersionedNoxuInstance(namespace, name, version string) *unstructured.Unstructured { return &unstructured.Unstructured{ @@ -211,6 +243,19 @@ func servedVersions(crd *apiextensionsv1beta1.CustomResourceDefinition) []string return versions } +func servedV1Versions(crd *apiextensionsv1.CustomResourceDefinition) []string { + if len(crd.Spec.Versions) == 0 { + return []string{} + } + var versions []string + for _, v := range crd.Spec.Versions { + if v.Served { + versions = append(versions, v.Name) + } + } + return versions +} + func existsInDiscovery(crd *apiextensionsv1beta1.CustomResourceDefinition, apiExtensionsClient clientset.Interface, version string) (bool, error) { groupResource, err := apiExtensionsClient.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + version) if err != nil { @@ -227,6 +272,22 @@ func existsInDiscovery(crd *apiextensionsv1beta1.CustomResourceDefinition, apiEx return false, nil } +func existsInDiscoveryV1(crd *apiextensionsv1.CustomResourceDefinition, apiExtensionsClient clientset.Interface, version string) (bool, error) { + groupResource, err := apiExtensionsClient.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + version) + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + for _, g := range groupResource.APIResources { + if g.Name == crd.Spec.Names.Plural { + return true, nil + } + } + return false, nil +} + // CreateNewCustomResourceDefinitionWatchUnsafe creates the CRD and makes sure // the apiextension apiserver has installed the CRD. But it's not safe to watch // the created CR. Please call CreateNewCustomResourceDefinition if you need to @@ -428,6 +489,23 @@ func DeleteCustomResourceDefinition(crd *apiextensionsv1beta1.CustomResourceDefi return nil } +// DeleteV1CustomResourceDefinition deletes a CRD and waits until it disappears from discovery. +func DeleteV1CustomResourceDefinition(crd *apiextensionsv1.CustomResourceDefinition, apiExtensionsClient clientset.Interface) error { + if err := apiExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(crd.Name, nil); err != nil { + return err + } + for _, version := range servedV1Versions(crd) { + err := wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) { + exists, err := existsInDiscoveryV1(crd, apiExtensionsClient, version) + return !exists, err + }) + if err != nil { + return err + } + } + return nil +} + // DeleteCustomResourceDefinitions deletes all CRD matching the provided deleteListOpts and waits until all the CRDs disappear from discovery. func DeleteCustomResourceDefinitions(deleteListOpts metav1.ListOptions, apiExtensionsClient clientset.Interface) error { list, err := apiExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().List(deleteListOpts) @@ -451,6 +529,29 @@ func DeleteCustomResourceDefinitions(deleteListOpts metav1.ListOptions, apiExten return nil } +// DeleteV1CustomResourceDefinitions deletes all CRD matching the provided deleteListOpts and waits until all the CRDs disappear from discovery. +func DeleteV1CustomResourceDefinitions(deleteListOpts metav1.ListOptions, apiExtensionsClient clientset.Interface) error { + list, err := apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().List(deleteListOpts) + if err != nil { + return err + } + if err = apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().DeleteCollection(nil, deleteListOpts); err != nil { + return err + } + for _, crd := range list.Items { + for _, version := range servedV1Versions(&crd) { + err := wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) { + exists, err := existsInDiscoveryV1(&crd, apiExtensionsClient, version) + return !exists, err + }) + if err != nil { + return err + } + } + } + return nil +} + // CreateNewVersionedScaleClient returns a scale client. func CreateNewVersionedScaleClient(crd *apiextensionsv1beta1.CustomResourceDefinition, config *rest.Config, version string) (scale.ScalesGetter, error) { discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/helpers.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/helpers.go index 7a7d6611e86..121f065b616 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/helpers.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/helpers.go @@ -20,6 +20,7 @@ import ( "fmt" "testing" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/errors" @@ -95,6 +96,25 @@ func UpdateCustomResourceDefinitionWithRetry(client clientset.Interface, name st return nil, fmt.Errorf("too many retries after conflicts updating CustomResourceDefinition %q", name) } +// UpdateV1CustomResourceDefinitionWithRetry updates a CRD, retrying up to 5 times on version conflict errors. +func UpdateV1CustomResourceDefinitionWithRetry(client clientset.Interface, name string, update func(*apiextensionsv1.CustomResourceDefinition)) (*apiextensionsv1.CustomResourceDefinition, error) { + for i := 0; i < 5; i++ { + crd, err := client.ApiextensionsV1().CustomResourceDefinitions().Get(name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get CustomResourceDefinition %q: %v", name, err) + } + update(crd) + crd, err = client.ApiextensionsV1().CustomResourceDefinitions().Update(crd) + if err == nil { + return crd, nil + } + if !errors.IsConflict(err) { + return nil, fmt.Errorf("failed to update CustomResourceDefinition %q: %v", name, err) + } + } + return nil, fmt.Errorf("too many retries after conflicts updating CustomResourceDefinition %q", name) +} + // getSchemaForVersion returns the validation schema for given version in given CRD. func getSchemaForVersion(crd *apiextensionsv1beta1.CustomResourceDefinition, version string) (*apiextensionsv1beta1.CustomResourceValidation, error) { if !hasPerVersionSchema(crd.Spec.Versions) { diff --git a/test/e2e/apimachinery/BUILD b/test/e2e/apimachinery/BUILD index e55747fa178..876cfcfc700 100644 --- a/test/e2e/apimachinery/BUILD +++ b/test/e2e/apimachinery/BUILD @@ -41,6 +41,7 @@ go_library( "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/api/scheduling/v1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", diff --git a/test/e2e/apimachinery/crd_conversion_webhook.go b/test/e2e/apimachinery/crd_conversion_webhook.go index c139d4ba4a5..f0446efa7c0 100644 --- a/test/e2e/apimachinery/crd_conversion_webhook.go +++ b/test/e2e/apimachinery/crd_conversion_webhook.go @@ -39,7 +39,7 @@ import ( imageutils "k8s.io/kubernetes/test/utils/image" "k8s.io/utils/pointer" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apiextensions-apiserver/test/integration" // ensure libs have a chance to initialize @@ -56,15 +56,15 @@ const ( var serverCRDConversionWebhookVersion = utilversion.MustParseSemantic("v1.13.0-alpha") -var apiVersions = []v1beta1.CustomResourceDefinitionVersion{ +var apiVersions = []apiextensionsv1.CustomResourceDefinitionVersion{ { Name: "v1", Served: true, Storage: true, - Schema: &v1beta1.CustomResourceValidation{ - OpenAPIV3Schema: &v1beta1.JSONSchemaProps{ + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ Type: "object", - Properties: map[string]v1beta1.JSONSchemaProps{ + Properties: map[string]apiextensionsv1.JSONSchemaProps{ "hostPort": {Type: "string"}, }, }, @@ -74,10 +74,10 @@ var apiVersions = []v1beta1.CustomResourceDefinitionVersion{ Name: "v2", Served: true, Storage: false, - Schema: &v1beta1.CustomResourceValidation{ - OpenAPIV3Schema: &v1beta1.JSONSchemaProps{ + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ Type: "object", - Properties: map[string]v1beta1.JSONSchemaProps{ + Properties: map[string]apiextensionsv1.JSONSchemaProps{ "host": {Type: "string"}, "port": {Type: "string"}, }, @@ -86,15 +86,15 @@ var apiVersions = []v1beta1.CustomResourceDefinitionVersion{ }, } -var alternativeAPIVersions = []v1beta1.CustomResourceDefinitionVersion{ +var alternativeAPIVersions = []apiextensionsv1.CustomResourceDefinitionVersion{ { Name: "v1", Served: true, Storage: false, - Schema: &v1beta1.CustomResourceValidation{ - OpenAPIV3Schema: &v1beta1.JSONSchemaProps{ + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ Type: "object", - Properties: map[string]v1beta1.JSONSchemaProps{ + Properties: map[string]apiextensionsv1.JSONSchemaProps{ "hostPort": {Type: "string"}, }, }, @@ -104,10 +104,10 @@ var alternativeAPIVersions = []v1beta1.CustomResourceDefinitionVersion{ Name: "v2", Served: true, Storage: true, - Schema: &v1beta1.CustomResourceValidation{ - OpenAPIV3Schema: &v1beta1.JSONSchemaProps{ + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ Type: "object", - Properties: map[string]v1beta1.JSONSchemaProps{ + Properties: map[string]apiextensionsv1.JSONSchemaProps{ "host": {Type: "string"}, "port": {Type: "string"}, }, @@ -142,21 +142,24 @@ var _ = SIGDescribe("CustomResourceConversionWebhook", func() { }) ginkgo.It("Should be able to convert from CR v1 to CR v2", func() { - testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *v1beta1.CustomResourceDefinition) { + testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *apiextensionsv1.CustomResourceDefinition) { crd.Spec.Versions = apiVersions - crd.Spec.Conversion = &v1beta1.CustomResourceConversion{ - Strategy: v1beta1.WebhookConverter, - WebhookClientConfig: &v1beta1.WebhookClientConfig{ - CABundle: context.signingCert, - Service: &v1beta1.ServiceReference{ - Namespace: f.Namespace.Name, - Name: serviceCRDName, - Path: pointer.StringPtr("/crdconvert"), - Port: pointer.Int32Ptr(serviceCRDPort), + crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{ + Strategy: apiextensionsv1.WebhookConverter, + Webhook: &apiextensionsv1.WebhookConversion{ + ClientConfig: &apiextensionsv1.WebhookClientConfig{ + CABundle: context.signingCert, + Service: &apiextensionsv1.ServiceReference{ + Namespace: f.Namespace.Name, + Name: serviceCRDName, + Path: pointer.StringPtr("/crdconvert"), + Port: pointer.Int32Ptr(serviceCRDPort), + }, }, + ConversionReviewVersions: []string{"v1", "v1beta1"}, }, } - crd.Spec.PreserveUnknownFields = pointer.BoolPtr(false) + crd.Spec.PreserveUnknownFields = false }) if err != nil { return @@ -166,21 +169,24 @@ var _ = SIGDescribe("CustomResourceConversionWebhook", func() { }) ginkgo.It("Should be able to convert a non homogeneous list of CRs", func() { - testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *v1beta1.CustomResourceDefinition) { + testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *apiextensionsv1.CustomResourceDefinition) { crd.Spec.Versions = apiVersions - crd.Spec.Conversion = &v1beta1.CustomResourceConversion{ - Strategy: v1beta1.WebhookConverter, - WebhookClientConfig: &v1beta1.WebhookClientConfig{ - CABundle: context.signingCert, - Service: &v1beta1.ServiceReference{ - Namespace: f.Namespace.Name, - Name: serviceCRDName, - Path: pointer.StringPtr("/crdconvert"), - Port: pointer.Int32Ptr(serviceCRDPort), + crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{ + Strategy: apiextensionsv1.WebhookConverter, + Webhook: &apiextensionsv1.WebhookConversion{ + ClientConfig: &apiextensionsv1.WebhookClientConfig{ + CABundle: context.signingCert, + Service: &apiextensionsv1.ServiceReference{ + Namespace: f.Namespace.Name, + Name: serviceCRDName, + Path: pointer.StringPtr("/crdconvert"), + Port: pointer.Int32Ptr(serviceCRDPort), + }, }, + ConversionReviewVersions: []string{"v1", "v1beta1"}, }, } - crd.Spec.PreserveUnknownFields = pointer.BoolPtr(false) + crd.Spec.PreserveUnknownFields = false }) if err != nil { return @@ -339,7 +345,7 @@ func deployCustomResourceWebhookAndService(f *framework.Framework, image string, framework.ExpectNoError(err, "waiting for service %s/%s have %d endpoint", namespace, serviceCRDName, 1) } -func verifyV1Object(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, obj *unstructured.Unstructured) { +func verifyV1Object(f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, obj *unstructured.Unstructured) { gomega.Expect(obj.GetAPIVersion()).To(gomega.BeEquivalentTo(crd.Spec.Group + "/v1")) hostPort, exists := obj.Object["hostPort"] gomega.Expect(exists).To(gomega.BeTrue()) @@ -350,7 +356,7 @@ func verifyV1Object(f *framework.Framework, crd *v1beta1.CustomResourceDefinitio gomega.Expect(portExists).To(gomega.BeFalse()) } -func verifyV2Object(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, obj *unstructured.Unstructured) { +func verifyV2Object(f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, obj *unstructured.Unstructured) { gomega.Expect(obj.GetAPIVersion()).To(gomega.BeEquivalentTo(crd.Spec.Group + "/v2")) _, hostPortExists := obj.Object["hostPort"] gomega.Expect(hostPortExists).To(gomega.BeFalse()) @@ -362,7 +368,7 @@ func verifyV2Object(f *framework.Framework, crd *v1beta1.CustomResourceDefinitio gomega.Expect(port).To(gomega.BeEquivalentTo("8080")) } -func testCustomResourceConversionWebhook(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, customResourceClients map[string]dynamic.ResourceInterface) { +func testCustomResourceConversionWebhook(f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClients map[string]dynamic.ResourceInterface) { name := "cr-instance-1" ginkgo.By("Creating a v1 custom resource") crInstance := &unstructured.Unstructured{ @@ -380,6 +386,7 @@ func testCustomResourceConversionWebhook(f *framework.Framework, crd *v1beta1.Cu gomega.Expect(err).To(gomega.BeNil()) ginkgo.By("v2 custom resource should be converted") v2crd, err := customResourceClients["v2"].Get(name, metav1.GetOptions{}) + framework.ExpectNoError(err, "Getting v2 of custom resource %s", name) verifyV2Object(f, crd, v2crd) } @@ -404,7 +411,7 @@ func testCRListConversion(f *framework.Framework, testCrd *crd.TestCrd) { gomega.Expect(err).To(gomega.BeNil()) // Now cr-instance-1 is stored as v1. lets change storage version - crd, err = integration.UpdateCustomResourceDefinitionWithRetry(testCrd.APIExtensionClient, crd.Name, func(c *v1beta1.CustomResourceDefinition) { + crd, err = integration.UpdateV1CustomResourceDefinitionWithRetry(testCrd.APIExtensionClient, crd.Name, func(c *apiextensionsv1.CustomResourceDefinition) { c.Spec.Versions = alternativeAPIVersions }) gomega.Expect(err).To(gomega.BeNil()) diff --git a/test/e2e/apimachinery/crd_publish_openapi.go b/test/e2e/apimachinery/crd_publish_openapi.go index caa28026e62..b285679db54 100644 --- a/test/e2e/apimachinery/crd_publish_openapi.go +++ b/test/e2e/apimachinery/crd_publish_openapi.go @@ -27,9 +27,10 @@ import ( "github.com/go-openapi/spec" "github.com/onsi/ginkgo" + "k8s.io/utils/pointer" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -337,8 +338,11 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() { } ginkgo.By("rename a version") - patch := []byte(`{"spec":{"versions":[{"name":"v2","served":true,"storage":true},{"name":"v4","served":true,"storage":false}]}}`) - crdMultiVer.Crd, err = crdMultiVer.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crdMultiVer.Crd.Name, types.MergePatchType, patch) + patch := []byte(`[ + {"op":"test","path":"/spec/versions/1/name","value":"v3"}, + {"op": "replace", "path": "/spec/versions/1/name", "value": "v4"} + ]`) + crdMultiVer.Crd, err = crdMultiVer.APIExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Patch(crdMultiVer.Crd.Name, types.JSONPatchType, patch) if err != nil { e2elog.Failf("%v", err) } @@ -379,12 +383,12 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() { } ginkgo.By("mark a version not serverd") - crd.Crd, err = crd.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Crd.Name, metav1.GetOptions{}) + crd.Crd, err = crd.APIExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Get(crd.Crd.Name, metav1.GetOptions{}) if err != nil { e2elog.Failf("%v", err) } crd.Crd.Spec.Versions[1].Served = false - crd.Crd, err = crd.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd.Crd) + crd.Crd, err = crd.APIExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Update(crd.Crd) if err != nil { e2elog.Failf("%v", err) } @@ -415,35 +419,42 @@ func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, version } func setupCRDAndVerifySchema(f *framework.Framework, schema, expect []byte, groupSuffix string, versions ...string) (*crd.TestCrd, error) { - group := fmt.Sprintf("%s-test-%s.k8s.io", f.BaseName, groupSuffix) + group := fmt.Sprintf("%s-test-%s.example.com", f.BaseName, groupSuffix) if len(versions) == 0 { return nil, fmt.Errorf("require at least one version for CRD") } - props := &v1beta1.JSONSchemaProps{} + props := &apiextensionsv1.JSONSchemaProps{} if schema != nil { if err := yaml.Unmarshal(schema, props); err != nil { return nil, err } } - crd, err := crd.CreateMultiVersionTestCRD(f, group, func(crd *v1beta1.CustomResourceDefinition) { - var apiVersions []v1beta1.CustomResourceDefinitionVersion + crd, err := crd.CreateMultiVersionTestCRD(f, group, func(crd *apiextensionsv1.CustomResourceDefinition) { + var apiVersions []apiextensionsv1.CustomResourceDefinitionVersion for i, version := range versions { - apiVersions = append(apiVersions, v1beta1.CustomResourceDefinitionVersion{ + version := apiextensionsv1.CustomResourceDefinitionVersion{ Name: version, Served: true, Storage: i == 0, - }) + } + // set up validation when input schema isn't nil + if schema != nil { + version.Schema = &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: props, + } + } else { + version.Schema = &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + XPreserveUnknownFields: pointer.BoolPtr(true), + Type: "object", + }, + } + } + apiVersions = append(apiVersions, version) } crd.Spec.Versions = apiVersions - - // set up validation when input schema isn't nil - if schema != nil { - crd.Spec.Validation = &v1beta1.CustomResourceValidation{ - OpenAPIV3Schema: props, - } - } }) if err != nil { return nil, fmt.Errorf("failed to create CRD: %v", err) @@ -573,12 +584,12 @@ func waitForOpenAPISchema(c k8sclientset.Interface, pred func(*spec.Swagger) (bo // convertJSONSchemaProps converts JSONSchemaProps in YAML to spec.Schema func convertJSONSchemaProps(in []byte, out *spec.Schema) error { - external := v1beta1.JSONSchemaProps{} + external := apiextensionsv1.JSONSchemaProps{} if err := yaml.UnmarshalStrict(in, &external); err != nil { return err } internal := apiextensions.JSONSchemaProps{} - if err := v1beta1.Convert_v1beta1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(&external, &internal, nil); err != nil { + if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(&external, &internal, nil); err != nil { return err } if err := validation.ConvertJSONSchemaProps(&internal, out); err != nil { diff --git a/test/e2e/apimachinery/crd_watch.go b/test/e2e/apimachinery/crd_watch.go index d4a0264ec67..5162dd82553 100644 --- a/test/e2e/apimachinery/crd_watch.go +++ b/test/e2e/apimachinery/crd_watch.go @@ -19,7 +19,7 @@ package apimachinery import ( "fmt" - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apiextensions-apiserver/test/integration/fixtures" "k8s.io/apimachinery/pkg/api/meta" @@ -63,21 +63,22 @@ var _ = SIGDescribe("CustomResourceDefinition Watch", func() { e2elog.Failf("failed to initialize apiExtensionClient: %v", err) } - noxuDefinition := fixtures.NewNoxuCustomResourceDefinition(apiextensionsv1beta1.ClusterScoped) - noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, f.DynamicClient) + noxuDefinition := fixtures.NewNoxuV1CustomResourceDefinition(apiextensionsv1.ClusterScoped) + noxuDefinition, err = fixtures.CreateNewV1CustomResourceDefinition(noxuDefinition, apiExtensionClient, f.DynamicClient) if err != nil { e2elog.Failf("failed to create CustomResourceDefinition: %v", err) } defer func() { - err = fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient) + err = fixtures.DeleteV1CustomResourceDefinition(noxuDefinition, apiExtensionClient) if err != nil { e2elog.Failf("failed to delete CustomResourceDefinition: %v", err) } }() ns := "" - noxuResourceClient := newNamespacedCustomResourceClient(ns, f.DynamicClient, noxuDefinition) + noxuResourceClient, err := newNamespacedCustomResourceClient(ns, f.DynamicClient, noxuDefinition) + framework.ExpectNoError(err, "creating custom resource client") watchA, err := watchCRWithName(noxuResourceClient, watchCRNameA) framework.ExpectNoError(err, "failed to watch custom resource: %s", watchCRNameA) @@ -124,7 +125,7 @@ func watchCRWithName(crdResourceClient dynamic.ResourceInterface, name string) ( ) } -func instantiateCustomResource(instanceToCreate *unstructured.Unstructured, client dynamic.ResourceInterface, definition *apiextensionsv1beta1.CustomResourceDefinition) (*unstructured.Unstructured, error) { +func instantiateCustomResource(instanceToCreate *unstructured.Unstructured, client dynamic.ResourceInterface, definition *apiextensionsv1.CustomResourceDefinition) (*unstructured.Unstructured, error) { createdInstance, err := client.Create(instanceToCreate, metav1.CreateOptions{}) if err != nil { return nil, err @@ -141,7 +142,10 @@ func instantiateCustomResource(instanceToCreate *unstructured.Unstructured, clie if err != nil { return nil, err } - if e, a := definition.Spec.Group+"/"+definition.Spec.Version, createdTypeMeta.GetAPIVersion(); e != a { + if len(definition.Spec.Versions) != 1 { + return nil, fmt.Errorf("expected exactly one version, got %v", definition.Spec.Versions) + } + if e, a := definition.Spec.Group+"/"+definition.Spec.Versions[0].Name, createdTypeMeta.GetAPIVersion(); e != a { return nil, fmt.Errorf("expected %v, got %v", e, a) } if e, a := definition.Spec.Names.Kind, createdTypeMeta.GetKind(); e != a { @@ -154,12 +158,15 @@ func deleteCustomResource(client dynamic.ResourceInterface, name string) error { return client.Delete(name, &metav1.DeleteOptions{}) } -func newNamespacedCustomResourceClient(ns string, client dynamic.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) dynamic.ResourceInterface { - gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: crd.Spec.Version, Resource: crd.Spec.Names.Plural} - - if crd.Spec.Scope != apiextensionsv1beta1.ClusterScoped { - return client.Resource(gvr).Namespace(ns) +func newNamespacedCustomResourceClient(ns string, client dynamic.Interface, crd *apiextensionsv1.CustomResourceDefinition) (dynamic.ResourceInterface, error) { + if len(crd.Spec.Versions) != 1 { + return nil, fmt.Errorf("expected exactly one version, got %v", crd.Spec.Versions) } - return client.Resource(gvr) + gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: crd.Spec.Versions[0].Name, Resource: crd.Spec.Names.Plural} + + if crd.Spec.Scope != apiextensionsv1.ClusterScoped { + return client.Resource(gvr).Namespace(ns), nil + } + return client.Resource(gvr), nil } diff --git a/test/e2e/apimachinery/webhook.go b/test/e2e/apimachinery/webhook.go index 69e2bc666d8..daaca21c9f3 100644 --- a/test/e2e/apimachinery/webhook.go +++ b/test/e2e/apimachinery/webhook.go @@ -28,7 +28,7 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" crdclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -195,22 +195,24 @@ var _ = SIGDescribe("AdmissionWebhook", func() { ginkgo.It("Should mutate custom resource with pruning", func() { const prune = true - testcrd, err := createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f, func(crd *apiextensionsv1beta1.CustomResourceDefinition) { - crd.Spec.PreserveUnknownFields = pointer.BoolPtr(false) - crd.Spec.Validation = &apiextensionsv1beta1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ - Type: "object", - Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ - "data": { - Type: "object", - Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ - "mutation-start": {Type: "string"}, - "mutation-stage-1": {Type: "string"}, - // mutation-stage-2 is intentionally missing such that it is pruned + testcrd, err := createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f, func(crd *apiextensionsv1.CustomResourceDefinition) { + crd.Spec.PreserveUnknownFields = false + for i := range crd.Spec.Versions { + crd.Spec.Versions[i].Schema = &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "data": { + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "mutation-start": {Type: "string"}, + "mutation-stage-1": {Type: "string"}, + // mutation-stage-2 is intentionally missing such that it is pruned + }, }, }, }, - }, + } } }) if err != nil { @@ -733,7 +735,7 @@ func registerWebhookForAttachingPod(f *framework.Framework, configName string, c CABundle: context.signingCert, }, SideEffects: &sideEffectsNone, - AdmissionReviewVersions: []string{"v1beta1"}, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, // Scope the webhook to just this namespace NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{f.UniqueName: "true"}, @@ -821,7 +823,7 @@ func registerMutatingWebhookForPod(f *framework.Framework, configName string, co CABundle: context.signingCert, }, SideEffects: &sideEffectsNone, - AdmissionReviewVersions: []string{"v1beta1"}, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, // Scope the webhook to just this namespace NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{f.UniqueName: "true"}, @@ -1042,7 +1044,7 @@ func failingWebhook(namespace, name string) admissionregistrationv1.ValidatingWe CABundle: nil, }, SideEffects: &sideEffectsNone, - AdmissionReviewVersions: []string{"v1beta1"}, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, } } @@ -1150,7 +1152,7 @@ func registerValidatingWebhookForWebhookConfigurations(f *framework.Framework, c CABundle: context.signingCert, }, SideEffects: &sideEffectsNone, - AdmissionReviewVersions: []string{"v1beta1"}, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, FailurePolicy: &failurePolicy, // Scope the webhook to just this namespace NamespaceSelector: &metav1.LabelSelector{ @@ -1209,7 +1211,7 @@ func registerMutatingWebhookForWebhookConfigurations(f *framework.Framework, con CABundle: context.signingCert, }, SideEffects: &sideEffectsNone, - AdmissionReviewVersions: []string{"v1beta1"}, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, FailurePolicy: &failurePolicy, // Scope the webhook to just this namespace NamespaceSelector: &metav1.LabelSelector{ @@ -1270,7 +1272,7 @@ func testWebhooksForWebhookConfigurations(f *framework.Framework, configName str CABundle: nil, }, SideEffects: &sideEffectsNone, - AdmissionReviewVersions: []string{"v1beta1"}, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, FailurePolicy: &failurePolicy, // Scope the webhook to just this namespace NamespaceSelector: &metav1.LabelSelector{ @@ -1324,7 +1326,7 @@ func testWebhooksForWebhookConfigurations(f *framework.Framework, configName str CABundle: nil, }, SideEffects: &sideEffectsNone, - AdmissionReviewVersions: []string{"v1beta1"}, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, FailurePolicy: &failurePolicy, // Scope the webhook to just this namespace NamespaceSelector: &metav1.LabelSelector{ @@ -1542,7 +1544,7 @@ func registerWebhookForCustomResource(f *framework.Framework, configName string, CABundle: context.signingCert, }, SideEffects: &sideEffectsNone, - AdmissionReviewVersions: []string{"v1beta1"}, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, // Scope the webhook to just this namespace NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{f.UniqueName: "true"}, @@ -1591,7 +1593,7 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, configName CABundle: context.signingCert, }, SideEffects: &sideEffectsNone, - AdmissionReviewVersions: []string{"v1beta1"}, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, // Scope the webhook to just this namespace NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{f.UniqueName: "true"}, @@ -1617,7 +1619,7 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, configName CABundle: context.signingCert, }, SideEffects: &sideEffectsNone, - AdmissionReviewVersions: []string{"v1beta1"}, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, // Scope the webhook to just this namespace NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{f.UniqueName: "true"}, @@ -1633,13 +1635,13 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, configName return func() { client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(configName, nil) } } -func testCustomResourceWebhook(f *framework.Framework, crd *apiextensionsv1beta1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface) { +func testCustomResourceWebhook(f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface) { ginkgo.By("Creating a custom resource that should be denied by the webhook") crInstanceName := "cr-instance-1" crInstance := &unstructured.Unstructured{ Object: map[string]interface{}{ "kind": crd.Spec.Names.Kind, - "apiVersion": crd.Spec.Group + "/" + crd.Spec.Version, + "apiVersion": crd.Spec.Group + "/" + crd.Spec.Versions[0].Name, "metadata": map[string]interface{}{ "name": crInstanceName, "namespace": f.Namespace.Name, @@ -1657,13 +1659,13 @@ func testCustomResourceWebhook(f *framework.Framework, crd *apiextensionsv1beta1 } } -func testBlockingCustomResourceDeletion(f *framework.Framework, crd *apiextensionsv1beta1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface) { +func testBlockingCustomResourceDeletion(f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface) { ginkgo.By("Creating a custom resource whose deletion would be denied by the webhook") crInstanceName := "cr-instance-2" crInstance := &unstructured.Unstructured{ Object: map[string]interface{}{ "kind": crd.Spec.Names.Kind, - "apiVersion": crd.Spec.Group + "/" + crd.Spec.Version, + "apiVersion": crd.Spec.Group + "/" + crd.Spec.Versions[0].Name, "metadata": map[string]interface{}{ "name": crInstanceName, "namespace": f.Namespace.Name, @@ -1701,13 +1703,13 @@ func testBlockingCustomResourceDeletion(f *framework.Framework, crd *apiextensio } -func testMutatingCustomResourceWebhook(f *framework.Framework, crd *apiextensionsv1beta1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface, prune bool) { +func testMutatingCustomResourceWebhook(f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface, prune bool) { ginkgo.By("Creating a custom resource that should be mutated by the webhook") crName := "cr-instance-1" cr := &unstructured.Unstructured{ Object: map[string]interface{}{ "kind": crd.Spec.Names.Kind, - "apiVersion": crd.Spec.Group + "/" + crd.Spec.Version, + "apiVersion": crd.Spec.Group + "/" + crd.Spec.Versions[0].Name, "metadata": map[string]interface{}{ "name": crName, "namespace": f.Namespace.Name, @@ -1738,7 +1740,7 @@ func testMultiVersionCustomResourceWebhook(f *framework.Framework, testcrd *crd. cr := &unstructured.Unstructured{ Object: map[string]interface{}{ "kind": testcrd.Crd.Spec.Names.Kind, - "apiVersion": testcrd.Crd.Spec.Group + "/" + testcrd.Crd.Spec.Version, + "apiVersion": testcrd.Crd.Spec.Group + "/" + testcrd.Crd.Spec.Versions[0].Name, "metadata": map[string]interface{}{ "name": crName, "namespace": f.Namespace.Name, @@ -1752,8 +1754,29 @@ func testMultiVersionCustomResourceWebhook(f *framework.Framework, testcrd *crd. framework.ExpectNoError(err, "failed to create custom resource %s in namespace: %s", crName, f.Namespace.Name) ginkgo.By("Patching Custom Resource Definition to set v2 as storage") - apiVersionWithV2StoragePatch := fmt.Sprint(`{"spec": {"versions": [{"name": "v1", "storage": false, "served": true},{"name": "v2", "storage": true, "served": true}]}}`) - _, err = testcrd.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(testcrd.Crd.Name, types.StrategicMergePatchType, []byte(apiVersionWithV2StoragePatch)) + apiVersionWithV2StoragePatch := `{ + "spec": { + "versions": [ + { + "name": "v1", + "storage": false, + "served": true, + "schema": { + "openAPIV3Schema": {"x-kubernetes-preserve-unknown-fields": true, "type": "object"} + } + }, + { + "name": "v2", + "storage": true, + "served": true, + "schema": { + "openAPIV3Schema": {"x-kubernetes-preserve-unknown-fields": true, "type": "object"} + } + } + ] + } + }` + _, err = testcrd.APIExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Patch(testcrd.Crd.Name, types.StrategicMergePatchType, []byte(apiVersionWithV2StoragePatch)) framework.ExpectNoError(err, "failed to patch custom resource definition %s in namespace: %s", testcrd.Crd.Name, f.Namespace.Name) ginkgo.By("Patching the custom resource while v2 is storage version") @@ -1798,7 +1821,7 @@ func registerValidatingWebhookForCRD(f *framework.Framework, configName string, CABundle: context.signingCert, }, SideEffects: &sideEffectsNone, - AdmissionReviewVersions: []string{"v1beta1"}, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, // Scope the webhook to just this namespace NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{f.UniqueName: "true"}, @@ -1819,12 +1842,18 @@ func testCRDDenyWebhook(f *framework.Framework) { ginkgo.By("Creating a custom resource definition that should be denied by the webhook") name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, "deny") kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, "deny") - group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName) - apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + group := fmt.Sprintf("%s.example.com", f.BaseName) + apiVersions := []apiextensionsv1.CustomResourceDefinitionVersion{ { Name: "v1", Served: true, Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + XPreserveUnknownFields: pointer.BoolPtr(true), + Type: "object", + }, + }, }, } @@ -1839,28 +1868,28 @@ func testCRDDenyWebhook(f *framework.Framework) { e2elog.Failf("failed to initialize apiExtensionClient: %v", err) return } - crd := &apiextensionsv1beta1.CustomResourceDefinition{ + crd := &apiextensionsv1.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{ Name: name + "s." + group, Labels: map[string]string{ "webhook-e2e-test": "webhook-disallow", }, }, - Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ Group: group, Versions: apiVersions, - Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Names: apiextensionsv1.CustomResourceDefinitionNames{ Singular: name, Kind: kind, ListKind: kind + "List", Plural: name + "s", }, - Scope: apiextensionsv1beta1.NamespaceScoped, + Scope: apiextensionsv1.NamespaceScoped, }, } // create CRD - _, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd) + _, err = apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Create(crd) framework.ExpectError(err, "create custom resource definition %s should be denied by webhook", crd.Name) expectedErrMsg := "the crd contains unwanted label" if !strings.Contains(err.Error(), expectedErrMsg) { @@ -1920,7 +1949,7 @@ func registerSlowWebhook(f *framework.Framework, configName string, context *cer FailurePolicy: policy, TimeoutSeconds: timeout, SideEffects: &sideEffectsNone, - AdmissionReviewVersions: []string{"v1beta1"}, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, }, }, }) @@ -1958,25 +1987,37 @@ func testSlowWebhookTimeoutNoError(f *framework.Framework) { // createAdmissionWebhookMultiVersionTestCRDWithV1Storage creates a new CRD specifically // for the admissin webhook calling test. func createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f *framework.Framework, opts ...crd.Option) (*crd.TestCrd, error) { - group := fmt.Sprintf("%s-multiversion-crd-test.k8s.io", f.BaseName) - return crd.CreateMultiVersionTestCRD(f, group, append([]crd.Option{func(crd *apiextensionsv1beta1.CustomResourceDefinition) { - crd.Spec.Versions = []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + group := fmt.Sprintf("%s.example.com", f.BaseName) + return crd.CreateMultiVersionTestCRD(f, group, append([]crd.Option{func(crd *apiextensionsv1.CustomResourceDefinition) { + crd.Spec.Versions = []apiextensionsv1.CustomResourceDefinitionVersion{ { Name: "v1", Served: true, Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + XPreserveUnknownFields: pointer.BoolPtr(true), + Type: "object", + }, + }, }, { Name: "v2", Served: true, Storage: false, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + XPreserveUnknownFields: pointer.BoolPtr(true), + Type: "object", + }, + }, }, } }}, opts...)...) } // servedAPIVersions returns the API versions served by the CRD. -func servedAPIVersions(crd *apiextensionsv1beta1.CustomResourceDefinition) []string { +func servedAPIVersions(crd *apiextensionsv1.CustomResourceDefinition) []string { ret := []string{} for _, v := range crd.Spec.Versions { if v.Served { @@ -2038,7 +2079,7 @@ func newDenyPodWebhookFixture(f *framework.Framework, context *certContext) admi CABundle: context.signingCert, }, SideEffects: &sideEffectsNone, - AdmissionReviewVersions: []string{"v1beta1"}, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, // Scope the webhook to just this namespace NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{f.UniqueName: "true"}, @@ -2079,7 +2120,7 @@ func newDenyConfigMapWebhookFixture(f *framework.Framework, context *certContext CABundle: context.signingCert, }, SideEffects: &sideEffectsNone, - AdmissionReviewVersions: []string{"v1beta1"}, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, } } @@ -2105,7 +2146,7 @@ func newMutateConfigMapWebhookFixture(f *framework.Framework, context *certConte CABundle: context.signingCert, }, SideEffects: &sideEffectsNone, - AdmissionReviewVersions: []string{"v1beta1"}, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, // Scope the webhook to just this namespace NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{f.UniqueName: "true"}, diff --git a/test/e2e/kubectl/BUILD b/test/e2e/kubectl/BUILD index fce3c520b4e..bb2024d4e0b 100644 --- a/test/e2e/kubectl/BUILD +++ b/test/e2e/kubectl/BUILD @@ -17,7 +17,7 @@ go_library( "//pkg/controller:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -49,6 +49,7 @@ go_library( "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/golang.org/x/net/websocket:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/test/e2e/kubectl/kubectl.go b/test/e2e/kubectl/kubectl.go index dd2a0030bee..a37fcdb6355 100644 --- a/test/e2e/kubectl/kubectl.go +++ b/test/e2e/kubectl/kubectl.go @@ -41,10 +41,13 @@ import ( "github.com/elazarl/goproxy" openapi_v2 "github.com/googleapis/gnostic/OpenAPIv2" + uexec "k8s.io/utils/exec" + "k8s.io/utils/pointer" + "sigs.k8s.io/yaml" v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -69,8 +72,6 @@ import ( "k8s.io/kubernetes/test/e2e/scheduling" testutils "k8s.io/kubernetes/test/utils" "k8s.io/kubernetes/test/utils/crd" - uexec "k8s.io/utils/exec" - "sigs.k8s.io/yaml" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -927,12 +928,14 @@ metadata: ginkgo.It("should create/apply a valid CR for CRD with validation schema", func() { ginkgo.By("prepare CRD with validation schema") - crd, err := crd.CreateTestCRD(f, func(crd *v1beta1.CustomResourceDefinition) { - props := &v1beta1.JSONSchemaProps{} + crd, err := crd.CreateTestCRD(f, func(crd *apiextensionsv1.CustomResourceDefinition) { + props := &apiextensionsv1.JSONSchemaProps{} if err := yaml.Unmarshal(schemaFoo, props); err != nil { e2elog.Failf("failed to unmarshal schema: %v", err) } - crd.Spec.Validation = &v1beta1.CustomResourceValidation{OpenAPIV3Schema: props} + for i := range crd.Spec.Versions { + crd.Spec.Versions[i].Schema = &apiextensionsv1.CustomResourceValidation{OpenAPIV3Schema: props} + } }) if err != nil { e2elog.Failf("failed to create test CRD: %v", err) @@ -951,12 +954,16 @@ metadata: ginkgo.It("should create/apply a valid CR with arbitrary-extra properties for CRD with partially-specified validation schema", func() { ginkgo.By("prepare CRD with partially-specified validation schema") - crd, err := crd.CreateTestCRD(f, func(crd *v1beta1.CustomResourceDefinition) { - props := &v1beta1.JSONSchemaProps{} + crd, err := crd.CreateTestCRD(f, func(crd *apiextensionsv1.CustomResourceDefinition) { + props := &apiextensionsv1.JSONSchemaProps{} if err := yaml.Unmarshal(schemaFoo, props); err != nil { e2elog.Failf("failed to unmarshal schema: %v", err) } - crd.Spec.Validation = &v1beta1.CustomResourceValidation{OpenAPIV3Schema: props} + // Allow for arbitrary-extra properties. + props.XPreserveUnknownFields = pointer.BoolPtr(true) + for i := range crd.Spec.Versions { + crd.Spec.Versions[i].Schema = &apiextensionsv1.CustomResourceValidation{OpenAPIV3Schema: props} + } }) if err != nil { e2elog.Failf("failed to create test CRD: %v", err) @@ -966,31 +973,14 @@ metadata: ginkgo.By("sleep for 10s to wait for potential crd openapi publishing alpha feature") time.Sleep(10 * time.Second) - publishedSchema := schemaForGVK(schema.GroupVersionKind{Group: crd.Crd.Spec.Group, Version: crd.Crd.Spec.Version, Kind: crd.Crd.Spec.Names.Kind}) - expectSuccess := false - if publishedSchema == nil || publishedSchema.Properties == nil || publishedSchema.Properties.AdditionalProperties == nil || len(publishedSchema.Properties.AdditionalProperties) == 0 { - // expect success in the following cases: - // - no schema was published - // - a schema was published with no properties - expectSuccess = true - e2elog.Logf("no schema with properties found, expect apply with extra properties to succeed") - } else { - e2elog.Logf("schema with properties found, expect apply with extra properties to fail") - } + schema := schemaForGVK(schema.GroupVersionKind{Group: crd.Crd.Spec.Group, Version: crd.Crd.Spec.Versions[0].Name, Kind: crd.Crd.Spec.Names.Kind}) + framework.ExpectNotEqual(schema, nil, "retrieving a schema for the crd") meta := fmt.Sprintf(metaPattern, crd.Crd.Spec.Names.Kind, crd.Crd.Spec.Group, crd.Crd.Spec.Versions[0].Name, "test-cr") validArbitraryCR := fmt.Sprintf(`{%s,"spec":{"bars":[{"name":"test-bar"}],"extraProperty":"arbitrary-value"}}`, meta) - if err := createApplyCustomResource(validArbitraryCR, f.Namespace.Name, "test-cr", crd); err != nil { - if expectSuccess { - e2elog.Failf("%v", err) - } - } else { - if !expectSuccess { - e2elog.Failf("expected error, got none") - } - } + err = createApplyCustomResource(validArbitraryCR, f.Namespace.Name, "test-cr", crd) + framework.ExpectNoError(err, "creating custom resource") }) - }) framework.KubeDescribe("Kubectl cluster-info", func() { diff --git a/test/utils/crd/BUILD b/test/utils/crd/BUILD index e74b54755ba..db657884fca 100644 --- a/test/utils/crd/BUILD +++ b/test/utils/crd/BUILD @@ -6,7 +6,7 @@ go_library( importpath = "k8s.io/kubernetes/test/utils/crd", visibility = ["//visibility:public"], deps = [ - "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -14,6 +14,7 @@ go_library( "//staging/src/k8s.io/client-go/dynamic:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/log:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", ], ) diff --git a/test/utils/crd/crd_util.go b/test/utils/crd/crd_util.go index 2137eb2d87c..768ea4b826c 100644 --- a/test/utils/crd/crd_util.go +++ b/test/utils/crd/crd_util.go @@ -19,7 +19,9 @@ package crd import ( "fmt" - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/utils/pointer" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" crdclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apiextensions-apiserver/test/integration/fixtures" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -35,13 +37,13 @@ type CleanCrdFn func() error // TestCrd holds all the pieces needed to test with the CRD type TestCrd struct { APIExtensionClient *crdclientset.Clientset - Crd *apiextensionsv1beta1.CustomResourceDefinition + Crd *apiextensionsv1.CustomResourceDefinition DynamicClients map[string]dynamic.ResourceInterface CleanUp CleanCrdFn } // Option is a modifier for a CRD object used to customize CreateMultiVersionTestCRD and CreateTestCRD. -type Option func(crd *apiextensionsv1beta1.CustomResourceDefinition) +type Option func(crd *apiextensionsv1.CustomResourceDefinition) // CreateMultiVersionTestCRD creates a new CRD specifically for the calling test. func CreateMultiVersionTestCRD(f *framework.Framework, group string, opts ...Option) (*TestCrd, error) { @@ -67,28 +69,38 @@ func CreateMultiVersionTestCRD(f *framework.Framework, group string, opts ...Opt return nil, err } - crd := &apiextensionsv1beta1.CustomResourceDefinition{ + crd := &apiextensionsv1.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: name + "s." + group}, - Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ Group: group, - Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: name + "s", Singular: name, Kind: kind, ListKind: kind + "List", }, - Scope: apiextensionsv1beta1.NamespaceScoped, + Scope: apiextensionsv1.NamespaceScoped, }, } for _, opt := range opts { opt(crd) } - if len(crd.Spec.Versions) == 0 && len(crd.Spec.Version) == 0 { - crd.Spec.Version = "v1" + if len(crd.Spec.Versions) == 0 { + crd.Spec.Versions = []apiextensionsv1.CustomResourceDefinitionVersion{{ + Served: true, + Storage: true, + Name: "v1", + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + XPreserveUnknownFields: pointer.BoolPtr(true), + Type: "object", + }, + }, + }} } //create CRD and waits for the resource to be recognized and available. - crd, err = fixtures.CreateNewCustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient) + crd, err = fixtures.CreateNewV1CustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient) if err != nil { e2elog.Failf("failed to create CustomResourceDefinition: %v", err) return nil, err @@ -106,7 +118,7 @@ func CreateMultiVersionTestCRD(f *framework.Framework, group string, opts ...Opt testcrd.Crd = crd testcrd.DynamicClients = resourceClients testcrd.CleanUp = func() error { - err := fixtures.DeleteCustomResourceDefinition(crd, apiExtensionClient) + err := fixtures.DeleteV1CustomResourceDefinition(crd, apiExtensionClient) if err != nil { e2elog.Failf("failed to delete CustomResourceDefinition(%s): %v", name, err) } @@ -117,13 +129,19 @@ func CreateMultiVersionTestCRD(f *framework.Framework, group string, opts ...Opt // CreateTestCRD creates a new CRD specifically for the calling test. func CreateTestCRD(f *framework.Framework, opts ...Option) (*TestCrd, error) { - group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName) - return CreateMultiVersionTestCRD(f, group, append([]Option{func(crd *apiextensionsv1beta1.CustomResourceDefinition) { - crd.Spec.Versions = []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + group := fmt.Sprintf("%s.example.com", f.BaseName) + return CreateMultiVersionTestCRD(f, group, append([]Option{func(crd *apiextensionsv1.CustomResourceDefinition) { + crd.Spec.Versions = []apiextensionsv1.CustomResourceDefinitionVersion{ { Name: "v1", Served: true, Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + XPreserveUnknownFields: pointer.BoolPtr(true), + Type: "object", + }, + }, }, } }}, opts...)...) diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index 3821dc96f65..b149f54a939 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -204,7 +204,7 @@ const ( func initImageConfigs() map[int]Config { configs := map[int]Config{} - configs[Agnhost] = Config{e2eRegistry, "agnhost", "2.4"} + configs[Agnhost] = Config{e2eRegistry, "agnhost", "2.5"} configs[Alpine] = Config{dockerLibraryRegistry, "alpine", "3.7"} configs[AuthenticatedAlpine] = Config{gcAuthenticatedRegistry, "alpine", "3.7"} configs[AuthenticatedWindowsNanoServer] = Config{gcAuthenticatedRegistry, "windows-nanoserver", "v1"}