diff --git a/test/e2e/apimachinery/BUILD b/test/e2e/apimachinery/BUILD index d2c037f795d..df91e7781e2 100644 --- a/test/e2e/apimachinery/BUILD +++ b/test/e2e/apimachinery/BUILD @@ -63,7 +63,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library", diff --git a/test/e2e/apimachinery/crd_conversion_webhook.go b/test/e2e/apimachinery/crd_conversion_webhook.go index 3389e84f4c8..3cf73f9fed4 100644 --- a/test/e2e/apimachinery/crd_conversion_webhook.go +++ b/test/e2e/apimachinery/crd_conversion_webhook.go @@ -106,15 +106,20 @@ var _ = SIGDescribe("CustomResourceConversionWebhook [Feature:CustomResourceWebh }) ginkgo.It("Should be able to convert from CR v1 to CR v2", func() { - testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions, - &v1beta1.WebhookClientConfig{ - CABundle: context.signingCert, - Service: &v1beta1.ServiceReference{ - Namespace: f.Namespace.Name, - Name: serviceCRDName, - Path: pointer.StringPtr("/crdconvert"), - Port: pointer.Int32Ptr(serviceCRDPort), - }}) + testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *v1beta1.CustomResourceDefinition) { + crd.Spec.Conversion = &v1beta1.CustomResourceConversion{ + Strategy: v1beta1.WebhookConverter, + WebhookClientConfig: &v1beta1.WebhookClientConfig{ + CABundle: context.signingCert, + Service: &v1beta1.ServiceReference{ + Namespace: f.Namespace.Name, + Name: serviceCRDName, + Path: pointer.StringPtr("/crdconvert"), + Port: pointer.Int32Ptr(serviceCRDPort), + }, + }, + } + }) if err != nil { return } @@ -123,15 +128,20 @@ var _ = SIGDescribe("CustomResourceConversionWebhook [Feature:CustomResourceWebh }) ginkgo.It("Should be able to convert a non homogeneous list of CRs", func() { - testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions, - &v1beta1.WebhookClientConfig{ - CABundle: context.signingCert, - Service: &v1beta1.ServiceReference{ - Namespace: f.Namespace.Name, - Name: serviceCRDName, - Path: pointer.StringPtr("/crdconvert"), - Port: pointer.Int32Ptr(serviceCRDPort), - }}) + testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *v1beta1.CustomResourceDefinition) { + crd.Spec.Conversion = &v1beta1.CustomResourceConversion{ + Strategy: v1beta1.WebhookConverter, + WebhookClientConfig: &v1beta1.WebhookClientConfig{ + CABundle: context.signingCert, + Service: &v1beta1.ServiceReference{ + Namespace: f.Namespace.Name, + Name: serviceCRDName, + Path: pointer.StringPtr("/crdconvert"), + Port: pointer.Int32Ptr(serviceCRDPort), + }, + }, + } + }) if err != nil { return } diff --git a/test/e2e/apimachinery/crd_publish_openapi.go b/test/e2e/apimachinery/crd_publish_openapi.go index b3dc0e39e86..7f3e4e69f2e 100644 --- a/test/e2e/apimachinery/crd_publish_openapi.go +++ b/test/e2e/apimachinery/crd_publish_openapi.go @@ -35,7 +35,6 @@ import ( "k8s.io/apimachinery/pkg/types" utilversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" - utilyaml "k8s.io/apimachinery/pkg/util/yaml" k8sclientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" openapiutil "k8s.io/kube-openapi/pkg/util" @@ -62,7 +61,7 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish framework.Failf("%v", err) } - meta := fmt.Sprintf(metaPattern, crd.Kind, crd.APIGroup, crd.Versions[0].Name, "test-foo") + meta := fmt.Sprintf(metaPattern, crd.Crd.Spec.Names.Kind, crd.Crd.Spec.Group, crd.Crd.Spec.Versions[0].Name, "test-foo") ns := fmt.Sprintf("--namespace=%v", f.Namespace.Name) ginkgo.By("client-side validation (kubectl create and apply) allows request with known and required properties") @@ -70,13 +69,13 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish if _, err := framework.RunKubectlInput(validCR, ns, "create", "-f", "-"); err != nil { framework.Failf("failed to create valid CR %s: %v", validCR, err) } - if _, err := framework.RunKubectl(ns, "delete", crd.GetPluralName(), "test-foo"); err != nil { + if _, err := framework.RunKubectl(ns, "delete", crd.Crd.Spec.Names.Plural, "test-foo"); err != nil { framework.Failf("failed to delete valid CR: %v", err) } if _, err := framework.RunKubectlInput(validCR, ns, "apply", "-f", "-"); err != nil { framework.Failf("failed to apply valid CR %s: %v", validCR, err) } - if _, err := framework.RunKubectl(ns, "delete", crd.GetPluralName(), "test-foo"); err != nil { + if _, err := framework.RunKubectl(ns, "delete", crd.Crd.Spec.Names.Plural, "test-foo"); err != nil { framework.Failf("failed to delete valid CR: %v", err) } @@ -99,23 +98,23 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish } ginkgo.By("kubectl explain works to explain CR properties") - if err := verifyKubectlExplain(crd.GetPluralName(), `(?s)DESCRIPTION:.*Foo CRD for Testing.*FIELDS:.*apiVersion.*.*APIVersion defines.*spec.*.*Specification of Foo`); err != nil { + if err := verifyKubectlExplain(crd.Crd.Spec.Names.Plural, `(?s)DESCRIPTION:.*Foo CRD for Testing.*FIELDS:.*apiVersion.*.*APIVersion defines.*spec.*.*Specification of Foo`); err != nil { framework.Failf("%v", err) } ginkgo.By("kubectl explain works to explain CR properties recursively") - if err := verifyKubectlExplain(crd.GetPluralName()+".metadata", `(?s)DESCRIPTION:.*Standard object's metadata.*FIELDS:.*creationTimestamp.*.*CreationTimestamp is a timestamp`); err != nil { + if err := verifyKubectlExplain(crd.Crd.Spec.Names.Plural+".metadata", `(?s)DESCRIPTION:.*Standard object's metadata.*FIELDS:.*creationTimestamp.*.*CreationTimestamp is a timestamp`); err != nil { framework.Failf("%v", err) } - if err := verifyKubectlExplain(crd.GetPluralName()+".spec", `(?s)DESCRIPTION:.*Specification of Foo.*FIELDS:.*bars.*<\[\]Object>.*List of Bars and their specs`); err != nil { + if err := verifyKubectlExplain(crd.Crd.Spec.Names.Plural+".spec", `(?s)DESCRIPTION:.*Specification of Foo.*FIELDS:.*bars.*<\[\]Object>.*List of Bars and their specs`); err != nil { framework.Failf("%v", err) } - if err := verifyKubectlExplain(crd.GetPluralName()+".spec.bars", `(?s)RESOURCE:.*bars.*<\[\]Object>.*DESCRIPTION:.*List of Bars and their specs.*FIELDS:.*bazs.*<\[\]string>.*List of Bazs.*name.*.*Name of Bar`); err != nil { + if err := verifyKubectlExplain(crd.Crd.Spec.Names.Plural+".spec.bars", `(?s)RESOURCE:.*bars.*<\[\]Object>.*DESCRIPTION:.*List of Bars and their specs.*FIELDS:.*bazs.*<\[\]string>.*List of Bazs.*name.*.*Name of Bar`); err != nil { framework.Failf("%v", err) } ginkgo.By("kubectl explain works to return error when explain is called on property that doesn't exist") - if _, err := framework.RunKubectl("explain", crd.GetPluralName()+".spec.bars2"); err == nil || !strings.Contains(err.Error(), `field "bars2" does not exist`) { + if _, err := framework.RunKubectl("explain", crd.Crd.Spec.Names.Plural+".spec.bars2"); err == nil || !strings.Contains(err.Error(), `field "bars2" does not exist`) { framework.Failf("unexpected no error when explaining property that doesn't exist: %v", err) } @@ -130,7 +129,7 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish framework.Failf("%v", err) } - meta := fmt.Sprintf(metaPattern, crd.Kind, crd.APIGroup, crd.Versions[0].Name, "test-cr") + meta := fmt.Sprintf(metaPattern, crd.Crd.Spec.Names.Kind, crd.Crd.Spec.Group, crd.Crd.Spec.Versions[0].Name, "test-cr") ns := fmt.Sprintf("--namespace=%v", f.Namespace.Name) ginkgo.By("client-side validation (kubectl create and apply) allows request with any unknown properties") @@ -138,18 +137,18 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish if _, err := framework.RunKubectlInput(randomCR, ns, "create", "-f", "-"); err != nil { framework.Failf("failed to create random CR %s for CRD without schema: %v", randomCR, err) } - if _, err := framework.RunKubectl(ns, "delete", crd.GetPluralName(), "test-cr"); err != nil { + if _, err := framework.RunKubectl(ns, "delete", crd.Crd.Spec.Names.Plural, "test-cr"); err != nil { framework.Failf("failed to delete random CR: %v", err) } if _, err := framework.RunKubectlInput(randomCR, ns, "apply", "-f", "-"); err != nil { framework.Failf("failed to apply random CR %s for CRD without schema: %v", randomCR, err) } - if _, err := framework.RunKubectl(ns, "delete", crd.GetPluralName(), "test-cr"); err != nil { + if _, err := framework.RunKubectl(ns, "delete", crd.Crd.Spec.Names.Plural, "test-cr"); err != nil { framework.Failf("failed to delete random CR: %v", err) } ginkgo.By("kubectl explain works to explain CR without validation schema") - if err := verifyKubectlExplain(crd.GetPluralName(), `(?s)DESCRIPTION:.*`); err != nil { + if err := verifyKubectlExplain(crd.Crd.Spec.Names.Plural, `(?s)DESCRIPTION:.*`); err != nil { framework.Failf("%v", err) } @@ -168,8 +167,8 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish if err != nil { framework.Failf("%v", err) } - if crdFoo.APIGroup == crdWaldo.APIGroup { - framework.Failf("unexpected: CRDs should be of different group %v, %v", crdFoo.APIGroup, crdWaldo.APIGroup) + if crdFoo.Crd.Spec.Group == crdWaldo.Crd.Spec.Group { + framework.Failf("unexpected: CRDs should be of different group %v, %v", crdFoo.Crd.Spec.Group, crdWaldo.Crd.Spec.Group) } if err := waitForDefinition(f.ClientSet, definitionName(crdWaldo, "v1beta1"), schemaWaldo); err != nil { framework.Failf("%v", err) @@ -210,8 +209,8 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish if err != nil { framework.Failf("%v", err) } - if crdFoo.APIGroup != crdWaldo.APIGroup { - framework.Failf("unexpected: CRDs should be of the same group %v, %v", crdFoo.APIGroup, crdWaldo.APIGroup) + if crdFoo.Crd.Spec.Group != crdWaldo.Crd.Spec.Group { + framework.Failf("unexpected: CRDs should be of the same group %v, %v", crdFoo.Crd.Spec.Group, crdWaldo.Crd.Spec.Group) } if err := waitForDefinition(f.ClientSet, definitionName(crdWaldo, "v5"), schemaWaldo); err != nil { framework.Failf("%v", err) @@ -237,8 +236,8 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish if err != nil { framework.Failf("%v", err) } - if crdFoo.APIGroup != crdWaldo.APIGroup { - framework.Failf("unexpected: CRDs should be of the same group %v, %v", crdFoo.APIGroup, crdWaldo.APIGroup) + if crdFoo.Crd.Spec.Group != crdWaldo.Crd.Spec.Group { + framework.Failf("unexpected: CRDs should be of the same group %v, %v", crdFoo.Crd.Spec.Group, crdWaldo.Crd.Spec.Group) } if err := waitForDefinition(f.ClientSet, definitionName(crdWaldo, "v6"), schemaWaldo); err != nil { framework.Failf("%v", err) @@ -269,7 +268,7 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish ginkgo.By("rename a version") patch := []byte(`{"spec":{"versions":[{"name":"v2","served":true,"storage":true},{"name":"v4","served":true,"storage":false}]}}`) - crdMultiVer.Crd, err = crdMultiVer.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crdMultiVer.GetMetaName(), types.MergePatchType, patch) + crdMultiVer.Crd, err = crdMultiVer.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crdMultiVer.Crd.Name, types.MergePatchType, patch) if err != nil { framework.Failf("%v", err) } @@ -289,7 +288,7 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish // TestCrd.Versions is different from TestCrd.Crd.Versions, we have to manually // update the name there. Used by cleanupCRD - crdMultiVer.Versions[1].Name = "v4" + crdMultiVer.Crd.Spec.Versions[1].Name = "v4" if err := cleanupCRD(f, crdMultiVer); err != nil { framework.Failf("%v", err) } @@ -336,33 +335,36 @@ func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, version if len(versions) == 0 { return nil, fmt.Errorf("require at least one version for CRD") } - apiVersions := []v1beta1.CustomResourceDefinitionVersion{} - for _, version := range versions { - v := v1beta1.CustomResourceDefinitionVersion{ - Name: version, - Served: true, - Storage: false, - } - apiVersions = append(apiVersions, v) - } - apiVersions[0].Storage = true - crd, err := crd.CreateMultiVersionTestCRD(f, group, apiVersions, nil) + if schema == nil { + schema = []byte(`type: object`) + } + props := &v1beta1.JSONSchemaProps{} + if err := yaml.Unmarshal(schema, props); err != nil { + return nil, err + } + + crd, err := crd.CreateMultiVersionTestCRD(f, group, func(crd *v1beta1.CustomResourceDefinition) { + apiVersions := []v1beta1.CustomResourceDefinitionVersion{} + for _, version := range versions { + v := v1beta1.CustomResourceDefinitionVersion{ + Name: version, + Served: true, + Storage: false, + } + apiVersions = append(apiVersions, v) + } + apiVersions[0].Storage = true + + crd.Spec.Validation = &v1beta1.CustomResourceValidation{ + OpenAPIV3Schema: props, + } + }) if err != nil { return nil, fmt.Errorf("failed to create CRD: %v", err) } - if schema != nil { - // patch validation schema for all versions - if err := patchSchema(schema, crd); err != nil { - return nil, fmt.Errorf("failed to patch schema: %v", err) - } - } else { - // change expectation if CRD doesn't have schema - schema = []byte(`type: object`) - } - - for _, v := range crd.Versions { + for _, v := range crd.Crd.Spec.Versions { if err := waitForDefinition(f.ClientSet, definitionName(crd, v.Name), schema); err != nil { return nil, fmt.Errorf("%v", err) } @@ -372,7 +374,7 @@ func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, version func cleanupCRD(f *framework.Framework, crd *crd.TestCrd) error { crd.CleanUp() - for _, v := range crd.Versions { + for _, v := range crd.Crd.Spec.Versions { name := definitionName(crd, v.Name) if err := waitForDefinitionCleanup(f.ClientSet, name); err != nil { return fmt.Errorf("%v", err) @@ -381,17 +383,6 @@ func cleanupCRD(f *framework.Framework, crd *crd.TestCrd) error { return nil } -// patchSchema takes schema in YAML and patches it to given CRD in given version -func patchSchema(schema []byte, crd *crd.TestCrd) error { - s, err := utilyaml.ToJSON(schema) - if err != nil { - return fmt.Errorf("failed to create json patch: %v", err) - } - patch := []byte(fmt.Sprintf(`{"spec":{"validation":{"openAPIV3Schema":%s}}}`, string(s))) - crd.Crd, err = crd.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crd.GetMetaName(), types.MergePatchType, patch) - return err -} - const waitSuccessThreshold = 10 // mustSucceedMultipleTimes calls f multiple times on success and only returns true if all calls are successful. @@ -530,7 +521,7 @@ func verifyKubectlExplain(name, pattern string) error { // definitionName returns the openapi definition name for given CRD in given version func definitionName(crd *crd.TestCrd, version string) string { - return openapiutil.ToRESTFriendlyName(fmt.Sprintf("%s/%s/%s", crd.APIGroup, version, crd.Kind)) + return openapiutil.ToRESTFriendlyName(fmt.Sprintf("%s/%s/%s", crd.Crd.Spec.Group, version, crd.Crd.Spec.Names.Kind)) } var schemaFoo = []byte(`description: Foo CRD for Testing diff --git a/test/e2e/apimachinery/resource_quota.go b/test/e2e/apimachinery/resource_quota.go index 2b129db0351..ed36cbb7887 100644 --- a/test/e2e/apimachinery/resource_quota.go +++ b/test/e2e/apimachinery/resource_quota.go @@ -532,10 +532,10 @@ var _ = SIGDescribe("ResourceQuota", func() { framework.ExpectNoError(err) ginkgo.By("Creating a custom resource") - resourceClient := testcrd.GetV1DynamicClient() + resourceClient := testcrd.DynamicClients["v1"] testcr, err := instantiateCustomResource(&unstructured.Unstructured{ Object: map[string]interface{}{ - "apiVersion": testcrd.APIGroup + "/" + testcrd.GetAPIVersions()[0], + "apiVersion": testcrd.Crd.Spec.Group + "/" + testcrd.Crd.Spec.Versions[0].Name, "kind": testcrd.Crd.Spec.Names.Kind, "metadata": map[string]interface{}{ "name": "test-cr-1", @@ -553,7 +553,7 @@ var _ = SIGDescribe("ResourceQuota", func() { ginkgo.By("Creating a second custom resource") _, err = instantiateCustomResource(&unstructured.Unstructured{ Object: map[string]interface{}{ - "apiVersion": testcrd.APIGroup + "/" + testcrd.GetAPIVersions()[0], + "apiVersion": testcrd.Crd.Spec.Group + "/" + testcrd.Crd.Spec.Versions[0].Name, "kind": testcrd.Crd.Spec.Names.Kind, "metadata": map[string]interface{}{ "name": "test-cr-2", diff --git a/test/e2e/apimachinery/webhook.go b/test/e2e/apimachinery/webhook.go index a7cff4f6c34..d63d239e121 100644 --- a/test/e2e/apimachinery/webhook.go +++ b/test/e2e/apimachinery/webhook.go @@ -24,7 +24,7 @@ import ( "k8s.io/api/admissionregistration/v1beta1" apps "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" rbacv1beta1 "k8s.io/api/rbac/v1beta1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" crdclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" @@ -46,6 +46,7 @@ import ( "github.com/onsi/ginkgo" "github.com/onsi/gomega" + // ensure libs have a chance to initialize _ "github.com/stretchr/testify/assert" ) @@ -143,7 +144,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() { defer testcrd.CleanUp() webhookCleanup := registerWebhookForCustomResource(f, context, testcrd) defer webhookCleanup() - testCustomResourceWebhook(f, testcrd.Crd, testcrd.GetV1DynamicClient()) + testCustomResourceWebhook(f, testcrd.Crd, testcrd.DynamicClients["v1"]) }) ginkgo.It("Should unconditionally reject operations on fail closed webhook", func() { @@ -180,7 +181,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() { defer testcrd.CleanUp() webhookCleanup := registerMutatingWebhookForCustomResource(f, context, testcrd) defer webhookCleanup() - testMutatingCustomResourceWebhook(f, testcrd.Crd, testcrd.GetV1DynamicClient()) + testMutatingCustomResourceWebhook(f, testcrd.Crd, testcrd.DynamicClients["v1"]) }) ginkgo.It("Should deny crd creation", func() { @@ -191,7 +192,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() { }) ginkgo.It("Should mutate custom resource with different stored version", func() { - testcrd, err := crd.CreateMultiVersionTestCRDWithV1Storage(f) + testcrd, err := createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f) if err != nil { return } @@ -1217,9 +1218,9 @@ func registerWebhookForCustomResource(f *framework.Framework, context *certConte Rules: []v1beta1.RuleWithOperations{{ Operations: []v1beta1.OperationType{v1beta1.Create, v1beta1.Update}, Rule: v1beta1.Rule{ - APIGroups: []string{testcrd.APIGroup}, - APIVersions: testcrd.GetAPIVersions(), - Resources: []string{testcrd.GetPluralName()}, + APIGroups: []string{testcrd.Crd.Spec.Group}, + APIVersions: servedAPIVersions(testcrd.Crd), + Resources: []string{testcrd.Crd.Spec.Names.Plural}, }, }}, ClientConfig: v1beta1.WebhookClientConfig{ @@ -1259,9 +1260,9 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, context *c Rules: []v1beta1.RuleWithOperations{{ Operations: []v1beta1.OperationType{v1beta1.Create, v1beta1.Update}, Rule: v1beta1.Rule{ - APIGroups: []string{testcrd.APIGroup}, - APIVersions: testcrd.GetAPIVersions(), - Resources: []string{testcrd.GetPluralName()}, + APIGroups: []string{testcrd.Crd.Spec.Group}, + APIVersions: servedAPIVersions(testcrd.Crd), + Resources: []string{testcrd.Crd.Spec.Names.Plural}, }, }}, ClientConfig: v1beta1.WebhookClientConfig{ @@ -1279,9 +1280,9 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, context *c Rules: []v1beta1.RuleWithOperations{{ Operations: []v1beta1.OperationType{v1beta1.Create}, Rule: v1beta1.Rule{ - APIGroups: []string{testcrd.APIGroup}, - APIVersions: testcrd.GetAPIVersions(), - Resources: []string{testcrd.GetPluralName()}, + APIGroups: []string{testcrd.Crd.Spec.Group}, + APIVersions: servedAPIVersions(testcrd.Crd), + Resources: []string{testcrd.Crd.Spec.Names.Plural}, }, }}, ClientConfig: v1beta1.WebhookClientConfig{ @@ -1357,7 +1358,7 @@ func testMutatingCustomResourceWebhook(f *framework.Framework, crd *apiextension } func testMultiVersionCustomResourceWebhook(f *framework.Framework, testcrd *crd.TestCrd) { - customResourceClient := testcrd.GetV1DynamicClient() + customResourceClient := testcrd.DynamicClients["v1"] ginkgo.By("Creating a custom resource while v1 is storage version") crName := "cr-instance-1" cr := &unstructured.Unstructured{ @@ -1446,12 +1447,6 @@ func testCRDDenyWebhook(f *framework.Framework) { Storage: true, }, } - testcrd := &crd.TestCrd{ - Name: name, - Kind: kind, - APIGroup: group, - Versions: apiVersions, - } // Creating a custom resource definition for use by assorted tests. config, err := framework.LoadConfig() @@ -1466,19 +1461,19 @@ func testCRDDenyWebhook(f *framework.Framework) { } crd := &apiextensionsv1beta1.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{ - Name: testcrd.GetMetaName(), + Name: name + "s." + group, Labels: map[string]string{ "webhook-e2e-test": "webhook-disallow", }, }, Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ - Group: testcrd.APIGroup, - Versions: testcrd.Versions, + Group: group, + Versions: apiVersions, Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ - Plural: testcrd.GetPluralName(), - Singular: testcrd.Name, - Kind: testcrd.Kind, - ListKind: testcrd.GetListName(), + Singular: name, + Kind: kind, + ListKind: kind + "List", + Plural: name + "s", }, Scope: apiextensionsv1beta1.NamespaceScoped, }, @@ -1486,7 +1481,7 @@ func testCRDDenyWebhook(f *framework.Framework) { // create CRD _, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd) - gomega.Expect(err).To(gomega.HaveOccurred(), "create custom resource definition %s should be denied by webhook", testcrd.GetMetaName()) + gomega.Expect(err).To(gomega.HaveOccurred(), "create custom resource definition %s should be denied by webhook", crd.Name) expectedErrMsg := "the crd contains unwanted label" if !strings.Contains(err.Error(), expectedErrMsg) { framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error()) @@ -1573,3 +1568,34 @@ func testSlowWebhookTimeoutNoError(f *framework.Framework) { err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(name, &metav1.DeleteOptions{}) gomega.Expect(err).To(gomega.BeNil()) } + +// createAdmissionWebhookMultiVersionTestCRDWithV1Storage creates a new CRD specifically +// for the admissin webhook calling test. +func createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f *framework.Framework) (*crd.TestCrd, error) { + group := fmt.Sprintf("%s-multiversion-crd-test.k8s.io", f.BaseName) + return crd.CreateMultiVersionTestCRD(f, group, func(crd *apiextensionsv1beta1.CustomResourceDefinition) { + crd.Spec.Versions = []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + }, + { + Name: "v2", + Served: true, + Storage: false, + }, + } + }) +} + +// servedAPIVersions returns the API versions served by the CRD. +func servedAPIVersions(crd *apiextensionsv1beta1.CustomResourceDefinition) []string { + ret := []string{} + for _, v := range crd.Spec.Versions { + if v.Served { + ret = append(ret, v.Name) + } + } + return ret +} diff --git a/test/e2e/kubectl/BUILD b/test/e2e/kubectl/BUILD index 2989fb29af6..f740c529be0 100644 --- a/test/e2e/kubectl/BUILD +++ b/test/e2e/kubectl/BUILD @@ -18,6 +18,7 @@ go_library( "//pkg/kubectl/polymorphichelpers:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/test/e2e/kubectl/kubectl.go b/test/e2e/kubectl/kubectl.go index 3f305eaca79..de52df35e73 100644 --- a/test/e2e/kubectl/kubectl.go +++ b/test/e2e/kubectl/kubectl.go @@ -44,6 +44,7 @@ import ( v1 "k8s.io/api/core/v1" rbacv1beta1 "k8s.io/api/rbac/v1beta1" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -69,6 +70,7 @@ import ( "github.com/onsi/ginkgo" "github.com/onsi/gomega" + "k8s.io/kubernetes/pkg/kubectl/polymorphichelpers" imageutils "k8s.io/kubernetes/test/utils/image" ) @@ -861,7 +863,7 @@ metadata: ginkgo.By("sleep for 10s to wait for potential crd openapi publishing alpha feature") time.Sleep(10 * time.Second) - meta := fmt.Sprintf(metaPattern, crd.Kind, crd.APIGroup, crd.Versions[0].Name, "test-cr") + meta := fmt.Sprintf(metaPattern, crd.Crd.Spec.Names.Kind, crd.Crd.Spec.Group, crd.Crd.Spec.Versions[0].Name, "test-cr") randomCR := fmt.Sprintf(`{%s,"a":{"b":[{"c":"d"}]}}`, meta) if err := createApplyCustomResource(randomCR, f.Namespace.Name, "test-cr", crd); err != nil { framework.Failf("%v", err) @@ -870,19 +872,21 @@ metadata: ginkgo.It("should create/apply a valid CR for CRD with validation schema", func() { ginkgo.By("prepare CRD with validation schema") - crd, err := crd.CreateTestCRD(f) + crd, err := crd.CreateTestCRD(f, func(crd *v1beta1.CustomResourceDefinition) { + props := &v1beta1.JSONSchemaProps{} + if err := yaml.Unmarshal(schemaFoo, props); err != nil { + framework.Failf("failed to unmarshal schema: %v", err) + } + }) if err != nil { framework.Failf("failed to create test CRD: %v", err) } defer crd.CleanUp() - if err := crd.PatchSchema(schemaFoo); err != nil { - framework.Failf("%v", err) - } ginkgo.By("sleep for 10s to wait for potential crd openapi publishing alpha feature") time.Sleep(10 * time.Second) - meta := fmt.Sprintf(metaPattern, crd.Kind, crd.APIGroup, crd.Versions[0].Name, "test-cr") + meta := fmt.Sprintf(metaPattern, crd.Crd.Spec.Names.Kind, crd.Crd.Spec.Group, crd.Crd.Spec.Versions[0].Name, "test-cr") validCR := fmt.Sprintf(`{%s,"spec":{"bars":[{"name":"test-bar"}]}}`, meta) if err := createApplyCustomResource(validCR, f.Namespace.Name, "test-cr", crd); err != nil { framework.Failf("%v", err) @@ -891,19 +895,21 @@ metadata: ginkgo.It("should create/apply a valid CR with arbitrary-extra properties for CRD with partially-specified validation schema", func() { ginkgo.By("prepare CRD with partially-specified validation schema") - crd, err := crd.CreateTestCRD(f) + crd, err := crd.CreateTestCRD(f, func(crd *v1beta1.CustomResourceDefinition) { + props := &v1beta1.JSONSchemaProps{} + if err := yaml.Unmarshal(schemaFoo, props); err != nil { + framework.Failf("failed to unmarshal schema: %v", err) + } + }) if err != nil { framework.Failf("failed to create test CRD: %v", err) } defer crd.CleanUp() - if err := crd.PatchSchema(schemaFoo); err != nil { - framework.Failf("%v", err) - } ginkgo.By("sleep for 10s to wait for potential crd openapi publishing alpha feature") time.Sleep(10 * time.Second) - meta := fmt.Sprintf(metaPattern, crd.Kind, crd.APIGroup, crd.Versions[0].Name, "test-cr") + meta := fmt.Sprintf(metaPattern, crd.Crd.Spec.Names.Kind, crd.Crd.Spec.Group, crd.Crd.Spec.Versions[0].Name, "test-cr") validArbitraryCR := fmt.Sprintf(`{%s,"spec":{"bars":[{"name":"test-bar"}],"extraProperty":"arbitrary-value"}}`, meta) if err := createApplyCustomResource(validArbitraryCR, f.Namespace.Name, "test-cr", crd); err != nil { framework.Failf("%v", err) @@ -2258,14 +2264,14 @@ func createApplyCustomResource(resource, namespace, name string, crd *crd.TestCr if _, err := framework.RunKubectlInput(resource, ns, "create", "-f", "-"); err != nil { return fmt.Errorf("failed to create CR %s in namespace %s: %v", resource, ns, err) } - if _, err := framework.RunKubectl(ns, "delete", crd.GetPluralName(), name); err != nil { + if _, err := framework.RunKubectl(ns, "delete", crd.Crd.Spec.Names.Plural, name); err != nil { return fmt.Errorf("failed to delete CR %s: %v", name, err) } ginkgo.By("successfully apply CR") if _, err := framework.RunKubectlInput(resource, ns, "apply", "-f", "-"); err != nil { return fmt.Errorf("failed to apply CR %s in namespace %s: %v", resource, ns, err) } - if _, err := framework.RunKubectl(ns, "delete", crd.GetPluralName(), name); err != nil { + if _, err := framework.RunKubectl(ns, "delete", crd.Crd.Spec.Names.Plural, name); err != nil { return fmt.Errorf("failed to delete CR %s: %v", name, err) } return nil diff --git a/test/utils/crd/BUILD b/test/utils/crd/BUILD index 6a577b464b1..4ed4991479e 100644 --- a/test/utils/crd/BUILD +++ b/test/utils/crd/BUILD @@ -11,8 +11,6 @@ go_library( "//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//staging/src/k8s.io/client-go/dynamic:go_default_library", "//test/e2e/framework:go_default_library", ], diff --git a/test/utils/crd/crd_util.go b/test/utils/crd/crd_util.go index 33088508999..42ecdf1139a 100644 --- a/test/utils/crd/crd_util.go +++ b/test/utils/crd/crd_util.go @@ -24,8 +24,6 @@ import ( "k8s.io/apiextensions-apiserver/test/integration/fixtures" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - utilyaml "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/dynamic" "k8s.io/kubernetes/test/e2e/framework" ) @@ -35,27 +33,21 @@ type CleanCrdFn func() error // TestCrd holds all the pieces needed to test with the CRD type TestCrd struct { - Name string - Kind string - APIGroup string - Versions []apiextensionsv1beta1.CustomResourceDefinitionVersion APIExtensionClient *crdclientset.Clientset Crd *apiextensionsv1beta1.CustomResourceDefinition DynamicClients map[string]dynamic.ResourceInterface CleanUp CleanCrdFn } +// Option is a modifier for a CRD object used to customize CreateMultiVersionTestCRD and CreateTestCRD. +type Option func(crd *apiextensionsv1beta1.CustomResourceDefinition) + // CreateMultiVersionTestCRD creates a new CRD specifically for the calling test. -func CreateMultiVersionTestCRD(f *framework.Framework, group string, apiVersions []apiextensionsv1beta1.CustomResourceDefinitionVersion, conversionWebhook *apiextensionsv1beta1.WebhookClientConfig) (*TestCrd, error) { +func CreateMultiVersionTestCRD(f *framework.Framework, group string, opts ...Option) (*TestCrd, error) { suffix := framework.RandomSuffix() name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, suffix) kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, suffix) - testcrd := &TestCrd{ - Name: name, - Kind: kind, - APIGroup: group, - Versions: apiVersions, - } + testcrd := &TestCrd{} // Creating a custom resource definition for use by assorted tests. config, err := framework.LoadConfig() @@ -74,13 +66,24 @@ func CreateMultiVersionTestCRD(f *framework.Framework, group string, apiVersions return nil, err } - crd := newCRDForTest(testcrd) - - if conversionWebhook != nil { - crd.Spec.Conversion = &apiextensionsv1beta1.CustomResourceConversion{ - Strategy: "Webhook", - WebhookClientConfig: conversionWebhook, - } + crd := &apiextensionsv1beta1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: name + "s." + group}, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: group, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: name + "s", + Singular: name, + Kind: kind, + ListKind: kind + "List", + }, + Scope: apiextensionsv1beta1.NamespaceScoped, + }, + } + for _, opt := range opts { + opt(crd) + } + if len(crd.Spec.Versions) == 0 && len(crd.Spec.Version) == 0 { + crd.Spec.Version = "v1" } //create CRD and waits for the resource to be recognized and available. @@ -112,92 +115,15 @@ func CreateMultiVersionTestCRD(f *framework.Framework, group string, apiVersions } // CreateTestCRD creates a new CRD specifically for the calling test. -func CreateTestCRD(f *framework.Framework) (*TestCrd, error) { +func CreateTestCRD(f *framework.Framework, opts ...Option) (*TestCrd, error) { group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName) - apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{ - { - Name: "v1", - Served: true, - Storage: true, - }, - } - return CreateMultiVersionTestCRD(f, group, apiVersions, nil) -} - -// CreateMultiVersionTestCRDWithV1Storage creates a new CRD specifically for the calling test. -func CreateMultiVersionTestCRDWithV1Storage(f *framework.Framework) (*TestCrd, error) { - group := fmt.Sprintf("%s-multiversion-crd-test.k8s.io", f.BaseName) - apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{ - { - Name: "v1", - Served: true, - Storage: true, - }, - { - Name: "v2", - Served: true, - Storage: false, - }, - } - return CreateMultiVersionTestCRD(f, group, apiVersions, nil) -} - -// newCRDForTest generates a CRD definition for the test -func newCRDForTest(testcrd *TestCrd) *apiextensionsv1beta1.CustomResourceDefinition { - return &apiextensionsv1beta1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{Name: testcrd.GetMetaName()}, - Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ - Group: testcrd.APIGroup, - Versions: testcrd.Versions, - Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ - Plural: testcrd.GetPluralName(), - Singular: testcrd.Name, - Kind: testcrd.Kind, - ListKind: testcrd.GetListName(), + return CreateMultiVersionTestCRD(f, group, append([]Option{func(crd *apiextensionsv1beta1.CustomResourceDefinition) { + crd.Spec.Versions = []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, }, - Scope: apiextensionsv1beta1.NamespaceScoped, - }, - } -} - -// GetMetaName returns the metaname for the CRD. -func (c *TestCrd) GetMetaName() string { - return c.Name + "s." + c.APIGroup -} - -// GetPluralName returns the plural form of the CRD name -func (c *TestCrd) GetPluralName() string { - return c.Name + "s" -} - -// GetListName returns the name for the CRD list resources -func (c *TestCrd) GetListName() string { - return c.Name + "List" -} - -// GetAPIVersions returns the API versions served by the CRD. -func (c *TestCrd) GetAPIVersions() []string { - ret := []string{} - for _, v := range c.Versions { - if v.Served { - ret = append(ret, v.Name) } - } - return ret -} - -// GetV1DynamicClient returns the dynamic client for v1. -func (c *TestCrd) GetV1DynamicClient() dynamic.ResourceInterface { - return c.DynamicClients["v1"] -} - -// PatchSchema takes validation schema in YAML and patches it to given CRD -func (c *TestCrd) PatchSchema(schema []byte) error { - s, err := utilyaml.ToJSON(schema) - if err != nil { - return fmt.Errorf("failed to create json patch: %v", err) - } - patch := []byte(fmt.Sprintf(`{"spec":{"validation":{"openAPIV3Schema":%s}}}`, string(s))) - c.Crd, err = c.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(c.GetMetaName(), types.MergePatchType, patch) - return err + }}, opts...)...) }