From c7919e64ff337e7404fbf32d5d84c553008c210b Mon Sep 17 00:00:00 2001 From: Joe Betz Date: Wed, 28 Aug 2019 18:45:26 -0700 Subject: [PATCH] Promote CRD e2e tests to conformance --- test/conformance/testdata/conformance.txt | 14 ++++ .../apimachinery/crd_conversion_webhook.go | 19 ++++- test/e2e/apimachinery/crd_publish_openapi.go | 84 ++++++++++++++++--- test/e2e/apimachinery/crd_watch.go | 11 +-- .../custom_resource_definition.go | 18 ++-- 5 files changed, 122 insertions(+), 24 deletions(-) diff --git a/test/conformance/testdata/conformance.txt b/test/conformance/testdata/conformance.txt index d609cb9c00b..8803586808b 100644 --- a/test/conformance/testdata/conformance.txt +++ b/test/conformance/testdata/conformance.txt @@ -1,5 +1,19 @@ test/e2e/apimachinery/aggregator.go: "Should be able to support the 1.10 Sample API Server using the current Aggregator" +test/e2e/apimachinery/crd_conversion_webhook.go: "should be able to convert from CR v1 to CR v2" +test/e2e/apimachinery/crd_conversion_webhook.go: "should be able to convert a non homogeneous list of CRs" +test/e2e/apimachinery/crd_publish_openapi.go: "works for CRD with validation schema" +test/e2e/apimachinery/crd_publish_openapi.go: "works for CRD without validation schema" +test/e2e/apimachinery/crd_publish_openapi.go: "works for CRD preserving unknown fields at the schema root" +test/e2e/apimachinery/crd_publish_openapi.go: "works for CRD preserving unknown fields in an embedded object" +test/e2e/apimachinery/crd_publish_openapi.go: "works for multiple CRDs of different groups" +test/e2e/apimachinery/crd_publish_openapi.go: "works for multiple CRDs of same group but different versions" +test/e2e/apimachinery/crd_publish_openapi.go: "works for multiple CRDs of same group and version but different kinds" +test/e2e/apimachinery/crd_publish_openapi.go: "updates the published spec when one version gets renamed" +test/e2e/apimachinery/crd_publish_openapi.go: "removes definition from spec when one version gets changed to not be served" +test/e2e/apimachinery/crd_watch.go: "watch on custom resource definition objects" test/e2e/apimachinery/custom_resource_definition.go: "creating/deleting custom resource definition objects works" +test/e2e/apimachinery/custom_resource_definition.go: "listing custom resource definition objects works" +test/e2e/apimachinery/custom_resource_definition.go: "getting/updating/patching custom resource definition status sub-resource works" test/e2e/apimachinery/garbage_collector.go: "should delete pods created by rc when not orphaning" test/e2e/apimachinery/garbage_collector.go: "should orphan pods created by rc if delete options say so" test/e2e/apimachinery/garbage_collector.go: "should delete RS created by deployment when not orphaning" diff --git a/test/e2e/apimachinery/crd_conversion_webhook.go b/test/e2e/apimachinery/crd_conversion_webhook.go index 3af639c12d3..f6a28daa78c 100644 --- a/test/e2e/apimachinery/crd_conversion_webhook.go +++ b/test/e2e/apimachinery/crd_conversion_webhook.go @@ -113,7 +113,7 @@ var alternativeAPIVersions = []apiextensionsv1.CustomResourceDefinitionVersion{ }, } -var _ = SIGDescribe("CustomResourceConversionWebhook", func() { +var _ = SIGDescribe("CustomResourceConversionWebhook [Privileged:ClusterAdmin]", func() { var context *certContext f := framework.NewDefaultFramework("crd-webhook") servicePort := int32(9443) @@ -137,7 +137,13 @@ var _ = SIGDescribe("CustomResourceConversionWebhook", func() { cleanCRDWebhookTest(client, namespaceName) }) - ginkgo.It("Should be able to convert from CR v1 to CR v2", func() { + /* + Release : v1.16 + Testname: Custom Resource Definition Conversion Webhook, conversion custom resource + Description: Register a conversion webhook and a custom resource definition. Create a v1 custom + resource. Attempts to read it at v2 MUST succeed. + */ + framework.ConformanceIt("should be able to convert from CR v1 to CR v2", func() { testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *apiextensionsv1.CustomResourceDefinition) { crd.Spec.Versions = apiVersions crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{ @@ -164,7 +170,14 @@ var _ = SIGDescribe("CustomResourceConversionWebhook", func() { testCustomResourceConversionWebhook(f, testcrd.Crd, testcrd.DynamicClients) }) - ginkgo.It("Should be able to convert a non homogeneous list of CRs", func() { + /* + Release : v1.16 + Testname: Custom Resource Definition Conversion Webhook, convert mixed version list + Description: Register a conversion webhook and a custom resource definition. Create a custom resource stored at + v1. Change the custom resource definition storage to v2. Create a custom resource stored at v2. Attempt to list + the custom resources at v2; the list result MUST contain both custom resources at v2. + */ + framework.ConformanceIt("should be able to convert a non homogeneous list of CRs", func() { testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *apiextensionsv1.CustomResourceDefinition) { crd.Spec.Versions = apiVersions crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{ diff --git a/test/e2e/apimachinery/crd_publish_openapi.go b/test/e2e/apimachinery/crd_publish_openapi.go index 6dce7a38d59..f613e8eb30b 100644 --- a/test/e2e/apimachinery/crd_publish_openapi.go +++ b/test/e2e/apimachinery/crd_publish_openapi.go @@ -49,10 +49,20 @@ var ( metaPattern = `"kind":"%s","apiVersion":"%s/%s","metadata":{"name":"%s"}` ) -var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() { +var _ = SIGDescribe("CustomResourcePublishOpenAPI [Privileged:ClusterAdmin]", func() { f := framework.NewDefaultFramework("crd-publish-openapi") - ginkgo.It("works for CRD with validation schema", func() { + /* + Release: v1.16 + Testname: Custom Resource OpenAPI Publish, with validation schema + Description: Register a custom resource definition with a validating schema consisting of objects, arrays and + primitives. Attempt to create and apply a change a custom resource using valid properties, via kubectl; + client-side validation MUST pass. Attempt both operations with unknown properties and without required + properties; client-side validation MUST reject the operations. Attempt kubectl explain; the output MUST + explain the custom resource properties. Attempt kubectl explain on custom resource properties; the output MUST + explain the nested custom resource properties. + */ + framework.ConformanceIt("works for CRD with validation schema", func() { crd, err := setupCRD(f, schemaFoo, "foo", "v1") if err != nil { e2elog.Failf("%v", err) @@ -120,7 +130,14 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() { } }) - ginkgo.It("works for CRD without validation schema", func() { + /* + Release: v1.16 + Testname: Custom Resource OpenAPI Publish, with x-preserve-unknown-fields in object + Description: Register a custom resource definition with x-preserve-unknown-fields in the top level object. + Attempt to create and apply a change a custom resource, via kubectl; client-side validation MUST accept unknown + properties. Attempt kubectl explain; the output MUST contain a valid DESCRIPTION stanza. + */ + framework.ConformanceIt("works for CRD without validation schema", func() { crd, err := setupCRD(f, nil, "empty", "v1") if err != nil { e2elog.Failf("%v", err) @@ -154,7 +171,14 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() { } }) - ginkgo.It("works for CRD preserving unknown fields at the schema root", func() { + /* + Release: v1.16 + Testname: Custom Resource OpenAPI Publish, with x-preserve-unknown-fields at root + Description: Register a custom resource definition with x-preserve-unknown-fields in the schema root. + Attempt to create and apply a change a custom resource, via kubectl; client-side validation MUST accept unknown + properties. Attempt kubectl explain; the output MUST show the custom resource KIND. + */ + framework.ConformanceIt("works for CRD preserving unknown fields at the schema root", func() { crd, err := setupCRDAndVerifySchema(f, schemaPreserveRoot, nil, "unknown-at-root", "v1") if err != nil { e2elog.Failf("%v", err) @@ -188,7 +212,15 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() { } }) - ginkgo.It("works for CRD preserving unknown fields in an embedded object", func() { + /* + Release: v1.16 + Testname: Custom Resource OpenAPI Publish, with x-preserve-unknown-fields in embedded object + Description: Register a custom resource definition with x-preserve-unknown-fields in an embedded object. + Attempt to create and apply a change a custom resource, via kubectl; client-side validation MUST accept unknown + properties. Attempt kubectl explain; the output MUST show that x-preserve-unknown-properties is used on the + nested field. + */ + framework.ConformanceIt("works for CRD preserving unknown fields in an embedded object", func() { crd, err := setupCRDAndVerifySchema(f, schemaPreserveNested, nil, "unknown-in-nested", "v1") if err != nil { e2elog.Failf("%v", err) @@ -222,7 +254,13 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() { } }) - ginkgo.It("works for multiple CRDs of different groups", func() { + /* + Release: v1.16 + Testname: Custom Resource OpenAPI Publish, varying groups + Description: Register multiple custom resource definitions spanning different groups and versions; + OpenAPI definitions MUST be published for custom resource definitions. + */ + framework.ConformanceIt("works for multiple CRDs of different groups", func() { ginkgo.By("CRs in different groups (two CRDs) show up in OpenAPI documentation") crdFoo, err := setupCRD(f, schemaFoo, "foo", "v1") if err != nil { @@ -249,7 +287,13 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() { } }) - ginkgo.It("works for multiple CRDs of same group but different versions", func() { + /* + Release: v1.16 + Testname: Custom Resource OpenAPI Publish, varying versions + Description: Register a custom resource definition with multiple versions; OpenAPI definitions MUST be published + for custom resource definitions. + */ + framework.ConformanceIt("works for multiple CRDs of same group but different versions", func() { ginkgo.By("CRs in the same group but different versions (one multiversion CRD) show up in OpenAPI documentation") crdMultiVer, err := setupCRD(f, schemaFoo, "multi-ver", "v2", "v3") if err != nil { @@ -291,7 +335,13 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() { } }) - ginkgo.It("works for multiple CRDs of same group and version but different kinds", func() { + /* + Release: v1.16 + Testname: Custom Resource OpenAPI Publish, varying kinds + Description: Register multiple custom resource definitions in the same group and version but spanning different kinds; + OpenAPI definitions MUST be published for custom resource definitions. + */ + framework.ConformanceIt("works for multiple CRDs of same group and version but different kinds", func() { ginkgo.By("CRs in the same group and version but different kinds (two CRDs) show up in OpenAPI documentation") crdFoo, err := setupCRD(f, schemaFoo, "common-group", "v6") if err != nil { @@ -318,7 +368,14 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() { } }) - ginkgo.It("updates the published spec when one versin gets renamed", func() { + /* + Release: v1.16 + Testname: Custom Resource OpenAPI Publish, version rename + Description: Register a custom resource definition with multiple versions; OpenAPI definitions MUST be published + for custom resource definitions. Rename one of the versions of the custom resource definition via a patch; + OpenAPI definitions MUST update to reflect the rename. + */ + framework.ConformanceIt("updates the published spec when one version gets renamed", func() { ginkgo.By("set up a multi version CRD") crdMultiVer, err := setupCRD(f, schemaFoo, "multi-ver", "v2", "v3") if err != nil { @@ -362,7 +419,14 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() { } }) - ginkgo.It("removes definition from spec when one versin gets changed to not be served", func() { + /* + Release: v1.16 + Testname: Custom Resource OpenAPI Publish, stop serving version + Description: Register a custom resource definition with multiple versions. OpenAPI definitions MUST be published + for custom resource definitions. Update the custom resource definition to not serve one of the versions. OpenAPI + definitions MUST be updated to not contain the version that is no longer served. + */ + framework.ConformanceIt("removes definition from spec when one version gets changed to not be served", func() { ginkgo.By("set up a multi version CRD") crd, err := setupCRD(f, schemaFoo, "multi-to-single-ver", "v5", "v6alpha1") if err != nil { diff --git a/test/e2e/apimachinery/crd_watch.go b/test/e2e/apimachinery/crd_watch.go index ecee88f5d18..8162e347d12 100644 --- a/test/e2e/apimachinery/crd_watch.go +++ b/test/e2e/apimachinery/crd_watch.go @@ -35,17 +35,18 @@ import ( "github.com/onsi/ginkgo" ) -var _ = SIGDescribe("CustomResourceDefinition Watch", func() { +var _ = SIGDescribe("CustomResourceDefinition Watch [Privileged:ClusterAdmin]", func() { f := framework.NewDefaultFramework("crd-watch") ginkgo.Context("CustomResourceDefinition Watch", func() { /* - Testname: crd-watch - Description: Create a Custom Resource Definition and make sure - watches observe events on create/delete. + Release: v1.16 + Testname: Custom Resource Definition, watch + Description: Create a Custom Resource Definition. Attempt to watch it; the watch MUST observe create, + modify and delete events. */ - ginkgo.It("watch on custom resource definition objects", func() { + framework.ConformanceIt("watch on custom resource definition objects", func() { const ( watchCRNameA = "name1" diff --git a/test/e2e/apimachinery/custom_resource_definition.go b/test/e2e/apimachinery/custom_resource_definition.go index 6be793b0ce8..cd317cdf0fb 100644 --- a/test/e2e/apimachinery/custom_resource_definition.go +++ b/test/e2e/apimachinery/custom_resource_definition.go @@ -35,7 +35,7 @@ import ( e2elog "k8s.io/kubernetes/test/e2e/framework/log" ) -var _ = SIGDescribe("CustomResourceDefinition resources", func() { +var _ = SIGDescribe("CustomResourceDefinition resources [Privileged:ClusterAdmin]", func() { framework.NewDefaultFramework("custom-resource-definition") @@ -43,7 +43,9 @@ var _ = SIGDescribe("CustomResourceDefinition resources", func() { /* Release : v1.9 Testname: Custom Resource Definition, create - Description: Create a API extension client, define a random custom resource definition, create the custom resource. API server MUST be able to create the custom resource. + Description: Create a API extension client and define a random custom resource definition. + Create the custom resource definition and then delete it. The creation and deletion MUST + be successful. */ framework.ConformanceIt("creating/deleting custom resource definition objects works ", func() { @@ -67,9 +69,12 @@ var _ = SIGDescribe("CustomResourceDefinition resources", func() { /* Release : v1.16 Testname: Custom Resource Definition, list - Description: Create a API extension client, define 10 random custom resource definitions and list them using a label selector. API server MUST be able to list the custom resource definitions and delete them via delete collection. + Description: Create a API extension client, define 10 labeled custom resource definitions and list them using + a label selector; the list result MUST contain only the labeled custom resource definitions. Delete the labeled + custom resource definitions via delete collection; the delete MUST be successful and MUST delete only the + labeled custom resource definitions. */ - ginkgo.It("listing custom resource definition objects works ", func() { + framework.ConformanceIt("listing custom resource definition objects works ", func() { testListSize := 10 config, err := framework.LoadConfig() framework.ExpectNoError(err, "loading config") @@ -126,9 +131,10 @@ var _ = SIGDescribe("CustomResourceDefinition resources", func() { /* Release : v1.16 Testname: Custom Resource Definition, status sub-resource - Description: Create a API extension client, create a custom resource definition and then read, update and patch its status sub-resource. API server MUST be able to perform the operations against the status sub-resource. + Description: Create a custom resource definition. Attempt to read, update and patch its status sub-resource; + all mutating sub-resource operations MUST be visible to subsequent reads. */ - ginkgo.It("getting/updating/patching custom resource definition status sub-resource works ", func() { + framework.ConformanceIt("getting/updating/patching custom resource definition status sub-resource works ", func() { config, err := framework.LoadConfig() framework.ExpectNoError(err, "loading config") apiExtensionClient, err := clientset.NewForConfig(config)