Promote CRD e2e tests to conformance

test/conformance/testdata/conformance.txt

@@ -1,5 +1,19 @@
 test/e2e/apimachinery/aggregator.go: "Should be able to support the 1.10 Sample API Server using the current Aggregator"
+test/e2e/apimachinery/crd_conversion_webhook.go: "should be able to convert from CR v1 to CR v2"
+test/e2e/apimachinery/crd_conversion_webhook.go: "should be able to convert a non homogeneous list of CRs"
+test/e2e/apimachinery/crd_publish_openapi.go: "works for CRD with validation schema"
+test/e2e/apimachinery/crd_publish_openapi.go: "works for CRD without validation schema"
+test/e2e/apimachinery/crd_publish_openapi.go: "works for CRD preserving unknown fields at the schema root"
+test/e2e/apimachinery/crd_publish_openapi.go: "works for CRD preserving unknown fields in an embedded object"
+test/e2e/apimachinery/crd_publish_openapi.go: "works for multiple CRDs of different groups"
+test/e2e/apimachinery/crd_publish_openapi.go: "works for multiple CRDs of same group but different versions"
+test/e2e/apimachinery/crd_publish_openapi.go: "works for multiple CRDs of same group and version but different kinds"
+test/e2e/apimachinery/crd_publish_openapi.go: "updates the published spec when one version gets renamed"
+test/e2e/apimachinery/crd_publish_openapi.go: "removes definition from spec when one version gets changed to not be served"
+test/e2e/apimachinery/crd_watch.go: "watch on custom resource definition objects"
 test/e2e/apimachinery/custom_resource_definition.go: "creating/deleting custom resource definition objects works"
+test/e2e/apimachinery/custom_resource_definition.go: "listing custom resource definition objects works"
+test/e2e/apimachinery/custom_resource_definition.go: "getting/updating/patching custom resource definition status sub-resource works"
 test/e2e/apimachinery/garbage_collector.go: "should delete pods created by rc when not orphaning"
 test/e2e/apimachinery/garbage_collector.go: "should orphan pods created by rc if delete options say so"
 test/e2e/apimachinery/garbage_collector.go: "should delete RS created by deployment when not orphaning"

test/e2e/apimachinery/crd_conversion_webhook.go

@@ -113,7 +113,7 @@ var alternativeAPIVersions = []apiextensionsv1.CustomResourceDefinitionVersion{
 	},
 }
 
-var _ = SIGDescribe("CustomResourceConversionWebhook", func() {
+var _ = SIGDescribe("CustomResourceConversionWebhook [Privileged:ClusterAdmin]", func() {
 	var context *certContext
 	f := framework.NewDefaultFramework("crd-webhook")
 	servicePort := int32(9443)
@@ -137,7 +137,13 @@ var _ = SIGDescribe("CustomResourceConversionWebhook", func() {
 		cleanCRDWebhookTest(client, namespaceName)
 	})
 
-	ginkgo.It("Should be able to convert from CR v1 to CR v2", func() {
+	/*
+		Release : v1.16
+		Testname: Custom Resource Definition Conversion Webhook, conversion custom resource
+		Description: Register a conversion webhook and a custom resource definition. Create a v1 custom
+		resource. Attempts to read it at v2 MUST succeed.
+	*/
+	framework.ConformanceIt("should be able to convert from CR v1 to CR v2", func() {
 		testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *apiextensionsv1.CustomResourceDefinition) {
 			crd.Spec.Versions = apiVersions
 			crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{
@@ -164,7 +170,14 @@ var _ = SIGDescribe("CustomResourceConversionWebhook", func() {
 		testCustomResourceConversionWebhook(f, testcrd.Crd, testcrd.DynamicClients)
 	})
 
-	ginkgo.It("Should be able to convert a non homogeneous list of CRs", func() {
+	/*
+		Release : v1.16
+		Testname: Custom Resource Definition Conversion Webhook, convert mixed version list
+		Description: Register a conversion webhook and a custom resource definition. Create a custom resource stored at
+		v1. Change the custom resource definition storage to v2. Create a custom resource stored at v2. Attempt to list
+		the custom resources at v2; the list result MUST contain both custom resources at v2.
+	*/
+	framework.ConformanceIt("should be able to convert a non homogeneous list of CRs", func() {
 		testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *apiextensionsv1.CustomResourceDefinition) {
 			crd.Spec.Versions = apiVersions
 			crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{

test/e2e/apimachinery/crd_publish_openapi.go

@@ -49,10 +49,20 @@ var (
 	metaPattern = `"kind":"%s","apiVersion":"%s/%s","metadata":{"name":"%s"}`
 )
 
-var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() {
+var _ = SIGDescribe("CustomResourcePublishOpenAPI [Privileged:ClusterAdmin]", func() {
 	f := framework.NewDefaultFramework("crd-publish-openapi")
 
-	ginkgo.It("works for CRD with validation schema", func() {
+	/*
+		Release: v1.16
+		Testname: Custom Resource OpenAPI Publish, with validation schema
+		Description: Register a custom resource definition with a validating schema consisting of objects, arrays and
+		primitives. Attempt to create and apply a change a custom resource using valid properties, via kubectl;
+		client-side validation MUST pass. Attempt both operations with unknown properties and without required
+		properties; client-side validation MUST reject the operations. Attempt kubectl explain; the output MUST
+		explain the custom resource properties. Attempt kubectl explain on custom resource properties; the output MUST
+		explain the nested custom resource properties.
+	*/
+	framework.ConformanceIt("works for CRD with validation schema", func() {
 		crd, err := setupCRD(f, schemaFoo, "foo", "v1")
 		if err != nil {
 			e2elog.Failf("%v", err)
@@ -120,7 +130,14 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() {
 		}
 	})
 
-	ginkgo.It("works for CRD without validation schema", func() {
+	/*
+		Release: v1.16
+		Testname: Custom Resource OpenAPI Publish, with x-preserve-unknown-fields in object
+		Description: Register a custom resource definition with x-preserve-unknown-fields in the top level object.
+		Attempt to create and apply a change a custom resource, via kubectl; client-side validation MUST accept unknown
+		properties. Attempt kubectl explain; the output MUST contain a valid DESCRIPTION stanza.
+	*/
+	framework.ConformanceIt("works for CRD without validation schema", func() {
 		crd, err := setupCRD(f, nil, "empty", "v1")
 		if err != nil {
 			e2elog.Failf("%v", err)
@@ -154,7 +171,14 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() {
 		}
 	})
 
-	ginkgo.It("works for CRD preserving unknown fields at the schema root", func() {
+	/*
+		Release: v1.16
+		Testname: Custom Resource OpenAPI Publish, with x-preserve-unknown-fields at root
+		Description: Register a custom resource definition with x-preserve-unknown-fields in the schema root.
+		Attempt to create and apply a change a custom resource, via kubectl; client-side validation MUST accept unknown
+		properties. Attempt kubectl explain; the output MUST show the custom resource KIND.
+	*/
+	framework.ConformanceIt("works for CRD preserving unknown fields at the schema root", func() {
 		crd, err := setupCRDAndVerifySchema(f, schemaPreserveRoot, nil, "unknown-at-root", "v1")
 		if err != nil {
 			e2elog.Failf("%v", err)
@@ -188,7 +212,15 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() {
 		}
 	})
 
-	ginkgo.It("works for CRD preserving unknown fields in an embedded object", func() {
+	/*
+		Release: v1.16
+		Testname: Custom Resource OpenAPI Publish, with x-preserve-unknown-fields in embedded object
+		Description: Register a custom resource definition with x-preserve-unknown-fields in an embedded object.
+		Attempt to create and apply a change a custom resource, via kubectl; client-side validation MUST accept unknown
+		properties. Attempt kubectl explain; the output MUST show that x-preserve-unknown-properties is used on the
+		nested field.
+	*/
+	framework.ConformanceIt("works for CRD preserving unknown fields in an embedded object", func() {
 		crd, err := setupCRDAndVerifySchema(f, schemaPreserveNested, nil, "unknown-in-nested", "v1")
 		if err != nil {
 			e2elog.Failf("%v", err)
@@ -222,7 +254,13 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() {
 		}
 	})
 
-	ginkgo.It("works for multiple CRDs of different groups", func() {
+	/*
+		Release: v1.16
+		Testname: Custom Resource OpenAPI Publish, varying groups
+		Description: Register multiple custom resource definitions spanning different groups and versions;
+		OpenAPI definitions MUST be published for custom resource definitions.
+	*/
+	framework.ConformanceIt("works for multiple CRDs of different groups", func() {
 		ginkgo.By("CRs in different groups (two CRDs) show up in OpenAPI documentation")
 		crdFoo, err := setupCRD(f, schemaFoo, "foo", "v1")
 		if err != nil {
@@ -249,7 +287,13 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() {
 		}
 	})
 
-	ginkgo.It("works for multiple CRDs of same group but different versions", func() {
+	/*
+		Release: v1.16
+		Testname: Custom Resource OpenAPI Publish, varying versions
+		Description: Register a custom resource definition with multiple versions; OpenAPI definitions MUST be published
+		for custom resource definitions.
+	*/
+	framework.ConformanceIt("works for multiple CRDs of same group but different versions", func() {
 		ginkgo.By("CRs in the same group but different versions (one multiversion CRD) show up in OpenAPI documentation")
 		crdMultiVer, err := setupCRD(f, schemaFoo, "multi-ver", "v2", "v3")
 		if err != nil {
@@ -291,7 +335,13 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() {
 		}
 	})
 
-	ginkgo.It("works for multiple CRDs of same group and version but different kinds", func() {
+	/*
+		Release: v1.16
+		Testname: Custom Resource OpenAPI Publish, varying kinds
+		Description: Register multiple custom resource definitions in the same group and version but spanning different kinds;
+		OpenAPI definitions MUST be published for custom resource definitions.
+	*/
+	framework.ConformanceIt("works for multiple CRDs of same group and version but different kinds", func() {
 		ginkgo.By("CRs in the same group and version but different kinds (two CRDs) show up in OpenAPI documentation")
 		crdFoo, err := setupCRD(f, schemaFoo, "common-group", "v6")
 		if err != nil {
@@ -318,7 +368,14 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() {
 		}
 	})
 
-	ginkgo.It("updates the published spec when one versin gets renamed", func() {
+	/*
+		Release: v1.16
+		Testname: Custom Resource OpenAPI Publish, version rename
+		Description: Register a custom resource definition with multiple versions; OpenAPI definitions MUST be published
+		for custom resource definitions. Rename one of the versions of the custom resource definition via a patch;
+		OpenAPI definitions MUST update to reflect the rename.
+	*/
+	framework.ConformanceIt("updates the published spec when one version gets renamed", func() {
 		ginkgo.By("set up a multi version CRD")
 		crdMultiVer, err := setupCRD(f, schemaFoo, "multi-ver", "v2", "v3")
 		if err != nil {
@@ -362,7 +419,14 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() {
 		}
 	})
 
-	ginkgo.It("removes definition from spec when one versin gets changed to not be served", func() {
+	/*
+		Release: v1.16
+		Testname: Custom Resource OpenAPI Publish, stop serving version
+		Description: Register a custom resource definition with multiple versions. OpenAPI definitions MUST be published
+		for custom resource definitions. Update the custom resource definition to not serve one of the versions. OpenAPI
+		definitions MUST be updated to not contain the version that is no longer served.
+	*/
+	framework.ConformanceIt("removes definition from spec when one version gets changed to not be served", func() {
 		ginkgo.By("set up a multi version CRD")
 		crd, err := setupCRD(f, schemaFoo, "multi-to-single-ver", "v5", "v6alpha1")
 		if err != nil {

test/e2e/apimachinery/crd_watch.go

@@ -35,17 +35,18 @@ import (
 	"github.com/onsi/ginkgo"
 )
 
-var _ = SIGDescribe("CustomResourceDefinition Watch", func() {
+var _ = SIGDescribe("CustomResourceDefinition Watch [Privileged:ClusterAdmin]", func() {
 
 	f := framework.NewDefaultFramework("crd-watch")
 
 	ginkgo.Context("CustomResourceDefinition Watch", func() {
 		/*
-			   	   Testname: crd-watch
+			Release: v1.16
-			   	   Description: Create a Custom Resource Definition and make sure
+			Testname: Custom Resource Definition, watch
-				   watches observe events on create/delete.
+			Description: Create a Custom Resource Definition. Attempt to watch it; the watch MUST observe create,
+			modify and delete events.
 		*/
-		ginkgo.It("watch on custom resource definition objects", func() {
+		framework.ConformanceIt("watch on custom resource definition objects", func() {
 
 			const (
 				watchCRNameA = "name1"

test/e2e/apimachinery/custom_resource_definition.go

@@ -35,7 +35,7 @@ import (
 	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
 )
 
-var _ = SIGDescribe("CustomResourceDefinition resources", func() {
+var _ = SIGDescribe("CustomResourceDefinition resources [Privileged:ClusterAdmin]", func() {
 
 	framework.NewDefaultFramework("custom-resource-definition")
 
@@ -43,7 +43,9 @@ var _ = SIGDescribe("CustomResourceDefinition resources", func() {
 		/*
 			Release : v1.9
 			Testname: Custom Resource Definition, create
-			Description: Create a API extension client, define a random custom resource definition, create the custom resource. API server MUST be able to create the custom resource.
+			Description: Create a API extension client and define a random custom resource definition.
+			Create the custom resource definition and then delete it. The creation and deletion MUST
+			be successful.
 		*/
 		framework.ConformanceIt("creating/deleting custom resource definition objects works ", func() {
 
@@ -67,9 +69,12 @@ var _ = SIGDescribe("CustomResourceDefinition resources", func() {
 		/*
 			Release : v1.16
 			Testname: Custom Resource Definition, list
-			Description: Create a API extension client, define 10 random custom resource definitions and list them using a label selector. API server MUST be able to list the custom resource definitions and delete them via delete collection.
+			Description: Create a API extension client, define 10 labeled custom resource definitions and list them using
+			a label selector; the list result MUST contain only the labeled custom resource definitions. Delete the labeled
+			custom resource definitions via delete collection; the delete MUST be successful and MUST delete only the
+			labeled custom resource definitions.
 		*/
-		ginkgo.It("listing custom resource definition objects works ", func() {
+		framework.ConformanceIt("listing custom resource definition objects works ", func() {
 			testListSize := 10
 			config, err := framework.LoadConfig()
 			framework.ExpectNoError(err, "loading config")
@@ -126,9 +131,10 @@ var _ = SIGDescribe("CustomResourceDefinition resources", func() {
 		/*
 			Release : v1.16
 			Testname: Custom Resource Definition, status sub-resource
-			Description: Create a API extension client, create a custom resource definition and then read, update and patch its status sub-resource. API server MUST be able to perform the operations against the status sub-resource.
+			Description: Create a custom resource definition. Attempt to read, update and patch its status sub-resource;
+			all mutating sub-resource operations MUST be visible to subsequent reads.
 		*/
-		ginkgo.It("getting/updating/patching custom resource definition status sub-resource works ", func() {
+		framework.ConformanceIt("getting/updating/patching custom resource definition status sub-resource works ", func() {
 			config, err := framework.LoadConfig()
 			framework.ExpectNoError(err, "loading config")
 			apiExtensionClient, err := clientset.NewForConfig(config)