Merge pull request #81497 from jpbetz/crd-e2e-v1

Upgrade e2e tests to use CRD v1 APIs

staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD

@@ -89,6 +89,7 @@ go_library(
     importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/test/integration",
     importpath = "k8s.io/apiextensions-apiserver/test/integration",
     deps = [
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",

staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/BUILD

@@ -32,6 +32,7 @@ go_library(
         "//staging/src/k8s.io/client-go/restmapper:go_default_library",
         "//staging/src/k8s.io/client-go/scale:go_default_library",
         "//vendor/github.com/pborman/uuid:go_default_library",
+        "//vendor/k8s.io/utils/pointer:go_default_library",
     ],
 )
 

staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/resources.go

@@ -20,6 +20,8 @@ import (
 	"fmt"
 	"time"
 
+	"k8s.io/utils/pointer"
+
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
@@ -81,6 +83,36 @@ func NewNoxuCustomResourceDefinition(scope apiextensionsv1beta1.ResourceScope) *
 	}
 }
 
+// NewNoxuV1CustomResourceDefinition returns a WishIHadChosenNoxu CRD.
+func NewNoxuV1CustomResourceDefinition(scope apiextensionsv1.ResourceScope) *apiextensionsv1.CustomResourceDefinition {
+	return &apiextensionsv1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"},
+		Spec: apiextensionsv1.CustomResourceDefinitionSpec{
+			Group: "mygroup.example.com",
+			Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{
+				Name:    "v1beta1",
+				Served:  true,
+				Storage: true,
+				Schema: &apiextensionsv1.CustomResourceValidation{
+					OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
+						XPreserveUnknownFields: pointer.BoolPtr(true),
+						Type:                   "object",
+					},
+				},
+			}},
+			Names: apiextensionsv1.CustomResourceDefinitionNames{
+				Plural:     "noxus",
+				Singular:   "nonenglishnoxu",
+				Kind:       "WishIHadChosenNoxu",
+				ShortNames: []string{"foo", "bar", "abc", "def"},
+				ListKind:   "NoxuItemList",
+				Categories: []string{"all"},
+			},
+			Scope: scope,
+		},
+	}
+}
+
 // NewVersionedNoxuInstance returns a WishIHadChosenNoxu instance for a given version
 func NewVersionedNoxuInstance(namespace, name, version string) *unstructured.Unstructured {
 	return &unstructured.Unstructured{
@@ -211,6 +243,19 @@ func servedVersions(crd *apiextensionsv1beta1.CustomResourceDefinition) []string
 	return versions
 }
 
+func servedV1Versions(crd *apiextensionsv1.CustomResourceDefinition) []string {
+	if len(crd.Spec.Versions) == 0 {
+		return []string{}
+	}
+	var versions []string
+	for _, v := range crd.Spec.Versions {
+		if v.Served {
+			versions = append(versions, v.Name)
+		}
+	}
+	return versions
+}
+
 func existsInDiscovery(crd *apiextensionsv1beta1.CustomResourceDefinition, apiExtensionsClient clientset.Interface, version string) (bool, error) {
 	groupResource, err := apiExtensionsClient.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + version)
 	if err != nil {
@@ -227,6 +272,22 @@ func existsInDiscovery(crd *apiextensionsv1beta1.CustomResourceDefinition, apiEx
 	return false, nil
 }
 
+func existsInDiscoveryV1(crd *apiextensionsv1.CustomResourceDefinition, apiExtensionsClient clientset.Interface, version string) (bool, error) {
+	groupResource, err := apiExtensionsClient.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + version)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	for _, g := range groupResource.APIResources {
+		if g.Name == crd.Spec.Names.Plural {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
 // CreateNewCustomResourceDefinitionWatchUnsafe creates the CRD and makes sure
 // the apiextension apiserver has installed the CRD. But it's not safe to watch
 // the created CR. Please call CreateNewCustomResourceDefinition if you need to
@@ -428,6 +489,23 @@ func DeleteCustomResourceDefinition(crd *apiextensionsv1beta1.CustomResourceDefi
 	return nil
 }
 
+// DeleteV1CustomResourceDefinition deletes a CRD and waits until it disappears from discovery.
+func DeleteV1CustomResourceDefinition(crd *apiextensionsv1.CustomResourceDefinition, apiExtensionsClient clientset.Interface) error {
+	if err := apiExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(crd.Name, nil); err != nil {
+		return err
+	}
+	for _, version := range servedV1Versions(crd) {
+		err := wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) {
+			exists, err := existsInDiscoveryV1(crd, apiExtensionsClient, version)
+			return !exists, err
+		})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // DeleteCustomResourceDefinitions deletes all CRD matching the provided deleteListOpts and waits until all the CRDs disappear from discovery.
 func DeleteCustomResourceDefinitions(deleteListOpts metav1.ListOptions, apiExtensionsClient clientset.Interface) error {
 	list, err := apiExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().List(deleteListOpts)
@@ -451,6 +529,29 @@ func DeleteCustomResourceDefinitions(deleteListOpts metav1.ListOptions, apiExten
 	return nil
 }
 
+// DeleteV1CustomResourceDefinitions deletes all CRD matching the provided deleteListOpts and waits until all the CRDs disappear from discovery.
+func DeleteV1CustomResourceDefinitions(deleteListOpts metav1.ListOptions, apiExtensionsClient clientset.Interface) error {
+	list, err := apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().List(deleteListOpts)
+	if err != nil {
+		return err
+	}
+	if err = apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().DeleteCollection(nil, deleteListOpts); err != nil {
+		return err
+	}
+	for _, crd := range list.Items {
+		for _, version := range servedV1Versions(&crd) {
+			err := wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) {
+				exists, err := existsInDiscoveryV1(&crd, apiExtensionsClient, version)
+				return !exists, err
+			})
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 // CreateNewVersionedScaleClient returns a scale client.
 func CreateNewVersionedScaleClient(crd *apiextensionsv1beta1.CustomResourceDefinition, config *rest.Config, version string) (scale.ScalesGetter, error) {
 	discoveryClient, err := discovery.NewDiscoveryClientForConfig(config)

staging/src/k8s.io/apiextensions-apiserver/test/integration/helpers.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"testing"
 
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -95,6 +96,25 @@ func UpdateCustomResourceDefinitionWithRetry(client clientset.Interface, name st
 	return nil, fmt.Errorf("too many retries after conflicts updating CustomResourceDefinition %q", name)
 }
 
+// UpdateV1CustomResourceDefinitionWithRetry updates a CRD, retrying up to 5 times on version conflict errors.
+func UpdateV1CustomResourceDefinitionWithRetry(client clientset.Interface, name string, update func(*apiextensionsv1.CustomResourceDefinition)) (*apiextensionsv1.CustomResourceDefinition, error) {
+	for i := 0; i < 5; i++ {
+		crd, err := client.ApiextensionsV1().CustomResourceDefinitions().Get(name, metav1.GetOptions{})
+		if err != nil {
+			return nil, fmt.Errorf("failed to get CustomResourceDefinition %q: %v", name, err)
+		}
+		update(crd)
+		crd, err = client.ApiextensionsV1().CustomResourceDefinitions().Update(crd)
+		if err == nil {
+			return crd, nil
+		}
+		if !errors.IsConflict(err) {
+			return nil, fmt.Errorf("failed to update CustomResourceDefinition %q: %v", name, err)
+		}
+	}
+	return nil, fmt.Errorf("too many retries after conflicts updating CustomResourceDefinition %q", name)
+}
+
 // getSchemaForVersion returns the validation schema for given version in given CRD.
 func getSchemaForVersion(crd *apiextensionsv1beta1.CustomResourceDefinition, version string) (*apiextensionsv1beta1.CustomResourceValidation, error) {
 	if !hasPerVersionSchema(crd.Spec.Versions) {

test/e2e/apimachinery/BUILD

@@ -41,6 +41,7 @@ go_library(
         "//staging/src/k8s.io/api/rbac/v1:go_default_library",
         "//staging/src/k8s.io/api/scheduling/v1:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",

test/e2e/apimachinery/crd_conversion_webhook.go

@@ -39,7 +39,7 @@ import (
 	imageutils "k8s.io/kubernetes/test/utils/image"
 	"k8s.io/utils/pointer"
 
-	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apiextensions-apiserver/test/integration"
 
 	// ensure libs have a chance to initialize
@@ -56,15 +56,15 @@ const (
 
 var serverCRDConversionWebhookVersion = utilversion.MustParseSemantic("v1.13.0-alpha")
 
-var apiVersions = []v1beta1.CustomResourceDefinitionVersion{
+var apiVersions = []apiextensionsv1.CustomResourceDefinitionVersion{
 	{
 		Name:    "v1",
 		Served:  true,
 		Storage: true,
-		Schema: &v1beta1.CustomResourceValidation{
+		Schema: &apiextensionsv1.CustomResourceValidation{
-			OpenAPIV3Schema: &v1beta1.JSONSchemaProps{
+			OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
 				Type: "object",
-				Properties: map[string]v1beta1.JSONSchemaProps{
+				Properties: map[string]apiextensionsv1.JSONSchemaProps{
 					"hostPort": {Type: "string"},
 				},
 			},
@@ -74,10 +74,10 @@ var apiVersions = []v1beta1.CustomResourceDefinitionVersion{
 		Name:    "v2",
 		Served:  true,
 		Storage: false,
-		Schema: &v1beta1.CustomResourceValidation{
+		Schema: &apiextensionsv1.CustomResourceValidation{
-			OpenAPIV3Schema: &v1beta1.JSONSchemaProps{
+			OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
 				Type: "object",
-				Properties: map[string]v1beta1.JSONSchemaProps{
+				Properties: map[string]apiextensionsv1.JSONSchemaProps{
 					"host": {Type: "string"},
 					"port": {Type: "string"},
 				},
@@ -86,15 +86,15 @@ var apiVersions = []v1beta1.CustomResourceDefinitionVersion{
 	},
 }
 
-var alternativeAPIVersions = []v1beta1.CustomResourceDefinitionVersion{
+var alternativeAPIVersions = []apiextensionsv1.CustomResourceDefinitionVersion{
 	{
 		Name:    "v1",
 		Served:  true,
 		Storage: false,
-		Schema: &v1beta1.CustomResourceValidation{
+		Schema: &apiextensionsv1.CustomResourceValidation{
-			OpenAPIV3Schema: &v1beta1.JSONSchemaProps{
+			OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
 				Type: "object",
-				Properties: map[string]v1beta1.JSONSchemaProps{
+				Properties: map[string]apiextensionsv1.JSONSchemaProps{
 					"hostPort": {Type: "string"},
 				},
 			},
@@ -104,10 +104,10 @@ var alternativeAPIVersions = []v1beta1.CustomResourceDefinitionVersion{
 		Name:    "v2",
 		Served:  true,
 		Storage: true,
-		Schema: &v1beta1.CustomResourceValidation{
+		Schema: &apiextensionsv1.CustomResourceValidation{
-			OpenAPIV3Schema: &v1beta1.JSONSchemaProps{
+			OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
 				Type: "object",
-				Properties: map[string]v1beta1.JSONSchemaProps{
+				Properties: map[string]apiextensionsv1.JSONSchemaProps{
 					"host": {Type: "string"},
 					"port": {Type: "string"},
 				},
@@ -142,21 +142,24 @@ var _ = SIGDescribe("CustomResourceConversionWebhook", func() {
 	})
 
 	ginkgo.It("Should be able to convert from CR v1 to CR v2", func() {
-		testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *v1beta1.CustomResourceDefinition) {
+		testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *apiextensionsv1.CustomResourceDefinition) {
 			crd.Spec.Versions = apiVersions
-			crd.Spec.Conversion = &v1beta1.CustomResourceConversion{
+			crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{
-				Strategy: v1beta1.WebhookConverter,
+				Strategy: apiextensionsv1.WebhookConverter,
-				WebhookClientConfig: &v1beta1.WebhookClientConfig{
+				Webhook: &apiextensionsv1.WebhookConversion{
-					CABundle: context.signingCert,
+					ClientConfig: &apiextensionsv1.WebhookClientConfig{
-					Service: &v1beta1.ServiceReference{
+						CABundle: context.signingCert,
-						Namespace: f.Namespace.Name,
+						Service: &apiextensionsv1.ServiceReference{
-						Name:      serviceCRDName,
+							Namespace: f.Namespace.Name,
-						Path:      pointer.StringPtr("/crdconvert"),
+							Name:      serviceCRDName,
-						Port:      pointer.Int32Ptr(serviceCRDPort),
+							Path:      pointer.StringPtr("/crdconvert"),
+							Port:      pointer.Int32Ptr(serviceCRDPort),
+						},
 					},
+					ConversionReviewVersions: []string{"v1", "v1beta1"},
 				},
 			}
-			crd.Spec.PreserveUnknownFields = pointer.BoolPtr(false)
+			crd.Spec.PreserveUnknownFields = false
 		})
 		if err != nil {
 			return
@@ -166,21 +169,24 @@ var _ = SIGDescribe("CustomResourceConversionWebhook", func() {
 	})
 
 	ginkgo.It("Should be able to convert a non homogeneous list of CRs", func() {
-		testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *v1beta1.CustomResourceDefinition) {
+		testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *apiextensionsv1.CustomResourceDefinition) {
 			crd.Spec.Versions = apiVersions
-			crd.Spec.Conversion = &v1beta1.CustomResourceConversion{
+			crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{
-				Strategy: v1beta1.WebhookConverter,
+				Strategy: apiextensionsv1.WebhookConverter,
-				WebhookClientConfig: &v1beta1.WebhookClientConfig{
+				Webhook: &apiextensionsv1.WebhookConversion{
-					CABundle: context.signingCert,
+					ClientConfig: &apiextensionsv1.WebhookClientConfig{
-					Service: &v1beta1.ServiceReference{
+						CABundle: context.signingCert,
-						Namespace: f.Namespace.Name,
+						Service: &apiextensionsv1.ServiceReference{
-						Name:      serviceCRDName,
+							Namespace: f.Namespace.Name,
-						Path:      pointer.StringPtr("/crdconvert"),
+							Name:      serviceCRDName,
-						Port:      pointer.Int32Ptr(serviceCRDPort),
+							Path:      pointer.StringPtr("/crdconvert"),
+							Port:      pointer.Int32Ptr(serviceCRDPort),
+						},
 					},
+					ConversionReviewVersions: []string{"v1", "v1beta1"},
 				},
 			}
-			crd.Spec.PreserveUnknownFields = pointer.BoolPtr(false)
+			crd.Spec.PreserveUnknownFields = false
 		})
 		if err != nil {
 			return
@@ -339,7 +345,7 @@ func deployCustomResourceWebhookAndService(f *framework.Framework, image string,
 	framework.ExpectNoError(err, "waiting for service %s/%s have %d endpoint", namespace, serviceCRDName, 1)
 }
 
-func verifyV1Object(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, obj *unstructured.Unstructured) {
+func verifyV1Object(f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, obj *unstructured.Unstructured) {
 	gomega.Expect(obj.GetAPIVersion()).To(gomega.BeEquivalentTo(crd.Spec.Group + "/v1"))
 	hostPort, exists := obj.Object["hostPort"]
 	gomega.Expect(exists).To(gomega.BeTrue())
@@ -350,7 +356,7 @@ func verifyV1Object(f *framework.Framework, crd *v1beta1.CustomResourceDefinitio
 	gomega.Expect(portExists).To(gomega.BeFalse())
 }
 
-func verifyV2Object(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, obj *unstructured.Unstructured) {
+func verifyV2Object(f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, obj *unstructured.Unstructured) {
 	gomega.Expect(obj.GetAPIVersion()).To(gomega.BeEquivalentTo(crd.Spec.Group + "/v2"))
 	_, hostPortExists := obj.Object["hostPort"]
 	gomega.Expect(hostPortExists).To(gomega.BeFalse())
@@ -362,7 +368,7 @@ func verifyV2Object(f *framework.Framework, crd *v1beta1.CustomResourceDefinitio
 	gomega.Expect(port).To(gomega.BeEquivalentTo("8080"))
 }
 
-func testCustomResourceConversionWebhook(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, customResourceClients map[string]dynamic.ResourceInterface) {
+func testCustomResourceConversionWebhook(f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClients map[string]dynamic.ResourceInterface) {
 	name := "cr-instance-1"
 	ginkgo.By("Creating a v1 custom resource")
 	crInstance := &unstructured.Unstructured{
@@ -380,6 +386,7 @@ func testCustomResourceConversionWebhook(f *framework.Framework, crd *v1beta1.Cu
 	gomega.Expect(err).To(gomega.BeNil())
 	ginkgo.By("v2 custom resource should be converted")
 	v2crd, err := customResourceClients["v2"].Get(name, metav1.GetOptions{})
+	framework.ExpectNoError(err, "Getting v2 of custom resource %s", name)
 	verifyV2Object(f, crd, v2crd)
 }
 
@@ -404,7 +411,7 @@ func testCRListConversion(f *framework.Framework, testCrd *crd.TestCrd) {
 	gomega.Expect(err).To(gomega.BeNil())
 
 	// Now cr-instance-1 is stored as v1. lets change storage version
-	crd, err = integration.UpdateCustomResourceDefinitionWithRetry(testCrd.APIExtensionClient, crd.Name, func(c *v1beta1.CustomResourceDefinition) {
+	crd, err = integration.UpdateV1CustomResourceDefinitionWithRetry(testCrd.APIExtensionClient, crd.Name, func(c *apiextensionsv1.CustomResourceDefinition) {
 		c.Spec.Versions = alternativeAPIVersions
 	})
 	gomega.Expect(err).To(gomega.BeNil())

test/e2e/apimachinery/crd_publish_openapi.go

@@ -27,9 +27,10 @@ import (
 
 	"github.com/go-openapi/spec"
 	"github.com/onsi/ginkgo"
+	"k8s.io/utils/pointer"
 
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
-	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apiextensions-apiserver/pkg/apiserver/validation"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -337,8 +338,11 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() {
 		}
 
 		ginkgo.By("rename a version")
-		patch := []byte(`{"spec":{"versions":[{"name":"v2","served":true,"storage":true},{"name":"v4","served":true,"storage":false}]}}`)
+		patch := []byte(`[
-		crdMultiVer.Crd, err = crdMultiVer.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crdMultiVer.Crd.Name, types.MergePatchType, patch)
+			{"op":"test","path":"/spec/versions/1/name","value":"v3"},
+			{"op": "replace", "path": "/spec/versions/1/name", "value": "v4"}
+		]`)
+		crdMultiVer.Crd, err = crdMultiVer.APIExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Patch(crdMultiVer.Crd.Name, types.JSONPatchType, patch)
 		if err != nil {
 			e2elog.Failf("%v", err)
 		}
@@ -379,12 +383,12 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() {
 		}
 
 		ginkgo.By("mark a version not serverd")
-		crd.Crd, err = crd.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Crd.Name, metav1.GetOptions{})
+		crd.Crd, err = crd.APIExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Get(crd.Crd.Name, metav1.GetOptions{})
 		if err != nil {
 			e2elog.Failf("%v", err)
 		}
 		crd.Crd.Spec.Versions[1].Served = false
-		crd.Crd, err = crd.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd.Crd)
+		crd.Crd, err = crd.APIExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Update(crd.Crd)
 		if err != nil {
 			e2elog.Failf("%v", err)
 		}
@@ -415,35 +419,42 @@ func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, version
 }
 
 func setupCRDAndVerifySchema(f *framework.Framework, schema, expect []byte, groupSuffix string, versions ...string) (*crd.TestCrd, error) {
-	group := fmt.Sprintf("%s-test-%s.k8s.io", f.BaseName, groupSuffix)
+	group := fmt.Sprintf("%s-test-%s.example.com", f.BaseName, groupSuffix)
 	if len(versions) == 0 {
 		return nil, fmt.Errorf("require at least one version for CRD")
 	}
 
-	props := &v1beta1.JSONSchemaProps{}
+	props := &apiextensionsv1.JSONSchemaProps{}
 	if schema != nil {
 		if err := yaml.Unmarshal(schema, props); err != nil {
 			return nil, err
 		}
 	}
 
-	crd, err := crd.CreateMultiVersionTestCRD(f, group, func(crd *v1beta1.CustomResourceDefinition) {
+	crd, err := crd.CreateMultiVersionTestCRD(f, group, func(crd *apiextensionsv1.CustomResourceDefinition) {
-		var apiVersions []v1beta1.CustomResourceDefinitionVersion
+		var apiVersions []apiextensionsv1.CustomResourceDefinitionVersion
 		for i, version := range versions {
-			apiVersions = append(apiVersions, v1beta1.CustomResourceDefinitionVersion{
+			version := apiextensionsv1.CustomResourceDefinitionVersion{
 				Name:    version,
 				Served:  true,
 				Storage: i == 0,
-			})
+			}
+			// set up validation when input schema isn't nil
+			if schema != nil {
+				version.Schema = &apiextensionsv1.CustomResourceValidation{
+					OpenAPIV3Schema: props,
+				}
+			} else {
+				version.Schema = &apiextensionsv1.CustomResourceValidation{
+					OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
+						XPreserveUnknownFields: pointer.BoolPtr(true),
+						Type:                   "object",
+					},
+				}
+			}
+			apiVersions = append(apiVersions, version)
 		}
 		crd.Spec.Versions = apiVersions
-
-		// set up validation when input schema isn't nil
-		if schema != nil {
-			crd.Spec.Validation = &v1beta1.CustomResourceValidation{
-				OpenAPIV3Schema: props,
-			}
-		}
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to create CRD: %v", err)
@@ -573,12 +584,12 @@ func waitForOpenAPISchema(c k8sclientset.Interface, pred func(*spec.Swagger) (bo
 
 // convertJSONSchemaProps converts JSONSchemaProps in YAML to spec.Schema
 func convertJSONSchemaProps(in []byte, out *spec.Schema) error {
-	external := v1beta1.JSONSchemaProps{}
+	external := apiextensionsv1.JSONSchemaProps{}
 	if err := yaml.UnmarshalStrict(in, &external); err != nil {
 		return err
 	}
 	internal := apiextensions.JSONSchemaProps{}
-	if err := v1beta1.Convert_v1beta1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(&external, &internal, nil); err != nil {
+	if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(&external, &internal, nil); err != nil {
 		return err
 	}
 	if err := validation.ConvertJSONSchemaProps(&internal, out); err != nil {

test/e2e/apimachinery/crd_watch.go

@@ -19,7 +19,7 @@ package apimachinery
 import (
 	"fmt"
 
-	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apiextensions-apiserver/test/integration/fixtures"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -63,21 +63,22 @@ var _ = SIGDescribe("CustomResourceDefinition Watch", func() {
 				e2elog.Failf("failed to initialize apiExtensionClient: %v", err)
 			}
 
-			noxuDefinition := fixtures.NewNoxuCustomResourceDefinition(apiextensionsv1beta1.ClusterScoped)
+			noxuDefinition := fixtures.NewNoxuV1CustomResourceDefinition(apiextensionsv1.ClusterScoped)
-			noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, f.DynamicClient)
+			noxuDefinition, err = fixtures.CreateNewV1CustomResourceDefinition(noxuDefinition, apiExtensionClient, f.DynamicClient)
 			if err != nil {
 				e2elog.Failf("failed to create CustomResourceDefinition: %v", err)
 			}
 
 			defer func() {
-				err = fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient)
+				err = fixtures.DeleteV1CustomResourceDefinition(noxuDefinition, apiExtensionClient)
 				if err != nil {
 					e2elog.Failf("failed to delete CustomResourceDefinition: %v", err)
 				}
 			}()
 
 			ns := ""
-			noxuResourceClient := newNamespacedCustomResourceClient(ns, f.DynamicClient, noxuDefinition)
+			noxuResourceClient, err := newNamespacedCustomResourceClient(ns, f.DynamicClient, noxuDefinition)
+			framework.ExpectNoError(err, "creating custom resource client")
 
 			watchA, err := watchCRWithName(noxuResourceClient, watchCRNameA)
 			framework.ExpectNoError(err, "failed to watch custom resource: %s", watchCRNameA)
@@ -124,7 +125,7 @@ func watchCRWithName(crdResourceClient dynamic.ResourceInterface, name string) (
 	)
 }
 
-func instantiateCustomResource(instanceToCreate *unstructured.Unstructured, client dynamic.ResourceInterface, definition *apiextensionsv1beta1.CustomResourceDefinition) (*unstructured.Unstructured, error) {
+func instantiateCustomResource(instanceToCreate *unstructured.Unstructured, client dynamic.ResourceInterface, definition *apiextensionsv1.CustomResourceDefinition) (*unstructured.Unstructured, error) {
 	createdInstance, err := client.Create(instanceToCreate, metav1.CreateOptions{})
 	if err != nil {
 		return nil, err
@@ -141,7 +142,10 @@ func instantiateCustomResource(instanceToCreate *unstructured.Unstructured, clie
 	if err != nil {
 		return nil, err
 	}
-	if e, a := definition.Spec.Group+"/"+definition.Spec.Version, createdTypeMeta.GetAPIVersion(); e != a {
+	if len(definition.Spec.Versions) != 1 {
+		return nil, fmt.Errorf("expected exactly one version, got %v", definition.Spec.Versions)
+	}
+	if e, a := definition.Spec.Group+"/"+definition.Spec.Versions[0].Name, createdTypeMeta.GetAPIVersion(); e != a {
 		return nil, fmt.Errorf("expected %v, got %v", e, a)
 	}
 	if e, a := definition.Spec.Names.Kind, createdTypeMeta.GetKind(); e != a {
@@ -154,12 +158,15 @@ func deleteCustomResource(client dynamic.ResourceInterface, name string) error {
 	return client.Delete(name, &metav1.DeleteOptions{})
 }
 
-func newNamespacedCustomResourceClient(ns string, client dynamic.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) dynamic.ResourceInterface {
+func newNamespacedCustomResourceClient(ns string, client dynamic.Interface, crd *apiextensionsv1.CustomResourceDefinition) (dynamic.ResourceInterface, error) {
-	gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: crd.Spec.Version, Resource: crd.Spec.Names.Plural}
+	if len(crd.Spec.Versions) != 1 {
-
+		return nil, fmt.Errorf("expected exactly one version, got %v", crd.Spec.Versions)
-	if crd.Spec.Scope != apiextensionsv1beta1.ClusterScoped {
-		return client.Resource(gvr).Namespace(ns)
 	}
-	return client.Resource(gvr)
+	gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: crd.Spec.Versions[0].Name, Resource: crd.Spec.Names.Plural}
+
+	if crd.Spec.Scope != apiextensionsv1.ClusterScoped {
+		return client.Resource(gvr).Namespace(ns), nil
+	}
+	return client.Resource(gvr), nil
 
 }

test/e2e/apimachinery/webhook.go

@@ -28,7 +28,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
-	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	crdclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -195,22 +195,24 @@ var _ = SIGDescribe("AdmissionWebhook", func() {
 
 	ginkgo.It("Should mutate custom resource with pruning", func() {
 		const prune = true
-		testcrd, err := createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f, func(crd *apiextensionsv1beta1.CustomResourceDefinition) {
+		testcrd, err := createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f, func(crd *apiextensionsv1.CustomResourceDefinition) {
-			crd.Spec.PreserveUnknownFields = pointer.BoolPtr(false)
+			crd.Spec.PreserveUnknownFields = false
-			crd.Spec.Validation = &apiextensionsv1beta1.CustomResourceValidation{
+			for i := range crd.Spec.Versions {
-				OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
+				crd.Spec.Versions[i].Schema = &apiextensionsv1.CustomResourceValidation{
-					Type: "object",
+					OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
-					Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
+						Type: "object",
-						"data": {
+						Properties: map[string]apiextensionsv1.JSONSchemaProps{
-							Type: "object",
+							"data": {
-							Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
+								Type: "object",
-								"mutation-start":   {Type: "string"},
+								Properties: map[string]apiextensionsv1.JSONSchemaProps{
-								"mutation-stage-1": {Type: "string"},
+									"mutation-start":   {Type: "string"},
-								// mutation-stage-2 is intentionally missing such that it is pruned
+									"mutation-stage-1": {Type: "string"},
+									// mutation-stage-2 is intentionally missing such that it is pruned
+								},
 							},
 						},
 					},
-				},
+				}
 			}
 		})
 		if err != nil {
@@ -733,7 +735,7 @@ func registerWebhookForAttachingPod(f *framework.Framework, configName string, c
 					CABundle: context.signingCert,
 				},
 				SideEffects:             &sideEffectsNone,
-				AdmissionReviewVersions: []string{"v1beta1"},
+				AdmissionReviewVersions: []string{"v1", "v1beta1"},
 				// Scope the webhook to just this namespace
 				NamespaceSelector: &metav1.LabelSelector{
 					MatchLabels: map[string]string{f.UniqueName: "true"},
@@ -821,7 +823,7 @@ func registerMutatingWebhookForPod(f *framework.Framework, configName string, co
 					CABundle: context.signingCert,
 				},
 				SideEffects:             &sideEffectsNone,
-				AdmissionReviewVersions: []string{"v1beta1"},
+				AdmissionReviewVersions: []string{"v1", "v1beta1"},
 				// Scope the webhook to just this namespace
 				NamespaceSelector: &metav1.LabelSelector{
 					MatchLabels: map[string]string{f.UniqueName: "true"},
@@ -1042,7 +1044,7 @@ func failingWebhook(namespace, name string) admissionregistrationv1.ValidatingWe
 			CABundle: nil,
 		},
 		SideEffects:             &sideEffectsNone,
-		AdmissionReviewVersions: []string{"v1beta1"},
+		AdmissionReviewVersions: []string{"v1", "v1beta1"},
 	}
 }
 
@@ -1150,7 +1152,7 @@ func registerValidatingWebhookForWebhookConfigurations(f *framework.Framework, c
 					CABundle: context.signingCert,
 				},
 				SideEffects:             &sideEffectsNone,
-				AdmissionReviewVersions: []string{"v1beta1"},
+				AdmissionReviewVersions: []string{"v1", "v1beta1"},
 				FailurePolicy:           &failurePolicy,
 				// Scope the webhook to just this namespace
 				NamespaceSelector: &metav1.LabelSelector{
@@ -1209,7 +1211,7 @@ func registerMutatingWebhookForWebhookConfigurations(f *framework.Framework, con
 					CABundle: context.signingCert,
 				},
 				SideEffects:             &sideEffectsNone,
-				AdmissionReviewVersions: []string{"v1beta1"},
+				AdmissionReviewVersions: []string{"v1", "v1beta1"},
 				FailurePolicy:           &failurePolicy,
 				// Scope the webhook to just this namespace
 				NamespaceSelector: &metav1.LabelSelector{
@@ -1270,7 +1272,7 @@ func testWebhooksForWebhookConfigurations(f *framework.Framework, configName str
 					CABundle: nil,
 				},
 				SideEffects:             &sideEffectsNone,
-				AdmissionReviewVersions: []string{"v1beta1"},
+				AdmissionReviewVersions: []string{"v1", "v1beta1"},
 				FailurePolicy:           &failurePolicy,
 				// Scope the webhook to just this namespace
 				NamespaceSelector: &metav1.LabelSelector{
@@ -1324,7 +1326,7 @@ func testWebhooksForWebhookConfigurations(f *framework.Framework, configName str
 					CABundle: nil,
 				},
 				SideEffects:             &sideEffectsNone,
-				AdmissionReviewVersions: []string{"v1beta1"},
+				AdmissionReviewVersions: []string{"v1", "v1beta1"},
 				FailurePolicy:           &failurePolicy,
 				// Scope the webhook to just this namespace
 				NamespaceSelector: &metav1.LabelSelector{
@@ -1542,7 +1544,7 @@ func registerWebhookForCustomResource(f *framework.Framework, configName string,
 					CABundle: context.signingCert,
 				},
 				SideEffects:             &sideEffectsNone,
-				AdmissionReviewVersions: []string{"v1beta1"},
+				AdmissionReviewVersions: []string{"v1", "v1beta1"},
 				// Scope the webhook to just this namespace
 				NamespaceSelector: &metav1.LabelSelector{
 					MatchLabels: map[string]string{f.UniqueName: "true"},
@@ -1591,7 +1593,7 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, configName
 					CABundle: context.signingCert,
 				},
 				SideEffects:             &sideEffectsNone,
-				AdmissionReviewVersions: []string{"v1beta1"},
+				AdmissionReviewVersions: []string{"v1", "v1beta1"},
 				// Scope the webhook to just this namespace
 				NamespaceSelector: &metav1.LabelSelector{
 					MatchLabels: map[string]string{f.UniqueName: "true"},
@@ -1617,7 +1619,7 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, configName
 					CABundle: context.signingCert,
 				},
 				SideEffects:             &sideEffectsNone,
-				AdmissionReviewVersions: []string{"v1beta1"},
+				AdmissionReviewVersions: []string{"v1", "v1beta1"},
 				// Scope the webhook to just this namespace
 				NamespaceSelector: &metav1.LabelSelector{
 					MatchLabels: map[string]string{f.UniqueName: "true"},
@@ -1633,13 +1635,13 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, configName
 	return func() { client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(configName, nil) }
 }
 
-func testCustomResourceWebhook(f *framework.Framework, crd *apiextensionsv1beta1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface) {
+func testCustomResourceWebhook(f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface) {
 	ginkgo.By("Creating a custom resource that should be denied by the webhook")
 	crInstanceName := "cr-instance-1"
 	crInstance := &unstructured.Unstructured{
 		Object: map[string]interface{}{
 			"kind":       crd.Spec.Names.Kind,
-			"apiVersion": crd.Spec.Group + "/" + crd.Spec.Version,
+			"apiVersion": crd.Spec.Group + "/" + crd.Spec.Versions[0].Name,
 			"metadata": map[string]interface{}{
 				"name":      crInstanceName,
 				"namespace": f.Namespace.Name,
@@ -1657,13 +1659,13 @@ func testCustomResourceWebhook(f *framework.Framework, crd *apiextensionsv1beta1
 	}
 }
 
-func testBlockingCustomResourceDeletion(f *framework.Framework, crd *apiextensionsv1beta1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface) {
+func testBlockingCustomResourceDeletion(f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface) {
 	ginkgo.By("Creating a custom resource whose deletion would be denied by the webhook")
 	crInstanceName := "cr-instance-2"
 	crInstance := &unstructured.Unstructured{
 		Object: map[string]interface{}{
 			"kind":       crd.Spec.Names.Kind,
-			"apiVersion": crd.Spec.Group + "/" + crd.Spec.Version,
+			"apiVersion": crd.Spec.Group + "/" + crd.Spec.Versions[0].Name,
 			"metadata": map[string]interface{}{
 				"name":      crInstanceName,
 				"namespace": f.Namespace.Name,
@@ -1701,13 +1703,13 @@ func testBlockingCustomResourceDeletion(f *framework.Framework, crd *apiextensio
 
 }
 
-func testMutatingCustomResourceWebhook(f *framework.Framework, crd *apiextensionsv1beta1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface, prune bool) {
+func testMutatingCustomResourceWebhook(f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface, prune bool) {
 	ginkgo.By("Creating a custom resource that should be mutated by the webhook")
 	crName := "cr-instance-1"
 	cr := &unstructured.Unstructured{
 		Object: map[string]interface{}{
 			"kind":       crd.Spec.Names.Kind,
-			"apiVersion": crd.Spec.Group + "/" + crd.Spec.Version,
+			"apiVersion": crd.Spec.Group + "/" + crd.Spec.Versions[0].Name,
 			"metadata": map[string]interface{}{
 				"name":      crName,
 				"namespace": f.Namespace.Name,
@@ -1738,7 +1740,7 @@ func testMultiVersionCustomResourceWebhook(f *framework.Framework, testcrd *crd.
 	cr := &unstructured.Unstructured{
 		Object: map[string]interface{}{
 			"kind":       testcrd.Crd.Spec.Names.Kind,
-			"apiVersion": testcrd.Crd.Spec.Group + "/" + testcrd.Crd.Spec.Version,
+			"apiVersion": testcrd.Crd.Spec.Group + "/" + testcrd.Crd.Spec.Versions[0].Name,
 			"metadata": map[string]interface{}{
 				"name":      crName,
 				"namespace": f.Namespace.Name,
@@ -1752,8 +1754,29 @@ func testMultiVersionCustomResourceWebhook(f *framework.Framework, testcrd *crd.
 	framework.ExpectNoError(err, "failed to create custom resource %s in namespace: %s", crName, f.Namespace.Name)
 
 	ginkgo.By("Patching Custom Resource Definition to set v2 as storage")
-	apiVersionWithV2StoragePatch := fmt.Sprint(`{"spec": {"versions": [{"name": "v1", "storage": false, "served": true},{"name": "v2", "storage": true, "served": true}]}}`)
+	apiVersionWithV2StoragePatch := `{
-	_, err = testcrd.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(testcrd.Crd.Name, types.StrategicMergePatchType, []byte(apiVersionWithV2StoragePatch))
+		"spec": {
+		  "versions": [
+		    {
+			  "name": "v1",
+			  "storage": false,
+			  "served": true,
+			  "schema": {
+			    "openAPIV3Schema": {"x-kubernetes-preserve-unknown-fields": true, "type": "object"}
+			  }
+            },
+		    {
+			  "name": "v2",
+			  "storage": true,
+			  "served": true,
+			  "schema": {
+			    "openAPIV3Schema": {"x-kubernetes-preserve-unknown-fields": true, "type": "object"}
+			  }
+            }
+          ]
+       }
+    }`
+	_, err = testcrd.APIExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Patch(testcrd.Crd.Name, types.StrategicMergePatchType, []byte(apiVersionWithV2StoragePatch))
 	framework.ExpectNoError(err, "failed to patch custom resource definition %s in namespace: %s", testcrd.Crd.Name, f.Namespace.Name)
 
 	ginkgo.By("Patching the custom resource while v2 is storage version")
@@ -1798,7 +1821,7 @@ func registerValidatingWebhookForCRD(f *framework.Framework, configName string,
 					CABundle: context.signingCert,
 				},
 				SideEffects:             &sideEffectsNone,
-				AdmissionReviewVersions: []string{"v1beta1"},
+				AdmissionReviewVersions: []string{"v1", "v1beta1"},
 				// Scope the webhook to just this namespace
 				NamespaceSelector: &metav1.LabelSelector{
 					MatchLabels: map[string]string{f.UniqueName: "true"},
@@ -1819,12 +1842,18 @@ func testCRDDenyWebhook(f *framework.Framework) {
 	ginkgo.By("Creating a custom resource definition that should be denied by the webhook")
 	name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, "deny")
 	kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, "deny")
-	group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName)
+	group := fmt.Sprintf("%s.example.com", f.BaseName)
-	apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+	apiVersions := []apiextensionsv1.CustomResourceDefinitionVersion{
 		{
 			Name:    "v1",
 			Served:  true,
 			Storage: true,
+			Schema: &apiextensionsv1.CustomResourceValidation{
+				OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
+					XPreserveUnknownFields: pointer.BoolPtr(true),
+					Type:                   "object",
+				},
+			},
 		},
 	}
 
@@ -1839,28 +1868,28 @@ func testCRDDenyWebhook(f *framework.Framework) {
 		e2elog.Failf("failed to initialize apiExtensionClient: %v", err)
 		return
 	}
-	crd := &apiextensionsv1beta1.CustomResourceDefinition{
+	crd := &apiextensionsv1.CustomResourceDefinition{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: name + "s." + group,
 			Labels: map[string]string{
 				"webhook-e2e-test": "webhook-disallow",
 			},
 		},
-		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+		Spec: apiextensionsv1.CustomResourceDefinitionSpec{
 			Group:    group,
 			Versions: apiVersions,
-			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+			Names: apiextensionsv1.CustomResourceDefinitionNames{
 				Singular: name,
 				Kind:     kind,
 				ListKind: kind + "List",
 				Plural:   name + "s",
 			},
-			Scope: apiextensionsv1beta1.NamespaceScoped,
+			Scope: apiextensionsv1.NamespaceScoped,
 		},
 	}
 
 	// create CRD
-	_, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
+	_, err = apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Create(crd)
 	framework.ExpectError(err, "create custom resource definition %s should be denied by webhook", crd.Name)
 	expectedErrMsg := "the crd contains unwanted label"
 	if !strings.Contains(err.Error(), expectedErrMsg) {
@@ -1920,7 +1949,7 @@ func registerSlowWebhook(f *framework.Framework, configName string, context *cer
 				FailurePolicy:           policy,
 				TimeoutSeconds:          timeout,
 				SideEffects:             &sideEffectsNone,
-				AdmissionReviewVersions: []string{"v1beta1"},
+				AdmissionReviewVersions: []string{"v1", "v1beta1"},
 			},
 		},
 	})
@@ -1958,25 +1987,37 @@ func testSlowWebhookTimeoutNoError(f *framework.Framework) {
 // createAdmissionWebhookMultiVersionTestCRDWithV1Storage creates a new CRD specifically
 // for the admissin webhook calling test.
 func createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f *framework.Framework, opts ...crd.Option) (*crd.TestCrd, error) {
-	group := fmt.Sprintf("%s-multiversion-crd-test.k8s.io", f.BaseName)
+	group := fmt.Sprintf("%s.example.com", f.BaseName)
-	return crd.CreateMultiVersionTestCRD(f, group, append([]crd.Option{func(crd *apiextensionsv1beta1.CustomResourceDefinition) {
+	return crd.CreateMultiVersionTestCRD(f, group, append([]crd.Option{func(crd *apiextensionsv1.CustomResourceDefinition) {
-		crd.Spec.Versions = []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+		crd.Spec.Versions = []apiextensionsv1.CustomResourceDefinitionVersion{
 			{
 				Name:    "v1",
 				Served:  true,
 				Storage: true,
+				Schema: &apiextensionsv1.CustomResourceValidation{
+					OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
+						XPreserveUnknownFields: pointer.BoolPtr(true),
+						Type:                   "object",
+					},
+				},
 			},
 			{
 				Name:    "v2",
 				Served:  true,
 				Storage: false,
+				Schema: &apiextensionsv1.CustomResourceValidation{
+					OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
+						XPreserveUnknownFields: pointer.BoolPtr(true),
+						Type:                   "object",
+					},
+				},
 			},
 		}
 	}}, opts...)...)
 }
 
 // servedAPIVersions returns the API versions served by the CRD.
-func servedAPIVersions(crd *apiextensionsv1beta1.CustomResourceDefinition) []string {
+func servedAPIVersions(crd *apiextensionsv1.CustomResourceDefinition) []string {
 	ret := []string{}
 	for _, v := range crd.Spec.Versions {
 		if v.Served {
@@ -2038,7 +2079,7 @@ func newDenyPodWebhookFixture(f *framework.Framework, context *certContext) admi
 			CABundle: context.signingCert,
 		},
 		SideEffects:             &sideEffectsNone,
-		AdmissionReviewVersions: []string{"v1beta1"},
+		AdmissionReviewVersions: []string{"v1", "v1beta1"},
 		// Scope the webhook to just this namespace
 		NamespaceSelector: &metav1.LabelSelector{
 			MatchLabels: map[string]string{f.UniqueName: "true"},
@@ -2079,7 +2120,7 @@ func newDenyConfigMapWebhookFixture(f *framework.Framework, context *certContext
 			CABundle: context.signingCert,
 		},
 		SideEffects:             &sideEffectsNone,
-		AdmissionReviewVersions: []string{"v1beta1"},
+		AdmissionReviewVersions: []string{"v1", "v1beta1"},
 	}
 }
 
@@ -2105,7 +2146,7 @@ func newMutateConfigMapWebhookFixture(f *framework.Framework, context *certConte
 			CABundle: context.signingCert,
 		},
 		SideEffects:             &sideEffectsNone,
-		AdmissionReviewVersions: []string{"v1beta1"},
+		AdmissionReviewVersions: []string{"v1", "v1beta1"},
 		// Scope the webhook to just this namespace
 		NamespaceSelector: &metav1.LabelSelector{
 			MatchLabels: map[string]string{f.UniqueName: "true"},

test/e2e/kubectl/BUILD

@@ -17,7 +17,7 @@ go_library(
         "//pkg/controller:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/rbac/v1:go_default_library",
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
@@ -49,6 +49,7 @@ go_library(
         "//vendor/github.com/onsi/gomega:go_default_library",
         "//vendor/golang.org/x/net/websocket:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",
+        "//vendor/k8s.io/utils/pointer:go_default_library",
         "//vendor/sigs.k8s.io/yaml:go_default_library",
     ],
 )

test/e2e/kubectl/kubectl.go

@@ -41,10 +41,13 @@ import (
 
 	"github.com/elazarl/goproxy"
 	openapi_v2 "github.com/googleapis/gnostic/OpenAPIv2"
+	uexec "k8s.io/utils/exec"
+	"k8s.io/utils/pointer"
+	"sigs.k8s.io/yaml"
 
 	v1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
-	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -69,8 +72,6 @@ import (
 	"k8s.io/kubernetes/test/e2e/scheduling"
 	testutils "k8s.io/kubernetes/test/utils"
 	"k8s.io/kubernetes/test/utils/crd"
-	uexec "k8s.io/utils/exec"
-	"sigs.k8s.io/yaml"
 
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/gomega"
@@ -927,12 +928,14 @@ metadata:
 
 		ginkgo.It("should create/apply a valid CR for CRD with validation schema", func() {
 			ginkgo.By("prepare CRD with validation schema")
-			crd, err := crd.CreateTestCRD(f, func(crd *v1beta1.CustomResourceDefinition) {
+			crd, err := crd.CreateTestCRD(f, func(crd *apiextensionsv1.CustomResourceDefinition) {
-				props := &v1beta1.JSONSchemaProps{}
+				props := &apiextensionsv1.JSONSchemaProps{}
 				if err := yaml.Unmarshal(schemaFoo, props); err != nil {
 					e2elog.Failf("failed to unmarshal schema: %v", err)
 				}
-				crd.Spec.Validation = &v1beta1.CustomResourceValidation{OpenAPIV3Schema: props}
+				for i := range crd.Spec.Versions {
+					crd.Spec.Versions[i].Schema = &apiextensionsv1.CustomResourceValidation{OpenAPIV3Schema: props}
+				}
 			})
 			if err != nil {
 				e2elog.Failf("failed to create test CRD: %v", err)
@@ -951,12 +954,16 @@ metadata:
 
 		ginkgo.It("should create/apply a valid CR with arbitrary-extra properties for CRD with partially-specified validation schema", func() {
 			ginkgo.By("prepare CRD with partially-specified validation schema")
-			crd, err := crd.CreateTestCRD(f, func(crd *v1beta1.CustomResourceDefinition) {
+			crd, err := crd.CreateTestCRD(f, func(crd *apiextensionsv1.CustomResourceDefinition) {
-				props := &v1beta1.JSONSchemaProps{}
+				props := &apiextensionsv1.JSONSchemaProps{}
 				if err := yaml.Unmarshal(schemaFoo, props); err != nil {
 					e2elog.Failf("failed to unmarshal schema: %v", err)
 				}
-				crd.Spec.Validation = &v1beta1.CustomResourceValidation{OpenAPIV3Schema: props}
+				// Allow for arbitrary-extra properties.
+				props.XPreserveUnknownFields = pointer.BoolPtr(true)
+				for i := range crd.Spec.Versions {
+					crd.Spec.Versions[i].Schema = &apiextensionsv1.CustomResourceValidation{OpenAPIV3Schema: props}
+				}
 			})
 			if err != nil {
 				e2elog.Failf("failed to create test CRD: %v", err)
@@ -966,31 +973,14 @@ metadata:
 			ginkgo.By("sleep for 10s to wait for potential crd openapi publishing alpha feature")
 			time.Sleep(10 * time.Second)
 
-			publishedSchema := schemaForGVK(schema.GroupVersionKind{Group: crd.Crd.Spec.Group, Version: crd.Crd.Spec.Version, Kind: crd.Crd.Spec.Names.Kind})
+			schema := schemaForGVK(schema.GroupVersionKind{Group: crd.Crd.Spec.Group, Version: crd.Crd.Spec.Versions[0].Name, Kind: crd.Crd.Spec.Names.Kind})
-			expectSuccess := false
+			framework.ExpectNotEqual(schema, nil, "retrieving a schema for the crd")
-			if publishedSchema == nil || publishedSchema.Properties == nil || publishedSchema.Properties.AdditionalProperties == nil || len(publishedSchema.Properties.AdditionalProperties) == 0 {
-				// expect success in the following cases:
-				// - no schema was published
-				// - a schema was published with no properties
-				expectSuccess = true
-				e2elog.Logf("no schema with properties found, expect apply with extra properties to succeed")
-			} else {
-				e2elog.Logf("schema with properties found, expect apply with extra properties to fail")
-			}
 
 			meta := fmt.Sprintf(metaPattern, crd.Crd.Spec.Names.Kind, crd.Crd.Spec.Group, crd.Crd.Spec.Versions[0].Name, "test-cr")
 			validArbitraryCR := fmt.Sprintf(`{%s,"spec":{"bars":[{"name":"test-bar"}],"extraProperty":"arbitrary-value"}}`, meta)
-			if err := createApplyCustomResource(validArbitraryCR, f.Namespace.Name, "test-cr", crd); err != nil {
+			err = createApplyCustomResource(validArbitraryCR, f.Namespace.Name, "test-cr", crd)
-				if expectSuccess {
+			framework.ExpectNoError(err, "creating custom resource")
-					e2elog.Failf("%v", err)
-				}
-			} else {
-				if !expectSuccess {
-					e2elog.Failf("expected error, got none")
-				}
-			}
 		})
-
 	})
 
 	framework.KubeDescribe("Kubectl cluster-info", func() {

test/utils/crd/BUILD

@@ -6,7 +6,7 @@ go_library(
     importpath = "k8s.io/kubernetes/test/utils/crd",
     visibility = ["//visibility:public"],
     deps = [
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
@@ -14,6 +14,7 @@ go_library(
         "//staging/src/k8s.io/client-go/dynamic:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e/framework/log:go_default_library",
+        "//vendor/k8s.io/utils/pointer:go_default_library",
     ],
 )
 

test/utils/crd/crd_util.go

@@ -19,7 +19,9 @@ package crd
 import (
 	"fmt"
 
-	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	"k8s.io/utils/pointer"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	crdclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apiextensions-apiserver/test/integration/fixtures"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,13 +37,13 @@ type CleanCrdFn func() error
 // TestCrd holds all the pieces needed to test with the CRD
 type TestCrd struct {
 	APIExtensionClient *crdclientset.Clientset
-	Crd                *apiextensionsv1beta1.CustomResourceDefinition
+	Crd                *apiextensionsv1.CustomResourceDefinition
 	DynamicClients     map[string]dynamic.ResourceInterface
 	CleanUp            CleanCrdFn
 }
 
 // Option is a modifier for a CRD object used to customize CreateMultiVersionTestCRD and CreateTestCRD.
-type Option func(crd *apiextensionsv1beta1.CustomResourceDefinition)
+type Option func(crd *apiextensionsv1.CustomResourceDefinition)
 
 // CreateMultiVersionTestCRD creates a new CRD specifically for the calling test.
 func CreateMultiVersionTestCRD(f *framework.Framework, group string, opts ...Option) (*TestCrd, error) {
@@ -67,28 +69,38 @@ func CreateMultiVersionTestCRD(f *framework.Framework, group string, opts ...Opt
 		return nil, err
 	}
 
-	crd := &apiextensionsv1beta1.CustomResourceDefinition{
+	crd := &apiextensionsv1.CustomResourceDefinition{
 		ObjectMeta: metav1.ObjectMeta{Name: name + "s." + group},
-		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+		Spec: apiextensionsv1.CustomResourceDefinitionSpec{
 			Group: group,
-			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+			Names: apiextensionsv1.CustomResourceDefinitionNames{
 				Plural:   name + "s",
 				Singular: name,
 				Kind:     kind,
 				ListKind: kind + "List",
 			},
-			Scope: apiextensionsv1beta1.NamespaceScoped,
+			Scope: apiextensionsv1.NamespaceScoped,
 		},
 	}
 	for _, opt := range opts {
 		opt(crd)
 	}
-	if len(crd.Spec.Versions) == 0 && len(crd.Spec.Version) == 0 {
+	if len(crd.Spec.Versions) == 0 {
-		crd.Spec.Version = "v1"
+		crd.Spec.Versions = []apiextensionsv1.CustomResourceDefinitionVersion{{
+			Served:  true,
+			Storage: true,
+			Name:    "v1",
+			Schema: &apiextensionsv1.CustomResourceValidation{
+				OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
+					XPreserveUnknownFields: pointer.BoolPtr(true),
+					Type:                   "object",
+				},
+			},
+		}}
 	}
 
 	//create CRD and waits for the resource to be recognized and available.
-	crd, err = fixtures.CreateNewCustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient)
+	crd, err = fixtures.CreateNewV1CustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient)
 	if err != nil {
 		e2elog.Failf("failed to create CustomResourceDefinition: %v", err)
 		return nil, err
@@ -106,7 +118,7 @@ func CreateMultiVersionTestCRD(f *framework.Framework, group string, opts ...Opt
 	testcrd.Crd = crd
 	testcrd.DynamicClients = resourceClients
 	testcrd.CleanUp = func() error {
-		err := fixtures.DeleteCustomResourceDefinition(crd, apiExtensionClient)
+		err := fixtures.DeleteV1CustomResourceDefinition(crd, apiExtensionClient)
 		if err != nil {
 			e2elog.Failf("failed to delete CustomResourceDefinition(%s): %v", name, err)
 		}
@@ -117,13 +129,19 @@ func CreateMultiVersionTestCRD(f *framework.Framework, group string, opts ...Opt
 
 // CreateTestCRD creates a new CRD specifically for the calling test.
 func CreateTestCRD(f *framework.Framework, opts ...Option) (*TestCrd, error) {
-	group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName)
+	group := fmt.Sprintf("%s.example.com", f.BaseName)
-	return CreateMultiVersionTestCRD(f, group, append([]Option{func(crd *apiextensionsv1beta1.CustomResourceDefinition) {
+	return CreateMultiVersionTestCRD(f, group, append([]Option{func(crd *apiextensionsv1.CustomResourceDefinition) {
-		crd.Spec.Versions = []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+		crd.Spec.Versions = []apiextensionsv1.CustomResourceDefinitionVersion{
 			{
 				Name:    "v1",
 				Served:  true,
 				Storage: true,
+				Schema: &apiextensionsv1.CustomResourceValidation{
+					OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
+						XPreserveUnknownFields: pointer.BoolPtr(true),
+						Type:                   "object",
+					},
+				},
 			},
 		}
 	}}, opts...)...)

test/utils/image/manifest.go

@@ -204,7 +204,7 @@ const (
 
 func initImageConfigs() map[int]Config {
 	configs := map[int]Config{}
-	configs[Agnhost] = Config{e2eRegistry, "agnhost", "2.4"}
+	configs[Agnhost] = Config{e2eRegistry, "agnhost", "2.5"}
 	configs[Alpine] = Config{dockerLibraryRegistry, "alpine", "3.7"}
 	configs[AuthenticatedAlpine] = Config{gcAuthenticatedRegistry, "alpine", "3.7"}
 	configs[AuthenticatedWindowsNanoServer] = Config{gcAuthenticatedRegistry, "windows-nanoserver", "v1"}