Merge pull request #63830 from mbohlool/crd_versioning_nop

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Crd versioning with nop Conversion

Implements Custom Resource Definition versioning according to[ design doc](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/customresources-versioning.md).

Note: I recreated this PR instead of #63518. Huge number of comments there broke github. 

@sttts @nikhita @deads2k @liggitt @lavalamp 

```release-note
Add CRD Versioning with NOP converter
```

api/openapi-spec/swagger.json

@@ -84943,7 +84943,6 @@
     "description": "CustomResourceDefinitionSpec describes how a user wants their resource to appear",
     "required": [
      "group",
-     "version",
      "names",
      "scope"
     ],
@@ -84969,8 +84968,15 @@
       "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceValidation"
      },
      "version": {
-      "description": "Version is the version this resource belongs in",
+      "description": "Version is the version this resource belongs in Should be always first item in Versions field if provided. Optional, but at least one of Version or Versions must be set. Deprecated: Please use `Versions`.",
       "type": "string"
+     },
+     "versions": {
+      "description": "Versions is the list of all supported versions for this resource. If Version field is provided, this field is optional. Validation: All versions must use the same validation schema for now. i.e., top level Validation field is applied to all of these versions. Order: The version name will be used to compute the order. If the version string is \"kube-like\", it will sort above non \"kube-like\" version strings, which are ordered lexicographically. \"Kube-like\" versions start with a \"v\", then are followed by a number (the major version), then optionally the string \"alpha\" or \"beta\" and another number (the minor version). These are sorted first by GA \u003e beta \u003e alpha, and then by comparing major version, then minor version. An example sorted list of versions: v10, v2, v1, v11beta2, v10beta3, v3beta1, v12alpha1, v11alpha2, foo1, foo10.",
+      "type": "array",
+      "items": {
+       "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceDefinitionVersion"
+      }
      }
     }
    },
@@ -84978,7 +84984,8 @@
     "description": "CustomResourceDefinitionStatus indicates the state of the CustomResourceDefinition",
     "required": [
      "conditions",
-     "acceptedNames"
+     "acceptedNames",
+     "storedVersions"
     ],
     "properties": {
      "acceptedNames": {
@@ -84991,6 +84998,34 @@
       "items": {
        "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceDefinitionCondition"
       }
+     },
+     "storedVersions": {
+      "description": "StoredVersions are all versions of CustomResources that were ever persisted. Tracking these versions allows a migration path for stored versions in etcd. The field is mutable so the migration controller can first finish a migration to another version (i.e. that no old objects are left in the storage), and then remove the rest of the versions from this list. None of the versions in this list can be removed from the spec.Versions field.",
+      "type": "array",
+      "items": {
+       "type": "string"
+      }
+     }
+    }
+   },
+   "io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceDefinitionVersion": {
+    "required": [
+     "name",
+     "served",
+     "storage"
+    ],
+    "properties": {
+     "name": {
+      "description": "Name is the version name, e.g. “v1”, “v2beta1”, etc.",
+      "type": "string"
+     },
+     "served": {
+      "description": "Served is a flag enabling/disabling this version from being served via REST APIs",
+      "type": "boolean"
+     },
+     "storage": {
+      "description": "Storage flags the version as storage version. There must be exactly one flagged as storage version.",
+      "type": "boolean"
      }
     }
    },

pkg/master/controller/crdregistration/crdregistration_controller.go

@@ -77,9 +77,11 @@ func NewAutoRegistrationController(crdinformer crdinformers.CustomResourceDefini
 			cast := obj.(*apiextensions.CustomResourceDefinition)
 			c.enqueueCRD(cast)
 		},
-		UpdateFunc: func(_, obj interface{}) {
+		UpdateFunc: func(oldObj, newObj interface{}) {
-			cast := obj.(*apiextensions.CustomResourceDefinition)
+			// Enqueue both old and new object to make sure we remove and add appropriate API services.
-			c.enqueueCRD(cast)
+			// The working queue will resolve any duplicates and only changes will stay in the queue.
+			c.enqueueCRD(oldObj.(*apiextensions.CustomResourceDefinition))
+			c.enqueueCRD(newObj.(*apiextensions.CustomResourceDefinition))
 		},
 		DeleteFunc: func(obj interface{}) {
 			cast, ok := obj.(*apiextensions.CustomResourceDefinition)
@@ -120,8 +122,10 @@ func (c *crdRegistrationController) Run(threadiness int, stopCh <-chan struct{})
 		utilruntime.HandleError(err)
 	} else {
 		for _, crd := range crds {
-			if err := c.syncHandler(schema.GroupVersion{Group: crd.Spec.Group, Version: crd.Spec.Version}); err != nil {
+			for _, version := range crd.Spec.Versions {
-				utilruntime.HandleError(err)
+				if err := c.syncHandler(schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name}); err != nil {
+					utilruntime.HandleError(err)
+				}
 			}
 		}
 	}
@@ -182,11 +186,12 @@ func (c *crdRegistrationController) processNextWorkItem() bool {
 }
 
 func (c *crdRegistrationController) enqueueCRD(crd *apiextensions.CustomResourceDefinition) {
-	c.queue.Add(schema.GroupVersion{Group: crd.Spec.Group, Version: crd.Spec.Version})
+	for _, version := range crd.Spec.Versions {
+		c.queue.Add(schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name})
+	}
 }
 
 func (c *crdRegistrationController) handleVersionUpdate(groupVersion schema.GroupVersion) error {
-	found := false
 	apiServiceName := groupVersion.Version + "." + groupVersion.Group
 
 	// check all CRDs.  There shouldn't that many, but if we have problems later we can index them
@@ -195,26 +200,27 @@ func (c *crdRegistrationController) handleVersionUpdate(groupVersion schema.Grou
 		return err
 	}
 	for _, crd := range crds {
-		if crd.Spec.Version == groupVersion.Version && crd.Spec.Group == groupVersion.Group {
+		if crd.Spec.Group != groupVersion.Group {
-			found = true
+			continue
-			break
+		}
+		for _, version := range crd.Spec.Versions {
+			if version.Name != groupVersion.Version || !version.Served {
+				continue
+			}
+
+			c.apiServiceRegistration.AddAPIServiceToSync(&apiregistration.APIService{
+				ObjectMeta: metav1.ObjectMeta{Name: apiServiceName},
+				Spec: apiregistration.APIServiceSpec{
+					Group:                groupVersion.Group,
+					Version:              groupVersion.Version,
+					GroupPriorityMinimum: 1000, // CRDs should have relatively low priority
+					VersionPriority:      100,  // CRDs will be sorted by kube-like versions like any other APIService with the same VersionPriority
+				},
+			})
+			return nil
 		}
 	}
 
-	if !found {
+	c.apiServiceRegistration.RemoveAPIServiceToSync(apiServiceName)
-		c.apiServiceRegistration.RemoveAPIServiceToSync(apiServiceName)
-		return nil
-	}
-
-	c.apiServiceRegistration.AddAPIServiceToSync(&apiregistration.APIService{
-		ObjectMeta: metav1.ObjectMeta{Name: apiServiceName},
-		Spec: apiregistration.APIServiceSpec{
-			Group:                groupVersion.Group,
-			Version:              groupVersion.Version,
-			GroupPriorityMinimum: 1000, // CRDs should have relatively low priority
-			VersionPriority:      100,  // CRDs should have relatively low priority
-		},
-	})
-
 	return nil
 }

pkg/master/controller/crdregistration/crdregistration_controller_test.go

@@ -42,8 +42,16 @@ func TestHandleVersionUpdate(t *testing.T) {
 			startingCRDs: []*apiextensions.CustomResourceDefinition{
 				{
 					Spec: apiextensions.CustomResourceDefinitionSpec{
-						Group:   "group.com",
+						Group: "group.com",
-						Version: "v1",
+						// Version field is deprecated and crd registration won't rely on it at all.
+						// defaulting route will fill up Versions field if user only provided version field.
+						Versions: []apiextensions.CustomResourceDefinitionVersion{
+							{
+								Name:    "v1",
+								Served:  true,
+								Storage: true,
+							},
+						},
 					},
 				},
 			},
@@ -66,8 +74,14 @@ func TestHandleVersionUpdate(t *testing.T) {
 			startingCRDs: []*apiextensions.CustomResourceDefinition{
 				{
 					Spec: apiextensions.CustomResourceDefinitionSpec{
-						Group:   "group.com",
+						Group: "group.com",
-						Version: "v1",
+						Versions: []apiextensions.CustomResourceDefinitionVersion{
+							{
+								Name:    "v1",
+								Served:  true,
+								Storage: true,
+							},
+						},
 					},
 				},
 			},
@@ -98,7 +112,6 @@ func TestHandleVersionUpdate(t *testing.T) {
 			t.Errorf("%s expected %v, got %v", test.name, test.expectedRemoved, registration.removed)
 		}
 	}
-
 }
 
 type fakeAPIServiceRegistration struct {

staging/src/k8s.io/api/Godeps/Godeps.json

@@ -118,6 +118,10 @@
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
+		{
+			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1beta1",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/fuzzer/fuzzer.go

@@ -42,6 +42,28 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 			if len(obj.Names.ListKind) == 0 && len(obj.Names.Kind) > 0 {
 				obj.Names.ListKind = obj.Names.Kind + "List"
 			}
+			if len(obj.Versions) == 0 && len(obj.Version) != 0 {
+				obj.Versions = []apiextensions.CustomResourceDefinitionVersion{
+					{
+						Name:    obj.Version,
+						Served:  true,
+						Storage: true,
+					},
+				}
+			} else if len(obj.Versions) != 0 {
+				obj.Version = obj.Versions[0].Name
+			}
+		},
+		func(obj *apiextensions.CustomResourceDefinition, c fuzz.Continue) {
+			c.FuzzNoCustom(obj)
+
+			if len(obj.Status.StoredVersions) == 0 {
+				for _, v := range obj.Spec.Versions {
+					if v.Storage && !apiextensions.IsStoredVersion(obj, v.Name) {
+						obj.Status.StoredVersions = append(obj.Status.StoredVersions, v.Name)
+					}
+				}
+			}
 		},
 		func(obj *apiextensions.JSONSchemaProps, c fuzz.Continue) {
 			// we cannot use c.FuzzNoCustom because of the interface{} fields. So let's loop with reflection.

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/helpers.go

@@ -17,6 +17,7 @@ limitations under the License.
 package apiextensions
 
 import (
+	"fmt"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -116,3 +117,33 @@ func CRDRemoveFinalizer(crd *CustomResourceDefinition, needle string) {
 	}
 	crd.Finalizers = newFinalizers
 }
+
+// HasServedCRDVersion returns true if `version` is in the list of CRD's versions and the Served flag is set.
+func HasServedCRDVersion(crd *CustomResourceDefinition, version string) bool {
+	for _, v := range crd.Spec.Versions {
+		if v.Name == version {
+			return v.Served
+		}
+	}
+	return false
+}
+
+// GetCRDStorageVersion returns the storage version for given CRD.
+func GetCRDStorageVersion(crd *CustomResourceDefinition) (string, error) {
+	for _, v := range crd.Spec.Versions {
+		if v.Storage {
+			return v.Name, nil
+		}
+	}
+	// This should not happened if crd is valid
+	return "", fmt.Errorf("invalid CustomResourceDefinition, no storage version")
+}
+
+func IsStoredVersion(crd *CustomResourceDefinition, version string) bool {
+	for _, v := range crd.Status.StoredVersions {
+		if version == v {
+			return true
+		}
+	}
+	return false
+}

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go

@@ -25,6 +25,9 @@ type CustomResourceDefinitionSpec struct {
 	// Group is the group this resource belongs in
 	Group string
 	// Version is the version this resource belongs in
+	// Should be always first item in Versions field if provided.
+	// Optional, but at least one of Version or Versions must be set.
+	// Deprecated: Please use `Versions`.
 	Version string
 	// Names are the names used to describe this custom resource
 	Names CustomResourceDefinitionNames
@@ -34,6 +37,27 @@ type CustomResourceDefinitionSpec struct {
 	Validation *CustomResourceValidation
 	// Subresources describes the subresources for CustomResources
 	Subresources *CustomResourceSubresources
+	// Versions is the list of all supported versions for this resource.
+	// If Version field is provided, this field is optional.
+	// Validation: All versions must use the same validation schema for now. i.e., top
+	// level Validation field is applied to all of these versions.
+	// Order: The version name will be used to compute the order.
+	// If the version string is "kube-like", it will sort above non "kube-like" version strings, which are ordered
+	// lexicographically. "Kube-like" versions start with a "v", then are followed by a number (the major version),
+	// then optionally the string "alpha" or "beta" and another number (the minor version). These are sorted first
+	// by GA > beta > alpha, and then by comparing major version, then minor version. An example sorted list of
+	// versions: v10, v2, v1, v11beta2, v10beta3, v3beta1, v12alpha1, v11alpha2, foo1, foo10.
+	Versions []CustomResourceDefinitionVersion
+}
+
+type CustomResourceDefinitionVersion struct {
+	// Name is the version name, e.g. “v1”, “v2beta1”, etc.
+	Name string
+	// Served is a flag enabling/disabling this version from being served via REST APIs
+	Served bool
+	// Storage flags the version as storage version. There must be exactly one flagged
+	// as storage version.
+	Storage bool
 }
 
 // CustomResourceDefinitionNames indicates the names to serve this CustomResourceDefinition
@@ -115,6 +139,14 @@ type CustomResourceDefinitionStatus struct {
 	// AcceptedNames are the names that are actually being used to serve discovery
 	// They may be different than the names in spec.
 	AcceptedNames CustomResourceDefinitionNames
+
+	// StoredVersions are all versions of CustomResources that were ever persisted. Tracking these
+	// versions allows a migration path for stored versions in etcd. The field is mutable
+	// so the migration controller can first finish a migration to another version (i.e.
+	// that no old objects are left in the storage), and then remove the rest of the
+	// versions from this list.
+	// None of the versions in this list can be removed from the spec.Versions field.
+	StoredVersions []string
 }
 
 // CustomResourceCleanupFinalizer is the name of the finalizer which will delete instances of

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/defaults.go

@@ -31,6 +31,14 @@ func addDefaultingFuncs(scheme *runtime.Scheme) error {
 
 func SetDefaults_CustomResourceDefinition(obj *CustomResourceDefinition) {
 	SetDefaults_CustomResourceDefinitionSpec(&obj.Spec)
+	if len(obj.Status.StoredVersions) == 0 {
+		for _, v := range obj.Spec.Versions {
+			if v.Storage {
+				obj.Status.StoredVersions = append(obj.Status.StoredVersions, v.Name)
+				break
+			}
+		}
+	}
 }
 
 func SetDefaults_CustomResourceDefinitionSpec(obj *CustomResourceDefinitionSpec) {
@@ -43,4 +51,16 @@ func SetDefaults_CustomResourceDefinitionSpec(obj *CustomResourceDefinitionSpec)
 	if len(obj.Names.ListKind) == 0 && len(obj.Names.Kind) > 0 {
 		obj.Names.ListKind = obj.Names.Kind + "List"
 	}
+	// If there is no list of versions, create on using deprecated Version field.
+	if len(obj.Versions) == 0 && len(obj.Version) != 0 {
+		obj.Versions = []CustomResourceDefinitionVersion{{
+			Name:    obj.Version,
+			Storage: true,
+			Served:  true,
+		}}
+	}
+	// For backward compatibility set the version field to the first item in versions list.
+	if len(obj.Version) == 0 && len(obj.Versions) != 0 {
+		obj.Version = obj.Versions[0].Name
+	}
 }

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.pb.go

@@ -31,6 +31,7 @@ limitations under the License.
 		CustomResourceDefinitionNames
 		CustomResourceDefinitionSpec
 		CustomResourceDefinitionStatus
+		CustomResourceDefinitionVersion
 		CustomResourceSubresourceScale
 		CustomResourceSubresourceStatus
 		CustomResourceSubresources
@@ -102,54 +103,60 @@ func (*CustomResourceDefinitionStatus) Descriptor() ([]byte, []int) {
 	return fileDescriptorGenerated, []int{5}
 }
 
+func (m *CustomResourceDefinitionVersion) Reset()      { *m = CustomResourceDefinitionVersion{} }
+func (*CustomResourceDefinitionVersion) ProtoMessage() {}
+func (*CustomResourceDefinitionVersion) Descriptor() ([]byte, []int) {
+	return fileDescriptorGenerated, []int{6}
+}
+
 func (m *CustomResourceSubresourceScale) Reset()      { *m = CustomResourceSubresourceScale{} }
 func (*CustomResourceSubresourceScale) ProtoMessage() {}
 func (*CustomResourceSubresourceScale) Descriptor() ([]byte, []int) {
-	return fileDescriptorGenerated, []int{6}
+	return fileDescriptorGenerated, []int{7}
 }
 
 func (m *CustomResourceSubresourceStatus) Reset()      { *m = CustomResourceSubresourceStatus{} }
 func (*CustomResourceSubresourceStatus) ProtoMessage() {}
 func (*CustomResourceSubresourceStatus) Descriptor() ([]byte, []int) {
-	return fileDescriptorGenerated, []int{7}
+	return fileDescriptorGenerated, []int{8}
 }
 
 func (m *CustomResourceSubresources) Reset()      { *m = CustomResourceSubresources{} }
 func (*CustomResourceSubresources) ProtoMessage() {}
 func (*CustomResourceSubresources) Descriptor() ([]byte, []int) {
-	return fileDescriptorGenerated, []int{8}
+	return fileDescriptorGenerated, []int{9}
 }
 
 func (m *CustomResourceValidation) Reset()      { *m = CustomResourceValidation{} }
 func (*CustomResourceValidation) ProtoMessage() {}
 func (*CustomResourceValidation) Descriptor() ([]byte, []int) {
-	return fileDescriptorGenerated, []int{9}
+	return fileDescriptorGenerated, []int{10}
 }
 
 func (m *ExternalDocumentation) Reset()                    { *m = ExternalDocumentation{} }
 func (*ExternalDocumentation) ProtoMessage()               {}
-func (*ExternalDocumentation) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{10} }
+func (*ExternalDocumentation) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{11} }
 
 func (m *JSON) Reset()                    { *m = JSON{} }
 func (*JSON) ProtoMessage()               {}
-func (*JSON) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{11} }
+func (*JSON) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{12} }
 
 func (m *JSONSchemaProps) Reset()                    { *m = JSONSchemaProps{} }
 func (*JSONSchemaProps) ProtoMessage()               {}
-func (*JSONSchemaProps) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{12} }
+func (*JSONSchemaProps) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{13} }
 
 func (m *JSONSchemaPropsOrArray) Reset()                    { *m = JSONSchemaPropsOrArray{} }
 func (*JSONSchemaPropsOrArray) ProtoMessage()               {}
-func (*JSONSchemaPropsOrArray) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{13} }
+func (*JSONSchemaPropsOrArray) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{14} }
 
 func (m *JSONSchemaPropsOrBool) Reset()                    { *m = JSONSchemaPropsOrBool{} }
 func (*JSONSchemaPropsOrBool) ProtoMessage()               {}
-func (*JSONSchemaPropsOrBool) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{14} }
+func (*JSONSchemaPropsOrBool) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{15} }
 
 func (m *JSONSchemaPropsOrStringArray) Reset()      { *m = JSONSchemaPropsOrStringArray{} }
 func (*JSONSchemaPropsOrStringArray) ProtoMessage() {}
 func (*JSONSchemaPropsOrStringArray) Descriptor() ([]byte, []int) {
-	return fileDescriptorGenerated, []int{15}
+	return fileDescriptorGenerated, []int{16}
 }
 
 func init() {
@@ -159,6 +166,7 @@ func init() {
 	proto.RegisterType((*CustomResourceDefinitionNames)(nil), "k8s.io.apiextensions_apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceDefinitionNames")
 	proto.RegisterType((*CustomResourceDefinitionSpec)(nil), "k8s.io.apiextensions_apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceDefinitionSpec")
 	proto.RegisterType((*CustomResourceDefinitionStatus)(nil), "k8s.io.apiextensions_apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceDefinitionStatus")
+	proto.RegisterType((*CustomResourceDefinitionVersion)(nil), "k8s.io.apiextensions_apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceDefinitionVersion")
 	proto.RegisterType((*CustomResourceSubresourceScale)(nil), "k8s.io.apiextensions_apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceSubresourceScale")
 	proto.RegisterType((*CustomResourceSubresourceStatus)(nil), "k8s.io.apiextensions_apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceSubresourceStatus")
 	proto.RegisterType((*CustomResourceSubresources)(nil), "k8s.io.apiextensions_apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceSubresources")
@@ -411,6 +419,18 @@ func (m *CustomResourceDefinitionSpec) MarshalTo(dAtA []byte) (int, error) {
 		}
 		i += n8
 	}
+	if len(m.Versions) > 0 {
+		for _, msg := range m.Versions {
+			dAtA[i] = 0x3a
+			i++
+			i = encodeVarintGenerated(dAtA, i, uint64(msg.Size()))
+			n, err := msg.MarshalTo(dAtA[i:])
+			if err != nil {
+				return 0, err
+			}
+			i += n
+		}
+	}
 	return i, nil
 }
 
@@ -449,6 +469,59 @@ func (m *CustomResourceDefinitionStatus) MarshalTo(dAtA []byte) (int, error) {
 		return 0, err
 	}
 	i += n9
+	if len(m.StoredVersions) > 0 {
+		for _, s := range m.StoredVersions {
+			dAtA[i] = 0x1a
+			i++
+			l = len(s)
+			for l >= 1<<7 {
+				dAtA[i] = uint8(uint64(l)&0x7f | 0x80)
+				l >>= 7
+				i++
+			}
+			dAtA[i] = uint8(l)
+			i++
+			i += copy(dAtA[i:], s)
+		}
+	}
+	return i, nil
+}
+
+func (m *CustomResourceDefinitionVersion) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalTo(dAtA)
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *CustomResourceDefinitionVersion) MarshalTo(dAtA []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	dAtA[i] = 0xa
+	i++
+	i = encodeVarintGenerated(dAtA, i, uint64(len(m.Name)))
+	i += copy(dAtA[i:], m.Name)
+	dAtA[i] = 0x10
+	i++
+	if m.Served {
+		dAtA[i] = 1
+	} else {
+		dAtA[i] = 0
+	}
+	i++
+	dAtA[i] = 0x18
+	i++
+	if m.Storage {
+		dAtA[i] = 1
+	} else {
+		dAtA[i] = 0
+	}
+	i++
 	return i, nil
 }
 
@@ -1271,6 +1344,12 @@ func (m *CustomResourceDefinitionSpec) Size() (n int) {
 		l = m.Subresources.Size()
 		n += 1 + l + sovGenerated(uint64(l))
 	}
+	if len(m.Versions) > 0 {
+		for _, e := range m.Versions {
+			l = e.Size()
+			n += 1 + l + sovGenerated(uint64(l))
+		}
+	}
 	return n
 }
 
@@ -1285,6 +1364,22 @@ func (m *CustomResourceDefinitionStatus) Size() (n int) {
 	}
 	l = m.AcceptedNames.Size()
 	n += 1 + l + sovGenerated(uint64(l))
+	if len(m.StoredVersions) > 0 {
+		for _, s := range m.StoredVersions {
+			l = len(s)
+			n += 1 + l + sovGenerated(uint64(l))
+		}
+	}
+	return n
+}
+
+func (m *CustomResourceDefinitionVersion) Size() (n int) {
+	var l int
+	_ = l
+	l = len(m.Name)
+	n += 1 + l + sovGenerated(uint64(l))
+	n += 2
+	n += 2
 	return n
 }
 
@@ -1619,6 +1714,7 @@ func (this *CustomResourceDefinitionSpec) String() string {
 		`Scope:` + fmt.Sprintf("%v", this.Scope) + `,`,
 		`Validation:` + strings.Replace(fmt.Sprintf("%v", this.Validation), "CustomResourceValidation", "CustomResourceValidation", 1) + `,`,
 		`Subresources:` + strings.Replace(fmt.Sprintf("%v", this.Subresources), "CustomResourceSubresources", "CustomResourceSubresources", 1) + `,`,
+		`Versions:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.Versions), "CustomResourceDefinitionVersion", "CustomResourceDefinitionVersion", 1), `&`, ``, 1) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -1630,6 +1726,19 @@ func (this *CustomResourceDefinitionStatus) String() string {
 	s := strings.Join([]string{`&CustomResourceDefinitionStatus{`,
 		`Conditions:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.Conditions), "CustomResourceDefinitionCondition", "CustomResourceDefinitionCondition", 1), `&`, ``, 1) + `,`,
 		`AcceptedNames:` + strings.Replace(strings.Replace(this.AcceptedNames.String(), "CustomResourceDefinitionNames", "CustomResourceDefinitionNames", 1), `&`, ``, 1) + `,`,
+		`StoredVersions:` + fmt.Sprintf("%v", this.StoredVersions) + `,`,
+		`}`,
+	}, "")
+	return s
+}
+func (this *CustomResourceDefinitionVersion) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&CustomResourceDefinitionVersion{`,
+		`Name:` + fmt.Sprintf("%v", this.Name) + `,`,
+		`Served:` + fmt.Sprintf("%v", this.Served) + `,`,
+		`Storage:` + fmt.Sprintf("%v", this.Storage) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -2706,6 +2815,37 @@ func (m *CustomResourceDefinitionSpec) Unmarshal(dAtA []byte) error {
 				return err
 			}
 			iNdEx = postIndex
+		case 7:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Versions", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowGenerated
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthGenerated
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Versions = append(m.Versions, CustomResourceDefinitionVersion{})
+			if err := m.Versions[len(m.Versions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipGenerated(dAtA[iNdEx:])
@@ -2817,6 +2957,154 @@ func (m *CustomResourceDefinitionStatus) Unmarshal(dAtA []byte) error {
 				return err
 			}
 			iNdEx = postIndex
+		case 3:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field StoredVersions", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowGenerated
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthGenerated
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.StoredVersions = append(m.StoredVersions, string(dAtA[iNdEx:postIndex]))
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipGenerated(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthGenerated
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *CustomResourceDefinitionVersion) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowGenerated
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: CustomResourceDefinitionVersion: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: CustomResourceDefinitionVersion: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowGenerated
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthGenerated
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Name = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Served", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowGenerated
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.Served = bool(v != 0)
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Storage", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowGenerated
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.Storage = bool(v != 0)
 		default:
 			iNdEx = preIndex
 			skippy, err := skipGenerated(dAtA[iNdEx:])
@@ -5241,137 +5529,143 @@ func init() {
 }
 
 var fileDescriptorGenerated = []byte{
-	// 2102 bytes of a gzipped FileDescriptorProto
+	// 2200 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x58, 0xcb, 0x6f, 0x63, 0x49,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x59, 0xcd, 0x6f, 0x1c, 0x49,
-	0xd5, 0x4f, 0xd9, 0x79, 0x56, 0x92, 0x49, 0x52, 0xdd, 0xe9, 0xef, 0x76, 0xbe, 0x6e, 0x3b, 0xf1,
+	0x15, 0x77, 0xcf, 0x78, 0xfc, 0x51, 0xb6, 0x63, 0xbb, 0x12, 0x87, 0x8e, 0x49, 0x66, 0xec, 0x59,
-	0x30, 0xa3, 0x00, 0xd3, 0x36, 0x3d, 0x0f, 0x66, 0x40, 0x62, 0x11, 0x27, 0x01, 0xf5, 0x90, 0x74,
+	0x76, 0x65, 0x60, 0x33, 0x43, 0xf6, 0x83, 0x5d, 0x56, 0xe2, 0xe0, 0xb1, 0x0d, 0xca, 0x62, 0xc7,
-	0xa2, 0x72, 0x77, 0x23, 0x98, 0x67, 0xe5, 0xba, 0xec, 0x54, 0xe7, 0xbe, 0xfa, 0x56, 0x5d, 0x77,
+	0x56, 0x4d, 0x12, 0x04, 0xfb, 0x59, 0xee, 0xae, 0x19, 0x77, 0xdc, 0x5f, 0xe9, 0xaa, 0x9e, 0xd8,
-	0x22, 0x01, 0xe2, 0xa1, 0x11, 0x12, 0x12, 0x0f, 0x41, 0x6f, 0x90, 0xd8, 0x80, 0xc4, 0x06, 0x21,
+	0x12, 0x20, 0x3e, 0xb4, 0x42, 0x42, 0xc0, 0x22, 0x88, 0x90, 0x90, 0xb8, 0x80, 0xc4, 0x05, 0x21,
-	0x58, 0xc0, 0x92, 0x25, 0x8b, 0x5e, 0x8e, 0xc4, 0x66, 0x56, 0x16, 0x6d, 0xfe, 0x05, 0x24, 0xa4,
+	0x38, 0xc0, 0x91, 0x3f, 0x20, 0xc7, 0x95, 0xb8, 0xec, 0x69, 0x44, 0x86, 0x7f, 0x01, 0x09, 0xc9,
-	0xac, 0x50, 0x3d, 0xee, 0xcb, 0x8e, 0x67, 0x5a, 0x1a, 0x7b, 0x7a, 0xe7, 0x7b, 0x5e, 0xbf, 0x5f,
+	0x27, 0x54, 0x1f, 0x5d, 0xdd, 0x3d, 0xe3, 0xd9, 0x44, 0xda, 0x99, 0xcd, 0xcd, 0xfd, 0xde, 0xab,
-	0x9d, 0x3a, 0x75, 0xea, 0x94, 0x61, 0xeb, 0xe4, 0x0d, 0x5e, 0x65, 0x7e, 0xed, 0x24, 0x3a, 0xa2,
+	0xf7, 0xfb, 0xd5, 0xab, 0x57, 0xaf, 0xde, 0x1b, 0x83, 0xd6, 0xf1, 0xeb, 0xb4, 0xe6, 0x04, 0xf5,
-	0xa1, 0x47, 0x05, 0xe5, 0xb5, 0x0e, 0xf5, 0x9a, 0x7e, 0x58, 0x33, 0x0a, 0x12, 0x30, 0x7a, 0x2a,
+	0xe3, 0xf8, 0x90, 0x44, 0x3e, 0x61, 0x84, 0xd6, 0x3b, 0xc4, 0xb7, 0x83, 0xa8, 0xae, 0x14, 0x38,
-	0xa8, 0xc7, 0x99, 0xef, 0xf1, 0x1b, 0x24, 0x60, 0x9c, 0x86, 0x1d, 0x1a, 0xd6, 0x82, 0x93, 0xb6,
+	0x74, 0xc8, 0x09, 0x23, 0x3e, 0x75, 0x02, 0x9f, 0x5e, 0xc7, 0xa1, 0x43, 0x49, 0xd4, 0x21, 0x51,
-	0xd4, 0xf1, 0xbc, 0x41, 0xad, 0x73, 0xf3, 0x88, 0x0a, 0x72, 0xb3, 0xd6, 0xa6, 0x1e, 0x0d, 0x89,
+	0x3d, 0x3c, 0x6e, 0x73, 0x1d, 0xcd, 0x1b, 0xd4, 0x3b, 0x37, 0x0e, 0x09, 0xc3, 0x37, 0xea, 0x6d,
-	0xa0, 0xcd, 0x6a, 0x10, 0xfa, 0xc2, 0x47, 0x5f, 0xd3, 0xe1, 0xaa, 0x39, 0xeb, 0xf7, 0x92, 0x70,
+	0xe2, 0x93, 0x08, 0x33, 0x62, 0xd7, 0xc2, 0x28, 0x60, 0x01, 0xfc, 0xba, 0x74, 0x57, 0xcb, 0x59,
-	0xd5, 0xe0, 0xa4, 0x2d, 0x75, 0x3c, 0x6f, 0x50, 0x35, 0xe1, 0xd6, 0x6e, 0xb4, 0x99, 0x38, 0x8e,
+	0xbf, 0xa7, 0xdd, 0xd5, 0xc2, 0xe3, 0x36, 0xd7, 0xd1, 0xbc, 0x41, 0x4d, 0xb9, 0x5b, 0xbd, 0xde,
-	0x8e, 0xaa, 0xb6, 0xef, 0xd6, 0xda, 0x7e, 0xdb, 0xaf, 0xa9, 0xa8, 0x47, 0x51, 0x4b, 0x7d, 0xa9,
+	0x76, 0xd8, 0x51, 0x7c, 0x58, 0xb3, 0x02, 0xaf, 0xde, 0x0e, 0xda, 0x41, 0x5d, 0x78, 0x3d, 0x8c,
-	0x0f, 0xf5, 0x4b, 0xa3, 0xad, 0xbd, 0x9a, 0x92, 0x77, 0x89, 0x7d, 0xcc, 0x3c, 0x1a, 0x9e, 0xa5,
+	0x5b, 0xe2, 0x4b, 0x7c, 0x88, 0xbf, 0x24, 0xda, 0xea, 0x2b, 0x29, 0x79, 0x0f, 0x5b, 0x47, 0x8e,
-	0x8c, 0x5d, 0x2a, 0x48, 0xad, 0x33, 0xc0, 0x71, 0xad, 0x36, 0xcc, 0x2b, 0x8c, 0x3c, 0xc1, 0x5c,
+	0x4f, 0xa2, 0xd3, 0x94, 0xb1, 0x47, 0x18, 0xae, 0x77, 0x06, 0x38, 0xae, 0xd6, 0x87, 0xad, 0x8a,
-	0x3a, 0xe0, 0xf0, 0xe5, 0x4f, 0x72, 0xe0, 0xf6, 0x31, 0x75, 0xc9, 0x80, 0xdf, 0x2b, 0xc3, 0xfc,
+	0x62, 0x9f, 0x39, 0x1e, 0x19, 0x58, 0xf0, 0xd5, 0x27, 0x2d, 0xa0, 0xd6, 0x11, 0xf1, 0xf0, 0xc0,
-	0x22, 0xc1, 0x9c, 0x1a, 0xf3, 0x04, 0x17, 0x61, 0xbf, 0x53, 0xe5, 0xc7, 0x45, 0x68, 0x6d, 0x47,
+	0xba, 0x97, 0x87, 0xad, 0x8b, 0x99, 0xe3, 0xd6, 0x1d, 0x9f, 0x51, 0x16, 0xf5, 0x2f, 0xaa, 0xfe,
-	0x5c, 0xf8, 0x2e, 0xa6, 0xdc, 0x8f, 0x42, 0x9b, 0xee, 0xd0, 0x16, 0xf3, 0x98, 0x60, 0xbe, 0x87,
+	0xa4, 0x08, 0xcc, 0xad, 0x98, 0xb2, 0xc0, 0x43, 0x84, 0x06, 0x71, 0x64, 0x91, 0x6d, 0xd2, 0x72,
-	0xde, 0x87, 0xb3, 0x72, 0x55, 0x4d, 0x22, 0x88, 0x05, 0xd6, 0xc1, 0xe6, 0xfc, 0xcb, 0x5f, 0xaa,
+	0x7c, 0x87, 0x39, 0x81, 0x0f, 0xdf, 0x07, 0x33, 0x7c, 0x57, 0x36, 0x66, 0xd8, 0x34, 0xd6, 0x8c,
-	0xa6, 0x19, 0x4f, 0x40, 0xd2, 0x34, 0x4b, 0xeb, 0x6a, 0xe7, 0x66, 0xf5, 0xe0, 0xe8, 0x3e, 0xb5,
+	0x8d, 0xb9, 0x97, 0xbe, 0x52, 0x4b, 0x23, 0xae, 0x41, 0xd2, 0x30, 0x73, 0xeb, 0x5a, 0xe7, 0x46,
-	0xc5, 0x3e, 0x15, 0xa4, 0x8e, 0x1e, 0x77, 0xcb, 0x13, 0xbd, 0x6e, 0x19, 0xa6, 0x32, 0x9c, 0x44,
+	0x6d, 0xff, 0xf0, 0x1e, 0xb1, 0xd8, 0x1e, 0x61, 0xb8, 0x01, 0x1f, 0x75, 0x2b, 0x13, 0xbd, 0x6e,
-	0x45, 0xdf, 0x83, 0x93, 0x3c, 0xa0, 0xb6, 0x55, 0x50, 0xd1, 0xdf, 0xaa, 0x7e, 0xaa, 0xfd, 0xac,
+	0x05, 0xa4, 0x32, 0xa4, 0xbd, 0xc2, 0xef, 0x83, 0x49, 0x1a, 0x12, 0xcb, 0x2c, 0x08, 0xef, 0x6f,
-	0x0e, 0x5b, 0x48, 0x23, 0xa0, 0x76, 0x7d, 0xc1, 0x10, 0x99, 0x94, 0x5f, 0x58, 0xc1, 0xa2, 0x0f,
+	0xd5, 0x3e, 0xd5, 0x79, 0xd6, 0x86, 0x6d, 0xa4, 0x19, 0x12, 0xab, 0x31, 0xaf, 0x88, 0x4c, 0xf2,
-	0x00, 0x9c, 0xe6, 0x82, 0x88, 0x88, 0x5b, 0x45, 0xc5, 0xe0, 0x9d, 0x71, 0x31, 0x50, 0x20, 0xf5,
+	0x2f, 0x24, 0x60, 0xe1, 0x07, 0x06, 0x98, 0xa2, 0x0c, 0xb3, 0x98, 0x9a, 0x45, 0xc1, 0xe0, 0x9d,
-	0xe7, 0x0c, 0x87, 0x69, 0xfd, 0x8d, 0x0d, 0x78, 0xe5, 0x3f, 0x05, 0xb8, 0x31, 0xcc, 0x75, 0xdb,
+	0x71, 0x31, 0x10, 0x20, 0x8d, 0x0b, 0x8a, 0xc3, 0x94, 0xfc, 0x46, 0x0a, 0xbc, 0xfa, 0xdf, 0x02,
-	0xf7, 0x9a, 0x7a, 0x3b, 0x6e, 0xc1, 0x49, 0x71, 0x16, 0x50, 0xb5, 0x15, 0x73, 0xf5, 0xd7, 0xe2,
+	0x58, 0x1f, 0xb6, 0x74, 0x2b, 0xf0, 0x6d, 0x79, 0x1c, 0x37, 0xc1, 0x24, 0x3b, 0x0d, 0x89, 0x38,
-	0xf5, 0xdc, 0x39, 0x0b, 0xe8, 0x79, 0xb7, 0xfc, 0xc2, 0x27, 0x06, 0x90, 0x86, 0x58, 0x85, 0x40,
+	0x8a, 0xd9, 0xc6, 0xab, 0xc9, 0x7e, 0x6e, 0x9f, 0x86, 0xe4, 0xac, 0x5b, 0x79, 0xfe, 0x89, 0x0e,
-	0x5f, 0x49, 0xd6, 0x5d, 0x50, 0xc1, 0x36, 0xf2, 0xc4, 0xce, 0xbb, 0xe5, 0xa5, 0xc4, 0x2d, 0xcf,
+	0xb8, 0x21, 0x12, 0x2e, 0xe0, 0xd7, 0xf4, 0xbe, 0x0b, 0xc2, 0xd9, 0x7a, 0x9e, 0xd8, 0x59, 0xb7,
-	0x15, 0x75, 0x20, 0x72, 0x08, 0x17, 0x77, 0x42, 0xe2, 0x71, 0x1d, 0x96, 0xb9, 0xd4, 0xa4, 0xef,
+	0xb2, 0xa8, 0x97, 0xe5, 0xb9, 0xc2, 0x0e, 0x80, 0x2e, 0xa6, 0xec, 0x76, 0x84, 0x7d, 0x2a, 0xdd,
-	0x0b, 0x4f, 0x57, 0x1e, 0xd2, 0xa3, 0xbe, 0x66, 0x20, 0xd1, 0xde, 0x40, 0x34, 0x7c, 0x01, 0x02,
+	0x3a, 0x1e, 0x51, 0xe1, 0xfb, 0xd2, 0xd3, 0xa5, 0x07, 0x5f, 0xd1, 0x58, 0x55, 0x90, 0x70, 0x77,
-	0x7a, 0x11, 0x4e, 0x87, 0x94, 0x70, 0xdf, 0xb3, 0x26, 0x15, 0xe5, 0x24, 0x97, 0x58, 0x49, 0xb1,
+	0xc0, 0x1b, 0x3a, 0x07, 0x01, 0xbe, 0x00, 0xa6, 0x22, 0x82, 0x69, 0xe0, 0x9b, 0x93, 0x82, 0xb2,
-	0xd1, 0xa2, 0xcf, 0xc3, 0x19, 0x97, 0x72, 0x4e, 0xda, 0xd4, 0x9a, 0x52, 0x86, 0x4b, 0xc6, 0x70,
+	0x8e, 0x25, 0x12, 0x52, 0xa4, 0xb4, 0xf0, 0x8b, 0x60, 0xda, 0x23, 0x94, 0xe2, 0x36, 0x31, 0x4b,
-	0x66, 0x5f, 0x8b, 0x71, 0xac, 0xaf, 0x9c, 0x03, 0x78, 0x6d, 0x58, 0xd6, 0xf6, 0x18, 0x17, 0xe8,
+	0xc2, 0x70, 0x51, 0x19, 0x4e, 0xef, 0x49, 0x31, 0x4a, 0xf4, 0xd5, 0x33, 0x03, 0x5c, 0x1d, 0x16,
-	0xed, 0x81, 0x03, 0x50, 0x7d, 0xba, 0x15, 0x4a, 0x6f, 0x55, 0xfe, 0xcb, 0x06, 0x7c, 0x36, 0x96,
+	0xb5, 0x5d, 0x87, 0x32, 0xf8, 0xf6, 0xc0, 0x05, 0xa8, 0x3d, 0xdd, 0x0e, 0xf9, 0x6a, 0x91, 0xfe,
-	0x64, 0x8a, 0xff, 0xbb, 0x70, 0x8a, 0x09, 0xea, 0xca, 0x3d, 0x28, 0x6e, 0xce, 0xbf, 0xfc, 0xad,
+	0x4b, 0x0a, 0x7c, 0x26, 0x91, 0x64, 0x92, 0xff, 0x7b, 0xa0, 0xe4, 0x30, 0xe2, 0xf1, 0x33, 0x28,
-	0x31, 0xd5, 0x5e, 0x7d, 0xd1, 0x70, 0x98, 0xba, 0x25, 0xd1, 0xb0, 0x06, 0xad, 0xfc, 0xa1, 0x00,
+	0x6e, 0xcc, 0xbd, 0xf4, 0xed, 0x31, 0xe5, 0x5e, 0x63, 0x41, 0x71, 0x28, 0xdd, 0xe4, 0x68, 0x48,
-	0xaf, 0x0f, 0x73, 0xb9, 0x4d, 0x5c, 0xca, 0x65, 0xc6, 0x03, 0x27, 0x0a, 0x89, 0x63, 0x2a, 0x2e,
+	0x82, 0x56, 0xff, 0x54, 0x00, 0xd7, 0x86, 0x2d, 0xb9, 0x85, 0x3d, 0x42, 0x79, 0xc4, 0x43, 0x37,
-	0xc9, 0xf8, 0xa1, 0x92, 0x62, 0xa3, 0x45, 0x2f, 0xc1, 0x59, 0xce, 0xbc, 0x76, 0xe4, 0x90, 0xd0,
+	0x8e, 0xb0, 0xab, 0x32, 0x4e, 0x47, 0xfc, 0x40, 0x48, 0x91, 0xd2, 0xc2, 0x17, 0xc1, 0x0c, 0x75,
-	0x94, 0x53, 0xb2, 0xea, 0x86, 0x91, 0xe3, 0xc4, 0x02, 0x55, 0x21, 0xe4, 0xc7, 0x7e, 0x28, 0x14,
+	0xfc, 0x76, 0xec, 0xe2, 0x48, 0xa5, 0x93, 0xde, 0x75, 0x53, 0xc9, 0x91, 0xb6, 0x80, 0x35, 0x00,
-	0x86, 0x55, 0x5c, 0x2f, 0xca, 0xc8, 0xb2, 0x41, 0x34, 0x12, 0x29, 0xce, 0x58, 0xa0, 0x75, 0x38,
+	0xe8, 0x51, 0x10, 0x31, 0x81, 0x61, 0x16, 0xd7, 0x8a, 0xdc, 0x33, 0x2f, 0x10, 0x4d, 0x2d, 0x45,
-	0x79, 0xc2, 0xbc, 0xa6, 0xd9, 0xf5, 0xe4, 0x14, 0x7f, 0x93, 0x79, 0x4d, 0xac, 0x34, 0x12, 0xdf,
+	0x19, 0x0b, 0xb8, 0x06, 0x26, 0x8f, 0x1d, 0xdf, 0x56, 0xa7, 0xae, 0x6f, 0xf1, 0xb7, 0x1c, 0xdf,
-	0x61, 0x5c, 0x48, 0x89, 0xd9, 0xf2, 0x5c, 0xd6, 0x95, 0x65, 0x62, 0x21, 0xf1, 0x6d, 0x22, 0x68,
+	0x46, 0x42, 0xc3, 0xf1, 0x5d, 0x87, 0x32, 0x2e, 0x51, 0x47, 0x9e, 0x8b, 0xba, 0xb0, 0xd4, 0x16,
-	0xdb, 0x0f, 0x19, 0xe5, 0xd6, 0x74, 0x8a, 0xbf, 0x9d, 0x48, 0x71, 0xc6, 0xa2, 0xf2, 0x8f, 0xc9,
+	0x1c, 0xdf, 0xc2, 0x8c, 0xb4, 0x83, 0xc8, 0x21, 0xd4, 0x9c, 0x4a, 0xf1, 0xb7, 0xb4, 0x14, 0x65,
-	0xe1, 0x45, 0x22, 0x5b, 0x09, 0x7a, 0x1e, 0x4e, 0xb5, 0x43, 0x3f, 0x0a, 0x4c, 0x96, 0x92, 0x6c,
+	0x2c, 0xaa, 0xff, 0x2a, 0x0d, 0x4f, 0x12, 0x5e, 0x4a, 0xe0, 0x73, 0xa0, 0xd4, 0x8e, 0x82, 0x38,
-	0x7f, 0x43, 0x0a, 0xb1, 0xd6, 0xc9, 0xaa, 0xec, 0xd0, 0x50, 0x6e, 0x98, 0x49, 0x51, 0x52, 0x95,
+	0x54, 0x51, 0xd2, 0xd1, 0xfe, 0x26, 0x17, 0x22, 0xa9, 0xe3, 0x59, 0xd9, 0x21, 0x11, 0x3f, 0x30,
-	0xf7, 0xb4, 0x18, 0xc7, 0x7a, 0xf4, 0x43, 0x00, 0xa7, 0x3c, 0x93, 0x1c, 0x59, 0x72, 0x6f, 0x8f,
+	0x15, 0x22, 0x9d, 0x95, 0x77, 0xa5, 0x18, 0x25, 0x7a, 0xf8, 0x23, 0x03, 0x94, 0x7c, 0x15, 0x1c,
-	0xa9, 0x2e, 0x54, 0x7a, 0x53, 0xba, 0x3a, 0xf3, 0x1a, 0x19, 0xbd, 0x0a, 0xa7, 0xb8, 0xed, 0x07,
+	0x9e, 0x72, 0x6f, 0x8f, 0x29, 0x2f, 0x44, 0x78, 0x53, 0xba, 0x32, 0xf2, 0x12, 0x19, 0xbe, 0x02,
-	0xd4, 0x64, 0xbd, 0x14, 0x1b, 0x35, 0xa4, 0xf0, 0xbc, 0x5b, 0x5e, 0x8c, 0xc3, 0x29, 0x01, 0xd6,
+	0x4a, 0xd4, 0x0a, 0x42, 0xa2, 0xa2, 0x5e, 0x4e, 0x8c, 0x9a, 0x5c, 0x78, 0xd6, 0xad, 0x2c, 0x24,
-	0xc6, 0xe8, 0x27, 0x00, 0xc2, 0x0e, 0x71, 0x58, 0x93, 0xc8, 0xf8, 0x6a, 0x2f, 0x46, 0x5d, 0xd6,
+	0xee, 0x84, 0x00, 0x49, 0x63, 0xf8, 0x53, 0x03, 0x80, 0x0e, 0x76, 0x1d, 0x1b, 0x73, 0xff, 0xe2,
-	0xf7, 0x92, 0xf0, 0x7a, 0xd3, 0xd2, 0x6f, 0x9c, 0x81, 0x46, 0xbf, 0x00, 0x70, 0x81, 0x47, 0x47,
+	0x2c, 0x46, 0x9d, 0xd6, 0x77, 0xb5, 0x7b, 0x79, 0x68, 0xe9, 0x37, 0xca, 0x40, 0xc3, 0x0f, 0x0d,
-	0xa1, 0xf1, 0x92, 0xfb, 0x2c, 0xb9, 0x7c, 0x7b, 0xa4, 0x5c, 0x1a, 0x19, 0x80, 0xfa, 0x72, 0xaf,
+	0x30, 0x4f, 0xe3, 0xc3, 0x48, 0xad, 0xe2, 0xe7, 0xcc, 0xb9, 0x7c, 0x67, 0xa4, 0x5c, 0x9a, 0x19,
-	0x5b, 0x5e, 0xc8, 0x4a, 0x70, 0x8e, 0x40, 0xe5, 0x9f, 0x05, 0x58, 0xfa, 0xf8, 0xdb, 0x01, 0x3d,
+	0x80, 0xc6, 0x52, 0xaf, 0x5b, 0x99, 0xcf, 0x4a, 0x50, 0x8e, 0x00, 0xfc, 0xb9, 0x01, 0x66, 0xd4,
-	0x02, 0x10, 0xda, 0x71, 0xd7, 0xe5, 0x16, 0x50, 0x5d, 0xe1, 0xfd, 0x31, 0xed, 0x7e, 0xd2, 0xde,
+	0x09, 0x53, 0x73, 0x5a, 0x5c, 0xf8, 0x77, 0xc7, 0x74, 0xb0, 0x2a, 0xa3, 0xd2, 0x5b, 0xa0, 0x04,
-	0xd3, 0x1b, 0x3a, 0x11, 0xc9, 0x03, 0x90, 0xfc, 0x46, 0xbf, 0x01, 0x70, 0x91, 0xd8, 0x36, 0x0d,
+	0x14, 0x69, 0x06, 0xd5, 0x0f, 0x8b, 0xa0, 0xfc, 0xc9, 0x8f, 0x15, 0x7c, 0x68, 0x00, 0x60, 0x25,
-	0x04, 0x6d, 0xea, 0x43, 0x5b, 0xf8, 0x0c, 0xea, 0x72, 0xd5, 0xb0, 0x5a, 0xdc, 0xca, 0x42, 0xe3,
+	0x8f, 0x00, 0x35, 0x0d, 0xc1, 0xf9, 0xfd, 0x31, 0x71, 0xd6, 0xaf, 0x4d, 0xda, 0x30, 0x68, 0x11,
-	0x3c, 0x93, 0xca, 0x7f, 0x41, 0x7f, 0x56, 0x33, 0x5b, 0xd0, 0xb0, 0x89, 0x43, 0xd1, 0x0e, 0x5c,
+	0xbf, 0x8f, 0xfa, 0x6f, 0xf8, 0x3b, 0x03, 0x2c, 0x60, 0xcb, 0x22, 0x21, 0x23, 0xb6, 0xac, 0x21,
-	0x96, 0x77, 0x3d, 0xa6, 0x81, 0xc3, 0x6c, 0xc2, 0x0f, 0x89, 0x38, 0x36, 0x27, 0xd5, 0x32, 0x10,
+	0x85, 0xcf, 0xe0, 0x9a, 0xac, 0x28, 0x56, 0x0b, 0x9b, 0x59, 0x68, 0x94, 0x67, 0x02, 0xdf, 0x00,
-	0xcb, 0x8d, 0x3e, 0x3d, 0x1e, 0xf0, 0x40, 0x6f, 0x42, 0xa4, 0xef, 0xbf, 0x5c, 0x1c, 0x7d, 0x94,
+	0x17, 0x28, 0x0b, 0x22, 0x62, 0x27, 0x11, 0x57, 0xf5, 0x0d, 0xf6, 0xba, 0x95, 0x0b, 0xcd, 0x9c,
-	0x93, 0x9b, 0xac, 0x31, 0x60, 0x81, 0x2f, 0xf0, 0x42, 0xdb, 0x70, 0xc5, 0x21, 0x47, 0xd4, 0x69,
+	0x06, 0xf5, 0x59, 0x56, 0x7f, 0x6b, 0x80, 0xca, 0x13, 0x4e, 0x94, 0xd7, 0x42, 0x7e, 0x3f, 0x55,
-	0x50, 0x87, 0xda, 0xc2, 0x0f, 0x55, 0xa8, 0xa2, 0x0a, 0xb5, 0xda, 0xeb, 0x96, 0x57, 0xf6, 0xfa,
+	0xa5, 0xd1, 0xb5, 0x90, 0x83, 0x23, 0xa1, 0xe1, 0x35, 0x5b, 0x6c, 0xd7, 0x16, 0x51, 0x99, 0xc9,
-	0x95, 0x78, 0xd0, 0xbe, 0xb2, 0x01, 0xcb, 0xc3, 0x17, 0xae, 0xa7, 0x8a, 0xdf, 0x15, 0xe0, 0xda,
+	0x74, 0x1c, 0x42, 0x8a, 0x94, 0x96, 0xd7, 0x23, 0x8e, 0xcf, 0x5f, 0xc9, 0xa2, 0x30, 0xd4, 0xf5,
-	0xf0, 0x8a, 0x45, 0x3f, 0x4a, 0x87, 0x1f, 0x7d, 0xb7, 0xbd, 0x3b, 0xae, 0xd3, 0x61, 0xa6, 0x1f,
+	0xa8, 0x29, 0xc5, 0x28, 0xd1, 0x57, 0xff, 0x67, 0xf4, 0xa7, 0x4a, 0x26, 0xcd, 0x9b, 0x16, 0x76,
-	0x38, 0x38, 0xf9, 0xa0, 0xef, 0xcb, 0x46, 0x43, 0x1c, 0x6a, 0x6a, 0xea, 0x9d, 0xb1, 0x51, 0x90,
+	0x09, 0xdc, 0x06, 0x4b, 0xbc, 0x9f, 0x42, 0x24, 0x74, 0x1d, 0x0b, 0xd3, 0x03, 0xcc, 0x8e, 0x14,
-	0x20, 0xf5, 0x39, 0xdd, 0xc3, 0x88, 0xa3, 0x5a, 0x16, 0x71, 0x68, 0xe5, 0x8f, 0xa0, 0x7f, 0xfe,
+	0x47, 0x53, 0xb9, 0x5d, 0x6a, 0xf6, 0xe9, 0xd1, 0xc0, 0x0a, 0xf8, 0x26, 0x80, 0xb2, 0xc7, 0xc8,
-	0x4d, 0x3b, 0x0a, 0xfa, 0x19, 0x80, 0x4b, 0x7e, 0x40, 0xbd, 0xad, 0xc3, 0x5b, 0xf7, 0x5e, 0x69,
+	0xf9, 0x91, 0xe5, 0x52, 0x77, 0x0b, 0xcd, 0x01, 0x0b, 0x74, 0xce, 0x2a, 0xb8, 0x05, 0x96, 0x5d,
-	0xa8, 0xa9, 0xdb, 0xa4, 0xea, 0xf6, 0xa7, 0xe4, 0xf9, 0x66, 0xe3, 0xe0, 0xb6, 0x0e, 0x78, 0x18,
+	0x7c, 0x48, 0xdc, 0x26, 0x71, 0x89, 0xc5, 0x82, 0x48, 0xb8, 0x2a, 0x0a, 0x57, 0x2b, 0xbd, 0x6e,
-	0xfa, 0x01, 0xaf, 0x5f, 0xea, 0x75, 0xcb, 0x4b, 0x07, 0x79, 0x28, 0xdc, 0x8f, 0x5d, 0x71, 0xe1,
+	0x65, 0x79, 0xb7, 0x5f, 0x89, 0x06, 0xed, 0xab, 0xeb, 0xfd, 0x27, 0x92, 0xdd, 0xb8, 0xec, 0xdc,
-	0xea, 0xee, 0xa9, 0xa0, 0xa1, 0x47, 0x9c, 0x1d, 0xdf, 0x8e, 0x5c, 0xea, 0x09, 0x4d, 0xf4, 0x35,
+	0xfe, 0x50, 0x00, 0xab, 0xc3, 0xab, 0x02, 0xfc, 0x71, 0xda, 0x60, 0xca, 0xfe, 0xe1, 0xdd, 0x71,
-	0x38, 0xdf, 0xa4, 0xdc, 0x0e, 0x59, 0xa0, 0x1a, 0xaf, 0x2e, 0xef, 0x4b, 0xa6, 0x2c, 0xe7, 0x77,
+	0x55, 0x20, 0xd5, 0x61, 0x82, 0xc1, 0xee, 0x12, 0xfe, 0x80, 0x17, 0x73, 0xec, 0x12, 0x75, 0x51,
-	0x52, 0x15, 0xce, 0xda, 0xa1, 0xeb, 0xb0, 0x18, 0x85, 0x8e, 0xa9, 0xe2, 0x79, 0x63, 0x5e, 0xbc,
+	0xde, 0x19, 0x1b, 0x05, 0x0e, 0xd2, 0x98, 0x95, 0xef, 0x04, 0x76, 0xc5, 0xb3, 0x80, 0x5d, 0x52,
-	0x8b, 0xf7, 0xb0, 0x94, 0x57, 0x36, 0xe0, 0xa4, 0xe4, 0x89, 0xae, 0xc2, 0x62, 0x48, 0x1e, 0xaa,
+	0xfd, 0xb3, 0xd1, 0x3f, 0x63, 0xa4, 0x55, 0x1b, 0xfe, 0xc2, 0x00, 0x8b, 0x41, 0x48, 0xfc, 0xcd,
-	0xa8, 0x0b, 0xf5, 0x19, 0x69, 0x82, 0xc9, 0x43, 0x2c, 0x65, 0x95, 0x3f, 0x5d, 0x83, 0x4b, 0x7d,
+	0x83, 0x9b, 0x77, 0x5f, 0x6e, 0x8a, 0xc9, 0x46, 0x85, 0xea, 0xd6, 0xa7, 0xe4, 0xf9, 0x66, 0x73,
-	0x6b, 0x41, 0x6b, 0xb0, 0xc0, 0x9a, 0x86, 0x03, 0x34, 0x41, 0x0b, 0xb7, 0x76, 0x70, 0x81, 0x35,
+	0xff, 0x96, 0x74, 0x78, 0x10, 0x05, 0x21, 0x6d, 0x5c, 0xec, 0x75, 0x2b, 0x8b, 0xfb, 0x79, 0x28,
-	0xd1, 0xeb, 0x70, 0x5a, 0xbf, 0x5e, 0x0c, 0x68, 0x39, 0x99, 0x3b, 0x95, 0x54, 0xde, 0x2c, 0x69,
+	0xd4, 0x8f, 0x5d, 0xf5, 0xc0, 0xca, 0xce, 0x09, 0x23, 0x91, 0x8f, 0xdd, 0xed, 0xc0, 0x8a, 0x3d,
-	0x38, 0x49, 0xc4, 0x98, 0x2b, 0x0e, 0xb4, 0x65, 0x4e, 0x89, 0xe6, 0x40, 0x5b, 0x58, 0xca, 0xfa,
+	0xe2, 0x33, 0x49, 0xf4, 0x55, 0x30, 0x67, 0x13, 0x6a, 0x45, 0x4e, 0x28, 0x1e, 0x37, 0x99, 0xde,
-	0x17, 0x3f, 0xf9, 0x94, 0x8b, 0x5f, 0x37, 0xd3, 0xf4, 0x54, 0x7e, 0xae, 0xc8, 0x0c, 0xc9, 0x2f,
+	0x17, 0x55, 0x5a, 0xce, 0x6d, 0xa7, 0x2a, 0x94, 0xb5, 0x83, 0xd7, 0x40, 0x31, 0x8e, 0x5c, 0x95,
-	0xc2, 0xe9, 0x96, 0x1f, 0xba, 0x44, 0xa8, 0xdb, 0x23, 0x33, 0xff, 0x7c, 0x5d, 0x49, 0xb1, 0xd1,
+	0xc5, 0x73, 0xca, 0xbc, 0x78, 0x07, 0xed, 0x22, 0x2e, 0xaf, 0xae, 0x83, 0x49, 0xce, 0x13, 0x5e,
-	0xca, 0x01, 0x40, 0x30, 0xe1, 0x50, 0x6b, 0x26, 0x3f, 0x00, 0xdc, 0x91, 0x42, 0xac, 0x75, 0xe8,
+	0x01, 0xc5, 0x08, 0x3f, 0x10, 0x5e, 0xe7, 0x1b, 0xd3, 0xdc, 0x04, 0xe1, 0x07, 0x88, 0xcb, 0xaa,
-	0x3e, 0x9c, 0x69, 0xd2, 0x16, 0x89, 0x1c, 0x61, 0xcd, 0xaa, 0x12, 0xda, 0x1e, 0x41, 0x09, 0xd5,
+	0x7f, 0xb9, 0x0a, 0x16, 0xfb, 0xf6, 0x02, 0x57, 0x41, 0xc1, 0xb1, 0x15, 0x07, 0xa0, 0x9c, 0x16,
-	0xe7, 0xe5, 0x04, 0xb1, 0xa3, 0xe3, 0xe2, 0x18, 0x00, 0xbd, 0x00, 0x67, 0x5c, 0x72, 0xca, 0xdc,
+	0x6e, 0x6e, 0xa3, 0x82, 0x63, 0xc3, 0xd7, 0xc0, 0x94, 0x9c, 0x10, 0x15, 0x68, 0x45, 0x97, 0x00,
-	0xc8, 0xb5, 0xe6, 0xd6, 0xc1, 0x26, 0xd0, 0x66, 0xfb, 0x5a, 0x84, 0x63, 0x9d, 0xec, 0x8c, 0xf4,
+	0x21, 0xe5, 0xaf, 0x77, 0xea, 0x8e, 0x13, 0x51, 0xe6, 0x82, 0x03, 0x69, 0xa9, 0x5b, 0x22, 0x39,
-	0xd4, 0x76, 0x22, 0xce, 0x3a, 0xd4, 0x28, 0x2d, 0xb8, 0x0e, 0x36, 0x67, 0xd3, 0xce, 0xb8, 0xdb,
+	0x90, 0x16, 0xe2, 0xb2, 0xfe, 0xcd, 0x4f, 0x3e, 0xe5, 0xe6, 0xd7, 0xd4, 0xc4, 0x52, 0xca, 0xd7,
-	0xa7, 0xc7, 0x03, 0x1e, 0x0a, 0x8c, 0x79, 0xca, 0x79, 0x3e, 0x03, 0xa6, 0x45, 0x38, 0xd6, 0xe5,
+	0xab, 0xcc, 0x20, 0xf2, 0x02, 0x98, 0x6a, 0x05, 0x91, 0x87, 0x99, 0x78, 0xa1, 0x33, 0x3d, 0xe6,
-	0xc1, 0x8c, 0xfd, 0xc2, 0x30, 0x30, 0xe3, 0x3c, 0xe0, 0x81, 0xbe, 0x08, 0xe7, 0x5c, 0x72, 0xba,
+	0x37, 0x84, 0x14, 0x29, 0x2d, 0x6f, 0xb2, 0x98, 0xc3, 0x5c, 0x62, 0x4e, 0xe7, 0x9b, 0xac, 0xdb,
-	0x47, 0xbd, 0xb6, 0x38, 0xb6, 0x16, 0xd7, 0xc1, 0x66, 0xb1, 0xbe, 0xd8, 0xeb, 0x96, 0xe7, 0xf6,
+	0x5c, 0x88, 0xa4, 0x0e, 0xde, 0x03, 0xd3, 0x36, 0x69, 0xe1, 0xd8, 0x65, 0xe6, 0x8c, 0x48, 0xa1,
-	0x63, 0x21, 0x4e, 0xf5, 0xca, 0x98, 0x79, 0xc6, 0xf8, 0xb9, 0x8c, 0x71, 0x2c, 0xc4, 0xa9, 0x5e,
+	0xad, 0x11, 0xa4, 0x50, 0x63, 0x8e, 0x57, 0xc5, 0x6d, 0xe9, 0x17, 0x25, 0x00, 0xf0, 0x79, 0x30,
-	0x0e, 0x68, 0x01, 0x11, 0xf2, 0x70, 0x59, 0x4b, 0xf9, 0x01, 0xed, 0x50, 0x8b, 0x71, 0xac, 0x47,
+	0xed, 0xe1, 0x13, 0xc7, 0x8b, 0x3d, 0x73, 0x76, 0xcd, 0xd8, 0x30, 0xa4, 0xd9, 0x9e, 0x14, 0xa1,
-	0x9b, 0x70, 0xd6, 0x25, 0xa7, 0x6a, 0x98, 0xb6, 0x96, 0x55, 0xd8, 0x05, 0x39, 0x6b, 0xee, 0x1b,
+	0x44, 0xc7, 0x2b, 0x23, 0x39, 0xb1, 0xdc, 0x98, 0x3a, 0x1d, 0xa2, 0x94, 0x26, 0x10, 0x05, 0x57,
-	0x19, 0x4e, 0xb4, 0xca, 0x92, 0x79, 0xda, 0x72, 0x25, 0x63, 0x69, 0x64, 0x38, 0xd1, 0xca, 0x22,
+	0x57, 0xc6, 0x9d, 0x3e, 0x3d, 0x1a, 0x58, 0x21, 0xc0, 0x1c, 0x5f, 0x2c, 0x9e, 0xcb, 0x80, 0x49,
-	0x8e, 0x3c, 0xf6, 0x20, 0xa2, 0xda, 0x18, 0xa9, 0xcc, 0x24, 0x45, 0x7c, 0x37, 0x55, 0xe1, 0xac,
+	0x11, 0x4a, 0x74, 0x79, 0x30, 0x65, 0x3f, 0x3f, 0x0c, 0x4c, 0x2d, 0x1e, 0x58, 0x01, 0xbf, 0x0c,
-	0x9d, 0x1c, 0x66, 0xdd, 0xc8, 0x11, 0x2c, 0x70, 0xe8, 0x41, 0xcb, 0xba, 0xa4, 0xf2, 0xaf, 0xe6,
+	0x66, 0x3d, 0x7c, 0xb2, 0x4b, 0xfc, 0x36, 0x3b, 0x32, 0x17, 0xd6, 0x8c, 0x8d, 0x62, 0x63, 0xa1,
-	0xa2, 0xfd, 0x44, 0x8a, 0x33, 0x16, 0x88, 0xc2, 0x49, 0xea, 0x45, 0xae, 0x75, 0x59, 0xcd, 0x16,
+	0xd7, 0xad, 0xcc, 0xee, 0x25, 0x42, 0x94, 0xea, 0x85, 0xb1, 0xe3, 0x2b, 0xe3, 0x0b, 0x19, 0xe3,
-	0x23, 0x29, 0xc1, 0xe4, 0xe4, 0xec, 0x7a, 0x91, 0x8b, 0x55, 0x78, 0xf4, 0x3a, 0x5c, 0x74, 0xc9,
+	0x44, 0x88, 0x52, 0x3d, 0x7f, 0x74, 0x42, 0xcc, 0xf8, 0xe5, 0x32, 0x17, 0xf3, 0x4d, 0xf0, 0x81,
-	0xa9, 0x6c, 0x07, 0x34, 0x14, 0x72, 0xcc, 0x5e, 0x55, 0x8b, 0x5f, 0x91, 0xf7, 0xf9, 0x7e, 0x56,
+	0x14, 0xa3, 0x44, 0x0f, 0x37, 0xc0, 0x8c, 0x87, 0x4f, 0xc4, 0xc0, 0x62, 0x2e, 0x09, 0xb7, 0xf3,
-	0x81, 0xf3, 0x76, 0xca, 0x91, 0x79, 0x19, 0xc7, 0x2b, 0x19, 0xc7, 0xac, 0x02, 0xe7, 0xed, 0x64,
+	0xbc, 0x93, 0xd9, 0x53, 0x32, 0xa4, 0xb5, 0xc2, 0xd2, 0xf1, 0xa5, 0xe5, 0x72, 0xc6, 0x52, 0xc9,
-	0xa6, 0x43, 0xfa, 0x20, 0x62, 0x21, 0x6d, 0x5a, 0xff, 0xa7, 0x66, 0x7a, 0x95, 0x69, 0x6c, 0x64,
+	0x90, 0xd6, 0xf2, 0x24, 0x8e, 0x7d, 0xe7, 0x7e, 0x4c, 0xa4, 0x31, 0x14, 0x91, 0xd1, 0x49, 0x7c,
-	0x38, 0xd1, 0xa2, 0x4e, 0xfc, 0xea, 0xb2, 0xd4, 0x31, 0xbc, 0x3b, 0xda, 0x4e, 0x7e, 0x10, 0x6e,
+	0x27, 0x55, 0xa1, 0xac, 0x1d, 0x1f, 0x18, 0xbc, 0xd8, 0x65, 0x4e, 0xe8, 0x92, 0xfd, 0x96, 0x79,
-	0x85, 0x21, 0x39, 0xd3, 0x37, 0x4d, 0xf6, 0xbd, 0x85, 0x38, 0x9c, 0x22, 0x8e, 0x73, 0xd0, 0xb2,
+	0x51, 0xc4, 0x5f, 0xf4, 0x9e, 0x7b, 0x5a, 0x8a, 0x32, 0x16, 0x90, 0x80, 0x49, 0xe2, 0xc7, 0x9e,
-	0xae, 0xaa, 0xdc, 0x8f, 0xfa, 0x06, 0x49, 0xba, 0xce, 0x96, 0x04, 0xc1, 0x1a, 0x4b, 0x82, 0xfa,
+	0x79, 0x49, 0x34, 0x4c, 0x23, 0x49, 0x41, 0x7d, 0x73, 0x76, 0xfc, 0xd8, 0x43, 0xc2, 0x3d, 0x7c,
-	0x9e, 0x2c, 0x8d, 0xb5, 0xf1, 0x82, 0x1e, 0x48, 0x10, 0xac, 0xb1, 0xd4, 0x4a, 0xbd, 0xb3, 0x83,
+	0x0d, 0x2c, 0x78, 0xf8, 0x84, 0x97, 0x03, 0x12, 0x31, 0x3e, 0xca, 0xac, 0x88, 0xcd, 0x2f, 0xf3,
-	0x96, 0xf5, 0xff, 0x63, 0x5e, 0xa9, 0x04, 0xc1, 0x1a, 0x0b, 0x31, 0x58, 0xf4, 0x7c, 0x61, 0x5d,
+	0x26, 0x65, 0x2f, 0xab, 0x40, 0x79, 0x3b, 0xb1, 0xd0, 0xf1, 0x33, 0x0b, 0x2f, 0x67, 0x16, 0x66,
-	0x1b, 0xcb, 0xf5, 0xac, 0x2e, 0x9c, 0xdb, 0xbe, 0xc0, 0x12, 0x03, 0xfd, 0x0a, 0x40, 0x18, 0xa4,
+	0x15, 0x28, 0x6f, 0xc7, 0x23, 0x1d, 0x91, 0xfb, 0xb1, 0x13, 0x11, 0xdb, 0xfc, 0x9c, 0xe8, 0x6b,
-	0x25, 0x7a, 0x5d, 0xad, 0xf2, 0xdd, 0xd1, 0x42, 0x56, 0xd3, 0xda, 0xde, 0xf5, 0x44, 0x78, 0x96,
+	0x44, 0xa4, 0x91, 0x92, 0x21, 0xad, 0x85, 0x9d, 0x64, 0xb2, 0x35, 0xc5, 0x35, 0xbc, 0x33, 0xda,
-	0x4e, 0xe9, 0x99, 0x33, 0x90, 0x61, 0x81, 0x7e, 0x0f, 0xe0, 0x65, 0xd2, 0xd4, 0x33, 0x3b, 0x71,
+	0x4a, 0xbe, 0x1f, 0x6d, 0x46, 0x11, 0x3e, 0x95, 0x2f, 0x4d, 0x76, 0xa6, 0x85, 0x14, 0x94, 0xb0,
-	0x32, 0x27, 0xa8, 0xa4, 0x32, 0x72, 0x67, 0xd4, 0x65, 0x5e, 0xf7, 0x7d, 0xa7, 0x6e, 0xf5, 0xba,
+	0xeb, 0xee, 0xb7, 0xcc, 0x2b, 0x22, 0xf6, 0xa3, 0x7e, 0x41, 0x74, 0xd5, 0xd9, 0xe4, 0x20, 0x48,
-	0xe5, 0xcb, 0x5b, 0x17, 0xa0, 0xe2, 0x0b, 0xb9, 0xa0, 0x3f, 0x03, 0xb8, 0x62, 0xba, 0x68, 0x86,
+	0x62, 0x71, 0xd0, 0xc0, 0xe7, 0xa9, 0xb1, 0x3a, 0x5e, 0xd0, 0x7d, 0x0e, 0x82, 0x24, 0x96, 0xd8,
-	0x61, 0x59, 0x25, 0x90, 0x8e, 0x3a, 0x81, 0xfd, 0x38, 0x3a, 0x8f, 0x57, 0x4d, 0x1e, 0x57, 0x06,
+	0xa9, 0x7f, 0xba, 0xdf, 0x32, 0x3f, 0x3f, 0xe6, 0x9d, 0x72, 0x10, 0x24, 0xb1, 0xa0, 0x03, 0x8a,
-	0xf4, 0x78, 0x90, 0x1a, 0xfa, 0x1b, 0x80, 0x0b, 0x4d, 0x1a, 0x50, 0xaf, 0x49, 0x3d, 0x5b, 0x72,
+	0x7e, 0xc0, 0xcc, 0xab, 0x63, 0x79, 0x9e, 0xc5, 0x83, 0x73, 0x2b, 0x60, 0x88, 0x63, 0xc0, 0x5f,
-	0x5d, 0x1f, 0xc9, 0xa3, 0xac, 0x9f, 0xeb, 0x4e, 0x06, 0x42, 0xd3, 0xac, 0x1a, 0x9a, 0x0b, 0x59,
+	0x1b, 0x00, 0x84, 0x69, 0x8a, 0x5e, 0x1b, 0xc9, 0xc0, 0xd4, 0x07, 0x59, 0x4b, 0x73, 0x7b, 0xc7,
-	0xd5, 0x79, 0xb7, 0x7c, 0x25, 0x75, 0xcd, 0x6a, 0x70, 0x8e, 0x25, 0xfa, 0x35, 0x80, 0x4b, 0xe9,
+	0x67, 0xd1, 0x69, 0x3a, 0x7a, 0x64, 0xee, 0x40, 0x86, 0x05, 0xfc, 0xa3, 0x01, 0x2e, 0x61, 0x5b,
-	0x06, 0xe8, 0x2b, 0x65, 0x63, 0x8c, 0x75, 0xa0, 0xc6, 0xd7, 0xad, 0x3c, 0x20, 0xee, 0x67, 0x80,
+	0x0e, 0x22, 0xd8, 0xcd, 0xdc, 0xa0, 0xb2, 0x88, 0xc8, 0xed, 0x51, 0xa7, 0x79, 0x23, 0x08, 0xdc,
-	0xfe, 0x02, 0xe4, 0xa4, 0x16, 0x3f, 0xf3, 0xb8, 0x55, 0x51, 0xb9, 0x7c, 0x6f, 0xe4, 0xb9, 0x4c,
+	0x86, 0xd9, 0xeb, 0x56, 0x2e, 0x6d, 0x9e, 0x83, 0x8a, 0xce, 0xe5, 0x02, 0xff, 0x6a, 0x80, 0x65,
-	0x10, 0x74, 0x2a, 0x5f, 0x4a, 0x47, 0xc1, 0x44, 0x73, 0xde, 0x2d, 0xaf, 0x66, 0x33, 0x99, 0x28,
+	0x55, 0x45, 0x33, 0x0c, 0x2b, 0x22, 0x80, 0x64, 0xd4, 0x01, 0xec, 0xc7, 0x91, 0x71, 0xbc, 0xa2,
-	0x70, 0x96, 0x21, 0xfa, 0x29, 0x80, 0x0b, 0x34, 0x9d, 0xb8, 0xb9, 0xf5, 0xfc, 0x48, 0x92, 0x78,
+	0xe2, 0xb8, 0x3c, 0xa0, 0x47, 0x83, 0xd4, 0xe0, 0x3f, 0x0c, 0x30, 0x6f, 0x93, 0x90, 0xf8, 0x36,
-	0xe1, 0x10, 0xaf, 0xff, 0x41, 0xc8, 0xa8, 0x38, 0xce, 0x61, 0xcb, 0x09, 0x92, 0x9e, 0x12, 0x37,
+	0xf1, 0x2d, 0xce, 0x75, 0x6d, 0x24, 0x93, 0x66, 0x3f, 0xd7, 0xed, 0x0c, 0x84, 0xa4, 0x59, 0x53,
-	0x70, 0xa8, 0xf5, 0xb9, 0x11, 0x4f, 0x90, 0xbb, 0x3a, 0x2e, 0x8e, 0x01, 0xd6, 0xe4, 0xcb, 0xa7,
+	0x34, 0xe7, 0xb3, 0xaa, 0xb3, 0x6e, 0xe5, 0x72, 0xba, 0x34, 0xab, 0x41, 0x39, 0x96, 0xf0, 0x37,
-	0xef, 0xe4, 0xa0, 0x65, 0x58, 0x3c, 0xa1, 0x67, 0x7a, 0xb0, 0xc7, 0xf2, 0x27, 0x6a, 0xc2, 0xa9,
+	0x06, 0x58, 0x4c, 0x0f, 0x40, 0x3e, 0x29, 0xeb, 0x63, 0xcc, 0x03, 0xd1, 0xbe, 0x6e, 0xe6, 0x01,
-	0x0e, 0x71, 0xa2, 0xf8, 0xf1, 0x36, 0xe2, 0xae, 0x8b, 0x75, 0xf0, 0xaf, 0x16, 0xde, 0x00, 0x6b,
+	0x51, 0x3f, 0x03, 0xf8, 0x37, 0x83, 0x77, 0x6a, 0xc9, 0xdc, 0x48, 0xcd, 0xaa, 0x88, 0xe5, 0x7b,
-	0x8f, 0x00, 0xbc, 0x72, 0xf1, 0x81, 0x7e, 0xa6, 0xb4, 0x7e, 0x0b, 0xe0, 0xca, 0xc0, 0xd9, 0xbd,
+	0x23, 0x8f, 0xa5, 0x46, 0x90, 0xa1, 0x7c, 0x31, 0x6d, 0x05, 0xb5, 0xe6, 0xac, 0x5b, 0x59, 0xc9,
-	0x80, 0xd1, 0x83, 0x3c, 0xa3, 0xb7, 0x46, 0x7d, 0x08, 0x1b, 0x22, 0x64, 0x5e, 0x5b, 0x4d, 0x1e,
+	0x46, 0x52, 0x2b, 0x50, 0x96, 0x21, 0xfc, 0x99, 0x01, 0xe6, 0x49, 0xda, 0x71, 0x53, 0xf3, 0xb9,
-	0x59, 0x7a, 0x3f, 0x07, 0x70, 0xb9, 0xff, 0x38, 0x3c, 0xcb, 0x7c, 0x55, 0x1e, 0x15, 0xe0, 0x95,
+	0x91, 0x04, 0xf1, 0xdc, 0x26, 0x5e, 0xfe, 0x4a, 0x93, 0x51, 0x51, 0x94, 0xc3, 0xe6, 0x1d, 0x24,
-	0x8b, 0x07, 0x26, 0x14, 0x26, 0x2f, 0xc3, 0xf1, 0xbc, 0xb0, 0x61, 0xfa, 0xca, 0x4c, 0x1e, 0x95,
+	0x39, 0xc1, 0x5e, 0xe8, 0x12, 0xf3, 0x0b, 0x23, 0xee, 0x20, 0x77, 0xa4, 0x5f, 0x94, 0x00, 0xac,
-	0x1f, 0x00, 0x38, 0x7f, 0x3f, 0xb1, 0x8b, 0xff, 0x87, 0x1f, 0xf9, 0xdb, 0x3e, 0xee, 0x3f, 0xa9,
+	0xf2, 0xc9, 0xa7, 0xef, 0xe6, 0xc0, 0x25, 0x50, 0x3c, 0x26, 0xa7, 0xb2, 0xb1, 0x47, 0xfc, 0x4f,
-	0x82, 0xe3, 0x2c, 0x6e, 0xe5, 0xaf, 0x00, 0xae, 0x5e, 0xd8, 0x58, 0xe5, 0x13, 0x94, 0x38, 0x8e,
+	0x68, 0x83, 0x52, 0x07, 0xbb, 0x71, 0x32, 0xbc, 0x8d, 0xb8, 0xea, 0x22, 0xe9, 0xfc, 0x8d, 0xc2,
-	0xff, 0x50, 0xff, 0x45, 0x33, 0x9b, 0x3e, 0x41, 0xb7, 0x94, 0x14, 0x1b, 0x6d, 0x26, 0x7b, 0x85,
+	0xeb, 0xc6, 0xea, 0x43, 0x03, 0x5c, 0x3e, 0xff, 0x42, 0x3f, 0x53, 0x5a, 0xbf, 0x37, 0xc0, 0xf2,
-	0xcf, 0x2a, 0x7b, 0x95, 0xbf, 0x03, 0x78, 0xed, 0xe3, 0x2a, 0xf1, 0x99, 0x6c, 0xe9, 0x26, 0x9c,
+	0xc0, 0xdd, 0x3d, 0x87, 0xd1, 0xfd, 0x3c, 0xa3, 0xb7, 0x46, 0x7d, 0x09, 0x9b, 0x2c, 0x72, 0xfc,
-	0x35, 0x43, 0xd1, 0x99, 0xda, 0x4e, 0xf3, 0x0e, 0x30, 0x4d, 0xe3, 0x0c, 0x27, 0xda, 0xfa, 0x8d,
+	0xb6, 0xe8, 0x3c, 0xb2, 0xf4, 0x7e, 0x69, 0x80, 0xa5, 0xfe, 0xeb, 0xf0, 0x2c, 0xe3, 0x55, 0x7d,
-	0xc7, 0x4f, 0x4a, 0x13, 0x1f, 0x3e, 0x29, 0x4d, 0x7c, 0xf4, 0xa4, 0x34, 0xf1, 0x83, 0x5e, 0x09,
+	0x58, 0x00, 0x97, 0xcf, 0x6f, 0x98, 0x60, 0xa4, 0x27, 0xc3, 0xf1, 0x4c, 0xd8, 0x20, 0x9d, 0x32,
-	0x3c, 0xee, 0x95, 0xc0, 0x87, 0xbd, 0x12, 0xf8, 0xa8, 0x57, 0x02, 0xff, 0xea, 0x95, 0xc0, 0x2f,
+	0xf5, 0x50, 0xf9, 0x81, 0x01, 0xe6, 0xee, 0x69, 0xbb, 0xe4, 0x7f, 0x1d, 0x23, 0x9f, 0xed, 0x93,
-	0xff, 0x5d, 0x9a, 0xf8, 0xce, 0x8c, 0x01, 0xff, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x0a, 0x6f,
+	0xfa, 0x93, 0x2a, 0x28, 0xca, 0xe2, 0x56, 0xff, 0x6e, 0x80, 0x95, 0x73, 0x0b, 0x2b, 0x1f, 0x41,
-	0x04, 0x49, 0xd3, 0x1e, 0x00, 0x00,
+	0xb1, 0xeb, 0x06, 0x0f, 0xe4, 0x4f, 0x34, 0x99, 0x9f, 0xcc, 0x36, 0x85, 0x14, 0x29, 0x6d, 0x26,
+	0x7a, 0x85, 0xcf, 0x2a, 0x7a, 0xd5, 0x7f, 0x1a, 0xe0, 0xea, 0x27, 0x65, 0xe2, 0x33, 0x39, 0xd2,
+	0x0d, 0x30, 0xa3, 0x9a, 0xa2, 0x53, 0x71, 0x9c, 0x6a, 0x0e, 0x50, 0x45, 0xe3, 0x14, 0x69, 0x6d,
+	0xe3, 0xfa, 0xa3, 0xc7, 0xe5, 0x89, 0x8f, 0x1e, 0x97, 0x27, 0x3e, 0x7e, 0x5c, 0x9e, 0xf8, 0x61,
+	0xaf, 0x6c, 0x3c, 0xea, 0x95, 0x8d, 0x8f, 0x7a, 0x65, 0xe3, 0xe3, 0x5e, 0xd9, 0xf8, 0x77, 0xaf,
+	0x6c, 0xfc, 0xea, 0x3f, 0xe5, 0x89, 0xef, 0x4e, 0x2b, 0xf0, 0xff, 0x07, 0x00, 0x00, 0xff, 0xff,
+	0x8c, 0x3a, 0x89, 0x32, 0x37, 0x20, 0x00, 0x00,
 }

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto

@@ -100,6 +100,10 @@ message CustomResourceDefinitionSpec {
   optional string group = 1;
 
   // Version is the version this resource belongs in
+  // Should be always first item in Versions field if provided.
+  // Optional, but at least one of Version or Versions must be set.
+  // Deprecated: Please use `Versions`.
+  // +optional
   optional string version = 2;
 
   // Names are the names used to describe this custom resource
@@ -115,6 +119,18 @@ message CustomResourceDefinitionSpec {
   // Subresources describes the subresources for CustomResources
   // +optional
   optional CustomResourceSubresources subresources = 6;
+
+  // Versions is the list of all supported versions for this resource.
+  // If Version field is provided, this field is optional.
+  // Validation: All versions must use the same validation schema for now. i.e., top
+  // level Validation field is applied to all of these versions.
+  // Order: The version name will be used to compute the order.
+  // If the version string is "kube-like", it will sort above non "kube-like" version strings, which are ordered
+  // lexicographically. "Kube-like" versions start with a "v", then are followed by a number (the major version),
+  // then optionally the string "alpha" or "beta" and another number (the minor version). These are sorted first
+  // by GA > beta > alpha, and then by comparing major version, then minor version. An example sorted list of
+  // versions: v10, v2, v1, v11beta2, v10beta3, v3beta1, v12alpha1, v11alpha2, foo1, foo10.
+  repeated CustomResourceDefinitionVersion versions = 7;
 }
 
 // CustomResourceDefinitionStatus indicates the state of the CustomResourceDefinition
@@ -125,6 +141,26 @@ message CustomResourceDefinitionStatus {
   // AcceptedNames are the names that are actually being used to serve discovery
   // They may be different than the names in spec.
   optional CustomResourceDefinitionNames acceptedNames = 2;
+
+  // StoredVersions are all versions of CustomResources that were ever persisted. Tracking these
+  // versions allows a migration path for stored versions in etcd. The field is mutable
+  // so the migration controller can first finish a migration to another version (i.e.
+  // that no old objects are left in the storage), and then remove the rest of the
+  // versions from this list.
+  // None of the versions in this list can be removed from the spec.Versions field.
+  repeated string storedVersions = 3;
+}
+
+message CustomResourceDefinitionVersion {
+  // Name is the version name, e.g. “v1”, “v2beta1”, etc.
+  optional string name = 1;
+
+  // Served is a flag enabling/disabling this version from being served via REST APIs
+  optional bool served = 2;
+
+  // Storage flags the version as storage version. There must be exactly one
+  // flagged as storage version.
+  optional bool storage = 3;
 }
 
 // CustomResourceSubresourceScale defines how to serve the scale subresource for CustomResources.

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go

@@ -25,7 +25,11 @@ type CustomResourceDefinitionSpec struct {
 	// Group is the group this resource belongs in
 	Group string `json:"group" protobuf:"bytes,1,opt,name=group"`
 	// Version is the version this resource belongs in
-	Version string `json:"version" protobuf:"bytes,2,opt,name=version"`
+	// Should be always first item in Versions field if provided.
+	// Optional, but at least one of Version or Versions must be set.
+	// Deprecated: Please use `Versions`.
+	// +optional
+	Version string `json:"version,omitempty" protobuf:"bytes,2,opt,name=version"`
 	// Names are the names used to describe this custom resource
 	Names CustomResourceDefinitionNames `json:"names" protobuf:"bytes,3,opt,name=names"`
 	// Scope indicates whether this resource is cluster or namespace scoped.  Default is namespaced
@@ -36,6 +40,27 @@ type CustomResourceDefinitionSpec struct {
 	// Subresources describes the subresources for CustomResources
 	// +optional
 	Subresources *CustomResourceSubresources `json:"subresources,omitempty" protobuf:"bytes,6,opt,name=subresources"`
+	// Versions is the list of all supported versions for this resource.
+	// If Version field is provided, this field is optional.
+	// Validation: All versions must use the same validation schema for now. i.e., top
+	// level Validation field is applied to all of these versions.
+	// Order: The version name will be used to compute the order.
+	// If the version string is "kube-like", it will sort above non "kube-like" version strings, which are ordered
+	// lexicographically. "Kube-like" versions start with a "v", then are followed by a number (the major version),
+	// then optionally the string "alpha" or "beta" and another number (the minor version). These are sorted first
+	// by GA > beta > alpha, and then by comparing major version, then minor version. An example sorted list of
+	// versions: v10, v2, v1, v11beta2, v10beta3, v3beta1, v12alpha1, v11alpha2, foo1, foo10.
+	Versions []CustomResourceDefinitionVersion `json:"versions,omitempty" protobuf:"bytes,7,rep,name=versions"`
+}
+
+type CustomResourceDefinitionVersion struct {
+	// Name is the version name, e.g. “v1”, “v2beta1”, etc.
+	Name string `json:"name" protobuf:"bytes,1,opt,name=name"`
+	// Served is a flag enabling/disabling this version from being served via REST APIs
+	Served bool `json:"served" protobuf:"varint,2,opt,name=served"`
+	// Storage flags the version as storage version. There must be exactly one
+	// flagged as storage version.
+	Storage bool `json:"storage" protobuf:"varint,3,opt,name=storage"`
 }
 
 // CustomResourceDefinitionNames indicates the names to serve this CustomResourceDefinition
@@ -117,6 +142,14 @@ type CustomResourceDefinitionStatus struct {
 	// AcceptedNames are the names that are actually being used to serve discovery
 	// They may be different than the names in spec.
 	AcceptedNames CustomResourceDefinitionNames `json:"acceptedNames" protobuf:"bytes,2,opt,name=acceptedNames"`
+
+	// StoredVersions are all versions of CustomResources that were ever persisted. Tracking these
+	// versions allows a migration path for stored versions in etcd. The field is mutable
+	// so the migration controller can first finish a migration to another version (i.e.
+	// that no old objects are left in the storage), and then remove the rest of the
+	// versions from this list.
+	// None of the versions in this list can be removed from the spec.Versions field.
+	StoredVersions []string `json:"storedVersions" protobuf:"bytes,3,rep,name=storedVersions"`
 }
 
 // CustomResourceCleanupFinalizer is the name of the finalizer which will delete instances of

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.conversion.go

@@ -48,6 +48,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_apiextensions_CustomResourceDefinitionSpec_To_v1beta1_CustomResourceDefinitionSpec,
 		Convert_v1beta1_CustomResourceDefinitionStatus_To_apiextensions_CustomResourceDefinitionStatus,
 		Convert_apiextensions_CustomResourceDefinitionStatus_To_v1beta1_CustomResourceDefinitionStatus,
+		Convert_v1beta1_CustomResourceDefinitionVersion_To_apiextensions_CustomResourceDefinitionVersion,
+		Convert_apiextensions_CustomResourceDefinitionVersion_To_v1beta1_CustomResourceDefinitionVersion,
 		Convert_v1beta1_CustomResourceSubresourceScale_To_apiextensions_CustomResourceSubresourceScale,
 		Convert_apiextensions_CustomResourceSubresourceScale_To_v1beta1_CustomResourceSubresourceScale,
 		Convert_v1beta1_CustomResourceSubresourceStatus_To_apiextensions_CustomResourceSubresourceStatus,
@@ -220,6 +222,7 @@ func autoConvert_v1beta1_CustomResourceDefinitionSpec_To_apiextensions_CustomRes
 		out.Validation = nil
 	}
 	out.Subresources = (*apiextensions.CustomResourceSubresources)(unsafe.Pointer(in.Subresources))
+	out.Versions = *(*[]apiextensions.CustomResourceDefinitionVersion)(unsafe.Pointer(&in.Versions))
 	return nil
 }
 
@@ -245,6 +248,7 @@ func autoConvert_apiextensions_CustomResourceDefinitionSpec_To_v1beta1_CustomRes
 		out.Validation = nil
 	}
 	out.Subresources = (*CustomResourceSubresources)(unsafe.Pointer(in.Subresources))
+	out.Versions = *(*[]CustomResourceDefinitionVersion)(unsafe.Pointer(&in.Versions))
 	return nil
 }
 
@@ -258,6 +262,7 @@ func autoConvert_v1beta1_CustomResourceDefinitionStatus_To_apiextensions_CustomR
 	if err := Convert_v1beta1_CustomResourceDefinitionNames_To_apiextensions_CustomResourceDefinitionNames(&in.AcceptedNames, &out.AcceptedNames, s); err != nil {
 		return err
 	}
+	out.StoredVersions = *(*[]string)(unsafe.Pointer(&in.StoredVersions))
 	return nil
 }
 
@@ -271,6 +276,7 @@ func autoConvert_apiextensions_CustomResourceDefinitionStatus_To_v1beta1_CustomR
 	if err := Convert_apiextensions_CustomResourceDefinitionNames_To_v1beta1_CustomResourceDefinitionNames(&in.AcceptedNames, &out.AcceptedNames, s); err != nil {
 		return err
 	}
+	out.StoredVersions = *(*[]string)(unsafe.Pointer(&in.StoredVersions))
 	return nil
 }
 
@@ -279,6 +285,30 @@ func Convert_apiextensions_CustomResourceDefinitionStatus_To_v1beta1_CustomResou
 	return autoConvert_apiextensions_CustomResourceDefinitionStatus_To_v1beta1_CustomResourceDefinitionStatus(in, out, s)
 }
 
+func autoConvert_v1beta1_CustomResourceDefinitionVersion_To_apiextensions_CustomResourceDefinitionVersion(in *CustomResourceDefinitionVersion, out *apiextensions.CustomResourceDefinitionVersion, s conversion.Scope) error {
+	out.Name = in.Name
+	out.Served = in.Served
+	out.Storage = in.Storage
+	return nil
+}
+
+// Convert_v1beta1_CustomResourceDefinitionVersion_To_apiextensions_CustomResourceDefinitionVersion is an autogenerated conversion function.
+func Convert_v1beta1_CustomResourceDefinitionVersion_To_apiextensions_CustomResourceDefinitionVersion(in *CustomResourceDefinitionVersion, out *apiextensions.CustomResourceDefinitionVersion, s conversion.Scope) error {
+	return autoConvert_v1beta1_CustomResourceDefinitionVersion_To_apiextensions_CustomResourceDefinitionVersion(in, out, s)
+}
+
+func autoConvert_apiextensions_CustomResourceDefinitionVersion_To_v1beta1_CustomResourceDefinitionVersion(in *apiextensions.CustomResourceDefinitionVersion, out *CustomResourceDefinitionVersion, s conversion.Scope) error {
+	out.Name = in.Name
+	out.Served = in.Served
+	out.Storage = in.Storage
+	return nil
+}
+
+// Convert_apiextensions_CustomResourceDefinitionVersion_To_v1beta1_CustomResourceDefinitionVersion is an autogenerated conversion function.
+func Convert_apiextensions_CustomResourceDefinitionVersion_To_v1beta1_CustomResourceDefinitionVersion(in *apiextensions.CustomResourceDefinitionVersion, out *CustomResourceDefinitionVersion, s conversion.Scope) error {
+	return autoConvert_apiextensions_CustomResourceDefinitionVersion_To_v1beta1_CustomResourceDefinitionVersion(in, out, s)
+}
+
 func autoConvert_v1beta1_CustomResourceSubresourceScale_To_apiextensions_CustomResourceSubresourceScale(in *CustomResourceSubresourceScale, out *apiextensions.CustomResourceSubresourceScale, s conversion.Scope) error {
 	out.SpecReplicasPath = in.SpecReplicasPath
 	out.StatusReplicasPath = in.StatusReplicasPath

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.deepcopy.go

@@ -150,6 +150,11 @@ func (in *CustomResourceDefinitionSpec) DeepCopyInto(out *CustomResourceDefiniti
 			(*in).DeepCopyInto(*out)
 		}
 	}
+	if in.Versions != nil {
+		in, out := &in.Versions, &out.Versions
+		*out = make([]CustomResourceDefinitionVersion, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -174,6 +179,11 @@ func (in *CustomResourceDefinitionStatus) DeepCopyInto(out *CustomResourceDefini
 		}
 	}
 	in.AcceptedNames.DeepCopyInto(&out.AcceptedNames)
+	if in.StoredVersions != nil {
+		in, out := &in.StoredVersions, &out.StoredVersions
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -187,6 +197,22 @@ func (in *CustomResourceDefinitionStatus) DeepCopy() *CustomResourceDefinitionSt
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CustomResourceDefinitionVersion) DeepCopyInto(out *CustomResourceDefinitionVersion) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomResourceDefinitionVersion.
+func (in *CustomResourceDefinitionVersion) DeepCopy() *CustomResourceDefinitionVersion {
+	if in == nil {
+		return nil
+	}
+	out := new(CustomResourceDefinitionVersion)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *CustomResourceSubresourceScale) DeepCopyInto(out *CustomResourceSubresourceScale) {
 	*out = *in

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go

@@ -45,6 +45,7 @@ func ValidateCustomResourceDefinition(obj *apiextensions.CustomResourceDefinitio
 	allErrs := genericvalidation.ValidateObjectMeta(&obj.ObjectMeta, false, nameValidationFn, field.NewPath("metadata"))
 	allErrs = append(allErrs, ValidateCustomResourceDefinitionSpec(&obj.Spec, field.NewPath("spec"))...)
 	allErrs = append(allErrs, ValidateCustomResourceDefinitionStatus(&obj.Status, field.NewPath("status"))...)
+	allErrs = append(allErrs, ValidateCustomResourceDefinitionStoredVersions(obj.Status.StoredVersions, obj.Spec.Versions, field.NewPath("status").Child("storedVersions"))...)
 	return allErrs
 }
 
@@ -53,6 +54,34 @@ func ValidateCustomResourceDefinitionUpdate(obj, oldObj *apiextensions.CustomRes
 	allErrs := genericvalidation.ValidateObjectMetaUpdate(&obj.ObjectMeta, &oldObj.ObjectMeta, field.NewPath("metadata"))
 	allErrs = append(allErrs, ValidateCustomResourceDefinitionSpecUpdate(&obj.Spec, &oldObj.Spec, apiextensions.IsCRDConditionTrue(oldObj, apiextensions.Established), field.NewPath("spec"))...)
 	allErrs = append(allErrs, ValidateCustomResourceDefinitionStatus(&obj.Status, field.NewPath("status"))...)
+	allErrs = append(allErrs, ValidateCustomResourceDefinitionStoredVersions(obj.Status.StoredVersions, obj.Spec.Versions, field.NewPath("status").Child("storedVersions"))...)
+	return allErrs
+}
+
+// ValidateCustomResourceDefinitionStoredVersions statically validates
+func ValidateCustomResourceDefinitionStoredVersions(storedVersions []string, versions []apiextensions.CustomResourceDefinitionVersion, fldPath *field.Path) field.ErrorList {
+	if len(storedVersions) == 0 {
+		return field.ErrorList{field.Invalid(fldPath, storedVersions, "must have at least one stored version")}
+	}
+	allErrs := field.ErrorList{}
+	storedVersionsMap := map[string]int{}
+	for i, v := range storedVersions {
+		storedVersionsMap[v] = i
+	}
+	for _, v := range versions {
+		_, ok := storedVersionsMap[v.Name]
+		if v.Storage && !ok {
+			allErrs = append(allErrs, field.Invalid(fldPath, v, "must have the storage version "+v.Name))
+		}
+		if ok {
+			delete(storedVersionsMap, v.Name)
+		}
+	}
+
+	for v, i := range storedVersionsMap {
+		allErrs = append(allErrs, field.Invalid(fldPath.Index(i), v, "must appear in spec.versions"))
+	}
+
 	return allErrs
 }
 
@@ -75,12 +104,6 @@ func ValidateCustomResourceDefinitionSpec(spec *apiextensions.CustomResourceDefi
 		allErrs = append(allErrs, field.Invalid(fldPath.Child("group"), spec.Group, "should be a domain with at least one dot"))
 	}
 
-	if len(spec.Version) == 0 {
-		allErrs = append(allErrs, field.Required(fldPath.Child("version"), ""))
-	} else if errs := validationutil.IsDNS1035Label(spec.Version); len(errs) > 0 {
-		allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), spec.Version, strings.Join(errs, ",")))
-	}
-
 	switch spec.Scope {
 	case "":
 		allErrs = append(allErrs, field.Required(fldPath.Child("scope"), ""))
@@ -89,6 +112,37 @@ func ValidateCustomResourceDefinitionSpec(spec *apiextensions.CustomResourceDefi
 		allErrs = append(allErrs, field.NotSupported(fldPath.Child("scope"), spec.Scope, []string{string(apiextensions.ClusterScoped), string(apiextensions.NamespaceScoped)}))
 	}
 
+	storageFlagCount := 0
+	versionsMap := map[string]bool{}
+	uniqueNames := true
+	for i, version := range spec.Versions {
+		if version.Storage {
+			storageFlagCount++
+		}
+		if versionsMap[version.Name] {
+			uniqueNames = false
+		} else {
+			versionsMap[version.Name] = true
+		}
+		if errs := validationutil.IsDNS1035Label(version.Name); len(errs) > 0 {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("versions").Index(i).Child("name"), spec.Versions[i].Name, strings.Join(errs, ",")))
+		}
+	}
+	if !uniqueNames {
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("versions"), spec.Versions, "must contain unique version names"))
+	}
+	if storageFlagCount != 1 {
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("versions"), spec.Versions, "must have exactly one version marked as storage version"))
+	}
+	if len(spec.Version) != 0 {
+		if errs := validationutil.IsDNS1035Label(spec.Version); len(errs) > 0 {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), spec.Version, strings.Join(errs, ",")))
+		}
+		if len(spec.Versions) >= 1 && spec.Versions[0].Name != spec.Version {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), spec.Version, "must match the first version in spec.versions"))
+		}
+	}
+
 	// in addition to the basic name restrictions, some names are required for spec, but not for status
 	if len(spec.Names.Plural) == 0 {
 		allErrs = append(allErrs, field.Required(fldPath.Child("names", "plural"), ""))
@@ -130,7 +184,6 @@ func ValidateCustomResourceDefinitionSpecUpdate(spec, oldSpec *apiextensions.Cus
 
 	if established {
 		// these effect the storage and cannot be changed therefore
-		allErrs = append(allErrs, genericvalidation.ValidateImmutableField(spec.Version, oldSpec.Version, fldPath.Child("version"))...)
 		allErrs = append(allErrs, genericvalidation.ValidateImmutableField(spec.Scope, oldSpec.Scope, fldPath.Child("scope"))...)
 		allErrs = append(allErrs, genericvalidation.ValidateImmutableField(spec.Names.Kind, oldSpec.Names.Kind, fldPath.Child("names", "kind"))...)
 	}

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go

@@ -19,10 +19,9 @@ package validation
 import (
 	"testing"
 
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/validation/field"
-
-	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
 )
 
 type validationMatch struct {
@@ -51,11 +50,150 @@ func (v validationMatch) matches(err *field.Error) bool {
 }
 
 func TestValidateCustomResourceDefinition(t *testing.T) {
+	singleVersionList := []apiextensions.CustomResourceDefinitionVersion{
+		{
+			Name:    "version",
+			Served:  true,
+			Storage: true,
+		},
+	}
 	tests := []struct {
 		name     string
 		resource *apiextensions.CustomResourceDefinition
 		errors   []validationMatch
 	}{
+		{
+			name: "no_storage_version",
+			resource: &apiextensions.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"},
+				Spec: apiextensions.CustomResourceDefinitionSpec{
+					Group: "group.com",
+					Scope: apiextensions.ResourceScope("Cluster"),
+					Names: apiextensions.CustomResourceDefinitionNames{
+						Plural:   "plural",
+						Singular: "singular",
+						Kind:     "Plural",
+						ListKind: "PluralList",
+					},
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:    "version",
+							Served:  true,
+							Storage: false,
+						},
+						{
+							Name:    "version2",
+							Served:  true,
+							Storage: false,
+						},
+					},
+				},
+				Status: apiextensions.CustomResourceDefinitionStatus{
+					StoredVersions: []string{"version"},
+				},
+			},
+			errors: []validationMatch{
+				invalid("spec", "versions"),
+			},
+		},
+		{
+			name: "multiple_storage_version",
+			resource: &apiextensions.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"},
+				Spec: apiextensions.CustomResourceDefinitionSpec{
+					Group: "group.com",
+					Scope: apiextensions.ResourceScope("Cluster"),
+					Names: apiextensions.CustomResourceDefinitionNames{
+						Plural:   "plural",
+						Singular: "singular",
+						Kind:     "Plural",
+						ListKind: "PluralList",
+					},
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:    "version",
+							Served:  true,
+							Storage: true,
+						},
+						{
+							Name:    "version2",
+							Served:  true,
+							Storage: true,
+						},
+					},
+				},
+				Status: apiextensions.CustomResourceDefinitionStatus{
+					StoredVersions: []string{"version"},
+				},
+			},
+			errors: []validationMatch{
+				invalid("spec", "versions"),
+				invalid("status", "storedVersions"),
+			},
+		},
+		{
+			name: "missing_storage_version_in_stored_versions",
+			resource: &apiextensions.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"},
+				Spec: apiextensions.CustomResourceDefinitionSpec{
+					Group: "group.com",
+					Scope: apiextensions.ResourceScope("Cluster"),
+					Names: apiextensions.CustomResourceDefinitionNames{
+						Plural:   "plural",
+						Singular: "singular",
+						Kind:     "Plural",
+						ListKind: "PluralList",
+					},
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:    "version",
+							Served:  true,
+							Storage: false,
+						},
+						{
+							Name:    "version2",
+							Served:  true,
+							Storage: true,
+						},
+					},
+				},
+				Status: apiextensions.CustomResourceDefinitionStatus{
+					StoredVersions: []string{"version"},
+				},
+			},
+			errors: []validationMatch{
+				invalid("status", "storedVersions"),
+			},
+		},
+		{
+			name: "empty_stored_version",
+			resource: &apiextensions.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"},
+				Spec: apiextensions.CustomResourceDefinitionSpec{
+					Group: "group.com",
+					Scope: apiextensions.ResourceScope("Cluster"),
+					Names: apiextensions.CustomResourceDefinitionNames{
+						Plural:   "plural",
+						Singular: "singular",
+						Kind:     "Plural",
+						ListKind: "PluralList",
+					},
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:    "version",
+							Served:  true,
+							Storage: true,
+						},
+					},
+				},
+				Status: apiextensions.CustomResourceDefinitionStatus{
+					StoredVersions: []string{},
+				},
+			},
+			errors: []validationMatch{
+				invalid("status", "storedVersions"),
+			},
+		},
 		{
 			name: "mismatched name",
 			resource: &apiextensions.CustomResourceDefinition{
@@ -68,8 +206,9 @@ func TestValidateCustomResourceDefinition(t *testing.T) {
 				},
 			},
 			errors: []validationMatch{
+				invalid("status", "storedVersions"),
 				invalid("metadata", "name"),
-				required("spec", "version"),
+				invalid("spec", "versions"),
 				required("spec", "scope"),
 				required("spec", "names", "singular"),
 				required("spec", "names", "kind"),
@@ -82,9 +221,10 @@ func TestValidateCustomResourceDefinition(t *testing.T) {
 				ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"},
 			},
 			errors: []validationMatch{
+				invalid("status", "storedVersions"),
 				invalid("metadata", "name"),
+				invalid("spec", "versions"),
 				required("spec", "group"),
-				required("spec", "version"),
 				required("spec", "scope"),
 				required("spec", "names", "plural"),
 				required("spec", "names", "singular"),
@@ -117,9 +257,9 @@ func TestValidateCustomResourceDefinition(t *testing.T) {
 				},
 			},
 			errors: []validationMatch{
+				invalid("status", "storedVersions"),
 				invalid("metadata", "name"),
 				invalid("spec", "group"),
-				invalid("spec", "version"),
 				unsupported("spec", "scope"),
 				invalid("spec", "names", "plural"),
 				invalid("spec", "names", "singular"),
@@ -131,6 +271,8 @@ func TestValidateCustomResourceDefinition(t *testing.T) {
 				invalid("status", "acceptedNames", "kind"),
 				invalid("status", "acceptedNames", "listKind"), // invalid format
 				invalid("status", "acceptedNames", "listKind"), // kind == listKind
+				invalid("spec", "versions"),
+				invalid("spec", "version"),
 			},
 		},
 		{
@@ -138,8 +280,9 @@ func TestValidateCustomResourceDefinition(t *testing.T) {
 			resource: &apiextensions.CustomResourceDefinition{
 				ObjectMeta: metav1.ObjectMeta{Name: "plural.group"},
 				Spec: apiextensions.CustomResourceDefinitionSpec{
-					Group:   "group.c(*&om",
+					Group:    "group.c(*&om",
-					Version: "version",
+					Version:  "version",
+					Versions: singleVersionList,
 					Names: apiextensions.CustomResourceDefinitionNames{
 						Plural:   "plural",
 						Singular: "singular",
@@ -154,6 +297,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) {
 						Kind:     "matching",
 						ListKind: "matching",
 					},
+					StoredVersions: []string{"version"},
 				},
 			},
 			errors: []validationMatch{
@@ -169,9 +313,10 @@ func TestValidateCustomResourceDefinition(t *testing.T) {
 			resource: &apiextensions.CustomResourceDefinition{
 				ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"},
 				Spec: apiextensions.CustomResourceDefinitionSpec{
-					Group:   "group.com",
+					Group:    "group.com",
-					Version: "version",
+					Version:  "version",
-					Scope:   apiextensions.NamespaceScoped,
+					Versions: singleVersionList,
+					Scope:    apiextensions.NamespaceScoped,
 					Names: apiextensions.CustomResourceDefinitionNames{
 						Plural:   "plural",
 						Singular: "singular",
@@ -187,6 +332,9 @@ func TestValidateCustomResourceDefinition(t *testing.T) {
 						},
 					},
 				},
+				Status: apiextensions.CustomResourceDefinitionStatus{
+					StoredVersions: []string{"version"},
+				},
 			},
 			errors: []validationMatch{
 				forbidden("spec", "validation", "openAPIV3Schema", "additionalProperties"),
@@ -197,9 +345,10 @@ func TestValidateCustomResourceDefinition(t *testing.T) {
 			resource: &apiextensions.CustomResourceDefinition{
 				ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"},
 				Spec: apiextensions.CustomResourceDefinitionSpec{
-					Group:   "group.com",
+					Group:    "group.com",
-					Version: "version",
+					Version:  "version",
-					Scope:   apiextensions.NamespaceScoped,
+					Versions: singleVersionList,
+					Scope:    apiextensions.NamespaceScoped,
 					Names: apiextensions.CustomResourceDefinitionNames{
 						Plural:   "plural",
 						Singular: "singular",
@@ -217,6 +366,9 @@ func TestValidateCustomResourceDefinition(t *testing.T) {
 						},
 					},
 				},
+				Status: apiextensions.CustomResourceDefinitionStatus{
+					StoredVersions: []string{"version"},
+				},
 			},
 			errors: []validationMatch{},
 		},
@@ -266,7 +418,14 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) {
 				Spec: apiextensions.CustomResourceDefinitionSpec{
 					Group:   "group.com",
 					Version: "version",
-					Scope:   apiextensions.ResourceScope("Cluster"),
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:    "version",
+							Served:  true,
+							Storage: true,
+						},
+					},
+					Scope: apiextensions.ResourceScope("Cluster"),
 					Names: apiextensions.CustomResourceDefinitionNames{
 						Plural:   "plural",
 						Singular: "singular",
@@ -291,7 +450,14 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) {
 				Spec: apiextensions.CustomResourceDefinitionSpec{
 					Group:   "group.com",
 					Version: "version",
-					Scope:   apiextensions.ResourceScope("Cluster"),
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:    "version",
+							Served:  true,
+							Storage: true,
+						},
+					},
+					Scope: apiextensions.ResourceScope("Cluster"),
 					Names: apiextensions.CustomResourceDefinitionNames{
 						Plural:   "plural",
 						Singular: "singular",
@@ -306,6 +472,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) {
 						Kind:     "kind",
 						ListKind: "listkind",
 					},
+					StoredVersions: []string{"version"},
 				},
 			},
 			errors: []validationMatch{},
@@ -320,7 +487,14 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) {
 				Spec: apiextensions.CustomResourceDefinitionSpec{
 					Group:   "group.com",
 					Version: "version",
-					Scope:   apiextensions.ResourceScope("Cluster"),
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:    "version",
+							Served:  true,
+							Storage: true,
+						},
+					},
+					Scope: apiextensions.ResourceScope("Cluster"),
 					Names: apiextensions.CustomResourceDefinitionNames{
 						Plural:   "plural",
 						Singular: "singular",
@@ -348,7 +522,14 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) {
 				Spec: apiextensions.CustomResourceDefinitionSpec{
 					Group:   "group.com",
 					Version: "version",
-					Scope:   apiextensions.ResourceScope("Cluster"),
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:    "version",
+							Served:  true,
+							Storage: true,
+						},
+					},
+					Scope: apiextensions.ResourceScope("Cluster"),
 					Names: apiextensions.CustomResourceDefinitionNames{
 						Plural:   "plural",
 						Singular: "singular",
@@ -363,10 +544,91 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) {
 						Kind:     "kind",
 						ListKind: "listkind",
 					},
+					StoredVersions: []string{"version"},
 				},
 			},
 			errors: []validationMatch{},
 		},
+		{
+			name: "version-deleted",
+			old: &apiextensions.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:            "plural.group.com",
+					ResourceVersion: "42",
+				},
+				Spec: apiextensions.CustomResourceDefinitionSpec{
+					Group:   "group.com",
+					Version: "version",
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:    "version",
+							Served:  true,
+							Storage: true,
+						},
+						{
+							Name:    "version2",
+							Served:  true,
+							Storage: false,
+						},
+					},
+					Scope: apiextensions.ResourceScope("Cluster"),
+					Names: apiextensions.CustomResourceDefinitionNames{
+						Plural:   "plural",
+						Singular: "singular",
+						Kind:     "kind",
+						ListKind: "listkind",
+					},
+				},
+				Status: apiextensions.CustomResourceDefinitionStatus{
+					AcceptedNames: apiextensions.CustomResourceDefinitionNames{
+						Plural:   "plural",
+						Singular: "singular",
+						Kind:     "kind",
+						ListKind: "listkind",
+					},
+					StoredVersions: []string{"version", "version2"},
+					Conditions: []apiextensions.CustomResourceDefinitionCondition{
+						{Type: apiextensions.Established, Status: apiextensions.ConditionTrue},
+					},
+				},
+			},
+			resource: &apiextensions.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:            "plural.group.com",
+					ResourceVersion: "42",
+				},
+				Spec: apiextensions.CustomResourceDefinitionSpec{
+					Group:   "group.com",
+					Version: "version",
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:    "version",
+							Served:  true,
+							Storage: true,
+						},
+					},
+					Scope: apiextensions.ResourceScope("Cluster"),
+					Names: apiextensions.CustomResourceDefinitionNames{
+						Plural:   "plural",
+						Singular: "singular",
+						Kind:     "kind",
+						ListKind: "listkind",
+					},
+				},
+				Status: apiextensions.CustomResourceDefinitionStatus{
+					AcceptedNames: apiextensions.CustomResourceDefinitionNames{
+						Plural:   "plural",
+						Singular: "singular",
+						Kind:     "kind",
+						ListKind: "listkind",
+					},
+					StoredVersions: []string{"version", "version2"},
+				},
+			},
+			errors: []validationMatch{
+				invalid("status", "storedVersions[1]"),
+			},
+		},
 		{
 			name: "changes",
 			old: &apiextensions.CustomResourceDefinition{
@@ -377,7 +639,14 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) {
 				Spec: apiextensions.CustomResourceDefinitionSpec{
 					Group:   "group.com",
 					Version: "version",
-					Scope:   apiextensions.ResourceScope("Cluster"),
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:    "version",
+							Served:  true,
+							Storage: true,
+						},
+					},
+					Scope: apiextensions.ResourceScope("Cluster"),
 					Names: apiextensions.CustomResourceDefinitionNames{
 						Plural:   "plural",
 						Singular: "singular",
@@ -405,7 +674,14 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) {
 				Spec: apiextensions.CustomResourceDefinitionSpec{
 					Group:   "abc.com",
 					Version: "version2",
-					Scope:   apiextensions.ResourceScope("Namespaced"),
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:    "version2",
+							Served:  true,
+							Storage: true,
+						},
+					},
+					Scope: apiextensions.ResourceScope("Namespaced"),
 					Names: apiextensions.CustomResourceDefinitionNames{
 						Plural:   "plural2",
 						Singular: "singular2",
@@ -420,6 +696,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) {
 						Kind:     "kind2",
 						ListKind: "listkind2",
 					},
+					StoredVersions: []string{"version2"},
 				},
 			},
 			errors: []validationMatch{
@@ -437,7 +714,14 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) {
 				Spec: apiextensions.CustomResourceDefinitionSpec{
 					Group:   "group.com",
 					Version: "version",
-					Scope:   apiextensions.ResourceScope("Cluster"),
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:    "version",
+							Served:  true,
+							Storage: true,
+						},
+					},
+					Scope: apiextensions.ResourceScope("Cluster"),
 					Names: apiextensions.CustomResourceDefinitionNames{
 						Plural:   "plural",
 						Singular: "singular",
@@ -465,7 +749,14 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) {
 				Spec: apiextensions.CustomResourceDefinitionSpec{
 					Group:   "abc.com",
 					Version: "version2",
-					Scope:   apiextensions.ResourceScope("Namespaced"),
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:    "version2",
+							Served:  true,
+							Storage: true,
+						},
+					},
+					Scope: apiextensions.ResourceScope("Namespaced"),
 					Names: apiextensions.CustomResourceDefinitionNames{
 						Plural:   "plural2",
 						Singular: "singular2",
@@ -480,11 +771,11 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) {
 						Kind:     "kind2",
 						ListKind: "listkind2",
 					},
+					StoredVersions: []string{"version2"},
 				},
 			},
 			errors: []validationMatch{
 				immutable("spec", "group"),
-				immutable("spec", "version"),
 				immutable("spec", "scope"),
 				immutable("spec", "names", "kind"),
 				immutable("spec", "names", "plural"),

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/zz_generated.deepcopy.go

@@ -150,6 +150,11 @@ func (in *CustomResourceDefinitionSpec) DeepCopyInto(out *CustomResourceDefiniti
 			(*in).DeepCopyInto(*out)
 		}
 	}
+	if in.Versions != nil {
+		in, out := &in.Versions, &out.Versions
+		*out = make([]CustomResourceDefinitionVersion, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -174,6 +179,11 @@ func (in *CustomResourceDefinitionStatus) DeepCopyInto(out *CustomResourceDefini
 		}
 	}
 	in.AcceptedNames.DeepCopyInto(&out.AcceptedNames)
+	if in.StoredVersions != nil {
+		in, out := &in.StoredVersions, &out.StoredVersions
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -187,6 +197,22 @@ func (in *CustomResourceDefinitionStatus) DeepCopy() *CustomResourceDefinitionSt
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CustomResourceDefinitionVersion) DeepCopyInto(out *CustomResourceDefinitionVersion) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomResourceDefinitionVersion.
+func (in *CustomResourceDefinitionVersion) DeepCopy() *CustomResourceDefinitionVersion {
+	if in == nil {
+		return nil
+	}
+	out := new(CustomResourceDefinitionVersion)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *CustomResourceSubresourceScale) DeepCopyInto(out *CustomResourceSubresourceScale) {
 	*out = *in

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD

@@ -24,6 +24,7 @@ go_library(
         "//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library",
         "//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install:go_default_library",
         "//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//vendor/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion:go_default_library",
         "//vendor/k8s.io/apiextensions-apiserver/pkg/apiserver/validation:go_default_library",
         "//vendor/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//vendor/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset:go_default_library",
@@ -84,6 +85,7 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion:all-srcs",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation:all-srcs",
     ],
     tags = ["automanaged"],
@@ -93,5 +95,8 @@ go_test(
     name = "go_default_test",
     srcs = ["customresource_handler_test.go"],
     embed = [":go_default_library"],
-    deps = ["//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library"],
+    deps = [
+        "//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library",
+        "//vendor/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion:go_default_library",
+    ],
 )

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/BUILD

@@ -0,0 +1,31 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "converter.go",
+        "nop_converter.go",
+    ],
+    importpath = "k8s.io/apiextensions-apiserver/pkg/apiserver/conversion",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go

@@ -0,0 +1,67 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package conversion
+
+import (
+	"fmt"
+
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// NewCRDConverter returns a new CRD converter based on the conversion settings in crd object.
+func NewCRDConverter(crd *apiextensions.CustomResourceDefinition) (safe, unsafe runtime.ObjectConvertor) {
+	validVersions := map[schema.GroupVersion]bool{}
+	for _, version := range crd.Spec.Versions {
+		validVersions[schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name}] = true
+	}
+
+	// The only converter right now is nopConverter. More converters will be returned based on the
+	// CRD object when they introduced.
+	unsafe = &nopConverter{
+		clusterScoped: crd.Spec.Scope == apiextensions.ClusterScoped,
+		validVersions: validVersions,
+	}
+	return &safeConverterWrapper{unsafe}, unsafe
+}
+
+// safeConverterWrapper is a wrapper over an unsafe object converter that makes copy of the input and then delegate to the unsafe converter.
+type safeConverterWrapper struct {
+	unsafe runtime.ObjectConvertor
+}
+
+var _ runtime.ObjectConvertor = &nopConverter{}
+
+// ConvertFieldLabel delegate the call to the unsafe converter.
+func (c *safeConverterWrapper) ConvertFieldLabel(version, kind, label, value string) (string, string, error) {
+	return c.unsafe.ConvertFieldLabel(version, kind, label, value)
+}
+
+// Convert makes a copy of in object and then delegate the call to the unsafe converter.
+func (c *safeConverterWrapper) Convert(in, out, context interface{}) error {
+	inObject, ok := in.(runtime.Object)
+	if !ok {
+		return fmt.Errorf("input type %T in not valid for object conversion", in)
+	}
+	return c.unsafe.Convert(inObject.DeepCopyObject(), out, context)
+}
+
+// ConvertToVersion makes a copy of in object and then delegate the call to the unsafe converter.
+func (c *safeConverterWrapper) ConvertToVersion(in runtime.Object, target runtime.GroupVersioner) (runtime.Object, error) {
+	return c.unsafe.ConvertToVersion(in.DeepCopyObject(), target)
+}

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/nop_converter.go

@@ -0,0 +1,100 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package conversion
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// nopConverter is a converter that only sets the apiVersion fields, but does not real conversion. It supports fields selectors.
+type nopConverter struct {
+	clusterScoped bool
+	validVersions map[schema.GroupVersion]bool
+}
+
+var _ runtime.ObjectConvertor = &nopConverter{}
+
+func (c *nopConverter) ConvertFieldLabel(version, kind, label, value string) (string, string, error) {
+	// We currently only support metadata.namespace and metadata.name.
+	switch {
+	case label == "metadata.name":
+		return label, value, nil
+	case !c.clusterScoped && label == "metadata.namespace":
+		return label, value, nil
+	default:
+		return "", "", fmt.Errorf("field label not supported: %s", label)
+	}
+}
+
+func (c *nopConverter) Convert(in, out, context interface{}) error {
+	unstructIn, ok := in.(*unstructured.Unstructured)
+	if !ok {
+		return fmt.Errorf("input type %T in not valid for unstructured conversion", in)
+	}
+
+	unstructOut, ok := out.(*unstructured.Unstructured)
+	if !ok {
+		return fmt.Errorf("output type %T in not valid for unstructured conversion", out)
+	}
+
+	outGVK := unstructOut.GroupVersionKind()
+	if !c.validVersions[outGVK.GroupVersion()] {
+		return fmt.Errorf("request to convert CRD from an invalid group/version: %s", outGVK.String())
+	}
+	inGVK := unstructIn.GroupVersionKind()
+	if !c.validVersions[inGVK.GroupVersion()] {
+		return fmt.Errorf("request to convert CRD to an invalid group/version: %s", inGVK.String())
+	}
+
+	unstructOut.SetUnstructuredContent(unstructIn.UnstructuredContent())
+	_, err := c.ConvertToVersion(unstructOut, outGVK.GroupVersion())
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *nopConverter) convertToVersion(in runtime.Object, target runtime.GroupVersioner) error {
+	kind := in.GetObjectKind().GroupVersionKind()
+	gvk, ok := target.KindForGroupVersionKinds([]schema.GroupVersionKind{kind})
+	if !ok {
+		// TODO: should this be a typed error?
+		return fmt.Errorf("%v is unstructured and is not suitable for converting to %q", kind, target)
+	}
+	if !c.validVersions[gvk.GroupVersion()] {
+		return fmt.Errorf("request to convert CRD to an invalid group/version: %s", gvk.String())
+	}
+	in.GetObjectKind().SetGroupVersionKind(gvk)
+	return nil
+}
+
+// ConvertToVersion converts in object to the given gvk in place and returns the same `in` object.
+func (c *nopConverter) ConvertToVersion(in runtime.Object, target runtime.GroupVersioner) (runtime.Object, error) {
+	var err error
+	// Run the converter on the list items instead of list itself
+	if list, ok := in.(*unstructured.UnstructuredList); ok {
+		err = list.EachListItem(func(item runtime.Object) error {
+			return c.convertToVersion(item, target)
+		})
+	}
+	err = c.convertToVersion(in, target)
+	return in, err
+}

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go

@@ -18,6 +18,7 @@ package apiserver
 
 import (
 	"fmt"
+	"sort"
 	"time"
 
 	"github.com/golang/glog"
@@ -28,6 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/apiserver/pkg/endpoints/discovery"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/workqueue"
@@ -75,6 +77,7 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 
 	apiVersionsForDiscovery := []metav1.GroupVersionForDiscovery{}
 	apiResourcesForDiscovery := []metav1.APIResource{}
+	versionsForDiscoveryMap := map[metav1.GroupVersion]bool{}
 
 	crds, err := c.crdLister.List(labels.Everything())
 	if err != nil {
@@ -90,13 +93,29 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 		if crd.Spec.Group != version.Group {
 			continue
 		}
-		foundGroup = true
-		apiVersionsForDiscovery = append(apiVersionsForDiscovery, metav1.GroupVersionForDiscovery{
-			GroupVersion: crd.Spec.Group + "/" + crd.Spec.Version,
-			Version:      crd.Spec.Version,
-		})
 
-		if crd.Spec.Version != version.Version {
+		foundThisVersion := false
+		for _, v := range crd.Spec.Versions {
+			if !v.Served {
+				continue
+			}
+			// If there is any Served version, that means the group should show up in discovery
+			foundGroup = true
+
+			gv := metav1.GroupVersion{Group: crd.Spec.Group, Version: v.Name}
+			if !versionsForDiscoveryMap[gv] {
+				versionsForDiscoveryMap[gv] = true
+				apiVersionsForDiscovery = append(apiVersionsForDiscovery, metav1.GroupVersionForDiscovery{
+					GroupVersion: crd.Spec.Group + "/" + v.Name,
+					Version:      v.Name,
+				})
+			}
+			if v.Name == version.Version {
+				foundThisVersion = true
+			}
+		}
+
+		if !foundThisVersion {
 			continue
 		}
 		foundVersion = true
@@ -144,10 +163,13 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 		return nil
 	}
 
+	sortGroupDiscoveryByKubeAwareVersion(apiVersionsForDiscovery)
+
 	apiGroup := metav1.APIGroup{
 		Name:     version.Group,
 		Versions: apiVersionsForDiscovery,
-		// the preferred versions for a group is arbitrary since there cannot be duplicate resources
+		// the preferred versions for a group is the first item in
+		// apiVersionsForDiscovery after it put in the right ordered
 		PreferredVersion: apiVersionsForDiscovery[0],
 	}
 	c.groupHandler.setDiscovery(version.Group, discovery.NewAPIGroupHandler(Codecs, apiGroup))
@@ -163,6 +185,12 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 	return nil
 }
 
+func sortGroupDiscoveryByKubeAwareVersion(gd []metav1.GroupVersionForDiscovery) {
+	sort.Slice(gd, func(i, j int) bool {
+		return version.CompareKubeAwareVersionStrings(gd[i].Version, gd[j].Version) > 0
+	})
+}
+
 func (c *DiscoveryController) Run(stopCh <-chan struct{}) {
 	defer utilruntime.HandleCrash()
 	defer c.queue.ShutDown()
@@ -207,7 +235,9 @@ func (c *DiscoveryController) processNextWorkItem() bool {
 }
 
 func (c *DiscoveryController) enqueue(obj *apiextensions.CustomResourceDefinition) {
-	c.queue.Add(schema.GroupVersion{Group: obj.Spec.Group, Version: obj.Spec.Version})
+	for _, v := range obj.Spec.Versions {
+		c.queue.Add(schema.GroupVersion{Group: obj.Spec.Group, Version: v.Name})
+	}
 }
 
 func (c *DiscoveryController) addCustomResourceDefinition(obj interface{}) {
@@ -216,10 +246,14 @@ func (c *DiscoveryController) addCustomResourceDefinition(obj interface{}) {
 	c.enqueue(castObj)
 }
 
-func (c *DiscoveryController) updateCustomResourceDefinition(obj, _ interface{}) {
+func (c *DiscoveryController) updateCustomResourceDefinition(oldObj, newObj interface{}) {
-	castObj := obj.(*apiextensions.CustomResourceDefinition)
+	castNewObj := newObj.(*apiextensions.CustomResourceDefinition)
-	glog.V(4).Infof("Updating customresourcedefinition %s", castObj.Name)
+	castOldObj := oldObj.(*apiextensions.CustomResourceDefinition)
-	c.enqueue(castObj)
+	glog.V(4).Infof("Updating customresourcedefinition %s", castOldObj.Name)
+	// Enqueue both old and new object to make sure we remove and add appropriate Versions.
+	// The working queue will resolve any duplicates and only changes will stay in the queue.
+	c.enqueue(castNewObj)
+	c.enqueue(castOldObj)
 }
 
 func (c *DiscoveryController) deleteCustomResourceDefinition(obj interface{}) {

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go

@@ -58,6 +58,7 @@ import (
 	"k8s.io/client-go/tools/cache"
 
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
+	"k8s.io/apiextensions-apiserver/pkg/apiserver/conversion"
 	apiservervalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation"
 	informers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion/apiextensions/internalversion"
 	listers "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion"
@@ -94,11 +95,20 @@ type crdInfo struct {
 	spec          *apiextensions.CustomResourceDefinitionSpec
 	acceptedNames *apiextensions.CustomResourceDefinitionNames
 
-	storage customresource.CustomResourceStorage
+	// Storage per version
+	storages map[string]customresource.CustomResourceStorage
 
-	requestScope       handlers.RequestScope
+	// Request scope per version
-	scaleRequestScope  handlers.RequestScope
+	requestScopes map[string]handlers.RequestScope
-	statusRequestScope handlers.RequestScope
+
+	// Scale scope per version
+	scaleRequestScopes map[string]handlers.RequestScope
+
+	// Status scope per version
+	statusRequestScopes map[string]handlers.RequestScope
+
+	// storageVersion is the CRD version used when storing the object in etcd.
+	storageVersion string
 }
 
 // crdStorageMap goes from customresourcedefinition to its storage
@@ -120,7 +130,6 @@ func NewCustomResourceDefinitionHandler(
 		restOptionsGetter:       restOptionsGetter,
 		admission:               admission,
 	}
-
 	crdInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
 		UpdateFunc: ret.updateCustomResourceDefinition,
 		DeleteFunc: func(obj interface{}) {
@@ -168,7 +177,7 @@ func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	if crd.Spec.Version != requestInfo.APIVersion {
+	if !apiextensions.HasServedCRDVersion(crd, requestInfo.APIVersion) {
 		r.delegate.ServeHTTP(w, req)
 		return
 	}
@@ -214,8 +223,8 @@ func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 }
 
 func (r *crdHandler) serveResource(w http.ResponseWriter, req *http.Request, requestInfo *apirequest.RequestInfo, crdInfo *crdInfo, terminating bool, supportedTypes []string) http.HandlerFunc {
-	requestScope := crdInfo.requestScope
+	requestScope := crdInfo.requestScopes[requestInfo.APIVersion]
-	storage := crdInfo.storage.CustomResource
+	storage := crdInfo.storages[requestInfo.APIVersion].CustomResource
 	minRequestTimeout := 1 * time.Minute
 
 	switch requestInfo.Verb {
@@ -250,8 +259,8 @@ func (r *crdHandler) serveResource(w http.ResponseWriter, req *http.Request, req
 }
 
 func (r *crdHandler) serveStatus(w http.ResponseWriter, req *http.Request, requestInfo *apirequest.RequestInfo, crdInfo *crdInfo, terminating bool, supportedTypes []string) http.HandlerFunc {
-	requestScope := crdInfo.statusRequestScope
+	requestScope := crdInfo.statusRequestScopes[requestInfo.APIVersion]
-	storage := crdInfo.storage.Status
+	storage := crdInfo.storages[requestInfo.APIVersion].Status
 
 	switch requestInfo.Verb {
 	case "get":
@@ -267,8 +276,8 @@ func (r *crdHandler) serveStatus(w http.ResponseWriter, req *http.Request, reque
 }
 
 func (r *crdHandler) serveScale(w http.ResponseWriter, req *http.Request, requestInfo *apirequest.RequestInfo, crdInfo *crdInfo, terminating bool, supportedTypes []string) http.HandlerFunc {
-	requestScope := crdInfo.scaleRequestScope
+	requestScope := crdInfo.scaleRequestScopes[requestInfo.APIVersion]
-	storage := crdInfo.storage.Scale
+	storage := crdInfo.storages[requestInfo.APIVersion].Scale
 
 	switch requestInfo.Verb {
 	case "get":
@@ -306,8 +315,10 @@ func (r *crdHandler) updateCustomResourceDefinition(oldObj, newObj interface{})
 	// as it is used without locking elsewhere.
 	storageMap2 := storageMap.clone()
 	if oldInfo, ok := storageMap2[types.UID(oldCRD.UID)]; ok {
-		// destroy only the main storage. Those for the subresources share cacher and etcd clients.
+		for _, storage := range oldInfo.storages {
-		oldInfo.storage.CustomResource.DestroyFunc()
+			// destroy only the main storage. Those for the subresources share cacher and etcd clients.
+			storage.CustomResource.DestroyFunc()
+		}
 		delete(storageMap2, types.UID(oldCRD.UID))
 	}
 
@@ -338,9 +349,11 @@ func (r *crdHandler) removeDeadStorage() {
 			}
 		}
 		if !found {
-			glog.V(4).Infof("Removing dead CRD storage for %v", s.requestScope.Resource)
+			for version, storage := range s.storages {
-			// destroy only the main storage. Those for the subresources share cacher and etcd clients.
+				glog.V(4).Infof("Removing dead CRD storage for %v", s.requestScopes[version].Resource)
-			s.storage.CustomResource.DestroyFunc()
+				// destroy only the main storage. Those for the subresources share cacher and etcd clients.
+				storage.CustomResource.DestroyFunc()
+			}
 			delete(storageMap2, uid)
 		}
 	}
@@ -354,7 +367,7 @@ func (r *crdHandler) GetCustomResourceListerCollectionDeleter(crd *apiextensions
 	if err != nil {
 		return nil, err
 	}
-	return info.storage.CustomResource, nil
+	return info.storages[info.storageVersion].CustomResource, nil
 }
 
 func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResourceDefinition) (*crdInfo, error) {
@@ -371,140 +384,158 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource
 		return ret, nil
 	}
 
-	// In addition to Unstructured objects (Custom Resources), we also may sometimes need to
+	storageVersion, err := apiextensions.GetCRDStorageVersion(crd)
-	// decode unversioned Options objects, so we delegate to parameterScheme for such types.
-	parameterScheme := runtime.NewScheme()
-	parameterScheme.AddUnversionedTypes(schema.GroupVersion{Group: crd.Spec.Group, Version: crd.Spec.Version},
-		&metav1.ListOptions{},
-		&metav1.ExportOptions{},
-		&metav1.GetOptions{},
-		&metav1.DeleteOptions{},
-	)
-	parameterCodec := runtime.NewParameterCodec(parameterScheme)
-
-	kind := schema.GroupVersionKind{Group: crd.Spec.Group, Version: crd.Spec.Version, Kind: crd.Status.AcceptedNames.Kind}
-	typer := UnstructuredObjectTyper{
-		Delegate:          parameterScheme,
-		UnstructuredTyper: discovery.NewUnstructuredObjectTyper(),
-	}
-	creator := unstructuredCreator{}
-
-	validator, _, err := apiservervalidation.NewSchemaValidator(crd.Spec.Validation)
 	if err != nil {
 		return nil, err
 	}
 
-	var statusSpec *apiextensions.CustomResourceSubresourceStatus
+	// Scope/Storages per version.
-	var statusValidator *validate.SchemaValidator
+	requestScopes := map[string]handlers.RequestScope{}
-	if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) && crd.Spec.Subresources != nil && crd.Spec.Subresources.Status != nil {
+	storages := map[string]customresource.CustomResourceStorage{}
-		statusSpec = crd.Spec.Subresources.Status
+	statusScopes := map[string]handlers.RequestScope{}
+	scaleScopes := map[string]handlers.RequestScope{}
 
-		// for the status subresource, validate only against the status schema
+	for _, v := range crd.Spec.Versions {
-		if crd.Spec.Validation != nil && crd.Spec.Validation.OpenAPIV3Schema != nil && crd.Spec.Validation.OpenAPIV3Schema.Properties != nil {
+		safeConverter, unsafeConverter := conversion.NewCRDConverter(crd)
-			if statusSchema, ok := crd.Spec.Validation.OpenAPIV3Schema.Properties["status"]; ok {
+		// In addition to Unstructured objects (Custom Resources), we also may sometimes need to
-				openapiSchema := &spec.Schema{}
+		// decode unversioned Options objects, so we delegate to parameterScheme for such types.
-				if err := apiservervalidation.ConvertJSONSchemaProps(&statusSchema, openapiSchema); err != nil {
+		parameterScheme := runtime.NewScheme()
-					return nil, err
+		parameterScheme.AddUnversionedTypes(schema.GroupVersion{Group: crd.Spec.Group, Version: v.Name},
+			&metav1.ListOptions{},
+			&metav1.ExportOptions{},
+			&metav1.GetOptions{},
+			&metav1.DeleteOptions{},
+		)
+		parameterCodec := runtime.NewParameterCodec(parameterScheme)
+
+		kind := schema.GroupVersionKind{Group: crd.Spec.Group, Version: v.Name, Kind: crd.Status.AcceptedNames.Kind}
+		typer := newUnstructuredObjectTyper(parameterScheme)
+		creator := unstructuredCreator{}
+
+		validator, _, err := apiservervalidation.NewSchemaValidator(crd.Spec.Validation)
+		if err != nil {
+			return nil, err
+		}
+
+		var statusSpec *apiextensions.CustomResourceSubresourceStatus
+		var statusValidator *validate.SchemaValidator
+		if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) && crd.Spec.Subresources != nil && crd.Spec.Subresources.Status != nil {
+			statusSpec = crd.Spec.Subresources.Status
+
+			// for the status subresource, validate only against the status schema
+			if crd.Spec.Validation != nil && crd.Spec.Validation.OpenAPIV3Schema != nil && crd.Spec.Validation.OpenAPIV3Schema.Properties != nil {
+				if statusSchema, ok := crd.Spec.Validation.OpenAPIV3Schema.Properties["status"]; ok {
+					openapiSchema := &spec.Schema{}
+					if err := apiservervalidation.ConvertJSONSchemaProps(&statusSchema, openapiSchema); err != nil {
+						return nil, err
+					}
+					statusValidator = validate.NewSchemaValidator(openapiSchema, nil, "", strfmt.Default)
 				}
-				statusValidator = validate.NewSchemaValidator(openapiSchema, nil, "", strfmt.Default)
 			}
 		}
-	}
 
-	var scaleSpec *apiextensions.CustomResourceSubresourceScale
+		var scaleSpec *apiextensions.CustomResourceSubresourceScale
-	if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) && crd.Spec.Subresources != nil && crd.Spec.Subresources.Scale != nil {
+		if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) && crd.Spec.Subresources != nil && crd.Spec.Subresources.Scale != nil {
-		scaleSpec = crd.Spec.Subresources.Scale
+			scaleSpec = crd.Spec.Subresources.Scale
-	}
+		}
 
-	// TODO: identify how to pass printer specification from the CRD
+		// TODO: identify how to pass printer specification from the CRD
-	table, err := tableconvertor.New(nil)
+		table, err := tableconvertor.New(nil)
-	if err != nil {
+		if err != nil {
-		glog.V(2).Infof("The CRD for %v has an invalid printer specification, falling back to default printing: %v", kind, err)
+			glog.V(2).Infof("The CRD for %v has an invalid printer specification, falling back to default printing: %v", kind, err)
-	}
+		}
 
-	customResourceStorage := customresource.NewStorage(
+		storages[v.Name] = customresource.NewStorage(
-		schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Status.AcceptedNames.Plural},
+			schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Status.AcceptedNames.Plural},
-		schema.GroupVersionKind{Group: crd.Spec.Group, Version: crd.Spec.Version, Kind: crd.Status.AcceptedNames.ListKind},
+			schema.GroupVersionKind{Group: crd.Spec.Group, Version: v.Name, Kind: crd.Status.AcceptedNames.ListKind},
-		customresource.NewStrategy(
+			customresource.NewStrategy(
-			typer,
+				typer,
-			crd.Spec.Scope == apiextensions.NamespaceScoped,
+				crd.Spec.Scope == apiextensions.NamespaceScoped,
-			kind,
+				kind,
-			validator,
+				validator,
-			statusValidator,
+				statusValidator,
-			statusSpec,
+				statusSpec,
-			scaleSpec,
+				scaleSpec,
-		),
+			),
-		r.restOptionsGetter,
+			crdConversionRESTOptionsGetter{
-		crd.Status.AcceptedNames.Categories,
+				RESTOptionsGetter: r.restOptionsGetter,
-		table,
+				converter:         safeConverter,
-	)
+				decoderVersion:    schema.GroupVersion{Group: crd.Spec.Group, Version: v.Name},
+				encoderVersion:    schema.GroupVersion{Group: crd.Spec.Group, Version: storageVersion},
+			},
+			crd.Status.AcceptedNames.Categories,
+			table,
+		)
 
-	selfLinkPrefix := ""
+		selfLinkPrefix := ""
-	switch crd.Spec.Scope {
+		switch crd.Spec.Scope {
-	case apiextensions.ClusterScoped:
+		case apiextensions.ClusterScoped:
-		selfLinkPrefix = "/" + path.Join("apis", crd.Spec.Group, crd.Spec.Version) + "/" + crd.Status.AcceptedNames.Plural + "/"
+			selfLinkPrefix = "/" + path.Join("apis", crd.Spec.Group, v.Name) + "/" + crd.Status.AcceptedNames.Plural + "/"
-	case apiextensions.NamespaceScoped:
+		case apiextensions.NamespaceScoped:
-		selfLinkPrefix = "/" + path.Join("apis", crd.Spec.Group, crd.Spec.Version, "namespaces") + "/"
+			selfLinkPrefix = "/" + path.Join("apis", crd.Spec.Group, v.Name, "namespaces") + "/"
-	}
+		}
 
-	clusterScoped := crd.Spec.Scope == apiextensions.ClusterScoped
+		clusterScoped := crd.Spec.Scope == apiextensions.ClusterScoped
 
-	requestScope := handlers.RequestScope{
+		requestScopes[v.Name] = handlers.RequestScope{
-		Namer: handlers.ContextBasedNaming{
+			Namer: handlers.ContextBasedNaming{
+				SelfLinker:         meta.NewAccessor(),
+				ClusterScoped:      clusterScoped,
+				SelfLinkPathPrefix: selfLinkPrefix,
+			},
+			Serializer:     unstructuredNegotiatedSerializer{typer: typer, creator: creator, converter: safeConverter},
+			ParameterCodec: parameterCodec,
+
+			Creater:         creator,
+			Convertor:       safeConverter,
+			Defaulter:       unstructuredDefaulter{parameterScheme},
+			Typer:           typer,
+			UnsafeConvertor: unsafeConverter,
+
+			Resource: schema.GroupVersionResource{Group: crd.Spec.Group, Version: v.Name, Resource: crd.Status.AcceptedNames.Plural},
+			Kind:     kind,
+
+			MetaGroupVersion: metav1.SchemeGroupVersion,
+
+			TableConvertor: storages[v.Name].CustomResource,
+		}
+
+		// override scaleSpec subresource values
+		// shallow copy
+		scaleScope := requestScopes[v.Name]
+		scaleConverter := scale.NewScaleConverter()
+		scaleScope.Subresource = "scale"
+		scaleScope.Serializer = serializer.NewCodecFactory(scaleConverter.Scheme())
+		scaleScope.Kind = autoscalingv1.SchemeGroupVersion.WithKind("Scale")
+		scaleScope.Namer = handlers.ContextBasedNaming{
 			SelfLinker:         meta.NewAccessor(),
 			ClusterScoped:      clusterScoped,
 			SelfLinkPathPrefix: selfLinkPrefix,
-		},
+			SelfLinkPathSuffix: "/scale",
+		}
+		scaleScopes[v.Name] = scaleScope
 
-		Serializer:     unstructuredNegotiatedSerializer{typer: typer, creator: creator},
+		// override status subresource values
-		ParameterCodec: parameterCodec,
+		// shallow copy
-
+		statusScope := requestScopes[v.Name]
-		Creater: creator,
+		statusScope.Subresource = "status"
-		Convertor: crdObjectConverter{
+		statusScope.Namer = handlers.ContextBasedNaming{
-			UnstructuredObjectConverter: unstructured.UnstructuredObjectConverter{},
+			SelfLinker:         meta.NewAccessor(),
-			clusterScoped:               clusterScoped,
+			ClusterScoped:      clusterScoped,
-		},
+			SelfLinkPathPrefix: selfLinkPrefix,
-		Defaulter:       unstructuredDefaulter{parameterScheme},
+			SelfLinkPathSuffix: "/status",
-		Typer:           typer,
+		}
-		UnsafeConvertor: unstructured.UnstructuredObjectConverter{},
+		statusScopes[v.Name] = statusScope
-
-		Resource: schema.GroupVersionResource{Group: crd.Spec.Group, Version: crd.Spec.Version, Resource: crd.Status.AcceptedNames.Plural},
-		Kind:     kind,
-
-		MetaGroupVersion: metav1.SchemeGroupVersion,
-
-		TableConvertor: customResourceStorage.CustomResource,
 	}
 
 	ret := &crdInfo{
-		spec:          &crd.Spec,
+		spec:                &crd.Spec,
-		acceptedNames: &crd.Status.AcceptedNames,
+		acceptedNames:       &crd.Status.AcceptedNames,
-
+		storages:            storages,
-		storage:            customResourceStorage,
+		requestScopes:       requestScopes,
-		requestScope:       requestScope,
+		scaleRequestScopes:  scaleScopes,
-		scaleRequestScope:  requestScope, // shallow copy
+		statusRequestScopes: statusScopes,
-		statusRequestScope: requestScope, // shallow copy
+		storageVersion:      storageVersion,
-	}
-
-	// override scaleSpec subresource values
-	scaleConverter := scale.NewScaleConverter()
-	ret.scaleRequestScope.Subresource = "scale"
-	ret.scaleRequestScope.Serializer = serializer.NewCodecFactory(scaleConverter.Scheme())
-	ret.scaleRequestScope.Kind = autoscalingv1.SchemeGroupVersion.WithKind("Scale")
-	ret.scaleRequestScope.Namer = handlers.ContextBasedNaming{
-		SelfLinker:         meta.NewAccessor(),
-		ClusterScoped:      clusterScoped,
-		SelfLinkPathPrefix: selfLinkPrefix,
-		SelfLinkPathSuffix: "/scale",
-	}
-
-	// override status subresource values
-	ret.statusRequestScope.Subresource = "status"
-	ret.statusRequestScope.Namer = handlers.ContextBasedNaming{
-		SelfLinker:         meta.NewAccessor(),
-		ClusterScoped:      clusterScoped,
-		SelfLinkPathPrefix: selfLinkPrefix,
-		SelfLinkPathSuffix: "/status",
 	}
 
 	// Copy because we cannot write to storageMap without a race
@@ -517,27 +548,10 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource
 	return ret, nil
 }
 
-// crdObjectConverter is a converter that supports field selectors for CRDs.
-type crdObjectConverter struct {
-	unstructured.UnstructuredObjectConverter
-	clusterScoped bool
-}
-
-func (c crdObjectConverter) ConvertFieldLabel(version, kind, label, value string) (string, string, error) {
-	// We currently only support metadata.namespace and metadata.name.
-	switch {
-	case label == "metadata.name":
-		return label, value, nil
-	case !c.clusterScoped && label == "metadata.namespace":
-		return label, value, nil
-	default:
-		return "", "", fmt.Errorf("field label not supported: %s", label)
-	}
-}
-
 type unstructuredNegotiatedSerializer struct {
-	typer   runtime.ObjectTyper
+	typer     runtime.ObjectTyper
-	creator runtime.ObjectCreater
+	creator   runtime.ObjectCreater
+	converter runtime.ObjectConvertor
 }
 
 func (s unstructuredNegotiatedSerializer) SupportedMediaTypes() []runtime.SerializerInfo {
@@ -562,7 +576,7 @@ func (s unstructuredNegotiatedSerializer) SupportedMediaTypes() []runtime.Serial
 }
 
 func (s unstructuredNegotiatedSerializer) EncoderForVersion(encoder runtime.Encoder, gv runtime.GroupVersioner) runtime.Encoder {
-	return versioning.NewDefaultingCodecForScheme(Scheme, encoder, nil, gv, nil)
+	return versioning.NewCodec(encoder, nil, s.converter, Scheme, Scheme, Scheme, gv, nil)
 }
 
 func (s unstructuredNegotiatedSerializer) DecoderToVersion(decoder runtime.Decoder, gv runtime.GroupVersioner) runtime.Decoder {
@@ -574,6 +588,13 @@ type UnstructuredObjectTyper struct {
 	UnstructuredTyper runtime.ObjectTyper
 }
 
+func newUnstructuredObjectTyper(Delegate runtime.ObjectTyper) UnstructuredObjectTyper {
+	return UnstructuredObjectTyper{
+		Delegate:          Delegate,
+		UnstructuredTyper: discovery.NewUnstructuredObjectTyper(),
+	}
+}
+
 func (t UnstructuredObjectTyper) ObjectKinds(obj runtime.Object) ([]schema.GroupVersionKind, bool, error) {
 	// Delegate for things other than Unstructured.
 	if _, ok := obj.(runtime.Unstructured); !ok {
@@ -640,3 +661,20 @@ func (in crdStorageMap) clone() crdStorageMap {
 	}
 	return out
 }
+
+// crdConversionRESTOptionsGetter overrides the codec with one using the
+// provided custom converter and custom encoder and decoder version.
+type crdConversionRESTOptionsGetter struct {
+	generic.RESTOptionsGetter
+	converter      runtime.ObjectConvertor
+	encoderVersion schema.GroupVersion
+	decoderVersion schema.GroupVersion
+}
+
+func (t crdConversionRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
+	ret, err := t.RESTOptionsGetter.GetRESTOptions(resource)
+	if err == nil {
+		ret.StorageConfig.Codec = versioning.NewCodec(ret.StorageConfig.Codec, ret.StorageConfig.Codec, t.converter, &unstructuredCreator{}, discovery.NewUnstructuredObjectTyper(), &unstructuredDefaulter{delegate: Scheme}, t.encoderVersion, t.decoderVersion)
+	}
+	return ret, err
+}

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler_test.go

@@ -19,7 +19,8 @@ package apiserver
 import (
 	"testing"
 
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
+	conversion "k8s.io/apiextensions-apiserver/pkg/apiserver/conversion"
 )
 
 func TestConvertFieldLabel(t *testing.T) {
@@ -64,10 +65,14 @@ func TestConvertFieldLabel(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 
-			c := crdObjectConverter{
+			crd := apiextensions.CustomResourceDefinition{}
-				UnstructuredObjectConverter: unstructured.UnstructuredObjectConverter{},
+
-				clusterScoped:               test.clusterScoped,
+			if test.clusterScoped {
+				crd.Spec.Scope = apiextensions.ClusterScoped
+			} else {
+				crd.Spec.Scope = apiextensions.NamespaceScoped
 			}
+			_, c := conversion.NewCRDConverter(&crd)
 
 			label, value, err := c.ConvertFieldLabel("", "", test.label, "value")
 			if e, a := test.expectError, err != nil; e != a {

staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy.go

@@ -62,6 +62,15 @@ func (strategy) PrepareForCreate(ctx context.Context, obj runtime.Object) {
 	if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) {
 		crd.Spec.Subresources = nil
 	}
+
+	for _, v := range crd.Spec.Versions {
+		if v.Storage {
+			if !apiextensions.IsStoredVersion(crd, v.Name) {
+				crd.Status.StoredVersions = append(crd.Status.StoredVersions, v.Name)
+			}
+			break
+		}
+	}
 }
 
 // PrepareForUpdate clears fields that are not allowed to be set by end users on update.
@@ -90,6 +99,15 @@ func (strategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
 		newCRD.Spec.Subresources = nil
 		oldCRD.Spec.Subresources = nil
 	}
+
+	for _, v := range newCRD.Spec.Versions {
+		if v.Storage {
+			if !apiextensions.IsStoredVersion(newCRD, v.Name) {
+				newCRD.Status.StoredVersions = append(newCRD.Status.StoredVersions, v.Name)
+			}
+			break
+		}
+	}
 }
 
 // Validate validates a new CustomResourceDefinition.

staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD

@@ -13,6 +13,7 @@ go_test(
         "registration_test.go",
         "subresources_test.go",
         "validation_test.go",
+        "versioning_test.go",
         "yaml_test.go",
     ],
     tags = ["integration"],

staging/src/k8s.io/apiextensions-apiserver/test/integration/registration_test.go

@@ -68,6 +68,42 @@ func instantiateCustomResource(t *testing.T, instanceToCreate *unstructured.Unst
 	return createdInstance, nil
 }
 
+func instantiateVersionedCustomResource(t *testing.T, instanceToCreate *unstructured.Unstructured, client dynamic.ResourceInterface, definition *apiextensionsv1beta1.CustomResourceDefinition, version string) (*unstructured.Unstructured, error) {
+	createdInstance, err := client.Create(instanceToCreate)
+	if err != nil {
+		t.Logf("%#v", createdInstance)
+		return nil, err
+	}
+	createdObjectMeta, err := meta.Accessor(createdInstance)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// it should have a UUID
+	if len(createdObjectMeta.GetUID()) == 0 {
+		t.Errorf("missing uuid: %#v", createdInstance)
+	}
+	createdTypeMeta, err := meta.TypeAccessor(createdInstance)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e, a := definition.Spec.Group+"/"+version, createdTypeMeta.GetAPIVersion(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+	if e, a := definition.Spec.Names.Kind, createdTypeMeta.GetKind(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+	return createdInstance, nil
+}
+
+func NewNamespacedCustomResourceVersionedClient(ns string, client dynamic.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition, version string) dynamic.ResourceInterface {
+	gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: version, Resource: crd.Spec.Names.Plural}
+
+	if crd.Spec.Scope != apiextensionsv1beta1.ClusterScoped {
+		return client.Resource(gvr).Namespace(ns)
+	}
+	return client.Resource(gvr)
+}
+
 func NewNamespacedCustomResourceClient(ns string, client dynamic.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) dynamic.ResourceInterface {
 	gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: crd.Spec.Version, Resource: crd.Spec.Names.Plural}
 

staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/resources.go

@@ -100,6 +100,62 @@ func NewNoxuInstance(namespace, name string) *unstructured.Unstructured {
 	}
 }
 
+func NewMultipleVersionNoxuCRD(scope apiextensionsv1beta1.ResourceScope) *apiextensionsv1beta1.CustomResourceDefinition {
+	return &apiextensionsv1beta1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"},
+		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+			Group:   "mygroup.example.com",
+			Version: "v1beta1",
+			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+				Plural:     "noxus",
+				Singular:   "nonenglishnoxu",
+				Kind:       "WishIHadChosenNoxu",
+				ShortNames: []string{"foo", "bar", "abc", "def"},
+				ListKind:   "NoxuItemList",
+				Categories: []string{"all"},
+			},
+			Scope: scope,
+			Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+				{
+					Name:    "v1beta1",
+					Served:  true,
+					Storage: false,
+				},
+				{
+					Name:    "v1beta2",
+					Served:  true,
+					Storage: true,
+				},
+				{
+					Name:    "v0",
+					Served:  false,
+					Storage: false,
+				},
+			},
+		},
+	}
+}
+
+func NewVersionedNoxuInstance(namespace, name, version string) *unstructured.Unstructured {
+	return &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "mygroup.example.com/" + version,
+			"kind":       "WishIHadChosenNoxu",
+			"metadata": map[string]interface{}{
+				"namespace": namespace,
+				"name":      name,
+			},
+			"content": map[string]interface{}{
+				"key": "value",
+			},
+			"num": map[string]interface{}{
+				"num1": noxuInstanceNum,
+				"num2": 1000000,
+			},
+		},
+	}
+}
+
 func NewNoxu2CustomResourceDefinition(scope apiextensionsv1beta1.ResourceScope) *apiextensionsv1beta1.CustomResourceDefinition {
 	return &apiextensionsv1beta1.CustomResourceDefinition{
 		ObjectMeta: metav1.ObjectMeta{Name: "noxus2.mygroup.example.com"},

staging/src/k8s.io/apiextensions-apiserver/test/integration/versioning_test.go

@@ -0,0 +1,304 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	"k8s.io/apiextensions-apiserver/test/integration/testserver"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/dynamic"
+)
+
+func TestVersionedNamspacedScopedCRD(t *testing.T) {
+	stopCh, apiExtensionClient, dynamicClient, err := testserver.StartDefaultServerWithClients()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer close(stopCh)
+
+	noxuDefinition := testserver.NewMultipleVersionNoxuCRD(apiextensionsv1beta1.NamespaceScoped)
+	err = testserver.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ns := "not-the-default"
+	testSimpleVersionedCRUD(t, ns, noxuDefinition, dynamicClient)
+}
+
+func TestVersionedClusterScopedCRD(t *testing.T) {
+	stopCh, apiExtensionClient, dynamicClient, err := testserver.StartDefaultServerWithClients()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer close(stopCh)
+
+	noxuDefinition := testserver.NewMultipleVersionNoxuCRD(apiextensionsv1beta1.ClusterScoped)
+	err = testserver.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ns := ""
+	testSimpleVersionedCRUD(t, ns, noxuDefinition, dynamicClient)
+}
+
+func TestStoragedVersionInNamespacedCRDStatus(t *testing.T) {
+	noxuDefinition := testserver.NewMultipleVersionNoxuCRD(apiextensionsv1beta1.NamespaceScoped)
+	ns := "not-the-default"
+	testStoragedVersionInCRDStatus(t, ns, noxuDefinition)
+}
+
+func TestStoragedVersionInClusterScopedCRDStatus(t *testing.T) {
+	noxuDefinition := testserver.NewMultipleVersionNoxuCRD(apiextensionsv1beta1.ClusterScoped)
+	ns := ""
+	testStoragedVersionInCRDStatus(t, ns, noxuDefinition)
+}
+
+func testStoragedVersionInCRDStatus(t *testing.T, ns string, noxuDefinition *apiextensionsv1beta1.CustomResourceDefinition) {
+	versionsV1Beta1Storage := []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+		{
+			Name:    "v1beta1",
+			Served:  true,
+			Storage: true,
+		},
+		{
+			Name:    "v1beta2",
+			Served:  true,
+			Storage: false,
+		},
+	}
+	versionsV1Beta2Storage := []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+		{
+			Name:    "v1beta1",
+			Served:  true,
+			Storage: false,
+		},
+		{
+			Name:    "v1beta2",
+			Served:  true,
+			Storage: true,
+		},
+	}
+	stopCh, apiExtensionClient, dynamicClient, err := testserver.StartDefaultServerWithClients()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer close(stopCh)
+
+	noxuDefinition.Spec.Versions = versionsV1Beta1Storage
+	err = testserver.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// The storage version list should be initilized to storage version
+	crd, err := testserver.GetCustomResourceDefinition(noxuDefinition, apiExtensionClient)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e, a := []string{"v1beta1"}, crd.Status.StoredVersions; !reflect.DeepEqual(e, a) {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+
+	// Changing CRD storage version should be reflected immediately
+	crd.Spec.Versions = versionsV1Beta2Storage
+	_, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd)
+	if err != nil {
+		t.Fatal(err)
+	}
+	crd, err = testserver.GetCustomResourceDefinition(noxuDefinition, apiExtensionClient)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e, a := []string{"v1beta1", "v1beta2"}, crd.Status.StoredVersions; !reflect.DeepEqual(e, a) {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+
+	err = testserver.DeleteCustomResourceDefinition(crd, apiExtensionClient)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func testSimpleVersionedCRUD(t *testing.T, ns string, noxuDefinition *apiextensionsv1beta1.CustomResourceDefinition, dynamicClient dynamic.Interface) {
+	noxuResourceClients := map[string]dynamic.ResourceInterface{}
+	noxuWatchs := map[string]watch.Interface{}
+	disbaledVersions := map[string]bool{}
+	for _, v := range noxuDefinition.Spec.Versions {
+		disbaledVersions[v.Name] = !v.Served
+	}
+	for _, v := range noxuDefinition.Spec.Versions {
+		noxuResourceClients[v.Name] = NewNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name)
+
+		noxuWatch, err := noxuResourceClients[v.Name].Watch(metav1.ListOptions{})
+		if disbaledVersions[v.Name] {
+			if err == nil {
+				t.Errorf("expected the watch creation fail for disabled version %s", v.Name)
+			}
+		} else {
+			if err != nil {
+				t.Fatal(err)
+			}
+			noxuWatchs[v.Name] = noxuWatch
+		}
+	}
+	defer func() {
+		for _, w := range noxuWatchs {
+			w.Stop()
+		}
+	}()
+
+	for version, noxuResourceClient := range noxuResourceClients {
+		createdNoxuInstance, err := instantiateVersionedCustomResource(t, testserver.NewVersionedNoxuInstance(ns, "foo", version), noxuResourceClient, noxuDefinition, version)
+		if disbaledVersions[version] {
+			if err == nil {
+				t.Errorf("expected the CR creation fail for disabled version %s", version)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("unable to create noxu Instance:%v", err)
+		}
+		if e, a := noxuDefinition.Spec.Group+"/"+version, createdNoxuInstance.GetAPIVersion(); e != a {
+			t.Errorf("expected %v, got %v", e, a)
+		}
+		for watchVersion, noxuWatch := range noxuWatchs {
+			select {
+			case watchEvent := <-noxuWatch.ResultChan():
+				if e, a := watch.Added, watchEvent.Type; e != a {
+					t.Errorf("expected %v, got %v", e, a)
+					break
+				}
+				createdObjectMeta, err := meta.Accessor(watchEvent.Object)
+				if err != nil {
+					t.Fatal(err)
+				}
+				// it should have a UUID
+				if len(createdObjectMeta.GetUID()) == 0 {
+					t.Errorf("missing uuid: %#v", watchEvent.Object)
+				}
+				if e, a := ns, createdObjectMeta.GetNamespace(); e != a {
+					t.Errorf("expected %v, got %v", e, a)
+				}
+				createdTypeMeta, err := meta.TypeAccessor(watchEvent.Object)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if e, a := noxuDefinition.Spec.Group+"/"+watchVersion, createdTypeMeta.GetAPIVersion(); e != a {
+					t.Errorf("expected %v, got %v", e, a)
+				}
+				if e, a := noxuDefinition.Spec.Names.Kind, createdTypeMeta.GetKind(); e != a {
+					t.Errorf("expected %v, got %v", e, a)
+				}
+			case <-time.After(5 * time.Second):
+				t.Errorf("missing watch event")
+			}
+		}
+
+		// Check get for all versions
+		for version2, noxuResourceClient2 := range noxuResourceClients {
+			// Get test
+			gottenNoxuInstance, err := noxuResourceClient2.Get("foo", metav1.GetOptions{})
+
+			if disbaledVersions[version2] {
+				if err == nil {
+					t.Errorf("expected the get operation fail for disabled version %s", version2)
+				}
+			} else {
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				if e, a := version2, gottenNoxuInstance.GroupVersionKind().Version; !reflect.DeepEqual(e, a) {
+					t.Errorf("expected %v, got %v", e, a)
+				}
+			}
+
+			// List test
+			listWithItem, err := noxuResourceClient2.List(metav1.ListOptions{})
+			if disbaledVersions[version2] {
+				if err == nil {
+					t.Errorf("expected the list operation fail for disabled version %s", version2)
+				}
+			} else {
+				if err != nil {
+					t.Fatal(err)
+				}
+				if e, a := 1, len(listWithItem.Items); e != a {
+					t.Errorf("expected %v, got %v", e, a)
+				}
+				if e, a := version2, listWithItem.GroupVersionKind().Version; !reflect.DeepEqual(e, a) {
+					t.Errorf("expected %v, got %v", e, a)
+				}
+				if e, a := version2, listWithItem.Items[0].GroupVersionKind().Version; !reflect.DeepEqual(e, a) {
+					t.Errorf("expected %v, got %v", e, a)
+				}
+			}
+		}
+
+		// Delete test
+		if err := noxuResourceClient.Delete("foo", metav1.NewDeleteOptions(0)); err != nil {
+			t.Fatal(err)
+		}
+
+		listWithoutItem, err := noxuResourceClient.List(metav1.ListOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if e, a := 0, len(listWithoutItem.Items); e != a {
+			t.Errorf("expected %v, got %v", e, a)
+		}
+
+		for _, noxuWatch := range noxuWatchs {
+			select {
+			case watchEvent := <-noxuWatch.ResultChan():
+				if e, a := watch.Deleted, watchEvent.Type; e != a {
+					t.Errorf("expected %v, got %v", e, a)
+					break
+				}
+				deletedObjectMeta, err := meta.Accessor(watchEvent.Object)
+				if err != nil {
+					t.Fatal(err)
+				}
+				// it should have a UUID
+				createdObjectMeta, err := meta.Accessor(createdNoxuInstance)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if e, a := createdObjectMeta.GetUID(), deletedObjectMeta.GetUID(); e != a {
+					t.Errorf("expected %v, got %v", e, a)
+				}
+
+			case <-time.After(5 * time.Second):
+				t.Errorf("missing watch event")
+			}
+		}
+
+		// Delete test
+		if err := noxuResourceClient.DeleteCollection(metav1.NewDeleteOptions(0), metav1.ListOptions{}); err != nil {
+			t.Fatal(err)
+		}
+
+	}
+}

staging/src/k8s.io/apimachinery/pkg/runtime/serializer/versioning/BUILD

@@ -22,6 +22,7 @@ go_library(
     srcs = ["versioning.go"],
     importpath = "k8s.io/apimachinery/pkg/runtime/serializer/versioning",
     deps = [
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
     ],

staging/src/k8s.io/apimachinery/pkg/runtime/serializer/versioning/versioning.go

@@ -19,6 +19,7 @@ package versioning
 import (
 	"io"
 
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )
@@ -170,17 +171,22 @@ func (c *codec) Encode(obj runtime.Object, w io.Writer) error {
 	case *runtime.Unknown:
 		return c.encoder.Encode(obj, w)
 	case runtime.Unstructured:
-		// avoid conversion roundtrip if GVK is the right one already or is empty (yes, this is a hack, but the old behaviour we rely on in kubectl)
+		// An unstructured list can contain objects of multiple group version kinds. don't short-circuit just
-		objGVK := obj.GetObjectKind().GroupVersionKind()
+		// because the top-level type matches our desired destination type. actually send the object to the converter
-		if len(objGVK.Version) == 0 {
+		// to give it a chance to convert the list items if needed.
-			return c.encoder.Encode(obj, w)
+		if _, ok := obj.(*unstructured.UnstructuredList); !ok {
-		}
+			// avoid conversion roundtrip if GVK is the right one already or is empty (yes, this is a hack, but the old behaviour we rely on in kubectl)
-		targetGVK, ok := c.encodeVersion.KindForGroupVersionKinds([]schema.GroupVersionKind{objGVK})
+			objGVK := obj.GetObjectKind().GroupVersionKind()
-		if !ok {
+			if len(objGVK.Version) == 0 {
-			return runtime.NewNotRegisteredGVKErrForTarget(objGVK, c.encodeVersion)
+				return c.encoder.Encode(obj, w)
-		}
+			}
-		if targetGVK == objGVK {
+			targetGVK, ok := c.encodeVersion.KindForGroupVersionKinds([]schema.GroupVersionKind{objGVK})
-			return c.encoder.Encode(obj, w)
+			if !ok {
+				return runtime.NewNotRegisteredGVKErrForTarget(objGVK, c.encodeVersion)
+			}
+			if targetGVK == objGVK {
+				return c.encoder.Encode(obj, w)
+			}
 		}
 	}
 

staging/src/k8s.io/metrics/Godeps/Godeps.json

@@ -278,6 +278,10 @@
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
+		{
+			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1beta1",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"