Merge pull request #131196 from siyuanfoundation/forward-api

bug fix: fix version order in emulation forward compatibility.

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -156,8 +156,9 @@ type Config struct {
 	EffectiveVersion basecompatibility.EffectiveVersion
 	// EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and
 	// have higher priority than APIs of the same group resource enabled at the emulation version.
-	// If true, all APIs that have higher priority than the APIs of the same group resource enabled at the emulation version will be installed.
+	// If true, all APIs that have higher priority than the APIs(beta+) of the same group resource enabled at the emulation version will be installed.
 	// This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set.
+	// Not applicable to alpha APIs.
 	EmulationForwardCompatible bool
 	// RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config.
 	// If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version.

staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"os"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -28,6 +29,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	apimachineryversion "k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/apiserver/pkg/registry/rest"
 	serverstorage "k8s.io/apiserver/pkg/server/storage"
 	"k8s.io/klog/v2"
@@ -71,6 +73,7 @@ type ResourceExpirationEvaluatorOptions struct {
 	Prerelease string
 	// EmulationForwardCompatible indicates whether the apiserver should serve resources that are introduced after the current version,
 	// when resources of the same group and resource name but with lower priority are served.
+	// Not applicable to alpha APIs.
 	EmulationForwardCompatible bool
 	// RuntimeConfigEmulationForwardCompatible indicates whether the apiserver should serve resources that are introduced after the current version,
 	// when the resource is explicitly enabled in runtime-config.
@@ -222,6 +225,9 @@ func (e *resourceExpirationEvaluator) RemoveUnavailableKinds(groupName string, v
 func (e *resourceExpirationEvaluator) removeUnintroducedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage, apiResourceConfigSource serverstorage.APIResourceConfigSource) error {
 	versionsToRemove := sets.NewString()
 	prioritizedVersions := versioner.PrioritizedVersionsForGroup(groupName)
+	sort.Slice(prioritizedVersions, func(i, j int) bool {
+		return version.CompareKubeAwareVersionStrings(prioritizedVersions[i].Version, prioritizedVersions[j].Version) > 0
+	})
 	enabledResources := sets.NewString()
 
 	// iterate from the end to the front, so that we remove the lower priority versions first.

staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go

@@ -801,13 +801,272 @@ func Test_removeUnIntroducedKinds(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "both-runtime-config-and-emulation-forward-compatible-runtime-config-alpha-api",
+			resourceExpirationEvaluator: resourceExpirationEvaluator{
+				currentVersion:                          apimachineryversion.MajorMinor(1, 19),
+				runtimeConfigEmulationForwardCompatible: true,
+				emulationForwardCompatible:              true,
+			},
+			runtimeConfig: map[string]string{
+				groupName + "/v2alpha1": "true",
+			},
+			versionedResourcesStorageMap: map[string]map[string]rest.Storage{
+				"v1": {
+					resource1: storageIntroducedIn(1, 19),
+				},
+				"v2alpha1": {
+					resource1: storageIntroducedAndRemovedIn(1, 20, 1, 21),
+					resource2: storageIntroducedAndRemovedIn(1, 20, 1, 21),
+				},
+				"v2beta1": {
+					resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
+				},
+				"v2beta2": {
+					resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+					resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+				},
+				"v2": {
+					resource1: storageIntroducedIn(1, 23),
+					resource2: storageIntroducedIn(1, 23),
+				},
+			},
+			expectedStorage: map[string]map[string]rest.Storage{
+				"v1": {
+					resource1: storageIntroducedIn(1, 19),
+				},
+				"v2alpha1": {
+					resource1: storageIntroducedAndRemovedIn(1, 20, 1, 21),
+					resource2: storageIntroducedAndRemovedIn(1, 20, 1, 21),
+				},
+				"v2": {
+					resource1: storageIntroducedIn(1, 23),
+				},
+			},
+		},
+		{
+			name: "both-runtime-config-and-emulation-forward-compatible-runtime-config-alpha-resource",
+			resourceExpirationEvaluator: resourceExpirationEvaluator{
+				currentVersion:                          apimachineryversion.MajorMinor(1, 19),
+				runtimeConfigEmulationForwardCompatible: true,
+				emulationForwardCompatible:              true,
+			},
+			runtimeConfig: map[string]string{
+				groupName + "/v2alpha1/resource2": "true",
+			},
+			versionedResourcesStorageMap: map[string]map[string]rest.Storage{
+				"v1": {
+					resource1: storageIntroducedIn(1, 19),
+				},
+				"v2alpha1": {
+					resource1: storageIntroducedAndRemovedIn(1, 20, 1, 21),
+					resource2: storageIntroducedAndRemovedIn(1, 20, 1, 21),
+				},
+				"v2beta1": {
+					resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
+				},
+				"v2beta2": {
+					resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+					resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+				},
+				"v2": {
+					resource1: storageIntroducedIn(1, 23),
+					resource2: storageIntroducedIn(1, 23),
+				},
+			},
+			expectedStorage: map[string]map[string]rest.Storage{
+				"v1": {
+					resource1: storageIntroducedIn(1, 19),
+				},
+				"v2alpha1": {
+					resource2: storageIntroducedAndRemovedIn(1, 20, 1, 21),
+				},
+				"v2": {
+					resource1: storageIntroducedIn(1, 23),
+				},
+			},
+		},
+		{
+			name: "emulation-forward-compatible-beta1-api",
+			resourceExpirationEvaluator: resourceExpirationEvaluator{
+				currentVersion:             apimachineryversion.MajorMinor(1, 21),
+				emulationForwardCompatible: true,
+			},
+			runtimeConfig: map[string]string{
+				groupName + "/v2beta1": "true",
+			},
+			versionedResourcesStorageMap: map[string]map[string]rest.Storage{
+				"v1": {
+					resource1: storageIntroducedIn(1, 20),
+				},
+				"v2beta1": {
+					resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
+					resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
+				},
+				"v2beta2": {
+					resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+					resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+				},
+				"v2": {
+					resource1: storageIntroducedIn(1, 23),
+					resource2: storageIntroducedIn(1, 23),
+				},
+			},
+			expectedStorage: map[string]map[string]rest.Storage{
+				"v1": {
+					resource1: storageIntroducedIn(1, 20),
+				},
+				"v2beta1": {
+					resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
+					resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
+				},
+				"v2beta2": {
+					resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+					resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+				},
+				"v2": {
+					resource1: storageIntroducedIn(1, 23),
+					resource2: storageIntroducedIn(1, 23),
+				},
+			},
+		},
+		{
+			name: "emulation-forward-compatible-beta1-resource",
+			resourceExpirationEvaluator: resourceExpirationEvaluator{
+				currentVersion:             apimachineryversion.MajorMinor(1, 21),
+				emulationForwardCompatible: true,
+			},
+			runtimeConfig: map[string]string{
+				groupName + "/v2beta1/resource2": "true",
+			},
+			versionedResourcesStorageMap: map[string]map[string]rest.Storage{
+				"v1": {
+					resource1: storageIntroducedIn(1, 20),
+				},
+				"v2beta1": {
+					resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
+				},
+				"v2beta2": {
+					resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+					resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+				},
+				"v2": {
+					resource1: storageIntroducedIn(1, 23),
+					resource2: storageIntroducedIn(1, 23),
+				},
+			},
+			expectedStorage: map[string]map[string]rest.Storage{
+				"v1": {
+					resource1: storageIntroducedIn(1, 20),
+				},
+				"v2beta1": {
+					resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
+				},
+				"v2beta2": {
+					resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+				},
+				"v2": {
+					resource1: storageIntroducedIn(1, 23),
+					resource2: storageIntroducedIn(1, 23),
+				},
+			},
+		},
+		{
+			name: "both-runtime-config-and-emulation-forward-compatible-runtime-config-beta1-api",
+			resourceExpirationEvaluator: resourceExpirationEvaluator{
+				currentVersion:                          apimachineryversion.MajorMinor(1, 20),
+				emulationForwardCompatible:              true,
+				runtimeConfigEmulationForwardCompatible: true,
+			},
+			runtimeConfig: map[string]string{
+				groupName + "/v2beta1": "true",
+			},
+			versionedResourcesStorageMap: map[string]map[string]rest.Storage{
+				"v1": {
+					resource1: storageIntroducedIn(1, 20),
+				},
+				"v2beta1": {
+					resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
+					resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
+				},
+				"v2beta2": {
+					resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+					resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+				},
+				"v2": {
+					resource1: storageIntroducedIn(1, 23),
+					resource2: storageIntroducedIn(1, 23),
+				},
+			},
+			expectedStorage: map[string]map[string]rest.Storage{
+				"v1": {
+					resource1: storageIntroducedIn(1, 20),
+				},
+				"v2beta1": {
+					resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
+					resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
+				},
+				"v2beta2": {
+					resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+					resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+				},
+				"v2": {
+					resource1: storageIntroducedIn(1, 23),
+					resource2: storageIntroducedIn(1, 23),
+				},
+			},
+		},
+		{
+			name: "both-runtime-config-and-emulation-forward-compatible-runtime-config-beta1-resource",
+			resourceExpirationEvaluator: resourceExpirationEvaluator{
+				currentVersion:                          apimachineryversion.MajorMinor(1, 20),
+				emulationForwardCompatible:              true,
+				runtimeConfigEmulationForwardCompatible: true,
+			},
+			runtimeConfig: map[string]string{
+				groupName + "/v2beta1/resource2": "true",
+			},
+			versionedResourcesStorageMap: map[string]map[string]rest.Storage{
+				"v1": {
+					resource1: storageIntroducedIn(1, 20),
+				},
+				"v2beta1": {
+					resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
+					resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
+				},
+				"v2beta2": {
+					resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+					resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+				},
+				"v2": {
+					resource1: storageIntroducedIn(1, 23),
+					resource2: storageIntroducedIn(1, 23),
+				},
+			},
+			expectedStorage: map[string]map[string]rest.Storage{
+				"v1": {
+					resource1: storageIntroducedIn(1, 20),
+				},
+				"v2beta1": {
+					resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
+				},
+				"v2beta2": {
+					resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
+				},
+				"v2": {
+					resource1: storageIntroducedIn(1, 23),
+					resource2: storageIntroducedIn(1, 23),
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			resourceConfig := serverstorage.NewResourceConfig()
 			convertor := &dummyConvertor{prioritizedVersions: []schema.GroupVersion{
 				{Group: groupName, Version: "v2"}, {Group: groupName, Version: "v1"},
-				{Group: groupName, Version: "v2beta2"}, {Group: groupName, Version: "v2beta1"},
+				{Group: groupName, Version: "v2beta1"},
+				{Group: groupName, Version: "v2beta2"},
 				{Group: groupName, Version: "v2alpha1"}}}
 			resourceConfig.EnableVersions(convertor.PrioritizedVersionsForGroup(groupName)...)
 			resourceConfig, err := resourceconfig.MergeAPIResourceConfigs(resourceConfig, tt.runtimeConfig, convertor)

staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go

@@ -247,8 +247,9 @@ type GenericAPIServer struct {
 	EffectiveVersion basecompatibility.EffectiveVersion
 	// EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and
 	// have higher priority than APIs of the same group resource enabled at the emulation version.
-	// If true, all APIs that have higher priority than the APIs of the same group resource enabled at the emulation version will be installed.
+	// If true, all APIs that have higher priority than the APIs(beta+) of the same group resource enabled at the emulation version will be installed.
 	// This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set.
+	// Not applicable to alpha APIs.
 	EmulationForwardCompatible bool
 	// RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config.
 	// If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version.

staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go

@@ -100,8 +100,9 @@ type ServerRunOptions struct {
 	ComponentName string
 	// EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and
 	// have higher priority than APIs of the same group resource enabled at the emulation version.
-	// If true, all APIs that have higher priority than the APIs of the same group resource enabled at the emulation version will be installed.
+	// If true, all APIs that have higher priority than the APIs(beta+) of the same group resource enabled at the emulation version will be installed.
 	// This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set.
+	// Not applicable to alpha APIs.
 	EmulationForwardCompatible bool
 	// RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config.
 	// If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version.
@@ -399,10 +400,11 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 
 	s.ComponentGlobalsRegistry.AddFlags(fs)
 	fs.BoolVar(&s.EmulationForwardCompatible, "emulation-forward-compatible", s.EmulationForwardCompatible, ""+
-		"If true all APIs that have higher priority than the APIs enabled at the emulation version of the same group resource will be installed. "+
+		"If true, for any beta+ APIs enabled by default or by --runtime-config at the emulation version, their future versions with higher priority/stability will be auto enabled even if they introduced after the emulation version. "+
 		"Can only be set to true if the emulation version is lower than the binary version.")
 	fs.BoolVar(&s.RuntimeConfigEmulationForwardCompatible, "runtime-config-emulation-forward-compatible", s.RuntimeConfigEmulationForwardCompatible, ""+
 		"If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. "+
+		"If false, server would fail to start if any APIs identified by group/version that are enabled in the --runtime-config flag are introduced after the emulation version. "+
 		"Can only be set to true if the emulation version is lower than the binary version.")
 }
 

staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers.go

@@ -19,11 +19,13 @@ package resourceconfig
 import (
 	"fmt"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/version"
 	serverstore "k8s.io/apiserver/pkg/server/storage"
 	cliflag "k8s.io/component-base/cli/flag"
 )
@@ -258,7 +260,11 @@ func EmulationForwardCompatibleResourceConfig(
 			continue
 		}
 		// if a gv is enabled, all the versions with higher priority (all the versions before gv in PrioritizedVersionsForGroup) are also implicitly enabled for emulation forward compatibility.
-		for _, pgv := range registry.PrioritizedVersionsForGroup(gv.Group) {
+		prioritizedVersions := registry.PrioritizedVersionsForGroup(gv.Group)
+		sort.Slice(prioritizedVersions, func(i, j int) bool {
+			return version.CompareKubeAwareVersionStrings(prioritizedVersions[i].Version, prioritizedVersions[j].Version) > 0
+		})
+		for _, pgv := range prioritizedVersions {
 			if pgv.Version == gv.Version {
 				break
 			}
@@ -275,7 +281,11 @@ func EmulationForwardCompatibleResourceConfig(
 			continue
 		}
 		// if a gvr is enabled, all the versions with the same resource name and higher priority (all the versions before gv in PrioritizedVersionsForGroup) are also implicitly enabled for emulation forward compatibility.
-		for _, pgv := range registry.PrioritizedVersionsForGroup(gvr.Group) {
+		prioritizedVersions := registry.PrioritizedVersionsForGroup(gvr.Group)
+		sort.Slice(prioritizedVersions, func(i, j int) bool {
+			return version.CompareKubeAwareVersionStrings(prioritizedVersions[i].Version, prioritizedVersions[j].Version) > 0
+		})
+		for _, pgv := range prioritizedVersions {
 			if pgv.Version == gvr.Version {
 				break
 			}

staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers_test.go

@@ -802,5 +802,5 @@ func addTestGVs(t *testing.T, s *runtime.Scheme) {
 	s.AddKnownTypes(v2, &runtimetesting.TestType1{}, &runtimetesting.TestType2{})
 
 	require.NoError(t, runtimetesting.RegisterConversions(s))
-	require.NoError(t, s.SetVersionPriority(v2, v1, v2beta2, v2beta1))
+	require.NoError(t, s.SetVersionPriority(v2, v1, v2beta1, v2beta2))
 }

test/integration/etcd/data.go

@@ -294,18 +294,16 @@ func GetEtcdStorageDataForNamespaceServedAt(namespace string, v string, removeAl
 			IntroducedVersion: "1.7",
 		},
 		gvr("networking.k8s.io", "v1", "ipaddresses"): {
-			Stub:                                   `{"metadata": {"name": "192.168.2.3"}, "spec": {"parentRef": {"resource": "services","name": "test", "namespace": "ns"}}}`,
+			Stub:              `{"metadata": {"name": "192.168.2.3"}, "spec": {"parentRef": {"resource": "services","name": "test", "namespace": "ns"}}}`,
-			ExpectedEtcdPath:                       "/registry/ipaddresses/192.168.2.3",
+			ExpectedEtcdPath:  "/registry/ipaddresses/192.168.2.3",
-			ExpectedGVK:                            gvkP("networking.k8s.io", "v1beta1", "IPAddress"),
+			ExpectedGVK:       gvkP("networking.k8s.io", "v1beta1", "IPAddress"),
-			IntroducedVersion:                      "1.33",
+			IntroducedVersion: "1.33",
-			EmulationForwardCompatibleSinceVersion: "1.31",
 		},
 		gvr("networking.k8s.io", "v1", "servicecidrs"): {
-			Stub:                                   `{"metadata": {"name": "range-b2"}, "spec": {"cidrs": ["192.168.0.0/16","fd00:1::/120"]}}`,
+			Stub:              `{"metadata": {"name": "range-b2"}, "spec": {"cidrs": ["192.168.0.0/16","fd00:1::/120"]}}`,
-			ExpectedEtcdPath:                       "/registry/servicecidrs/range-b2",
+			ExpectedEtcdPath:  "/registry/servicecidrs/range-b2",
-			ExpectedGVK:                            gvkP("networking.k8s.io", "v1beta1", "ServiceCIDR"),
+			ExpectedGVK:       gvkP("networking.k8s.io", "v1beta1", "ServiceCIDR"),
-			IntroducedVersion:                      "1.33",
+			IntroducedVersion: "1.33",
-			EmulationForwardCompatibleSinceVersion: "1.31",
 		},
 		// --
 
@@ -680,6 +678,22 @@ func GetEtcdStorageDataForNamespaceServedAt(namespace string, v string, removeAl
 		// --
 	}
 
+	// get the min version the Beta api of a group is introduced, and it would be used to determine emulation forward compatibility.
+	minBetaVersions := map[schema.GroupResource]*version.Version{}
+	for key, data := range etcdStorageData {
+		if !strings.Contains(key.Version, "beta") {
+			continue
+		}
+		introduced := version.MustParse(data.IntroducedVersion)
+		if ver, ok := minBetaVersions[key.GroupResource()]; ok {
+			if introduced.LessThan(ver) {
+				minBetaVersions[key.GroupResource()] = introduced
+			}
+		} else {
+			minBetaVersions[key.GroupResource()] = introduced
+		}
+	}
+
 	// Delete types no longer served or not yet added at a particular emulated version.
 	for key, data := range etcdStorageData {
 		if data.RemovedVersion != "" && version.MustParse(v).AtLeast(version.MustParse(data.RemovedVersion)) {
@@ -688,7 +702,8 @@ func GetEtcdStorageDataForNamespaceServedAt(namespace string, v string, removeAl
 		if data.IntroducedVersion == "" || version.MustParse(v).AtLeast(version.MustParse(data.IntroducedVersion)) {
 			continue
 		}
-		if data.EmulationForwardCompatibleSinceVersion != "" && version.MustParse(v).AtLeast(version.MustParse(data.EmulationForwardCompatibleSinceVersion)) {
+		minBetaVersion, ok := minBetaVersions[key.GroupResource()]
+		if ok && version.MustParse(v).AtLeast(minBetaVersion) {
 			continue
 		}
 		delete(etcdStorageData, key)
@@ -735,10 +750,6 @@ type StorageData struct {
 	ExpectedGVK       *schema.GroupVersionKind // The GVK that we expect this object to be stored as - leave this nil to use the default
 	IntroducedVersion string                   // The version that this type is introduced
 	RemovedVersion    string                   // The version that this type is removed. May be empty for stable resources
-	// EmulationForwardCompatibleSinceVersion indicates the api should be kept if the emulation version is >= this version, even when the api is introduced after the emulation version.
-	// Only needed for some Beta (Beta2+) and GA APIs, and is the version when the lowest Beta api is introduced.
-	// This is needed to enable some Beta features where the api used in the corresponding controller has GAed.
-	EmulationForwardCompatibleSinceVersion string
 }
 
 // Prerequisite contains information required to create a resource (but not verify it)