expand etcd storage tests to test for 1.31..1.33

test/integration/controlplane/transformation/kms_transformation_test.go

@@ -618,9 +618,6 @@ resources:
         endpoint: unix:///@encrypt-all-kms-provider.sock
 `
 
-	// KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE allows for APIs pending removal to not block tests
-	t.Setenv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE", "true")
-
 	t.Run("encrypt all resources", func(t *testing.T) {
 		_ = mock.NewBase64Plugin(t, "@encrypt-all-kms-provider.sock")
 		// To ensure we are checking all REST resources

test/integration/etcd/data.go

@@ -17,26 +17,54 @@ limitations under the License.
 package etcd
 
 import (
+	"strings"
+
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apiextensions-apiserver/test/integration/fixtures"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilversion "k8s.io/component-base/version"
 
 	"k8s.io/kubernetes/test/utils/image"
 )
 
-// GetEtcdStorageData returns etcd data for all persisted objects.
+// GetSupportedEmulatedVersions provides the list of supported emulated versions in the etcd data.
+// Tests aiming for full coverage of versions should test fixtures of all supported versions.
+func GetSupportedEmulatedVersions() []string {
+	return []string{
+		utilversion.DefaultKubeEffectiveVersion().BinaryVersion().SubtractMinor(2).String(),
+		utilversion.DefaultKubeEffectiveVersion().BinaryVersion().SubtractMinor(1).String(),
+		utilversion.DefaultKubeEffectiveVersion().BinaryVersion().String(),
+	}
+}
+
+// GetEtcdStorageData returns etcd data for all persisted objects at the latest release version.
 // It is exported so that it can be reused across multiple tests.
 // It returns a new map on every invocation to prevent different tests from mutating shared state.
 func GetEtcdStorageData() map[schema.GroupVersionResource]StorageData {
-	return GetEtcdStorageDataForNamespace("etcdstoragepathtestnamespace")
+	return GetEtcdStorageDataServedAt(utilversion.DefaultKubeBinaryVersion, false)
 }
 
-// GetEtcdStorageDataForNamespace returns etcd data for all persisted objects.
+// GetEtcdStorageDataServedAt returns etcd data for all persisted objects at a particular release version.
+// It is exported so that it can be reused across multiple tests.
+// It returns a new map on every invocation to prevent different tests from mutating shared state.
+func GetEtcdStorageDataServedAt(version string, removeAlphas bool) map[schema.GroupVersionResource]StorageData {
+	return GetEtcdStorageDataForNamespaceServedAt("etcdstoragepathtestnamespace", version, removeAlphas)
+}
+
+// GetEtcdStorageDataForNamespace returns etcd data for all persisted objects at the latest release version.
 // It is exported so that it can be reused across multiple tests.
 // It returns a new map on every invocation to prevent different tests from mutating shared state.
 // Namespaced objects keys are computed for the specified namespace.
 func GetEtcdStorageDataForNamespace(namespace string) map[schema.GroupVersionResource]StorageData {
+	return GetEtcdStorageDataForNamespaceServedAt(namespace, utilversion.DefaultKubeBinaryVersion, false)
+}
+
+// GetEtcdStorageDataForNamespaceServedAt returns etcd data for all persisted objects at a particular release version.
+// It is exported so that it can be reused across multiple tests.
+// It returns a new map on every invocation to prevent different tests from mutating shared state.
+// Namespaced objects keys are computed for the specified namespace.
+func GetEtcdStorageDataForNamespaceServedAt(namespace string, version string, removeAlphas bool) map[schema.GroupVersionResource]StorageData {
 	image := image.GetE2EImage(image.BusyBox)
 	etcdStorageData := map[schema.GroupVersionResource]StorageData{
 		// k8s.io/kubernetes/pkg/api/v1
@@ -465,22 +493,46 @@ func GetEtcdStorageDataForNamespace(namespace string) map[schema.GroupVersionRes
 		},
 		// --
 
-	}
-
-	// add csinodes
 		// k8s.io/kubernetes/pkg/apis/storage/v1
-	etcdStorageData[gvr("storage.k8s.io", "v1", "csinodes")] = StorageData{
+		gvr("storage.k8s.io", "v1", "csinodes"): {
 			Stub:             `{"metadata": {"name": "csini2"}, "spec": {"drivers": [{"name": "test-driver", "nodeID": "localhost", "topologyKeys": ["company.com/zone1", "company.com/zone2"]}]}}`,
 			ExpectedEtcdPath: "/registry/csinodes/csini2",
-	}
+		},
+		// --
 
-	// add csidrivers
 		// k8s.io/kubernetes/pkg/apis/storage/v1
-	etcdStorageData[gvr("storage.k8s.io", "v1", "csidrivers")] = StorageData{
+		gvr("storage.k8s.io", "v1", "csidrivers"): {
 			Stub:             `{"metadata": {"name": "csid2"}, "spec": {"attachRequired": true, "podInfoOnMount": true}}`,
 			ExpectedEtcdPath: "/registry/csidrivers/csid2",
+		},
+		// --
 	}
 
+	// Delete types no longer served or not yet added at a particular emulated version.
+	// When adding a brand new type non-alpha type in the latest release, please ensure that
+	// it is removed in previous emulated versions otherwise emulated version tests will fail.
+	// TODO: derive this programatically from gvk --> instance --> APILifecycle info
+	switch version {
+	case "1.33":
+		delete(etcdStorageData, gvr("flowcontrol.apiserver.k8s.io", "v1beta3", "flowschemas"))
+		delete(etcdStorageData, gvr("flowcontrol.apiserver.k8s.io", "v1beta3", "prioritylevelconfigurations"))
+	case "1.32":
+		delete(etcdStorageData, gvr("flowcontrol.apiserver.k8s.io", "v1beta3", "flowschemas"))
+		delete(etcdStorageData, gvr("flowcontrol.apiserver.k8s.io", "v1beta3", "prioritylevelconfigurations"))
+	case "1.31":
+		delete(etcdStorageData, gvr("resource.k8s.io", "v1beta1", "deviceclasses"))
+		delete(etcdStorageData, gvr("resource.k8s.io", "v1beta1", "resourceclaims"))
+		delete(etcdStorageData, gvr("resource.k8s.io", "v1beta1", "resourceclaimtemplates"))
+		delete(etcdStorageData, gvr("resource.k8s.io", "v1beta1", "resourceslices"))
+	}
+
+	if removeAlphas {
+		for key := range etcdStorageData {
+			if strings.Contains(key.Version, "alpha") {
+				delete(etcdStorageData, key)
+			}
+		}
+	}
 	return etcdStorageData
 }
 

test/integration/etcd/etcd_storage_path_test.go

@@ -39,9 +39,12 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/serializer/recognizer"
 	utiljson "k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/dynamic"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	componentbaseversion "k8s.io/component-base/version"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 )
 
 // Only add kinds to this list when this a virtual resource with get and create verbs that doesn't actually
@@ -75,13 +78,32 @@ var allowMissingTestdataFixtures = map[schema.GroupVersionKind]bool{
 // It will also fail when a type gets moved to a different location. Be very careful in this situation because
 // it essentially means that you will be break old clusters unless you create some migration path for the old data.
 func TestEtcdStoragePath(t *testing.T) {
-	// KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE allows for APIs pending removal to not block tests
+	supportedVersions := GetSupportedEmulatedVersions()
-	t.Setenv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE", "true")
+	for _, v := range supportedVersions {
+		t.Run(v, func(t *testing.T) {
+			testEtcdStoragePathWithVersion(t, v)
+		})
+	}
+}
 
+func testEtcdStoragePathWithVersion(t *testing.T, v string) {
+	if v == componentbaseversion.DefaultKubeBinaryVersion {
 		featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)
 		featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)
+	} else {
+		// Only test for beta and GA APIs with emulated version.
+		featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, feature.DefaultFeatureGate, version.MustParse(v))
+		featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)
+		registerEffectiveEmulationVersion(t)
+	}
+
+	apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) {
+		// Disable alphas when emulating previous versions.
+		if v != componentbaseversion.DefaultKubeBinaryVersion {
+			opts.Options.APIEnablement.RuntimeConfig["api/alpha"] = "false"
+		}
+	})
 
-	apiServer := StartRealAPIServerOrDie(t)
 	defer apiServer.Cleanup()
 	defer dumpEtcdKVOnFailure(t, apiServer.KV)
 
@@ -91,7 +113,14 @@ func TestEtcdStoragePath(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	etcdStorageData := GetEtcdStorageData()
+	var etcdStorageData map[schema.GroupVersionResource]StorageData
+	if v == componentbaseversion.DefaultKubeBinaryVersion {
+		etcdStorageData = GetEtcdStorageDataForNamespaceServedAt("etcdstoragepathtestnamespace", v, false)
+	} else {
+		// Drop alphas from etcd data fixtures when emulating previous versions
+		// as alphas are not supported with emulation.
+		etcdStorageData = GetEtcdStorageDataForNamespaceServedAt("etcdstoragepathtestnamespace", v, true)
+	}
 
 	kindSeen := sets.NewString()
 	pathSeen := map[string][]schema.GroupVersionResource{}

test/integration/etcd/server.go

@@ -36,14 +36,19 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/json"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	genericapiserveroptions "k8s.io/apiserver/pkg/server/options"
+	"k8s.io/apiserver/pkg/util/feature"
 	cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
 	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/restmapper"
 	utiltesting "k8s.io/client-go/util/testing"
+	"k8s.io/component-base/featuregate"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	utilversion "k8s.io/component-base/version"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	"k8s.io/kubernetes/test/integration"
@@ -62,11 +67,23 @@ AwEHoUQDQgAEH6cuzP8XuD5wal6wf9M6xDljTOPLX2i8uIp/C/ASqiIGUeeKQtX0
 /IR3qCXyThP/dbCiHrF3v1cuhBOHY8CLVg==
 -----END EC PRIVATE KEY-----`
 
+func registerEffectiveEmulationVersion(t *testing.T) {
+	featureGate := feature.DefaultMutableFeatureGate
+	featureGate.AddMetrics()
+
+	effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
+	effectiveVersion.SetEmulationVersion(featureGate.EmulationVersion())
+	featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, featureGate, effectiveVersion.EmulationVersion())
+	featuregate.DefaultComponentGlobalsRegistry.Reset()
+	utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, effectiveVersion, featureGate))
+}
+
 // StartRealAPIServerOrDie starts an API server that is appropriate for use in tests that require one of every resource
 func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRunOptions)) *APIServer {
 	tCtx := ktesting.Init(t)
 
-	certDir, err := os.MkdirTemp("", t.Name())
+	// Strip out "/" in subtests
+	certDir, err := os.MkdirTemp("", strings.ReplaceAll(t.Name(), "/", ""))
 	if err != nil {
 		t.Fatal(err)
 	}