diff --git a/test/integration/controlplane/transformation/kms_transformation_test.go b/test/integration/controlplane/transformation/kms_transformation_test.go index 01858351ea3..9e70ef3c321 100644 --- a/test/integration/controlplane/transformation/kms_transformation_test.go +++ b/test/integration/controlplane/transformation/kms_transformation_test.go @@ -618,9 +618,6 @@ resources: endpoint: unix:///@encrypt-all-kms-provider.sock ` - // KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE allows for APIs pending removal to not block tests - t.Setenv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE", "true") - t.Run("encrypt all resources", func(t *testing.T) { _ = mock.NewBase64Plugin(t, "@encrypt-all-kms-provider.sock") // To ensure we are checking all REST resources diff --git a/test/integration/etcd/data.go b/test/integration/etcd/data.go index d2ef43b7c9c..2d8618f01b3 100644 --- a/test/integration/etcd/data.go +++ b/test/integration/etcd/data.go @@ -17,26 +17,54 @@ limitations under the License. package etcd import ( + "strings" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apiextensions-apiserver/test/integration/fixtures" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + utilversion "k8s.io/component-base/version" "k8s.io/kubernetes/test/utils/image" ) -// GetEtcdStorageData returns etcd data for all persisted objects. +// GetSupportedEmulatedVersions provides the list of supported emulated versions in the etcd data. +// Tests aiming for full coverage of versions should test fixtures of all supported versions. +func GetSupportedEmulatedVersions() []string { + return []string{ + utilversion.DefaultKubeEffectiveVersion().BinaryVersion().SubtractMinor(2).String(), + utilversion.DefaultKubeEffectiveVersion().BinaryVersion().SubtractMinor(1).String(), + utilversion.DefaultKubeEffectiveVersion().BinaryVersion().String(), + } +} + +// GetEtcdStorageData returns etcd data for all persisted objects at the latest release version. // It is exported so that it can be reused across multiple tests. // It returns a new map on every invocation to prevent different tests from mutating shared state. func GetEtcdStorageData() map[schema.GroupVersionResource]StorageData { - return GetEtcdStorageDataForNamespace("etcdstoragepathtestnamespace") + return GetEtcdStorageDataServedAt(utilversion.DefaultKubeBinaryVersion, false) } -// GetEtcdStorageDataForNamespace returns etcd data for all persisted objects. +// GetEtcdStorageDataServedAt returns etcd data for all persisted objects at a particular release version. +// It is exported so that it can be reused across multiple tests. +// It returns a new map on every invocation to prevent different tests from mutating shared state. +func GetEtcdStorageDataServedAt(version string, removeAlphas bool) map[schema.GroupVersionResource]StorageData { + return GetEtcdStorageDataForNamespaceServedAt("etcdstoragepathtestnamespace", version, removeAlphas) +} + +// GetEtcdStorageDataForNamespace returns etcd data for all persisted objects at the latest release version. // It is exported so that it can be reused across multiple tests. // It returns a new map on every invocation to prevent different tests from mutating shared state. // Namespaced objects keys are computed for the specified namespace. func GetEtcdStorageDataForNamespace(namespace string) map[schema.GroupVersionResource]StorageData { + return GetEtcdStorageDataForNamespaceServedAt(namespace, utilversion.DefaultKubeBinaryVersion, false) +} + +// GetEtcdStorageDataForNamespaceServedAt returns etcd data for all persisted objects at a particular release version. +// It is exported so that it can be reused across multiple tests. +// It returns a new map on every invocation to prevent different tests from mutating shared state. +// Namespaced objects keys are computed for the specified namespace. +func GetEtcdStorageDataForNamespaceServedAt(namespace string, version string, removeAlphas bool) map[schema.GroupVersionResource]StorageData { image := image.GetE2EImage(image.BusyBox) etcdStorageData := map[schema.GroupVersionResource]StorageData{ // k8s.io/kubernetes/pkg/api/v1 @@ -465,22 +493,46 @@ func GetEtcdStorageDataForNamespace(namespace string) map[schema.GroupVersionRes }, // -- + // k8s.io/kubernetes/pkg/apis/storage/v1 + gvr("storage.k8s.io", "v1", "csinodes"): { + Stub: `{"metadata": {"name": "csini2"}, "spec": {"drivers": [{"name": "test-driver", "nodeID": "localhost", "topologyKeys": ["company.com/zone1", "company.com/zone2"]}]}}`, + ExpectedEtcdPath: "/registry/csinodes/csini2", + }, + // -- + + // k8s.io/kubernetes/pkg/apis/storage/v1 + gvr("storage.k8s.io", "v1", "csidrivers"): { + Stub: `{"metadata": {"name": "csid2"}, "spec": {"attachRequired": true, "podInfoOnMount": true}}`, + ExpectedEtcdPath: "/registry/csidrivers/csid2", + }, + // -- } - // add csinodes - // k8s.io/kubernetes/pkg/apis/storage/v1 - etcdStorageData[gvr("storage.k8s.io", "v1", "csinodes")] = StorageData{ - Stub: `{"metadata": {"name": "csini2"}, "spec": {"drivers": [{"name": "test-driver", "nodeID": "localhost", "topologyKeys": ["company.com/zone1", "company.com/zone2"]}]}}`, - ExpectedEtcdPath: "/registry/csinodes/csini2", + // Delete types no longer served or not yet added at a particular emulated version. + // When adding a brand new type non-alpha type in the latest release, please ensure that + // it is removed in previous emulated versions otherwise emulated version tests will fail. + // TODO: derive this programatically from gvk --> instance --> APILifecycle info + switch version { + case "1.33": + delete(etcdStorageData, gvr("flowcontrol.apiserver.k8s.io", "v1beta3", "flowschemas")) + delete(etcdStorageData, gvr("flowcontrol.apiserver.k8s.io", "v1beta3", "prioritylevelconfigurations")) + case "1.32": + delete(etcdStorageData, gvr("flowcontrol.apiserver.k8s.io", "v1beta3", "flowschemas")) + delete(etcdStorageData, gvr("flowcontrol.apiserver.k8s.io", "v1beta3", "prioritylevelconfigurations")) + case "1.31": + delete(etcdStorageData, gvr("resource.k8s.io", "v1beta1", "deviceclasses")) + delete(etcdStorageData, gvr("resource.k8s.io", "v1beta1", "resourceclaims")) + delete(etcdStorageData, gvr("resource.k8s.io", "v1beta1", "resourceclaimtemplates")) + delete(etcdStorageData, gvr("resource.k8s.io", "v1beta1", "resourceslices")) } - // add csidrivers - // k8s.io/kubernetes/pkg/apis/storage/v1 - etcdStorageData[gvr("storage.k8s.io", "v1", "csidrivers")] = StorageData{ - Stub: `{"metadata": {"name": "csid2"}, "spec": {"attachRequired": true, "podInfoOnMount": true}}`, - ExpectedEtcdPath: "/registry/csidrivers/csid2", + if removeAlphas { + for key := range etcdStorageData { + if strings.Contains(key.Version, "alpha") { + delete(etcdStorageData, key) + } + } } - return etcdStorageData } diff --git a/test/integration/etcd/etcd_storage_path_test.go b/test/integration/etcd/etcd_storage_path_test.go index bedc000856b..e4e8ed4473a 100644 --- a/test/integration/etcd/etcd_storage_path_test.go +++ b/test/integration/etcd/etcd_storage_path_test.go @@ -39,9 +39,12 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer/recognizer" utiljson "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/dynamic" featuregatetesting "k8s.io/component-base/featuregate/testing" + componentbaseversion "k8s.io/component-base/version" + "k8s.io/kubernetes/cmd/kube-apiserver/app/options" ) // Only add kinds to this list when this a virtual resource with get and create verbs that doesn't actually @@ -75,13 +78,32 @@ var allowMissingTestdataFixtures = map[schema.GroupVersionKind]bool{ // It will also fail when a type gets moved to a different location. Be very careful in this situation because // it essentially means that you will be break old clusters unless you create some migration path for the old data. func TestEtcdStoragePath(t *testing.T) { - // KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE allows for APIs pending removal to not block tests - t.Setenv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE", "true") + supportedVersions := GetSupportedEmulatedVersions() + for _, v := range supportedVersions { + t.Run(v, func(t *testing.T) { + testEtcdStoragePathWithVersion(t, v) + }) + } +} - featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true) - featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true) +func testEtcdStoragePathWithVersion(t *testing.T, v string) { + if v == componentbaseversion.DefaultKubeBinaryVersion { + featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true) + featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true) + } else { + // Only test for beta and GA APIs with emulated version. + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, feature.DefaultFeatureGate, version.MustParse(v)) + featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true) + registerEffectiveEmulationVersion(t) + } + + apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { + // Disable alphas when emulating previous versions. + if v != componentbaseversion.DefaultKubeBinaryVersion { + opts.Options.APIEnablement.RuntimeConfig["api/alpha"] = "false" + } + }) - apiServer := StartRealAPIServerOrDie(t) defer apiServer.Cleanup() defer dumpEtcdKVOnFailure(t, apiServer.KV) @@ -91,7 +113,14 @@ func TestEtcdStoragePath(t *testing.T) { t.Fatal(err) } - etcdStorageData := GetEtcdStorageData() + var etcdStorageData map[schema.GroupVersionResource]StorageData + if v == componentbaseversion.DefaultKubeBinaryVersion { + etcdStorageData = GetEtcdStorageDataForNamespaceServedAt("etcdstoragepathtestnamespace", v, false) + } else { + // Drop alphas from etcd data fixtures when emulating previous versions + // as alphas are not supported with emulation. + etcdStorageData = GetEtcdStorageDataForNamespaceServedAt("etcdstoragepathtestnamespace", v, true) + } kindSeen := sets.NewString() pathSeen := map[string][]schema.GroupVersionResource{} diff --git a/test/integration/etcd/server.go b/test/integration/etcd/server.go index 03911859aaf..cb8d71a335b 100644 --- a/test/integration/etcd/server.go +++ b/test/integration/etcd/server.go @@ -36,14 +36,19 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/json" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" genericapiserveroptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/apiserver/pkg/util/feature" cacheddiscovery "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/restmapper" utiltesting "k8s.io/client-go/util/testing" + "k8s.io/component-base/featuregate" + featuregatetesting "k8s.io/component-base/featuregate/testing" + utilversion "k8s.io/component-base/version" "k8s.io/kubernetes/cmd/kube-apiserver/app" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/test/integration" @@ -62,11 +67,23 @@ AwEHoUQDQgAEH6cuzP8XuD5wal6wf9M6xDljTOPLX2i8uIp/C/ASqiIGUeeKQtX0 /IR3qCXyThP/dbCiHrF3v1cuhBOHY8CLVg== -----END EC PRIVATE KEY-----` +func registerEffectiveEmulationVersion(t *testing.T) { + featureGate := feature.DefaultMutableFeatureGate + featureGate.AddMetrics() + + effectiveVersion := utilversion.DefaultKubeEffectiveVersion() + effectiveVersion.SetEmulationVersion(featureGate.EmulationVersion()) + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, featureGate, effectiveVersion.EmulationVersion()) + featuregate.DefaultComponentGlobalsRegistry.Reset() + utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, effectiveVersion, featureGate)) +} + // StartRealAPIServerOrDie starts an API server that is appropriate for use in tests that require one of every resource func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRunOptions)) *APIServer { tCtx := ktesting.Init(t) - certDir, err := os.MkdirTemp("", t.Name()) + // Strip out "/" in subtests + certDir, err := os.MkdirTemp("", strings.ReplaceAll(t.Name(), "/", "")) if err != nil { t.Fatal(err) }