From 56bd8d86a58d6060741ca8a8bd9873e77087d5d9 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 20 Feb 2025 21:21:53 +0100 Subject: [PATCH] test/integration: use default API groups in test apiserver The goal is to make the test apiserver behave as much as kube-apiserver as possible. This ensures that tests are as realistic as possible out-of-the-box. If a test needs a special setup, then that should be visible in the test because it passes additional flags or options. One historic deviation from that goal was enabling all API groups. That change (from 71856246886a7100703ada714e723088bb5e2979) gets reverted and tests which happened to rely on this get updated. --- cmd/kube-apiserver/app/testing/testserver.go | 1 - test/integration/apiserver/apiserver_test.go | 11 ++++--- .../cel/authorizerselector/helper.go | 2 +- .../cel/mutatingadmissionpolicy_test.go | 11 ++++--- .../coordinated_leader_election_test.go | 15 ++++++---- .../apiserver/peerproxy/peer_proxy_test.go | 11 ++++--- .../client/metrics/metrics_test.go | 4 ++- .../admission_establishtrust_test.go | 3 +- .../apiserversigner_test.go | 1 + .../field_selector_test.go | 3 +- .../signer_name_change_forbidden_test.go | 2 +- .../transformation/all_transformation_test.go | 2 +- .../transformation/kms_transformation_test.go | 16 +++++----- .../kmsv2_transformation_test.go | 22 +++++++------- .../secrets_transformation_test.go | 8 ++--- .../transformation/transformation_test.go | 30 +++++++++++++------ test/integration/metrics/metrics_test.go | 5 +++- .../feature_enable_disable_test.go | 2 ++ test/integration/scheduler_perf/util.go | 9 +++--- test/integration/storageversion/gc_test.go | 4 ++- .../storage_version_filter_test.go | 4 ++- .../storageversionmigrator/util.go | 1 + 22 files changed, 102 insertions(+), 65 deletions(-) diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index de615a9b1ca..221afcd4221 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -339,7 +339,6 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, s.ServiceClusterIPRanges = "10.0.0.0/16" s.Etcd.StorageConfig = *storageConfig - s.APIEnablement.RuntimeConfig.Set("api/all=true") if err := fs.Parse(customFlags); err != nil { return result, err diff --git a/test/integration/apiserver/apiserver_test.go b/test/integration/apiserver/apiserver_test.go index 3eeb0bc0e45..54d0ddb7248 100644 --- a/test/integration/apiserver/apiserver_test.go +++ b/test/integration/apiserver/apiserver_test.go @@ -34,6 +34,7 @@ import ( "time" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -1683,12 +1684,14 @@ func TestGetScaleSubresourceAsTableForAllBuiltins(t *testing.T) { // KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE allows for APIs pending removal to not block tests t.Setenv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE", "true") - // Enable all features for testing + // Enable all features and apis for testing + flags := framework.DefaultTestServerFlags() + flags = append(flags, "--runtime-config=api/all=true") featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, "AllAlpha", true) featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, "AllBeta", true) testNamespace := "test-scale" - server := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd()) + server := kubeapiservertesting.StartTestServerOrDie(t, nil, flags, framework.SharedEtcd()) defer server.TearDownFn() clientset := clientset.NewForConfigOrDie(server.ClientConfig) @@ -3199,7 +3202,7 @@ func TestEmulatedStorageVersion(t *testing.T) { for emulatedVersion, cases := range groupedCases { t.Run(emulatedVersion, func(t *testing.T) { server := kubeapiservertesting.StartTestServerOrDie( - t, nil, []string{"--emulated-version=kube=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd()) + t, nil, []string{"--emulated-version=kube=" + emulatedVersion, `--storage-media-type=application/json`, fmt.Sprintf("--runtime-config=%s=true", admissionregistrationv1beta1.SchemeGroupVersion)}, framework.SharedEtcd()) defer server.TearDownFn() client := clientset.NewForConfigOrDie(server.ClientConfig) @@ -3340,7 +3343,7 @@ func TestEnableEmulationVersion(t *testing.T) { featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.32")) server := kubeapiservertesting.StartTestServerOrDie(t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"}, - []string{"--emulated-version=kube=1.31"}, framework.SharedEtcd()) + []string{"--emulated-version=kube=1.31", "--runtime-config=api/beta=true"}, framework.SharedEtcd()) defer server.TearDownFn() rt, err := restclient.TransportFor(server.ClientConfig) diff --git a/test/integration/apiserver/cel/authorizerselector/helper.go b/test/integration/apiserver/cel/authorizerselector/helper.go index 506583c4d6c..58761685cd4 100644 --- a/test/integration/apiserver/cel/authorizerselector/helper.go +++ b/test/integration/apiserver/cel/authorizerselector/helper.go @@ -49,7 +49,7 @@ func RunAuthzSelectorsLibraryTests(t *testing.T, featureEnabled bool) { // Start the server with the desired feature enablement server, err := apiservertesting.StartTestServer(t, nil, []string{ fmt.Sprintf("--feature-gates=AuthorizeNodeWithSelectors=%v,AuthorizeWithSelectors=%v", featureEnabled, featureEnabled), - "--runtime-config=resource.k8s.io/v1alpha3=true", + fmt.Sprintf("--runtime-config=%s=true", resourceapi.SchemeGroupVersion), }, framework.SharedEtcd()) if err != nil { t.Fatal(err) diff --git a/test/integration/apiserver/cel/mutatingadmissionpolicy_test.go b/test/integration/apiserver/cel/mutatingadmissionpolicy_test.go index 011a521a28f..e95fbdf401b 100644 --- a/test/integration/apiserver/cel/mutatingadmissionpolicy_test.go +++ b/test/integration/apiserver/cel/mutatingadmissionpolicy_test.go @@ -20,13 +20,13 @@ import ( "context" "errors" "fmt" - "github.com/google/go-cmp/cmp/cmpopts" "reflect" "strings" "testing" "time" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/require" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" @@ -493,7 +493,8 @@ func TestMutatingAdmissionPolicy(t *testing.T) { // Run all tests in a shared apiserver featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.MutatingAdmissionPolicy, true) - server, err := apiservertesting.StartTestServer(t, nil, nil, framework.SharedEtcd()) + flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha1.SchemeGroupVersion)} + server, err := apiservertesting.StartTestServer(t, nil, flags, framework.SharedEtcd()) require.NoError(t, err) defer server.TearDownFn() @@ -1006,7 +1007,8 @@ func TestMutatingAdmissionPolicy_Slow(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.MutatingAdmissionPolicy, true) - server, err := apiservertesting.StartTestServer(t, nil, nil, framework.SharedEtcd()) + flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha1.SchemeGroupVersion)} + server, err := apiservertesting.StartTestServer(t, nil, flags, framework.SharedEtcd()) require.NoError(t, err) defer server.TearDownFn() @@ -1091,7 +1093,8 @@ func TestMutatingAdmissionPolicy_Slow(t *testing.T) { // tested. func Test_MutatingAdmissionPolicy_CustomResources(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.MutatingAdmissionPolicy, true) - server, err := apiservertesting.StartTestServer(t, nil, nil, framework.SharedEtcd()) + flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha1.SchemeGroupVersion)} + server, err := apiservertesting.StartTestServer(t, nil, flags, framework.SharedEtcd()) etcd.CreateTestCRDs(t, apiextensions.NewForConfigOrDie(server.ClientConfig), false, versionedCustomResourceDefinition()) if err != nil { t.Fatal(err) diff --git a/test/integration/apiserver/coordinated_leader_election_test.go b/test/integration/apiserver/coordinated_leader_election_test.go index 5bfe1946fc6..505b9ee2308 100644 --- a/test/integration/apiserver/coordinated_leader_election_test.go +++ b/test/integration/apiserver/coordinated_leader_election_test.go @@ -44,7 +44,8 @@ import ( func TestSingleLeaseCandidate(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CoordinatedLeaderElection, true) - server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd()) + flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)} + server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd()) if err != nil { t.Fatal(err) } @@ -62,7 +63,8 @@ func TestSingleLeaseCandidate(t *testing.T) { func TestMultipleLeaseCandidate(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CoordinatedLeaderElection, true) - server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd()) + flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)} + server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd()) if err != nil { t.Fatal(err) } @@ -84,7 +86,8 @@ func TestMultipleLeaseCandidate(t *testing.T) { func TestLeaseSwapIfBetterAvailable(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CoordinatedLeaderElection, true) - server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd()) + flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)} + server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd()) if err != nil { t.Fatal(err) } @@ -105,7 +108,8 @@ func TestLeaseSwapIfBetterAvailable(t *testing.T) { func TestUpgradeSkew(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CoordinatedLeaderElection, true) - server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd()) + flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)} + server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd()) if err != nil { t.Fatal(err) } @@ -134,7 +138,8 @@ func TestLeaseCandidateCleanup(t *testing.T) { apiserver.LeaseCandidateGCPeriod = 30 * time.Minute }() - server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd()) + flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)} + server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd()) if err != nil { t.Fatal(err) } diff --git a/test/integration/apiserver/peerproxy/peer_proxy_test.go b/test/integration/apiserver/peerproxy/peer_proxy_test.go index 3c9a2e13f99..705489a44bf 100644 --- a/test/integration/apiserver/peerproxy/peer_proxy_test.go +++ b/test/integration/apiserver/peerproxy/peer_proxy_test.go @@ -18,7 +18,6 @@ package peerproxy import ( "context" - "fmt" "testing" "time" @@ -77,7 +76,7 @@ func TestPeerProxiedRequest(t *testing.T) { serverA := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{ EnableCertAuth: true, ProxyCA: &proxyCA}, - []string{}, etcd) + []string{"--runtime-config=api/all=true"}, etcd) t.Cleanup(serverA.TearDownFn) // start another test server with some api disabled @@ -86,7 +85,7 @@ func TestPeerProxiedRequest(t *testing.T) { serverB := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{ EnableCertAuth: true, ProxyCA: &proxyCA}, - []string{fmt.Sprintf("--runtime-config=%s", "batch/v1=false")}, etcd) + []string{"--runtime-config=api/all=true,batch/v1=false"}, etcd) t.Cleanup(serverB.TearDownFn) kubeClientSetA, err := kubernetes.NewForConfig(serverA.ClientConfig) @@ -144,7 +143,7 @@ func TestPeerProxiedRequestToThirdServerAfterFirstDies(t *testing.T) { // override hostname to ensure unique ips server.SetHostnameFuncForTests("test-server-a") t.Log("starting apiserver for ServerA") - serverA := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, ProxyCA: &proxyCA}, []string{}, etcd) + serverA := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, ProxyCA: &proxyCA}, []string{"--runtime-config=api/all=true"}, etcd) kubeClientSetA, err := kubernetes.NewForConfig(serverA.ClientConfig) require.NoError(t, err) // ensure storageversion garbage collector ctlr is set up @@ -160,7 +159,7 @@ func TestPeerProxiedRequestToThirdServerAfterFirstDies(t *testing.T) { server.SetHostnameFuncForTests("test-server-b") t.Log("starting apiserver for ServerB") serverB := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, ProxyCA: &proxyCA}, []string{ - fmt.Sprintf("--runtime-config=%v", "batch/v1=false")}, etcd) + "--runtime-config=api/all=true,batch/v1=false"}, etcd) t.Cleanup(serverB.TearDownFn) kubeClientSetB, err := kubernetes.NewForConfig(serverB.ClientConfig) require.NoError(t, err) @@ -172,7 +171,7 @@ func TestPeerProxiedRequestToThirdServerAfterFirstDies(t *testing.T) { // override hostname to ensure unique ips server.SetHostnameFuncForTests("test-server-c") t.Log("starting apiserver for ServerC") - serverC := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, ProxyCA: &proxyCA}, []string{}, etcd) + serverC := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, ProxyCA: &proxyCA}, []string{"--runtime-config=api/all=true"}, etcd) t.Cleanup(serverC.TearDownFn) // create jobs resource using serverA diff --git a/test/integration/client/metrics/metrics_test.go b/test/integration/client/metrics/metrics_test.go index af88050deba..e8f7fec02f2 100644 --- a/test/integration/client/metrics/metrics_test.go +++ b/test/integration/client/metrics/metrics_test.go @@ -50,7 +50,9 @@ func TestAPIServerTransportMetrics(t *testing.T) { // reset default registry metrics legacyregistry.Reset() - result := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd()) + flags := framework.DefaultTestServerFlags() + flags = append(flags, "--runtime-config=api/all=true,api/beta=true") + result := kubeapiservertesting.StartTestServerOrDie(t, nil, flags, framework.SharedEtcd()) defer result.TearDownFn() client := clientset.NewForConfigOrDie(result.ClientConfig) diff --git a/test/integration/clustertrustbundles/admission_establishtrust_test.go b/test/integration/clustertrustbundles/admission_establishtrust_test.go index 95eaa37ac8f..da720b815cc 100644 --- a/test/integration/clustertrustbundles/admission_establishtrust_test.go +++ b/test/integration/clustertrustbundles/admission_establishtrust_test.go @@ -20,6 +20,7 @@ import ( "context" "crypto/x509" "crypto/x509/pkix" + "fmt" "math/big" "testing" @@ -77,7 +78,7 @@ func TestCTBAttestPlugin(t *testing.T) { t.Run(tc.description, func(t *testing.T) { ctx := context.Background() - server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--authorization-mode=RBAC", "--feature-gates=ClusterTrustBundle=true"}, framework.SharedEtcd()) + server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--authorization-mode=RBAC", "--feature-gates=ClusterTrustBundle=true", fmt.Sprintf("--runtime-config=%s=true", certsv1alpha1.SchemeGroupVersion)}, framework.SharedEtcd()) defer server.TearDownFn() client := kubernetes.NewForConfigOrDie(server.ClientConfig) diff --git a/test/integration/clustertrustbundles/apiserversigner_test.go b/test/integration/clustertrustbundles/apiserversigner_test.go index d651089f4aa..3d1a7918d59 100644 --- a/test/integration/clustertrustbundles/apiserversigner_test.go +++ b/test/integration/clustertrustbundles/apiserversigner_test.go @@ -73,6 +73,7 @@ func TestClusterTrustBundlesPublisherController(t *testing.T) { "--disable-admission-plugins", "ServiceAccount", "--authorization-mode=RBAC", "--feature-gates", "ClusterTrustBundle=true", + fmt.Sprintf("--runtime-config=%s=true", v1alpha1.SchemeGroupVersion), } storageConfig := framework.SharedEtcd() server := kubeapiservertesting.StartTestServerOrDie(t, nil, apiServerFlags, storageConfig) diff --git a/test/integration/clustertrustbundles/field_selector_test.go b/test/integration/clustertrustbundles/field_selector_test.go index 0b0d6d6b162..a1257c83674 100644 --- a/test/integration/clustertrustbundles/field_selector_test.go +++ b/test/integration/clustertrustbundles/field_selector_test.go @@ -20,6 +20,7 @@ import ( "context" "crypto/x509" "crypto/x509/pkix" + "fmt" "math/big" "testing" @@ -37,7 +38,7 @@ func TestCTBSignerNameFieldSelector(t *testing.T) { ctx := context.Background() - server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--feature-gates=ClusterTrustBundle=true"}, framework.SharedEtcd()) + server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--feature-gates=ClusterTrustBundle=true", fmt.Sprintf("--runtime-config=%s=true", certsv1alpha1.SchemeGroupVersion)}, framework.SharedEtcd()) defer server.TearDownFn() client := kubernetes.NewForConfigOrDie(server.ClientConfig) diff --git a/test/integration/clustertrustbundles/signer_name_change_forbidden_test.go b/test/integration/clustertrustbundles/signer_name_change_forbidden_test.go index 3e218cc23cc..8afa1cdbc1b 100644 --- a/test/integration/clustertrustbundles/signer_name_change_forbidden_test.go +++ b/test/integration/clustertrustbundles/signer_name_change_forbidden_test.go @@ -63,7 +63,7 @@ func TestCTBSignerNameChangeForbidden(t *testing.T) { ctx := context.Background() - server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--feature-gates=ClusterTrustBundle=true"}, framework.SharedEtcd()) + server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--feature-gates=ClusterTrustBundle=true", fmt.Sprintf("--runtime-config=%s=true", certsv1alpha1.SchemeGroupVersion)}, framework.SharedEtcd()) defer server.TearDownFn() client := kubernetes.NewForConfigOrDie(server.ClientConfig) diff --git a/test/integration/controlplane/transformation/all_transformation_test.go b/test/integration/controlplane/transformation/all_transformation_test.go index 4ca250f234a..1496dcf9521 100644 --- a/test/integration/controlplane/transformation/all_transformation_test.go +++ b/test/integration/controlplane/transformation/all_transformation_test.go @@ -94,7 +94,7 @@ resources: - name: key1 secret: c2VjcmV0IGlzIHNlY3VyZQ== ` - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start Kube API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } diff --git a/test/integration/controlplane/transformation/kms_transformation_test.go b/test/integration/controlplane/transformation/kms_transformation_test.go index 9e70ef3c321..22d54abd081 100644 --- a/test/integration/controlplane/transformation/kms_transformation_test.go +++ b/test/integration/controlplane/transformation/kms_transformation_test.go @@ -145,7 +145,7 @@ resources: ` providerName := "kms-provider" pluginMock := mock.NewBase64Plugin(t, "@kms-provider.sock") - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -329,7 +329,7 @@ resources: genericapiserver.SetHostnameFuncForTests("testAPIServerID") _ = mock.NewBase64Plugin(t, "@kms-provider.sock") var restarted bool - test, err := newTransformTest(t, encryptionConfig, true, "", storageConfig) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, reload: true, storageConfig: storageConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -550,7 +550,7 @@ resources: previousConfigDir := test.configDir test.shutdownAPIServer() restarted = true - test, err = newTransformTest(t, test.transformerConfig, true, previousConfigDir, storageConfig) + test, err = newTransformTest(t, transformTestConfig{transformerConfigYAML: test.transformerConfig, reload: true, configDir: previousConfigDir, storageConfig: storageConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -626,7 +626,7 @@ resources: // Need to enable this explicitly as the feature is deprecated featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true) - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, runtimeConfig: []string{"api/alpha=true", "api/beta=true"}}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig") } @@ -752,7 +752,7 @@ resources: featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true) - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -899,7 +899,7 @@ resources: ` _ = mock.NewBase64Plugin(t, "@kms-provider.sock") - test, err := newTransformTest(t, encryptionConfig, true, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, reload: true}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -1111,7 +1111,7 @@ resources: pluginMock1 := mock.NewBase64Plugin(t, "@kms-provider-1.sock") pluginMock2 := mock.NewBase64Plugin(t, "@kms-provider-2.sock") - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start kube-apiserver, error: %v", err) } @@ -1174,7 +1174,7 @@ resources: pluginMock1 := mock.NewBase64Plugin(t, "@kms-provider-1.sock") pluginMock2 := mock.NewBase64Plugin(t, "@kms-provider-2.sock") - test, err := newTransformTest(t, encryptionConfig, true, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, reload: true}) if err != nil { t.Fatalf("Failed to start kube-apiserver, error: %v", err) } diff --git a/test/integration/controlplane/transformation/kmsv2_transformation_test.go b/test/integration/controlplane/transformation/kmsv2_transformation_test.go index 31d684a6640..6e6899e1c25 100644 --- a/test/integration/controlplane/transformation/kmsv2_transformation_test.go +++ b/test/integration/controlplane/transformation/kmsv2_transformation_test.go @@ -193,7 +193,7 @@ resources: ` _ = kmsv2mock.NewBase64Plugin(t, "@kms-provider-defaults.sock") - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -279,7 +279,7 @@ resources: genericapiserver.SetHostnameFuncForTests("testAPIServerID") pluginMock := kmsv2mock.NewBase64Plugin(t, "@"+kmsName+".sock") - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -432,7 +432,7 @@ resources: ` pluginMock := kmsv2mock.NewBase64Plugin(t, "@"+kmsName+".sock") - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -730,7 +730,7 @@ resources: ` _ = kmsv2mock.NewBase64Plugin(t, "@"+kmsName+".sock") - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -871,7 +871,7 @@ resources: pluginMock1 := kmsv2mock.NewBase64Plugin(t, "@kms-provider-1.sock") pluginMock2 := kmsv2mock.NewBase64Plugin(t, "@kms-provider-2.sock") - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("Failed to start kube-apiserver, error: %v", err) } @@ -949,7 +949,7 @@ resources: _ = kmsv2mock.NewBase64Plugin(t, "@kms-provider-single-service.sock") - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -1006,7 +1006,7 @@ resources: storageConfig := framework.SharedEtcd() // KMSv2 is enabled by default. Loading a encryptionConfig with KMSv2 should work - test, err := newTransformTest(t, encryptionConfig, false, "", storageConfig) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, storageConfig: storageConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -1078,7 +1078,7 @@ resources: // After a restart, loading a encryptionConfig with the same KMSv2 plugin before the restart should work, decryption of data encrypted with v2 should work - test, err = newTransformTest(t, encryptionConfig, false, "", storageConfig) + test, err = newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, storageConfig: storageConfig}) if err != nil { t.Fatalf("Failed to restart api server, error: %v", err) } @@ -1126,7 +1126,7 @@ resources: ` _ = kmsv2mock.NewBase64Plugin(b, "@kms-provider-bench.sock") - test, err := newTransformTest(b, encryptionConfig, false, "", nil) + test, err := newTransformTest(b, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { b.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -1279,7 +1279,7 @@ resources: ` _ = kmsv2mock.NewBase64Plugin(b, "@kms-provider-bench-rest.sock") - test, err := newTransformTest(b, encryptionConfig, false, "", nil) + test, err := newTransformTest(b, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { b.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -1378,7 +1378,7 @@ resources: storageConfig := storagebackend.NewDefaultConfig(path.Join(legacyDataEtcdPrefix, "registry"), nil) storageConfig.Transport.ServerList = []string{framework.GetEtcdURL()} - test, err := newTransformTest(t, encryptionConfig, false, "", storageConfig) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, storageConfig: storageConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } diff --git a/test/integration/controlplane/transformation/secrets_transformation_test.go b/test/integration/controlplane/transformation/secrets_transformation_test.go index 8767b2f5acb..432cb74287a 100644 --- a/test/integration/controlplane/transformation/secrets_transformation_test.go +++ b/test/integration/controlplane/transformation/secrets_transformation_test.go @@ -106,7 +106,7 @@ func TestSecretsShouldBeTransformed(t *testing.T) { // TODO: add secretbox } for _, tt := range testCases { - test, err := newTransformTest(t, tt.transformerConfigContent, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: tt.transformerConfigContent}) if err != nil { t.Fatalf("failed to setup test for envelop %s, error was %v", tt.transformerPrefix, err) continue @@ -195,7 +195,7 @@ func TestAllowUnsafeMalformedObjectDeletionFeature(t *testing.T) { t.Run(fmt.Sprintf("%s/%t", string(genericfeatures.AllowUnsafeMalformedObjectDeletion), tc.featureEnabled), func(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AllowUnsafeMalformedObjectDeletion, tc.featureEnabled) - test, err := newTransformTest(t, aesGCMConfigYAML, true, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: aesGCMConfigYAML, reload: true}) if err != nil { t.Fatalf("failed to setup test for envelop %s, error was %v", aesGCMPrefix, err) } @@ -498,7 +498,7 @@ func TestListCorruptObjects(t *testing.T) { t.Run(fmt.Sprintf("%s/%t", string(genericfeatures.AllowUnsafeMalformedObjectDeletion), tc.featureEnabled), func(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AllowUnsafeMalformedObjectDeletion, tc.featureEnabled) - test, err := newTransformTest(t, aesGCMConfigYAML, true, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: aesGCMConfigYAML, reload: true}) if err != nil { t.Fatalf("failed to setup test for envelop %s, error was %v", aesGCMPrefix, err) } @@ -651,7 +651,7 @@ func BenchmarkAESCBCEnvelopeWrite(b *testing.B) { func runBenchmark(b *testing.B, transformerConfig string) { b.StopTimer() - test, err := newTransformTest(b, transformerConfig, false, "", nil) + test, err := newTransformTest(b, transformTestConfig{transformerConfigYAML: transformerConfig}) if err != nil { b.Fatalf("failed to setup benchmark for config %s, error was %v", transformerConfig, err) } diff --git a/test/integration/controlplane/transformation/transformation_test.go b/test/integration/controlplane/transformation/transformation_test.go index c0c95ce230a..96446920fe0 100644 --- a/test/integration/controlplane/transformation/transformation_test.go +++ b/test/integration/controlplane/transformation/transformation_test.go @@ -87,27 +87,35 @@ type transformTest struct { secret *corev1.Secret } -func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool, configDir string, storageConfig *storagebackend.Config) (*transformTest, error) { +type transformTestConfig struct { + transformerConfigYAML string + reload bool + configDir string + storageConfig *storagebackend.Config + runtimeConfig []string +} + +func newTransformTest(tb testing.TB, config transformTestConfig) (*transformTest, error) { tCtx := ktesting.Init(tb) - if storageConfig == nil { - storageConfig = framework.SharedEtcd() + if config.storageConfig == nil { + config.storageConfig = framework.SharedEtcd() } e := transformTest{ TContext: tCtx, - transformerConfig: transformerConfigYAML, - storageConfig: storageConfig, + transformerConfig: config.transformerConfigYAML, + storageConfig: config.storageConfig, } var err error // create config dir with provided config yaml - if transformerConfigYAML != "" && configDir == "" { + if config.transformerConfigYAML != "" && config.configDir == "" { if e.configDir, err = e.createEncryptionConfig(); err != nil { e.cleanUp() return nil, fmt.Errorf("error while creating KubeAPIServer encryption config: %w", err) } } else { // configDir already exists. api-server must be restarting with existing encryption config - e.configDir = configDir + e.configDir = config.configDir } configFile := filepath.Join(e.configDir, encryptionConfigFileName) _, err = os.ReadFile(configFile) @@ -116,9 +124,13 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool, return nil, fmt.Errorf("failed to read config file: %w", err) } + flags := e.getEncryptionOptions(config.reload) + if len(config.runtimeConfig) > 0 { + flags = append(flags, "--runtime-config="+strings.Join(config.runtimeConfig, ",")) + } if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer( tb, nil, - e.getEncryptionOptions(reload), e.storageConfig); err != nil { + flags, e.storageConfig); err != nil { e.cleanUp() return nil, fmt.Errorf("failed to start KubeAPI server: %w", err) } @@ -134,7 +146,7 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool, return nil, err } - if transformerConfigYAML != "" && reload { + if config.transformerConfigYAML != "" && config.reload { // when reloading is enabled, this healthz endpoint is always present mustBeHealthy(tCtx, "/kms-providers", "ok", e.kubeAPIServer.ClientConfig) mustNotHaveLivez(tCtx, "/kms-providers", "404 page not found", e.kubeAPIServer.ClientConfig) diff --git a/test/integration/metrics/metrics_test.go b/test/integration/metrics/metrics_test.go index f38f1f33220..dd1328a40d4 100644 --- a/test/integration/metrics/metrics_test.go +++ b/test/integration/metrics/metrics_test.go @@ -26,6 +26,7 @@ import ( "testing" "github.com/prometheus/common/model" + admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/endpoints/metrics" @@ -108,7 +109,9 @@ func TestAPIServerMetrics(t *testing.T) { // KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE allows for APIs pending removal to not block tests t.Setenv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE", "true") - s := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd()) + flags := framework.DefaultTestServerFlags() + flags = append(flags, fmt.Sprintf("--runtime-config=%s=true", admissionregistrationv1beta1.SchemeGroupVersion)) + s := kubeapiservertesting.StartTestServerOrDie(t, nil, flags, framework.SharedEtcd()) defer s.TearDownFn() // Make a request to the apiserver to ensure there's at least one data point diff --git a/test/integration/resourceclaim/feature_enable_disable_test.go b/test/integration/resourceclaim/feature_enable_disable_test.go index 7921190e604..ca4a6d8fd16 100644 --- a/test/integration/resourceclaim/feature_enable_disable_test.go +++ b/test/integration/resourceclaim/feature_enable_disable_test.go @@ -42,6 +42,7 @@ func TestEnableDisableDRAResourceClaimDeviceStatus(t *testing.T) { // apiserver with the feature disabled server1 := kubeapiservertesting.StartTestServerOrDie(t, apiServerOptions, []string{ + fmt.Sprintf("--runtime-config=%s=true", v1beta1.SchemeGroupVersion), fmt.Sprintf("--feature-gates=%s=true,%s=false", features.DynamicResourceAllocation, features.DRAResourceClaimDeviceStatus), }, etcdOptions) @@ -114,6 +115,7 @@ func TestEnableDisableDRAResourceClaimDeviceStatus(t *testing.T) { // apiserver with the feature enabled server2 := kubeapiservertesting.StartTestServerOrDie(t, apiServerOptions, []string{ + fmt.Sprintf("--runtime-config=%s=true", v1beta1.SchemeGroupVersion), fmt.Sprintf("--feature-gates=%s=true,%s=true", features.DynamicResourceAllocation, features.DRAResourceClaimDeviceStatus), }, etcdOptions) diff --git a/test/integration/scheduler_perf/util.go b/test/integration/scheduler_perf/util.go index 279f309a4fc..b3e5801ac11 100644 --- a/test/integration/scheduler_perf/util.go +++ b/test/integration/scheduler_perf/util.go @@ -30,6 +30,8 @@ import ( "time" v1 "k8s.io/api/core/v1" + resourceapialpha "k8s.io/api/resource/v1alpha3" + resourceapi "k8s.io/api/resource/v1beta1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -86,11 +88,10 @@ func newDefaultComponentConfig() (*config.KubeSchedulerConfiguration, error) { // Notes on rate limiter: // - client rate limit is set to 5000. func mustSetupCluster(tCtx ktesting.TContext, config *config.KubeSchedulerConfiguration, enabledFeatures map[featuregate.Feature]bool, outOfTreePluginRegistry frameworkruntime.Registry) (informers.SharedInformerFactory, ktesting.TContext) { - // No alpha APIs (overrides api/all=true in https://github.com/kubernetes/kubernetes/blob/d647d19f6aef811bace300eec96a67644ff303d4/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go#L136), - // except for DRA API group when needed. - runtimeConfig := []string{"api/alpha=false"} + var runtimeConfig []string if enabledFeatures[features.DynamicResourceAllocation] { - runtimeConfig = append(runtimeConfig, "resource.k8s.io/v1alpha3=true") + runtimeConfig = append(runtimeConfig, fmt.Sprintf("%s=true", resourceapi.SchemeGroupVersion)) + runtimeConfig = append(runtimeConfig, fmt.Sprintf("%s=true", resourceapialpha.SchemeGroupVersion)) } customFlags := []string{ // Disable ServiceAccount admission plugin as we don't have serviceaccount controller running. diff --git a/test/integration/storageversion/gc_test.go b/test/integration/storageversion/gc_test.go index 13065ebb20b..af521415714 100644 --- a/test/integration/storageversion/gc_test.go +++ b/test/integration/storageversion/gc_test.go @@ -52,7 +52,9 @@ const ( func TestStorageVersionGarbageCollection(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true) featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionAPI, true) - result := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd()) + flags := framework.DefaultTestServerFlags() + flags = append(flags, fmt.Sprintf("--runtime-config=%s=true", apiserverinternalv1alpha1.SchemeGroupVersion)) + result := kubeapiservertesting.StartTestServerOrDie(t, nil, flags, framework.SharedEtcd()) defer result.TearDownFn() kubeclient, err := kubernetes.NewForConfig(result.ClientConfig) diff --git a/test/integration/storageversion/storage_version_filter_test.go b/test/integration/storageversion/storage_version_filter_test.go index b95fa5505cd..5e41929cf7b 100644 --- a/test/integration/storageversion/storage_version_filter_test.go +++ b/test/integration/storageversion/storage_version_filter_test.go @@ -148,7 +148,9 @@ func testBuiltinResourceRead(t *testing.T, cfg *rest.Config, shouldBlock bool) { func TestStorageVersionBootstrap(t *testing.T) { // Start server and create CRD etcdConfig := framework.SharedEtcd() - server := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), etcdConfig) + flags := framework.DefaultTestServerFlags() + flags = append(flags, "--runtime-config=api/all=true") + server := kubeapiservertesting.StartTestServerOrDie(t, nil, flags, etcdConfig) etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(server.ClientConfig), false, etcd.GetCustomResourceDefinitionData()[0]) server.TearDownFn() diff --git a/test/integration/storageversionmigrator/util.go b/test/integration/storageversionmigrator/util.go index 7b78e673209..111632a14e5 100644 --- a/test/integration/storageversionmigrator/util.go +++ b/test/integration/storageversionmigrator/util.go @@ -275,6 +275,7 @@ func svmSetup(ctx context.Context, t *testing.T) *svmTest { "--audit-log-mode", "blocking", "--audit-log-path", logFile.Name(), "--authorization-mode=RBAC", + fmt.Sprintf("--runtime-config=%s=true", svmv1alpha1.SchemeGroupVersion), } storageConfig := framework.SharedEtcd() server := kubeapiservertesting.StartTestServerOrDie(t, nil, apiServerFlags, storageConfig)