diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index de615a9b1ca..221afcd4221 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -339,7 +339,6 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, s.ServiceClusterIPRanges = "10.0.0.0/16" s.Etcd.StorageConfig = *storageConfig - s.APIEnablement.RuntimeConfig.Set("api/all=true") if err := fs.Parse(customFlags); err != nil { return result, err diff --git a/test/integration/apiserver/apiserver_test.go b/test/integration/apiserver/apiserver_test.go index 3eeb0bc0e45..54d0ddb7248 100644 --- a/test/integration/apiserver/apiserver_test.go +++ b/test/integration/apiserver/apiserver_test.go @@ -34,6 +34,7 @@ import ( "time" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -1683,12 +1684,14 @@ func TestGetScaleSubresourceAsTableForAllBuiltins(t *testing.T) { // KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE allows for APIs pending removal to not block tests t.Setenv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE", "true") - // Enable all features for testing + // Enable all features and apis for testing + flags := framework.DefaultTestServerFlags() + flags = append(flags, "--runtime-config=api/all=true") featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, "AllAlpha", true) featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, "AllBeta", true) testNamespace := "test-scale" - server := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd()) + server := kubeapiservertesting.StartTestServerOrDie(t, nil, flags, framework.SharedEtcd()) defer server.TearDownFn() clientset := clientset.NewForConfigOrDie(server.ClientConfig) @@ -3199,7 +3202,7 @@ func TestEmulatedStorageVersion(t *testing.T) { for emulatedVersion, cases := range groupedCases { t.Run(emulatedVersion, func(t *testing.T) { server := kubeapiservertesting.StartTestServerOrDie( - t, nil, []string{"--emulated-version=kube=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd()) + t, nil, []string{"--emulated-version=kube=" + emulatedVersion, `--storage-media-type=application/json`, fmt.Sprintf("--runtime-config=%s=true", admissionregistrationv1beta1.SchemeGroupVersion)}, framework.SharedEtcd()) defer server.TearDownFn() client := clientset.NewForConfigOrDie(server.ClientConfig) @@ -3340,7 +3343,7 @@ func TestEnableEmulationVersion(t *testing.T) { featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.32")) server := kubeapiservertesting.StartTestServerOrDie(t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"}, - []string{"--emulated-version=kube=1.31"}, framework.SharedEtcd()) + []string{"--emulated-version=kube=1.31", "--runtime-config=api/beta=true"}, framework.SharedEtcd()) defer server.TearDownFn() rt, err := restclient.TransportFor(server.ClientConfig) diff --git a/test/integration/apiserver/cel/authorizerselector/helper.go b/test/integration/apiserver/cel/authorizerselector/helper.go index 506583c4d6c..58761685cd4 100644 --- a/test/integration/apiserver/cel/authorizerselector/helper.go +++ b/test/integration/apiserver/cel/authorizerselector/helper.go @@ -49,7 +49,7 @@ func RunAuthzSelectorsLibraryTests(t *testing.T, featureEnabled bool) { // Start the server with the desired feature enablement server, err := apiservertesting.StartTestServer(t, nil, []string{ fmt.Sprintf("--feature-gates=AuthorizeNodeWithSelectors=%v,AuthorizeWithSelectors=%v", featureEnabled, featureEnabled), - "--runtime-config=resource.k8s.io/v1alpha3=true", + fmt.Sprintf("--runtime-config=%s=true", resourceapi.SchemeGroupVersion), }, framework.SharedEtcd()) if err != nil { t.Fatal(err) diff --git a/test/integration/apiserver/cel/mutatingadmissionpolicy_test.go b/test/integration/apiserver/cel/mutatingadmissionpolicy_test.go index 011a521a28f..e95fbdf401b 100644 --- a/test/integration/apiserver/cel/mutatingadmissionpolicy_test.go +++ b/test/integration/apiserver/cel/mutatingadmissionpolicy_test.go @@ -20,13 +20,13 @@ import ( "context" "errors" "fmt" - "github.com/google/go-cmp/cmp/cmpopts" "reflect" "strings" "testing" "time" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/require" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" @@ -493,7 +493,8 @@ func TestMutatingAdmissionPolicy(t *testing.T) { // Run all tests in a shared apiserver featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.MutatingAdmissionPolicy, true) - server, err := apiservertesting.StartTestServer(t, nil, nil, framework.SharedEtcd()) + flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha1.SchemeGroupVersion)} + server, err := apiservertesting.StartTestServer(t, nil, flags, framework.SharedEtcd()) require.NoError(t, err) defer server.TearDownFn() @@ -1006,7 +1007,8 @@ func TestMutatingAdmissionPolicy_Slow(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.MutatingAdmissionPolicy, true) - server, err := apiservertesting.StartTestServer(t, nil, nil, framework.SharedEtcd()) + flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha1.SchemeGroupVersion)} + server, err := apiservertesting.StartTestServer(t, nil, flags, framework.SharedEtcd()) require.NoError(t, err) defer server.TearDownFn() @@ -1091,7 +1093,8 @@ func TestMutatingAdmissionPolicy_Slow(t *testing.T) { // tested. func Test_MutatingAdmissionPolicy_CustomResources(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.MutatingAdmissionPolicy, true) - server, err := apiservertesting.StartTestServer(t, nil, nil, framework.SharedEtcd()) + flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha1.SchemeGroupVersion)} + server, err := apiservertesting.StartTestServer(t, nil, flags, framework.SharedEtcd()) etcd.CreateTestCRDs(t, apiextensions.NewForConfigOrDie(server.ClientConfig), false, versionedCustomResourceDefinition()) if err != nil { t.Fatal(err) diff --git a/test/integration/apiserver/coordinated_leader_election_test.go b/test/integration/apiserver/coordinated_leader_election_test.go index 5bfe1946fc6..505b9ee2308 100644 --- a/test/integration/apiserver/coordinated_leader_election_test.go +++ b/test/integration/apiserver/coordinated_leader_election_test.go @@ -44,7 +44,8 @@ import ( func TestSingleLeaseCandidate(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CoordinatedLeaderElection, true) - server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd()) + flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)} + server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd()) if err != nil { t.Fatal(err) } @@ -62,7 +63,8 @@ func TestSingleLeaseCandidate(t *testing.T) { func TestMultipleLeaseCandidate(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CoordinatedLeaderElection, true) - server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd()) + flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)} + server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd()) if err != nil { t.Fatal(err) } @@ -84,7 +86,8 @@ func TestMultipleLeaseCandidate(t *testing.T) { func TestLeaseSwapIfBetterAvailable(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CoordinatedLeaderElection, true) - server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd()) + flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)} + server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd()) if err != nil { t.Fatal(err) } @@ -105,7 +108,8 @@ func TestLeaseSwapIfBetterAvailable(t *testing.T) { func TestUpgradeSkew(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CoordinatedLeaderElection, true) - server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd()) + flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)} + server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd()) if err != nil { t.Fatal(err) } @@ -134,7 +138,8 @@ func TestLeaseCandidateCleanup(t *testing.T) { apiserver.LeaseCandidateGCPeriod = 30 * time.Minute }() - server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd()) + flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)} + server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd()) if err != nil { t.Fatal(err) } diff --git a/test/integration/apiserver/peerproxy/peer_proxy_test.go b/test/integration/apiserver/peerproxy/peer_proxy_test.go index 3c9a2e13f99..705489a44bf 100644 --- a/test/integration/apiserver/peerproxy/peer_proxy_test.go +++ b/test/integration/apiserver/peerproxy/peer_proxy_test.go @@ -18,7 +18,6 @@ package peerproxy import ( "context" - "fmt" "testing" "time" @@ -77,7 +76,7 @@ func TestPeerProxiedRequest(t *testing.T) { serverA := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{ EnableCertAuth: true, ProxyCA: &proxyCA}, - []string{}, etcd) + []string{"--runtime-config=api/all=true"}, etcd) t.Cleanup(serverA.TearDownFn) // start another test server with some api disabled @@ -86,7 +85,7 @@ func TestPeerProxiedRequest(t *testing.T) { serverB := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{ EnableCertAuth: true, ProxyCA: &proxyCA}, - []string{fmt.Sprintf("--runtime-config=%s", "batch/v1=false")}, etcd) + []string{"--runtime-config=api/all=true,batch/v1=false"}, etcd) t.Cleanup(serverB.TearDownFn) kubeClientSetA, err := kubernetes.NewForConfig(serverA.ClientConfig) @@ -144,7 +143,7 @@ func TestPeerProxiedRequestToThirdServerAfterFirstDies(t *testing.T) { // override hostname to ensure unique ips server.SetHostnameFuncForTests("test-server-a") t.Log("starting apiserver for ServerA") - serverA := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, ProxyCA: &proxyCA}, []string{}, etcd) + serverA := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, ProxyCA: &proxyCA}, []string{"--runtime-config=api/all=true"}, etcd) kubeClientSetA, err := kubernetes.NewForConfig(serverA.ClientConfig) require.NoError(t, err) // ensure storageversion garbage collector ctlr is set up @@ -160,7 +159,7 @@ func TestPeerProxiedRequestToThirdServerAfterFirstDies(t *testing.T) { server.SetHostnameFuncForTests("test-server-b") t.Log("starting apiserver for ServerB") serverB := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, ProxyCA: &proxyCA}, []string{ - fmt.Sprintf("--runtime-config=%v", "batch/v1=false")}, etcd) + "--runtime-config=api/all=true,batch/v1=false"}, etcd) t.Cleanup(serverB.TearDownFn) kubeClientSetB, err := kubernetes.NewForConfig(serverB.ClientConfig) require.NoError(t, err) @@ -172,7 +171,7 @@ func TestPeerProxiedRequestToThirdServerAfterFirstDies(t *testing.T) { // override hostname to ensure unique ips server.SetHostnameFuncForTests("test-server-c") t.Log("starting apiserver for ServerC") - serverC := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, ProxyCA: &proxyCA}, []string{}, etcd) + serverC := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, ProxyCA: &proxyCA}, []string{"--runtime-config=api/all=true"}, etcd) t.Cleanup(serverC.TearDownFn) // create jobs resource using serverA diff --git a/test/integration/client/metrics/metrics_test.go b/test/integration/client/metrics/metrics_test.go index af88050deba..e8f7fec02f2 100644 --- a/test/integration/client/metrics/metrics_test.go +++ b/test/integration/client/metrics/metrics_test.go @@ -50,7 +50,9 @@ func TestAPIServerTransportMetrics(t *testing.T) { // reset default registry metrics legacyregistry.Reset() - result := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd()) + flags := framework.DefaultTestServerFlags() + flags = append(flags, "--runtime-config=api/all=true,api/beta=true") + result := kubeapiservertesting.StartTestServerOrDie(t, nil, flags, framework.SharedEtcd()) defer result.TearDownFn() client := clientset.NewForConfigOrDie(result.ClientConfig) diff --git a/test/integration/clustertrustbundles/admission_establishtrust_test.go b/test/integration/clustertrustbundles/admission_establishtrust_test.go index 95eaa37ac8f..da720b815cc 100644 --- a/test/integration/clustertrustbundles/admission_establishtrust_test.go +++ b/test/integration/clustertrustbundles/admission_establishtrust_test.go @@ -20,6 +20,7 @@ import ( "context" "crypto/x509" "crypto/x509/pkix" + "fmt" "math/big" "testing" @@ -77,7 +78,7 @@ func TestCTBAttestPlugin(t *testing.T) { t.Run(tc.description, func(t *testing.T) { ctx := context.Background() - server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--authorization-mode=RBAC", "--feature-gates=ClusterTrustBundle=true"}, framework.SharedEtcd()) + server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--authorization-mode=RBAC", "--feature-gates=ClusterTrustBundle=true", fmt.Sprintf("--runtime-config=%s=true", certsv1alpha1.SchemeGroupVersion)}, framework.SharedEtcd()) defer server.TearDownFn() client := kubernetes.NewForConfigOrDie(server.ClientConfig) diff --git a/test/integration/clustertrustbundles/apiserversigner_test.go b/test/integration/clustertrustbundles/apiserversigner_test.go index d651089f4aa..3d1a7918d59 100644 --- a/test/integration/clustertrustbundles/apiserversigner_test.go +++ b/test/integration/clustertrustbundles/apiserversigner_test.go @@ -73,6 +73,7 @@ func TestClusterTrustBundlesPublisherController(t *testing.T) { "--disable-admission-plugins", "ServiceAccount", "--authorization-mode=RBAC", "--feature-gates", "ClusterTrustBundle=true", + fmt.Sprintf("--runtime-config=%s=true", v1alpha1.SchemeGroupVersion), } storageConfig := framework.SharedEtcd() server := kubeapiservertesting.StartTestServerOrDie(t, nil, apiServerFlags, storageConfig) diff --git a/test/integration/clustertrustbundles/field_selector_test.go b/test/integration/clustertrustbundles/field_selector_test.go index 0b0d6d6b162..a1257c83674 100644 --- a/test/integration/clustertrustbundles/field_selector_test.go +++ b/test/integration/clustertrustbundles/field_selector_test.go @@ -20,6 +20,7 @@ import ( "context" "crypto/x509" "crypto/x509/pkix" + "fmt" "math/big" "testing" @@ -37,7 +38,7 @@ func TestCTBSignerNameFieldSelector(t *testing.T) { ctx := context.Background() - server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--feature-gates=ClusterTrustBundle=true"}, framework.SharedEtcd()) + server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--feature-gates=ClusterTrustBundle=true", fmt.Sprintf("--runtime-config=%s=true", certsv1alpha1.SchemeGroupVersion)}, framework.SharedEtcd()) defer server.TearDownFn() client := kubernetes.NewForConfigOrDie(server.ClientConfig) diff --git a/test/integration/clustertrustbundles/signer_name_change_forbidden_test.go b/test/integration/clustertrustbundles/signer_name_change_forbidden_test.go index 3e218cc23cc..8afa1cdbc1b 100644 --- a/test/integration/clustertrustbundles/signer_name_change_forbidden_test.go +++ b/test/integration/clustertrustbundles/signer_name_change_forbidden_test.go @@ -63,7 +63,7 @@ func TestCTBSignerNameChangeForbidden(t *testing.T) { ctx := context.Background() - server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--feature-gates=ClusterTrustBundle=true"}, framework.SharedEtcd()) + server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--feature-gates=ClusterTrustBundle=true", fmt.Sprintf("--runtime-config=%s=true", certsv1alpha1.SchemeGroupVersion)}, framework.SharedEtcd()) defer server.TearDownFn() client := kubernetes.NewForConfigOrDie(server.ClientConfig) diff --git a/test/integration/controlplane/transformation/all_transformation_test.go b/test/integration/controlplane/transformation/all_transformation_test.go index 4ca250f234a..1496dcf9521 100644 --- a/test/integration/controlplane/transformation/all_transformation_test.go +++ b/test/integration/controlplane/transformation/all_transformation_test.go @@ -94,7 +94,7 @@ resources: - name: key1 secret: c2VjcmV0IGlzIHNlY3VyZQ== ` - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start Kube API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } diff --git a/test/integration/controlplane/transformation/kms_transformation_test.go b/test/integration/controlplane/transformation/kms_transformation_test.go index 9e70ef3c321..22d54abd081 100644 --- a/test/integration/controlplane/transformation/kms_transformation_test.go +++ b/test/integration/controlplane/transformation/kms_transformation_test.go @@ -145,7 +145,7 @@ resources: ` providerName := "kms-provider" pluginMock := mock.NewBase64Plugin(t, "@kms-provider.sock") - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -329,7 +329,7 @@ resources: genericapiserver.SetHostnameFuncForTests("testAPIServerID") _ = mock.NewBase64Plugin(t, "@kms-provider.sock") var restarted bool - test, err := newTransformTest(t, encryptionConfig, true, "", storageConfig) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, reload: true, storageConfig: storageConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -550,7 +550,7 @@ resources: previousConfigDir := test.configDir test.shutdownAPIServer() restarted = true - test, err = newTransformTest(t, test.transformerConfig, true, previousConfigDir, storageConfig) + test, err = newTransformTest(t, transformTestConfig{transformerConfigYAML: test.transformerConfig, reload: true, configDir: previousConfigDir, storageConfig: storageConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -626,7 +626,7 @@ resources: // Need to enable this explicitly as the feature is deprecated featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true) - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, runtimeConfig: []string{"api/alpha=true", "api/beta=true"}}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig") } @@ -752,7 +752,7 @@ resources: featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true) - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -899,7 +899,7 @@ resources: ` _ = mock.NewBase64Plugin(t, "@kms-provider.sock") - test, err := newTransformTest(t, encryptionConfig, true, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, reload: true}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -1111,7 +1111,7 @@ resources: pluginMock1 := mock.NewBase64Plugin(t, "@kms-provider-1.sock") pluginMock2 := mock.NewBase64Plugin(t, "@kms-provider-2.sock") - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start kube-apiserver, error: %v", err) } @@ -1174,7 +1174,7 @@ resources: pluginMock1 := mock.NewBase64Plugin(t, "@kms-provider-1.sock") pluginMock2 := mock.NewBase64Plugin(t, "@kms-provider-2.sock") - test, err := newTransformTest(t, encryptionConfig, true, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, reload: true}) if err != nil { t.Fatalf("Failed to start kube-apiserver, error: %v", err) } diff --git a/test/integration/controlplane/transformation/kmsv2_transformation_test.go b/test/integration/controlplane/transformation/kmsv2_transformation_test.go index 31d684a6640..6e6899e1c25 100644 --- a/test/integration/controlplane/transformation/kmsv2_transformation_test.go +++ b/test/integration/controlplane/transformation/kmsv2_transformation_test.go @@ -193,7 +193,7 @@ resources: ` _ = kmsv2mock.NewBase64Plugin(t, "@kms-provider-defaults.sock") - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -279,7 +279,7 @@ resources: genericapiserver.SetHostnameFuncForTests("testAPIServerID") pluginMock := kmsv2mock.NewBase64Plugin(t, "@"+kmsName+".sock") - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -432,7 +432,7 @@ resources: ` pluginMock := kmsv2mock.NewBase64Plugin(t, "@"+kmsName+".sock") - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -730,7 +730,7 @@ resources: ` _ = kmsv2mock.NewBase64Plugin(t, "@"+kmsName+".sock") - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -871,7 +871,7 @@ resources: pluginMock1 := kmsv2mock.NewBase64Plugin(t, "@kms-provider-1.sock") pluginMock2 := kmsv2mock.NewBase64Plugin(t, "@kms-provider-2.sock") - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("Failed to start kube-apiserver, error: %v", err) } @@ -949,7 +949,7 @@ resources: _ = kmsv2mock.NewBase64Plugin(t, "@kms-provider-single-service.sock") - test, err := newTransformTest(t, encryptionConfig, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -1006,7 +1006,7 @@ resources: storageConfig := framework.SharedEtcd() // KMSv2 is enabled by default. Loading a encryptionConfig with KMSv2 should work - test, err := newTransformTest(t, encryptionConfig, false, "", storageConfig) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, storageConfig: storageConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -1078,7 +1078,7 @@ resources: // After a restart, loading a encryptionConfig with the same KMSv2 plugin before the restart should work, decryption of data encrypted with v2 should work - test, err = newTransformTest(t, encryptionConfig, false, "", storageConfig) + test, err = newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, storageConfig: storageConfig}) if err != nil { t.Fatalf("Failed to restart api server, error: %v", err) } @@ -1126,7 +1126,7 @@ resources: ` _ = kmsv2mock.NewBase64Plugin(b, "@kms-provider-bench.sock") - test, err := newTransformTest(b, encryptionConfig, false, "", nil) + test, err := newTransformTest(b, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { b.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -1279,7 +1279,7 @@ resources: ` _ = kmsv2mock.NewBase64Plugin(b, "@kms-provider-bench-rest.sock") - test, err := newTransformTest(b, encryptionConfig, false, "", nil) + test, err := newTransformTest(b, transformTestConfig{transformerConfigYAML: encryptionConfig}) if err != nil { b.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -1378,7 +1378,7 @@ resources: storageConfig := storagebackend.NewDefaultConfig(path.Join(legacyDataEtcdPrefix, "registry"), nil) storageConfig.Transport.ServerList = []string{framework.GetEtcdURL()} - test, err := newTransformTest(t, encryptionConfig, false, "", storageConfig) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, storageConfig: storageConfig}) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } diff --git a/test/integration/controlplane/transformation/secrets_transformation_test.go b/test/integration/controlplane/transformation/secrets_transformation_test.go index 8767b2f5acb..432cb74287a 100644 --- a/test/integration/controlplane/transformation/secrets_transformation_test.go +++ b/test/integration/controlplane/transformation/secrets_transformation_test.go @@ -106,7 +106,7 @@ func TestSecretsShouldBeTransformed(t *testing.T) { // TODO: add secretbox } for _, tt := range testCases { - test, err := newTransformTest(t, tt.transformerConfigContent, false, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: tt.transformerConfigContent}) if err != nil { t.Fatalf("failed to setup test for envelop %s, error was %v", tt.transformerPrefix, err) continue @@ -195,7 +195,7 @@ func TestAllowUnsafeMalformedObjectDeletionFeature(t *testing.T) { t.Run(fmt.Sprintf("%s/%t", string(genericfeatures.AllowUnsafeMalformedObjectDeletion), tc.featureEnabled), func(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AllowUnsafeMalformedObjectDeletion, tc.featureEnabled) - test, err := newTransformTest(t, aesGCMConfigYAML, true, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: aesGCMConfigYAML, reload: true}) if err != nil { t.Fatalf("failed to setup test for envelop %s, error was %v", aesGCMPrefix, err) } @@ -498,7 +498,7 @@ func TestListCorruptObjects(t *testing.T) { t.Run(fmt.Sprintf("%s/%t", string(genericfeatures.AllowUnsafeMalformedObjectDeletion), tc.featureEnabled), func(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AllowUnsafeMalformedObjectDeletion, tc.featureEnabled) - test, err := newTransformTest(t, aesGCMConfigYAML, true, "", nil) + test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: aesGCMConfigYAML, reload: true}) if err != nil { t.Fatalf("failed to setup test for envelop %s, error was %v", aesGCMPrefix, err) } @@ -651,7 +651,7 @@ func BenchmarkAESCBCEnvelopeWrite(b *testing.B) { func runBenchmark(b *testing.B, transformerConfig string) { b.StopTimer() - test, err := newTransformTest(b, transformerConfig, false, "", nil) + test, err := newTransformTest(b, transformTestConfig{transformerConfigYAML: transformerConfig}) if err != nil { b.Fatalf("failed to setup benchmark for config %s, error was %v", transformerConfig, err) } diff --git a/test/integration/controlplane/transformation/transformation_test.go b/test/integration/controlplane/transformation/transformation_test.go index c0c95ce230a..96446920fe0 100644 --- a/test/integration/controlplane/transformation/transformation_test.go +++ b/test/integration/controlplane/transformation/transformation_test.go @@ -87,27 +87,35 @@ type transformTest struct { secret *corev1.Secret } -func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool, configDir string, storageConfig *storagebackend.Config) (*transformTest, error) { +type transformTestConfig struct { + transformerConfigYAML string + reload bool + configDir string + storageConfig *storagebackend.Config + runtimeConfig []string +} + +func newTransformTest(tb testing.TB, config transformTestConfig) (*transformTest, error) { tCtx := ktesting.Init(tb) - if storageConfig == nil { - storageConfig = framework.SharedEtcd() + if config.storageConfig == nil { + config.storageConfig = framework.SharedEtcd() } e := transformTest{ TContext: tCtx, - transformerConfig: transformerConfigYAML, - storageConfig: storageConfig, + transformerConfig: config.transformerConfigYAML, + storageConfig: config.storageConfig, } var err error // create config dir with provided config yaml - if transformerConfigYAML != "" && configDir == "" { + if config.transformerConfigYAML != "" && config.configDir == "" { if e.configDir, err = e.createEncryptionConfig(); err != nil { e.cleanUp() return nil, fmt.Errorf("error while creating KubeAPIServer encryption config: %w", err) } } else { // configDir already exists. api-server must be restarting with existing encryption config - e.configDir = configDir + e.configDir = config.configDir } configFile := filepath.Join(e.configDir, encryptionConfigFileName) _, err = os.ReadFile(configFile) @@ -116,9 +124,13 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool, return nil, fmt.Errorf("failed to read config file: %w", err) } + flags := e.getEncryptionOptions(config.reload) + if len(config.runtimeConfig) > 0 { + flags = append(flags, "--runtime-config="+strings.Join(config.runtimeConfig, ",")) + } if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer( tb, nil, - e.getEncryptionOptions(reload), e.storageConfig); err != nil { + flags, e.storageConfig); err != nil { e.cleanUp() return nil, fmt.Errorf("failed to start KubeAPI server: %w", err) } @@ -134,7 +146,7 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool, return nil, err } - if transformerConfigYAML != "" && reload { + if config.transformerConfigYAML != "" && config.reload { // when reloading is enabled, this healthz endpoint is always present mustBeHealthy(tCtx, "/kms-providers", "ok", e.kubeAPIServer.ClientConfig) mustNotHaveLivez(tCtx, "/kms-providers", "404 page not found", e.kubeAPIServer.ClientConfig) diff --git a/test/integration/metrics/metrics_test.go b/test/integration/metrics/metrics_test.go index f38f1f33220..dd1328a40d4 100644 --- a/test/integration/metrics/metrics_test.go +++ b/test/integration/metrics/metrics_test.go @@ -26,6 +26,7 @@ import ( "testing" "github.com/prometheus/common/model" + admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/endpoints/metrics" @@ -108,7 +109,9 @@ func TestAPIServerMetrics(t *testing.T) { // KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE allows for APIs pending removal to not block tests t.Setenv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE", "true") - s := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd()) + flags := framework.DefaultTestServerFlags() + flags = append(flags, fmt.Sprintf("--runtime-config=%s=true", admissionregistrationv1beta1.SchemeGroupVersion)) + s := kubeapiservertesting.StartTestServerOrDie(t, nil, flags, framework.SharedEtcd()) defer s.TearDownFn() // Make a request to the apiserver to ensure there's at least one data point diff --git a/test/integration/resourceclaim/feature_enable_disable_test.go b/test/integration/resourceclaim/feature_enable_disable_test.go index 7921190e604..ca4a6d8fd16 100644 --- a/test/integration/resourceclaim/feature_enable_disable_test.go +++ b/test/integration/resourceclaim/feature_enable_disable_test.go @@ -42,6 +42,7 @@ func TestEnableDisableDRAResourceClaimDeviceStatus(t *testing.T) { // apiserver with the feature disabled server1 := kubeapiservertesting.StartTestServerOrDie(t, apiServerOptions, []string{ + fmt.Sprintf("--runtime-config=%s=true", v1beta1.SchemeGroupVersion), fmt.Sprintf("--feature-gates=%s=true,%s=false", features.DynamicResourceAllocation, features.DRAResourceClaimDeviceStatus), }, etcdOptions) @@ -114,6 +115,7 @@ func TestEnableDisableDRAResourceClaimDeviceStatus(t *testing.T) { // apiserver with the feature enabled server2 := kubeapiservertesting.StartTestServerOrDie(t, apiServerOptions, []string{ + fmt.Sprintf("--runtime-config=%s=true", v1beta1.SchemeGroupVersion), fmt.Sprintf("--feature-gates=%s=true,%s=true", features.DynamicResourceAllocation, features.DRAResourceClaimDeviceStatus), }, etcdOptions) diff --git a/test/integration/scheduler_perf/util.go b/test/integration/scheduler_perf/util.go index 279f309a4fc..b3e5801ac11 100644 --- a/test/integration/scheduler_perf/util.go +++ b/test/integration/scheduler_perf/util.go @@ -30,6 +30,8 @@ import ( "time" v1 "k8s.io/api/core/v1" + resourceapialpha "k8s.io/api/resource/v1alpha3" + resourceapi "k8s.io/api/resource/v1beta1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -86,11 +88,10 @@ func newDefaultComponentConfig() (*config.KubeSchedulerConfiguration, error) { // Notes on rate limiter: // - client rate limit is set to 5000. func mustSetupCluster(tCtx ktesting.TContext, config *config.KubeSchedulerConfiguration, enabledFeatures map[featuregate.Feature]bool, outOfTreePluginRegistry frameworkruntime.Registry) (informers.SharedInformerFactory, ktesting.TContext) { - // No alpha APIs (overrides api/all=true in https://github.com/kubernetes/kubernetes/blob/d647d19f6aef811bace300eec96a67644ff303d4/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go#L136), - // except for DRA API group when needed. - runtimeConfig := []string{"api/alpha=false"} + var runtimeConfig []string if enabledFeatures[features.DynamicResourceAllocation] { - runtimeConfig = append(runtimeConfig, "resource.k8s.io/v1alpha3=true") + runtimeConfig = append(runtimeConfig, fmt.Sprintf("%s=true", resourceapi.SchemeGroupVersion)) + runtimeConfig = append(runtimeConfig, fmt.Sprintf("%s=true", resourceapialpha.SchemeGroupVersion)) } customFlags := []string{ // Disable ServiceAccount admission plugin as we don't have serviceaccount controller running. diff --git a/test/integration/storageversion/gc_test.go b/test/integration/storageversion/gc_test.go index 13065ebb20b..af521415714 100644 --- a/test/integration/storageversion/gc_test.go +++ b/test/integration/storageversion/gc_test.go @@ -52,7 +52,9 @@ const ( func TestStorageVersionGarbageCollection(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true) featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionAPI, true) - result := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd()) + flags := framework.DefaultTestServerFlags() + flags = append(flags, fmt.Sprintf("--runtime-config=%s=true", apiserverinternalv1alpha1.SchemeGroupVersion)) + result := kubeapiservertesting.StartTestServerOrDie(t, nil, flags, framework.SharedEtcd()) defer result.TearDownFn() kubeclient, err := kubernetes.NewForConfig(result.ClientConfig) diff --git a/test/integration/storageversion/storage_version_filter_test.go b/test/integration/storageversion/storage_version_filter_test.go index b95fa5505cd..5e41929cf7b 100644 --- a/test/integration/storageversion/storage_version_filter_test.go +++ b/test/integration/storageversion/storage_version_filter_test.go @@ -148,7 +148,9 @@ func testBuiltinResourceRead(t *testing.T, cfg *rest.Config, shouldBlock bool) { func TestStorageVersionBootstrap(t *testing.T) { // Start server and create CRD etcdConfig := framework.SharedEtcd() - server := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), etcdConfig) + flags := framework.DefaultTestServerFlags() + flags = append(flags, "--runtime-config=api/all=true") + server := kubeapiservertesting.StartTestServerOrDie(t, nil, flags, etcdConfig) etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(server.ClientConfig), false, etcd.GetCustomResourceDefinitionData()[0]) server.TearDownFn() diff --git a/test/integration/storageversionmigrator/util.go b/test/integration/storageversionmigrator/util.go index 7b78e673209..111632a14e5 100644 --- a/test/integration/storageversionmigrator/util.go +++ b/test/integration/storageversionmigrator/util.go @@ -275,6 +275,7 @@ func svmSetup(ctx context.Context, t *testing.T) *svmTest { "--audit-log-mode", "blocking", "--audit-log-path", logFile.Name(), "--authorization-mode=RBAC", + fmt.Sprintf("--runtime-config=%s=true", svmv1alpha1.SchemeGroupVersion), } storageConfig := framework.SharedEtcd() server := kubeapiservertesting.StartTestServerOrDie(t, nil, apiServerFlags, storageConfig)