github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-10 12:32:03 +00:00
Code Issues Projects Releases Wiki Activity

test/integration: use default API groups in test apiserver

Browse Source 
The goal is to make the test apiserver behave as much as kube-apiserver as
possible. This ensures that tests are as realistic as possible out-of-the-box.
If a test needs a special setup, then that should be visible in the test
because it passes additional flags or options.

One historic deviation from that goal was enabling all API groups. That
change (from 7185624688) gets reverted and tests
which happened to rely on this get updated.
This commit is contained in:
Patrick Ohly 2025-02-20 21:21:53 +01:00
parent 87fcae2bc7
commit 56bd8d86a5
22 changed files with 102 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/kube-apiserver/app/testing/testserver.go
View File

@ -339,7 +339,6 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
s.ServiceClusterIPRanges = "10.0.0.0/16"
s.Etcd.StorageConfig = *storageConfig
s.APIEnablement.RuntimeConfig.Set("api/all=true")
if err := fs.Parse(customFlags); err != nil {
return result, err

11
test/integration/apiserver/apiserver_test.go
View File

@ -34,6 +34,7 @@ import (
"time"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@ -1683,12 +1684,14 @@ func TestGetScaleSubresourceAsTableForAllBuiltins(t *testing.T) {
// KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE allows for APIs pending removal to not block tests
t.Setenv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE", "true")
// Enable all features for testing
// Enable all features and apis for testing
flags := framework.DefaultTestServerFlags()
flags = append(flags, "--runtime-config=api/all=true")
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, "AllAlpha", true)
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, "AllBeta", true)
testNamespace := "test-scale"
server := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd())
server := kubeapiservertesting.StartTestServerOrDie(t, nil, flags, framework.SharedEtcd())
defer server.TearDownFn()
clientset := clientset.NewForConfigOrDie(server.ClientConfig)
@ -3199,7 +3202,7 @@ func TestEmulatedStorageVersion(t *testing.T) {
for emulatedVersion, cases := range groupedCases {
t.Run(emulatedVersion, func(t *testing.T) {
server := kubeapiservertesting.StartTestServerOrDie(
t, nil, []string{"--emulated-version=kube=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd())
t, nil, []string{"--emulated-version=kube=" + emulatedVersion, `--storage-media-type=application/json`, fmt.Sprintf("--runtime-config=%s=true", admissionregistrationv1beta1.SchemeGroupVersion)}, framework.SharedEtcd())
defer server.TearDownFn()
client := clientset.NewForConfigOrDie(server.ClientConfig)
@ -3340,7 +3343,7 @@ func TestEnableEmulationVersion(t *testing.T) {
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.32"))
server := kubeapiservertesting.StartTestServerOrDie(t,
&kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"},
[]string{"--emulated-version=kube=1.31"}, framework.SharedEtcd())
[]string{"--emulated-version=kube=1.31", "--runtime-config=api/beta=true"}, framework.SharedEtcd())
defer server.TearDownFn()
rt, err := restclient.TransportFor(server.ClientConfig)

2
test/integration/apiserver/cel/authorizerselector/helper.go
View File

@ -49,7 +49,7 @@ func RunAuthzSelectorsLibraryTests(t *testing.T, featureEnabled bool) {
// Start the server with the desired feature enablement
server, err := apiservertesting.StartTestServer(t, nil, []string{
fmt.Sprintf("--feature-gates=AuthorizeNodeWithSelectors=%v,AuthorizeWithSelectors=%v", featureEnabled, featureEnabled),
"--runtime-config=resource.k8s.io/v1alpha3=true",
fmt.Sprintf("--runtime-config=%s=true", resourceapi.SchemeGroupVersion),
}, framework.SharedEtcd())
if err != nil {
t.Fatal(err)

11
test/integration/apiserver/cel/mutatingadmissionpolicy_test.go
View File

@ -20,13 +20,13 @@ import (
"context"
"errors"
"fmt"
"github.com/google/go-cmp/cmp/cmpopts"
"reflect"
"strings"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/stretchr/testify/require"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
@ -493,7 +493,8 @@ func TestMutatingAdmissionPolicy(t *testing.T) {
// Run all tests in a shared apiserver
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.MutatingAdmissionPolicy, true)
server, err := apiservertesting.StartTestServer(t, nil, nil, framework.SharedEtcd())
flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha1.SchemeGroupVersion)}
server, err := apiservertesting.StartTestServer(t, nil, flags, framework.SharedEtcd())
require.NoError(t, err)
defer server.TearDownFn()
@ -1006,7 +1007,8 @@ func TestMutatingAdmissionPolicy_Slow(t *testing.T) {
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.MutatingAdmissionPolicy, true)
server, err := apiservertesting.StartTestServer(t, nil, nil, framework.SharedEtcd())
flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha1.SchemeGroupVersion)}
server, err := apiservertesting.StartTestServer(t, nil, flags, framework.SharedEtcd())
require.NoError(t, err)
defer server.TearDownFn()
@ -1091,7 +1093,8 @@ func TestMutatingAdmissionPolicy_Slow(t *testing.T) {
// tested.
func Test_MutatingAdmissionPolicy_CustomResources(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.MutatingAdmissionPolicy, true)
server, err := apiservertesting.StartTestServer(t, nil, nil, framework.SharedEtcd())
flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha1.SchemeGroupVersion)}
server, err := apiservertesting.StartTestServer(t, nil, flags, framework.SharedEtcd())
etcd.CreateTestCRDs(t, apiextensions.NewForConfigOrDie(server.ClientConfig), false, versionedCustomResourceDefinition())
if err != nil {
t.Fatal(err)

15
test/integration/apiserver/coordinated_leader_election_test.go
View File

@ -44,7 +44,8 @@ import (
func TestSingleLeaseCandidate(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CoordinatedLeaderElection, true)
server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd())
flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)}
server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd())
if err != nil {
t.Fatal(err)
}
@ -62,7 +63,8 @@ func TestSingleLeaseCandidate(t *testing.T) {
func TestMultipleLeaseCandidate(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CoordinatedLeaderElection, true)
server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd())
flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)}
server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd())
if err != nil {
t.Fatal(err)
}
@ -84,7 +86,8 @@ func TestMultipleLeaseCandidate(t *testing.T) {
func TestLeaseSwapIfBetterAvailable(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CoordinatedLeaderElection, true)
server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd())
flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)}
server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd())
if err != nil {
t.Fatal(err)
}
@ -105,7 +108,8 @@ func TestLeaseSwapIfBetterAvailable(t *testing.T) {
func TestUpgradeSkew(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CoordinatedLeaderElection, true)
server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd())
flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)}
server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd())
if err != nil {
t.Fatal(err)
}
@ -134,7 +138,8 @@ func TestLeaseCandidateCleanup(t *testing.T) {
apiserver.LeaseCandidateGCPeriod = 30 * time.Minute
}()
server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd())
flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)}
server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd())
if err != nil {
t.Fatal(err)
}

11
test/integration/apiserver/peerproxy/peer_proxy_test.go
View File

@ -18,7 +18,6 @@ package peerproxy
import (
"context"
"fmt"
"testing"
"time"
@ -77,7 +76,7 @@ func TestPeerProxiedRequest(t *testing.T) {
serverA := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{
EnableCertAuth: true,
ProxyCA: &proxyCA},
[]string{}, etcd)
[]string{"--runtime-config=api/all=true"}, etcd)
t.Cleanup(serverA.TearDownFn)
// start another test server with some api disabled
@ -86,7 +85,7 @@ func TestPeerProxiedRequest(t *testing.T) {
serverB := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{
EnableCertAuth: true,
ProxyCA: &proxyCA},
[]string{fmt.Sprintf("--runtime-config=%s", "batch/v1=false")}, etcd)
[]string{"--runtime-config=api/all=true,batch/v1=false"}, etcd)
t.Cleanup(serverB.TearDownFn)
kubeClientSetA, err := kubernetes.NewForConfig(serverA.ClientConfig)
@ -144,7 +143,7 @@ func TestPeerProxiedRequestToThirdServerAfterFirstDies(t *testing.T) {
// override hostname to ensure unique ips
server.SetHostnameFuncForTests("test-server-a")
t.Log("starting apiserver for ServerA")
serverA := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, ProxyCA: &proxyCA}, []string{}, etcd)
serverA := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, ProxyCA: &proxyCA}, []string{"--runtime-config=api/all=true"}, etcd)
kubeClientSetA, err := kubernetes.NewForConfig(serverA.ClientConfig)
require.NoError(t, err)
// ensure storageversion garbage collector ctlr is set up
@ -160,7 +159,7 @@ func TestPeerProxiedRequestToThirdServerAfterFirstDies(t *testing.T) {
server.SetHostnameFuncForTests("test-server-b")
t.Log("starting apiserver for ServerB")
serverB := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, ProxyCA: &proxyCA}, []string{
fmt.Sprintf("--runtime-config=%v", "batch/v1=false")}, etcd)
"--runtime-config=api/all=true,batch/v1=false"}, etcd)
t.Cleanup(serverB.TearDownFn)
kubeClientSetB, err := kubernetes.NewForConfig(serverB.ClientConfig)
require.NoError(t, err)
@ -172,7 +171,7 @@ func TestPeerProxiedRequestToThirdServerAfterFirstDies(t *testing.T) {
// override hostname to ensure unique ips
server.SetHostnameFuncForTests("test-server-c")
t.Log("starting apiserver for ServerC")
serverC := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, ProxyCA: &proxyCA}, []string{}, etcd)
serverC := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, ProxyCA: &proxyCA}, []string{"--runtime-config=api/all=true"}, etcd)
t.Cleanup(serverC.TearDownFn)
// create jobs resource using serverA

4
test/integration/client/metrics/metrics_test.go
View File

@ -50,7 +50,9 @@ func TestAPIServerTransportMetrics(t *testing.T) {
// reset default registry metrics
legacyregistry.Reset()
result := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd())
flags := framework.DefaultTestServerFlags()
flags = append(flags, "--runtime-config=api/all=true,api/beta=true")
result := kubeapiservertesting.StartTestServerOrDie(t, nil, flags, framework.SharedEtcd())
defer result.TearDownFn()
client := clientset.NewForConfigOrDie(result.ClientConfig)

3
test/integration/clustertrustbundles/admission_establishtrust_test.go
View File

@ -20,6 +20,7 @@ import (
"context"
"crypto/x509"
"crypto/x509/pkix"
"fmt"
"math/big"
"testing"
@ -77,7 +78,7 @@ func TestCTBAttestPlugin(t *testing.T) {
t.Run(tc.description, func(t *testing.T) {
ctx := context.Background()
server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--authorization-mode=RBAC", "--feature-gates=ClusterTrustBundle=true"}, framework.SharedEtcd())
server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--authorization-mode=RBAC", "--feature-gates=ClusterTrustBundle=true", fmt.Sprintf("--runtime-config=%s=true", certsv1alpha1.SchemeGroupVersion)}, framework.SharedEtcd())
defer server.TearDownFn()
client := kubernetes.NewForConfigOrDie(server.ClientConfig)

1
test/integration/clustertrustbundles/apiserversigner_test.go
View File

@ -73,6 +73,7 @@ func TestClusterTrustBundlesPublisherController(t *testing.T) {
"--disable-admission-plugins", "ServiceAccount",
"--authorization-mode=RBAC",
"--feature-gates", "ClusterTrustBundle=true",
fmt.Sprintf("--runtime-config=%s=true", v1alpha1.SchemeGroupVersion),
}
storageConfig := framework.SharedEtcd()
server := kubeapiservertesting.StartTestServerOrDie(t, nil, apiServerFlags, storageConfig)

3
test/integration/clustertrustbundles/field_selector_test.go
View File

@ -20,6 +20,7 @@ import (
"context"
"crypto/x509"
"crypto/x509/pkix"
"fmt"
"math/big"
"testing"
@ -37,7 +38,7 @@ func TestCTBSignerNameFieldSelector(t *testing.T) {
ctx := context.Background()
server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--feature-gates=ClusterTrustBundle=true"}, framework.SharedEtcd())
server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--feature-gates=ClusterTrustBundle=true", fmt.Sprintf("--runtime-config=%s=true", certsv1alpha1.SchemeGroupVersion)}, framework.SharedEtcd())
defer server.TearDownFn()
client := kubernetes.NewForConfigOrDie(server.ClientConfig)

2
test/integration/clustertrustbundles/signer_name_change_forbidden_test.go
View File

@ -63,7 +63,7 @@ func TestCTBSignerNameChangeForbidden(t *testing.T) {
ctx := context.Background()
server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--feature-gates=ClusterTrustBundle=true"}, framework.SharedEtcd())
server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--feature-gates=ClusterTrustBundle=true", fmt.Sprintf("--runtime-config=%s=true", certsv1alpha1.SchemeGroupVersion)}, framework.SharedEtcd())
defer server.TearDownFn()
client := kubernetes.NewForConfigOrDie(server.ClientConfig)

2
test/integration/controlplane/transformation/all_transformation_test.go
View File

@ -94,7 +94,7 @@ resources:
- name: key1
secret: c2VjcmV0IGlzIHNlY3VyZQ==
`
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig})
if err != nil {
t.Fatalf("failed to start Kube API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}

16
test/integration/controlplane/transformation/kms_transformation_test.go
View File

@ -145,7 +145,7 @@ resources:
`
providerName := "kms-provider"
pluginMock := mock.NewBase64Plugin(t, "@kms-provider.sock")
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig})
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}
@ -329,7 +329,7 @@ resources:
genericapiserver.SetHostnameFuncForTests("testAPIServerID")
_ = mock.NewBase64Plugin(t, "@kms-provider.sock")
var restarted bool
test, err := newTransformTest(t, encryptionConfig, true, "", storageConfig)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, reload: true, storageConfig: storageConfig})
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}
@ -550,7 +550,7 @@ resources:
previousConfigDir := test.configDir
test.shutdownAPIServer()
restarted = true
test, err = newTransformTest(t, test.transformerConfig, true, previousConfigDir, storageConfig)
test, err = newTransformTest(t, transformTestConfig{transformerConfigYAML: test.transformerConfig, reload: true, configDir: previousConfigDir, storageConfig: storageConfig})
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}
@ -626,7 +626,7 @@ resources:
// Need to enable this explicitly as the feature is deprecated
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, runtimeConfig: []string{"api/alpha=true", "api/beta=true"}})
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig")
}
@ -752,7 +752,7 @@ resources:
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig})
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}
@ -899,7 +899,7 @@ resources:
`
_ = mock.NewBase64Plugin(t, "@kms-provider.sock")
test, err := newTransformTest(t, encryptionConfig, true, "", nil)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, reload: true})
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}
@ -1111,7 +1111,7 @@ resources:
pluginMock1 := mock.NewBase64Plugin(t, "@kms-provider-1.sock")
pluginMock2 := mock.NewBase64Plugin(t, "@kms-provider-2.sock")
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig})
if err != nil {
t.Fatalf("failed to start kube-apiserver, error: %v", err)
}
@ -1174,7 +1174,7 @@ resources:
pluginMock1 := mock.NewBase64Plugin(t, "@kms-provider-1.sock")
pluginMock2 := mock.NewBase64Plugin(t, "@kms-provider-2.sock")
test, err := newTransformTest(t, encryptionConfig, true, "", nil)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, reload: true})
if err != nil {
t.Fatalf("Failed to start kube-apiserver, error: %v", err)
}

22
test/integration/controlplane/transformation/kmsv2_transformation_test.go
View File

@ -193,7 +193,7 @@ resources:
`
_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider-defaults.sock")
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig})
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}
@ -279,7 +279,7 @@ resources:
genericapiserver.SetHostnameFuncForTests("testAPIServerID")
pluginMock := kmsv2mock.NewBase64Plugin(t, "@"+kmsName+".sock")
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig})
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}
@ -432,7 +432,7 @@ resources:
`
pluginMock := kmsv2mock.NewBase64Plugin(t, "@"+kmsName+".sock")
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig})
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}
@ -730,7 +730,7 @@ resources:
`
_ = kmsv2mock.NewBase64Plugin(t, "@"+kmsName+".sock")
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig})
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}
@ -871,7 +871,7 @@ resources:
pluginMock1 := kmsv2mock.NewBase64Plugin(t, "@kms-provider-1.sock")
pluginMock2 := kmsv2mock.NewBase64Plugin(t, "@kms-provider-2.sock")
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig})
if err != nil {
t.Fatalf("Failed to start kube-apiserver, error: %v", err)
}
@ -949,7 +949,7 @@ resources:
_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider-single-service.sock")
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig})
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}
@ -1006,7 +1006,7 @@ resources:
storageConfig := framework.SharedEtcd()
// KMSv2 is enabled by default. Loading a encryptionConfig with KMSv2 should work
test, err := newTransformTest(t, encryptionConfig, false, "", storageConfig)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, storageConfig: storageConfig})
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}
@ -1078,7 +1078,7 @@ resources:
// After a restart, loading a encryptionConfig with the same KMSv2 plugin before the restart should work, decryption of data encrypted with v2 should work
test, err = newTransformTest(t, encryptionConfig, false, "", storageConfig)
test, err = newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, storageConfig: storageConfig})
if err != nil {
t.Fatalf("Failed to restart api server, error: %v", err)
}
@ -1126,7 +1126,7 @@ resources:
`
_ = kmsv2mock.NewBase64Plugin(b, "@kms-provider-bench.sock")
test, err := newTransformTest(b, encryptionConfig, false, "", nil)
test, err := newTransformTest(b, transformTestConfig{transformerConfigYAML: encryptionConfig})
if err != nil {
b.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}
@ -1279,7 +1279,7 @@ resources:
`
_ = kmsv2mock.NewBase64Plugin(b, "@kms-provider-bench-rest.sock")
test, err := newTransformTest(b, encryptionConfig, false, "", nil)
test, err := newTransformTest(b, transformTestConfig{transformerConfigYAML: encryptionConfig})
if err != nil {
b.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}
@ -1378,7 +1378,7 @@ resources:
storageConfig := storagebackend.NewDefaultConfig(path.Join(legacyDataEtcdPrefix, "registry"), nil)
storageConfig.Transport.ServerList = []string{framework.GetEtcdURL()}
test, err := newTransformTest(t, encryptionConfig, false, "", storageConfig)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: encryptionConfig, storageConfig: storageConfig})
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}

8
test/integration/controlplane/transformation/secrets_transformation_test.go
View File

@ -106,7 +106,7 @@ func TestSecretsShouldBeTransformed(t *testing.T) {
// TODO: add secretbox
}
for _, tt := range testCases {
test, err := newTransformTest(t, tt.transformerConfigContent, false, "", nil)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: tt.transformerConfigContent})
if err != nil {
t.Fatalf("failed to setup test for envelop %s, error was %v", tt.transformerPrefix, err)
continue
@ -195,7 +195,7 @@ func TestAllowUnsafeMalformedObjectDeletionFeature(t *testing.T) {
t.Run(fmt.Sprintf("%s/%t", string(genericfeatures.AllowUnsafeMalformedObjectDeletion), tc.featureEnabled), func(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AllowUnsafeMalformedObjectDeletion, tc.featureEnabled)
test, err := newTransformTest(t, aesGCMConfigYAML, true, "", nil)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: aesGCMConfigYAML, reload: true})
if err != nil {
t.Fatalf("failed to setup test for envelop %s, error was %v", aesGCMPrefix, err)
}
@ -498,7 +498,7 @@ func TestListCorruptObjects(t *testing.T) {
t.Run(fmt.Sprintf("%s/%t", string(genericfeatures.AllowUnsafeMalformedObjectDeletion), tc.featureEnabled), func(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AllowUnsafeMalformedObjectDeletion, tc.featureEnabled)
test, err := newTransformTest(t, aesGCMConfigYAML, true, "", nil)
test, err := newTransformTest(t, transformTestConfig{transformerConfigYAML: aesGCMConfigYAML, reload: true})
if err != nil {
t.Fatalf("failed to setup test for envelop %s, error was %v", aesGCMPrefix, err)
}
@ -651,7 +651,7 @@ func BenchmarkAESCBCEnvelopeWrite(b *testing.B) {
func runBenchmark(b *testing.B, transformerConfig string) {
b.StopTimer()
test, err := newTransformTest(b, transformerConfig, false, "", nil)
test, err := newTransformTest(b, transformTestConfig{transformerConfigYAML: transformerConfig})
if err != nil {
b.Fatalf("failed to setup benchmark for config %s, error was %v", transformerConfig, err)
}

30
test/integration/controlplane/transformation/transformation_test.go
View File

@ -87,27 +87,35 @@ type transformTest struct {
secret *corev1.Secret
}
func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool, configDir string, storageConfig *storagebackend.Config) (*transformTest, error) {
type transformTestConfig struct {
transformerConfigYAML string
reload bool
configDir string
storageConfig *storagebackend.Config
runtimeConfig []string
}
func newTransformTest(tb testing.TB, config transformTestConfig) (*transformTest, error) {
tCtx := ktesting.Init(tb)
if storageConfig == nil {
storageConfig = framework.SharedEtcd()
if config.storageConfig == nil {
config.storageConfig = framework.SharedEtcd()
}
e := transformTest{
TContext: tCtx,
transformerConfig: transformerConfigYAML,
storageConfig: storageConfig,
transformerConfig: config.transformerConfigYAML,
storageConfig: config.storageConfig,
}
var err error
// create config dir with provided config yaml
if transformerConfigYAML != "" && configDir == "" {
if config.transformerConfigYAML != "" && config.configDir == "" {
if e.configDir, err = e.createEncryptionConfig(); err != nil {
e.cleanUp()
return nil, fmt.Errorf("error while creating KubeAPIServer encryption config: %w", err)
}
} else {
// configDir already exists. api-server must be restarting with existing encryption config
e.configDir = configDir
e.configDir = config.configDir
}
configFile := filepath.Join(e.configDir, encryptionConfigFileName)
_, err = os.ReadFile(configFile)
@ -116,9 +124,13 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool,
return nil, fmt.Errorf("failed to read config file: %w", err)
}
flags := e.getEncryptionOptions(config.reload)
if len(config.runtimeConfig) > 0 {
flags = append(flags, "--runtime-config="+strings.Join(config.runtimeConfig, ","))
}
if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(
tb, nil,
e.getEncryptionOptions(reload), e.storageConfig); err != nil {
flags, e.storageConfig); err != nil {
e.cleanUp()
return nil, fmt.Errorf("failed to start KubeAPI server: %w", err)
}
@ -134,7 +146,7 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool,
return nil, err
}
if transformerConfigYAML != "" && reload {
if config.transformerConfigYAML != "" && config.reload {
// when reloading is enabled, this healthz endpoint is always present
mustBeHealthy(tCtx, "/kms-providers", "ok", e.kubeAPIServer.ClientConfig)
mustNotHaveLivez(tCtx, "/kms-providers", "404 page not found", e.kubeAPIServer.ClientConfig)

5
test/integration/metrics/metrics_test.go
View File

@ -26,6 +26,7 @@ import (
"testing"
"github.com/prometheus/common/model"
admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/endpoints/metrics"
@ -108,7 +109,9 @@ func TestAPIServerMetrics(t *testing.T) {
// KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE allows for APIs pending removal to not block tests
t.Setenv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE", "true")
s := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd())
flags := framework.DefaultTestServerFlags()
flags = append(flags, fmt.Sprintf("--runtime-config=%s=true", admissionregistrationv1beta1.SchemeGroupVersion))
s := kubeapiservertesting.StartTestServerOrDie(t, nil, flags, framework.SharedEtcd())
defer s.TearDownFn()
// Make a request to the apiserver to ensure there's at least one data point

2
test/integration/resourceclaim/feature_enable_disable_test.go
View File

@ -42,6 +42,7 @@ func TestEnableDisableDRAResourceClaimDeviceStatus(t *testing.T) {
// apiserver with the feature disabled
server1 := kubeapiservertesting.StartTestServerOrDie(t, apiServerOptions,
[]string{
fmt.Sprintf("--runtime-config=%s=true", v1beta1.SchemeGroupVersion),
fmt.Sprintf("--feature-gates=%s=true,%s=false", features.DynamicResourceAllocation, features.DRAResourceClaimDeviceStatus),
},
etcdOptions)
@ -114,6 +115,7 @@ func TestEnableDisableDRAResourceClaimDeviceStatus(t *testing.T) {
// apiserver with the feature enabled
server2 := kubeapiservertesting.StartTestServerOrDie(t, apiServerOptions,
[]string{
fmt.Sprintf("--runtime-config=%s=true", v1beta1.SchemeGroupVersion),
fmt.Sprintf("--feature-gates=%s=true,%s=true", features.DynamicResourceAllocation, features.DRAResourceClaimDeviceStatus),
},
etcdOptions)

9
test/integration/scheduler_perf/util.go
View File

@ -30,6 +30,8 @@ import (
"time"
v1 "k8s.io/api/core/v1"
resourceapialpha "k8s.io/api/resource/v1alpha3"
resourceapi "k8s.io/api/resource/v1beta1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
@ -86,11 +88,10 @@ func newDefaultComponentConfig() (*config.KubeSchedulerConfiguration, error) {
// Notes on rate limiter:
// - client rate limit is set to 5000.
func mustSetupCluster(tCtx ktesting.TContext, config *config.KubeSchedulerConfiguration, enabledFeatures map[featuregate.Feature]bool, outOfTreePluginRegistry frameworkruntime.Registry) (informers.SharedInformerFactory, ktesting.TContext) {
// No alpha APIs (overrides api/all=true in https://github.com/kubernetes/kubernetes/blob/d647d19f6aef811bace300eec96a67644ff303d4/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go#L136),
// except for DRA API group when needed.
runtimeConfig := []string{"api/alpha=false"}
var runtimeConfig []string
if enabledFeatures[features.DynamicResourceAllocation] {
runtimeConfig = append(runtimeConfig, "resource.k8s.io/v1alpha3=true")
runtimeConfig = append(runtimeConfig, fmt.Sprintf("%s=true", resourceapi.SchemeGroupVersion))
runtimeConfig = append(runtimeConfig, fmt.Sprintf("%s=true", resourceapialpha.SchemeGroupVersion))
}
customFlags := []string{
// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.

4
test/integration/storageversion/gc_test.go
View File

@ -52,7 +52,9 @@ const (
func TestStorageVersionGarbageCollection(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true)
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionAPI, true)
result := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd())
flags := framework.DefaultTestServerFlags()
flags = append(flags, fmt.Sprintf("--runtime-config=%s=true", apiserverinternalv1alpha1.SchemeGroupVersion))
result := kubeapiservertesting.StartTestServerOrDie(t, nil, flags, framework.SharedEtcd())
defer result.TearDownFn()
kubeclient, err := kubernetes.NewForConfig(result.ClientConfig)

4
test/integration/storageversion/storage_version_filter_test.go
View File

@ -148,7 +148,9 @@ func testBuiltinResourceRead(t *testing.T, cfg *rest.Config, shouldBlock bool) {
func TestStorageVersionBootstrap(t *testing.T) {
// Start server and create CRD
etcdConfig := framework.SharedEtcd()
server := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), etcdConfig)
flags := framework.DefaultTestServerFlags()
flags = append(flags, "--runtime-config=api/all=true")
server := kubeapiservertesting.StartTestServerOrDie(t, nil, flags, etcdConfig)
etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(server.ClientConfig), false, etcd.GetCustomResourceDefinitionData()[0])
server.TearDownFn()

1
test/integration/storageversionmigrator/util.go
View File

@ -275,6 +275,7 @@ func svmSetup(ctx context.Context, t *testing.T) *svmTest {
"--audit-log-mode", "blocking",
"--audit-log-path", logFile.Name(),
"--authorization-mode=RBAC",
fmt.Sprintf("--runtime-config=%s=true", svmv1alpha1.SchemeGroupVersion),
}
storageConfig := framework.SharedEtcd()
server := kubeapiservertesting.StartTestServerOrDie(t, nil, apiServerFlags, storageConfig)