Merge pull request #124250 from aramase/aramase/t/transformation_tests_parallel

Run `controlplane/transformation` integration tests in parallel
This commit is contained in:
Kubernetes Prow Robot 2024-04-22 11:53:21 -07:00 committed by GitHub
commit 13784cfa60
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 142 additions and 105 deletions

View File

@ -131,7 +131,8 @@ func (r envelope) plainTextPayload(secretETCDPath string) ([]byte, error) {
func TestKMSProvider(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
encryptionConfig := `
socketPath := getSocketPath()
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -141,10 +142,10 @@ resources:
- kms:
name: kms-provider
cachesize: 1000
endpoint: unix:///@kms-provider.sock
`
endpoint: unix:///%s
`, socketPath)
providerName := "kms-provider"
pluginMock := mock.NewBase64Plugin(t, "@kms-provider.sock")
pluginMock := mock.NewBase64Plugin(t, socketPath)
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
@ -312,8 +313,9 @@ func TestEncryptionConfigHotReload(t *testing.T) {
// this makes the test super responsive. It's set to a default of 1 minute.
encryptionconfigcontroller.EncryptionConfigFileChangePollDuration = time.Second
socketPath := getSocketPath()
storageConfig := framework.SharedEtcd()
encryptionConfig := `
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -323,11 +325,11 @@ resources:
- kms:
name: kms-provider
cachesize: 1000
endpoint: unix:///@kms-provider.sock
`
endpoint: unix:///%s
`, socketPath)
genericapiserver.SetHostnameFuncForTests("testAPIServerID")
_ = mock.NewBase64Plugin(t, "@kms-provider.sock")
_ = mock.NewBase64Plugin(t, socketPath)
var restarted bool
test, err := newTransformTest(t, encryptionConfig, true, "", storageConfig)
if err != nil {
@ -379,7 +381,8 @@ resources:
// test if hot reload controller is healthy
mustBeHealthy(t, "/poststarthook/start-encryption-provider-config-automatic-reload", "ok", test.kubeAPIServer.ClientConfig)
encryptionConfigWithNewProvider := `
newKMSProviderSocketPath := getSocketPath()
encryptionConfigWithNewProvider := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -389,22 +392,22 @@ resources:
- kms:
name: new-kms-provider-for-secrets
cachesize: 1000
endpoint: unix:///@new-kms-provider.sock
endpoint: unix:///%s
- kms:
name: kms-provider
cachesize: 1000
endpoint: unix:///@kms-provider.sock
endpoint: unix:///%s
- resources:
- configmaps
providers:
- kms:
name: new-kms-provider-for-configmaps
cachesize: 1000
endpoint: unix:///@new-kms-provider.sock
endpoint: unix:///%s
- identity: {}
`
`, newKMSProviderSocketPath, socketPath, newKMSProviderSocketPath)
// start new KMS Plugin
_ = mock.NewBase64Plugin(t, "@new-kms-provider.sock")
_ = mock.NewBase64Plugin(t, newKMSProviderSocketPath)
// update encryption config
updateFile(t, test.configDir, encryptionConfigFileName, []byte(encryptionConfigWithNewProvider))
@ -481,12 +484,13 @@ resources:
t.Fatalf("expected configmap to be prefixed with %s, but got %s", wantPrefixForConfigmaps, rawConfigmapEnvelope.Kvs[0].Value)
}
newEncryptAllProviderSocketPath := getSocketPath()
// remove old KMS provider
// verifyIfKMSTransformersSwapped sometimes passes even before the changes in the encryption config file are observed.
// this causes the metrics tests to fail, which validate two config changes.
// this may happen when an existing KMS provider is already running (e.g., new-kms-provider-for-secrets in this case).
// to ensure that the changes are observed, we added one more provider (kms-provider-to-encrypt-all) and are validating it in verifyIfKMSTransformersSwapped.
encryptionConfigWithoutOldProvider := `
encryptionConfigWithoutOldProvider := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -496,25 +500,25 @@ resources:
- kms:
name: new-kms-provider-for-secrets
cachesize: 1000
endpoint: unix:///@new-kms-provider.sock
endpoint: unix:///%s
- resources:
- configmaps
providers:
- kms:
name: new-kms-provider-for-configmaps
cachesize: 1000
endpoint: unix:///@new-kms-provider.sock
endpoint: unix:///%s
- resources:
- '*.*'
providers:
- kms:
name: kms-provider-to-encrypt-all
cachesize: 1000
endpoint: unix:///@new-encrypt-all-kms-provider.sock
endpoint: unix:///%s
- identity: {}
`
`, newKMSProviderSocketPath, newKMSProviderSocketPath, newEncryptAllProviderSocketPath)
// start new KMS Plugin
_ = mock.NewBase64Plugin(t, "@new-encrypt-all-kms-provider.sock")
_ = mock.NewBase64Plugin(t, newEncryptAllProviderSocketPath)
// update encryption config and wait for hot reload
updateFile(t, test.configDir, encryptionConfigFileName, []byte(encryptionConfigWithoutOldProvider))
@ -605,7 +609,8 @@ resources:
}
func TestEncryptAll(t *testing.T) {
encryptionConfig := `
socketPath := getSocketPath()
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -615,11 +620,11 @@ resources:
- kms:
name: encrypt-all-kms-provider
cachesize: 1000
endpoint: unix:///@encrypt-all-kms-provider.sock
`
endpoint: unix:///%s
`, socketPath)
t.Run("encrypt all resources", func(t *testing.T) {
_ = mock.NewBase64Plugin(t, "@encrypt-all-kms-provider.sock")
_ = mock.NewBase64Plugin(t, socketPath)
// To ensure we are checking all REST resources
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, "AllAlpha", true)()
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, "AllBeta", true)()
@ -724,7 +729,9 @@ resources:
}
func TestEncryptAllWithWildcard(t *testing.T) {
encryptionConfig := `
socketPath1 := getSocketPath()
socketPath2 := getSocketPath()
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -738,17 +745,17 @@ resources:
- kms:
name: kms-provider
cachesize: 1000
endpoint: unix:///@kms-provider.sock
endpoint: unix:///%s
- resources:
- '*.*'
providers:
- kms:
name: encrypt-all-kms-provider
cachesize: 1000
endpoint: unix:///@encrypt-all-kms-provider.sock
`
_ = mock.NewBase64Plugin(t, "@kms-provider.sock")
_ = mock.NewBase64Plugin(t, "@encrypt-all-kms-provider.sock")
endpoint: unix:///%s
`, socketPath1, socketPath2)
_ = mock.NewBase64Plugin(t, socketPath1)
_ = mock.NewBase64Plugin(t, socketPath2)
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
@ -883,8 +890,12 @@ func TestEncryptionConfigHotReloadFilePolling(t *testing.T) {
}
for _, tc := range testCases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
encryptionConfig := `
t.Parallel()
socketPath := getSocketPath()
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -894,10 +905,10 @@ resources:
- kms:
name: kms-provider
cachesize: 1000
endpoint: unix:///@kms-provider.sock
endpoint: unix:///%s
timeout: 1s
`
_ = mock.NewBase64Plugin(t, "@kms-provider.sock")
`, socketPath)
_ = mock.NewBase64Plugin(t, socketPath)
test, err := newTransformTest(t, encryptionConfig, true, "", nil)
if err != nil {
@ -913,7 +924,8 @@ resources:
// test if hot reload controller is healthy
mustBeHealthy(t, "/poststarthook/start-encryption-provider-config-automatic-reload", "ok", test.kubeAPIServer.ClientConfig)
encryptionConfigWithNewProvider := `
newKMSProviderSocketPath := getSocketPath()
encryptionConfigWithNewProvider := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -923,12 +935,12 @@ resources:
- kms:
name: new-kms-provider-for-secrets
cachesize: 1000
endpoint: unix:///@new-kms-provider.sock
endpoint: unix:///%s
timeout: 1s
- kms:
name: kms-provider
cachesize: 1000
endpoint: unix:///@kms-provider.sock
endpoint: unix:///%s
timeout: 1s
- resources:
- configmaps
@ -936,12 +948,12 @@ resources:
- kms:
name: new-kms-provider-for-configmaps
cachesize: 1000
endpoint: unix:///@new-kms-provider.sock
endpoint: unix:///%s
timeout: 1s
- identity: {}
`
`, newKMSProviderSocketPath, socketPath, newKMSProviderSocketPath)
// start new KMS Plugin
_ = mock.NewBase64Plugin(t, "@new-kms-provider.sock")
_ = mock.NewBase64Plugin(t, newKMSProviderSocketPath)
// update encryption config
if err := tc.updateFile(filepath.Join(test.configDir, encryptionConfigFileName), encryptionConfigWithNewProvider); err != nil {
t.Fatalf("failed to update encryption config, err: %v", err)
@ -1093,7 +1105,9 @@ func updateFile(t *testing.T, configDir, filename string, newContent []byte) {
func TestKMSHealthz(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
encryptionConfig := `
socketPath1 := getSocketPath()
socketPath2 := getSocketPath()
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -1102,14 +1116,14 @@ resources:
providers:
- kms:
name: provider-1
endpoint: unix:///@kms-provider-1.sock
endpoint: unix:///%s
- kms:
name: provider-2
endpoint: unix:///@kms-provider-2.sock
`
endpoint: unix:///%s
`, socketPath1, socketPath2)
pluginMock1 := mock.NewBase64Plugin(t, "@kms-provider-1.sock")
pluginMock2 := mock.NewBase64Plugin(t, "@kms-provider-2.sock")
pluginMock1 := mock.NewBase64Plugin(t, socketPath1)
pluginMock2 := mock.NewBase64Plugin(t, socketPath2)
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
if err != nil {
@ -1156,7 +1170,9 @@ resources:
func TestKMSHealthzWithReload(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
encryptionConfig := `
socketPath1 := getSocketPath()
socketPath2 := getSocketPath()
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -1165,14 +1181,14 @@ resources:
providers:
- kms:
name: provider-1
endpoint: unix:///@kms-provider-1.sock
endpoint: unix:///%s
- kms:
name: provider-2
endpoint: unix:///@kms-provider-2.sock
`
endpoint: unix:///%s
`, socketPath1, socketPath2)
pluginMock1 := mock.NewBase64Plugin(t, "@kms-provider-1.sock")
pluginMock2 := mock.NewBase64Plugin(t, "@kms-provider-2.sock")
pluginMock1 := mock.NewBase64Plugin(t, socketPath1)
pluginMock2 := mock.NewBase64Plugin(t, socketPath2)
test, err := newTransformTest(t, encryptionConfig, true, "", nil)
if err != nil {

View File

@ -179,7 +179,8 @@ func TestDefaultValues(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
t.Cleanup(cancel)
encryptionConfig := `
socketPath := getSocketPath()
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -189,9 +190,9 @@ resources:
- kms:
apiVersion: v2
name: kms-provider
endpoint: unix:///@kms-provider.sock
`
_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
endpoint: unix:///%s
`, socketPath)
_ = kmsv2mock.NewBase64Plugin(t, socketPath)
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
if err != nil {
@ -261,7 +262,8 @@ func TestKMSv2Provider(t *testing.T) {
}
func testKMSv2Provider(t *testing.T, useSeed bool) {
encryptionConfig := `
socketPath := getSocketPath()
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -271,11 +273,11 @@ resources:
- kms:
apiVersion: v2
name: kms-provider
endpoint: unix:///@kms-provider.sock
`
endpoint: unix:///%s
`, socketPath)
genericapiserver.SetHostnameFuncForTests("testAPIServerID")
providerName := "kms-provider"
pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
pluginMock := kmsv2mock.NewBase64Plugin(t, socketPath)
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
if err != nil {
@ -410,7 +412,9 @@ func TestKMSv2ProviderKeyIDStaleness(t *testing.T) {
}
func testKMSv2ProviderKeyIDStaleness(t *testing.T) {
encryptionConfig := `
t.Parallel()
socketPath := getSocketPath()
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -421,9 +425,9 @@ resources:
- kms:
apiVersion: v2
name: kms-provider
endpoint: unix:///@kms-provider.sock
`
pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
endpoint: unix:///%s
`, socketPath)
pluginMock := kmsv2mock.NewBase64Plugin(t, socketPath)
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
if err != nil {
@ -703,7 +707,8 @@ func testKMSv2ProviderDEKSourceReuse(t *testing.T, f checkFunc) {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
t.Cleanup(cancel)
encryptionConfig := `
socketPath := getSocketPath()
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -713,9 +718,9 @@ resources:
- kms:
apiVersion: v2
name: kms-provider
endpoint: unix:///@kms-provider.sock
`
_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
endpoint: unix:///%s
`, socketPath)
_ = kmsv2mock.NewBase64Plugin(t, socketPath)
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
if err != nil {
@ -835,7 +840,9 @@ func assertPodDEKSources(ctx context.Context, t *testing.T, config storagebacken
func TestKMSv2Healthz(t *testing.T) {
defer encryptionconfig.SetKDFForTests(randomBool())()
encryptionConfig := `
socketPath1 := getSocketPath()
socketPath2 := getSocketPath()
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -845,15 +852,15 @@ resources:
- kms:
apiVersion: v2
name: provider-1
endpoint: unix:///@kms-provider-1.sock
endpoint: unix:///%s
- kms:
apiVersion: v2
name: provider-2
endpoint: unix:///@kms-provider-2.sock
`
endpoint: unix:///%s
`, socketPath1, socketPath2)
pluginMock1 := kmsv2mock.NewBase64Plugin(t, "@kms-provider-1.sock")
pluginMock2 := kmsv2mock.NewBase64Plugin(t, "@kms-provider-2.sock")
pluginMock1 := kmsv2mock.NewBase64Plugin(t, socketPath1)
pluginMock2 := kmsv2mock.NewBase64Plugin(t, socketPath2)
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
if err != nil {
@ -910,11 +917,12 @@ func TestKMSv2SingleService(t *testing.T) {
encryptionconfig.EnvelopeKMSv2ServiceFactory = origEnvelopeKMSv2ServiceFactory
})
socketPath := getSocketPath()
// check resources provided by the three servers that we have wired together
// - pods and config maps from KAS
// - CRDs and CRs from API extensions
// - API services from aggregator
encryptionConfig := `
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -928,10 +936,10 @@ resources:
- kms:
apiVersion: v2
name: kms-provider
endpoint: unix:///@kms-provider.sock
`
endpoint: unix:///%s
`, socketPath)
_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
_ = kmsv2mock.NewBase64Plugin(t, socketPath)
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
if err != nil {
@ -971,7 +979,8 @@ resources:
// 2. After a restart, loading a encryptionConfig with the same KMSv2 plugin from 1 should work,
// decryption of data encrypted with v2 should work
func TestKMSv2FeatureFlag(t *testing.T) {
encryptionConfig := `
socketPath := getSocketPath()
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -981,10 +990,10 @@ resources:
- kms:
apiVersion: v2
name: kms-provider
endpoint: unix:///@kms-provider.sock
`
endpoint: unix:///%s
`, socketPath)
providerName := "kms-provider"
pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
pluginMock := kmsv2mock.NewBase64Plugin(t, socketPath)
storageConfig := framework.SharedEtcd()
// KMSv2 is enabled by default. Loading a encryptionConfig with KMSv2 should work
@ -1094,7 +1103,8 @@ func BenchmarkKMSv2KDF(b *testing.B) {
ctx = request.WithNamespace(ctx, testNamespace)
encryptionConfig := `
socketPath := getSocketPath()
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -1104,9 +1114,9 @@ resources:
- kms:
apiVersion: v2
name: kms-provider
endpoint: unix:///@kms-provider.sock
`
_ = kmsv2mock.NewBase64Plugin(b, "@kms-provider.sock")
endpoint: unix:///%s
`, socketPath)
_ = kmsv2mock.NewBase64Plugin(b, socketPath)
test, err := newTransformTest(b, encryptionConfig, false, "", nil)
if err != nil {
@ -1247,7 +1257,8 @@ func BenchmarkKMSv2REST(b *testing.B) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
b.Cleanup(cancel)
encryptionConfig := `
socketPath := getSocketPath()
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -1257,9 +1268,9 @@ resources:
- kms:
apiVersion: v2
name: kms-provider
endpoint: unix:///@kms-provider.sock
`
_ = kmsv2mock.NewBase64Plugin(b, "@kms-provider.sock")
endpoint: unix:///%s
`, socketPath)
_ = kmsv2mock.NewBase64Plugin(b, socketPath)
test, err := newTransformTest(b, encryptionConfig, false, "", nil)
if err != nil {
@ -1337,7 +1348,9 @@ func TestKMSv2ProviderLegacyData(t *testing.T) {
}
func testKMSv2ProviderLegacyData(t *testing.T) {
encryptionConfig := `
t.Parallel()
socketPath := getSocketPath()
encryptionConfig := fmt.Sprintf(`
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
@ -1347,10 +1360,10 @@ resources:
- kms:
apiVersion: v2
name: kms-provider
endpoint: unix:///@kms-provider.sock
`
endpoint: unix:///%s
`, socketPath)
_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
_ = kmsv2mock.NewBase64Plugin(t, socketPath)
// the value.Context.AuthenticatedData during read is the etcd storage path of the associated resource
// thus we need to manually construct the storage config so that we can have a static path

View File

@ -85,17 +85,20 @@ func TestSecretsShouldBeTransformed(t *testing.T) {
// TODO: add secretbox
}
for _, tt := range testCases {
test, err := newTransformTest(t, tt.transformerConfigContent, false, "", nil)
if err != nil {
t.Fatalf("failed to setup test for envelop %s, error was %v", tt.transformerPrefix, err)
continue
}
test.secret, err = test.createSecret(testSecret, testNamespace)
if err != nil {
t.Fatalf("Failed to create test secret, error: %v", err)
}
test.runResource(test.logger, tt.unSealFunc, tt.transformerPrefix, "", "v1", "secrets", test.secret.Name, test.secret.Namespace)
test.cleanUp()
tt := tt
t.Run(tt.transformerPrefix, func(t *testing.T) {
t.Parallel()
test, err := newTransformTest(t, tt.transformerConfigContent, false, "", nil)
if err != nil {
t.Fatalf("failed to setup test for envelop %s, error was %v", tt.transformerPrefix, err)
}
test.secret, err = test.createSecret(testSecret, testNamespace)
if err != nil {
t.Fatalf("Failed to create test secret, error: %v", err)
}
test.runResource(test.logger, tt.unSealFunc, tt.transformerPrefix, "", "v1", "secrets", test.secret.Name, test.secret.Namespace)
test.cleanUp()
})
}
}

View File

@ -37,6 +37,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/rand"
"k8s.io/apimachinery/pkg/util/wait"
apiserverv1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
"k8s.io/apiserver/pkg/storage/storagebackend"
@ -638,3 +639,7 @@ func getLivez(checkName string, clientConfig *rest.Config, excludes ...string) (
body, err := req.DoRaw(context.TODO()) // we can still have a response body during an error case
return string(body), err == nil, nil
}
func getSocketPath() string {
return fmt.Sprintf("@%s.sock", rand.String(10))
}