diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go index d7b4fc05a70..6056975cd10 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go @@ -107,24 +107,35 @@ const ( var codecs serializer.CodecFactory -// this atomic bool allows us to swap enablement of the KMSv2KDF feature in tests +// this map allows us to swap enablement of the KMSv2KDF feature in tests // as the feature gate is now locked to true starting with v1.29 // Note: it cannot be set by an end user -var kdfDisabled atomic.Bool +// KDF enablement is tracked per KMS provider to allow tests to run in parallel. +var kdfEnabledPerKMS sync.Map // map[string]bool, KMS name -> KDF enabled // this function should only be called in tests to swap enablement of the KMSv2KDF feature -func SetKDFForTests(b bool) func() { - kdfDisabled.Store(!b) - return func() { - kdfDisabled.Store(false) +// Caller must guarantee that all KMS providers have distinct names across all tests. +func SetKDFForTests(kmsName string, b bool) func() { + if len(kmsName) == 0 { // guarantee that GetKDF("") returns the default value + panic("empty KMS name used in test") } + if _, loaded := kdfEnabledPerKMS.LoadOrStore(kmsName, b); loaded { + panic("duplicate KMS name used in test") + } + return func() { kdfEnabledPerKMS.Delete(kmsName) } } // this function should be used to determine enablement of the KMSv2KDF feature // instead of getting it from DefaultFeatureGate as the feature gate is now locked // to true starting with v1.29 -func GetKDF() bool { - return !kdfDisabled.Load() +// to allow integration tests to run in parallel, this "feature flag" can be set +// per KMS provider as long as all providers use distinct names. +func GetKDF(kmsName string) bool { + kdfEnabled, ok := kdfEnabledPerKMS.Load(kmsName) + if !ok { + return true // explicit config is missing, but KDF is enabled by default + } + return kdfEnabled.(bool) // this will panic if a non-bool ever gets stored, which should never happen } func init() { @@ -390,7 +401,7 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey // this gate can only change during tests, but the check is cheap enough to always make // this allows us to easily exercise both modes without restarting the API server // TODO integration test that this dynamically takes effect - useSeed := GetKDF() + useSeed := GetKDF(h.name) stateUseSeed := state.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED // state is valid and status keyID is unchanged from when we generated this DEK/seed so there is no need to rotate it diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go index 26c259530e1..45684866aa0 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go @@ -1851,7 +1851,7 @@ func TestComputeEncryptionConfigHash(t *testing.T) { } func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) { - defaultUseSeed := GetKDF() + defaultUseSeed := GetKDF("") origNowFunc := envelopekmsv2.NowFunc now := origNowFunc() // freeze time @@ -2074,9 +2074,10 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) { `encryptKeyIDHash="", stateKeyIDHash="sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35", expirationTimestamp=` + now.Format(time.RFC3339), }, } - for _, tt := range tests { + for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { - defer SetKDFForTests(tt.useSeed)() + kmsName := fmt.Sprintf("panda-%d", i) + defer SetKDFForTests(kmsName, tt.useSeed)() var buf bytes.Buffer klog.SetOutput(&buf) @@ -2084,7 +2085,7 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) { ctx := testContext(t) h := &kmsv2PluginProbe{ - name: "panda", + name: kmsName, service: tt.service, } h.state.Store(&tt.state) @@ -2133,7 +2134,7 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) { if _, stateErr := h.getCurrentState(); stateErr == nil || err == nil { transformer := envelopekmsv2.NewEnvelopeTransformer( &testKMSv2EnvelopeService{err: fmt.Errorf("broken")}, // not called - "panda", + kmsName, h.getCurrentState, "", ) diff --git a/test/integration/controlplane/transformation/kmsv2_transformation_test.go b/test/integration/controlplane/transformation/kmsv2_transformation_test.go index 94cdd0f60da..31d684a6640 100644 --- a/test/integration/controlplane/transformation/kmsv2_transformation_test.go +++ b/test/integration/controlplane/transformation/kmsv2_transformation_test.go @@ -165,12 +165,15 @@ func (r envelopekmsv2) plainTextPayload(secretETCDPath string) ([]byte, error) { // TestDefaultValues tests default flag values without setting any of the feature flags or // calling SetKDFForTests, and assert that the data stored in etcd is using KDF func TestDefaultValues(t *testing.T) { - if encryptionconfig.GetKDF() != true { + if encryptionconfig.GetKDF("") != true { t.Fatalf("without updating the feature flags, default value of KMSv2KDF should be enabled.") } if utilfeature.DefaultFeatureGate.Enabled(features.KMSv1) != false { t.Fatalf("without updating the feature flags, default value of KMSv1 should be disabled.") } + + t.Parallel() // the feature gate check above uses global state, so only run in parallel after that point + // since encryptionconfig.GetKDF() is true by default, following test should verify if // object.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED ctx, cancel := context.WithTimeout(context.Background(), time.Minute) @@ -185,10 +188,10 @@ resources: providers: - kms: apiVersion: v2 - name: kms-provider - endpoint: unix:///@kms-provider.sock + name: kms-provider-defaults + endpoint: unix:///@kms-provider-defaults.sock ` - _ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock") + _ = kmsv2mock.NewBase64Plugin(t, "@kms-provider-defaults.sock") test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { @@ -228,7 +231,7 @@ resources: t.Fatalf("expected 1 KVs, but got %d", len(response.Kvs)) } object := kmstypes.EncryptedObject{} - v := bytes.TrimPrefix(response.Kvs[0].Value, []byte("k8s:enc:kms:v2:kms-provider:")) + v := bytes.TrimPrefix(response.Kvs[0].Value, []byte("k8s:enc:kms:v2:kms-provider-defaults:")) if err := proto.Unmarshal(v, &object); err != nil { t.Fatal(err) } @@ -245,19 +248,22 @@ resources: // 4. The cipherTextPayload (ex. Secret) should be encrypted via AES GCM transform / extended nonce GCM // 5. kmstypes.EncryptedObject structure should be serialized and deposited in ETCD func TestKMSv2Provider(t *testing.T) { - defaultUseSeed := encryptionconfig.GetKDF() + // this test cannot be run in parallel because both the API server ID and the metrics are global state + defaultUseSeed := encryptionconfig.GetKDF("") t.Run("regular gcm", func(t *testing.T) { - defer encryptionconfig.SetKDFForTests(false)() - testKMSv2Provider(t, !defaultUseSeed) + kmsName := "kms-provider-v2-false" + defer encryptionconfig.SetKDFForTests(kmsName, false)() + testKMSv2Provider(t, kmsName, !defaultUseSeed) }) t.Run("extended nonce gcm", func(t *testing.T) { - defer encryptionconfig.SetKDFForTests(true)() - testKMSv2Provider(t, defaultUseSeed) + kmsName := "kms-provider-v2-true" + defer encryptionconfig.SetKDFForTests(kmsName, true)() + testKMSv2Provider(t, kmsName, defaultUseSeed) }) } -func testKMSv2Provider(t *testing.T, useSeed bool) { +func testKMSv2Provider(t *testing.T, kmsName string, useSeed bool) { encryptionConfig := ` kind: EncryptionConfiguration apiVersion: apiserver.config.k8s.io/v1 @@ -267,12 +273,11 @@ resources: providers: - kms: apiVersion: v2 - name: kms-provider - endpoint: unix:///@kms-provider.sock + name: ` + kmsName + ` + endpoint: unix:///@` + kmsName + `.sock ` genericapiserver.SetHostnameFuncForTests("testAPIServerID") - providerName := "kms-provider" - pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock") + pluginMock := kmsv2mock.NewBase64Plugin(t, "@"+kmsName+".sock") test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { @@ -296,13 +301,13 @@ resources: // assert that the metrics we collect during the test run match expectations wantMetricStrings := []string{ - `apiserver_envelope_encryption_dek_source_cache_size{provider_name="kms-provider"} 1`, - `apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} FP`, - `apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} FP`, - `apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} 2`, - `apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} 1`, - `apiserver_storage_transformation_operations_total{resource="secrets",status="OK",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v2:kms-provider:"} 2`, - `apiserver_storage_transformation_operations_total{resource="secrets",status="OK",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v2:kms-provider:"} 1`, + `apiserver_envelope_encryption_dek_source_cache_size{provider_name="` + kmsName + `"} 1`, + `apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="` + kmsName + `",transformation_type="from_storage"} FP`, + `apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="` + kmsName + `",transformation_type="to_storage"} FP`, + `apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="` + kmsName + `",transformation_type="from_storage"} 2`, + `apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="` + kmsName + `",transformation_type="to_storage"} 1`, + `apiserver_storage_transformation_operations_total{resource="secrets",status="OK",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v2:` + kmsName + `:"} 2`, + `apiserver_storage_transformation_operations_total{resource="secrets",status="OK",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v2:` + kmsName + `:"} 1`, } defer func() { body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx) @@ -343,7 +348,7 @@ resources: } envelopeData := envelopekmsv2{ - providerName: providerName, + providerName: kmsName, rawEnvelope: rawEnvelope, plainTextDEKSource: plainTextDEKSource, useSeed: useSeed, @@ -399,16 +404,19 @@ resources: // 6. When kms-plugin is down, expect creation of new pod and encryption to fail once the DEK/seed is invalid // 7. when kms-plugin is down, no-op update for a pod should succeed and not result in RV change even once the DEK/seed is valid func TestKMSv2ProviderKeyIDStaleness(t *testing.T) { + t.Parallel() t.Run("regular gcm", func(t *testing.T) { - defer encryptionconfig.SetKDFForTests(false)() - testKMSv2ProviderKeyIDStaleness(t) + t.Parallel() + kmsName := "kms-provider-key-id-stale-false" + testKMSv2ProviderKeyIDStaleness(t, kmsName, encryptionconfig.SetKDFForTests(kmsName, false)) }) t.Run("extended nonce gcm", func(t *testing.T) { - testKMSv2ProviderKeyIDStaleness(t) + t.Parallel() + testKMSv2ProviderKeyIDStaleness(t, "kms-provider-key-id-stale-true", func() {}) }) } -func testKMSv2ProviderKeyIDStaleness(t *testing.T) { +func testKMSv2ProviderKeyIDStaleness(t *testing.T, kmsName string, resetSetKDFForTests func()) { encryptionConfig := ` kind: EncryptionConfiguration apiVersion: apiserver.config.k8s.io/v1 @@ -419,10 +427,10 @@ resources: providers: - kms: apiVersion: v2 - name: kms-provider - endpoint: unix:///@kms-provider.sock + name: ` + kmsName + ` + endpoint: unix:///@` + kmsName + `.sock ` - pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock") + pluginMock := kmsv2mock.NewBase64Plugin(t, "@"+kmsName+".sock") test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { @@ -451,7 +459,7 @@ resources: ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) t.Cleanup(cancel) - useSeed := encryptionconfig.GetKDF() + useSeed := encryptionconfig.GetKDF(kmsName) var firstEncryptedDEKSource []byte var f checkFunc @@ -480,7 +488,7 @@ resources: } } assertPodDEKSources(ctx, t, test.kubeAPIServer.ServerOpts.Etcd.StorageConfig, - 1, 1, "k8s:enc:kms:v2:kms-provider:", f, + 1, 1, kmsName, "k8s:enc:kms:v2:"+kmsName+":", f, ) if len(firstEncryptedDEKSource) == 0 { t.Fatal("unexpected empty DEK or seed") @@ -618,7 +626,7 @@ resources: } assertPodDEKSources(ctx, t, test.kubeAPIServer.ServerOpts.Etcd.StorageConfig, - 1, 1, "k8s:enc:kms:v2:kms-provider:", checkDEK, + 1, 1, kmsName, "k8s:enc:kms:v2:"+kmsName+":", checkDEK, ) // fix plugin and wait for new writes to start working again @@ -646,8 +654,9 @@ resources: if version7 != version8 { t.Fatalf("Resource version should not have changed after plugin health is restored. old pod: %v, new pod: %v", updatedNewPod, updatedNewPod2) } + resetSetKDFForTests() // prevent the config flip from panic-ing // flip the current config - defer encryptionconfig.SetKDFForTests(!useSeed)() + defer encryptionconfig.SetKDFForTests(kmsName, !useSeed)() // 9. confirm that no-op update for a pod results in RV change due to KDF config change var version9 string @@ -670,9 +679,12 @@ resources: } func TestKMSv2ProviderDEKSourceReuse(t *testing.T) { + t.Parallel() t.Run("regular gcm", func(t *testing.T) { - defer encryptionconfig.SetKDFForTests(false)() - testKMSv2ProviderDEKSourceReuse(t, + t.Parallel() + kmsName := "kms-provider-dek-reuse-false" + defer encryptionconfig.SetKDFForTests(kmsName, false)() + testKMSv2ProviderDEKSourceReuse(t, kmsName, func(i int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject) { if obj.KeyID != "1" { t.Errorf("key %s: want key ID %s, got %s", etcdKey, "1", obj.KeyID) @@ -687,8 +699,10 @@ func TestKMSv2ProviderDEKSourceReuse(t *testing.T) { ) }) t.Run("extended nonce gcm", func(t *testing.T) { - defer encryptionconfig.SetKDFForTests(true)() - testKMSv2ProviderDEKSourceReuse(t, + t.Parallel() + kmsName := "kms-provider-dek-reuse-true" + defer encryptionconfig.SetKDFForTests(kmsName, true)() + testKMSv2ProviderDEKSourceReuse(t, kmsName, func(_ int, _ uint64, etcdKey string, obj kmstypes.EncryptedObject) { if obj.KeyID != "1" { t.Errorf("key %s: want key ID %s, got %s", etcdKey, "1", obj.KeyID) @@ -698,7 +712,7 @@ func TestKMSv2ProviderDEKSourceReuse(t *testing.T) { }) } -func testKMSv2ProviderDEKSourceReuse(t *testing.T, f checkFunc) { +func testKMSv2ProviderDEKSourceReuse(t *testing.T, kmsName string, f checkFunc) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) t.Cleanup(cancel) @@ -711,10 +725,10 @@ resources: providers: - kms: apiVersion: v2 - name: kms-provider - endpoint: unix:///@kms-provider.sock + name: ` + kmsName + ` + endpoint: unix:///@` + kmsName + `.sock ` - _ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock") + _ = kmsv2mock.NewBase64Plugin(t, "@"+kmsName+".sock") test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { @@ -746,13 +760,13 @@ resources: assertPodDEKSources(ctx, t, test.kubeAPIServer.ServerOpts.Etcd.StorageConfig, podCount, 1, // key ID does not change during the test so we should only have a single DEK - "k8s:enc:kms:v2:kms-provider:", f, + kmsName, "k8s:enc:kms:v2:"+kmsName+":", f, ) } type checkFunc func(i int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject) -func assertPodDEKSources(ctx context.Context, t *testing.T, config storagebackend.Config, podCount, dekSourcesCount int, kmsPrefix string, f checkFunc) { +func assertPodDEKSources(ctx context.Context, t *testing.T, config storagebackend.Config, podCount, dekSourcesCount int, kmsName, kmsPrefix string, f checkFunc) { t.Helper() rawClient, etcdClient, err := integration.GetEtcdClients(config.Transport) @@ -770,7 +784,7 @@ func assertPodDEKSources(ctx context.Context, t *testing.T, config storagebacken t.Fatalf("expected %d KVs, but got %d", podCount, len(response.Kvs)) } - useSeed := encryptionconfig.GetKDF() + useSeed := encryptionconfig.GetKDF(kmsName) out := make([]kmstypes.EncryptedObject, len(response.Kvs)) for i, kv := range response.Kvs { @@ -832,7 +846,10 @@ func assertPodDEKSources(ctx context.Context, t *testing.T, config storagebacken } func TestKMSv2Healthz(t *testing.T) { - defer encryptionconfig.SetKDFForTests(randomBool())() + t.Parallel() + + defer encryptionconfig.SetKDFForTests("provider-1", randomBool())() + defer encryptionconfig.SetKDFForTests("provider-2", randomBool())() encryptionConfig := ` kind: EncryptionConfiguration @@ -897,7 +914,7 @@ resources: } func TestKMSv2SingleService(t *testing.T) { - defer encryptionconfig.SetKDFForTests(randomBool())() + defer encryptionconfig.SetKDFForTests("kms-provider-single-service", randomBool())() // this test must run serially due to the global state below var kmsv2Calls int origEnvelopeKMSv2ServiceFactory := encryptionconfig.EnvelopeKMSv2ServiceFactory @@ -926,11 +943,11 @@ resources: providers: - kms: apiVersion: v2 - name: kms-provider - endpoint: unix:///@kms-provider.sock + name: kms-provider-single-service + endpoint: unix:///@kms-provider-single-service.sock ` - _ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock") + _ = kmsv2mock.NewBase64Plugin(t, "@kms-provider-single-service.sock") test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { @@ -970,6 +987,8 @@ resources: // 2. After a restart, loading a encryptionConfig with the same KMSv2 plugin from 1 should work, // decryption of data encrypted with v2 should work func TestKMSv2FeatureFlag(t *testing.T) { + t.Parallel() + encryptionConfig := ` kind: EncryptionConfiguration apiVersion: apiserver.config.k8s.io/v1 @@ -979,11 +998,11 @@ resources: providers: - kms: apiVersion: v2 - name: kms-provider - endpoint: unix:///@kms-provider.sock + name: kms-provider-feature-flag + endpoint: unix:///@kms-provider-feature-flag.sock ` - providerName := "kms-provider" - pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock") + providerName := "kms-provider-feature-flag" + pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider-feature-flag.sock") storageConfig := framework.SharedEtcd() // KMSv2 is enabled by default. Loading a encryptionConfig with KMSv2 should work @@ -1086,7 +1105,7 @@ func BenchmarkKMSv2KDF(b *testing.B) { klog.SetOutput(io.Discard) klog.LogToStderr(false) - defer encryptionconfig.SetKDFForTests(false)() + defer encryptionconfig.SetKDFForTests("kms-provider-bench", false)() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) b.Cleanup(cancel) @@ -1102,10 +1121,10 @@ resources: providers: - kms: apiVersion: v2 - name: kms-provider - endpoint: unix:///@kms-provider.sock + name: kms-provider-bench + endpoint: unix:///@kms-provider-bench.sock ` - _ = kmsv2mock.NewBase64Plugin(b, "@kms-provider.sock") + _ = kmsv2mock.NewBase64Plugin(b, "@kms-provider-bench.sock") test, err := newTransformTest(b, encryptionConfig, false, "", nil) if err != nil { @@ -1241,7 +1260,7 @@ func BenchmarkKMSv2REST(b *testing.B) { klog.SetOutput(io.Discard) klog.LogToStderr(false) - defer encryptionconfig.SetKDFForTests(true)() + defer encryptionconfig.SetKDFForTests("kms-provider-bench-rest", true)() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) b.Cleanup(cancel) @@ -1255,10 +1274,10 @@ resources: providers: - kms: apiVersion: v2 - name: kms-provider - endpoint: unix:///@kms-provider.sock + name: kms-provider-bench-rest + endpoint: unix:///@kms-provider-bench-rest.sock ` - _ = kmsv2mock.NewBase64Plugin(b, "@kms-provider.sock") + _ = kmsv2mock.NewBase64Plugin(b, "@kms-provider-bench-rest.sock") test, err := newTransformTest(b, encryptionConfig, false, "", nil) if err != nil { @@ -1325,12 +1344,13 @@ func randomBool() bool { return utilrand.Int()%2 == 1 } // TestKMSv2ProviderLegacyData confirms that legacy data recorded from the earliest released commit can still be read. func TestKMSv2ProviderLegacyData(t *testing.T) { + // this test must run serially because the provider name is encoded in the legacy data and thus cannot be changed t.Run("regular gcm", func(t *testing.T) { - defer encryptionconfig.SetKDFForTests(false)() + defer encryptionconfig.SetKDFForTests("kms-provider", false)() testKMSv2ProviderLegacyData(t) }) t.Run("extended nonce gcm", func(t *testing.T) { - defer encryptionconfig.SetKDFForTests(true)() + defer encryptionconfig.SetKDFForTests("kms-provider", true)() testKMSv2ProviderLegacyData(t) }) } diff --git a/test/integration/controlplane/transformation/transformation_test.go b/test/integration/controlplane/transformation/transformation_test.go index f47177ff036..448c7e0554a 100644 --- a/test/integration/controlplane/transformation/transformation_test.go +++ b/test/integration/controlplane/transformation/transformation_test.go @@ -25,6 +25,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "testing" "time" @@ -116,7 +117,7 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool, return nil, fmt.Errorf("failed to read config file: %w", err) } - if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer( + if e.kubeAPIServer, err = startTestServerLocked( tb, nil, e.getEncryptionOptions(reload), e.storageConfig); err != nil { e.cleanUp() @@ -147,6 +148,15 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool, return &e, nil } +var startTestServerLock sync.Mutex + +// startTestServerLocked prevents parallel calls to kubeapiservertesting.StartTestServer because it messes with global state. +func startTestServerLocked(t ktesting.TB, instanceOptions *kubeapiservertesting.TestServerInstanceOptions, customFlags []string, storageConfig *storagebackend.Config) (result kubeapiservertesting.TestServer, err error) { + startTestServerLock.Lock() + defer startTestServerLock.Unlock() + return kubeapiservertesting.StartTestServer(t, instanceOptions, customFlags, storageConfig) +} + func (e *transformTest) cleanUp() { if e.configDir != "" { os.RemoveAll(e.configDir)