mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-29 06:27:05 +00:00
kmsv2: run KDF tests in parallel
This change updates the KDF "feature flag" to be per KMS provider instead of global to the API server. This allows integration tests that use distinct provider names to run in parallel. Locally this change reduced the runtime of test/integration/controlplane/transformation by 3.5 minutes. Signed-off-by: Monis Khan <mok@microsoft.com>
This commit is contained in:
Monis Khan
parent
a0db55c3a6
commit
43740c0def
@ -107,24 +107,35 @@ const (
|
||||
|
||||
var codecs serializer.CodecFactory
|
||||
|
||||
// this atomic bool allows us to swap enablement of the KMSv2KDF feature in tests
|
||||
// this map allows us to swap enablement of the KMSv2KDF feature in tests
|
||||
// as the feature gate is now locked to true starting with v1.29
|
||||
// Note: it cannot be set by an end user
|
||||
var kdfDisabled atomic.Bool
|
||||
// KDF enablement is tracked per KMS provider to allow tests to run in parallel.
|
||||
var kdfEnabledPerKMS sync.Map // map[string]bool, KMS name -> KDF enabled
|
||||
|
||||
// this function should only be called in tests to swap enablement of the KMSv2KDF feature
|
||||
func SetKDFForTests(b bool) func() {
|
||||
kdfDisabled.Store(!b)
|
||||
return func() {
|
||||
kdfDisabled.Store(false)
|
||||
// Caller must guarantee that all KMS providers have distinct names across all tests.
|
||||
func SetKDFForTests(kmsName string, b bool) func() {
|
||||
if len(kmsName) == 0 { // guarantee that GetKDF("") returns the default value
|
||||
panic("empty KMS name used in test")
|
||||
}
|
||||
if _, loaded := kdfEnabledPerKMS.LoadOrStore(kmsName, b); loaded {
|
||||
panic("duplicate KMS name used in test")
|
||||
}
|
||||
return func() { kdfEnabledPerKMS.Delete(kmsName) }
|
||||
}
|
||||
|
||||
// this function should be used to determine enablement of the KMSv2KDF feature
|
||||
// instead of getting it from DefaultFeatureGate as the feature gate is now locked
|
||||
// to true starting with v1.29
|
||||
func GetKDF() bool {
|
||||
return !kdfDisabled.Load()
|
||||
// to allow integration tests to run in parallel, this "feature flag" can be set
|
||||
// per KMS provider as long as all providers use distinct names.
|
||||
func GetKDF(kmsName string) bool {
|
||||
kdfEnabled, ok := kdfEnabledPerKMS.Load(kmsName)
|
||||
if !ok {
|
||||
return true // explicit config is missing, but KDF is enabled by default
|
||||
}
|
||||
return kdfEnabled.(bool) // this will panic if a non-bool ever gets stored, which should never happen
|
||||
}
|
||||
|
||||
func init() {
|
||||
@ -390,7 +401,7 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey
|
||||
// this gate can only change during tests, but the check is cheap enough to always make
|
||||
// this allows us to easily exercise both modes without restarting the API server
|
||||
// TODO integration test that this dynamically takes effect
|
||||
useSeed := GetKDF()
|
||||
useSeed := GetKDF(h.name)
|
||||
stateUseSeed := state.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
|
||||
|
||||
// state is valid and status keyID is unchanged from when we generated this DEK/seed so there is no need to rotate it
|
||||
|
||||
@ -1851,7 +1851,7 @@ func TestComputeEncryptionConfigHash(t *testing.T) {
|
||||
}
|
||||
|
||||
func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
|
||||
defaultUseSeed := GetKDF()
|
||||
defaultUseSeed := GetKDF("")
|
||||
|
||||
origNowFunc := envelopekmsv2.NowFunc
|
||||
now := origNowFunc() // freeze time
|
||||
@ -2074,9 +2074,10 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
|
||||
`encryptKeyIDHash="", stateKeyIDHash="sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35", expirationTimestamp=` + now.Format(time.RFC3339),
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
for i, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
defer SetKDFForTests(tt.useSeed)()
|
||||
kmsName := fmt.Sprintf("panda-%d", i)
|
||||
defer SetKDFForTests(kmsName, tt.useSeed)()
|
||||
|
||||
var buf bytes.Buffer
|
||||
klog.SetOutput(&buf)
|
||||
@ -2084,7 +2085,7 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
|
||||
ctx := testContext(t)
|
||||
|
||||
h := &kmsv2PluginProbe{
|
||||
name: "panda",
|
||||
name: kmsName,
|
||||
service: tt.service,
|
||||
}
|
||||
h.state.Store(&tt.state)
|
||||
@ -2133,7 +2134,7 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
|
||||
if _, stateErr := h.getCurrentState(); stateErr == nil || err == nil {
|
||||
transformer := envelopekmsv2.NewEnvelopeTransformer(
|
||||
&testKMSv2EnvelopeService{err: fmt.Errorf("broken")}, // not called
|
||||
"panda",
|
||||
kmsName,
|
||||
h.getCurrentState,
|
||||
"",
|
||||
)
|
||||
|
||||
@ -165,12 +165,15 @@ func (r envelopekmsv2) plainTextPayload(secretETCDPath string) ([]byte, error) {
|
||||
// TestDefaultValues tests default flag values without setting any of the feature flags or
|
||||
// calling SetKDFForTests, and assert that the data stored in etcd is using KDF
|
||||
func TestDefaultValues(t *testing.T) {
|
||||
if encryptionconfig.GetKDF() != true {
|
||||
if encryptionconfig.GetKDF("") != true {
|
||||
t.Fatalf("without updating the feature flags, default value of KMSv2KDF should be enabled.")
|
||||
}
|
||||
if utilfeature.DefaultFeatureGate.Enabled(features.KMSv1) != false {
|
||||
t.Fatalf("without updating the feature flags, default value of KMSv1 should be disabled.")
|
||||
}
|
||||
|
||||
t.Parallel() // the feature gate check above uses global state, so only run in parallel after that point
|
||||
|
||||
// since encryptionconfig.GetKDF() is true by default, following test should verify if
|
||||
// object.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
@ -185,10 +188,10 @@ resources:
|
||||
providers:
|
||||
- kms:
|
||||
apiVersion: v2
|
||||
name: kms-provider
|
||||
endpoint: unix:///@kms-provider.sock
|
||||
name: kms-provider-defaults
|
||||
endpoint: unix:///@kms-provider-defaults.sock
|
||||
`
|
||||
_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
|
||||
_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider-defaults.sock")
|
||||
|
||||
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
|
||||
if err != nil {
|
||||
@ -228,7 +231,7 @@ resources:
|
||||
t.Fatalf("expected 1 KVs, but got %d", len(response.Kvs))
|
||||
}
|
||||
object := kmstypes.EncryptedObject{}
|
||||
v := bytes.TrimPrefix(response.Kvs[0].Value, []byte("k8s:enc:kms:v2:kms-provider:"))
|
||||
v := bytes.TrimPrefix(response.Kvs[0].Value, []byte("k8s:enc:kms:v2:kms-provider-defaults:"))
|
||||
if err := proto.Unmarshal(v, &object); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -245,19 +248,22 @@ resources:
|
||||
// 4. The cipherTextPayload (ex. Secret) should be encrypted via AES GCM transform / extended nonce GCM
|
||||
// 5. kmstypes.EncryptedObject structure should be serialized and deposited in ETCD
|
||||
func TestKMSv2Provider(t *testing.T) {
|
||||
defaultUseSeed := encryptionconfig.GetKDF()
|
||||
// this test cannot be run in parallel because both the API server ID and the metrics are global state
|
||||
defaultUseSeed := encryptionconfig.GetKDF("")
|
||||
|
||||
t.Run("regular gcm", func(t *testing.T) {
|
||||
defer encryptionconfig.SetKDFForTests(false)()
|
||||
testKMSv2Provider(t, !defaultUseSeed)
|
||||
kmsName := "kms-provider-v2-false"
|
||||
defer encryptionconfig.SetKDFForTests(kmsName, false)()
|
||||
testKMSv2Provider(t, kmsName, !defaultUseSeed)
|
||||
})
|
||||
t.Run("extended nonce gcm", func(t *testing.T) {
|
||||
defer encryptionconfig.SetKDFForTests(true)()
|
||||
testKMSv2Provider(t, defaultUseSeed)
|
||||
kmsName := "kms-provider-v2-true"
|
||||
defer encryptionconfig.SetKDFForTests(kmsName, true)()
|
||||
testKMSv2Provider(t, kmsName, defaultUseSeed)
|
||||
})
|
||||
}
|
||||
|
||||
func testKMSv2Provider(t *testing.T, useSeed bool) {
|
||||
func testKMSv2Provider(t *testing.T, kmsName string, useSeed bool) {
|
||||
encryptionConfig := `
|
||||
kind: EncryptionConfiguration
|
||||
apiVersion: apiserver.config.k8s.io/v1
|
||||
@ -267,12 +273,11 @@ resources:
|
||||
providers:
|
||||
- kms:
|
||||
apiVersion: v2
|
||||
name: kms-provider
|
||||
endpoint: unix:///@kms-provider.sock
|
||||
name: ` + kmsName + `
|
||||
endpoint: unix:///@` + kmsName + `.sock
|
||||
`
|
||||
genericapiserver.SetHostnameFuncForTests("testAPIServerID")
|
||||
providerName := "kms-provider"
|
||||
pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
|
||||
pluginMock := kmsv2mock.NewBase64Plugin(t, "@"+kmsName+".sock")
|
||||
|
||||
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
|
||||
if err != nil {
|
||||
@ -296,13 +301,13 @@ resources:
|
||||
|
||||
// assert that the metrics we collect during the test run match expectations
|
||||
wantMetricStrings := []string{
|
||||
`apiserver_envelope_encryption_dek_source_cache_size{provider_name="kms-provider"} 1`,
|
||||
`apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} FP`,
|
||||
`apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} FP`,
|
||||
`apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} 2`,
|
||||
`apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} 1`,
|
||||
`apiserver_storage_transformation_operations_total{resource="secrets",status="OK",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v2:kms-provider:"} 2`,
|
||||
`apiserver_storage_transformation_operations_total{resource="secrets",status="OK",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v2:kms-provider:"} 1`,
|
||||
`apiserver_envelope_encryption_dek_source_cache_size{provider_name="` + kmsName + `"} 1`,
|
||||
`apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="` + kmsName + `",transformation_type="from_storage"} FP`,
|
||||
`apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="` + kmsName + `",transformation_type="to_storage"} FP`,
|
||||
`apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="` + kmsName + `",transformation_type="from_storage"} 2`,
|
||||
`apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="` + kmsName + `",transformation_type="to_storage"} 1`,
|
||||
`apiserver_storage_transformation_operations_total{resource="secrets",status="OK",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v2:` + kmsName + `:"} 2`,
|
||||
`apiserver_storage_transformation_operations_total{resource="secrets",status="OK",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v2:` + kmsName + `:"} 1`,
|
||||
}
|
||||
defer func() {
|
||||
body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx)
|
||||
@ -343,7 +348,7 @@ resources:
|
||||
}
|
||||
|
||||
envelopeData := envelopekmsv2{
|
||||
providerName: providerName,
|
||||
providerName: kmsName,
|
||||
rawEnvelope: rawEnvelope,
|
||||
plainTextDEKSource: plainTextDEKSource,
|
||||
useSeed: useSeed,
|
||||
@ -399,16 +404,19 @@ resources:
|
||||
// 6. When kms-plugin is down, expect creation of new pod and encryption to fail once the DEK/seed is invalid
|
||||
// 7. when kms-plugin is down, no-op update for a pod should succeed and not result in RV change even once the DEK/seed is valid
|
||||
func TestKMSv2ProviderKeyIDStaleness(t *testing.T) {
|
||||
t.Parallel()
|
||||
t.Run("regular gcm", func(t *testing.T) {
|
||||
defer encryptionconfig.SetKDFForTests(false)()
|
||||
testKMSv2ProviderKeyIDStaleness(t)
|
||||
t.Parallel()
|
||||
kmsName := "kms-provider-key-id-stale-false"
|
||||
testKMSv2ProviderKeyIDStaleness(t, kmsName, encryptionconfig.SetKDFForTests(kmsName, false))
|
||||
})
|
||||
t.Run("extended nonce gcm", func(t *testing.T) {
|
||||
testKMSv2ProviderKeyIDStaleness(t)
|
||||
t.Parallel()
|
||||
testKMSv2ProviderKeyIDStaleness(t, "kms-provider-key-id-stale-true", func() {})
|
||||
})
|
||||
}
|
||||
|
||||
func testKMSv2ProviderKeyIDStaleness(t *testing.T) {
|
||||
func testKMSv2ProviderKeyIDStaleness(t *testing.T, kmsName string, resetSetKDFForTests func()) {
|
||||
encryptionConfig := `
|
||||
kind: EncryptionConfiguration
|
||||
apiVersion: apiserver.config.k8s.io/v1
|
||||
@ -419,10 +427,10 @@ resources:
|
||||
providers:
|
||||
- kms:
|
||||
apiVersion: v2
|
||||
name: kms-provider
|
||||
endpoint: unix:///@kms-provider.sock
|
||||
name: ` + kmsName + `
|
||||
endpoint: unix:///@` + kmsName + `.sock
|
||||
`
|
||||
pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
|
||||
pluginMock := kmsv2mock.NewBase64Plugin(t, "@"+kmsName+".sock")
|
||||
|
||||
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
|
||||
if err != nil {
|
||||
@ -451,7 +459,7 @@ resources:
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
t.Cleanup(cancel)
|
||||
|
||||
useSeed := encryptionconfig.GetKDF()
|
||||
useSeed := encryptionconfig.GetKDF(kmsName)
|
||||
|
||||
var firstEncryptedDEKSource []byte
|
||||
var f checkFunc
|
||||
@ -480,7 +488,7 @@ resources:
|
||||
}
|
||||
}
|
||||
assertPodDEKSources(ctx, t, test.kubeAPIServer.ServerOpts.Etcd.StorageConfig,
|
||||
1, 1, "k8s:enc:kms:v2:kms-provider:", f,
|
||||
1, 1, kmsName, "k8s:enc:kms:v2:"+kmsName+":", f,
|
||||
)
|
||||
if len(firstEncryptedDEKSource) == 0 {
|
||||
t.Fatal("unexpected empty DEK or seed")
|
||||
@ -618,7 +626,7 @@ resources:
|
||||
}
|
||||
|
||||
assertPodDEKSources(ctx, t, test.kubeAPIServer.ServerOpts.Etcd.StorageConfig,
|
||||
1, 1, "k8s:enc:kms:v2:kms-provider:", checkDEK,
|
||||
1, 1, kmsName, "k8s:enc:kms:v2:"+kmsName+":", checkDEK,
|
||||
)
|
||||
|
||||
// fix plugin and wait for new writes to start working again
|
||||
@ -646,8 +654,9 @@ resources:
|
||||
if version7 != version8 {
|
||||
t.Fatalf("Resource version should not have changed after plugin health is restored. old pod: %v, new pod: %v", updatedNewPod, updatedNewPod2)
|
||||
}
|
||||
resetSetKDFForTests() // prevent the config flip from panic-ing
|
||||
// flip the current config
|
||||
defer encryptionconfig.SetKDFForTests(!useSeed)()
|
||||
defer encryptionconfig.SetKDFForTests(kmsName, !useSeed)()
|
||||
|
||||
// 9. confirm that no-op update for a pod results in RV change due to KDF config change
|
||||
var version9 string
|
||||
@ -670,9 +679,12 @@ resources:
|
||||
}
|
||||
|
||||
func TestKMSv2ProviderDEKSourceReuse(t *testing.T) {
|
||||
t.Parallel()
|
||||
t.Run("regular gcm", func(t *testing.T) {
|
||||
defer encryptionconfig.SetKDFForTests(false)()
|
||||
testKMSv2ProviderDEKSourceReuse(t,
|
||||
t.Parallel()
|
||||
kmsName := "kms-provider-dek-reuse-false"
|
||||
defer encryptionconfig.SetKDFForTests(kmsName, false)()
|
||||
testKMSv2ProviderDEKSourceReuse(t, kmsName,
|
||||
func(i int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject) {
|
||||
if obj.KeyID != "1" {
|
||||
t.Errorf("key %s: want key ID %s, got %s", etcdKey, "1", obj.KeyID)
|
||||
@ -687,8 +699,10 @@ func TestKMSv2ProviderDEKSourceReuse(t *testing.T) {
|
||||
)
|
||||
})
|
||||
t.Run("extended nonce gcm", func(t *testing.T) {
|
||||
defer encryptionconfig.SetKDFForTests(true)()
|
||||
testKMSv2ProviderDEKSourceReuse(t,
|
||||
t.Parallel()
|
||||
kmsName := "kms-provider-dek-reuse-true"
|
||||
defer encryptionconfig.SetKDFForTests(kmsName, true)()
|
||||
testKMSv2ProviderDEKSourceReuse(t, kmsName,
|
||||
func(_ int, _ uint64, etcdKey string, obj kmstypes.EncryptedObject) {
|
||||
if obj.KeyID != "1" {
|
||||
t.Errorf("key %s: want key ID %s, got %s", etcdKey, "1", obj.KeyID)
|
||||
@ -698,7 +712,7 @@ func TestKMSv2ProviderDEKSourceReuse(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func testKMSv2ProviderDEKSourceReuse(t *testing.T, f checkFunc) {
|
||||
func testKMSv2ProviderDEKSourceReuse(t *testing.T, kmsName string, f checkFunc) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
t.Cleanup(cancel)
|
||||
|
||||
@ -711,10 +725,10 @@ resources:
|
||||
providers:
|
||||
- kms:
|
||||
apiVersion: v2
|
||||
name: kms-provider
|
||||
endpoint: unix:///@kms-provider.sock
|
||||
name: ` + kmsName + `
|
||||
endpoint: unix:///@` + kmsName + `.sock
|
||||
`
|
||||
_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
|
||||
_ = kmsv2mock.NewBase64Plugin(t, "@"+kmsName+".sock")
|
||||
|
||||
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
|
||||
if err != nil {
|
||||
@ -746,13 +760,13 @@ resources:
|
||||
|
||||
assertPodDEKSources(ctx, t, test.kubeAPIServer.ServerOpts.Etcd.StorageConfig,
|
||||
podCount, 1, // key ID does not change during the test so we should only have a single DEK
|
||||
"k8s:enc:kms:v2:kms-provider:", f,
|
||||
kmsName, "k8s:enc:kms:v2:"+kmsName+":", f,
|
||||
)
|
||||
}
|
||||
|
||||
type checkFunc func(i int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject)
|
||||
|
||||
func assertPodDEKSources(ctx context.Context, t *testing.T, config storagebackend.Config, podCount, dekSourcesCount int, kmsPrefix string, f checkFunc) {
|
||||
func assertPodDEKSources(ctx context.Context, t *testing.T, config storagebackend.Config, podCount, dekSourcesCount int, kmsName, kmsPrefix string, f checkFunc) {
|
||||
t.Helper()
|
||||
|
||||
rawClient, etcdClient, err := integration.GetEtcdClients(config.Transport)
|
||||
@ -770,7 +784,7 @@ func assertPodDEKSources(ctx context.Context, t *testing.T, config storagebacken
|
||||
t.Fatalf("expected %d KVs, but got %d", podCount, len(response.Kvs))
|
||||
}
|
||||
|
||||
useSeed := encryptionconfig.GetKDF()
|
||||
useSeed := encryptionconfig.GetKDF(kmsName)
|
||||
|
||||
out := make([]kmstypes.EncryptedObject, len(response.Kvs))
|
||||
for i, kv := range response.Kvs {
|
||||
@ -832,7 +846,10 @@ func assertPodDEKSources(ctx context.Context, t *testing.T, config storagebacken
|
||||
}
|
||||
|
||||
func TestKMSv2Healthz(t *testing.T) {
|
||||
defer encryptionconfig.SetKDFForTests(randomBool())()
|
||||
t.Parallel()
|
||||
|
||||
defer encryptionconfig.SetKDFForTests("provider-1", randomBool())()
|
||||
defer encryptionconfig.SetKDFForTests("provider-2", randomBool())()
|
||||
|
||||
encryptionConfig := `
|
||||
kind: EncryptionConfiguration
|
||||
@ -897,7 +914,7 @@ resources:
|
||||
}
|
||||
|
||||
func TestKMSv2SingleService(t *testing.T) {
|
||||
defer encryptionconfig.SetKDFForTests(randomBool())()
|
||||
defer encryptionconfig.SetKDFForTests("kms-provider-single-service", randomBool())() // this test must run serially due to the global state below
|
||||
|
||||
var kmsv2Calls int
|
||||
origEnvelopeKMSv2ServiceFactory := encryptionconfig.EnvelopeKMSv2ServiceFactory
|
||||
@ -926,11 +943,11 @@ resources:
|
||||
providers:
|
||||
- kms:
|
||||
apiVersion: v2
|
||||
name: kms-provider
|
||||
endpoint: unix:///@kms-provider.sock
|
||||
name: kms-provider-single-service
|
||||
endpoint: unix:///@kms-provider-single-service.sock
|
||||
`
|
||||
|
||||
_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
|
||||
_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider-single-service.sock")
|
||||
|
||||
test, err := newTransformTest(t, encryptionConfig, false, "", nil)
|
||||
if err != nil {
|
||||
@ -970,6 +987,8 @@ resources:
|
||||
// 2. After a restart, loading a encryptionConfig with the same KMSv2 plugin from 1 should work,
|
||||
// decryption of data encrypted with v2 should work
|
||||
func TestKMSv2FeatureFlag(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
encryptionConfig := `
|
||||
kind: EncryptionConfiguration
|
||||
apiVersion: apiserver.config.k8s.io/v1
|
||||
@ -979,11 +998,11 @@ resources:
|
||||
providers:
|
||||
- kms:
|
||||
apiVersion: v2
|
||||
name: kms-provider
|
||||
endpoint: unix:///@kms-provider.sock
|
||||
name: kms-provider-feature-flag
|
||||
endpoint: unix:///@kms-provider-feature-flag.sock
|
||||
`
|
||||
providerName := "kms-provider"
|
||||
pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
|
||||
providerName := "kms-provider-feature-flag"
|
||||
pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider-feature-flag.sock")
|
||||
storageConfig := framework.SharedEtcd()
|
||||
|
||||
// KMSv2 is enabled by default. Loading a encryptionConfig with KMSv2 should work
|
||||
@ -1086,7 +1105,7 @@ func BenchmarkKMSv2KDF(b *testing.B) {
|
||||
klog.SetOutput(io.Discard)
|
||||
klog.LogToStderr(false)
|
||||
|
||||
defer encryptionconfig.SetKDFForTests(false)()
|
||||
defer encryptionconfig.SetKDFForTests("kms-provider-bench", false)()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
b.Cleanup(cancel)
|
||||
@ -1102,10 +1121,10 @@ resources:
|
||||
providers:
|
||||
- kms:
|
||||
apiVersion: v2
|
||||
name: kms-provider
|
||||
endpoint: unix:///@kms-provider.sock
|
||||
name: kms-provider-bench
|
||||
endpoint: unix:///@kms-provider-bench.sock
|
||||
`
|
||||
_ = kmsv2mock.NewBase64Plugin(b, "@kms-provider.sock")
|
||||
_ = kmsv2mock.NewBase64Plugin(b, "@kms-provider-bench.sock")
|
||||
|
||||
test, err := newTransformTest(b, encryptionConfig, false, "", nil)
|
||||
if err != nil {
|
||||
@ -1241,7 +1260,7 @@ func BenchmarkKMSv2REST(b *testing.B) {
|
||||
klog.SetOutput(io.Discard)
|
||||
klog.LogToStderr(false)
|
||||
|
||||
defer encryptionconfig.SetKDFForTests(true)()
|
||||
defer encryptionconfig.SetKDFForTests("kms-provider-bench-rest", true)()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
b.Cleanup(cancel)
|
||||
@ -1255,10 +1274,10 @@ resources:
|
||||
providers:
|
||||
- kms:
|
||||
apiVersion: v2
|
||||
name: kms-provider
|
||||
endpoint: unix:///@kms-provider.sock
|
||||
name: kms-provider-bench-rest
|
||||
endpoint: unix:///@kms-provider-bench-rest.sock
|
||||
`
|
||||
_ = kmsv2mock.NewBase64Plugin(b, "@kms-provider.sock")
|
||||
_ = kmsv2mock.NewBase64Plugin(b, "@kms-provider-bench-rest.sock")
|
||||
|
||||
test, err := newTransformTest(b, encryptionConfig, false, "", nil)
|
||||
if err != nil {
|
||||
@ -1325,12 +1344,13 @@ func randomBool() bool { return utilrand.Int()%2 == 1 }
|
||||
|
||||
// TestKMSv2ProviderLegacyData confirms that legacy data recorded from the earliest released commit can still be read.
|
||||
func TestKMSv2ProviderLegacyData(t *testing.T) {
|
||||
// this test must run serially because the provider name is encoded in the legacy data and thus cannot be changed
|
||||
t.Run("regular gcm", func(t *testing.T) {
|
||||
defer encryptionconfig.SetKDFForTests(false)()
|
||||
defer encryptionconfig.SetKDFForTests("kms-provider", false)()
|
||||
testKMSv2ProviderLegacyData(t)
|
||||
})
|
||||
t.Run("extended nonce gcm", func(t *testing.T) {
|
||||
defer encryptionconfig.SetKDFForTests(true)()
|
||||
defer encryptionconfig.SetKDFForTests("kms-provider", true)()
|
||||
testKMSv2ProviderLegacyData(t)
|
||||
})
|
||||
}
|
||||
|
||||
@ -25,6 +25,7 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -116,7 +117,7 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool,
|
||||
return nil, fmt.Errorf("failed to read config file: %w", err)
|
||||
}
|
||||
|
||||
if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(
|
||||
if e.kubeAPIServer, err = startTestServerLocked(
|
||||
tb, nil,
|
||||
e.getEncryptionOptions(reload), e.storageConfig); err != nil {
|
||||
e.cleanUp()
|
||||
@ -147,6 +148,15 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool,
|
||||
return &e, nil
|
||||
}
|
||||
|
||||
var startTestServerLock sync.Mutex
|
||||
|
||||
// startTestServerLocked prevents parallel calls to kubeapiservertesting.StartTestServer because it messes with global state.
|
||||
func startTestServerLocked(t ktesting.TB, instanceOptions *kubeapiservertesting.TestServerInstanceOptions, customFlags []string, storageConfig *storagebackend.Config) (result kubeapiservertesting.TestServer, err error) {
|
||||
startTestServerLock.Lock()
|
||||
defer startTestServerLock.Unlock()
|
||||
return kubeapiservertesting.StartTestServer(t, instanceOptions, customFlags, storageConfig)
|
||||
}
|
||||
|
||||
func (e *transformTest) cleanUp() {
|
||||
if e.configDir != "" {
|
||||
os.RemoveAll(e.configDir)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user