github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-10 20:42:26 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #121452 from nilekhc/test-hot-reload-test-flake

Browse Source 
tests: fixes flake in TestEncryptionConfigHotReload
This commit is contained in:
Kubernetes Prow Robot 2023-10-24 00:30:19 +02:00 committed by GitHub
commit 18d5a6fa64
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 44 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
test/integration/controlplane/transformation/kms_transformation_test.go
View File

@ -402,7 +402,7 @@ resources:
// implementing this brute force approach instead of fancy channel notification to avoid test specific code in prod.
// wait for config to be observed
verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, test)
verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, "", test)
// run storage migration
// get secrets
@ -472,6 +472,10 @@ resources:
}
// remove old KMS provider
// verifyIfKMSTransformersSwapped sometimes passes even before the changes in the encryption config file are observed.
// this causes the metrics tests to fail, which validate two config changes.
// this may happen when an existing KMS provider is already running (e.g., new-kms-provider-for-secrets in this case).
// to ensure that the changes are observed, we added one more provider (kms-provider-to-encrypt-all) and are validating it in verifyIfKMSTransformersSwapped.
encryptionConfigWithoutOldProvider := `
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
@ -490,15 +494,28 @@ resources:
name: new-kms-provider-for-configmaps
cachesize: 1000
endpoint: unix:///@new-kms-provider.sock
- resources:
- '*.*'
providers:
- kms:
name: kms-provider-to-encrypt-all
cachesize: 1000
endpoint: unix:///@new-encrypt-all-kms-provider.sock
- identity: {}
`
// start new KMS Plugin
_ = mock.NewBase64Plugin(t, "@new-encrypt-all-kms-provider.sock")
// update encryption config and wait for hot reload
if err := os.WriteFile(filepath.Join(test.configDir, encryptionConfigFileName), []byte(encryptionConfigWithoutOldProvider), 0644); err != nil {
t.Fatalf("failed to update encryption config, err: %v", err)
}
wantPrefixForEncryptAll := "k8s:enc:kms:v1:kms-provider-to-encrypt-all:"
// wait for config to be observed
verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, test)
verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, wantPrefixForEncryptAll, test)
// confirm that reading secrets still works
_, err = test.restClient.CoreV1().Secrets(testNamespace).Get(
@ -925,7 +942,7 @@ resources:
func verifyPrefixOfSecretResource(t *testing.T, wantPrefix string, test *transformTest) {
// implementing this brute force approach instead of fancy channel notification to avoid test specific code in prod.
// wait for config to be observed
verifyIfKMSTransformersSwapped(t, wantPrefix, test)
verifyIfKMSTransformersSwapped(t, wantPrefix, "", test)
// run storage migration
secretsList, err := test.restClient.CoreV1().Secrets("").List(
@ -959,7 +976,7 @@ func verifyPrefixOfSecretResource(t *testing.T, wantPrefix string, test *transfo
}
}
func verifyIfKMSTransformersSwapped(t *testing.T, wantPrefix string, test *transformTest) {
func verifyIfKMSTransformersSwapped(t *testing.T, wantPrefix, wantPrefixForEncryptAll string, test *transformTest) {
t.Helper()
var swapErr error
@ -990,6 +1007,29 @@ func verifyIfKMSTransformersSwapped(t *testing.T, wantPrefix string, test *trans
return false, nil
}
if wantPrefixForEncryptAll != "" {
deploymentName := fmt.Sprintf("deployment-%d", idx)
_, err := test.createDeployment(deploymentName, "default")
if err != nil {
t.Fatalf("Failed to create test secret, error: %v", err)
}
rawEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "deployments", deploymentName, "default"))
if err != nil {
t.Fatalf("failed to read %s from etcd: %v", test.getETCDPathForResource(test.storageConfig.Prefix, "", "deployments", deploymentName, "default"), err)
}
// check prefix
if !bytes.HasPrefix(rawEnvelope.Kvs[0].Value, []byte(wantPrefixForEncryptAll)) {
idx++
swapErr = fmt.Errorf("expected deployment to be prefixed with %s, but got %s", wantPrefixForEncryptAll, rawEnvelope.Kvs[0].Value)
// return nil error to continue polling till timeout
return false, nil
}
}
return true, nil
})
if pollErr == wait.ErrWaitTimeout {