diff --git a/test/integration/controlplane/transformation/kms_transformation_test.go b/test/integration/controlplane/transformation/kms_transformation_test.go index 941234f9b46..8964dbf009f 100644 --- a/test/integration/controlplane/transformation/kms_transformation_test.go +++ b/test/integration/controlplane/transformation/kms_transformation_test.go @@ -402,7 +402,7 @@ resources: // implementing this brute force approach instead of fancy channel notification to avoid test specific code in prod. // wait for config to be observed - verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, test) + verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, "", test) // run storage migration // get secrets @@ -472,6 +472,10 @@ resources: } // remove old KMS provider + // verifyIfKMSTransformersSwapped sometimes passes even before the changes in the encryption config file are observed. + // this causes the metrics tests to fail, which validate two config changes. + // this may happen when an existing KMS provider is already running (e.g., new-kms-provider-for-secrets in this case). + // to ensure that the changes are observed, we added one more provider (kms-provider-to-encrypt-all) and are validating it in verifyIfKMSTransformersSwapped. encryptionConfigWithoutOldProvider := ` kind: EncryptionConfiguration apiVersion: apiserver.config.k8s.io/v1 @@ -490,15 +494,28 @@ resources: name: new-kms-provider-for-configmaps cachesize: 1000 endpoint: unix:///@new-kms-provider.sock + - resources: + - '*.*' + providers: + - kms: + name: kms-provider-to-encrypt-all + cachesize: 1000 + endpoint: unix:///@new-encrypt-all-kms-provider.sock + - identity: {} ` + // start new KMS Plugin + _ = mock.NewBase64Plugin(t, "@new-encrypt-all-kms-provider.sock") + // update encryption config and wait for hot reload if err := os.WriteFile(filepath.Join(test.configDir, encryptionConfigFileName), []byte(encryptionConfigWithoutOldProvider), 0644); err != nil { t.Fatalf("failed to update encryption config, err: %v", err) } + wantPrefixForEncryptAll := "k8s:enc:kms:v1:kms-provider-to-encrypt-all:" + // wait for config to be observed - verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, test) + verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, wantPrefixForEncryptAll, test) // confirm that reading secrets still works _, err = test.restClient.CoreV1().Secrets(testNamespace).Get( @@ -925,7 +942,7 @@ resources: func verifyPrefixOfSecretResource(t *testing.T, wantPrefix string, test *transformTest) { // implementing this brute force approach instead of fancy channel notification to avoid test specific code in prod. // wait for config to be observed - verifyIfKMSTransformersSwapped(t, wantPrefix, test) + verifyIfKMSTransformersSwapped(t, wantPrefix, "", test) // run storage migration secretsList, err := test.restClient.CoreV1().Secrets("").List( @@ -959,7 +976,7 @@ func verifyPrefixOfSecretResource(t *testing.T, wantPrefix string, test *transfo } } -func verifyIfKMSTransformersSwapped(t *testing.T, wantPrefix string, test *transformTest) { +func verifyIfKMSTransformersSwapped(t *testing.T, wantPrefix, wantPrefixForEncryptAll string, test *transformTest) { t.Helper() var swapErr error @@ -990,6 +1007,29 @@ func verifyIfKMSTransformersSwapped(t *testing.T, wantPrefix string, test *trans return false, nil } + if wantPrefixForEncryptAll != "" { + deploymentName := fmt.Sprintf("deployment-%d", idx) + _, err := test.createDeployment(deploymentName, "default") + if err != nil { + t.Fatalf("Failed to create test secret, error: %v", err) + } + + rawEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "deployments", deploymentName, "default")) + if err != nil { + t.Fatalf("failed to read %s from etcd: %v", test.getETCDPathForResource(test.storageConfig.Prefix, "", "deployments", deploymentName, "default"), err) + } + + // check prefix + if !bytes.HasPrefix(rawEnvelope.Kvs[0].Value, []byte(wantPrefixForEncryptAll)) { + idx++ + + swapErr = fmt.Errorf("expected deployment to be prefixed with %s, but got %s", wantPrefixForEncryptAll, rawEnvelope.Kvs[0].Value) + + // return nil error to continue polling till timeout + return false, nil + } + } + return true, nil }) if pollErr == wait.ErrWaitTimeout {