diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go index 8af365af11d..5dc735a5866 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go @@ -161,7 +161,7 @@ func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool) (*E kmsHealthChecks = []healthz.HealthChecker{kmsHealthChecker(kmsHealthChecks)} } - // KMSTimeout is the duration we will wait before closing old transformers. + // KMSCloseGracePeriod is the duration we will wait before closing old transformers. // The way we calculate is as follows: // 1. Sum all timeouts across all KMS plugins. (check kmsPrefixTransformer for differences between v1 and v2) // 2. Multiply that by 2 (to allow for some buffer) diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go index d6eb61a1cb8..5e398d4b434 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go @@ -177,41 +177,71 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) { // Creates compound/prefix transformers with different ordering of available transformers. // Transforms data using one of them, and tries to untransform using the others. // Repeats this for all possible combinations. + // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 + expectedKMSCloseGracePeriod := 46 * time.Second correctConfigWithIdentityFirst := "testdata/valid-configs/identity-first.yaml" identityFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithIdentityFirst, false) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithIdentityFirst) } + if identityFirstEncryptionConfiguration.KMSCloseGracePeriod != expectedKMSCloseGracePeriod { + t.Fatalf("KMSCloseGracePeriod mismatch (-want +got):\n%s", cmp.Diff(expectedKMSCloseGracePeriod, identityFirstEncryptionConfiguration.KMSCloseGracePeriod)) + } + // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 + expectedKMSCloseGracePeriod = 32 * time.Second correctConfigWithAesGcmFirst := "testdata/valid-configs/aes-gcm-first.yaml" aesGcmFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithAesGcmFirst, false) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesGcmFirst) } + if aesGcmFirstEncryptionConfiguration.KMSCloseGracePeriod != expectedKMSCloseGracePeriod { + t.Fatalf("KMSCloseGracePeriod mismatch (-want +got):\n%s", cmp.Diff(expectedKMSCloseGracePeriod, aesGcmFirstEncryptionConfiguration.KMSCloseGracePeriod)) + } + // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 + expectedKMSCloseGracePeriod = 26 * time.Second correctConfigWithAesCbcFirst := "testdata/valid-configs/aes-cbc-first.yaml" aesCbcFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithAesCbcFirst, false) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesCbcFirst) } + if aesCbcFirstEncryptionConfiguration.KMSCloseGracePeriod != expectedKMSCloseGracePeriod { + t.Fatalf("KMSCloseGracePeriod mismatch (-want +got):\n%s", cmp.Diff(expectedKMSCloseGracePeriod, aesCbcFirstEncryptionConfiguration.KMSCloseGracePeriod)) + } + // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 + expectedKMSCloseGracePeriod = 14 * time.Second correctConfigWithSecretboxFirst := "testdata/valid-configs/secret-box-first.yaml" secretboxFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithSecretboxFirst, false) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithSecretboxFirst) } + if secretboxFirstEncryptionConfiguration.KMSCloseGracePeriod != expectedKMSCloseGracePeriod { + t.Fatalf("KMSCloseGracePeriod mismatch (-want +got):\n%s", cmp.Diff(expectedKMSCloseGracePeriod, secretboxFirstEncryptionConfiguration.KMSCloseGracePeriod)) + } + // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 + expectedKMSCloseGracePeriod = 34 * time.Second correctConfigWithKMSFirst := "testdata/valid-configs/kms-first.yaml" kmsFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithKMSFirst, false) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSFirst) } + if kmsFirstEncryptionConfiguration.KMSCloseGracePeriod != expectedKMSCloseGracePeriod { + t.Fatalf("KMSCloseGracePeriod mismatch (-want +got):\n%s", cmp.Diff(expectedKMSCloseGracePeriod, kmsFirstEncryptionConfiguration.KMSCloseGracePeriod)) + } + // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 + expectedKMSCloseGracePeriod = 42 * time.Second correctConfigWithKMSv2First := "testdata/valid-configs/kmsv2-first.yaml" kmsv2FirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithKMSv2First, false) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSv2First) } + if kmsv2FirstEncryptionConfiguration.KMSCloseGracePeriod != expectedKMSCloseGracePeriod { + t.Fatalf("KMSCloseGracePeriod mismatch (-want +got):\n%s", cmp.Diff(expectedKMSCloseGracePeriod, kmsv2FirstEncryptionConfiguration.KMSCloseGracePeriod)) + } // Pick the transformer for any of the returned resources. identityFirstTransformer := identityFirstEncryptionConfiguration.Transformers[schema.ParseGroupResource("secrets")] diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/aes-cbc-first.yaml b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/aes-cbc-first.yaml index ed1e5b2f695..3da61de63aa 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/aes-cbc-first.yaml +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/aes-cbc-first.yaml @@ -14,6 +14,7 @@ resources: name: testprovider endpoint: unix:///tmp/testprovider.sock cachesize: 10 + timeout: 5s - kms: apiVersion: v2 name: testproviderv2 diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/aes-gcm-first.yaml b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/aes-gcm-first.yaml index 9f98f5de232..f06ef230e63 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/aes-gcm-first.yaml +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/aes-gcm-first.yaml @@ -22,6 +22,7 @@ resources: apiVersion: v2 name: testproviderv2 endpoint: unix:///tmp/testprovider.sock + timeout: 10s - aescbc: keys: - name: key1 diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/identity-first.yaml b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/identity-first.yaml index 0ebea166bda..e57adc3f102 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/identity-first.yaml +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/identity-first.yaml @@ -16,6 +16,7 @@ resources: name: testprovider endpoint: unix:///tmp/testprovider.sock cachesize: 10 + timeout: 10s - kms: apiVersion: v2 name: testproviderv2 diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kms-first.yaml b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kms-first.yaml index 3bf881ae469..580cd76b508 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kms-first.yaml +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kms-first.yaml @@ -8,6 +8,7 @@ resources: name: testprovider endpoint: unix:///tmp/testprovider.sock cachesize: 10 + timeout: 7s - kms: apiVersion: v2 name: testproviderv2 diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kmsv2-first.yaml b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kmsv2-first.yaml index 673759ded23..afda2e06470 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kmsv2-first.yaml +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kmsv2-first.yaml @@ -8,6 +8,7 @@ resources: apiVersion: v2 name: testproviderv2 endpoint: unix:///tmp/testprovider.sock + timeout: 15s - kms: name: testprovider endpoint: unix:///tmp/testprovider.sock diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/secret-box-first.yaml b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/secret-box-first.yaml index 2bbdb6a8161..fc8839e6d62 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/secret-box-first.yaml +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/secret-box-first.yaml @@ -22,6 +22,7 @@ resources: apiVersion: v2 name: testproviderv2 endpoint: unix:///tmp/testprovider.sock + timeout: 1s - identity: {} - aesgcm: keys: diff --git a/test/integration/controlplane/transformation/all_transformation_test.go b/test/integration/controlplane/transformation/all_transformation_test.go index cf262683cce..3a9751cf445 100644 --- a/test/integration/controlplane/transformation/all_transformation_test.go +++ b/test/integration/controlplane/transformation/all_transformation_test.go @@ -94,7 +94,7 @@ resources: - name: key1 secret: c2VjcmV0IGlzIHNlY3VyZQ== ` - test, err := newTransformTest(t, encryptionConfig, false, "", false) + test, err := newTransformTest(t, encryptionConfig, false, "") if err != nil { t.Fatalf("failed to start Kube API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } diff --git a/test/integration/controlplane/transformation/kms_transformation_test.go b/test/integration/controlplane/transformation/kms_transformation_test.go index 3ef3df0382c..27920aaf1c1 100644 --- a/test/integration/controlplane/transformation/kms_transformation_test.go +++ b/test/integration/controlplane/transformation/kms_transformation_test.go @@ -29,6 +29,7 @@ import ( "math/rand" "os" "path" + "path/filepath" "strings" "testing" "time" @@ -131,7 +132,7 @@ resources: } defer pluginMock.CleanUp() - test, err := newTransformTest(t, encryptionConfig, false, "", false) + test, err := newTransformTest(t, encryptionConfig, false, "") if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -317,7 +318,7 @@ resources: defer pluginMock.CleanUp() var restarted bool - test, err := newTransformTest(t, encryptionConfig, true, "", false) + test, err := newTransformTest(t, encryptionConfig, true, "") if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -515,35 +516,80 @@ resources: previousConfigDir := test.configDir test.shutdownAPIServer() restarted = true - test, err = newTransformTest(t, "", true, previousConfigDir, false) + test, err = newTransformTest(t, "", true, previousConfigDir) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } - test.cleanUp() + defer test.cleanUp() + + // confirm that reading cluster wide secrets still works after restart + if _, err = test.restClient.CoreV1().Secrets("").List(context.TODO(), metav1.ListOptions{}); err != nil { + t.Fatalf("failed to list secrets, err: %v", err) + } + + // make sure cluster wide configmaps read still works + if _, err = test.restClient.CoreV1().ConfigMaps("").List(context.TODO(), metav1.ListOptions{}); err != nil { + t.Fatalf("failed to list configmaps, err: %v", err) + } } func TestEncryptionConfigHotReloadFileWatch(t *testing.T) { testCases := []struct { - fileUpdateMethod string - sleep time.Duration + sleep time.Duration + name string + updateFile func(filePath, fileContent string) error }{ { - fileUpdateMethod: "truncate", - sleep: 20 * time.Second, // significantly longer than KMSCloseGracePeriod + name: "truncate file", + updateFile: func(filePath string, fileContent string) error { + // os.WriteFile truncates the file before writing + return os.WriteFile(filePath, []byte(fileContent), 0644) + }, + // significantly longer than KMSCloseGracePeriod + sleep: 20 * time.Second, }, { - fileUpdateMethod: "deleteAndCreate", + name: "delete and create file", + updateFile: func(filePath, fileContent string) error { + // os.Remove deletes the file before creating a new one + if err := os.Remove(filePath); err != nil { + return fmt.Errorf("failed to remove encryption config, err: %w", err) + } + + file, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("failed to create encryption config, err: %w", err) + } + defer file.Close() + + if _, err := file.Write([]byte(fileContent)); err != nil { + return fmt.Errorf("failed to write encryption config, err: %w", err) + } + + return nil + }, }, { - fileUpdateMethod: "move", - }, - { - fileUpdateMethod: "symLink", + name: "move file", + updateFile: func(filePath, fileContent string) error { + // write new config to a temp file + tmpFilePath := filePath + ".tmp" + if err := os.WriteFile(tmpFilePath, []byte(fileContent), 0644); err != nil { + return fmt.Errorf("failed to write config to tmp file, err: %w", err) + } + + // move the temp file to the original file + if err := os.Rename(tmpFilePath, filePath); err != nil { + return fmt.Errorf("failed to move encryption config, err: %w", err) + } + + return nil + }, }, } for _, tc := range testCases { - t.Run(tc.fileUpdateMethod, func(t *testing.T) { + t.Run(tc.name, func(t *testing.T) { encryptionConfig := ` kind: EncryptionConfiguration apiVersion: apiserver.config.k8s.io/v1 @@ -568,17 +614,10 @@ resources: } defer pluginMock.CleanUp() - var test *transformTest - if tc.fileUpdateMethod == "symLink" { - test, err = newTransformTest(t, encryptionConfig, true, "", true) - if err != nil { - t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) - } - } else { - test, err = newTransformTest(t, encryptionConfig, true, "", false) - if err != nil { - t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) - } + test, err := newTransformTest(t, encryptionConfig, true, "") + if err != nil { + test.cleanUp() + t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } defer test.cleanUp() @@ -629,89 +668,13 @@ resources: } defer newPluginMock.CleanUp() - switch tc.fileUpdateMethod { - case "truncate": - // update encryption config - // os.WriteFile truncates the file before writing - if err := os.WriteFile(path.Join(test.configDir, encryptionConfigFileName), []byte(encryptionConfigWithNewProvider), 0644); err != nil { - t.Fatalf("failed to update encryption config, err: %v", err) - } - case "deleteAndCreate": - // update encryption config - // os.Remove deletes the file before creating a new one - if err := os.Remove(path.Join(test.configDir, encryptionConfigFileName)); err != nil { - t.Fatalf("failed to remove encryption config, err: %v", err) - } - file, err := os.Create(path.Join(test.configDir, encryptionConfigFileName)) - if err != nil { - t.Fatalf("failed to create encryption config, err: %v", err) - } - if _, err := file.Write([]byte(encryptionConfigWithNewProvider)); err != nil { - t.Fatalf("failed to write encryption config, err: %v", err) - } - if err := file.Close(); err != nil { - t.Fatalf("failed to close encryption config, err: %v", err) - } - case "move": - // update encryption config - // write new config to a temp file - if err := os.WriteFile(path.Join(test.configDir, encryptionConfigFileName+".tmp"), []byte(encryptionConfigWithNewProvider), 0644); err != nil { - t.Fatalf("failed to write config to tmp file, err: %v", err) - } - - // move the temp file to the original file - if err := os.Rename(path.Join(test.configDir, encryptionConfigFileName+".tmp"), path.Join(test.configDir, encryptionConfigFileName)); err != nil { - t.Fatalf("failed to move encryption config, err: %v", err) - } - case "symLink": - // update encryption config - // write new config in a parent directory. - if err := os.WriteFile(path.Join(test.configParentDir, encryptionConfigFileName), []byte(encryptionConfigWithNewProvider), 0644); err != nil { - t.Fatalf("failed to update encryption config, err: %v", err) - } - default: - t.Fatalf("unknown file update method: %s", tc.fileUpdateMethod) + // update encryption config + if err := tc.updateFile(filepath.Join(test.configDir, encryptionConfigFileName), encryptionConfigWithNewProvider); err != nil { + t.Fatalf("failed to update encryption config, err: %v", err) } wantPrefix := "k8s:enc:kms:v1:new-kms-provider-for-secrets:" - - // implementing this brute force approach instead of fancy channel notification to avoid test specific code in prod. - // wait for config to be observed - verifyIfKMSTransformersSwapped(t, wantPrefix, test) - - // run storage migration - // get secrets - secretsList, err := test.restClient.CoreV1().Secrets("").List( - context.TODO(), - metav1.ListOptions{}, - ) - if err != nil { - t.Fatalf("failed to list secrets, err: %v", err) - } - - for _, secret := range secretsList.Items { - // update secret - _, err = test.restClient.CoreV1().Secrets(secret.Namespace).Update( - context.TODO(), - &secret, - metav1.UpdateOptions{}, - ) - if err != nil { - t.Fatalf("failed to update secret, err: %v", err) - } - } - - // assert that resources has new prefix - secretETCDPath := test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", test.secret.Name, test.secret.Namespace) - rawEnvelope, err := test.getRawSecretFromETCD() - if err != nil { - t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err) - } - - // assert secret - if !bytes.HasPrefix(rawEnvelope, []byte(wantPrefix)) { - t.Fatalf("expected secret to be prefixed with %s, but got %s", wantPrefix, rawEnvelope) - } + verifyPrefixOfSecretResource(t, wantPrefix, test) // make sure things still work at a "later" time if tc.sleep != 0 { @@ -732,6 +695,43 @@ resources: } } +func verifyPrefixOfSecretResource(t *testing.T, wantPrefix string, test *transformTest) { + // implementing this brute force approach instead of fancy channel notification to avoid test specific code in prod. + // wait for config to be observed + verifyIfKMSTransformersSwapped(t, wantPrefix, test) + + // run storage migration + secretsList, err := test.restClient.CoreV1().Secrets("").List( + context.TODO(), + metav1.ListOptions{}, + ) + if err != nil { + t.Fatalf("failed to list secrets, err: %v", err) + } + + for _, secret := range secretsList.Items { + _, err = test.restClient.CoreV1().Secrets(secret.Namespace).Update( + context.TODO(), + &secret, + metav1.UpdateOptions{}, + ) + if err != nil { + t.Fatalf("failed to update secret, err: %v", err) + } + } + + secretETCDPath := test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", test.secret.Name, test.secret.Namespace) + rawEnvelope, err := test.getRawSecretFromETCD() + if err != nil { + t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err) + } + + // assert that resources has new prefix + if !bytes.HasPrefix(rawEnvelope, []byte(wantPrefix)) { + t.Fatalf("expected secret to be prefixed with %s, but got %s", wantPrefix, rawEnvelope) + } +} + func verifyIfKMSTransformersSwapped(t *testing.T, wantPrefix string, test *transformTest) { t.Helper() @@ -811,7 +811,7 @@ resources: t.Fatalf("Failed to start KMS Plugin #2: err: %v", err) } - test, err := newTransformTest(t, encryptionConfig, false, "", false) + test, err := newTransformTest(t, encryptionConfig, false, "") if err != nil { t.Fatalf("Failed to start kube-apiserver, error: %v", err) } @@ -889,7 +889,7 @@ resources: t.Fatalf("Failed to start KMS Plugin #2: err: %v", err) } - test, err := newTransformTest(t, encryptionConfig, true, "", false) + test, err := newTransformTest(t, encryptionConfig, true, "") if err != nil { t.Fatalf("Failed to start kube-apiserver, error: %v", err) } diff --git a/test/integration/controlplane/transformation/kmsv2_transformation_test.go b/test/integration/controlplane/transformation/kmsv2_transformation_test.go index 978f62cc728..82ab59f3f3e 100644 --- a/test/integration/controlplane/transformation/kmsv2_transformation_test.go +++ b/test/integration/controlplane/transformation/kmsv2_transformation_test.go @@ -140,7 +140,7 @@ resources: } defer pluginMock.CleanUp() - test, err := newTransformTest(t, encryptionConfig, false, "", false) + test, err := newTransformTest(t, encryptionConfig, false, "") if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -241,7 +241,7 @@ resources: } defer pluginMock.CleanUp() - test, err := newTransformTest(t, encryptionConfig, false, "", false) + test, err := newTransformTest(t, encryptionConfig, false, "") if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -367,7 +367,7 @@ resources: t.Fatalf("Failed to start KMS Plugin #2: err: %v", err) } - test, err := newTransformTest(t, encryptionConfig, false, "", false) + test, err := newTransformTest(t, encryptionConfig, false, "") if err != nil { t.Fatalf("Failed to start kube-apiserver, error: %v", err) } @@ -454,7 +454,7 @@ resources: } t.Cleanup(pluginMock.CleanUp) - test, err := newTransformTest(t, encryptionConfig, false, "", false) + test, err := newTransformTest(t, encryptionConfig, false, "") if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } diff --git a/test/integration/controlplane/transformation/secrets_transformation_test.go b/test/integration/controlplane/transformation/secrets_transformation_test.go index ab3b199de0f..820836bc19e 100644 --- a/test/integration/controlplane/transformation/secrets_transformation_test.go +++ b/test/integration/controlplane/transformation/secrets_transformation_test.go @@ -85,7 +85,7 @@ func TestSecretsShouldBeTransformed(t *testing.T) { // TODO: add secretbox } for _, tt := range testCases { - test, err := newTransformTest(t, tt.transformerConfigContent, false, "", false) + test, err := newTransformTest(t, tt.transformerConfigContent, false, "") if err != nil { test.cleanUp() t.Errorf("failed to setup test for envelop %s, error was %v", tt.transformerPrefix, err) @@ -120,7 +120,7 @@ func BenchmarkAESCBCEnvelopeWrite(b *testing.B) { func runBenchmark(b *testing.B, transformerConfig string) { b.StopTimer() - test, err := newTransformTest(b, transformerConfig, false, "", false) + test, err := newTransformTest(b, transformerConfig, false, "") defer test.cleanUp() if err != nil { b.Fatalf("failed to setup benchmark for config %s, error was %v", transformerConfig, err) diff --git a/test/integration/controlplane/transformation/transformation_test.go b/test/integration/controlplane/transformation/transformation_test.go index 050b86198ac..b090cfae133 100644 --- a/test/integration/controlplane/transformation/transformation_test.go +++ b/test/integration/controlplane/transformation/transformation_test.go @@ -22,7 +22,6 @@ import ( "encoding/json" "fmt" "os" - "path" "path/filepath" "strconv" "strings" @@ -75,7 +74,6 @@ type transformTest struct { logger kubeapiservertesting.Logger storageConfig *storagebackend.Config configDir string - configParentDir string transformerConfig string kubeAPIServer kubeapiservertesting.TestServer restClient *kubernetes.Clientset @@ -83,7 +81,7 @@ type transformTest struct { secret *corev1.Secret } -func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML string, reload bool, configDir string, ecSymLink bool) (*transformTest, error) { +func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML string, reload bool, configDir string) (*transformTest, error) { e := transformTest{ logger: l, transformerConfig: transformerConfigYAML, @@ -93,7 +91,7 @@ func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML strin var err error // create config dir with provided config yaml if transformerConfigYAML != "" && configDir == "" { - if e.configDir, e.configParentDir, err = e.createEncryptionConfig(ecSymLink); err != nil { + if e.configDir, err = e.createEncryptionConfig(); err != nil { return nil, fmt.Errorf("error while creating KubeAPIServer encryption config: %v", err) } } else { @@ -128,8 +126,10 @@ func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML strin func (e *transformTest) cleanUp() { os.RemoveAll(e.configDir) - os.RemoveAll(e.configParentDir) - e.shutdownAPIServer() + + if e.kubeAPIServer.ClientConfig != nil { + e.shutdownAPIServer() + } } func (e *transformTest) shutdownAPIServer() { @@ -254,7 +254,7 @@ func (e *transformTest) getRawSecretFromETCD() ([]byte, error) { func (e *transformTest) getEncryptionOptions(reload bool) []string { if e.transformerConfig != "" { return []string{ - "--encryption-provider-config", path.Join(e.configDir, encryptionConfigFileName), + "--encryption-provider-config", filepath.Join(e.configDir, encryptionConfigFileName), fmt.Sprintf("--encryption-provider-config-automatic-reload=%v", reload), "--disable-admission-plugins", "ServiceAccount"} } @@ -262,40 +262,21 @@ func (e *transformTest) getEncryptionOptions(reload bool) []string { return nil } -func (e *transformTest) createEncryptionConfig(ecSymLink bool) (string, string, error) { +func (e *transformTest) createEncryptionConfig() ( + filePathForEncryptionConfig string, + err error, +) { tempDir, err := os.MkdirTemp("", "secrets-encryption-test") if err != nil { - return "", "", fmt.Errorf("failed to create temp directory: %v", err) + return "", fmt.Errorf("failed to create temp directory: %v", err) } - if ecSymLink { - // create another temp dir - parentTempDir, err := os.MkdirTemp("", "secrets-encryption-symlink-test") - if err != nil { - return tempDir, "", fmt.Errorf("failed to create temp directory: %v", err) - } - - // create config file - if err := os.WriteFile(filepath.Join(parentTempDir, encryptionConfigFileName), []byte(e.transformerConfig), 0644); err != nil { - return tempDir, parentTempDir, fmt.Errorf("failed to write encryption config file: %v", err) - } - - // create symlink - if err := os.Symlink(filepath.Join(parentTempDir, encryptionConfigFileName), filepath.Join(tempDir, encryptionConfigFileName)); err != nil { - return tempDir, parentTempDir, fmt.Errorf("failed to create symlink: %v", err) - } - - return tempDir, parentTempDir, nil - } - - encryptionConfig := path.Join(tempDir, encryptionConfigFileName) - - if err := os.WriteFile(encryptionConfig, []byte(e.transformerConfig), 0644); err != nil { + if err = os.WriteFile(filepath.Join(tempDir, encryptionConfigFileName), []byte(e.transformerConfig), 0644); err != nil { os.RemoveAll(tempDir) - return tempDir, "", fmt.Errorf("error while writing encryption config: %v", err) + return tempDir, fmt.Errorf("error while writing encryption config: %v", err) } - return tempDir, "", nil + return tempDir, nil } func (e *transformTest) getEncryptionConfig() (*apiserverconfigv1.ProviderConfiguration, error) {