From 67769438e1dfa774c63135a2d526d769f8ff8c1d Mon Sep 17 00:00:00 2001 From: Rita Zhang Date: Wed, 19 Jul 2023 22:38:25 -0700 Subject: [PATCH] kmsv2 test feature enablement disablement and restart Signed-off-by: Rita Zhang --- .../transformation/all_transformation_test.go | 2 +- .../transformation/kms_transformation_test.go | 52 +++-- .../kmsv2_transformation_test.go | 188 +++++++++++++++++- .../controlplane/transformation/main_test.go | 8 + .../secrets_transformation_test.go | 9 +- .../transformation/transformation_test.go | 40 +++- 6 files changed, 256 insertions(+), 43 deletions(-) diff --git a/test/integration/controlplane/transformation/all_transformation_test.go b/test/integration/controlplane/transformation/all_transformation_test.go index 3a9751cf445..4ca250f234a 100644 --- a/test/integration/controlplane/transformation/all_transformation_test.go +++ b/test/integration/controlplane/transformation/all_transformation_test.go @@ -94,7 +94,7 @@ resources: - name: key1 secret: c2VjcmV0IGlzIHNlY3VyZQ== ` - test, err := newTransformTest(t, encryptionConfig, false, "") + test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { t.Fatalf("failed to start Kube API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } diff --git a/test/integration/controlplane/transformation/kms_transformation_test.go b/test/integration/controlplane/transformation/kms_transformation_test.go index 2bde85f284c..9d659205cf8 100644 --- a/test/integration/controlplane/transformation/kms_transformation_test.go +++ b/test/integration/controlplane/transformation/kms_transformation_test.go @@ -28,7 +28,6 @@ import ( "fmt" "math/rand" "os" - "path" "path/filepath" "strings" "testing" @@ -49,6 +48,7 @@ import ( kmsapi "k8s.io/kms/apis/v1beta1" "k8s.io/kubernetes/test/integration" "k8s.io/kubernetes/test/integration/etcd" + "k8s.io/kubernetes/test/integration/framework" ) const ( @@ -133,7 +133,7 @@ resources: ` providerName := "kms-provider" pluginMock := mock.NewBase64Plugin(t, "@kms-provider.sock") - test, err := newTransformTest(t, encryptionConfig, false, "") + test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -295,6 +295,7 @@ resources: // 10. confirm that cluster wide secret read still works // 11. confirm that api server can restart with last applied encryption config func TestEncryptionConfigHotReload(t *testing.T) { + storageConfig := framework.SharedEtcd() encryptionConfig := ` kind: EncryptionConfiguration apiVersion: apiserver.config.k8s.io/v1 @@ -309,7 +310,7 @@ resources: ` _ = mock.NewBase64Plugin(t, "@kms-provider.sock") var restarted bool - test, err := newTransformTest(t, encryptionConfig, true, "") + test, err := newTransformTest(t, encryptionConfig, true, "", storageConfig) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -365,7 +366,7 @@ resources: // start new KMS Plugin _ = mock.NewBase64Plugin(t, "@new-kms-provider.sock") // update encryption config - if err := os.WriteFile(path.Join(test.configDir, encryptionConfigFileName), []byte(encryptionConfigWithNewProvider), 0644); err != nil { + if err := os.WriteFile(filepath.Join(test.configDir, encryptionConfigFileName), []byte(encryptionConfigWithNewProvider), 0644); err != nil { t.Fatalf("failed to update encryption config, err: %v", err) } @@ -377,8 +378,9 @@ resources: // run storage migration // get secrets + ctx := testContext(t) secretsList, err := test.restClient.CoreV1().Secrets("").List( - context.TODO(), + ctx, metav1.ListOptions{}, ) if err != nil { @@ -388,7 +390,7 @@ resources: for _, secret := range secretsList.Items { // update secret _, err = test.restClient.CoreV1().Secrets(secret.Namespace).Update( - context.TODO(), + ctx, &secret, metav1.UpdateOptions{}, ) @@ -399,7 +401,7 @@ resources: // get configmaps configmapsList, err := test.restClient.CoreV1().ConfigMaps("").List( - context.TODO(), + ctx, metav1.ListOptions{}, ) if err != nil { @@ -409,7 +411,7 @@ resources: for _, configmap := range configmapsList.Items { // update configmap _, err = test.restClient.CoreV1().ConfigMaps(configmap.Namespace).Update( - context.TODO(), + ctx, &configmap, metav1.UpdateOptions{}, ) @@ -463,7 +465,7 @@ resources: ` // update encryption config and wait for hot reload - if err := os.WriteFile(path.Join(test.configDir, encryptionConfigFileName), []byte(encryptionConfigWithoutOldProvider), 0644); err != nil { + if err := os.WriteFile(filepath.Join(test.configDir, encryptionConfigFileName), []byte(encryptionConfigWithoutOldProvider), 0644); err != nil { t.Fatalf("failed to update encryption config, err: %v", err) } @@ -472,7 +474,7 @@ resources: // confirm that reading secrets still works _, err = test.restClient.CoreV1().Secrets(testNamespace).Get( - context.TODO(), + ctx, testSecret, metav1.GetOptions{}, ) @@ -481,13 +483,13 @@ resources: } // make sure cluster wide secrets read still works - _, err = test.restClient.CoreV1().Secrets("").List(context.TODO(), metav1.ListOptions{}) + _, err = test.restClient.CoreV1().Secrets("").List(ctx, metav1.ListOptions{}) if err != nil { t.Fatalf("failed to list secrets, err: %v", err) } // make sure cluster wide configmaps read still works - _, err = test.restClient.CoreV1().ConfigMaps("").List(context.TODO(), metav1.ListOptions{}) + _, err = test.restClient.CoreV1().ConfigMaps("").List(ctx, metav1.ListOptions{}) if err != nil { t.Fatalf("failed to list configmaps, err: %v", err) } @@ -496,19 +498,28 @@ resources: previousConfigDir := test.configDir test.shutdownAPIServer() restarted = true - test, err = newTransformTest(t, "", true, previousConfigDir) + test, err = newTransformTest(t, test.transformerConfig, true, previousConfigDir, storageConfig) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } defer test.cleanUp() + _, err = test.restClient.CoreV1().Secrets(testNamespace).Get( + ctx, + testSecret, + metav1.GetOptions{}, + ) + if err != nil { + t.Fatalf("failed to read secret, err: %v", err) + } + // confirm that reading cluster wide secrets still works after restart - if _, err = test.restClient.CoreV1().Secrets("").List(context.TODO(), metav1.ListOptions{}); err != nil { + if _, err = test.restClient.CoreV1().Secrets("").List(ctx, metav1.ListOptions{}); err != nil { t.Fatalf("failed to list secrets, err: %v", err) } // make sure cluster wide configmaps read still works - if _, err = test.restClient.CoreV1().ConfigMaps("").List(context.TODO(), metav1.ListOptions{}); err != nil { + if _, err = test.restClient.CoreV1().ConfigMaps("").List(ctx, metav1.ListOptions{}); err != nil { t.Fatalf("failed to list configmaps, err: %v", err) } } @@ -531,7 +542,7 @@ resources: _ = mock.NewBase64Plugin(t, "@encrypt-all-kms-provider.sock") defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)() defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)() - test, err := newTransformTest(t, encryptionConfig, false, "") + test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig") } @@ -643,7 +654,7 @@ resources: _ = mock.NewBase64Plugin(t, "@kms-provider.sock") _ = mock.NewBase64Plugin(t, "@encrypt-all-kms-provider.sock") - test, err := newTransformTest(t, encryptionConfig, false, "") + test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -785,9 +796,8 @@ resources: ` _ = mock.NewBase64Plugin(t, "@kms-provider.sock") - test, err := newTransformTest(t, encryptionConfig, true, "") + test, err := newTransformTest(t, encryptionConfig, true, "", nil) if err != nil { - test.cleanUp() t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } defer test.cleanUp() @@ -950,7 +960,7 @@ resources: pluginMock1 := mock.NewBase64Plugin(t, "@kms-provider-1.sock") pluginMock2 := mock.NewBase64Plugin(t, "@kms-provider-2.sock") - test, err := newTransformTest(t, encryptionConfig, false, "") + test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { t.Fatalf("failed to start kube-apiserver, error: %v", err) } @@ -1006,7 +1016,7 @@ resources: pluginMock1 := mock.NewBase64Plugin(t, "@kms-provider-1.sock") pluginMock2 := mock.NewBase64Plugin(t, "@kms-provider-2.sock") - test, err := newTransformTest(t, encryptionConfig, true, "") + test, err := newTransformTest(t, encryptionConfig, true, "", nil) if err != nil { t.Fatalf("Failed to start kube-apiserver, error: %v", err) } diff --git a/test/integration/controlplane/transformation/kmsv2_transformation_test.go b/test/integration/controlplane/transformation/kmsv2_transformation_test.go index e088c94adbd..c41db9726d0 100644 --- a/test/integration/controlplane/transformation/kmsv2_transformation_test.go +++ b/test/integration/controlplane/transformation/kmsv2_transformation_test.go @@ -69,6 +69,7 @@ import ( secretstore "k8s.io/kubernetes/pkg/registry/core/secret/storage" "k8s.io/kubernetes/test/integration" "k8s.io/kubernetes/test/integration/etcd" + "k8s.io/kubernetes/test/integration/framework" ) type envelopekmsv2 struct { @@ -194,7 +195,7 @@ resources: providerName := "kms-provider" pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock") - test, err := newTransformTest(t, encryptionConfig, false, "") + test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -301,7 +302,7 @@ resources: ` pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock") - test, err := newTransformTest(t, encryptionConfig, false, "") + test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -597,7 +598,7 @@ resources: ` _ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock") - test, err := newTransformTest(t, encryptionConfig, false, "") + test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -736,7 +737,7 @@ resources: pluginMock1 := kmsv2mock.NewBase64Plugin(t, "@kms-provider-1.sock") pluginMock2 := kmsv2mock.NewBase64Plugin(t, "@kms-provider-2.sock") - test, err := newTransformTest(t, encryptionConfig, false, "") + test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { t.Fatalf("Failed to start kube-apiserver, error: %v", err) } @@ -815,7 +816,7 @@ resources: _ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock") - test, err := newTransformTest(t, encryptionConfig, false, "") + test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -847,6 +848,179 @@ resources: } } +// TestKMSv2FeatureFlag is an integration test between KubeAPI and ETCD +// Concretely, this test verifies the following: +// 1. When feature flag is not enabled, loading a encryptionConfig with KMSv2 should fail +// 2. When feature flag is enabled, loading a encryptionConfig with KMSv2 should work +// 3. When feature flag is disabled, loading a encryptionConfig with a non-v2 provider should work. +// without performing a storage migration, decryption of existing data encrypted with v2 should fail for Get and List operations. +// New data stored in etcd will no longer be encrypted using the external kms provider with v2 API. +// 4. when feature flag is re-enabled, loading a encryptionConfig with the same KMSv2 plugin from 2 should work, +// decryption of data encrypted with v2 should work +func TestKMSv2FeatureFlag(t *testing.T) { + encryptionConfig := ` +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: + - resources: + - secrets + providers: + - kms: + apiVersion: v2 + name: kms-provider + endpoint: unix:///@kms-provider.sock +` + // TOODO: This test requires changes in the apiserver lifecycle code to correctly clean up goroutines after a server failure. + // When feature flag is not enabled, loading a encryptionConfig with KMSv2 should fail + // defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, false)() + // _, err := newTransformTest(t, encryptionConfig, true, "", nil) + // if err == nil || !strings.Contains(err.Error(), "KMSv2 feature is not enabled") { + // t.Fatalf("when feature flag is not enabled, loading a encryptionConfig with KMSv2 should have failed with: KMSv2 feature is not enabled, encryptionConfig:\n%s\n actual error: %v", encryptionConfig, err) + // } + + providerName := "kms-provider" + pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock") + storageConfig := framework.SharedEtcd() + + // When feature flag is enabled, loading a encryptionConfig with KMSv1 and v2 should work + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() + test, err := newTransformTest(t, encryptionConfig, false, "", storageConfig) + if err != nil { + t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) + } + defer func() { + test.cleanUp() + }() + + test.secret, err = test.createSecret(testSecret, testNamespace) + if err != nil { + t.Fatalf("Failed to create test secret, error: %v", err) + } + + // Since Data Encryption Key (DEK) is randomly generated, we need to ask KMS Mock for it. + plainTextDEKSource := pluginMock.LastEncryptRequest() + + secretETCDPath := test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", test.secret.Name, test.secret.Namespace) + rawEnvelope, err := test.getRawSecretFromETCD() + if err != nil { + t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err) + } + + envelopeData := envelopekmsv2{ + providerName: providerName, + rawEnvelope: rawEnvelope, + plainTextDEKSource: plainTextDEKSource, + } + + wantPrefix := envelopeData.prefix() + if !bytes.HasPrefix(rawEnvelope, []byte(wantPrefix)) { + t.Fatalf("expected secret to be prefixed with %s, but got %s", wantPrefix, rawEnvelope) + } + + ctx := testContext(t) + ciphertext, err := envelopeData.cipherTextDEKSource() + if err != nil { + t.Fatalf("failed to get ciphertext DEK from KMSv2 Plugin: %v", err) + } + decryptResponse, err := pluginMock.Decrypt(ctx, &kmsv2api.DecryptRequest{Uid: string(uuid.NewUUID()), Ciphertext: ciphertext}) + if err != nil { + t.Fatalf("failed to decrypt DEK, %v", err) + } + dekPlainAsWouldBeSeenByETCD := decryptResponse.Plaintext + + if !bytes.Equal(plainTextDEKSource, dekPlainAsWouldBeSeenByETCD) { + t.Fatalf("expected plainTextDEKSource %v to be passed to KMS Plugin, but got %s", + plainTextDEKSource, dekPlainAsWouldBeSeenByETCD) + } + + plainSecret, err := envelopeData.plainTextPayload(secretETCDPath) + if err != nil { + t.Fatalf("failed to transform from storage via AESGCM, err: %v", err) + } + + if !strings.Contains(string(plainSecret), secretVal) { + t.Fatalf("expected %q after decryption, but got %q", secretVal, string(plainSecret)) + } + + secretClient := test.restClient.CoreV1().Secrets(testNamespace) + // Secrets should be un-enveloped on direct reads from Kube API Server. + s, err := secretClient.Get(ctx, testSecret, metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get Secret from %s, err: %v", testNamespace, err) + } + if secretVal != string(s.Data[secretKey]) { + t.Fatalf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey])) + } + test.shutdownAPIServer() + + // When KMSv2 feature flag is disabled, loading a encryptionConfig with a non-v2 provider should work. without performing a storage migration, decryption of existing data encrypted with v2 should fail for Get and List operations. + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, false)() + + encryptionConfig1 := ` +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: + - resources: + - secrets + providers: + - aescbc: + keys: + - name: key1 + secret: c2VjcmV0IGlzIHNlY3VyZQ== +` + test, err = newTransformTest(t, encryptionConfig1, false, "", storageConfig) + if err != nil { + t.Fatalf("Failed to restart api server, error: %v", err) + } + + _, err = test.createSecret("test2", testNamespace) + if err != nil { + t.Fatalf("Failed to create test secret, error: %v", err) + } + test.runResource(t, unSealWithCBCTransformer, aesCBCPrefix, "", "v1", "secrets", "test2", testNamespace) + + secretClient = test.restClient.CoreV1().Secrets(testNamespace) + + // Getting an old secret that was encrypted by another provider should fail + _, err = secretClient.Get(ctx, testSecret, metav1.GetOptions{}) + if err == nil || !strings.Contains(err.Error(), "no matching prefix found") { + t.Fatalf("using a new provider, get Secret %s from %s should return err containing: no matching prefix found. Got err: %v", testSecret, testNamespace, err) + } + // List all cluster wide secrets should fail + _, err = test.restClient.CoreV1().Secrets("").List(ctx, metav1.ListOptions{}) + if err == nil || !strings.Contains(err.Error(), "no matching prefix found") { + t.Fatalf("using a new provider, LIST all Secrets should return err containing: no matching prefix found. Got err: %v", err) + } + test.shutdownAPIServer() + + // when feature flag is re-enabled, loading a encryptionConfig with the same KMSv2 plugin before the restart should work, decryption of data encrypted with v2 should work + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() + + test, err = newTransformTest(t, encryptionConfig, false, "", storageConfig) + if err != nil { + t.Fatalf("Failed to restart api server, error: %v", err) + } + + // Getting an old secret that was encrypted by the same plugin should not fail. + s, err = test.restClient.CoreV1().Secrets(testNamespace).Get( + ctx, + testSecret, + metav1.GetOptions{}, + ) + if err != nil { + t.Fatalf("failed to read secret, err: %v", err) + } + if secretVal != string(s.Data[secretKey]) { + t.Fatalf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey])) + } + secretClient = test.restClient.CoreV1().Secrets(testNamespace) + // Getting an old secret that was encrypted by another plugin should fail + _, err = secretClient.Get(ctx, "test2", metav1.GetOptions{}) + if err == nil || !strings.Contains(err.Error(), "no matching prefix found") { + t.Fatalf("after re-enabling feature gate, get test2 Secret from %s should return err containing: no matching prefix found. actual err: %v", testNamespace, err) + } +} + var benchSecret *api.Secret func BenchmarkKMSv2KDF(b *testing.B) { @@ -877,7 +1051,7 @@ resources: ` _ = kmsv2mock.NewBase64Plugin(b, "@kms-provider.sock") - test, err := newTransformTest(b, encryptionConfig, false, "") + test, err := newTransformTest(b, encryptionConfig, false, "", nil) if err != nil { b.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } @@ -1031,7 +1205,7 @@ resources: ` _ = kmsv2mock.NewBase64Plugin(b, "@kms-provider.sock") - test, err := newTransformTest(b, encryptionConfig, false, "") + test, err := newTransformTest(b, encryptionConfig, false, "", nil) if err != nil { b.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) } diff --git a/test/integration/controlplane/transformation/main_test.go b/test/integration/controlplane/transformation/main_test.go index 575b438c50c..31a3bbbb6fe 100644 --- a/test/integration/controlplane/transformation/main_test.go +++ b/test/integration/controlplane/transformation/main_test.go @@ -17,7 +17,9 @@ limitations under the License. package transformation import ( + "context" "testing" + "time" "k8s.io/kubernetes/test/integration/framework" ) @@ -25,3 +27,9 @@ import ( func TestMain(m *testing.M) { framework.EtcdMain(m.Run) } + +func testContext(t *testing.T) context.Context { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + t.Cleanup(cancel) + return ctx +} diff --git a/test/integration/controlplane/transformation/secrets_transformation_test.go b/test/integration/controlplane/transformation/secrets_transformation_test.go index 9cb940dfae7..1bbbc5c08c5 100644 --- a/test/integration/controlplane/transformation/secrets_transformation_test.go +++ b/test/integration/controlplane/transformation/secrets_transformation_test.go @@ -85,10 +85,9 @@ func TestSecretsShouldBeTransformed(t *testing.T) { // TODO: add secretbox } for _, tt := range testCases { - test, err := newTransformTest(t, tt.transformerConfigContent, false, "") + test, err := newTransformTest(t, tt.transformerConfigContent, false, "", nil) if err != nil { - test.cleanUp() - t.Errorf("failed to setup test for envelop %s, error was %v", tt.transformerPrefix, err) + t.Fatalf("failed to setup test for envelop %s, error was %v", tt.transformerPrefix, err) continue } test.secret, err = test.createSecret(testSecret, testNamespace) @@ -120,11 +119,11 @@ func BenchmarkAESCBCEnvelopeWrite(b *testing.B) { func runBenchmark(b *testing.B, transformerConfig string) { b.StopTimer() - test, err := newTransformTest(b, transformerConfig, false, "") - defer test.cleanUp() + test, err := newTransformTest(b, transformerConfig, false, "", nil) if err != nil { b.Fatalf("failed to setup benchmark for config %s, error was %v", transformerConfig, err) } + defer test.cleanUp() b.StartTimer() test.benchmark(b) diff --git a/test/integration/controlplane/transformation/transformation_test.go b/test/integration/controlplane/transformation/transformation_test.go index fa623da6d7e..a5fe45af6a5 100644 --- a/test/integration/controlplane/transformation/transformation_test.go +++ b/test/integration/controlplane/transformation/transformation_test.go @@ -33,6 +33,7 @@ import ( appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -84,34 +85,47 @@ type transformTest struct { secret *corev1.Secret } -func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML string, reload bool, configDir string) (*transformTest, error) { +func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML string, reload bool, configDir string, storageConfig *storagebackend.Config) (*transformTest, error) { + if storageConfig == nil { + storageConfig = framework.SharedEtcd() + } e := transformTest{ logger: l, transformerConfig: transformerConfigYAML, - storageConfig: framework.SharedEtcd(), + storageConfig: storageConfig, } var err error // create config dir with provided config yaml if transformerConfigYAML != "" && configDir == "" { if e.configDir, err = e.createEncryptionConfig(); err != nil { - return nil, fmt.Errorf("error while creating KubeAPIServer encryption config: %v", err) + e.cleanUp() + return nil, fmt.Errorf("error while creating KubeAPIServer encryption config: %w", err) } } else { // configDir already exists. api-server must be restarting with existing encryption config e.configDir = configDir } + configFile := filepath.Join(e.configDir, encryptionConfigFileName) + _, err = os.ReadFile(configFile) + if err != nil { + e.cleanUp() + return nil, fmt.Errorf("failed to read config file: %w", err) + } if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(l, nil, e.getEncryptionOptions(reload), e.storageConfig); err != nil { - return nil, fmt.Errorf("failed to start KubeAPI server: %v", err) + e.cleanUp() + return nil, fmt.Errorf("failed to start KubeAPI server: %w", err) } klog.Infof("Started kube-apiserver %v", e.kubeAPIServer.ClientConfig.Host) if e.restClient, err = kubernetes.NewForConfig(e.kubeAPIServer.ClientConfig); err != nil { - return nil, fmt.Errorf("error while creating rest client: %v", err) + e.cleanUp() + return nil, fmt.Errorf("error while creating rest client: %w", err) } if e.ns, err = e.createNamespace(testNamespace); err != nil { + e.cleanUp() return nil, err } @@ -128,7 +142,9 @@ func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML strin } func (e *transformTest) cleanUp() { - os.RemoveAll(e.configDir) + if e.configDir != "" { + os.RemoveAll(e.configDir) + } if e.kubeAPIServer.ClientConfig != nil { e.shutdownAPIServer() @@ -136,7 +152,6 @@ func (e *transformTest) cleanUp() { } func (e *transformTest) shutdownAPIServer() { - e.restClient.CoreV1().Namespaces().Delete(context.TODO(), e.ns.Name, *metav1.NewDeleteOptions(0)) e.kubeAPIServer.TearDownFn() } @@ -189,7 +204,7 @@ func (e *transformTest) runResource(l kubeapiservertesting.Logger, unSealSecretF // Data should be un-enveloped on direct reads from Kube API Server. if resource == "secrets" { - s, err := e.restClient.CoreV1().Secrets(testNamespace).Get(context.TODO(), testSecret, metav1.GetOptions{}) + s, err := e.restClient.CoreV1().Secrets(testNamespace).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { l.Fatalf("failed to get Secret from %s, err: %v", testNamespace, err) } @@ -300,7 +315,14 @@ func (e *transformTest) createNamespace(name string) (*corev1.Namespace, error) } if _, err := e.restClient.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}); err != nil { - return nil, fmt.Errorf("unable to create testing namespace %v", err) + if errors.IsAlreadyExists(err) { + existingNs, err := e.restClient.CoreV1().Namespaces().Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("unable to get testing namespace, err: [%v]", err) + } + return existingNs, nil + } + return nil, fmt.Errorf("unable to create testing namespace, err: [%v]", err) } return ns, nil