kmsv2 test feature enablement disablement and restart

Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com>

test/integration/controlplane/transformation/all_transformation_test.go

@@ -94,7 +94,7 @@ resources:
       - name: key1
         secret: c2VjcmV0IGlzIHNlY3VyZQ==
 `
-	test, err := newTransformTest(t, encryptionConfig, false, "")
+	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
 	if err != nil {
 		t.Fatalf("failed to start Kube API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}

test/integration/controlplane/transformation/kms_transformation_test.go

@@ -28,7 +28,6 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -49,6 +48,7 @@ import (
 	kmsapi "k8s.io/kms/apis/v1beta1"
 	"k8s.io/kubernetes/test/integration"
 	"k8s.io/kubernetes/test/integration/etcd"
+	"k8s.io/kubernetes/test/integration/framework"
 )
 
 const (
@@ -133,7 +133,7 @@ resources:
 `
 	providerName := "kms-provider"
 	pluginMock := mock.NewBase64Plugin(t, "@kms-provider.sock")
-	test, err := newTransformTest(t, encryptionConfig, false, "")
+	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
 	if err != nil {
 		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}
@@ -295,6 +295,7 @@ resources:
 // 10. confirm that cluster wide secret read still works
 // 11. confirm that api server can restart with last applied encryption config
 func TestEncryptionConfigHotReload(t *testing.T) {
+	storageConfig := framework.SharedEtcd()
 	encryptionConfig := `
 kind: EncryptionConfiguration
 apiVersion: apiserver.config.k8s.io/v1
@@ -309,7 +310,7 @@ resources:
 `
 	_ = mock.NewBase64Plugin(t, "@kms-provider.sock")
 	var restarted bool
-	test, err := newTransformTest(t, encryptionConfig, true, "")
+	test, err := newTransformTest(t, encryptionConfig, true, "", storageConfig)
 	if err != nil {
 		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}
@@ -365,7 +366,7 @@ resources:
 	// start new KMS Plugin
 	_ = mock.NewBase64Plugin(t, "@new-kms-provider.sock")
 	// update encryption config
-	if err := os.WriteFile(path.Join(test.configDir, encryptionConfigFileName), []byte(encryptionConfigWithNewProvider), 0644); err != nil {
+	if err := os.WriteFile(filepath.Join(test.configDir, encryptionConfigFileName), []byte(encryptionConfigWithNewProvider), 0644); err != nil {
 		t.Fatalf("failed to update encryption config, err: %v", err)
 	}
 
@@ -377,8 +378,9 @@ resources:
 
 	// run storage migration
 	// get secrets
+	ctx := testContext(t)
 	secretsList, err := test.restClient.CoreV1().Secrets("").List(
-		context.TODO(),
+		ctx,
 		metav1.ListOptions{},
 	)
 	if err != nil {
@@ -388,7 +390,7 @@ resources:
 	for _, secret := range secretsList.Items {
 		// update secret
 		_, err = test.restClient.CoreV1().Secrets(secret.Namespace).Update(
-			context.TODO(),
+			ctx,
 			&secret,
 			metav1.UpdateOptions{},
 		)
@@ -399,7 +401,7 @@ resources:
 
 	// get configmaps
 	configmapsList, err := test.restClient.CoreV1().ConfigMaps("").List(
-		context.TODO(),
+		ctx,
 		metav1.ListOptions{},
 	)
 	if err != nil {
@@ -409,7 +411,7 @@ resources:
 	for _, configmap := range configmapsList.Items {
 		// update configmap
 		_, err = test.restClient.CoreV1().ConfigMaps(configmap.Namespace).Update(
-			context.TODO(),
+			ctx,
 			&configmap,
 			metav1.UpdateOptions{},
 		)
@@ -463,7 +465,7 @@ resources:
 `
 
 	// update encryption config and wait for hot reload
-	if err := os.WriteFile(path.Join(test.configDir, encryptionConfigFileName), []byte(encryptionConfigWithoutOldProvider), 0644); err != nil {
+	if err := os.WriteFile(filepath.Join(test.configDir, encryptionConfigFileName), []byte(encryptionConfigWithoutOldProvider), 0644); err != nil {
 		t.Fatalf("failed to update encryption config, err: %v", err)
 	}
 
@@ -472,7 +474,7 @@ resources:
 
 	// confirm that reading secrets still works
 	_, err = test.restClient.CoreV1().Secrets(testNamespace).Get(
-		context.TODO(),
+		ctx,
 		testSecret,
 		metav1.GetOptions{},
 	)
@@ -481,13 +483,13 @@ resources:
 	}
 
 	// make sure cluster wide secrets read still works
-	_, err = test.restClient.CoreV1().Secrets("").List(context.TODO(), metav1.ListOptions{})
+	_, err = test.restClient.CoreV1().Secrets("").List(ctx, metav1.ListOptions{})
 	if err != nil {
 		t.Fatalf("failed to list secrets, err: %v", err)
 	}
 
 	// make sure cluster wide configmaps read still works
-	_, err = test.restClient.CoreV1().ConfigMaps("").List(context.TODO(), metav1.ListOptions{})
+	_, err = test.restClient.CoreV1().ConfigMaps("").List(ctx, metav1.ListOptions{})
 	if err != nil {
 		t.Fatalf("failed to list configmaps, err: %v", err)
 	}
@@ -496,19 +498,28 @@ resources:
 	previousConfigDir := test.configDir
 	test.shutdownAPIServer()
 	restarted = true
-	test, err = newTransformTest(t, "", true, previousConfigDir)
+	test, err = newTransformTest(t, test.transformerConfig, true, previousConfigDir, storageConfig)
 	if err != nil {
 		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}
 	defer test.cleanUp()
 
+	_, err = test.restClient.CoreV1().Secrets(testNamespace).Get(
+		ctx,
+		testSecret,
+		metav1.GetOptions{},
+	)
+	if err != nil {
+		t.Fatalf("failed to read secret, err: %v", err)
+	}
+
 	// confirm that reading cluster wide secrets still works after restart
-	if _, err = test.restClient.CoreV1().Secrets("").List(context.TODO(), metav1.ListOptions{}); err != nil {
+	if _, err = test.restClient.CoreV1().Secrets("").List(ctx, metav1.ListOptions{}); err != nil {
 		t.Fatalf("failed to list secrets, err: %v", err)
 	}
 
 	// make sure cluster wide configmaps read still works
-	if _, err = test.restClient.CoreV1().ConfigMaps("").List(context.TODO(), metav1.ListOptions{}); err != nil {
+	if _, err = test.restClient.CoreV1().ConfigMaps("").List(ctx, metav1.ListOptions{}); err != nil {
 		t.Fatalf("failed to list configmaps, err: %v", err)
 	}
 }
@@ -531,7 +542,7 @@ resources:
 		_ = mock.NewBase64Plugin(t, "@encrypt-all-kms-provider.sock")
 		defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)()
 		defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)()
-		test, err := newTransformTest(t, encryptionConfig, false, "")
+		test, err := newTransformTest(t, encryptionConfig, false, "", nil)
 		if err != nil {
 			t.Fatalf("failed to start KUBE API Server with encryptionConfig")
 		}
@@ -643,7 +654,7 @@ resources:
 	_ = mock.NewBase64Plugin(t, "@kms-provider.sock")
 	_ = mock.NewBase64Plugin(t, "@encrypt-all-kms-provider.sock")
 
-	test, err := newTransformTest(t, encryptionConfig, false, "")
+	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
 	if err != nil {
 		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}
@@ -785,9 +796,8 @@ resources:
 `
 			_ = mock.NewBase64Plugin(t, "@kms-provider.sock")
 
-			test, err := newTransformTest(t, encryptionConfig, true, "")
+			test, err := newTransformTest(t, encryptionConfig, true, "", nil)
 			if err != nil {
-				test.cleanUp()
 				t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 			}
 			defer test.cleanUp()
@@ -950,7 +960,7 @@ resources:
 	pluginMock1 := mock.NewBase64Plugin(t, "@kms-provider-1.sock")
 	pluginMock2 := mock.NewBase64Plugin(t, "@kms-provider-2.sock")
 
-	test, err := newTransformTest(t, encryptionConfig, false, "")
+	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
 	if err != nil {
 		t.Fatalf("failed to start kube-apiserver, error: %v", err)
 	}
@@ -1006,7 +1016,7 @@ resources:
 	pluginMock1 := mock.NewBase64Plugin(t, "@kms-provider-1.sock")
 	pluginMock2 := mock.NewBase64Plugin(t, "@kms-provider-2.sock")
 
-	test, err := newTransformTest(t, encryptionConfig, true, "")
+	test, err := newTransformTest(t, encryptionConfig, true, "", nil)
 	if err != nil {
 		t.Fatalf("Failed to start kube-apiserver, error: %v", err)
 	}

test/integration/controlplane/transformation/kmsv2_transformation_test.go

@@ -69,6 +69,7 @@ import (
 	secretstore "k8s.io/kubernetes/pkg/registry/core/secret/storage"
 	"k8s.io/kubernetes/test/integration"
 	"k8s.io/kubernetes/test/integration/etcd"
+	"k8s.io/kubernetes/test/integration/framework"
 )
 
 type envelopekmsv2 struct {
@@ -194,7 +195,7 @@ resources:
 	providerName := "kms-provider"
 	pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
 
-	test, err := newTransformTest(t, encryptionConfig, false, "")
+	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
 	if err != nil {
 		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}
@@ -301,7 +302,7 @@ resources:
 `
 	pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
 
-	test, err := newTransformTest(t, encryptionConfig, false, "")
+	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
 	if err != nil {
 		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}
@@ -597,7 +598,7 @@ resources:
 `
 	_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
 
-	test, err := newTransformTest(t, encryptionConfig, false, "")
+	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
 	if err != nil {
 		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}
@@ -736,7 +737,7 @@ resources:
 	pluginMock1 := kmsv2mock.NewBase64Plugin(t, "@kms-provider-1.sock")
 	pluginMock2 := kmsv2mock.NewBase64Plugin(t, "@kms-provider-2.sock")
 
-	test, err := newTransformTest(t, encryptionConfig, false, "")
+	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
 	if err != nil {
 		t.Fatalf("Failed to start kube-apiserver, error: %v", err)
 	}
@@ -815,7 +816,7 @@ resources:
 
 	_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
 
-	test, err := newTransformTest(t, encryptionConfig, false, "")
+	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
 	if err != nil {
 		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}
@@ -847,6 +848,179 @@ resources:
 	}
 }
 
+// TestKMSv2FeatureFlag is an integration test between KubeAPI and ETCD
+// Concretely, this test verifies the following:
+// 1. When feature flag is not enabled, loading a encryptionConfig with KMSv2 should fail
+// 2. When feature flag is enabled, loading a encryptionConfig with KMSv2 should work
+// 3. When feature flag is disabled, loading a encryptionConfig with a non-v2 provider should work.
+// without performing a storage migration, decryption of existing data encrypted with v2 should fail for Get and List operations.
+// New data stored in etcd will no longer be encrypted using the external kms provider with v2 API.
+// 4. when feature flag is re-enabled, loading a encryptionConfig with the same KMSv2 plugin from 2 should work,
+// decryption of data encrypted with v2 should work
+func TestKMSv2FeatureFlag(t *testing.T) {
+	encryptionConfig := `
+kind: EncryptionConfiguration
+apiVersion: apiserver.config.k8s.io/v1
+resources:
+  - resources:
+    - secrets
+    providers:
+    - kms:
+       apiVersion: v2
+       name: kms-provider
+       endpoint: unix:///@kms-provider.sock
+`
+	// TOODO: This test requires changes in the apiserver lifecycle code to correctly clean up goroutines after a server failure.
+	// When feature flag is not enabled, loading a encryptionConfig with KMSv2 should fail
+	// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, false)()
+	// _, err := newTransformTest(t, encryptionConfig, true, "", nil)
+	// if err == nil || !strings.Contains(err.Error(), "KMSv2 feature is not enabled") {
+	// 	t.Fatalf("when feature flag is not enabled, loading a encryptionConfig with KMSv2 should have failed with: KMSv2 feature is not enabled, encryptionConfig:\n%s\n actual error: %v", encryptionConfig, err)
+	// }
+
+	providerName := "kms-provider"
+	pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
+	storageConfig := framework.SharedEtcd()
+
+	// When feature flag is enabled, loading a encryptionConfig with KMSv1 and v2 should work
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
+	test, err := newTransformTest(t, encryptionConfig, false, "", storageConfig)
+	if err != nil {
+		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
+	}
+	defer func() {
+		test.cleanUp()
+	}()
+
+	test.secret, err = test.createSecret(testSecret, testNamespace)
+	if err != nil {
+		t.Fatalf("Failed to create test secret, error: %v", err)
+	}
+
+	// Since Data Encryption Key (DEK) is randomly generated, we need to ask KMS Mock for it.
+	plainTextDEKSource := pluginMock.LastEncryptRequest()
+
+	secretETCDPath := test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", test.secret.Name, test.secret.Namespace)
+	rawEnvelope, err := test.getRawSecretFromETCD()
+	if err != nil {
+		t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err)
+	}
+
+	envelopeData := envelopekmsv2{
+		providerName:       providerName,
+		rawEnvelope:        rawEnvelope,
+		plainTextDEKSource: plainTextDEKSource,
+	}
+
+	wantPrefix := envelopeData.prefix()
+	if !bytes.HasPrefix(rawEnvelope, []byte(wantPrefix)) {
+		t.Fatalf("expected secret to be prefixed with %s, but got %s", wantPrefix, rawEnvelope)
+	}
+
+	ctx := testContext(t)
+	ciphertext, err := envelopeData.cipherTextDEKSource()
+	if err != nil {
+		t.Fatalf("failed to get ciphertext DEK from KMSv2 Plugin: %v", err)
+	}
+	decryptResponse, err := pluginMock.Decrypt(ctx, &kmsv2api.DecryptRequest{Uid: string(uuid.NewUUID()), Ciphertext: ciphertext})
+	if err != nil {
+		t.Fatalf("failed to decrypt DEK, %v", err)
+	}
+	dekPlainAsWouldBeSeenByETCD := decryptResponse.Plaintext
+
+	if !bytes.Equal(plainTextDEKSource, dekPlainAsWouldBeSeenByETCD) {
+		t.Fatalf("expected plainTextDEKSource %v to be passed to KMS Plugin, but got %s",
+			plainTextDEKSource, dekPlainAsWouldBeSeenByETCD)
+	}
+
+	plainSecret, err := envelopeData.plainTextPayload(secretETCDPath)
+	if err != nil {
+		t.Fatalf("failed to transform from storage via AESGCM, err: %v", err)
+	}
+
+	if !strings.Contains(string(plainSecret), secretVal) {
+		t.Fatalf("expected %q after decryption, but got %q", secretVal, string(plainSecret))
+	}
+
+	secretClient := test.restClient.CoreV1().Secrets(testNamespace)
+	// Secrets should be un-enveloped on direct reads from Kube API Server.
+	s, err := secretClient.Get(ctx, testSecret, metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("failed to get Secret from %s, err: %v", testNamespace, err)
+	}
+	if secretVal != string(s.Data[secretKey]) {
+		t.Fatalf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
+	}
+	test.shutdownAPIServer()
+
+	// When KMSv2 feature flag is disabled, loading a encryptionConfig with a non-v2 provider should work. without performing a storage migration, decryption of existing data encrypted with v2 should fail for Get and List operations.
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, false)()
+
+	encryptionConfig1 := `
+kind: EncryptionConfiguration
+apiVersion: apiserver.config.k8s.io/v1
+resources:
+  - resources:
+    - secrets
+    providers:
+    - aescbc:
+        keys:
+        - name: key1
+          secret: c2VjcmV0IGlzIHNlY3VyZQ==
+`
+	test, err = newTransformTest(t, encryptionConfig1, false, "", storageConfig)
+	if err != nil {
+		t.Fatalf("Failed to restart api server, error: %v", err)
+	}
+
+	_, err = test.createSecret("test2", testNamespace)
+	if err != nil {
+		t.Fatalf("Failed to create test secret, error: %v", err)
+	}
+	test.runResource(t, unSealWithCBCTransformer, aesCBCPrefix, "", "v1", "secrets", "test2", testNamespace)
+
+	secretClient = test.restClient.CoreV1().Secrets(testNamespace)
+
+	// Getting an old secret that was encrypted by another provider should fail
+	_, err = secretClient.Get(ctx, testSecret, metav1.GetOptions{})
+	if err == nil || !strings.Contains(err.Error(), "no matching prefix found") {
+		t.Fatalf("using a new provider, get Secret %s from %s should return err containing: no matching prefix found. Got err: %v", testSecret, testNamespace, err)
+	}
+	// List all cluster wide secrets should fail
+	_, err = test.restClient.CoreV1().Secrets("").List(ctx, metav1.ListOptions{})
+	if err == nil || !strings.Contains(err.Error(), "no matching prefix found") {
+		t.Fatalf("using a new provider, LIST all Secrets should return err containing: no matching prefix found. Got err: %v", err)
+	}
+	test.shutdownAPIServer()
+
+	// when feature flag is re-enabled, loading a encryptionConfig with the same KMSv2 plugin before the restart should work, decryption of data encrypted with v2 should work
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
+
+	test, err = newTransformTest(t, encryptionConfig, false, "", storageConfig)
+	if err != nil {
+		t.Fatalf("Failed to restart api server, error: %v", err)
+	}
+
+	// Getting an old secret that was encrypted by the same plugin should not fail.
+	s, err = test.restClient.CoreV1().Secrets(testNamespace).Get(
+		ctx,
+		testSecret,
+		metav1.GetOptions{},
+	)
+	if err != nil {
+		t.Fatalf("failed to read secret, err: %v", err)
+	}
+	if secretVal != string(s.Data[secretKey]) {
+		t.Fatalf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
+	}
+	secretClient = test.restClient.CoreV1().Secrets(testNamespace)
+	// Getting an old secret that was encrypted by another plugin should fail
+	_, err = secretClient.Get(ctx, "test2", metav1.GetOptions{})
+	if err == nil || !strings.Contains(err.Error(), "no matching prefix found") {
+		t.Fatalf("after re-enabling feature gate, get test2 Secret from %s should return err containing: no matching prefix found. actual err: %v", testNamespace, err)
+	}
+}
+
 var benchSecret *api.Secret
 
 func BenchmarkKMSv2KDF(b *testing.B) {
@@ -877,7 +1051,7 @@ resources:
 `
 	_ = kmsv2mock.NewBase64Plugin(b, "@kms-provider.sock")
 
-	test, err := newTransformTest(b, encryptionConfig, false, "")
+	test, err := newTransformTest(b, encryptionConfig, false, "", nil)
 	if err != nil {
 		b.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}
@@ -1031,7 +1205,7 @@ resources:
 `
 	_ = kmsv2mock.NewBase64Plugin(b, "@kms-provider.sock")
 
-	test, err := newTransformTest(b, encryptionConfig, false, "")
+	test, err := newTransformTest(b, encryptionConfig, false, "", nil)
 	if err != nil {
 		b.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}

test/integration/controlplane/transformation/main_test.go

@@ -17,7 +17,9 @@ limitations under the License.
 package transformation
 
 import (
+	"context"
 	"testing"
+	"time"
 
 	"k8s.io/kubernetes/test/integration/framework"
 )
@@ -25,3 +27,9 @@ import (
 func TestMain(m *testing.M) {
 	framework.EtcdMain(m.Run)
 }
+
+func testContext(t *testing.T) context.Context {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	t.Cleanup(cancel)
+	return ctx
+}

test/integration/controlplane/transformation/secrets_transformation_test.go

@@ -85,10 +85,9 @@ func TestSecretsShouldBeTransformed(t *testing.T) {
 		// TODO: add secretbox
 	}
 	for _, tt := range testCases {
-		test, err := newTransformTest(t, tt.transformerConfigContent, false, "")
+		test, err := newTransformTest(t, tt.transformerConfigContent, false, "", nil)
 		if err != nil {
-			test.cleanUp()
+			t.Fatalf("failed to setup test for envelop %s, error was %v", tt.transformerPrefix, err)
-			t.Errorf("failed to setup test for envelop %s, error was %v", tt.transformerPrefix, err)
 			continue
 		}
 		test.secret, err = test.createSecret(testSecret, testNamespace)
@@ -120,11 +119,11 @@ func BenchmarkAESCBCEnvelopeWrite(b *testing.B) {
 
 func runBenchmark(b *testing.B, transformerConfig string) {
 	b.StopTimer()
-	test, err := newTransformTest(b, transformerConfig, false, "")
+	test, err := newTransformTest(b, transformerConfig, false, "", nil)
-	defer test.cleanUp()
 	if err != nil {
 		b.Fatalf("failed to setup benchmark for config %s, error was %v", transformerConfig, err)
 	}
+	defer test.cleanUp()
 
 	b.StartTimer()
 	test.benchmark(b)

test/integration/controlplane/transformation/transformation_test.go

@@ -33,6 +33,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -84,34 +85,47 @@ type transformTest struct {
 	secret            *corev1.Secret
 }
 
-func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML string, reload bool, configDir string) (*transformTest, error) {
+func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML string, reload bool, configDir string, storageConfig *storagebackend.Config) (*transformTest, error) {
+	if storageConfig == nil {
+		storageConfig = framework.SharedEtcd()
+	}
 	e := transformTest{
 		logger:            l,
 		transformerConfig: transformerConfigYAML,
-		storageConfig:     framework.SharedEtcd(),
+		storageConfig:     storageConfig,
 	}
 
 	var err error
 	// create config dir with provided config yaml
 	if transformerConfigYAML != "" && configDir == "" {
 		if e.configDir, err = e.createEncryptionConfig(); err != nil {
-			return nil, fmt.Errorf("error while creating KubeAPIServer encryption config: %v", err)
+			e.cleanUp()
+			return nil, fmt.Errorf("error while creating KubeAPIServer encryption config: %w", err)
 		}
 	} else {
 		// configDir already exists. api-server must be restarting with existing encryption config
 		e.configDir = configDir
 	}
+	configFile := filepath.Join(e.configDir, encryptionConfigFileName)
+	_, err = os.ReadFile(configFile)
+	if err != nil {
+		e.cleanUp()
+		return nil, fmt.Errorf("failed to read config file: %w", err)
+	}
 
 	if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(l, nil, e.getEncryptionOptions(reload), e.storageConfig); err != nil {
-		return nil, fmt.Errorf("failed to start KubeAPI server: %v", err)
+		e.cleanUp()
+		return nil, fmt.Errorf("failed to start KubeAPI server: %w", err)
 	}
 	klog.Infof("Started kube-apiserver %v", e.kubeAPIServer.ClientConfig.Host)
 
 	if e.restClient, err = kubernetes.NewForConfig(e.kubeAPIServer.ClientConfig); err != nil {
-		return nil, fmt.Errorf("error while creating rest client: %v", err)
+		e.cleanUp()
+		return nil, fmt.Errorf("error while creating rest client: %w", err)
 	}
 
 	if e.ns, err = e.createNamespace(testNamespace); err != nil {
+		e.cleanUp()
 		return nil, err
 	}
 
@@ -128,7 +142,9 @@ func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML strin
 }
 
 func (e *transformTest) cleanUp() {
-	os.RemoveAll(e.configDir)
+	if e.configDir != "" {
+		os.RemoveAll(e.configDir)
+	}
 
 	if e.kubeAPIServer.ClientConfig != nil {
 		e.shutdownAPIServer()
@@ -136,7 +152,6 @@ func (e *transformTest) cleanUp() {
 }
 
 func (e *transformTest) shutdownAPIServer() {
-	e.restClient.CoreV1().Namespaces().Delete(context.TODO(), e.ns.Name, *metav1.NewDeleteOptions(0))
 	e.kubeAPIServer.TearDownFn()
 }
 
@@ -189,7 +204,7 @@ func (e *transformTest) runResource(l kubeapiservertesting.Logger, unSealSecretF
 
 	// Data should be un-enveloped on direct reads from Kube API Server.
 	if resource == "secrets" {
-		s, err := e.restClient.CoreV1().Secrets(testNamespace).Get(context.TODO(), testSecret, metav1.GetOptions{})
+		s, err := e.restClient.CoreV1().Secrets(testNamespace).Get(context.TODO(), name, metav1.GetOptions{})
 		if err != nil {
 			l.Fatalf("failed to get Secret from %s, err: %v", testNamespace, err)
 		}
@@ -300,7 +315,14 @@ func (e *transformTest) createNamespace(name string) (*corev1.Namespace, error)
 	}
 
 	if _, err := e.restClient.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}); err != nil {
-		return nil, fmt.Errorf("unable to create testing namespace %v", err)
+		if errors.IsAlreadyExists(err) {
+			existingNs, err := e.restClient.CoreV1().Namespaces().Get(context.TODO(), name, metav1.GetOptions{})
+			if err != nil {
+				return nil, fmt.Errorf("unable to get testing namespace, err: [%v]", err)
+			}
+			return existingNs, nil
+		}
+		return nil, fmt.Errorf("unable to create testing namespace, err: [%v]", err)
 	}
 
 	return ns, nil