[KMSv2] promote KMSv2 and KMSv2KDF to GA

Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com>

staging/src/k8s.io/apiserver/pkg/features/kube_features.go

@@ -122,6 +122,7 @@ const (
 	// kep: https://kep.k8s.io/3299
 	// alpha: v1.25
 	// beta: v1.27
+	// stable: v1.29
 	//
 	// Enables KMS v2 API for encryption at rest.
 	KMSv2 featuregate.Feature = "KMSv2"
@@ -129,6 +130,7 @@ const (
 	// owner: @enj
 	// kep: https://kep.k8s.io/3299
 	// beta: v1.28
+	// stable: v1.29
 	//
 	// Enables the use of derived encryption keys with KMS v2.
 	KMSv2KDF featuregate.Feature = "KMSv2KDF"
@@ -288,11 +290,11 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 
 	EfficientWatchResumption: {Default: true, PreRelease: featuregate.GA, LockToDefault: true},
 
-	KMSv1: {Default: true, PreRelease: featuregate.Deprecated},
+	KMSv1: {Default: false, PreRelease: featuregate.Deprecated},
 
-	KMSv2: {Default: true, PreRelease: featuregate.Beta},
+	KMSv2: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31
 
-	KMSv2KDF: {Default: true, PreRelease: featuregate.Beta}, // lock to true in 1.29 once KMSv2 is GA, remove in 1.31
+	KMSv2KDF: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31
 
 	OpenAPIEnums: {Default: true, PreRelease: featuregate.Beta},
 

staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go

@@ -107,6 +107,26 @@ const (
 
 var codecs serializer.CodecFactory
 
+// this atomic bool allows us to swap enablement of the KMSv2KDF feature in tests
+// as the feature gate is now locked to true starting with v1.29
+// Note: it cannot be set by an end user
+var kdfDisabled atomic.Bool
+
+// this function should only be called in tests to swap enablement of the KMSv2KDF feature
+func SetKDFForTests(b bool) func() {
+	kdfDisabled.Store(!b)
+	return func() {
+		kdfDisabled.Store(false)
+	}
+}
+
+// this function should be used to determine enablement of the KMSv2KDF feature
+// instead of getting it from DefaultFeatureGate as the feature gate is now locked
+// to true starting with v1.29
+func GetKDF() bool {
+	return !kdfDisabled.Load()
+}
+
 func init() {
 	configScheme := runtime.NewScheme()
 	utilruntime.Must(apiserverconfig.AddToScheme(configScheme))
@@ -138,6 +158,7 @@ type kmsv2PluginProbe struct {
 	lastResponse *kmsPluginHealthzResponse
 	l            *sync.Mutex
 	apiServerID  string
+	version      string
 }
 
 type kmsHealthChecker []healthz.HealthChecker
@@ -369,7 +390,7 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey
 	// this gate can only change during tests, but the check is cheap enough to always make
 	// this allows us to easily exercise both modes without restarting the API server
 	// TODO integration test that this dynamically takes effect
-	useSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF)
+	useSeed := GetKDF()
 	stateUseSeed := state.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
 
 	// state is valid and status keyID is unchanged from when we generated this DEK/seed so there is no need to rotate it
@@ -454,8 +475,16 @@ func (h *kmsv2PluginProbe) isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx context.C
 	if response.Healthz != "ok" {
 		errs = append(errs, fmt.Errorf("got unexpected healthz status: %s", response.Healthz))
 	}
-	if response.Version != envelopekmsv2.KMSAPIVersion {
+	if response.Version != envelopekmsv2.KMSAPIVersionv2 && response.Version != envelopekmsv2.KMSAPIVersionv2beta1 {
-		errs = append(errs, fmt.Errorf("expected KMSv2 API version %s, got %s", envelopekmsv2.KMSAPIVersion, response.Version))
+		errs = append(errs, fmt.Errorf("expected KMSv2 API version %s, got %s", envelopekmsv2.KMSAPIVersionv2, response.Version))
+	} else {
+		// set version for the first status response
+		if len(h.version) == 0 {
+			h.version = response.Version
+		}
+		if h.version != response.Version {
+			errs = append(errs, fmt.Errorf("KMSv2 API version should not change after the initial status response version %s, got %s", h.version, response.Version))
+		}
 	}
 
 	if errCode, err := envelopekmsv2.ValidateKeyID(response.KeyID); err != nil {

staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go

@@ -187,7 +187,7 @@ func TestLegacyConfig(t *testing.T) {
 }
 
 func TestEncryptionProviderConfigCorrect(t *testing.T) {
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
 
 	// Set factory for mock envelope service
 	factory := envelopeServiceFactory
@@ -353,42 +353,33 @@ func TestKMSv1Deprecation(t *testing.T) {
 
 func TestKMSvsEnablement(t *testing.T) {
 	testCases := []struct {
-		name         string
+		name        string
-		kmsv2Enabled bool
+		filePath    string
-		filePath     string
+		expectedErr string
-		expectedErr  string
 	}{
 		{
-			name:         "config with kmsv2 and kmsv1, KMSv2=false",
+			name:        "config with kmsv2 and kmsv1, KMSv2=true, KMSv1=false, should fail when feature is disabled",
-			kmsv2Enabled: false,
+			filePath:    "testdata/valid-configs/kms/multiple-providers-mixed.yaml",
-			filePath:     "testdata/valid-configs/kms/multiple-providers-kmsv2.yaml",
+			expectedErr: "KMSv1 is deprecated and will only receive security updates going forward. Use KMSv2 instead",
-			expectedErr:  "KMSv2 feature is not enabled",
 		},
 		{
-			name:         "config with kmsv2 and kmsv1, KMSv2=true",
+			name:        "config with kmsv2, KMSv2=true, KMSv1=false",
-			kmsv2Enabled: true,
+			filePath:    "testdata/valid-configs/kms/multiple-providers-kmsv2.yaml",
-			filePath:     "testdata/valid-configs/kms/multiple-providers-kmsv2.yaml",
+			expectedErr: "",
-			expectedErr:  "",
-		},
-		{
-			name:         "config with kmsv1, KMSv2=false",
-			kmsv2Enabled: false,
-			filePath:     "testdata/valid-configs/kms/multiple-providers.yaml",
-			expectedErr:  "",
 		},
 	}
 
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
-			// Just testing KMSv2 feature flag
+			// only the KMSv2 feature flag is enabled
-			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
-
-			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, testCase.kmsv2Enabled)()
 			_, err := LoadEncryptionConfig(testContext(t), testCase.filePath, false, "")
 
-			if !strings.Contains(errString(err), testCase.expectedErr) {
+			if len(testCase.expectedErr) > 0 && !strings.Contains(errString(err), testCase.expectedErr) {
 				t.Fatalf("expected error %q, got %q", testCase.expectedErr, errString(err))
 			}
+			if len(testCase.expectedErr) == 0 && err != nil {
+				t.Fatalf("unexpected error %q", errString(err))
+			}
 
 		})
 	}
@@ -400,43 +391,6 @@ func TestKMSvsEnablement(t *testing.T) {
 		config          apiserverconfig.EncryptionConfiguration
 		wantV2Used      bool
 	}{
-		{
-			name:         "with kmsv1 and kmsv2, KMSv2=false",
-			kmsv2Enabled: false,
-			config: apiserverconfig.EncryptionConfiguration{
-				Resources: []apiserverconfig.ResourceConfiguration{
-					{
-						Resources: []string{"secrets"},
-						Providers: []apiserverconfig.ProviderConfiguration{
-							{
-								KMS: &apiserverconfig.KMSConfiguration{
-									Name:       "kms",
-									APIVersion: "v1",
-									Timeout: &metav1.Duration{
-										Duration: 1 * time.Second,
-									},
-									Endpoint:  "unix:///tmp/testprovider.sock",
-									CacheSize: pointer.Int32(1000),
-								},
-							},
-							{
-								KMS: &apiserverconfig.KMSConfiguration{
-									Name:       "another-kms",
-									APIVersion: "v2",
-									Timeout: &metav1.Duration{
-										Duration: 1 * time.Second,
-									},
-									Endpoint:  "unix:///tmp/anothertestprovider.sock",
-									CacheSize: pointer.Int32(1000),
-								},
-							},
-						},
-					},
-				},
-			},
-			expectedErr: "KMSv2 feature is not enabled",
-			wantV2Used:  false,
-		},
 		{
 			name:         "with kmsv1 and kmsv2, KMSv2=true",
 			kmsv2Enabled: true,
@@ -501,7 +455,7 @@ func TestKMSvsEnablement(t *testing.T) {
 }
 
 func TestKMSMaxTimeout(t *testing.T) {
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
 
 	testCases := []struct {
 		name            string
@@ -749,7 +703,7 @@ func TestKMSMaxTimeout(t *testing.T) {
 }
 
 func TestKMSPluginHealthz(t *testing.T) {
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
 
 	kmsv2Probe := &kmsv2PluginProbe{
 		name:        "foo",
@@ -823,7 +777,7 @@ func TestKMSPluginHealthz(t *testing.T) {
 		},
 		{
 			desc:   "Install multiple healthz with v1 and v2",
-			config: "testdata/valid-configs/kms/multiple-providers-kmsv2.yaml",
+			config: "testdata/valid-configs/kms/multiple-providers-mixed.yaml",
 			want: []healthChecker{
 				kmsv2Probe,
 				&kmsPluginProbe{
@@ -900,6 +854,7 @@ func TestKMSPluginHealthz(t *testing.T) {
 
 // tests for masking rules
 func TestWildcardMasking(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
 
 	testCases := []struct {
 		desc          string
@@ -1308,7 +1263,7 @@ func TestWildcardMasking(t *testing.T) {
 }
 
 func TestWildcardStructure(t *testing.T) {
-
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
 	testCases := []struct {
 		desc                         string
 		expectedResourceTransformers map[string]string
@@ -1752,7 +1707,7 @@ func TestIsKMSv2ProviderHealthyError(t *testing.T) {
 			statusResponse: &kmsservice.StatusResponse{
 				Healthz: "unhealthy",
 			},
-			expectedErr: "got unexpected healthz status: unhealthy, expected KMSv2 API version v2beta1, got , got invalid KMSv2 KeyID ",
+			expectedErr: "got unexpected healthz status: unhealthy, expected KMSv2 API version v2, got , got invalid KMSv2 KeyID ",
 			wantMetrics: `
 			# HELP apiserver_envelope_encryption_invalid_key_id_from_status_total [ALPHA] Number of times an invalid keyID is returned by the Status RPC call split by error.
 			# TYPE apiserver_envelope_encryption_invalid_key_id_from_status_total counter
@@ -1760,11 +1715,11 @@ func TestIsKMSv2ProviderHealthyError(t *testing.T) {
 			`,
 		},
 		{
-			desc: "version is not v2beta1",
+			desc: "version is not v2",
 			statusResponse: &kmsservice.StatusResponse{
 				Version: "v1beta1",
 			},
-			expectedErr: "got unexpected healthz status: , expected KMSv2 API version v2beta1, got v1beta1, got invalid KMSv2 KeyID ",
+			expectedErr: "got unexpected healthz status: , expected KMSv2 API version v2, got v1beta1, got invalid KMSv2 KeyID ",
 			wantMetrics: `
 			# HELP apiserver_envelope_encryption_invalid_key_id_from_status_total [ALPHA] Number of times an invalid keyID is returned by the Status RPC call split by error.
 			# TYPE apiserver_envelope_encryption_invalid_key_id_from_status_total counter
@@ -1788,7 +1743,7 @@ func TestIsKMSv2ProviderHealthyError(t *testing.T) {
 			desc: "invalid long keyID",
 			statusResponse: &kmsservice.StatusResponse{
 				Healthz: "ok",
-				Version: "v2beta1",
+				Version: "v2",
 				KeyID:   sampleInvalidKeyID,
 			},
 			expectedErr: "got invalid KMSv2 KeyID ",
@@ -1816,6 +1771,52 @@ func TestIsKMSv2ProviderHealthyError(t *testing.T) {
 	}
 }
 
+// test to ensure KMSv2 API version is not changed after the first status response
+func TestKMSv2SameVersionFromStatus(t *testing.T) {
+	probe := &kmsv2PluginProbe{name: "testplugin"}
+	service, _ := newMockEnvelopeKMSv2Service(testContext(t), "unix:///tmp/testprovider.sock", "providerName", 3*time.Second)
+	probe.l = &sync.Mutex{}
+	probe.state.Store(&envelopekmsv2.State{})
+	probe.service = service
+
+	testCases := []struct {
+		desc        string
+		expectedErr string
+		newVersion  string
+	}{
+		{
+			desc:        "version changed",
+			newVersion:  "v2",
+			expectedErr: "KMSv2 API version should not change",
+		},
+		{
+			desc:        "version unchanged",
+			newVersion:  "v2beta1",
+			expectedErr: "",
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.desc, func(t *testing.T) {
+			statusResponse := &kmsservice.StatusResponse{
+				Healthz: "ok",
+				Version: "v2beta1",
+				KeyID:   "1",
+			}
+			if err := probe.isKMSv2ProviderHealthyAndMaybeRotateDEK(testContext(t), statusResponse); err != nil {
+				t.Fatal(err)
+			}
+			statusResponse.Version = tt.newVersion
+			err := probe.isKMSv2ProviderHealthyAndMaybeRotateDEK(testContext(t), statusResponse)
+			if len(tt.expectedErr) > 0 && !strings.Contains(errString(err), tt.expectedErr) {
+				t.Errorf("expected err %q, got %q", tt.expectedErr, errString(err))
+			}
+			if len(tt.expectedErr) == 0 && err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
 func testContext(t *testing.T) context.Context {
 	ctx, cancel := context.WithCancel(context.Background())
 	t.Cleanup(cancel)
@@ -1840,7 +1841,7 @@ func TestComputeEncryptionConfigHash(t *testing.T) {
 }
 
 func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
-	defaultUseSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF)
+	defaultUseSeed := GetKDF()
 
 	origNowFunc := envelopekmsv2.NowFunc
 	now := origNowFunc() // freeze time
@@ -2065,7 +2066,7 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, tt.useSeed)()
+			defer SetKDFForTests(tt.useSeed)()
 
 			var buf bytes.Buffer
 			klog.SetOutput(&buf)

staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kms/multiple-providers-kmsv2.yaml

@@ -10,6 +10,7 @@ resources:
           endpoint: unix:///tmp/testprovider.sock
           timeout:   15s
       - kms:
+          apiVersion: v2
           name: bar
           endpoint: unix:///tmp/testprovider.sock
           timeout:   15s

staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kms/multiple-providers-mixed.yaml

@@ -0,0 +1,15 @@
+kind: EncryptionConfiguration
+apiVersion: apiserver.config.k8s.io/v1
+resources:
+  - resources:
+      - secrets
+    providers:
+      - kms:
+          apiVersion: v2
+          name: foo
+          endpoint: unix:///tmp/testprovider.sock
+          timeout:   15s
+      - kms:
+          name: bar
+          endpoint: unix:///tmp/testprovider.sock
+          timeout:   15s

staging/src/k8s.io/apiserver/pkg/server/options/etcd_test.go

@@ -229,7 +229,7 @@ func TestParseWatchCacheSizes(t *testing.T) {
 }
 
 func TestKMSHealthzEndpoint(t *testing.T) {
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
 
 	testCases := []struct {
 		name                 string

staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go

@@ -50,8 +50,10 @@ func init() {
 }
 
 const (
-	// KMSAPIVersion is the version of the KMS API.
+	// KMSAPIVersionv2 is a version of the KMS API.
-	KMSAPIVersion = "v2beta1"
+	KMSAPIVersionv2 = "v2"
+	// KMSAPIVersionv2beta1 is a version of the KMS API.
+	KMSAPIVersionv2beta1 = "v2beta1"
 	// annotationsMaxSize is the maximum size of the annotations.
 	annotationsMaxSize = 32 * 1024 // 32 kB
 	// KeyIDMaxSize is the maximum size of the keyID.

staging/src/k8s.io/kms/internal/plugins/_mock/pkcs11/pkcs11.go

@@ -111,7 +111,7 @@ func (s *pkcs11RemoteService) Decrypt(ctx context.Context, uid string, req *serv
 
 func (s *pkcs11RemoteService) Status(ctx context.Context) (*service.StatusResponse, error) {
 	return &service.StatusResponse{
-		Version: "v2beta1",
+		Version: "v2",
 		Healthz: "ok",
 		KeyID:   s.keyID,
 	}, nil

staging/src/k8s.io/kms/pkg/service/grpc_service_test.go

@@ -35,7 +35,7 @@ import (
 	kmsapi "k8s.io/kms/apis/v2"
 )
 
-const version = "v2alpha1"
+const version = "v2"
 
 func TestGRPCService(t *testing.T) {
 	t.Parallel()

test/integration/controlplane/transformation/kms_transformation_test.go

@@ -38,16 +38,19 @@ import (
 	clientv3 "go.etcd.io/etcd/client/v3"
 	"golang.org/x/crypto/cryptobyte"
 
+	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/features"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/storage/value"
 	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
 	mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/v1beta1"
-	"k8s.io/apiserver/pkg/util/feature"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/rest"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
@@ -125,6 +128,8 @@ func (r envelope) plainTextPayload(secretETCDPath string) ([]byte, error) {
 // 8. No-op updates to the secret should cause new AES GCM key to be used
 // 9. Direct AES GCM decryption works after the new AES GCM key is used
 func TestKMSProvider(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
+
 	encryptionConfig := `
 kind: EncryptionConfiguration
 apiVersion: apiserver.config.k8s.io/v1
@@ -301,6 +306,8 @@ resources:
 // 10. confirm that cluster wide secret read still works
 // 11. confirm that api server can restart with last applied encryption config
 func TestEncryptionConfigHotReload(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
+
 	storageConfig := framework.SharedEtcd()
 	encryptionConfig := `
 kind: EncryptionConfiguration
@@ -596,14 +603,20 @@ resources:
 
 	t.Run("encrypt all resources", func(t *testing.T) {
 		_ = mock.NewBase64Plugin(t, "@encrypt-all-kms-provider.sock")
-		defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)()
+		// To ensure we are checking all REST resources
-		defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)()
+		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, "AllAlpha", true)()
+		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, "AllBeta", true)()
+		// Need to enable this explicitly as the feature is deprecated
+		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
+
 		test, err := newTransformTest(t, encryptionConfig, false, "", nil)
 		if err != nil {
 			t.Fatalf("failed to start KUBE API Server with encryptionConfig")
 		}
 		defer test.cleanUp()
 
+		etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(test.kubeAPIServer.ClientConfig), false, etcd.GetCustomResourceDefinitionData()...)
+
 		_, serverResources, err := test.restClient.Discovery().ServerGroupsAndResources()
 		if err != nil {
 			t.Fatal(err)
@@ -612,6 +625,8 @@ resources:
 		client := dynamic.NewForConfigOrDie(test.kubeAPIServer.ClientConfig)
 
 		etcdStorageData := etcd.GetEtcdStorageDataForNamespace(testNamespace)
+		restResourceSet := sets.New[schema.GroupVersionResource]()
+		stubResourceSet := sets.New[schema.GroupVersionResource]()
 		for _, resource := range resources {
 			gvr := resource.Mapping.Resource
 			stub := etcdStorageData[gvr].Stub
@@ -621,7 +636,7 @@ resources:
 				t.Errorf("skipping resource %s because stub is empty", gvr)
 				continue
 			}
-
+			restResourceSet.Insert(gvr)
 			dynamicClient, obj, err := etcd.JSONToUnstructured(stub, testNamespace, &meta.RESTMapping{
 				Resource:         gvr,
 				GroupVersionKind: gvr.GroupVersion().WithKind(resource.Mapping.GroupVersionKind.Kind),
@@ -636,7 +651,15 @@ resources:
 				t.Fatal(err)
 			}
 		}
-
+		for gvr, data := range etcdStorageData {
+			if data.Stub == "" {
+				continue
+			}
+			stubResourceSet.Insert(gvr)
+		}
+		if !restResourceSet.Equal(stubResourceSet) {
+			t.Errorf("failed to check all REST resources: %q", restResourceSet.SymmetricDifference(stubResourceSet).UnsortedList())
+		}
 		rawClient, etcdClient, err := integration.GetEtcdClients(test.kubeAPIServer.ServerOpts.Etcd.StorageConfig.Transport)
 		if err != nil {
 			t.Fatalf("failed to create etcd client: %v", err)
@@ -710,6 +733,8 @@ resources:
 	_ = mock.NewBase64Plugin(t, "@kms-provider.sock")
 	_ = mock.NewBase64Plugin(t, "@encrypt-all-kms-provider.sock")
 
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
+
 	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
 	if err != nil {
 		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
@@ -781,6 +806,8 @@ resources:
 }
 
 func TestEncryptionConfigHotReloadFileWatch(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
+
 	testCases := []struct {
 		sleep      time.Duration
 		name       string
@@ -998,6 +1025,8 @@ func verifyIfKMSTransformersSwapped(t *testing.T, wantPrefix string, test *trans
 }
 
 func TestKMSHealthz(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
+
 	encryptionConfig := `
 kind: EncryptionConfiguration
 apiVersion: apiserver.config.k8s.io/v1
@@ -1059,6 +1088,8 @@ resources:
 }
 
 func TestKMSHealthzWithReload(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
+
 	encryptionConfig := `
 kind: EncryptionConfiguration
 apiVersion: apiserver.config.k8s.io/v1

test/integration/controlplane/transformation/kmsv2_transformation_test.go

@@ -64,7 +64,6 @@ import (
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
-	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	"k8s.io/klog/v2"
 	kmsv2api "k8s.io/kms/apis/v2"
 	kmsv2svc "k8s.io/kms/pkg/service"
@@ -163,6 +162,84 @@ func (r envelopekmsv2) plainTextPayload(secretETCDPath string) ([]byte, error) {
 	return plainSecret, nil
 }
 
+// TestDefaultValues tests default flag values without setting any of the feature flags or
+// calling SetKDFForTests, and assert that the data stored in etcd is using KDF
+func TestDefaultValues(t *testing.T) {
+	if encryptionconfig.GetKDF() != true {
+		t.Fatalf("without updating the feature flags, default value of KMSv2KDF should be enabled.")
+	}
+	if utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) != true {
+		t.Fatalf("without updating the feature flags, default value of KMSv2 should be enabled.")
+	}
+	if utilfeature.DefaultFeatureGate.Enabled(features.KMSv1) != false {
+		t.Fatalf("without updating the feature flags, default value of KMSv1 should be disabled.")
+	}
+	// since encryptionconfig.GetKDF() is true by default, following test should verify if
+	// object.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	t.Cleanup(cancel)
+
+	encryptionConfig := `
+kind: EncryptionConfiguration
+apiVersion: apiserver.config.k8s.io/v1
+resources:
+  - resources:
+    - pods
+    providers:
+    - kms:
+       apiVersion: v2
+       name: kms-provider
+       endpoint: unix:///@kms-provider.sock
+`
+	_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
+
+	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
+	if err != nil {
+		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
+	}
+	t.Cleanup(test.cleanUp)
+
+	client := kubernetes.NewForConfigOrDie(test.kubeAPIServer.ClientConfig)
+	if _, err := client.CoreV1().Pods(testNamespace).Create(ctx, &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+		},
+		Spec: corev1.PodSpec{
+			Containers: []corev1.Container{
+				{
+					Name:  "busybox",
+					Image: "busybox",
+				},
+			},
+		},
+	}, metav1.CreateOptions{}); err != nil {
+		t.Fatal(err)
+	}
+
+	config := test.kubeAPIServer.ServerOpts.Etcd.StorageConfig
+	rawClient, etcdClient, err := integration.GetEtcdClients(config.Transport)
+	if err != nil {
+		t.Fatalf("failed to create etcd client: %v", err)
+	}
+	t.Cleanup(func() { _ = rawClient.Close() })
+
+	response, err := etcdClient.Get(ctx, "/"+config.Prefix+"/pods/"+testNamespace+"/", clientv3.WithPrefix())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(response.Kvs) != 1 {
+		t.Fatalf("expected 1 KVs, but got %d", len(response.Kvs))
+	}
+	object := kmstypes.EncryptedObject{}
+	v := bytes.TrimPrefix(response.Kvs[0].Value, []byte("k8s:enc:kms:v2:kms-provider:"))
+	if err := proto.Unmarshal(v, &object); err != nil {
+		t.Fatal(err)
+	}
+	if object.EncryptedDEKSourceType != kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED {
+		t.Errorf("invalid type: %d", object.EncryptedDEKSourceType)
+	}
+}
+
 // TestKMSv2Provider is an integration test between KubeAPI, ETCD and KMSv2 Plugin
 // Concretely, this test verifies the following integration contracts:
 // 1. Raw records in ETCD that were processed by KMSv2 Provider should be prefixed with k8s:enc:kms:v2:<plugin name>:
@@ -171,22 +248,19 @@ func (r envelopekmsv2) plainTextPayload(secretETCDPath string) ([]byte, error) {
 // 4. The cipherTextPayload (ex. Secret) should be encrypted via AES GCM transform / extended nonce GCM
 // 5. kmstypes.EncryptedObject structure should be serialized and deposited in ETCD
 func TestKMSv2Provider(t *testing.T) {
-	defaultUseSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF)
+	defaultUseSeed := encryptionconfig.GetKDF()
 
 	t.Run("regular gcm", func(t *testing.T) {
-		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)()
+		defer encryptionconfig.SetKDFForTests(false)()
 		testKMSv2Provider(t, !defaultUseSeed)
 	})
-
 	t.Run("extended nonce gcm", func(t *testing.T) {
-		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, true)()
+		defer encryptionconfig.SetKDFForTests(true)()
 		testKMSv2Provider(t, defaultUseSeed)
 	})
 }
 
 func testKMSv2Provider(t *testing.T, useSeed bool) {
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
-
 	encryptionConfig := `
 kind: EncryptionConfiguration
 apiVersion: apiserver.config.k8s.io/v1
@@ -327,19 +401,15 @@ resources:
 // 7. when kms-plugin is down, no-op update for a pod should succeed and not result in RV change even once the DEK/seed is valid
 func TestKMSv2ProviderKeyIDStaleness(t *testing.T) {
 	t.Run("regular gcm", func(t *testing.T) {
-		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)()
+		defer encryptionconfig.SetKDFForTests(false)()
 		testKMSv2ProviderKeyIDStaleness(t)
 	})
-
 	t.Run("extended nonce gcm", func(t *testing.T) {
-		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, true)()
 		testKMSv2ProviderKeyIDStaleness(t)
 	})
 }
 
 func testKMSv2ProviderKeyIDStaleness(t *testing.T) {
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
-
 	encryptionConfig := `
 kind: EncryptionConfiguration
 apiVersion: apiserver.config.k8s.io/v1
@@ -382,7 +452,7 @@ resources:
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
 	t.Cleanup(cancel)
 
-	useSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF)
+	useSeed := encryptionconfig.GetKDF()
 
 	var firstEncryptedDEKSource []byte
 	var f checkFunc
@@ -577,9 +647,8 @@ resources:
 	if version7 != version8 {
 		t.Fatalf("Resource version should not have changed after plugin health is restored. old pod: %v, new pod: %v", updatedNewPod, updatedNewPod2)
 	}
-
 	// flip the current config
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, !useSeed)()
+	defer encryptionconfig.SetKDFForTests(!useSeed)()
 
 	// 9. confirm that no-op update for a pod results in RV change due to KDF config change
 	var version9 string
@@ -603,7 +672,7 @@ resources:
 
 func TestKMSv2ProviderDEKSourceReuse(t *testing.T) {
 	t.Run("regular gcm", func(t *testing.T) {
-		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)()
+		defer encryptionconfig.SetKDFForTests(false)()
 		testKMSv2ProviderDEKSourceReuse(t,
 			func(i int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject) {
 				if obj.KeyID != "1" {
@@ -618,9 +687,8 @@ func TestKMSv2ProviderDEKSourceReuse(t *testing.T) {
 			},
 		)
 	})
-
 	t.Run("extended nonce gcm", func(t *testing.T) {
-		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, true)()
+		defer encryptionconfig.SetKDFForTests(true)()
 		testKMSv2ProviderDEKSourceReuse(t,
 			func(_ int, _ uint64, etcdKey string, obj kmstypes.EncryptedObject) {
 				if obj.KeyID != "1" {
@@ -632,8 +700,6 @@ func TestKMSv2ProviderDEKSourceReuse(t *testing.T) {
 }
 
 func testKMSv2ProviderDEKSourceReuse(t *testing.T, f checkFunc) {
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
-
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 	t.Cleanup(cancel)
 
@@ -705,7 +771,7 @@ func assertPodDEKSources(ctx context.Context, t *testing.T, config storagebacken
 		t.Fatalf("expected %d KVs, but got %d", podCount, len(response.Kvs))
 	}
 
-	useSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF)
+	useSeed := encryptionconfig.GetKDF()
 
 	out := make([]kmstypes.EncryptedObject, len(response.Kvs))
 	for i, kv := range response.Kvs {
@@ -767,8 +833,7 @@ func assertPodDEKSources(ctx context.Context, t *testing.T, config storagebacken
 }
 
 func TestKMSv2Healthz(t *testing.T) {
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
+	defer encryptionconfig.SetKDFForTests(randomBool())()
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, randomBool())()
 
 	encryptionConfig := `
 kind: EncryptionConfiguration
@@ -833,8 +898,7 @@ resources:
 }
 
 func TestKMSv2SingleService(t *testing.T) {
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
+	defer encryptionconfig.SetKDFForTests(randomBool())()
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, randomBool())()
 
 	var kmsv2Calls int
 	origEnvelopeKMSv2ServiceFactory := encryptionconfig.EnvelopeKMSv2ServiceFactory
@@ -903,12 +967,8 @@ resources:
 
 // TestKMSv2FeatureFlag is an integration test between KubeAPI and ETCD
 // Concretely, this test verifies the following:
-// 1. When feature flag is not enabled, loading a encryptionConfig with KMSv2 should fail
+// 1. When feature flag is enabled, loading a encryptionConfig with KMSv2 should work
-// 2. When feature flag is enabled, loading a encryptionConfig with KMSv2 should work
+// 2. After a restart, loading a encryptionConfig with the same KMSv2 plugin from 1 should work,
-// 3. When feature flag is disabled, loading a encryptionConfig with a non-v2 provider should work.
-// without performing a storage migration, decryption of existing data encrypted with v2 should fail for Get and List operations.
-// New data stored in etcd will no longer be encrypted using the external kms provider with v2 API.
-// 4. when feature flag is re-enabled, loading a encryptionConfig with the same KMSv2 plugin from 2 should work,
 // decryption of data encrypted with v2 should work
 func TestKMSv2FeatureFlag(t *testing.T) {
 	encryptionConfig := `
@@ -927,8 +987,7 @@ resources:
 	pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
 	storageConfig := framework.SharedEtcd()
 
-	// When feature flag is enabled, loading a encryptionConfig with KMSv1 and v2 should work
+	// KMSv2 is enabled by default. Loading a encryptionConfig with KMSv2 should work
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
 	test, err := newTransformTest(t, encryptionConfig, false, "", storageConfig)
 	if err != nil {
 		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
@@ -999,48 +1058,7 @@ resources:
 	}
 	test.shutdownAPIServer()
 
-	// When KMSv2 feature flag is disabled, loading a encryptionConfig with a non-v2 provider should work. without performing a storage migration, decryption of existing data encrypted with v2 should fail for Get and List operations.
+	// After a restart, loading a encryptionConfig with the same KMSv2 plugin before the restart should work, decryption of data encrypted with v2 should work
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, false)()
-
-	encryptionConfig1 := `
-kind: EncryptionConfiguration
-apiVersion: apiserver.config.k8s.io/v1
-resources:
-  - resources:
-    - secrets
-    providers:
-    - aescbc:
-        keys:
-        - name: key1
-          secret: c2VjcmV0IGlzIHNlY3VyZQ==
-`
-	test, err = newTransformTest(t, encryptionConfig1, false, "", storageConfig)
-	if err != nil {
-		t.Fatalf("Failed to restart api server, error: %v", err)
-	}
-
-	_, err = test.createSecret("test2", testNamespace)
-	if err != nil {
-		t.Fatalf("Failed to create test secret, error: %v", err)
-	}
-	test.runResource(t, unSealWithCBCTransformer, aesCBCPrefix, "", "v1", "secrets", "test2", testNamespace)
-
-	secretClient = test.restClient.CoreV1().Secrets(testNamespace)
-
-	// Getting an old secret that was encrypted by another provider should fail
-	_, err = secretClient.Get(ctx, testSecret, metav1.GetOptions{})
-	if err == nil || !strings.Contains(err.Error(), "no matching prefix found") {
-		t.Fatalf("using a new provider, get Secret %s from %s should return err containing: no matching prefix found. Got err: %v", testSecret, testNamespace, err)
-	}
-	// List all cluster wide secrets should fail
-	_, err = test.restClient.CoreV1().Secrets("").List(ctx, metav1.ListOptions{})
-	if err == nil || !strings.Contains(err.Error(), "no matching prefix found") {
-		t.Fatalf("using a new provider, LIST all Secrets should return err containing: no matching prefix found. Got err: %v", err)
-	}
-	test.shutdownAPIServer()
-
-	// when feature flag is re-enabled, loading a encryptionConfig with the same KMSv2 plugin before the restart should work, decryption of data encrypted with v2 should work
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
 
 	test, err = newTransformTest(t, encryptionConfig, false, "", storageConfig)
 	if err != nil {
@@ -1059,12 +1077,6 @@ resources:
 	if secretVal != string(s.Data[secretKey]) {
 		t.Fatalf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
 	}
-	secretClient = test.restClient.CoreV1().Secrets(testNamespace)
-	// Getting an old secret that was encrypted by another plugin should fail
-	_, err = secretClient.Get(ctx, "test2", metav1.GetOptions{})
-	if err == nil || !strings.Contains(err.Error(), "no matching prefix found") {
-		t.Fatalf("after re-enabling feature gate, get test2 Secret from %s should return err containing: no matching prefix found. actual err: %v", testNamespace, err)
-	}
 }
 
 var benchSecret *api.Secret
@@ -1075,8 +1087,7 @@ func BenchmarkKMSv2KDF(b *testing.B) {
 	klog.SetOutput(io.Discard)
 	klog.LogToStderr(false)
 
-	defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
+	defer encryptionconfig.SetKDFForTests(false)()
-	defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)()
 
 	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
 	b.Cleanup(cancel)
@@ -1231,8 +1242,7 @@ func BenchmarkKMSv2REST(b *testing.B) {
 	klog.SetOutput(io.Discard)
 	klog.LogToStderr(false)
 
-	defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
+	defer encryptionconfig.SetKDFForTests(true)()
-	defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)()
 
 	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
 	b.Cleanup(cancel)
@@ -1317,19 +1327,16 @@ func randomBool() bool { return utilrand.Int()%2 == 1 }
 // TestKMSv2ProviderLegacyData confirms that legacy data recorded from the earliest released commit can still be read.
 func TestKMSv2ProviderLegacyData(t *testing.T) {
 	t.Run("regular gcm", func(t *testing.T) {
-		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)()
+		defer encryptionconfig.SetKDFForTests(false)()
 		testKMSv2ProviderLegacyData(t)
 	})
-
 	t.Run("extended nonce gcm", func(t *testing.T) {
-		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, true)()
+		defer encryptionconfig.SetKDFForTests(true)()
 		testKMSv2ProviderLegacyData(t)
 	})
 }
 
 func testKMSv2ProviderLegacyData(t *testing.T) {
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
-
 	encryptionConfig := `
 kind: EncryptionConfiguration
 apiVersion: apiserver.config.k8s.io/v1