diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go index c7d699d5405..7fa48a5c4f5 100644 --- a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go @@ -122,6 +122,7 @@ const ( // kep: https://kep.k8s.io/3299 // alpha: v1.25 // beta: v1.27 + // stable: v1.29 // // Enables KMS v2 API for encryption at rest. KMSv2 featuregate.Feature = "KMSv2" @@ -129,6 +130,7 @@ const ( // owner: @enj // kep: https://kep.k8s.io/3299 // beta: v1.28 + // stable: v1.29 // // Enables the use of derived encryption keys with KMS v2. KMSv2KDF featuregate.Feature = "KMSv2KDF" @@ -288,11 +290,11 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS EfficientWatchResumption: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, - KMSv1: {Default: true, PreRelease: featuregate.Deprecated}, + KMSv1: {Default: false, PreRelease: featuregate.Deprecated}, - KMSv2: {Default: true, PreRelease: featuregate.Beta}, + KMSv2: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31 - KMSv2KDF: {Default: true, PreRelease: featuregate.Beta}, // lock to true in 1.29 once KMSv2 is GA, remove in 1.31 + KMSv2KDF: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31 OpenAPIEnums: {Default: true, PreRelease: featuregate.Beta}, diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go index 4af2f1faf87..a7b35153966 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go @@ -107,6 +107,26 @@ const ( var codecs serializer.CodecFactory +// this atomic bool allows us to swap enablement of the KMSv2KDF feature in tests +// as the feature gate is now locked to true starting with v1.29 +// Note: it cannot be set by an end user +var kdfDisabled atomic.Bool + +// this function should only be called in tests to swap enablement of the KMSv2KDF feature +func SetKDFForTests(b bool) func() { + kdfDisabled.Store(!b) + return func() { + kdfDisabled.Store(false) + } +} + +// this function should be used to determine enablement of the KMSv2KDF feature +// instead of getting it from DefaultFeatureGate as the feature gate is now locked +// to true starting with v1.29 +func GetKDF() bool { + return !kdfDisabled.Load() +} + func init() { configScheme := runtime.NewScheme() utilruntime.Must(apiserverconfig.AddToScheme(configScheme)) @@ -138,6 +158,7 @@ type kmsv2PluginProbe struct { lastResponse *kmsPluginHealthzResponse l *sync.Mutex apiServerID string + version string } type kmsHealthChecker []healthz.HealthChecker @@ -369,7 +390,7 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey // this gate can only change during tests, but the check is cheap enough to always make // this allows us to easily exercise both modes without restarting the API server // TODO integration test that this dynamically takes effect - useSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF) + useSeed := GetKDF() stateUseSeed := state.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED // state is valid and status keyID is unchanged from when we generated this DEK/seed so there is no need to rotate it @@ -454,8 +475,16 @@ func (h *kmsv2PluginProbe) isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx context.C if response.Healthz != "ok" { errs = append(errs, fmt.Errorf("got unexpected healthz status: %s", response.Healthz)) } - if response.Version != envelopekmsv2.KMSAPIVersion { - errs = append(errs, fmt.Errorf("expected KMSv2 API version %s, got %s", envelopekmsv2.KMSAPIVersion, response.Version)) + if response.Version != envelopekmsv2.KMSAPIVersionv2 && response.Version != envelopekmsv2.KMSAPIVersionv2beta1 { + errs = append(errs, fmt.Errorf("expected KMSv2 API version %s, got %s", envelopekmsv2.KMSAPIVersionv2, response.Version)) + } else { + // set version for the first status response + if len(h.version) == 0 { + h.version = response.Version + } + if h.version != response.Version { + errs = append(errs, fmt.Errorf("KMSv2 API version should not change after the initial status response version %s, got %s", h.version, response.Version)) + } } if errCode, err := envelopekmsv2.ValidateKeyID(response.KeyID); err != nil { diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go index 46c76284b73..3a81e397d91 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go @@ -187,7 +187,7 @@ func TestLegacyConfig(t *testing.T) { } func TestEncryptionProviderConfigCorrect(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() // Set factory for mock envelope service factory := envelopeServiceFactory @@ -353,42 +353,33 @@ func TestKMSv1Deprecation(t *testing.T) { func TestKMSvsEnablement(t *testing.T) { testCases := []struct { - name string - kmsv2Enabled bool - filePath string - expectedErr string + name string + filePath string + expectedErr string }{ { - name: "config with kmsv2 and kmsv1, KMSv2=false", - kmsv2Enabled: false, - filePath: "testdata/valid-configs/kms/multiple-providers-kmsv2.yaml", - expectedErr: "KMSv2 feature is not enabled", + name: "config with kmsv2 and kmsv1, KMSv2=true, KMSv1=false, should fail when feature is disabled", + filePath: "testdata/valid-configs/kms/multiple-providers-mixed.yaml", + expectedErr: "KMSv1 is deprecated and will only receive security updates going forward. Use KMSv2 instead", }, { - name: "config with kmsv2 and kmsv1, KMSv2=true", - kmsv2Enabled: true, - filePath: "testdata/valid-configs/kms/multiple-providers-kmsv2.yaml", - expectedErr: "", - }, - { - name: "config with kmsv1, KMSv2=false", - kmsv2Enabled: false, - filePath: "testdata/valid-configs/kms/multiple-providers.yaml", - expectedErr: "", + name: "config with kmsv2, KMSv2=true, KMSv1=false", + filePath: "testdata/valid-configs/kms/multiple-providers-kmsv2.yaml", + expectedErr: "", }, } for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - // Just testing KMSv2 feature flag - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() - - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, testCase.kmsv2Enabled)() + // only the KMSv2 feature flag is enabled _, err := LoadEncryptionConfig(testContext(t), testCase.filePath, false, "") - if !strings.Contains(errString(err), testCase.expectedErr) { + if len(testCase.expectedErr) > 0 && !strings.Contains(errString(err), testCase.expectedErr) { t.Fatalf("expected error %q, got %q", testCase.expectedErr, errString(err)) } + if len(testCase.expectedErr) == 0 && err != nil { + t.Fatalf("unexpected error %q", errString(err)) + } }) } @@ -400,43 +391,6 @@ func TestKMSvsEnablement(t *testing.T) { config apiserverconfig.EncryptionConfiguration wantV2Used bool }{ - { - name: "with kmsv1 and kmsv2, KMSv2=false", - kmsv2Enabled: false, - config: apiserverconfig.EncryptionConfiguration{ - Resources: []apiserverconfig.ResourceConfiguration{ - { - Resources: []string{"secrets"}, - Providers: []apiserverconfig.ProviderConfiguration{ - { - KMS: &apiserverconfig.KMSConfiguration{ - Name: "kms", - APIVersion: "v1", - Timeout: &metav1.Duration{ - Duration: 1 * time.Second, - }, - Endpoint: "unix:///tmp/testprovider.sock", - CacheSize: pointer.Int32(1000), - }, - }, - { - KMS: &apiserverconfig.KMSConfiguration{ - Name: "another-kms", - APIVersion: "v2", - Timeout: &metav1.Duration{ - Duration: 1 * time.Second, - }, - Endpoint: "unix:///tmp/anothertestprovider.sock", - CacheSize: pointer.Int32(1000), - }, - }, - }, - }, - }, - }, - expectedErr: "KMSv2 feature is not enabled", - wantV2Used: false, - }, { name: "with kmsv1 and kmsv2, KMSv2=true", kmsv2Enabled: true, @@ -501,7 +455,7 @@ func TestKMSvsEnablement(t *testing.T) { } func TestKMSMaxTimeout(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() testCases := []struct { name string @@ -749,7 +703,7 @@ func TestKMSMaxTimeout(t *testing.T) { } func TestKMSPluginHealthz(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() kmsv2Probe := &kmsv2PluginProbe{ name: "foo", @@ -823,7 +777,7 @@ func TestKMSPluginHealthz(t *testing.T) { }, { desc: "Install multiple healthz with v1 and v2", - config: "testdata/valid-configs/kms/multiple-providers-kmsv2.yaml", + config: "testdata/valid-configs/kms/multiple-providers-mixed.yaml", want: []healthChecker{ kmsv2Probe, &kmsPluginProbe{ @@ -900,6 +854,7 @@ func TestKMSPluginHealthz(t *testing.T) { // tests for masking rules func TestWildcardMasking(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() testCases := []struct { desc string @@ -1308,7 +1263,7 @@ func TestWildcardMasking(t *testing.T) { } func TestWildcardStructure(t *testing.T) { - + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() testCases := []struct { desc string expectedResourceTransformers map[string]string @@ -1752,7 +1707,7 @@ func TestIsKMSv2ProviderHealthyError(t *testing.T) { statusResponse: &kmsservice.StatusResponse{ Healthz: "unhealthy", }, - expectedErr: "got unexpected healthz status: unhealthy, expected KMSv2 API version v2beta1, got , got invalid KMSv2 KeyID ", + expectedErr: "got unexpected healthz status: unhealthy, expected KMSv2 API version v2, got , got invalid KMSv2 KeyID ", wantMetrics: ` # HELP apiserver_envelope_encryption_invalid_key_id_from_status_total [ALPHA] Number of times an invalid keyID is returned by the Status RPC call split by error. # TYPE apiserver_envelope_encryption_invalid_key_id_from_status_total counter @@ -1760,11 +1715,11 @@ func TestIsKMSv2ProviderHealthyError(t *testing.T) { `, }, { - desc: "version is not v2beta1", + desc: "version is not v2", statusResponse: &kmsservice.StatusResponse{ Version: "v1beta1", }, - expectedErr: "got unexpected healthz status: , expected KMSv2 API version v2beta1, got v1beta1, got invalid KMSv2 KeyID ", + expectedErr: "got unexpected healthz status: , expected KMSv2 API version v2, got v1beta1, got invalid KMSv2 KeyID ", wantMetrics: ` # HELP apiserver_envelope_encryption_invalid_key_id_from_status_total [ALPHA] Number of times an invalid keyID is returned by the Status RPC call split by error. # TYPE apiserver_envelope_encryption_invalid_key_id_from_status_total counter @@ -1788,7 +1743,7 @@ func TestIsKMSv2ProviderHealthyError(t *testing.T) { desc: "invalid long keyID", statusResponse: &kmsservice.StatusResponse{ Healthz: "ok", - Version: "v2beta1", + Version: "v2", KeyID: sampleInvalidKeyID, }, expectedErr: "got invalid KMSv2 KeyID ", @@ -1816,6 +1771,52 @@ func TestIsKMSv2ProviderHealthyError(t *testing.T) { } } +// test to ensure KMSv2 API version is not changed after the first status response +func TestKMSv2SameVersionFromStatus(t *testing.T) { + probe := &kmsv2PluginProbe{name: "testplugin"} + service, _ := newMockEnvelopeKMSv2Service(testContext(t), "unix:///tmp/testprovider.sock", "providerName", 3*time.Second) + probe.l = &sync.Mutex{} + probe.state.Store(&envelopekmsv2.State{}) + probe.service = service + + testCases := []struct { + desc string + expectedErr string + newVersion string + }{ + { + desc: "version changed", + newVersion: "v2", + expectedErr: "KMSv2 API version should not change", + }, + { + desc: "version unchanged", + newVersion: "v2beta1", + expectedErr: "", + }, + } + for _, tt := range testCases { + t.Run(tt.desc, func(t *testing.T) { + statusResponse := &kmsservice.StatusResponse{ + Healthz: "ok", + Version: "v2beta1", + KeyID: "1", + } + if err := probe.isKMSv2ProviderHealthyAndMaybeRotateDEK(testContext(t), statusResponse); err != nil { + t.Fatal(err) + } + statusResponse.Version = tt.newVersion + err := probe.isKMSv2ProviderHealthyAndMaybeRotateDEK(testContext(t), statusResponse) + if len(tt.expectedErr) > 0 && !strings.Contains(errString(err), tt.expectedErr) { + t.Errorf("expected err %q, got %q", tt.expectedErr, errString(err)) + } + if len(tt.expectedErr) == 0 && err != nil { + t.Fatal(err) + } + }) + } +} + func testContext(t *testing.T) context.Context { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -1840,7 +1841,7 @@ func TestComputeEncryptionConfigHash(t *testing.T) { } func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) { - defaultUseSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF) + defaultUseSeed := GetKDF() origNowFunc := envelopekmsv2.NowFunc now := origNowFunc() // freeze time @@ -2065,7 +2066,7 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, tt.useSeed)() + defer SetKDFForTests(tt.useSeed)() var buf bytes.Buffer klog.SetOutput(&buf) diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kms/multiple-providers-kmsv2.yaml b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kms/multiple-providers-kmsv2.yaml index fd6e9079ea0..1265dd1b481 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kms/multiple-providers-kmsv2.yaml +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kms/multiple-providers-kmsv2.yaml @@ -10,6 +10,7 @@ resources: endpoint: unix:///tmp/testprovider.sock timeout: 15s - kms: + apiVersion: v2 name: bar endpoint: unix:///tmp/testprovider.sock timeout: 15s diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kms/multiple-providers-mixed.yaml b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kms/multiple-providers-mixed.yaml new file mode 100644 index 00000000000..fd6e9079ea0 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/testdata/valid-configs/kms/multiple-providers-mixed.yaml @@ -0,0 +1,15 @@ +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: + - resources: + - secrets + providers: + - kms: + apiVersion: v2 + name: foo + endpoint: unix:///tmp/testprovider.sock + timeout: 15s + - kms: + name: bar + endpoint: unix:///tmp/testprovider.sock + timeout: 15s diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/etcd_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/etcd_test.go index 89a1251bc1e..10cd38a863a 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/etcd_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/etcd_test.go @@ -229,7 +229,7 @@ func TestParseWatchCacheSizes(t *testing.T) { } func TestKMSHealthzEndpoint(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() testCases := []struct { name string diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go index 23de3717a18..0ccb1d0816e 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go @@ -50,8 +50,10 @@ func init() { } const ( - // KMSAPIVersion is the version of the KMS API. - KMSAPIVersion = "v2beta1" + // KMSAPIVersionv2 is a version of the KMS API. + KMSAPIVersionv2 = "v2" + // KMSAPIVersionv2beta1 is a version of the KMS API. + KMSAPIVersionv2beta1 = "v2beta1" // annotationsMaxSize is the maximum size of the annotations. annotationsMaxSize = 32 * 1024 // 32 kB // KeyIDMaxSize is the maximum size of the keyID. diff --git a/staging/src/k8s.io/kms/internal/plugins/_mock/pkcs11/pkcs11.go b/staging/src/k8s.io/kms/internal/plugins/_mock/pkcs11/pkcs11.go index 8762a077e85..f1cc51940fc 100644 --- a/staging/src/k8s.io/kms/internal/plugins/_mock/pkcs11/pkcs11.go +++ b/staging/src/k8s.io/kms/internal/plugins/_mock/pkcs11/pkcs11.go @@ -111,7 +111,7 @@ func (s *pkcs11RemoteService) Decrypt(ctx context.Context, uid string, req *serv func (s *pkcs11RemoteService) Status(ctx context.Context) (*service.StatusResponse, error) { return &service.StatusResponse{ - Version: "v2beta1", + Version: "v2", Healthz: "ok", KeyID: s.keyID, }, nil diff --git a/staging/src/k8s.io/kms/pkg/service/grpc_service_test.go b/staging/src/k8s.io/kms/pkg/service/grpc_service_test.go index e6f9b6a517a..0007059236e 100644 --- a/staging/src/k8s.io/kms/pkg/service/grpc_service_test.go +++ b/staging/src/k8s.io/kms/pkg/service/grpc_service_test.go @@ -35,7 +35,7 @@ import ( kmsapi "k8s.io/kms/apis/v2" ) -const version = "v2alpha1" +const version = "v2" func TestGRPCService(t *testing.T) { t.Parallel() diff --git a/test/integration/controlplane/transformation/kms_transformation_test.go b/test/integration/controlplane/transformation/kms_transformation_test.go index 941234f9b46..d6466cd9f4d 100644 --- a/test/integration/controlplane/transformation/kms_transformation_test.go +++ b/test/integration/controlplane/transformation/kms_transformation_test.go @@ -38,16 +38,19 @@ import ( clientv3 "go.etcd.io/etcd/client/v3" "golang.org/x/crypto/cryptobyte" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/features" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/storage/value" aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes" mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/v1beta1" - "k8s.io/apiserver/pkg/util/feature" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" featuregatetesting "k8s.io/component-base/featuregate/testing" @@ -125,6 +128,8 @@ func (r envelope) plainTextPayload(secretETCDPath string) ([]byte, error) { // 8. No-op updates to the secret should cause new AES GCM key to be used // 9. Direct AES GCM decryption works after the new AES GCM key is used func TestKMSProvider(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() + encryptionConfig := ` kind: EncryptionConfiguration apiVersion: apiserver.config.k8s.io/v1 @@ -301,6 +306,8 @@ resources: // 10. confirm that cluster wide secret read still works // 11. confirm that api server can restart with last applied encryption config func TestEncryptionConfigHotReload(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() + storageConfig := framework.SharedEtcd() encryptionConfig := ` kind: EncryptionConfiguration @@ -596,14 +603,20 @@ resources: t.Run("encrypt all resources", func(t *testing.T) { _ = mock.NewBase64Plugin(t, "@encrypt-all-kms-provider.sock") - defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)() - defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)() + // To ensure we are checking all REST resources + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, "AllAlpha", true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, "AllBeta", true)() + // Need to enable this explicitly as the feature is deprecated + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() + test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig") } defer test.cleanUp() + etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(test.kubeAPIServer.ClientConfig), false, etcd.GetCustomResourceDefinitionData()...) + _, serverResources, err := test.restClient.Discovery().ServerGroupsAndResources() if err != nil { t.Fatal(err) @@ -612,6 +625,8 @@ resources: client := dynamic.NewForConfigOrDie(test.kubeAPIServer.ClientConfig) etcdStorageData := etcd.GetEtcdStorageDataForNamespace(testNamespace) + restResourceSet := sets.New[schema.GroupVersionResource]() + stubResourceSet := sets.New[schema.GroupVersionResource]() for _, resource := range resources { gvr := resource.Mapping.Resource stub := etcdStorageData[gvr].Stub @@ -621,7 +636,7 @@ resources: t.Errorf("skipping resource %s because stub is empty", gvr) continue } - + restResourceSet.Insert(gvr) dynamicClient, obj, err := etcd.JSONToUnstructured(stub, testNamespace, &meta.RESTMapping{ Resource: gvr, GroupVersionKind: gvr.GroupVersion().WithKind(resource.Mapping.GroupVersionKind.Kind), @@ -636,7 +651,15 @@ resources: t.Fatal(err) } } - + for gvr, data := range etcdStorageData { + if data.Stub == "" { + continue + } + stubResourceSet.Insert(gvr) + } + if !restResourceSet.Equal(stubResourceSet) { + t.Errorf("failed to check all REST resources: %q", restResourceSet.SymmetricDifference(stubResourceSet).UnsortedList()) + } rawClient, etcdClient, err := integration.GetEtcdClients(test.kubeAPIServer.ServerOpts.Etcd.StorageConfig.Transport) if err != nil { t.Fatalf("failed to create etcd client: %v", err) @@ -710,6 +733,8 @@ resources: _ = mock.NewBase64Plugin(t, "@kms-provider.sock") _ = mock.NewBase64Plugin(t, "@encrypt-all-kms-provider.sock") + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() + test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) @@ -781,6 +806,8 @@ resources: } func TestEncryptionConfigHotReloadFileWatch(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() + testCases := []struct { sleep time.Duration name string @@ -998,6 +1025,8 @@ func verifyIfKMSTransformersSwapped(t *testing.T, wantPrefix string, test *trans } func TestKMSHealthz(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() + encryptionConfig := ` kind: EncryptionConfiguration apiVersion: apiserver.config.k8s.io/v1 @@ -1059,6 +1088,8 @@ resources: } func TestKMSHealthzWithReload(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() + encryptionConfig := ` kind: EncryptionConfiguration apiVersion: apiserver.config.k8s.io/v1 diff --git a/test/integration/controlplane/transformation/kmsv2_transformation_test.go b/test/integration/controlplane/transformation/kmsv2_transformation_test.go index 2ad70311baf..1b001681b7b 100644 --- a/test/integration/controlplane/transformation/kmsv2_transformation_test.go +++ b/test/integration/controlplane/transformation/kmsv2_transformation_test.go @@ -64,7 +64,6 @@ import ( "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/klog/v2" kmsv2api "k8s.io/kms/apis/v2" kmsv2svc "k8s.io/kms/pkg/service" @@ -163,6 +162,84 @@ func (r envelopekmsv2) plainTextPayload(secretETCDPath string) ([]byte, error) { return plainSecret, nil } +// TestDefaultValues tests default flag values without setting any of the feature flags or +// calling SetKDFForTests, and assert that the data stored in etcd is using KDF +func TestDefaultValues(t *testing.T) { + if encryptionconfig.GetKDF() != true { + t.Fatalf("without updating the feature flags, default value of KMSv2KDF should be enabled.") + } + if utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) != true { + t.Fatalf("without updating the feature flags, default value of KMSv2 should be enabled.") + } + if utilfeature.DefaultFeatureGate.Enabled(features.KMSv1) != false { + t.Fatalf("without updating the feature flags, default value of KMSv1 should be disabled.") + } + // since encryptionconfig.GetKDF() is true by default, following test should verify if + // object.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + t.Cleanup(cancel) + + encryptionConfig := ` +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: + - resources: + - pods + providers: + - kms: + apiVersion: v2 + name: kms-provider + endpoint: unix:///@kms-provider.sock +` + _ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock") + + test, err := newTransformTest(t, encryptionConfig, false, "", nil) + if err != nil { + t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) + } + t.Cleanup(test.cleanUp) + + client := kubernetes.NewForConfigOrDie(test.kubeAPIServer.ClientConfig) + if _, err := client.CoreV1().Pods(testNamespace).Create(ctx, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "busybox", + Image: "busybox", + }, + }, + }, + }, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + + config := test.kubeAPIServer.ServerOpts.Etcd.StorageConfig + rawClient, etcdClient, err := integration.GetEtcdClients(config.Transport) + if err != nil { + t.Fatalf("failed to create etcd client: %v", err) + } + t.Cleanup(func() { _ = rawClient.Close() }) + + response, err := etcdClient.Get(ctx, "/"+config.Prefix+"/pods/"+testNamespace+"/", clientv3.WithPrefix()) + if err != nil { + t.Fatal(err) + } + if len(response.Kvs) != 1 { + t.Fatalf("expected 1 KVs, but got %d", len(response.Kvs)) + } + object := kmstypes.EncryptedObject{} + v := bytes.TrimPrefix(response.Kvs[0].Value, []byte("k8s:enc:kms:v2:kms-provider:")) + if err := proto.Unmarshal(v, &object); err != nil { + t.Fatal(err) + } + if object.EncryptedDEKSourceType != kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED { + t.Errorf("invalid type: %d", object.EncryptedDEKSourceType) + } +} + // TestKMSv2Provider is an integration test between KubeAPI, ETCD and KMSv2 Plugin // Concretely, this test verifies the following integration contracts: // 1. Raw records in ETCD that were processed by KMSv2 Provider should be prefixed with k8s:enc:kms:v2:: @@ -171,22 +248,19 @@ func (r envelopekmsv2) plainTextPayload(secretETCDPath string) ([]byte, error) { // 4. The cipherTextPayload (ex. Secret) should be encrypted via AES GCM transform / extended nonce GCM // 5. kmstypes.EncryptedObject structure should be serialized and deposited in ETCD func TestKMSv2Provider(t *testing.T) { - defaultUseSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF) + defaultUseSeed := encryptionconfig.GetKDF() t.Run("regular gcm", func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)() + defer encryptionconfig.SetKDFForTests(false)() testKMSv2Provider(t, !defaultUseSeed) }) - t.Run("extended nonce gcm", func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, true)() + defer encryptionconfig.SetKDFForTests(true)() testKMSv2Provider(t, defaultUseSeed) }) } func testKMSv2Provider(t *testing.T, useSeed bool) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() - encryptionConfig := ` kind: EncryptionConfiguration apiVersion: apiserver.config.k8s.io/v1 @@ -327,19 +401,15 @@ resources: // 7. when kms-plugin is down, no-op update for a pod should succeed and not result in RV change even once the DEK/seed is valid func TestKMSv2ProviderKeyIDStaleness(t *testing.T) { t.Run("regular gcm", func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)() + defer encryptionconfig.SetKDFForTests(false)() testKMSv2ProviderKeyIDStaleness(t) }) - t.Run("extended nonce gcm", func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, true)() testKMSv2ProviderKeyIDStaleness(t) }) } func testKMSv2ProviderKeyIDStaleness(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() - encryptionConfig := ` kind: EncryptionConfiguration apiVersion: apiserver.config.k8s.io/v1 @@ -382,7 +452,7 @@ resources: ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) t.Cleanup(cancel) - useSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF) + useSeed := encryptionconfig.GetKDF() var firstEncryptedDEKSource []byte var f checkFunc @@ -577,9 +647,8 @@ resources: if version7 != version8 { t.Fatalf("Resource version should not have changed after plugin health is restored. old pod: %v, new pod: %v", updatedNewPod, updatedNewPod2) } - // flip the current config - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, !useSeed)() + defer encryptionconfig.SetKDFForTests(!useSeed)() // 9. confirm that no-op update for a pod results in RV change due to KDF config change var version9 string @@ -603,7 +672,7 @@ resources: func TestKMSv2ProviderDEKSourceReuse(t *testing.T) { t.Run("regular gcm", func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)() + defer encryptionconfig.SetKDFForTests(false)() testKMSv2ProviderDEKSourceReuse(t, func(i int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject) { if obj.KeyID != "1" { @@ -618,9 +687,8 @@ func TestKMSv2ProviderDEKSourceReuse(t *testing.T) { }, ) }) - t.Run("extended nonce gcm", func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, true)() + defer encryptionconfig.SetKDFForTests(true)() testKMSv2ProviderDEKSourceReuse(t, func(_ int, _ uint64, etcdKey string, obj kmstypes.EncryptedObject) { if obj.KeyID != "1" { @@ -632,8 +700,6 @@ func TestKMSv2ProviderDEKSourceReuse(t *testing.T) { } func testKMSv2ProviderDEKSourceReuse(t *testing.T, f checkFunc) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) t.Cleanup(cancel) @@ -705,7 +771,7 @@ func assertPodDEKSources(ctx context.Context, t *testing.T, config storagebacken t.Fatalf("expected %d KVs, but got %d", podCount, len(response.Kvs)) } - useSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF) + useSeed := encryptionconfig.GetKDF() out := make([]kmstypes.EncryptedObject, len(response.Kvs)) for i, kv := range response.Kvs { @@ -767,8 +833,7 @@ func assertPodDEKSources(ctx context.Context, t *testing.T, config storagebacken } func TestKMSv2Healthz(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, randomBool())() + defer encryptionconfig.SetKDFForTests(randomBool())() encryptionConfig := ` kind: EncryptionConfiguration @@ -833,8 +898,7 @@ resources: } func TestKMSv2SingleService(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, randomBool())() + defer encryptionconfig.SetKDFForTests(randomBool())() var kmsv2Calls int origEnvelopeKMSv2ServiceFactory := encryptionconfig.EnvelopeKMSv2ServiceFactory @@ -903,12 +967,8 @@ resources: // TestKMSv2FeatureFlag is an integration test between KubeAPI and ETCD // Concretely, this test verifies the following: -// 1. When feature flag is not enabled, loading a encryptionConfig with KMSv2 should fail -// 2. When feature flag is enabled, loading a encryptionConfig with KMSv2 should work -// 3. When feature flag is disabled, loading a encryptionConfig with a non-v2 provider should work. -// without performing a storage migration, decryption of existing data encrypted with v2 should fail for Get and List operations. -// New data stored in etcd will no longer be encrypted using the external kms provider with v2 API. -// 4. when feature flag is re-enabled, loading a encryptionConfig with the same KMSv2 plugin from 2 should work, +// 1. When feature flag is enabled, loading a encryptionConfig with KMSv2 should work +// 2. After a restart, loading a encryptionConfig with the same KMSv2 plugin from 1 should work, // decryption of data encrypted with v2 should work func TestKMSv2FeatureFlag(t *testing.T) { encryptionConfig := ` @@ -927,8 +987,7 @@ resources: pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock") storageConfig := framework.SharedEtcd() - // When feature flag is enabled, loading a encryptionConfig with KMSv1 and v2 should work - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() + // KMSv2 is enabled by default. Loading a encryptionConfig with KMSv2 should work test, err := newTransformTest(t, encryptionConfig, false, "", storageConfig) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) @@ -999,48 +1058,7 @@ resources: } test.shutdownAPIServer() - // When KMSv2 feature flag is disabled, loading a encryptionConfig with a non-v2 provider should work. without performing a storage migration, decryption of existing data encrypted with v2 should fail for Get and List operations. - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, false)() - - encryptionConfig1 := ` -kind: EncryptionConfiguration -apiVersion: apiserver.config.k8s.io/v1 -resources: - - resources: - - secrets - providers: - - aescbc: - keys: - - name: key1 - secret: c2VjcmV0IGlzIHNlY3VyZQ== -` - test, err = newTransformTest(t, encryptionConfig1, false, "", storageConfig) - if err != nil { - t.Fatalf("Failed to restart api server, error: %v", err) - } - - _, err = test.createSecret("test2", testNamespace) - if err != nil { - t.Fatalf("Failed to create test secret, error: %v", err) - } - test.runResource(t, unSealWithCBCTransformer, aesCBCPrefix, "", "v1", "secrets", "test2", testNamespace) - - secretClient = test.restClient.CoreV1().Secrets(testNamespace) - - // Getting an old secret that was encrypted by another provider should fail - _, err = secretClient.Get(ctx, testSecret, metav1.GetOptions{}) - if err == nil || !strings.Contains(err.Error(), "no matching prefix found") { - t.Fatalf("using a new provider, get Secret %s from %s should return err containing: no matching prefix found. Got err: %v", testSecret, testNamespace, err) - } - // List all cluster wide secrets should fail - _, err = test.restClient.CoreV1().Secrets("").List(ctx, metav1.ListOptions{}) - if err == nil || !strings.Contains(err.Error(), "no matching prefix found") { - t.Fatalf("using a new provider, LIST all Secrets should return err containing: no matching prefix found. Got err: %v", err) - } - test.shutdownAPIServer() - - // when feature flag is re-enabled, loading a encryptionConfig with the same KMSv2 plugin before the restart should work, decryption of data encrypted with v2 should work - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() + // After a restart, loading a encryptionConfig with the same KMSv2 plugin before the restart should work, decryption of data encrypted with v2 should work test, err = newTransformTest(t, encryptionConfig, false, "", storageConfig) if err != nil { @@ -1059,12 +1077,6 @@ resources: if secretVal != string(s.Data[secretKey]) { t.Fatalf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey])) } - secretClient = test.restClient.CoreV1().Secrets(testNamespace) - // Getting an old secret that was encrypted by another plugin should fail - _, err = secretClient.Get(ctx, "test2", metav1.GetOptions{}) - if err == nil || !strings.Contains(err.Error(), "no matching prefix found") { - t.Fatalf("after re-enabling feature gate, get test2 Secret from %s should return err containing: no matching prefix found. actual err: %v", testNamespace, err) - } } var benchSecret *api.Secret @@ -1075,8 +1087,7 @@ func BenchmarkKMSv2KDF(b *testing.B) { klog.SetOutput(io.Discard) klog.LogToStderr(false) - defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.KMSv2, true)() - defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)() + defer encryptionconfig.SetKDFForTests(false)() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) b.Cleanup(cancel) @@ -1231,8 +1242,7 @@ func BenchmarkKMSv2REST(b *testing.B) { klog.SetOutput(io.Discard) klog.LogToStderr(false) - defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.KMSv2, true)() - defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)() + defer encryptionconfig.SetKDFForTests(true)() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) b.Cleanup(cancel) @@ -1317,19 +1327,16 @@ func randomBool() bool { return utilrand.Int()%2 == 1 } // TestKMSv2ProviderLegacyData confirms that legacy data recorded from the earliest released commit can still be read. func TestKMSv2ProviderLegacyData(t *testing.T) { t.Run("regular gcm", func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)() + defer encryptionconfig.SetKDFForTests(false)() testKMSv2ProviderLegacyData(t) }) - t.Run("extended nonce gcm", func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, true)() + defer encryptionconfig.SetKDFForTests(true)() testKMSv2ProviderLegacyData(t) }) } func testKMSv2ProviderLegacyData(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() - encryptionConfig := ` kind: EncryptionConfiguration apiVersion: apiserver.config.k8s.io/v1