From db850931a8699e780dd794e1763fd0e54b4239b5 Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Mon, 29 Aug 2022 17:25:48 -0400 Subject: [PATCH] encryption config: no-op refactor to prepare for single loading Signed-off-by: Monis Khan --- .../server/options/encryptionconfig/config.go | 387 +++++++++--------- .../options/encryptionconfig/config_test.go | 151 ++++--- .../value/encrypt/envelope/envelope.go | 4 +- .../value/encrypt/envelope/envelope_test.go | 25 +- .../value/encrypt/envelope/kmsv2/envelope.go | 4 +- .../encrypt/envelope/kmsv2/envelope_test.go | 10 +- .../kmsv2_transformation_test.go | 2 +- 7 files changed, 269 insertions(+), 314 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go index c2b167104f1..bb6c948614c 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go @@ -67,125 +67,106 @@ type kmsPluginHealthzResponse struct { } type kmsPluginProbe struct { - name string - ttl time.Duration - envelope.Service + name string + ttl time.Duration + service envelope.Service lastResponse *kmsPluginHealthzResponse l *sync.Mutex } type kmsv2PluginProbe struct { - name string - ttl time.Duration - envelopekmsv2.Service + name string + ttl time.Duration + service envelopekmsv2.Service lastResponse *kmsPluginHealthzResponse l *sync.Mutex } func (h *kmsPluginProbe) toHealthzCheck(idx int) healthz.HealthChecker { return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error { - return h.Check() + return h.check() }) } func (p *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker { return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error { - return p.Check() + return p.check(r.Context()) }) } -// GetKMSPluginHealthzCheckers extracts KMSPluginProbes from the EncryptionConfig. func GetKMSPluginHealthzCheckers(filepath string, stopCh <-chan struct{}) ([]healthz.HealthChecker, error) { - f, err := os.Open(filepath) - if err != nil { - return nil, fmt.Errorf("error opening encryption provider configuration file %q: %v", filepath, err) - } - defer f.Close() - - var result []healthz.HealthChecker - probes, err := getKMSPluginProbes(f, stopCh) - if err != nil { - return nil, err - } - for i, p := range probes { - probe := p - switch t := probe.(type) { - case *kmsPluginProbe: - result = append(result, t.toHealthzCheck(i)) - case *kmsv2PluginProbe: - result = append(result, t.toHealthzCheck(i)) - default: - return nil, fmt.Errorf("unsupported KMS plugin type: %T", t) - } - } - - return result, nil + _, kmsHealthChecks, err := LoadEncryptionConfig(filepath, stopCh) + return kmsHealthChecks, err } -func getKMSPluginProbes(reader io.Reader, stopCh <-chan struct{}) ([]interface{}, error) { - // we ignore the cancel func because this context should only be canceled when stopCh is closed - ctx, _ := wait.ContextForChannel(stopCh) - - var result []interface{} - - configFileContents, err := io.ReadAll(reader) - if err != nil { - return nil, fmt.Errorf("could not read content of encryption provider configuration: %v", err) - } - - config, err := loadConfig(configFileContents) - if err != nil { - return nil, fmt.Errorf("error while parsing encryption provider configuration: %v", err) - } - - for _, r := range config.Resources { - for _, p := range r.Providers { - if p.KMS != nil { - switch p.KMS.APIVersion { - case kmsAPIVersionV1: - s, err := envelope.NewGRPCService(ctx, p.KMS.Endpoint, p.KMS.Timeout.Duration) - if err != nil { - return nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %v", p.KMS.Name, err) - } - - result = append(result, &kmsPluginProbe{ - name: p.KMS.Name, - ttl: kmsPluginHealthzNegativeTTL, - Service: s, - l: &sync.Mutex{}, - lastResponse: &kmsPluginHealthzResponse{}, - }) - - case kmsAPIVersionV2: - if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) { - return nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, KMSv2 feature is not enabled", p.KMS.Name) - } - - s, err := envelopekmsv2.NewGRPCService(ctx, p.KMS.Endpoint, p.KMS.Timeout.Duration) - if err != nil { - return nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %v", p.KMS.Name, err) - } - - result = append(result, &kmsv2PluginProbe{ - name: p.KMS.Name, - ttl: kmsPluginHealthzNegativeTTL, - Service: s, - l: &sync.Mutex{}, - lastResponse: &kmsPluginHealthzResponse{}, - }) - - default: - return nil, fmt.Errorf("could not configure KMS Plugin's probe %q, unsupported KMS API version %q", p.KMS.Name, p.KMS.APIVersion) - } - } - } - } - - return result, nil +func GetTransformerOverrides(filepath string, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, error) { + transformers, _, err := LoadEncryptionConfig(filepath, stopCh) + return transformers, err } -// Check encrypts and decrypts test data against KMS-Plugin's gRPC endpoint. -func (h *kmsPluginProbe) Check() error { +func LoadEncryptionConfig(filepath string, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, error) { + config, err := loadConfig(filepath) + if err != nil { + return nil, nil, fmt.Errorf("error while parsing file: %v", err) + } + + return getTransformerOverridesAndKMSPluginHealthzCheckers(config, stopCh) +} + +func getTransformerOverridesAndKMSPluginHealthzCheckers(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, error) { + var kmsHealthChecks []healthz.HealthChecker + transformers, probes, err := getTransformerOverridesAndKMSPluginProbes(config, stopCh) + if err != nil { + return nil, nil, err + } + for i := range probes { + probe := probes[i] + kmsHealthChecks = append(kmsHealthChecks, probe.toHealthzCheck(i)) + } + + return transformers, kmsHealthChecks, nil +} + +type healthChecker interface { + toHealthzCheck(idx int) healthz.HealthChecker +} + +func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthChecker, error) { + resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{} + var probes []healthChecker + + // For each entry in the configuration + for _, resourceConfig := range config.Resources { + resourceConfig := resourceConfig + + transformers, p, err := prefixTransformersAndProbes(resourceConfig, stopCh) + if err != nil { + return nil, nil, err + } + + // For each resource, create a list of providers to use + for _, resource := range resourceConfig.Resources { + resource := resource + gr := schema.ParseGroupResource(resource) + resourceToPrefixTransformer[gr] = append( + resourceToPrefixTransformer[gr], transformers...) + } + + probes = append(probes, p...) + } + + transformers := make(map[schema.GroupResource]value.Transformer, len(resourceToPrefixTransformer)) + for gr, transList := range resourceToPrefixTransformer { + gr := gr + transList := transList + transformers[gr] = value.NewMutableTransformer(value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...)) + } + + return transformers, probes, nil +} + +// check encrypts and decrypts test data against KMS-Plugin's gRPC endpoint. +func (h *kmsPluginProbe) check() error { h.l.Lock() defer h.l.Unlock() @@ -193,14 +174,14 @@ func (h *kmsPluginProbe) Check() error { return h.lastResponse.err } - p, err := h.Service.Encrypt([]byte("ping")) + p, err := h.service.Encrypt([]byte("ping")) if err != nil { h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()} h.ttl = kmsPluginHealthzNegativeTTL return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err) } - if _, err := h.Service.Decrypt(p); err != nil { + if _, err := h.service.Decrypt(p); err != nil { h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()} h.ttl = kmsPluginHealthzNegativeTTL return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err) @@ -211,8 +192,8 @@ func (h *kmsPluginProbe) Check() error { return nil } -// Check gets the healthz status of the KMSv2-Plugin using the Status() method. -func (h *kmsv2PluginProbe) Check() error { +// check gets the healthz status of the KMSv2-Plugin using the Status() method. +func (h *kmsv2PluginProbe) check(ctx context.Context) error { h.l.Lock() defer h.l.Unlock() @@ -220,8 +201,7 @@ func (h *kmsv2PluginProbe) Check() error { return h.lastResponse.err } - ctx := context.Background() - p, err := h.Service.Status(ctx) + p, err := h.service.Status(ctx) if err != nil { h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()} h.ttl = kmsPluginHealthzNegativeTTL @@ -258,58 +238,21 @@ func isKMSv2ProviderHealthy(name string, response *envelopekmsv2.StatusResponse) return nil } -// GetTransformerOverrides returns the transformer overrides by reading and parsing the encryption provider configuration file -func GetTransformerOverrides(filepath string, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, error) { +func loadConfig(filepath string) (*apiserverconfig.EncryptionConfiguration, error) { f, err := os.Open(filepath) if err != nil { return nil, fmt.Errorf("error opening encryption provider configuration file %q: %v", filepath, err) } defer f.Close() - result, err := parseEncryptionConfiguration(f, stopCh) - if err != nil { - return nil, fmt.Errorf("error while parsing encryption provider configuration file %q: %v", filepath, err) - } - return result, nil -} - -func parseEncryptionConfiguration(f io.Reader, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, error) { - configFileContents, err := io.ReadAll(f) + data, err := io.ReadAll(f) if err != nil { return nil, fmt.Errorf("could not read contents: %v", err) } - - config, err := loadConfig(configFileContents) - if err != nil { - return nil, fmt.Errorf("error while parsing file: %v", err) + if len(data) == 0 { + return nil, fmt.Errorf("encryption provider configuration file %q is empty", filepath) } - resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{} - - // For each entry in the configuration - for _, resourceConfig := range config.Resources { - transformers, err := prefixTransformers(&resourceConfig, stopCh) - if err != nil { - return nil, err - } - - // For each resource, create a list of providers to use - for _, resource := range resourceConfig.Resources { - gr := schema.ParseGroupResource(resource) - resourceToPrefixTransformer[gr] = append( - resourceToPrefixTransformer[gr], transformers...) - } - } - - result := map[schema.GroupResource]value.Transformer{} - for gr, transList := range resourceToPrefixTransformer { - result[gr] = value.NewMutableTransformer(value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...)) - } - return result, nil - -} - -func loadConfig(data []byte) (*apiserverconfig.EncryptionConfiguration, error) { scheme := runtime.NewScheme() codecs := serializer.NewCodecFactory(scheme) utilruntime.Must(apiserverconfig.AddToScheme(scheme)) @@ -327,68 +270,52 @@ func loadConfig(data []byte) (*apiserverconfig.EncryptionConfiguration, error) { return config, validation.ValidateEncryptionConfiguration(config).ToAggregate() } -var ( - // The factory to create kms service. This is to make writing test easier. - envelopeServiceFactory = envelope.NewGRPCService +func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, stopCh <-chan struct{}) ([]value.PrefixTransformer, []healthChecker, error) { + var transformers []value.PrefixTransformer + var probes []healthChecker - // The factory to create kmsv2 service. - envelopeKMSv2ServiceFactory = envelopekmsv2.NewGRPCService -) - -func prefixTransformers(config *apiserverconfig.ResourceConfiguration, stopCh <-chan struct{}) ([]value.PrefixTransformer, error) { - // we ignore the cancel func because this context should only be canceled when stopCh is closed - ctx, _ := wait.ContextForChannel(stopCh) - - var result []value.PrefixTransformer for _, provider := range config.Providers { + provider := provider var ( - transformer value.PrefixTransformer - err error + transformer value.PrefixTransformer + transformerErr error + probe healthChecker ) switch { case provider.AESGCM != nil: - transformer, err = aesPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1) - case provider.AESCBC != nil: - transformer, err = aesPrefixTransformer(provider.AESCBC, aestransformer.NewCBCTransformer, aesCBCTransformerPrefixV1) - case provider.Secretbox != nil: - transformer, err = secretboxPrefixTransformer(provider.Secretbox) - case provider.KMS != nil: - switch provider.KMS.APIVersion { - case kmsAPIVersionV1: - var envelopeService envelope.Service - if envelopeService, err = envelopeServiceFactory(ctx, provider.KMS.Endpoint, provider.KMS.Timeout.Duration); err != nil { - return nil, fmt.Errorf("could not configure KMS plugin %q, error: %v", provider.KMS.Name, err) - } - transformer, err = envelopePrefixTransformer(provider.KMS, envelopeService, kmsTransformerPrefixV1) - case kmsAPIVersionV2: - if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) { - return nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", provider.KMS.Name) - } + transformer, transformerErr = aesPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1) - var envelopeService envelopekmsv2.Service - if envelopeService, err = envelopeKMSv2ServiceFactory(ctx, provider.KMS.Endpoint, provider.KMS.Timeout.Duration); err != nil { - return nil, fmt.Errorf("could not configure KMSv2 plugin %q, error: %v", provider.KMS.Name, err) - } - transformer, err = envelopekmsv2PrefixTransformer(provider.KMS, envelopeService, kmsTransformerPrefixV2) - default: - return nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", provider.KMS.Name, provider.KMS.APIVersion) + case provider.AESCBC != nil: + transformer, transformerErr = aesPrefixTransformer(provider.AESCBC, aestransformer.NewCBCTransformer, aesCBCTransformerPrefixV1) + + case provider.Secretbox != nil: + transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox) + + case provider.KMS != nil: + transformer, probe, transformerErr = kmsPrefixTransformer(provider.KMS, stopCh) + if transformerErr == nil { + probes = append(probes, probe) } + case provider.Identity != nil: transformer = value.PrefixTransformer{ Transformer: identity.NewEncryptCheckTransformer(), Prefix: []byte{}, } + default: - return nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity") + return nil, nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity") } - if err != nil { - return result, err + if transformerErr != nil { + return nil, nil, transformerErr } - result = append(result, transformer) + + transformers = append(transformers, transformer) } - return result, nil + + return transformers, probes, nil } type blockTransformerFunc func(cipher.Block) value.Transformer @@ -400,6 +327,7 @@ func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTran return result, fmt.Errorf("aes provider has no valid keys") } for _, key := range config.Keys { + key := key if key.Name == "" { return result, fmt.Errorf("key with invalid name provided") } @@ -411,6 +339,7 @@ func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTran keyTransformers := []value.PrefixTransformer{} for _, keyData := range config.Keys { + keyData := keyData key, err := base64.StdEncoding.DecodeString(keyData.Secret) if err != nil { return result, fmt.Errorf("could not obtain secret for named key %s: %s", keyData.Name, err) @@ -447,6 +376,7 @@ func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) return result, fmt.Errorf("secretbox provider has no valid keys") } for _, key := range config.Keys { + key := key if key.Name == "" { return result, fmt.Errorf("key with invalid name provided") } @@ -458,6 +388,7 @@ func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) keyTransformers := []value.PrefixTransformer{} for _, keyData := range config.Keys { + keyData := keyData key, err := base64.StdEncoding.DecodeString(keyData.Secret) if err != nil { return result, fmt.Errorf("could not obtain secret for named key %s: %s", keyData.Name, err) @@ -490,7 +421,70 @@ func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) return result, nil } -func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) (value.PrefixTransformer, error) { +var ( + // The factory to create kms service. This is to make writing test easier. + envelopeServiceFactory = envelope.NewGRPCService + + // The factory to create kmsv2 service. + envelopeKMSv2ServiceFactory = envelopekmsv2.NewGRPCService +) + +func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-chan struct{}) (value.PrefixTransformer, healthChecker, error) { + // we ignore the cancel func because this context should only be canceled when stopCh is closed + ctx, _ := wait.ContextForChannel(stopCh) + + kmsName := config.Name + switch config.APIVersion { + case kmsAPIVersionV1: + envelopeService, err := envelopeServiceFactory(ctx, config.Endpoint, config.Timeout.Duration) + if err != nil { + return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %v", kmsName, err) + } + + probe := &kmsPluginProbe{ + name: kmsName, + ttl: kmsPluginHealthzNegativeTTL, + service: envelopeService, + l: &sync.Mutex{}, + lastResponse: &kmsPluginHealthzResponse{}, + } + + transformer := envelopePrefixTransformer(config, envelopeService, kmsTransformerPrefixV1) + + return transformer, probe, nil + + case kmsAPIVersionV2: + if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) { + return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName) + } + + envelopeService, err := envelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Timeout.Duration) + if err != nil { + return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %v", kmsName, err) + } + + probe := &kmsv2PluginProbe{ + name: kmsName, + ttl: kmsPluginHealthzNegativeTTL, + service: envelopeService, + l: &sync.Mutex{}, + lastResponse: &kmsPluginHealthzResponse{}, + } + + // using AES-GCM by default for encrypting data with KMSv2 + transformer := value.PrefixTransformer{ + Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), aestransformer.NewGCMTransformer), + Prefix: []byte(kmsTransformerPrefixV2 + kmsName + ":"), + } + + return transformer, probe, nil + + default: + return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion) + } +} + +func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) value.PrefixTransformer { baseTransformerFunc := func(block cipher.Block) value.Transformer { // v1.24: write using AES-CBC only but support reads via AES-CBC and AES-GCM (so we can move to AES-GCM) // v1.25: write using AES-GCM only but support reads via AES-GCM and fallback to AES-CBC for backwards compatibility @@ -499,33 +493,18 @@ func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelop return unionTransformers{aestransformer.NewGCMTransformer(block), aestransformer.NewCBCTransformer(block)} } - envelopeTransformer, err := envelope.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), baseTransformerFunc) - if err != nil { - return value.PrefixTransformer{}, err - } return value.PrefixTransformer{ - Transformer: envelopeTransformer, + Transformer: envelope.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), baseTransformerFunc), Prefix: []byte(prefix + config.Name + ":"), - }, nil -} - -func envelopekmsv2PrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelopekmsv2.Service, prefix string) (value.PrefixTransformer, error) { - // using AES-GCM by default for encrypting data with KMSv2 - envelopeTransformer, err := envelopekmsv2.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), aestransformer.NewGCMTransformer) - if err != nil { - return value.PrefixTransformer{}, err } - return value.PrefixTransformer{ - Transformer: envelopeTransformer, - Prefix: []byte(prefix + config.Name + ":"), - }, nil } type unionTransformers []value.Transformer func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, stale bool, err error) { var errs []error - for i, transformer := range u { + for i := range u { + transformer := u[i] result, stale, err := transformer.TransformFromStorage(ctx, data, dataCtx) if err != nil { errs = append(errs, err) diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go index aba6f65d2d7..edff1c2896a 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go @@ -21,8 +21,6 @@ import ( "context" "encoding/base64" "errors" - "io" - "os" "sync" "testing" "time" @@ -44,26 +42,6 @@ const ( sampleContextText = "0123456789" ) -func mustReadConfig(t *testing.T, path string) []byte { - t.Helper() - f, err := os.Open(path) - if err != nil { - t.Fatalf("error opening encryption configuration file %q: %v", path, err) - } - defer f.Close() - - configFileContents, err := io.ReadAll(f) - if err != nil { - t.Fatalf("could not read contents of encryption config: %v", err) - } - - return configFileContents -} - -func mustConfigReader(t *testing.T, path string) io.Reader { - return bytes.NewReader(mustReadConfig(t, path)) -} - // testEnvelopeService is a mock envelope service which can be used to simulate remote Envelope services // for testing of the envelope transformer with other transformers. type testEnvelopeService struct { @@ -136,7 +114,7 @@ func newMockErrorEnvelopeKMSv2Service(endpoint string, timeout time.Duration) (e func TestLegacyConfig(t *testing.T) { legacyV1Config := "testdata/valid-configs/legacy.yaml" - legacyConfigObject, err := loadConfig(mustReadConfig(t, legacyV1Config)) + legacyConfigObject, err := loadConfig(legacyV1Config) cacheSize := int32(10) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, legacyV1Config) @@ -199,37 +177,37 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) { // Transforms data using one of them, and tries to untransform using the others. // Repeats this for all possible combinations. correctConfigWithIdentityFirst := "testdata/valid-configs/identity-first.yaml" - identityFirstTransformerOverrides, err := parseEncryptionConfiguration(mustConfigReader(t, correctConfigWithIdentityFirst), ctx.Done()) + identityFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithIdentityFirst, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithIdentityFirst) } correctConfigWithAesGcmFirst := "testdata/valid-configs/aes-gcm-first.yaml" - aesGcmFirstTransformerOverrides, err := parseEncryptionConfiguration(mustConfigReader(t, correctConfigWithAesGcmFirst), ctx.Done()) + aesGcmFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesGcmFirst, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesGcmFirst) } correctConfigWithAesCbcFirst := "testdata/valid-configs/aes-cbc-first.yaml" - aesCbcFirstTransformerOverrides, err := parseEncryptionConfiguration(mustConfigReader(t, correctConfigWithAesCbcFirst), ctx.Done()) + aesCbcFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesCbcFirst, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesCbcFirst) } correctConfigWithSecretboxFirst := "testdata/valid-configs/secret-box-first.yaml" - secretboxFirstTransformerOverrides, err := parseEncryptionConfiguration(mustConfigReader(t, correctConfigWithSecretboxFirst), ctx.Done()) + secretboxFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithSecretboxFirst, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithSecretboxFirst) } correctConfigWithKMSFirst := "testdata/valid-configs/kms-first.yaml" - kmsFirstTransformerOverrides, err := parseEncryptionConfiguration(mustConfigReader(t, correctConfigWithKMSFirst), ctx.Done()) + kmsFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSFirst, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSFirst) } correctConfigWithKMSv2First := "testdata/valid-configs/kmsv2-first.yaml" - kmsv2FirstTransformerOverrides, err := parseEncryptionConfiguration(mustConfigReader(t, correctConfigWithKMSv2First), ctx.Done()) + kmsv2FirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSv2First, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSv2First) } @@ -281,44 +259,33 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) { func TestKMSPluginHealthz(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() - ctx := testContext(t) - - service, err := envelope.NewGRPCService(ctx, "unix:///tmp/testprovider.sock", 3*time.Second) - if err != nil { - t.Fatalf("Could not initialize envelopeService, error: %v", err) - } - serviceKMSv2, err := envelopekmsv2.NewGRPCService(ctx, "unix:///tmp/testprovider.sock", 3*time.Second) - if err != nil { - t.Fatalf("Could not initialize kmsv2 envelopeService, error: %v", err) - } - testCases := []struct { desc string config string - want []interface{} - wantErr bool + want []healthChecker + wantErr string }{ { desc: "Install Healthz", config: "testdata/valid-configs/kms/default-timeout.yaml", - want: []interface{}{ + want: []healthChecker{ &kmsPluginProbe{ - name: "foo", - Service: service, + name: "foo", + ttl: 3 * time.Second, }, }, }, { desc: "Install multiple healthz", config: "testdata/valid-configs/kms/multiple-providers.yaml", - want: []interface{}{ + want: []healthChecker{ &kmsPluginProbe{ - name: "foo", - Service: service, + name: "foo", + ttl: 3 * time.Second, }, &kmsPluginProbe{ - name: "bar", - Service: service, + name: "bar", + ttl: 3 * time.Second, }, }, }, @@ -329,14 +296,14 @@ func TestKMSPluginHealthz(t *testing.T) { { desc: "Install multiple healthz with v1 and v2", config: "testdata/valid-configs/kms/multiple-providers-kmsv2.yaml", - want: []interface{}{ + want: []healthChecker{ &kmsv2PluginProbe{ - name: "foo", - Service: serviceKMSv2, + name: "foo", + ttl: 3 * time.Second, }, &kmsPluginProbe{ - name: "bar", - Service: service, + name: "bar", + ttl: 3 * time.Second, }, }, }, @@ -344,18 +311,50 @@ func TestKMSPluginHealthz(t *testing.T) { desc: "Invalid API version", config: "testdata/invalid-configs/kms/invalid-apiversion.yaml", want: nil, - wantErr: true, + wantErr: `resources[0].providers[0].kms.apiVersion: Invalid value: "v3": unsupported apiVersion apiVersion for KMS provider, only v1 and v2 are supported`, }, } for _, tt := range testCases { t.Run(tt.desc, func(t *testing.T) { - got, err := getKMSPluginProbes(mustConfigReader(t, tt.config), ctx.Done()) - if err != nil && !tt.wantErr { - t.Fatalf("got %v, want nil for error", err) + config, err := loadConfig(tt.config) + if errStr := errString(err); errStr != tt.wantErr { + t.Fatalf("unexpected error state got=%s want=%s", errStr, tt.wantErr) + } + if len(tt.wantErr) > 0 { + return } - if d := cmp.Diff(tt.want, got, cmp.Comparer(serviceComparer), cmp.Comparer(serviceKMSv2Comparer)); d != "" { + _, got, err := getTransformerOverridesAndKMSPluginProbes(config, testContext(t).Done()) + if err != nil { + t.Fatal(err) + } + + // unset fields that are not relevant to the test + for i := range got { + checker := got[i] + switch p := checker.(type) { + case *kmsPluginProbe: + p.service = nil + p.l = nil + p.lastResponse = nil + case *kmsv2PluginProbe: + p.service = nil + p.l = nil + p.lastResponse = nil + default: + t.Fatalf("unexpected probe type %T", p) + } + } + + if d := cmp.Diff(tt.want, got, + cmp.Comparer(func(a, b *kmsPluginProbe) bool { + return *a == *b + }), + cmp.Comparer(func(a, b *kmsv2PluginProbe) bool { + return *a == *b + }), + ); d != "" { t.Fatalf("HealthzConfig mismatch (-want +got):\n%s", d) } }) @@ -378,7 +377,7 @@ func TestKMSPluginHealthzTTL(t *testing.T) { probe: &kmsPluginProbe{ name: "test", ttl: kmsPluginHealthzNegativeTTL, - Service: service, + service: service, l: &sync.Mutex{}, lastResponse: &kmsPluginHealthzResponse{}, }, @@ -389,7 +388,7 @@ func TestKMSPluginHealthzTTL(t *testing.T) { probe: &kmsPluginProbe{ name: "test", ttl: kmsPluginHealthzPositiveTTL, - Service: errService, + service: errService, l: &sync.Mutex{}, lastResponse: &kmsPluginHealthzResponse{}, }, @@ -399,7 +398,7 @@ func TestKMSPluginHealthzTTL(t *testing.T) { for _, tt := range testCases { t.Run(tt.desc, func(t *testing.T) { - tt.probe.Check() + _ = tt.probe.check() if tt.probe.ttl != tt.wantTTL { t.Fatalf("want ttl %v, got ttl %v", tt.wantTTL, tt.probe.ttl) } @@ -423,7 +422,7 @@ func TestKMSv2PluginHealthzTTL(t *testing.T) { probe: &kmsv2PluginProbe{ name: "test", ttl: kmsPluginHealthzNegativeTTL, - Service: service, + service: service, l: &sync.Mutex{}, lastResponse: &kmsPluginHealthzResponse{}, }, @@ -434,7 +433,7 @@ func TestKMSv2PluginHealthzTTL(t *testing.T) { probe: &kmsv2PluginProbe{ name: "test", ttl: kmsPluginHealthzPositiveTTL, - Service: errService, + service: errService, l: &sync.Mutex{}, lastResponse: &kmsPluginHealthzResponse{}, }, @@ -444,7 +443,7 @@ func TestKMSv2PluginHealthzTTL(t *testing.T) { for _, tt := range testCases { t.Run(tt.desc, func(t *testing.T) { - tt.probe.Check() + _ = tt.probe.check(ctx) if tt.probe.ttl != tt.wantTTL { t.Fatalf("want ttl %v, got ttl %v", tt.wantTTL, tt.probe.ttl) } @@ -452,16 +451,6 @@ func TestKMSv2PluginHealthzTTL(t *testing.T) { } } -// As long as got and want contain envelope.Service we will return true. -// If got has an envelope.Service and want does note (or vice versa) this will return false. -func serviceComparer(_, _ envelope.Service) bool { - return true -} - -func serviceKMSv2Comparer(_, _ envelopekmsv2.Service) bool { - return true -} - func TestCBCKeyRotationWithOverlappingProviders(t *testing.T) { testCBCKeyRotationWithProviders( t, @@ -539,7 +528,7 @@ func getTransformerFromEncryptionConfig(t *testing.T, encryptionConfigPath strin ctx := testContext(t) t.Helper() - transformers, err := parseEncryptionConfiguration(mustConfigReader(t, encryptionConfigPath), ctx.Done()) + transformers, _, err := LoadEncryptionConfig(encryptionConfigPath, ctx.Done()) if err != nil { t.Fatal(err) } @@ -592,3 +581,11 @@ func testContext(t *testing.T) context.Context { t.Cleanup(cancel) return ctx } + +func errString(err error) string { + if err == nil { + return "" + } + + return err.Error() +} diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope.go index 30795d41a87..43d2e00a22f 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope.go @@ -63,7 +63,7 @@ type envelopeTransformer struct { // It uses envelopeService to encrypt and decrypt DEKs. Respective DEKs (in encrypted form) are prepended to // the data items they encrypt. A cache (of size cacheSize) is maintained to store the most recently // used decrypted DEKs in memory. -func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransformerFunc func(cipher.Block) value.Transformer) (value.Transformer, error) { +func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransformerFunc func(cipher.Block) value.Transformer) value.Transformer { var ( cache *lru.Cache ) @@ -77,7 +77,7 @@ func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransfor baseTransformerFunc: baseTransformerFunc, cacheEnabled: cacheSize > 0, cacheSize: cacheSize, - }, nil + } } // TransformFromStorage decrypts data encrypted by this transformer using envelope encryption. diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope_test.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope_test.go index 4f030772eb9..e0c8b309eba 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope_test.go @@ -98,10 +98,7 @@ func TestEnvelopeCaching(t *testing.T) { for _, tt := range testCases { t.Run(tt.desc, func(t *testing.T) { envelopeService := newTestEnvelopeService() - envelopeTransformer, err := NewEnvelopeTransformer(envelopeService, tt.cacheSize, aestransformer.NewCBCTransformer) - if err != nil { - t.Fatalf("failed to initialize envelope transformer: %v", err) - } + envelopeTransformer := NewEnvelopeTransformer(envelopeService, tt.cacheSize, aestransformer.NewCBCTransformer) ctx := context.Background() dataCtx := value.DefaultContext([]byte(testContextText)) originalText := []byte(testText) @@ -133,10 +130,7 @@ func TestEnvelopeCaching(t *testing.T) { // Makes Envelope transformer hit cache limit, throws error if it misbehaves. func TestEnvelopeCacheLimit(t *testing.T) { - envelopeTransformer, err := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewCBCTransformer) - if err != nil { - t.Fatalf("failed to initialize envelope transformer: %v", err) - } + envelopeTransformer := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewCBCTransformer) ctx := context.Background() dataCtx := value.DefaultContext([]byte(testContextText)) @@ -169,10 +163,7 @@ func TestEnvelopeCacheLimit(t *testing.T) { } func BenchmarkEnvelopeCBCRead(b *testing.B) { - envelopeTransformer, err := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewCBCTransformer) - if err != nil { - b.Fatalf("failed to initialize envelope transformer: %v", err) - } + envelopeTransformer := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewCBCTransformer) benchmarkRead(b, envelopeTransformer, 1024) } @@ -187,10 +178,7 @@ func BenchmarkAESCBCRead(b *testing.B) { } func BenchmarkEnvelopeGCMRead(b *testing.B) { - envelopeTransformer, err := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewGCMTransformer) - if err != nil { - b.Fatalf("failed to initialize envelope transformer: %v", err) - } + envelopeTransformer := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewGCMTransformer) benchmarkRead(b, envelopeTransformer, 1024) } @@ -230,10 +218,7 @@ func benchmarkRead(b *testing.B, transformer value.Transformer, valueLength int) // remove after 1.13 func TestBackwardsCompatibility(t *testing.T) { envelopeService := newTestEnvelopeService() - envelopeTransformerInst, err := NewEnvelopeTransformer(envelopeService, testEnvelopeCacheSize, aestransformer.NewCBCTransformer) - if err != nil { - t.Fatalf("failed to initialize envelope transformer: %v", err) - } + envelopeTransformerInst := NewEnvelopeTransformer(envelopeService, testEnvelopeCacheSize, aestransformer.NewCBCTransformer) ctx := context.Background() dataCtx := value.DefaultContext([]byte(testContextText)) originalText := []byte(testText) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go index 775f14c44cc..7daa47d2d86 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go @@ -87,7 +87,7 @@ type StatusResponse struct { // It uses envelopeService to encrypt and decrypt DEKs. Respective DEKs (in encrypted form) are prepended to // the data items they encrypt. A cache (of size cacheSize) is maintained to store the most recently // used decrypted DEKs in memory. -func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransformerFunc func(cipher.Block) value.Transformer) (value.Transformer, error) { +func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransformerFunc func(cipher.Block) value.Transformer) value.Transformer { var cache *lru.Cache if cacheSize > 0 { @@ -102,7 +102,7 @@ func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransfor baseTransformerFunc: baseTransformerFunc, cacheEnabled: cacheSize > 0, cacheSize: cacheSize, - }, nil + } } // TransformFromStorage decrypts data encrypted by this transformer using envelope encryption. diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope_test.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope_test.go index 2229444d9d0..b3f531ca1f4 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope_test.go @@ -110,10 +110,7 @@ func TestEnvelopeCaching(t *testing.T) { for _, tt := range testCases { t.Run(tt.desc, func(t *testing.T) { envelopeService := newTestEnvelopeService() - envelopeTransformer, err := NewEnvelopeTransformer(envelopeService, tt.cacheSize, aestransformer.NewGCMTransformer) - if err != nil { - t.Fatalf("failed to initialize envelope transformer: %v", err) - } + envelopeTransformer := NewEnvelopeTransformer(envelopeService, tt.cacheSize, aestransformer.NewGCMTransformer) ctx := context.Background() dataCtx := value.DefaultContext([]byte(testContextText)) originalText := []byte(testText) @@ -145,10 +142,7 @@ func TestEnvelopeCaching(t *testing.T) { // Makes Envelope transformer hit cache limit, throws error if it misbehaves. func TestEnvelopeCacheLimit(t *testing.T) { - envelopeTransformer, err := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewGCMTransformer) - if err != nil { - t.Fatalf("failed to initialize envelope transformer: %v", err) - } + envelopeTransformer := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewGCMTransformer) ctx := context.Background() dataCtx := value.DefaultContext([]byte(testContextText)) diff --git a/test/integration/controlplane/transformation/kmsv2_transformation_test.go b/test/integration/controlplane/transformation/kmsv2_transformation_test.go index fe891a9b1ad..1c4cb7a3418 100644 --- a/test/integration/controlplane/transformation/kmsv2_transformation_test.go +++ b/test/integration/controlplane/transformation/kmsv2_transformation_test.go @@ -98,7 +98,7 @@ func (r envelopekmsv2) plainTextPayload(secretETCDPath string) ([]byte, error) { // TestKMSv2Provider is an integration test between KubeAPI, ETCD and KMSv2 Plugin // Concretely, this test verifies the following integration contracts: -// 1. Raw records in ETCD that were processed by KMSv2 Provider should be prefixed with []byte{'e', 'k', '8', 's', 0} +// 1. Raw records in ETCD that were processed by KMSv2 Provider should be prefixed with k8s:enc:kms:v2:: // 2. Data Encryption Key (DEK) should be generated by envelopeTransformer and passed to KMS gRPC Plugin // 3. KMS gRPC Plugin should encrypt the DEK with a Key Encryption Key (KEK) and pass it back to envelopeTransformer // 4. The cipherTextPayload (ex. Secret) should be encrypted via AES GCM transform