Merge pull request #112703 from enj/enj/r/kms_cleanup

encryption config: no-op refactor to prepare for single loading

staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go

@@ -67,125 +67,106 @@ type kmsPluginHealthzResponse struct {
 }
 
 type kmsPluginProbe struct {
-	name string
+	name         string
-	ttl  time.Duration
+	ttl          time.Duration
-	envelope.Service
+	service      envelope.Service
 	lastResponse *kmsPluginHealthzResponse
 	l            *sync.Mutex
 }
 
 type kmsv2PluginProbe struct {
-	name string
+	name         string
-	ttl  time.Duration
+	ttl          time.Duration
-	envelopekmsv2.Service
+	service      envelopekmsv2.Service
 	lastResponse *kmsPluginHealthzResponse
 	l            *sync.Mutex
 }
 
 func (h *kmsPluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
 	return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
-		return h.Check()
+		return h.check()
 	})
 }
 
 func (p *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
 	return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
-		return p.Check()
+		return p.check(r.Context())
 	})
 }
 
-// GetKMSPluginHealthzCheckers extracts KMSPluginProbes from the EncryptionConfig.
 func GetKMSPluginHealthzCheckers(filepath string, stopCh <-chan struct{}) ([]healthz.HealthChecker, error) {
-	f, err := os.Open(filepath)
+	_, kmsHealthChecks, err := LoadEncryptionConfig(filepath, stopCh)
-	if err != nil {
+	return kmsHealthChecks, err
-		return nil, fmt.Errorf("error opening encryption provider configuration file %q: %v", filepath, err)
-	}
-	defer f.Close()
-
-	var result []healthz.HealthChecker
-	probes, err := getKMSPluginProbes(f, stopCh)
-	if err != nil {
-		return nil, err
-	}
-	for i, p := range probes {
-		probe := p
-		switch t := probe.(type) {
-		case *kmsPluginProbe:
-			result = append(result, t.toHealthzCheck(i))
-		case *kmsv2PluginProbe:
-			result = append(result, t.toHealthzCheck(i))
-		default:
-			return nil, fmt.Errorf("unsupported KMS plugin type: %T", t)
-		}
-	}
-
-	return result, nil
 }
 
-func getKMSPluginProbes(reader io.Reader, stopCh <-chan struct{}) ([]interface{}, error) {
+func GetTransformerOverrides(filepath string, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, error) {
-	// we ignore the cancel func because this context should only be canceled when stopCh is closed
+	transformers, _, err := LoadEncryptionConfig(filepath, stopCh)
-	ctx, _ := wait.ContextForChannel(stopCh)
+	return transformers, err
-
-	var result []interface{}
-
-	configFileContents, err := io.ReadAll(reader)
-	if err != nil {
-		return nil, fmt.Errorf("could not read content of encryption provider configuration: %v", err)
-	}
-
-	config, err := loadConfig(configFileContents)
-	if err != nil {
-		return nil, fmt.Errorf("error while parsing encryption provider configuration: %v", err)
-	}
-
-	for _, r := range config.Resources {
-		for _, p := range r.Providers {
-			if p.KMS != nil {
-				switch p.KMS.APIVersion {
-				case kmsAPIVersionV1:
-					s, err := envelope.NewGRPCService(ctx, p.KMS.Endpoint, p.KMS.Timeout.Duration)
-					if err != nil {
-						return nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %v", p.KMS.Name, err)
-					}
-
-					result = append(result, &kmsPluginProbe{
-						name:         p.KMS.Name,
-						ttl:          kmsPluginHealthzNegativeTTL,
-						Service:      s,
-						l:            &sync.Mutex{},
-						lastResponse: &kmsPluginHealthzResponse{},
-					})
-
-				case kmsAPIVersionV2:
-					if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) {
-						return nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, KMSv2 feature is not enabled", p.KMS.Name)
-					}
-
-					s, err := envelopekmsv2.NewGRPCService(ctx, p.KMS.Endpoint, p.KMS.Timeout.Duration)
-					if err != nil {
-						return nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %v", p.KMS.Name, err)
-					}
-
-					result = append(result, &kmsv2PluginProbe{
-						name:         p.KMS.Name,
-						ttl:          kmsPluginHealthzNegativeTTL,
-						Service:      s,
-						l:            &sync.Mutex{},
-						lastResponse: &kmsPluginHealthzResponse{},
-					})
-
-				default:
-					return nil, fmt.Errorf("could not configure KMS Plugin's probe %q, unsupported KMS API version %q", p.KMS.Name, p.KMS.APIVersion)
-				}
-			}
-		}
-	}
-
-	return result, nil
 }
 
-// Check encrypts and decrypts test data against KMS-Plugin's gRPC endpoint.
+func LoadEncryptionConfig(filepath string, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, error) {
-func (h *kmsPluginProbe) Check() error {
+	config, err := loadConfig(filepath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error while parsing file: %v", err)
+	}
+
+	return getTransformerOverridesAndKMSPluginHealthzCheckers(config, stopCh)
+}
+
+func getTransformerOverridesAndKMSPluginHealthzCheckers(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, error) {
+	var kmsHealthChecks []healthz.HealthChecker
+	transformers, probes, err := getTransformerOverridesAndKMSPluginProbes(config, stopCh)
+	if err != nil {
+		return nil, nil, err
+	}
+	for i := range probes {
+		probe := probes[i]
+		kmsHealthChecks = append(kmsHealthChecks, probe.toHealthzCheck(i))
+	}
+
+	return transformers, kmsHealthChecks, nil
+}
+
+type healthChecker interface {
+	toHealthzCheck(idx int) healthz.HealthChecker
+}
+
+func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthChecker, error) {
+	resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{}
+	var probes []healthChecker
+
+	// For each entry in the configuration
+	for _, resourceConfig := range config.Resources {
+		resourceConfig := resourceConfig
+
+		transformers, p, err := prefixTransformersAndProbes(resourceConfig, stopCh)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		// For each resource, create a list of providers to use
+		for _, resource := range resourceConfig.Resources {
+			resource := resource
+			gr := schema.ParseGroupResource(resource)
+			resourceToPrefixTransformer[gr] = append(
+				resourceToPrefixTransformer[gr], transformers...)
+		}
+
+		probes = append(probes, p...)
+	}
+
+	transformers := make(map[schema.GroupResource]value.Transformer, len(resourceToPrefixTransformer))
+	for gr, transList := range resourceToPrefixTransformer {
+		gr := gr
+		transList := transList
+		transformers[gr] = value.NewMutableTransformer(value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...))
+	}
+
+	return transformers, probes, nil
+}
+
+// check encrypts and decrypts test data against KMS-Plugin's gRPC endpoint.
+func (h *kmsPluginProbe) check() error {
 	h.l.Lock()
 	defer h.l.Unlock()
 
@@ -193,14 +174,14 @@ func (h *kmsPluginProbe) Check() error {
 		return h.lastResponse.err
 	}
 
-	p, err := h.Service.Encrypt([]byte("ping"))
+	p, err := h.service.Encrypt([]byte("ping"))
 	if err != nil {
 		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
 		h.ttl = kmsPluginHealthzNegativeTTL
 		return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err)
 	}
 
-	if _, err := h.Service.Decrypt(p); err != nil {
+	if _, err := h.service.Decrypt(p); err != nil {
 		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
 		h.ttl = kmsPluginHealthzNegativeTTL
 		return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err)
@@ -211,8 +192,8 @@ func (h *kmsPluginProbe) Check() error {
 	return nil
 }
 
-// Check gets the healthz status of the KMSv2-Plugin using the Status() method.
+// check gets the healthz status of the KMSv2-Plugin using the Status() method.
-func (h *kmsv2PluginProbe) Check() error {
+func (h *kmsv2PluginProbe) check(ctx context.Context) error {
 	h.l.Lock()
 	defer h.l.Unlock()
 
@@ -220,8 +201,7 @@ func (h *kmsv2PluginProbe) Check() error {
 		return h.lastResponse.err
 	}
 
-	ctx := context.Background()
+	p, err := h.service.Status(ctx)
-	p, err := h.Service.Status(ctx)
 	if err != nil {
 		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
 		h.ttl = kmsPluginHealthzNegativeTTL
@@ -258,58 +238,21 @@ func isKMSv2ProviderHealthy(name string, response *envelopekmsv2.StatusResponse)
 	return nil
 }
 
-// GetTransformerOverrides returns the transformer overrides by reading and parsing the encryption provider configuration file
+func loadConfig(filepath string) (*apiserverconfig.EncryptionConfiguration, error) {
-func GetTransformerOverrides(filepath string, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, error) {
 	f, err := os.Open(filepath)
 	if err != nil {
 		return nil, fmt.Errorf("error opening encryption provider configuration file %q: %v", filepath, err)
 	}
 	defer f.Close()
 
-	result, err := parseEncryptionConfiguration(f, stopCh)
+	data, err := io.ReadAll(f)
-	if err != nil {
-		return nil, fmt.Errorf("error while parsing encryption provider configuration file %q: %v", filepath, err)
-	}
-	return result, nil
-}
-
-func parseEncryptionConfiguration(f io.Reader, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, error) {
-	configFileContents, err := io.ReadAll(f)
 	if err != nil {
 		return nil, fmt.Errorf("could not read contents: %v", err)
 	}
-
+	if len(data) == 0 {
-	config, err := loadConfig(configFileContents)
+		return nil, fmt.Errorf("encryption provider configuration file %q is empty", filepath)
-	if err != nil {
-		return nil, fmt.Errorf("error while parsing file: %v", err)
 	}
 
-	resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{}
-
-	// For each entry in the configuration
-	for _, resourceConfig := range config.Resources {
-		transformers, err := prefixTransformers(&resourceConfig, stopCh)
-		if err != nil {
-			return nil, err
-		}
-
-		// For each resource, create a list of providers to use
-		for _, resource := range resourceConfig.Resources {
-			gr := schema.ParseGroupResource(resource)
-			resourceToPrefixTransformer[gr] = append(
-				resourceToPrefixTransformer[gr], transformers...)
-		}
-	}
-
-	result := map[schema.GroupResource]value.Transformer{}
-	for gr, transList := range resourceToPrefixTransformer {
-		result[gr] = value.NewMutableTransformer(value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...))
-	}
-	return result, nil
-
-}
-
-func loadConfig(data []byte) (*apiserverconfig.EncryptionConfiguration, error) {
 	scheme := runtime.NewScheme()
 	codecs := serializer.NewCodecFactory(scheme)
 	utilruntime.Must(apiserverconfig.AddToScheme(scheme))
@@ -327,68 +270,52 @@ func loadConfig(data []byte) (*apiserverconfig.EncryptionConfiguration, error) {
 	return config, validation.ValidateEncryptionConfiguration(config).ToAggregate()
 }
 
-var (
+func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, stopCh <-chan struct{}) ([]value.PrefixTransformer, []healthChecker, error) {
-	// The factory to create kms service. This is to make writing test easier.
+	var transformers []value.PrefixTransformer
-	envelopeServiceFactory = envelope.NewGRPCService
+	var probes []healthChecker
 
-	// The factory to create kmsv2 service.
-	envelopeKMSv2ServiceFactory = envelopekmsv2.NewGRPCService
-)
-
-func prefixTransformers(config *apiserverconfig.ResourceConfiguration, stopCh <-chan struct{}) ([]value.PrefixTransformer, error) {
-	// we ignore the cancel func because this context should only be canceled when stopCh is closed
-	ctx, _ := wait.ContextForChannel(stopCh)
-
-	var result []value.PrefixTransformer
 	for _, provider := range config.Providers {
+		provider := provider
 		var (
-			transformer value.PrefixTransformer
+			transformer    value.PrefixTransformer
-			err         error
+			transformerErr error
+			probe          healthChecker
 		)
 
 		switch {
 		case provider.AESGCM != nil:
-			transformer, err = aesPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1)
+			transformer, transformerErr = aesPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1)
-		case provider.AESCBC != nil:
-			transformer, err = aesPrefixTransformer(provider.AESCBC, aestransformer.NewCBCTransformer, aesCBCTransformerPrefixV1)
-		case provider.Secretbox != nil:
-			transformer, err = secretboxPrefixTransformer(provider.Secretbox)
-		case provider.KMS != nil:
-			switch provider.KMS.APIVersion {
-			case kmsAPIVersionV1:
-				var envelopeService envelope.Service
-				if envelopeService, err = envelopeServiceFactory(ctx, provider.KMS.Endpoint, provider.KMS.Timeout.Duration); err != nil {
-					return nil, fmt.Errorf("could not configure KMS plugin %q, error: %v", provider.KMS.Name, err)
-				}
-				transformer, err = envelopePrefixTransformer(provider.KMS, envelopeService, kmsTransformerPrefixV1)
-			case kmsAPIVersionV2:
-				if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) {
-					return nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", provider.KMS.Name)
-				}
 
-				var envelopeService envelopekmsv2.Service
+		case provider.AESCBC != nil:
-				if envelopeService, err = envelopeKMSv2ServiceFactory(ctx, provider.KMS.Endpoint, provider.KMS.Timeout.Duration); err != nil {
+			transformer, transformerErr = aesPrefixTransformer(provider.AESCBC, aestransformer.NewCBCTransformer, aesCBCTransformerPrefixV1)
-					return nil, fmt.Errorf("could not configure KMSv2 plugin %q, error: %v", provider.KMS.Name, err)
+
-				}
+		case provider.Secretbox != nil:
-				transformer, err = envelopekmsv2PrefixTransformer(provider.KMS, envelopeService, kmsTransformerPrefixV2)
+			transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox)
-			default:
+
-				return nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", provider.KMS.Name, provider.KMS.APIVersion)
+		case provider.KMS != nil:
+			transformer, probe, transformerErr = kmsPrefixTransformer(provider.KMS, stopCh)
+			if transformerErr == nil {
+				probes = append(probes, probe)
 			}
+
 		case provider.Identity != nil:
 			transformer = value.PrefixTransformer{
 				Transformer: identity.NewEncryptCheckTransformer(),
 				Prefix:      []byte{},
 			}
+
 		default:
-			return nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity")
+			return nil, nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity")
 		}
 
-		if err != nil {
+		if transformerErr != nil {
-			return result, err
+			return nil, nil, transformerErr
 		}
-		result = append(result, transformer)
+
+		transformers = append(transformers, transformer)
 	}
-	return result, nil
+
+	return transformers, probes, nil
 }
 
 type blockTransformerFunc func(cipher.Block) value.Transformer
@@ -400,6 +327,7 @@ func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTran
 		return result, fmt.Errorf("aes provider has no valid keys")
 	}
 	for _, key := range config.Keys {
+		key := key
 		if key.Name == "" {
 			return result, fmt.Errorf("key with invalid name provided")
 		}
@@ -411,6 +339,7 @@ func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTran
 	keyTransformers := []value.PrefixTransformer{}
 
 	for _, keyData := range config.Keys {
+		keyData := keyData
 		key, err := base64.StdEncoding.DecodeString(keyData.Secret)
 		if err != nil {
 			return result, fmt.Errorf("could not obtain secret for named key %s: %s", keyData.Name, err)
@@ -447,6 +376,7 @@ func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration)
 		return result, fmt.Errorf("secretbox provider has no valid keys")
 	}
 	for _, key := range config.Keys {
+		key := key
 		if key.Name == "" {
 			return result, fmt.Errorf("key with invalid name provided")
 		}
@@ -458,6 +388,7 @@ func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration)
 	keyTransformers := []value.PrefixTransformer{}
 
 	for _, keyData := range config.Keys {
+		keyData := keyData
 		key, err := base64.StdEncoding.DecodeString(keyData.Secret)
 		if err != nil {
 			return result, fmt.Errorf("could not obtain secret for named key %s: %s", keyData.Name, err)
@@ -490,7 +421,70 @@ func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration)
 	return result, nil
 }
 
-func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) (value.PrefixTransformer, error) {
+var (
+	// The factory to create kms service. This is to make writing test easier.
+	envelopeServiceFactory = envelope.NewGRPCService
+
+	// The factory to create kmsv2 service.
+	envelopeKMSv2ServiceFactory = envelopekmsv2.NewGRPCService
+)
+
+func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-chan struct{}) (value.PrefixTransformer, healthChecker, error) {
+	// we ignore the cancel func because this context should only be canceled when stopCh is closed
+	ctx, _ := wait.ContextForChannel(stopCh)
+
+	kmsName := config.Name
+	switch config.APIVersion {
+	case kmsAPIVersionV1:
+		envelopeService, err := envelopeServiceFactory(ctx, config.Endpoint, config.Timeout.Duration)
+		if err != nil {
+			return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %v", kmsName, err)
+		}
+
+		probe := &kmsPluginProbe{
+			name:         kmsName,
+			ttl:          kmsPluginHealthzNegativeTTL,
+			service:      envelopeService,
+			l:            &sync.Mutex{},
+			lastResponse: &kmsPluginHealthzResponse{},
+		}
+
+		transformer := envelopePrefixTransformer(config, envelopeService, kmsTransformerPrefixV1)
+
+		return transformer, probe, nil
+
+	case kmsAPIVersionV2:
+		if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) {
+			return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName)
+		}
+
+		envelopeService, err := envelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Timeout.Duration)
+		if err != nil {
+			return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %v", kmsName, err)
+		}
+
+		probe := &kmsv2PluginProbe{
+			name:         kmsName,
+			ttl:          kmsPluginHealthzNegativeTTL,
+			service:      envelopeService,
+			l:            &sync.Mutex{},
+			lastResponse: &kmsPluginHealthzResponse{},
+		}
+
+		// using AES-GCM by default for encrypting data with KMSv2
+		transformer := value.PrefixTransformer{
+			Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), aestransformer.NewGCMTransformer),
+			Prefix:      []byte(kmsTransformerPrefixV2 + kmsName + ":"),
+		}
+
+		return transformer, probe, nil
+
+	default:
+		return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion)
+	}
+}
+
+func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) value.PrefixTransformer {
 	baseTransformerFunc := func(block cipher.Block) value.Transformer {
 		// v1.24: write using AES-CBC only but support reads via AES-CBC and AES-GCM (so we can move to AES-GCM)
 		// v1.25: write using AES-GCM only but support reads via AES-GCM and fallback to AES-CBC for backwards compatibility
@@ -499,33 +493,18 @@ func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelop
 		return unionTransformers{aestransformer.NewGCMTransformer(block), aestransformer.NewCBCTransformer(block)}
 	}
 
-	envelopeTransformer, err := envelope.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), baseTransformerFunc)
-	if err != nil {
-		return value.PrefixTransformer{}, err
-	}
 	return value.PrefixTransformer{
-		Transformer: envelopeTransformer,
+		Transformer: envelope.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), baseTransformerFunc),
 		Prefix:      []byte(prefix + config.Name + ":"),
-	}, nil
-}
-
-func envelopekmsv2PrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelopekmsv2.Service, prefix string) (value.PrefixTransformer, error) {
-	// using AES-GCM by default for encrypting data with KMSv2
-	envelopeTransformer, err := envelopekmsv2.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), aestransformer.NewGCMTransformer)
-	if err != nil {
-		return value.PrefixTransformer{}, err
 	}
-	return value.PrefixTransformer{
-		Transformer: envelopeTransformer,
-		Prefix:      []byte(prefix + config.Name + ":"),
-	}, nil
 }
 
 type unionTransformers []value.Transformer
 
 func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, stale bool, err error) {
 	var errs []error
-	for i, transformer := range u {
+	for i := range u {
+		transformer := u[i]
 		result, stale, err := transformer.TransformFromStorage(ctx, data, dataCtx)
 		if err != nil {
 			errs = append(errs, err)

staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go

@@ -21,8 +21,6 @@ import (
 	"context"
 	"encoding/base64"
 	"errors"
-	"io"
-	"os"
 	"sync"
 	"testing"
 	"time"
@@ -44,26 +42,6 @@ const (
 	sampleContextText = "0123456789"
 )
 
-func mustReadConfig(t *testing.T, path string) []byte {
-	t.Helper()
-	f, err := os.Open(path)
-	if err != nil {
-		t.Fatalf("error opening encryption configuration file %q: %v", path, err)
-	}
-	defer f.Close()
-
-	configFileContents, err := io.ReadAll(f)
-	if err != nil {
-		t.Fatalf("could not read contents of encryption config: %v", err)
-	}
-
-	return configFileContents
-}
-
-func mustConfigReader(t *testing.T, path string) io.Reader {
-	return bytes.NewReader(mustReadConfig(t, path))
-}
-
 // testEnvelopeService is a mock envelope service which can be used to simulate remote Envelope services
 // for testing of the envelope transformer with other transformers.
 type testEnvelopeService struct {
@@ -136,7 +114,7 @@ func newMockErrorEnvelopeKMSv2Service(endpoint string, timeout time.Duration) (e
 
 func TestLegacyConfig(t *testing.T) {
 	legacyV1Config := "testdata/valid-configs/legacy.yaml"
-	legacyConfigObject, err := loadConfig(mustReadConfig(t, legacyV1Config))
+	legacyConfigObject, err := loadConfig(legacyV1Config)
 	cacheSize := int32(10)
 	if err != nil {
 		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, legacyV1Config)
@@ -199,37 +177,37 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
 	// Transforms data using one of them, and tries to untransform using the others.
 	// Repeats this for all possible combinations.
 	correctConfigWithIdentityFirst := "testdata/valid-configs/identity-first.yaml"
-	identityFirstTransformerOverrides, err := parseEncryptionConfiguration(mustConfigReader(t, correctConfigWithIdentityFirst), ctx.Done())
+	identityFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithIdentityFirst, ctx.Done())
 	if err != nil {
 		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithIdentityFirst)
 	}
 
 	correctConfigWithAesGcmFirst := "testdata/valid-configs/aes-gcm-first.yaml"
-	aesGcmFirstTransformerOverrides, err := parseEncryptionConfiguration(mustConfigReader(t, correctConfigWithAesGcmFirst), ctx.Done())
+	aesGcmFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesGcmFirst, ctx.Done())
 	if err != nil {
 		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesGcmFirst)
 	}
 
 	correctConfigWithAesCbcFirst := "testdata/valid-configs/aes-cbc-first.yaml"
-	aesCbcFirstTransformerOverrides, err := parseEncryptionConfiguration(mustConfigReader(t, correctConfigWithAesCbcFirst), ctx.Done())
+	aesCbcFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesCbcFirst, ctx.Done())
 	if err != nil {
 		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesCbcFirst)
 	}
 
 	correctConfigWithSecretboxFirst := "testdata/valid-configs/secret-box-first.yaml"
-	secretboxFirstTransformerOverrides, err := parseEncryptionConfiguration(mustConfigReader(t, correctConfigWithSecretboxFirst), ctx.Done())
+	secretboxFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithSecretboxFirst, ctx.Done())
 	if err != nil {
 		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithSecretboxFirst)
 	}
 
 	correctConfigWithKMSFirst := "testdata/valid-configs/kms-first.yaml"
-	kmsFirstTransformerOverrides, err := parseEncryptionConfiguration(mustConfigReader(t, correctConfigWithKMSFirst), ctx.Done())
+	kmsFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSFirst, ctx.Done())
 	if err != nil {
 		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSFirst)
 	}
 
 	correctConfigWithKMSv2First := "testdata/valid-configs/kmsv2-first.yaml"
-	kmsv2FirstTransformerOverrides, err := parseEncryptionConfiguration(mustConfigReader(t, correctConfigWithKMSv2First), ctx.Done())
+	kmsv2FirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSv2First, ctx.Done())
 	if err != nil {
 		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSv2First)
 	}
@@ -281,44 +259,33 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
 func TestKMSPluginHealthz(t *testing.T) {
 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
 
-	ctx := testContext(t)
-
-	service, err := envelope.NewGRPCService(ctx, "unix:///tmp/testprovider.sock", 3*time.Second)
-	if err != nil {
-		t.Fatalf("Could not initialize envelopeService, error: %v", err)
-	}
-	serviceKMSv2, err := envelopekmsv2.NewGRPCService(ctx, "unix:///tmp/testprovider.sock", 3*time.Second)
-	if err != nil {
-		t.Fatalf("Could not initialize kmsv2 envelopeService, error: %v", err)
-	}
-
 	testCases := []struct {
 		desc    string
 		config  string
-		want    []interface{}
+		want    []healthChecker
-		wantErr bool
+		wantErr string
 	}{
 		{
 			desc:   "Install Healthz",
 			config: "testdata/valid-configs/kms/default-timeout.yaml",
-			want: []interface{}{
+			want: []healthChecker{
 				&kmsPluginProbe{
-					name:    "foo",
+					name: "foo",
-					Service: service,
+					ttl:  3 * time.Second,
 				},
 			},
 		},
 		{
 			desc:   "Install multiple healthz",
 			config: "testdata/valid-configs/kms/multiple-providers.yaml",
-			want: []interface{}{
+			want: []healthChecker{
 				&kmsPluginProbe{
-					name:    "foo",
+					name: "foo",
-					Service: service,
+					ttl:  3 * time.Second,
 				},
 				&kmsPluginProbe{
-					name:    "bar",
+					name: "bar",
-					Service: service,
+					ttl:  3 * time.Second,
 				},
 			},
 		},
@@ -329,14 +296,14 @@ func TestKMSPluginHealthz(t *testing.T) {
 		{
 			desc:   "Install multiple healthz with v1 and v2",
 			config: "testdata/valid-configs/kms/multiple-providers-kmsv2.yaml",
-			want: []interface{}{
+			want: []healthChecker{
 				&kmsv2PluginProbe{
-					name:    "foo",
+					name: "foo",
-					Service: serviceKMSv2,
+					ttl:  3 * time.Second,
 				},
 				&kmsPluginProbe{
-					name:    "bar",
+					name: "bar",
-					Service: service,
+					ttl:  3 * time.Second,
 				},
 			},
 		},
@@ -344,18 +311,50 @@ func TestKMSPluginHealthz(t *testing.T) {
 			desc:    "Invalid API version",
 			config:  "testdata/invalid-configs/kms/invalid-apiversion.yaml",
 			want:    nil,
-			wantErr: true,
+			wantErr: `resources[0].providers[0].kms.apiVersion: Invalid value: "v3": unsupported apiVersion apiVersion for KMS provider, only v1 and v2 are supported`,
 		},
 	}
 
 	for _, tt := range testCases {
 		t.Run(tt.desc, func(t *testing.T) {
-			got, err := getKMSPluginProbes(mustConfigReader(t, tt.config), ctx.Done())
+			config, err := loadConfig(tt.config)
-			if err != nil && !tt.wantErr {
+			if errStr := errString(err); errStr != tt.wantErr {
-				t.Fatalf("got %v, want nil for error", err)
+				t.Fatalf("unexpected error state got=%s want=%s", errStr, tt.wantErr)
+			}
+			if len(tt.wantErr) > 0 {
+				return
 			}
 
-			if d := cmp.Diff(tt.want, got, cmp.Comparer(serviceComparer), cmp.Comparer(serviceKMSv2Comparer)); d != "" {
+			_, got, err := getTransformerOverridesAndKMSPluginProbes(config, testContext(t).Done())
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// unset fields that are not relevant to the test
+			for i := range got {
+				checker := got[i]
+				switch p := checker.(type) {
+				case *kmsPluginProbe:
+					p.service = nil
+					p.l = nil
+					p.lastResponse = nil
+				case *kmsv2PluginProbe:
+					p.service = nil
+					p.l = nil
+					p.lastResponse = nil
+				default:
+					t.Fatalf("unexpected probe type %T", p)
+				}
+			}
+
+			if d := cmp.Diff(tt.want, got,
+				cmp.Comparer(func(a, b *kmsPluginProbe) bool {
+					return *a == *b
+				}),
+				cmp.Comparer(func(a, b *kmsv2PluginProbe) bool {
+					return *a == *b
+				}),
+			); d != "" {
 				t.Fatalf("HealthzConfig mismatch (-want +got):\n%s", d)
 			}
 		})
@@ -378,7 +377,7 @@ func TestKMSPluginHealthzTTL(t *testing.T) {
 			probe: &kmsPluginProbe{
 				name:         "test",
 				ttl:          kmsPluginHealthzNegativeTTL,
-				Service:      service,
+				service:      service,
 				l:            &sync.Mutex{},
 				lastResponse: &kmsPluginHealthzResponse{},
 			},
@@ -389,7 +388,7 @@ func TestKMSPluginHealthzTTL(t *testing.T) {
 			probe: &kmsPluginProbe{
 				name:         "test",
 				ttl:          kmsPluginHealthzPositiveTTL,
-				Service:      errService,
+				service:      errService,
 				l:            &sync.Mutex{},
 				lastResponse: &kmsPluginHealthzResponse{},
 			},
@@ -399,7 +398,7 @@ func TestKMSPluginHealthzTTL(t *testing.T) {
 
 	for _, tt := range testCases {
 		t.Run(tt.desc, func(t *testing.T) {
-			tt.probe.Check()
+			_ = tt.probe.check()
 			if tt.probe.ttl != tt.wantTTL {
 				t.Fatalf("want ttl %v, got ttl %v", tt.wantTTL, tt.probe.ttl)
 			}
@@ -423,7 +422,7 @@ func TestKMSv2PluginHealthzTTL(t *testing.T) {
 			probe: &kmsv2PluginProbe{
 				name:         "test",
 				ttl:          kmsPluginHealthzNegativeTTL,
-				Service:      service,
+				service:      service,
 				l:            &sync.Mutex{},
 				lastResponse: &kmsPluginHealthzResponse{},
 			},
@@ -434,7 +433,7 @@ func TestKMSv2PluginHealthzTTL(t *testing.T) {
 			probe: &kmsv2PluginProbe{
 				name:         "test",
 				ttl:          kmsPluginHealthzPositiveTTL,
-				Service:      errService,
+				service:      errService,
 				l:            &sync.Mutex{},
 				lastResponse: &kmsPluginHealthzResponse{},
 			},
@@ -444,7 +443,7 @@ func TestKMSv2PluginHealthzTTL(t *testing.T) {
 
 	for _, tt := range testCases {
 		t.Run(tt.desc, func(t *testing.T) {
-			tt.probe.Check()
+			_ = tt.probe.check(ctx)
 			if tt.probe.ttl != tt.wantTTL {
 				t.Fatalf("want ttl %v, got ttl %v", tt.wantTTL, tt.probe.ttl)
 			}
@@ -452,16 +451,6 @@ func TestKMSv2PluginHealthzTTL(t *testing.T) {
 	}
 }
 
-// As long as got and want contain envelope.Service we will return true.
-// If got has an envelope.Service and want does note (or vice versa) this will return false.
-func serviceComparer(_, _ envelope.Service) bool {
-	return true
-}
-
-func serviceKMSv2Comparer(_, _ envelopekmsv2.Service) bool {
-	return true
-}
-
 func TestCBCKeyRotationWithOverlappingProviders(t *testing.T) {
 	testCBCKeyRotationWithProviders(
 		t,
@@ -539,7 +528,7 @@ func getTransformerFromEncryptionConfig(t *testing.T, encryptionConfigPath strin
 	ctx := testContext(t)
 
 	t.Helper()
-	transformers, err := parseEncryptionConfiguration(mustConfigReader(t, encryptionConfigPath), ctx.Done())
+	transformers, _, err := LoadEncryptionConfig(encryptionConfigPath, ctx.Done())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -592,3 +581,11 @@ func testContext(t *testing.T) context.Context {
 	t.Cleanup(cancel)
 	return ctx
 }
+
+func errString(err error) string {
+	if err == nil {
+		return ""
+	}
+
+	return err.Error()
+}

staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope.go

@@ -63,7 +63,7 @@ type envelopeTransformer struct {
 // It uses envelopeService to encrypt and decrypt DEKs. Respective DEKs (in encrypted form) are prepended to
 // the data items they encrypt. A cache (of size cacheSize) is maintained to store the most recently
 // used decrypted DEKs in memory.
-func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransformerFunc func(cipher.Block) value.Transformer) (value.Transformer, error) {
+func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransformerFunc func(cipher.Block) value.Transformer) value.Transformer {
 	var (
 		cache *lru.Cache
 	)
@@ -77,7 +77,7 @@ func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransfor
 		baseTransformerFunc: baseTransformerFunc,
 		cacheEnabled:        cacheSize > 0,
 		cacheSize:           cacheSize,
-	}, nil
+	}
 }
 
 // TransformFromStorage decrypts data encrypted by this transformer using envelope encryption.

staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope_test.go

@@ -106,10 +106,7 @@ func TestEnvelopeCaching(t *testing.T) {
 	for _, tt := range testCases {
 		t.Run(tt.desc, func(t *testing.T) {
 			envelopeService := newTestEnvelopeService()
-			envelopeTransformer, err := NewEnvelopeTransformer(envelopeService, tt.cacheSize, aestransformer.NewCBCTransformer)
+			envelopeTransformer := NewEnvelopeTransformer(envelopeService, tt.cacheSize, aestransformer.NewCBCTransformer)
-			if err != nil {
-				t.Fatalf("failed to initialize envelope transformer: %v", err)
-			}
 			ctx := context.Background()
 			dataCtx := value.DefaultContext([]byte(testContextText))
 			originalText := []byte(testText)
@@ -149,10 +146,7 @@ func TestEnvelopeCaching(t *testing.T) {
 
 // Makes Envelope transformer hit cache limit, throws error if it misbehaves.
 func TestEnvelopeCacheLimit(t *testing.T) {
-	envelopeTransformer, err := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewCBCTransformer)
+	envelopeTransformer := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewCBCTransformer)
-	if err != nil {
-		t.Fatalf("failed to initialize envelope transformer: %v", err)
-	}
 	ctx := context.Background()
 	dataCtx := value.DefaultContext([]byte(testContextText))
 
@@ -185,10 +179,7 @@ func TestEnvelopeCacheLimit(t *testing.T) {
 }
 
 func BenchmarkEnvelopeCBCRead(b *testing.B) {
-	envelopeTransformer, err := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewCBCTransformer)
+	envelopeTransformer := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewCBCTransformer)
-	if err != nil {
-		b.Fatalf("failed to initialize envelope transformer: %v", err)
-	}
 	benchmarkRead(b, envelopeTransformer, 1024)
 }
 
@@ -203,10 +194,7 @@ func BenchmarkAESCBCRead(b *testing.B) {
 }
 
 func BenchmarkEnvelopeGCMRead(b *testing.B) {
-	envelopeTransformer, err := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewGCMTransformer)
+	envelopeTransformer := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewGCMTransformer)
-	if err != nil {
-		b.Fatalf("failed to initialize envelope transformer: %v", err)
-	}
 	benchmarkRead(b, envelopeTransformer, 1024)
 }
 
@@ -246,10 +234,7 @@ func benchmarkRead(b *testing.B, transformer value.Transformer, valueLength int)
 // remove after 1.13
 func TestBackwardsCompatibility(t *testing.T) {
 	envelopeService := newTestEnvelopeService()
-	envelopeTransformerInst, err := NewEnvelopeTransformer(envelopeService, testEnvelopeCacheSize, aestransformer.NewCBCTransformer)
+	envelopeTransformerInst := NewEnvelopeTransformer(envelopeService, testEnvelopeCacheSize, aestransformer.NewCBCTransformer)
-	if err != nil {
-		t.Fatalf("failed to initialize envelope transformer: %v", err)
-	}
 	ctx := context.Background()
 	dataCtx := value.DefaultContext([]byte(testContextText))
 	originalText := []byte(testText)

staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go

@@ -87,7 +87,7 @@ type StatusResponse struct {
 // It uses envelopeService to encrypt and decrypt DEKs. Respective DEKs (in encrypted form) are prepended to
 // the data items they encrypt. A cache (of size cacheSize) is maintained to store the most recently
 // used decrypted DEKs in memory.
-func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransformerFunc func(cipher.Block) value.Transformer) (value.Transformer, error) {
+func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransformerFunc func(cipher.Block) value.Transformer) value.Transformer {
 	var cache *lru.Cache
 
 	if cacheSize > 0 {
@@ -102,7 +102,7 @@ func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransfor
 		baseTransformerFunc: baseTransformerFunc,
 		cacheEnabled:        cacheSize > 0,
 		cacheSize:           cacheSize,
-	}, nil
+	}
 }
 
 // TransformFromStorage decrypts data encrypted by this transformer using envelope encryption.

staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope_test.go

@@ -118,10 +118,7 @@ func TestEnvelopeCaching(t *testing.T) {
 	for _, tt := range testCases {
 		t.Run(tt.desc, func(t *testing.T) {
 			envelopeService := newTestEnvelopeService()
-			envelopeTransformer, err := NewEnvelopeTransformer(envelopeService, tt.cacheSize, aestransformer.NewGCMTransformer)
+			envelopeTransformer := NewEnvelopeTransformer(envelopeService, tt.cacheSize, aestransformer.NewGCMTransformer)
-			if err != nil {
-				t.Fatalf("failed to initialize envelope transformer: %v", err)
-			}
 			ctx := context.Background()
 			dataCtx := value.DefaultContext([]byte(testContextText))
 			originalText := []byte(testText)
@@ -161,10 +158,7 @@ func TestEnvelopeCaching(t *testing.T) {
 
 // Makes Envelope transformer hit cache limit, throws error if it misbehaves.
 func TestEnvelopeCacheLimit(t *testing.T) {
-	envelopeTransformer, err := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewGCMTransformer)
+	envelopeTransformer := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewGCMTransformer)
-	if err != nil {
-		t.Fatalf("failed to initialize envelope transformer: %v", err)
-	}
 	ctx := context.Background()
 	dataCtx := value.DefaultContext([]byte(testContextText))
 

test/integration/controlplane/transformation/kmsv2_transformation_test.go

@@ -98,7 +98,7 @@ func (r envelopekmsv2) plainTextPayload(secretETCDPath string) ([]byte, error) {
 
 // TestKMSv2Provider is an integration test between KubeAPI, ETCD and KMSv2 Plugin
 // Concretely, this test verifies the following integration contracts:
-// 1. Raw records in ETCD that were processed by KMSv2 Provider should be prefixed with []byte{'e', 'k', '8', 's', 0}
+// 1. Raw records in ETCD that were processed by KMSv2 Provider should be prefixed with k8s:enc:kms:v2:<plugin name>:
 // 2. Data Encryption Key (DEK) should be generated by envelopeTransformer and passed to KMS gRPC Plugin
 // 3. KMS gRPC Plugin should encrypt the DEK with a Key Encryption Key (KEK) and pass it back to envelopeTransformer
 // 4. The cipherTextPayload (ex. Secret) should be encrypted via AES GCM transform