Merge pull request #49742 from sakshamsharma/kms-plugin-registry

Automatic merge from submit-queue

Add KMS plugin registry

Allows supporting KMS services as encryption providers using a plugin mechanism similar to admission plugins.

Simplifies #48574 

Progresses #48522

@deads2k PTAL

staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/BUILD

@@ -12,14 +12,17 @@ go_library(
     name = "go_default_library",
     srcs = [
         "config.go",
+        "plugins.go",
         "types.go",
     ],
     tags = ["automanaged"],
     deps = [
         "//vendor/github.com/ghodss/yaml:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/aes:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/identity:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/secretbox:go_default_library",
     ],
@@ -33,5 +36,6 @@ go_test(
     deps = [
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope:go_default_library",
     ],
 )

staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go

@@ -30,14 +30,17 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/storage/value"
 	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
+	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
 	"k8s.io/apiserver/pkg/storage/value/encrypt/identity"
 	"k8s.io/apiserver/pkg/storage/value/encrypt/secretbox"
 )
 
 const (
-	aesCBCTransformerPrefixV1    = "k8s:enc:aescbc:v1:"
+	aesCBCTransformerPrefixV1           = "k8s:enc:aescbc:v1:"
-	aesGCMTransformerPrefixV1    = "k8s:enc:aesgcm:v1:"
+	aesGCMTransformerPrefixV1           = "k8s:enc:aesgcm:v1:"
-	secretboxTransformerPrefixV1 = "k8s:enc:secretbox:v1:"
+	secretboxTransformerPrefixV1        = "k8s:enc:secretbox:v1:"
+	kmsTransformerPrefixV1              = "k8s:enc:kms:v1:"
+	cloudProvidedKMSTransformerPrefixV1 = "k8s:enc:cloudprovidedkms:v1:"
 )
 
 // GetTransformerOverrides returns the transformer overrides by reading and parsing the encryption provider configuration file
@@ -144,6 +147,38 @@ func GetPrefixTransformers(config *ResourceConfig) ([]value.PrefixTransformer, e
 			found = true
 		}
 
+		if provider.KMS != nil {
+			if found == true {
+				return nil, fmt.Errorf("more than one provider specified in a single element, should split into different list elements")
+			}
+			f, err := os.Open(provider.KMS.ConfigFile)
+			if err != nil {
+				return nil, fmt.Errorf("error opening KMS provider configuration file %q: %v", provider.KMS.ConfigFile, err)
+			}
+			defer f.Close()
+			envelopeService, pluginFound, err := KMSPluginRegistry.getPlugin(provider.KMS.Name, f)
+			if err != nil {
+				return nil, fmt.Errorf("could not configure KMS plugin %q, %v", provider.KMS.Name, err)
+			}
+			if pluginFound == false {
+				return nil, fmt.Errorf("KMS plugin %q not found", provider.KMS.Name)
+			}
+			transformer, err = getEnvelopePrefixTransformer(provider.KMS.CoreKMSConfig, envelopeService, kmsTransformerPrefixV1)
+			found = true
+		}
+
+		if provider.CloudProvidedKMS != nil {
+			if found == true {
+				return nil, fmt.Errorf("more than one provider specified in a single element, should split into different list elements")
+			}
+			envelopeService, err := KMSPluginRegistry.getCloudProvidedPlugin(provider.CloudProvidedKMS.Name)
+			if err != nil {
+				return nil, fmt.Errorf("could not configure Cloud-Provided KMS plugin %q, %v", provider.CloudProvidedKMS.Name, err)
+			}
+			transformer, err = getEnvelopePrefixTransformer(provider.CloudProvidedKMS.CoreKMSConfig, envelopeService, cloudProvidedKMSTransformerPrefixV1)
+			found = true
+		}
+
 		if err != nil {
 			return result, err
 		}
@@ -258,3 +293,16 @@ func GetSecretboxPrefixTransformer(config *SecretboxConfig) (value.PrefixTransfo
 	}
 	return result, nil
 }
+
+// getEnvelopePrefixTransformer returns a prefix transformer from the provided config.
+// envelopeService is used as the root of trust.
+func getEnvelopePrefixTransformer(config *CoreKMSConfig, envelopeService envelope.Service, prefix string) (value.PrefixTransformer, error) {
+	envelopeTransformer, err := envelope.NewEnvelopeTransformer(envelopeService, config.CacheSize, aestransformer.NewCBCTransformer)
+	if err != nil {
+		return value.PrefixTransformer{}, err
+	}
+	return value.PrefixTransformer{
+		Transformer: envelopeTransformer,
+		Prefix:      []byte(prefix + config.Name + ":"),
+	}, nil
+}

staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/encryptionconfig_test.go

@@ -18,11 +18,16 @@ package encryptionconfig
 
 import (
 	"bytes"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"os"
 	"strings"
 	"testing"
 
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/storage/value"
+	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
 )
 
 const (
@@ -30,6 +35,10 @@ const (
 
 	sampleContextText = "0123456789"
 
+	// Modify these in all configurations if changed
+	testEnvelopeServiceConfigPath   = "testproviderconfig"
+	testEnvelopeServiceProviderName = "testprovider"
+
 	correctConfigWithIdentityFirst = `
 kind: EncryptionConfig
 apiVersion: v1
@@ -45,6 +54,13 @@ resources:
           secret: c2VjcmV0IGlzIHNlY3VyZQ==
         - name: key2
           secret: dGhpcyBpcyBwYXNzd29yZA==
+    - kms:
+        name: testprovider
+        configfile: testproviderconfig
+        cachesize: 10
+    - cloudprovidedkms:
+        name: testprovider
+        cachesize: 10
     - aescbc:
         keys:
         - name: key1
@@ -74,12 +90,19 @@ resources:
         keys:
         - name: key1
           secret: YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXoxMjM0NTY=
+    - kms:
+        name: testprovider
+        configfile: testproviderconfig
+        cachesize: 10
     - aescbc:
         keys:
         - name: key1
           secret: c2VjcmV0IGlzIHNlY3VyZQ==
         - name: key2
           secret: dGhpcyBpcyBwYXNzd29yZA==
+    - cloudprovidedkms:
+        name: testprovider
+        cachesize: 10
     - identity: {}
 `
 
@@ -96,7 +119,14 @@ resources:
           secret: c2VjcmV0IGlzIHNlY3VyZQ==
         - name: key2
           secret: dGhpcyBpcyBwYXNzd29yZA==
+    - kms:
+        name: testprovider
+        configfile: testproviderconfig
+        cachesize: 10
     - identity: {}
+    - cloudprovidedkms:
+        name: testprovider
+        cachesize: 10
     - secretbox:
         keys:
         - name: key1
@@ -116,6 +146,43 @@ resources:
   - resources:
     - secrets
     providers:
+    - secretbox:
+        keys:
+        - name: key1
+          secret: YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXoxMjM0NTY=
+    - cloudprovidedkms:
+        name: testprovider
+        cachesize: 10
+    - aescbc:
+        keys:
+        - name: key1
+          secret: c2VjcmV0IGlzIHNlY3VyZQ==
+        - name: key2
+          secret: dGhpcyBpcyBwYXNzd29yZA==
+    - kms:
+        name: testprovider
+        configfile: testproviderconfig
+        cachesize: 10
+    - identity: {}
+    - aesgcm:
+        keys:
+        - name: key1
+          secret: c2VjcmV0IGlzIHNlY3VyZQ==
+        - name: key2
+          secret: dGhpcyBpcyBwYXNzd29yZA==
+`
+
+	correctConfigWithKMSFirst = `
+kind: EncryptionConfig
+apiVersion: v1
+resources:
+  - resources:
+    - secrets
+    providers:
+    - kms:
+        name: testprovider
+        configfile: testproviderconfig
+        cachesize: 10
     - secretbox:
         keys:
         - name: key1
@@ -127,6 +194,45 @@ resources:
         - name: key2
           secret: dGhpcyBpcyBwYXNzd29yZA==
     - identity: {}
+    - cloudprovidedkms:
+        name: testprovider
+        cachesize: 10
+    - aesgcm:
+        keys:
+        - name: key1
+          secret: c2VjcmV0IGlzIHNlY3VyZQ==
+        - name: key2
+          secret: dGhpcyBpcyBwYXNzd29yZA==
+`
+
+	correctConfigWithCloudProvidedKMSFirst = `
+kind: EncryptionConfig
+apiVersion: v1
+resources:
+  - resources:
+    - secrets
+    providers:
+    - cloudprovidedkms:
+        name: testprovider
+        cachesize: 10
+    - kms:
+        name: testprovider
+        configfile: testproviderconfig
+        cachesize: 10
+    - secretbox:
+        keys:
+        - name: key1
+          secret: YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXoxMjM0NTY=
+    - aescbc:
+        keys:
+        - name: key1
+          secret: c2VjcmV0IGlzIHNlY3VyZQ==
+        - name: key2
+          secret: dGhpcyBpcyBwYXNzd29yZA==
+    - identity: {}
+    - cloudprovidedkms:
+        name: testprovider
+        cachesize: 10
     - aesgcm:
         keys:
         - name: key1
@@ -165,11 +271,48 @@ resources:
 `
 )
 
-func TestEncryptionProviderConfigCorrect(t *testing.T) {
+// testEnvelopeService is a mock envelope service which can be used to simulate remote Envelope services
-	// Creates two transformers with different ordering of identity and AES transformers.
+// for testing of the envelope transformer with other transformers.
-	// Transforms data using one of them, and tries to untransform using both of them.
+type testEnvelopeService struct {
-	// Repeats this for both the possible combinations.
+	disabled bool
+}
 
+func (t *testEnvelopeService) Decrypt(data string) ([]byte, error) {
+	if t.disabled {
+		return nil, fmt.Errorf("Envelope service was disabled")
+	}
+	return base64.StdEncoding.DecodeString(data)
+}
+
+func (t *testEnvelopeService) Encrypt(data []byte) (string, error) {
+	if t.disabled {
+		return "", fmt.Errorf("Envelope service was disabled")
+	}
+	return base64.StdEncoding.EncodeToString(data), nil
+}
+
+func (t *testEnvelopeService) SetDisabledStatus(status bool) {
+	t.disabled = status
+}
+
+var _ envelope.Service = &testEnvelopeService{}
+
+func TestEncryptionProviderConfigCorrect(t *testing.T) {
+	os.OpenFile(testEnvelopeServiceConfigPath, os.O_CREATE, 0666)
+	defer os.Remove(testEnvelopeServiceConfigPath)
+	KMSPluginRegistry.Register(testEnvelopeServiceProviderName, func(config io.Reader) (envelope.Service, error) {
+		return &testEnvelopeService{}, nil
+	})
+	KMSPluginRegistry.RegisterCloudProvidedKMSPlugin(func(name string) (envelope.Service, error) {
+		if name == testEnvelopeServiceProviderName {
+			return &testEnvelopeService{}, nil
+		}
+		return nil, fmt.Errorf("no such cloud provided KMS plugin registered: %q", name)
+	})
+
+	// Creates compound/prefix transformers with different ordering of available transformers.
+	// Transforms data using one of them, and tries to untransform using the others.
+	// Repeats this for all possible combinations.
 	identityFirstTransformerOverrides, err := ParseEncryptionConfiguration(strings.NewReader(correctConfigWithIdentityFirst))
 	if err != nil {
 		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithIdentityFirst)
@@ -190,11 +333,23 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
 		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithSecretboxFirst)
 	}
 
+	kmsFirstTransformerOverrides, err := ParseEncryptionConfiguration(strings.NewReader(correctConfigWithKMSFirst))
+	if err != nil {
+		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSFirst)
+	}
+
+	cloudProvidedKMSFirstTransformerOverrides, err := ParseEncryptionConfiguration(strings.NewReader(correctConfigWithCloudProvidedKMSFirst))
+	if err != nil {
+		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithCloudProvidedKMSFirst)
+	}
+
 	// Pick the transformer for any of the returned resources.
 	identityFirstTransformer := identityFirstTransformerOverrides[schema.ParseGroupResource("secrets")]
 	aesGcmFirstTransformer := aesGcmFirstTransformerOverrides[schema.ParseGroupResource("secrets")]
 	aesCbcFirstTransformer := aesCbcFirstTransformerOverrides[schema.ParseGroupResource("secrets")]
 	secretboxFirstTransformer := secretboxFirstTransformerOverrides[schema.ParseGroupResource("secrets")]
+	kmsFirstTransformer := kmsFirstTransformerOverrides[schema.ParseGroupResource("secrets")]
+	cloudProvidedKMSFirstTransformer := cloudProvidedKMSFirstTransformerOverrides[schema.ParseGroupResource("secrets")]
 
 	context := value.DefaultContext([]byte(sampleContextText))
 	originalText := []byte(sampleText)
@@ -207,6 +362,8 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
 		{aesCbcFirstTransformer, "aesCbcFirst"},
 		{secretboxFirstTransformer, "secretboxFirst"},
 		{identityFirstTransformer, "identityFirst"},
+		{kmsFirstTransformer, "kmsFirst"},
+		{cloudProvidedKMSFirstTransformer, "cloudProvidedKMSFirst"},
 	}
 
 	for _, testCase := range transformers {

staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/plugins.go

@@ -0,0 +1,134 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package encryptionconfig
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"reflect"
+	"sync"
+
+	"github.com/golang/glog"
+
+	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
+)
+
+// Factory is a function that returns an envelope Service for encryption providers.
+// The config parameter provides an io.Reader handler to the factory in
+// order to load specific configurations. If no configuration is provided
+// the parameter is nil.
+type Factory func(config io.Reader) (envelope.Service, error)
+
+type cloudKMSFactory func(name string) (envelope.Service, error)
+
+// KMSPlugins contains all registered KMS options.
+type KMSPlugins struct {
+	lock     sync.RWMutex
+	registry map[string]Factory
+	cloudKMS cloudKMSFactory
+}
+
+var (
+	// PluginEnabledFn checks whether a plugin is enabled.  By default, if you ask about it, it's enabled.
+	PluginEnabledFn = func(name string, config io.Reader) bool {
+		return true
+	}
+
+	// KMSPluginRegistry contains the registered KMS plugins which can be used for configuring
+	// encryption providers.
+	KMSPluginRegistry = KMSPlugins{}
+)
+
+// PluginEnabledFunc is a function type that can provide an external check on whether an admission plugin may be enabled
+type PluginEnabledFunc func(name string, config io.Reader) bool
+
+// Register registers a plugin Factory by name. This
+// is expected to happen during app startup.
+func (ps *KMSPlugins) Register(name string, plugin Factory) {
+	ps.lock.Lock()
+	defer ps.lock.Unlock()
+	_, found := ps.registry[name]
+	if ps.registry == nil {
+		ps.registry = map[string]Factory{}
+	}
+	if found {
+		glog.Fatalf("KMS plugin %q was registered twice", name)
+	}
+	glog.V(1).Infof("Registered KMS plugin %q", name)
+	ps.registry[name] = plugin
+}
+
+// RegisterCloudProvidedKMSPlugin registers the cloud's KMS provider as
+// an envelope.Service. This service is provided by the cloudprovider interface.
+func (ps *KMSPlugins) RegisterCloudProvidedKMSPlugin(cloudKMSGetter cloudKMSFactory) {
+	ps.cloudKMS = cloudKMSGetter
+}
+
+// getPlugin creates an instance of the named plugin.  It returns `false` if the
+// the name is not known. The error is returned only when the named provider was
+// known but failed to initialize.  The config parameter specifies the io.Reader
+// handler of the configuration file for the cloud provider, or nil for no configuration.
+func (ps *KMSPlugins) getPlugin(name string, config io.Reader) (envelope.Service, bool, error) {
+	f, found := ps.fetchPluginFromRegistry(name)
+	if !found {
+		return nil, false, nil
+	}
+
+	config1, config2, err := splitStream(config)
+	if err != nil {
+		return nil, true, err
+	}
+	if !PluginEnabledFn(name, config1) {
+		return nil, true, nil
+	}
+
+	ret, err := f(config2)
+	return ret, true, err
+}
+
+// fetchPluginFromRegistry tries to get a registered plugin with the requested name.
+func (ps *KMSPlugins) fetchPluginFromRegistry(name string) (Factory, bool) {
+	ps.lock.RLock()
+	defer ps.lock.RUnlock()
+	// Map lookup defaults to single value context
+	f, found := ps.registry[name]
+	return f, found
+}
+
+// getCloudProvidedPlugin creates an instance of the named cloud provided KMS plugin.
+func (ps *KMSPlugins) getCloudProvidedPlugin(name string) (envelope.Service, error) {
+	if ps.cloudKMS == nil {
+		return nil, fmt.Errorf("no cloud registered for KMS plugins")
+	}
+	return ps.cloudKMS(name)
+}
+
+// splitStream reads the stream bytes and constructs two copies of it.
+func splitStream(config io.Reader) (io.Reader, io.Reader, error) {
+	if config == nil || reflect.ValueOf(config).IsNil() {
+		return nil, nil, nil
+	}
+
+	configBytes, err := ioutil.ReadAll(config)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return bytes.NewBuffer(configBytes), bytes.NewBuffer(configBytes), nil
+}

staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/types.go

@@ -45,6 +45,11 @@ type ProviderConfig struct {
 	Secretbox *SecretboxConfig `json:"secretbox,omitempty"`
 	// identity is the (empty) configuration for the identity transformer.
 	Identity *IdentityConfig `json:"identity,omitempty"`
+	// kms contains the name, cache size and path to configuration file for a KMS based envelope transformer.
+	KMS *KMSConfig `json:"kms,omitempty"`
+	// cloudProvidedKMSConfig contains the name and cache size for a KMS based envelope transformer which uses
+	// the KMS provided by the cloud.
+	CloudProvidedKMS *CloudProvidedKMSConfig `json:"cloudprovidedkms,omitempty"`
 }
 
 // AESConfig contains the API configuration for an AES transformer.
@@ -71,3 +76,25 @@ type Key struct {
 
 // IdentityConfig is an empty struct to allow identity transformer in provider configuration.
 type IdentityConfig struct{}
+
+// CoreKMSConfig contains the name and cache sized for a KMS based envelope transformer.
+type CoreKMSConfig struct {
+	// name is the name of the KMS plugin to be used.
+	Name string `json:"name"`
+	// cacheSize is the maximum number of secrets which are cached in memory. The default value is 1000.
+	// +optional
+	CacheSize int `json:"cachesize,omitempty"`
+}
+
+// KMSConfig contains the name, cache size and path to configuration file for a KMS based envelope transformer.
+type KMSConfig struct {
+	*CoreKMSConfig
+	// configfile is the path to the configuration file for the named KMS provider.
+	ConfigFile string `json:"configfile"`
+}
+
+// CloudProvidedKMSConfig contains the name and cache size for a KMS based envelope transformer which uses
+// the KMS provided by the cloud.
+type CloudProvidedKMSConfig struct {
+	*CoreKMSConfig
+}