From c3be7e189a8b0abc3cc15a0c85e22f8c0edb246e Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 22 Feb 2018 10:26:59 -0800 Subject: [PATCH] Adding Data Encryption Key (DEK) Key Encryption Key (KEK) integration tests via KMS Plugin Mock. --- test/integration/master/BUILD | 6 + test/integration/master/kms_plugin_mock.go | 111 ++++++++++++ .../master/kms_transformation_test.go | 163 ++++++++++++++++++ 3 files changed, 280 insertions(+) create mode 100644 test/integration/master/kms_plugin_mock.go create mode 100644 test/integration/master/kms_transformation_test.go diff --git a/test/integration/master/BUILD b/test/integration/master/BUILD index 397e2041e80..6b1afacbb6a 100644 --- a/test/integration/master/BUILD +++ b/test/integration/master/BUILD @@ -11,6 +11,7 @@ go_test( size = "large", srcs = [ "crd_test.go", + "kms_transformation_test.go", "kube_apiserver_test.go", "main_test.go", "secrets_transformation_test.go", @@ -48,6 +49,7 @@ go_test( "//vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/aes:go_default_library", + "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//vendor/k8s.io/apiserver/plugin/pkg/authenticator/token/tokentest:go_default_library", @@ -75,6 +77,7 @@ filegroup( go_library( name = "go_default_library", srcs = [ + "kms_plugin_mock.go", "transformation_testcase.go", ], importpath = "k8s.io/kubernetes/test/integration/master", @@ -84,11 +87,14 @@ go_library( "//test/integration/framework:go_default_library", "//vendor/github.com/coreos/etcd/clientv3:go_default_library", "//vendor/github.com/ghodss/yaml:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library", + "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", ], ) diff --git a/test/integration/master/kms_plugin_mock.go b/test/integration/master/kms_plugin_mock.go new file mode 100644 index 00000000000..ccaae296d05 --- /dev/null +++ b/test/integration/master/kms_plugin_mock.go @@ -0,0 +1,111 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package master + +import ( + "context" + "encoding/base64" + "fmt" + "net" + "os" + + "golang.org/x/sys/unix" + "google.golang.org/grpc" + + kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" +) + +const ( + kmsAPIVersion = "v1beta1" + sockFile = "/tmp/kms-provider.sock" + unixProtocol = "unix" +) + +// base64Plugin gRPC sever for a mock KMS provider. +// Uses base64 to simulate encrypt and decrypt. +type base64Plugin struct { + grpcServer *grpc.Server + listener net.Listener + + // Allow users of the plugin to sense requests that were passed to KMS. + encryptRequest chan *kmsapi.EncryptRequest + decryptRequest chan *kmsapi.DecryptRequest +} + +func NewBase64Plugin() (*base64Plugin, error) { + if err := cleanSockFile(); err != nil { + return nil, err + } + + listener, err := net.Listen(unixProtocol, sockFile) + if err != nil { + return nil, fmt.Errorf("failed to listen on the unix socket, error: %v", err) + } + + server := grpc.NewServer() + + result := &base64Plugin{ + grpcServer: server, + listener: listener, + encryptRequest: make(chan *kmsapi.EncryptRequest, 1), + decryptRequest: make(chan *kmsapi.DecryptRequest, 1), + } + + kmsapi.RegisterKeyManagementServiceServer(server, result) + + return result, nil +} + +func (s *base64Plugin) cleanUp() { + s.grpcServer.Stop() + s.listener.Close() + cleanSockFile() +} + +var testProviderAPIVersion = kmsAPIVersion + +func (s *base64Plugin) Version(ctx context.Context, request *kmsapi.VersionRequest) (*kmsapi.VersionResponse, error) { + return &kmsapi.VersionResponse{Version: testProviderAPIVersion, RuntimeName: "testKMS", RuntimeVersion: "0.0.1"}, nil +} + +func (s *base64Plugin) Decrypt(ctx context.Context, request *kmsapi.DecryptRequest) (*kmsapi.DecryptResponse, error) { + s.decryptRequest <- request + buf := make([]byte, base64.StdEncoding.DecodedLen(len(request.Cipher))) + n, err := base64.StdEncoding.Decode(buf, request.Cipher) + if err != nil { + return nil, err + } + + return &kmsapi.DecryptResponse{Plain: buf[:n]}, nil +} + +func (s *base64Plugin) Encrypt(ctx context.Context, request *kmsapi.EncryptRequest) (*kmsapi.EncryptResponse, error) { + s.encryptRequest <- request + + buf := make([]byte, base64.StdEncoding.EncodedLen(len(request.Plain))) + base64.StdEncoding.Encode(buf, request.Plain) + + return &kmsapi.EncryptResponse{Cipher: buf}, nil +} + +func cleanSockFile() error { + err := unix.Unlink(sockFile) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to delete the socket file, error: %v", err) + } + return nil +} diff --git a/test/integration/master/kms_transformation_test.go b/test/integration/master/kms_transformation_test.go new file mode 100644 index 00000000000..7fbfc1dbf8d --- /dev/null +++ b/test/integration/master/kms_transformation_test.go @@ -0,0 +1,163 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package master + +import ( + "bytes" + "context" + "crypto/aes" + "encoding/binary" + "fmt" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/storage/value" + aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes" + + kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" +) + +const ( + kmsPrefix = "k8s:enc:kms:v1:grpc-kms-provider:" + dekKeySizeLen = 2 + + kmsConfigYAML = ` +kind: EncryptionConfig +apiVersion: v1 +resources: + - resources: + - secrets + providers: + - kms: + name: grpc-kms-provider + cachesize: 1000 + endpoint: unix:///tmp/kms-provider.sock +` +) + +// rawDEKKEKSecret provides operations for working with secrets transformed with Data Encryption Key(DEK) Key Encryption Kye(KEK) envelop. +type rawDEKKEKSecret []byte + +func (r rawDEKKEKSecret) getDEKLen() int { + // DEK's length is stored in the two bytes that follow the prefix. + return int(binary.BigEndian.Uint16(r[len(kmsPrefix) : len(kmsPrefix)+dekKeySizeLen])) +} + +func (r rawDEKKEKSecret) getDEK() []byte { + return r[len(kmsPrefix)+dekKeySizeLen : len(kmsPrefix)+dekKeySizeLen+r.getDEKLen()] +} + +func (r rawDEKKEKSecret) getStartOfPayload() int { + return len(kmsPrefix) + dekKeySizeLen + r.getDEKLen() +} + +func (r rawDEKKEKSecret) getPayload() []byte { + return r[r.getStartOfPayload():] +} + +// TestKMSProvider is an integration test between KubAPI, ETCD and KMS Plugin +// Concretely, this test verifies the following integration contracts: +// 1. Raw records in ETCD that were processed by KMS Provider should be prefixed with k8s:enc:kms:v1:grpc-kms-provider-name: +// 2. Data Encryption Key (DEK) should be generated by envelopeTransformer and passed to KMS gRPC Plugin +// 3. KMS gRPC Plugin should encrypt the DEK with a Key Encryption Key (KEK) and pass it back to envelopeTransformer +// 4. The payload (ex. Secret) should be encrypted via AES CBC transform +// 5. Prefix-EncryptedDEK-EncryptedPayload structure should be deposited to ETCD +func TestKMSProvider(t *testing.T) { + pluginMock, err := NewBase64Plugin() + if err != nil { + t.Fatalf("failed to create mock of KMS Plugin: %v", err) + } + defer pluginMock.cleanUp() + go pluginMock.grpcServer.Serve(pluginMock.listener) + + test, err := newTransformTest(t, kmsConfigYAML) + if err != nil { + t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s", kmsConfigYAML) + } + defer test.cleanUp() + + secretETCDPath := test.getETCDPath() + var rawSecretAsSeenByETCD rawDEKKEKSecret + rawSecretAsSeenByETCD, err = test.getRawSecretFromETCD() + if err != nil { + t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err) + } + + if !bytes.HasPrefix(rawSecretAsSeenByETCD, []byte(kmsPrefix)) { + t.Fatalf("expected secret to be prefixed with %s, but got %s", kmsPrefix, rawSecretAsSeenByETCD) + } + + // Since Data Encryption Key (DEK) is randomly generated (per encryption operation), we need to ask KMS Mock for it. + dekPlainAsSeenByKMS, err := getDEKFromKMSPlugin(pluginMock) + if err != nil { + t.Fatalf("failed to get DEK from KMS: %v", err) + } + + decryptResponse, err := pluginMock.Decrypt(context.Background(), + &kmsapi.DecryptRequest{Version: kmsAPIVersion, Cipher: rawSecretAsSeenByETCD.getDEK()}) + if err != nil { + t.Fatalf("failed to decrypt DEK, %v", err) + } + dekPlainAsWouldBeSeenByETCD := decryptResponse.Plain + + if !bytes.Equal(dekPlainAsSeenByKMS, dekPlainAsWouldBeSeenByETCD) { + t.Fatalf("expected dekPlainAsSeenByKMS %v to be passed to KMS Plugin, but got %s", + dekPlainAsSeenByKMS, dekPlainAsWouldBeSeenByETCD) + } + + plainSecret, err := decryptPayload(dekPlainAsWouldBeSeenByETCD, rawSecretAsSeenByETCD, secretETCDPath) + if err != nil { + t.Fatalf("failed to transform from storage via AESCBC, err: %v", err) + } + + if !strings.Contains(string(plainSecret), secretVal) { + t.Fatalf("expected %q after decryption, but got %q", secretVal, string(plainSecret)) + } + + // Secrets should be un-enveloped on direct reads from Kube API Server. + s, err := test.restClient.CoreV1().Secrets(testNamespace).Get(testSecret, metav1.GetOptions{}) + if secretVal != string(s.Data[secretKey]) { + t.Fatalf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey])) + } +} + +func getDEKFromKMSPlugin(pluginMock *base64Plugin) ([]byte, error) { + select { + case e := <-pluginMock.encryptRequest: + return e.Plain, nil + case <-time.After(1 * time.Microsecond): + return nil, fmt.Errorf("timed-out while getting encryption request from KMS Plugin Mock") + } +} + +func decryptPayload(key []byte, secret rawDEKKEKSecret, secretETCDPath string) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, fmt.Errorf("failed to initialize AES Cipher: %v", err) + } + // etcd path of the key is used as the authenticated context - need to pass it to decrypt + ctx := value.DefaultContext([]byte(secretETCDPath)) + aescbcTransformer := aestransformer.NewCBCTransformer(block) + plainSecret, _, err := aescbcTransformer.TransformFromStorage(secret.getPayload(), ctx) + if err != nil { + return nil, fmt.Errorf("failed to transform from storage via AESCBC, err: %v", err) + } + + return plainSecret, nil +}