github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 19:56:01 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #83811 from immutableT/single-kms-mock

Browse Source 
Use single kms-plugin mock in unit and integration tests.
This commit is contained in:
Kubernetes Prow Robot 2019-10-17 18:17:57 -07:00 committed by GitHub
commit 1f8b3bfd98
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 416 additions and 353 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/BUILD
View File

@ -37,53 +37,43 @@ go_test(
] + select({
"@io_bazel_rules_go//go/platform:android": [
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
],
"//conditions:default": [],
}),
@ -100,6 +90,7 @@ filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:all-srcs",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:all-srcs",
],
tags = ["automanaged"],

2
staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service.go
View File

@ -68,6 +68,8 @@ func NewGRPCService(endpoint string, callTimeout time.Duration) (Service, error)
c, err := net.DialUnix(unixProtocol, nil, &net.UnixAddr{Name: addr})
if err != nil {
klog.Errorf("failed to create connection to unix socket: %s, error: %v", addr, err)
} else {
klog.V(4).Infof("Successfully dialed Unix socket %v", addr)
}
return c, err
}))

181
staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service_unix_test.go
View File

@ -20,44 +20,56 @@ limitations under the License.
package envelope
import (
"context"
"encoding/base64"
"fmt"
"net"
"os"
"reflect"
"runtime"
"strings"
"sync"
"testing"
"time"
"google.golang.org/grpc"
mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing"
"k8s.io/apimachinery/pkg/util/uuid"
kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
)
type testSocket struct {
path string
endpoint string
}
// newEndpoint constructs a unique name for a Linux Abstract Socket to be used in a test.
// This package uses Linux Domain Sockets to remove the need for clean-up of socket files.
func newEndpoint() *testSocket {
p := fmt.Sprintf("@%s.sock", uuid.NewUUID())
return &testSocket{
path: p,
endpoint: fmt.Sprintf("unix:///%s", p),
}
}
// TestKMSPluginLateStart tests the scenario where kms-plugin pod/container starts after kube-apiserver pod/container.
// Since the Dial to kms-plugin is non-blocking we expect the construction of gRPC service to succeed even when
// kms-plugin is not yet up - dialing happens in the background.
func TestKMSPluginLateStart(t *testing.T) {
t.Parallel()
callTimeout := 3 * time.Second
endpoint := getSocketName()
s := newEndpoint()
service, err := NewGRPCService(endpoint, callTimeout)
service, err := NewGRPCService(s.endpoint, callTimeout)
if err != nil {
t.Fatalf("failed to create envelope service, error: %v", err)
}
defer destroyService(service)
time.Sleep(callTimeout / 2)
f, err := startFakeKMSProvider(kmsapiVersion, endpoint)
f, err := mock.NewBase64Plugin(s.path)
if err != nil {
t.Fatalf("failed to start test KMS provider server, error: %v", err)
}
defer f.Stop()
if err := f.Start(); err != nil {
t.Fatalf("Failed to start kms-plugin, err: %v", err)
}
defer f.CleanUp()
data := []byte("test data")
_, err = service.Encrypt(data)
@ -113,7 +125,7 @@ func TestTimeouts(t *testing.T) {
kubeAPIServerWG sync.WaitGroup
kmsPluginWG sync.WaitGroup
testCompletedWG sync.WaitGroup
socketName = getSocketName()
socketName = newEndpoint()
)
testCompletedWG.Add(1)
@ -124,7 +136,7 @@ func TestTimeouts(t *testing.T) {
// Simulating late start of kube-apiserver - plugin is up before kube-apiserver, if requested by the testcase.
time.Sleep(tt.kubeAPIServerDelay)
service, err = NewGRPCService(socketName, tt.callTimeout)
service, err = NewGRPCService(socketName.endpoint, tt.callTimeout)
if err != nil {
t.Fatalf("failed to create envelope service, error: %v", err)
}
@ -139,11 +151,14 @@ func TestTimeouts(t *testing.T) {
// Simulating delayed start of kms-plugin, kube-apiserver is up before the plugin, if requested by the testcase.
time.Sleep(tt.pluginDelay)
f, err := startFakeKMSProvider(kmsapiVersion, socketName)
f, err := mock.NewBase64Plugin(socketName.path)
if err != nil {
t.Fatalf("failed to start test KMS provider server, error: %v", err)
t.Fatalf("failed to construct test KMS provider server, error: %v", err)
}
defer f.Stop()
if err := f.Start(); err != nil {
t.Fatalf("Failed to start test KMS provider server, error: %v", err)
}
defer f.CleanUp()
kmsPluginWG.Done()
// Keeping plugin up to process requests.
testCompletedWG.Wait()
@ -175,16 +190,19 @@ func TestIntermittentConnectionLoss(t *testing.T) {
timeout = 30 * time.Second
blackOut = 1 * time.Second
data = []byte("test data")
endpoint = getSocketName()
endpoint = newEndpoint()
)
// Start KMS Plugin
f, err := startFakeKMSProvider(kmsapiVersion, endpoint)
f, err := mock.NewBase64Plugin(endpoint.path)
if err != nil {
t.Fatalf("failed to start test KMS provider server, error: %v", err)
}
if err := f.Start(); err != nil {
t.Fatalf("Failed to start kms-plugin, err: %v", err)
}
// connect to kms plugin
service, err := NewGRPCService(endpoint, timeout)
service, err := NewGRPCService(endpoint.endpoint, timeout)
if err != nil {
t.Fatalf("failed to create envelope service, error: %v", err)
}
@ -198,7 +216,7 @@ func TestIntermittentConnectionLoss(t *testing.T) {
// Stop KMS Plugin - simulating connection loss
t.Log("KMS Plugin is stopping")
f.Stop()
f.CleanUp()
time.Sleep(2 * time.Second)
wg1.Add(1)
@ -217,11 +235,14 @@ func TestIntermittentConnectionLoss(t *testing.T) {
wg1.Wait()
time.Sleep(blackOut)
// Start KMS Plugin
f, err = startFakeKMSProvider(kmsapiVersion, endpoint)
f, err = mock.NewBase64Plugin(endpoint.path)
if err != nil {
t.Fatalf("failed to start test KMS provider server, error: %v", err)
}
defer f.Stop()
if err := f.Start(); err != nil {
t.Fatalf("Failed to start kms-plugin, err: %v", err)
}
defer f.CleanUp()
t.Log("Restarted KMS Plugin")
wg2.Wait()
@ -232,15 +253,19 @@ func TestUnsupportedVersion(t *testing.T) {
ver := "invalid"
data := []byte("test data")
wantErr := fmt.Errorf(versionErrorf, ver, kmsapiVersion)
endpoint := getSocketName()
endpoint := newEndpoint()
f, err := startFakeKMSProvider(ver, endpoint)
f, err := mock.NewBase64Plugin(endpoint.path)
if err != nil {
t.Fatalf("failed to start test KMS provider server, error: %ver", err)
}
defer f.Stop()
f.SetVersion(ver)
if err := f.Start(); err != nil {
t.Fatalf("Failed to start kms-plugin, err: %v", err)
}
defer f.CleanUp()
s, err := NewGRPCService(endpoint, 1*time.Second)
s, err := NewGRPCService(endpoint.endpoint, 1*time.Second)
if err != nil {
t.Fatal(err)
}
@ -254,7 +279,7 @@ func TestUnsupportedVersion(t *testing.T) {
destroyService(s)
s, err = NewGRPCService(endpoint, 1*time.Second)
s, err = NewGRPCService(endpoint.endpoint, 1*time.Second)
if err != nil {
t.Fatal(err)
}
@ -271,15 +296,18 @@ func TestUnsupportedVersion(t *testing.T) {
func TestGRPCService(t *testing.T) {
t.Parallel()
// Start a test gRPC server.
endpoint := getSocketName()
f, err := startFakeKMSProvider(kmsapiVersion, endpoint)
endpoint := newEndpoint()
f, err := mock.NewBase64Plugin(endpoint.path)
if err != nil {
t.Fatalf("failed to start test KMS provider server, error: %v", err)
t.Fatalf("failed to construct test KMS provider server, error: %v", err)
}
defer f.Stop()
if err := f.Start(); err != nil {
t.Fatalf("Failed to start kms-plugin, err: %v", err)
}
defer f.CleanUp()
// Create the gRPC client service.
service, err := NewGRPCService(endpoint, 1*time.Second)
service, err := NewGRPCService(endpoint.endpoint, 1*time.Second)
if err != nil {
t.Fatalf("failed to create envelope service, error: %v", err)
}
@ -307,15 +335,18 @@ func TestGRPCService(t *testing.T) {
func TestGRPCServiceConcurrentAccess(t *testing.T) {
t.Parallel()
// Start a test gRPC server.
endpoint := getSocketName()
f, err := startFakeKMSProvider(kmsapiVersion, endpoint)
endpoint := newEndpoint()
f, err := mock.NewBase64Plugin(endpoint.path)
if err != nil {
t.Fatalf("failed to start test KMS provider server, error: %v", err)
}
defer f.Stop()
if err := f.Start(); err != nil {
t.Fatalf("Failed to start kms-plugin, err: %v", err)
}
defer f.CleanUp()
// Create the gRPC client service.
service, err := NewGRPCService(endpoint, 15*time.Second)
service, err := NewGRPCService(endpoint.endpoint, 15*time.Second)
if err != nil {
t.Fatalf("failed to create envelope service, error: %v", err)
}
@ -356,32 +387,29 @@ func destroyService(service Service) {
}
}
func getSocketName() string {
return fmt.Sprintf("unix:///@%s.sock", uuid.NewUUID())
}
// Test all those invalid configuration for KMS provider.
func TestInvalidConfiguration(t *testing.T) {
t.Parallel()
// Start a test gRPC server.
f, err := startFakeKMSProvider(kmsapiVersion, getSocketName())
f, err := mock.NewBase64Plugin(newEndpoint().path)
if err != nil {
t.Fatalf("failed to start test KMS provider server, error: %v", err)
}
defer f.Stop()
if err := f.Start(); err != nil {
t.Fatalf("Failed to start kms-plugin, err: %v", err)
}
defer f.CleanUp()
invalidConfigs := []struct {
name string
apiVersion string
endpoint string
name string
endpoint string
}{
{"emptyConfiguration", kmsapiVersion, ""},
{"invalidScheme", kmsapiVersion, "tcp://localhost:6060"},
{"emptyConfiguration", ""},
{"invalidScheme", "tcp://localhost:6060"},
}
for _, testCase := range invalidConfigs {
t.Run(testCase.name, func(t *testing.T) {
f.apiVersion = testCase.apiVersion
_, err := NewGRPCService(testCase.endpoint, 1*time.Second)
if err == nil {
t.Fatalf("should fail to create envelope service for %s.", testCase.name)
@ -389,58 +417,3 @@ func TestInvalidConfiguration(t *testing.T) {
})
}
}
// Start the gRPC server that listens on unix socket.
func startFakeKMSProvider(version, endpoint string) (*fakeKMSPlugin, error) {
sockFile, err := parseEndpoint(endpoint)
if err != nil {
return nil, fmt.Errorf("failed to parse endpoint:%q, error %v", endpoint, err)
}
listener, err := net.Listen(unixProtocol, sockFile)
if err != nil {
return nil, fmt.Errorf("failed to listen on the unix socket, error: %v", err)
}
s := grpc.NewServer()
f := &fakeKMSPlugin{apiVersion: version, server: s, sockFile: sockFile}
kmsapi.RegisterKeyManagementServiceServer(s, f)
go s.Serve(listener)
return f, nil
}
// Fake gRPC sever for remote KMS provider.
// Use base64 to simulate encrypt and decrypt.
type fakeKMSPlugin struct {
apiVersion string
server *grpc.Server
sockFile string
}
func (s *fakeKMSPlugin) Stop() {
// Stop the server
s.server.Stop()
// If this isn't a Linux abstract namespace socket, or if we're on a non-linux platform, clean up the socket file
if !strings.HasPrefix(s.sockFile, "@") || runtime.GOOS != "linux" {
os.Remove(s.sockFile)
}
}
func (s *fakeKMSPlugin) Version(ctx context.Context, request *kmsapi.VersionRequest) (*kmsapi.VersionResponse, error) {
return &kmsapi.VersionResponse{Version: s.apiVersion, RuntimeName: "testKMS", RuntimeVersion: "0.0.1"}, nil
}
func (s *fakeKMSPlugin) Decrypt(ctx context.Context, request *kmsapi.DecryptRequest) (*kmsapi.DecryptResponse, error) {
buf := make([]byte, base64.StdEncoding.DecodedLen(len(request.Cipher)))
n, err := base64.StdEncoding.Decode(buf, request.Cipher)
if err != nil {
return nil, err
}
return &kmsapi.DecryptResponse{Plain: buf[:n]}, nil
}
func (s *fakeKMSPlugin) Encrypt(ctx context.Context, request *kmsapi.EncryptRequest) (*kmsapi.EncryptResponse, error) {
buf := make([]byte, base64.StdEncoding.EncodedLen(len(request.Plain)))
base64.StdEncoding.Encode(buf, request.Plain)
return &kmsapi.EncryptResponse{Cipher: buf}, nil
}

106
staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/BUILD Normal file
View File

@ -0,0 +1,106 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["kms_plugin_mock.go"],
importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing",
importpath = "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing",
visibility = ["//visibility:public"],
deps = select({
"@io_bazel_rules_go//go/platform:android": [
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
"//conditions:default": [],
}),
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

176
staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/kms_plugin_mock.go Normal file
View File

@ -0,0 +1,176 @@
// +build !windows
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package testing
import (
"context"
"encoding/base64"
"fmt"
"net"
"os"
"runtime"
"strings"
"sync"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"k8s.io/apimachinery/pkg/util/wait"
kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
"k8s.io/klog"
)
const (
// Now only supported unix domain socket.
unixProtocol = "unix"
// Current version for the protocol interface definition.
kmsapiVersion = "v1beta1"
)
// Base64Plugin gRPC sever for a mock KMS provider.
// Uses base64 to simulate encrypt and decrypt.
type Base64Plugin struct {
grpcServer *grpc.Server
listener net.Listener
mu *sync.Mutex
lastEncryptRequest *kmsapi.EncryptRequest
inFailedState bool
ver string
socketPath string
}
// NewBase64Plugin is a constructor for Base64Plugin.
func NewBase64Plugin(socketPath string) (*Base64Plugin, error) {
server := grpc.NewServer()
result := &Base64Plugin{
grpcServer: server,
mu: &sync.Mutex{},
ver: kmsapiVersion,
socketPath: socketPath,
}
kmsapi.RegisterKeyManagementServiceServer(server, result)
return result, nil
}
// WaitForBase64PluginToBeUp waits until the plugin is ready to serve requests.
func WaitForBase64PluginToBeUp(plugin *Base64Plugin) error {
var gRPCErr error
pollErr := wait.PollImmediate(1*time.Second, wait.ForeverTestTimeout, func() (bool, error) {
_, gRPCErr = plugin.Encrypt(context.Background(), &kmsapi.EncryptRequest{Plain: []byte("foo")})
return gRPCErr == nil, nil
})
if pollErr == wait.ErrWaitTimeout {
return fmt.Errorf("failed to start kms-plugin, error: %v", gRPCErr)
}
return nil
}
// LastEncryptRequest returns the last EncryptRequest.Plain sent to the plugin.
func (s *Base64Plugin) LastEncryptRequest() []byte {
return s.lastEncryptRequest.Plain
}
// SetVersion sets the version of kms-plugin.
func (s *Base64Plugin) SetVersion(ver string) {
s.ver = ver
}
// Start starts plugin's gRPC service.
func (s *Base64Plugin) Start() error {
var err error
s.listener, err = net.Listen(unixProtocol, s.socketPath)
if err != nil {
return fmt.Errorf("failed to listen on the unix socket, error: %v", err)
}
klog.Infof("Listening on %s", s.socketPath)
go s.grpcServer.Serve(s.listener)
return nil
}
// CleanUp stops gRPC server and the underlying listener.
func (s *Base64Plugin) CleanUp() {
s.grpcServer.Stop()
s.listener.Close()
if !strings.HasPrefix(s.socketPath, "@") || runtime.GOOS != "linux" {
os.Remove(s.socketPath)
}
}
// EnterFailedState places the plugin into failed state.
func (s *Base64Plugin) EnterFailedState() {
s.mu.Lock()
defer s.mu.Unlock()
s.inFailedState = true
}
// ExitFailedState removes the plugin from the failed state.
func (s *Base64Plugin) ExitFailedState() {
s.mu.Lock()
defer s.mu.Unlock()
s.inFailedState = false
}
// Version returns the version of the kms-plugin.
func (s *Base64Plugin) Version(ctx context.Context, request *kmsapi.VersionRequest) (*kmsapi.VersionResponse, error) {
klog.Infof("Received request for Version: %v", request)
return &kmsapi.VersionResponse{Version: s.ver, RuntimeName: "testKMS", RuntimeVersion: "0.0.1"}, nil
}
// Decrypt performs base64 decoding of the payload of kms.DecryptRequest.
func (s *Base64Plugin) Decrypt(ctx context.Context, request *kmsapi.DecryptRequest) (*kmsapi.DecryptResponse, error) {
klog.V(3).Infof("Received Decrypt Request for DEK: %s", string(request.Cipher))
s.mu.Lock()
defer s.mu.Unlock()
if s.inFailedState {
return nil, status.Error(codes.FailedPrecondition, "failed precondition - key disabled")
}
buf := make([]byte, base64.StdEncoding.DecodedLen(len(request.Cipher)))
n, err := base64.StdEncoding.Decode(buf, request.Cipher)
if err != nil {
return nil, err
}
return &kmsapi.DecryptResponse{Plain: buf[:n]}, nil
}
// Encrypt performs base64 encoding of the payload of kms.EncryptRequest.
func (s *Base64Plugin) Encrypt(ctx context.Context, request *kmsapi.EncryptRequest) (*kmsapi.EncryptResponse, error) {
klog.V(3).Infof("Received Encrypt Request for DEK: %x", request.Plain)
s.mu.Lock()
defer s.mu.Unlock()
s.lastEncryptRequest = request
if s.inFailedState {
return nil, status.Error(codes.FailedPrecondition, "failed precondition - key disabled")
}
buf := make([]byte, base64.StdEncoding.EncodedLen(len(request.Plain)))
base64.StdEncoding.Encode(buf, request.Plain)
return &kmsapi.EncryptResponse{Cipher: buf}, nil
}

79
test/integration/master/BUILD
View File

@ -74,33 +74,43 @@ go_test(
"//vendor/sigs.k8s.io/yaml:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:android": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
],
"//conditions:default": [],
@ -122,10 +132,7 @@ filegroup(
go_library(
name = "go_default_library",
srcs = [
"kms_plugin_mock.go",
"transformation_testcase.go",
],
srcs = ["transformation_testcase.go"],
importpath = "k8s.io/kubernetes/test/integration/master",
deps = [
"//cmd/kube-apiserver/app/testing:go_default_library",
@ -141,67 +148,5 @@ go_library(
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/sigs.k8s.io/yaml:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:android": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
],
"//conditions:default": [],
}),
],
)

126
test/integration/master/kms_plugin_mock.go
View File

@ -1,126 +0,0 @@
// +build !windows
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package master
import (
"context"
"encoding/base64"
"fmt"
"net"
"sync"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
"k8s.io/klog"
)
const (
kmsAPIVersion = "v1beta1"
unixProtocol = "unix"
)
// base64Plugin gRPC sever for a mock KMS provider.
// Uses base64 to simulate encrypt and decrypt.
type base64Plugin struct {
grpcServer *grpc.Server
listener net.Listener
mu *sync.Mutex
lastEncryptRequest *kmsapi.EncryptRequest
inFailedState bool
}
func newBase64Plugin(socketPath string) (*base64Plugin, error) {
listener, err := net.Listen(unixProtocol, socketPath)
if err != nil {
return nil, fmt.Errorf("failed to listen on the unix socket, error: %v", err)
}
klog.Infof("Listening on %s", socketPath)
server := grpc.NewServer()
result := &base64Plugin{
grpcServer: server,
listener: listener,
mu: &sync.Mutex{},
}
kmsapi.RegisterKeyManagementServiceServer(server, result)
return result, nil
}
func (s *base64Plugin) cleanUp() {
s.grpcServer.Stop()
s.listener.Close()
}
var testProviderAPIVersion = kmsAPIVersion
func (s *base64Plugin) enterFailedState() {
s.mu.Lock()
defer s.mu.Unlock()
s.inFailedState = true
}
func (s *base64Plugin) exitFailedState() {
s.mu.Lock()
defer s.mu.Unlock()
s.inFailedState = false
}
func (s *base64Plugin) Version(ctx context.Context, request *kmsapi.VersionRequest) (*kmsapi.VersionResponse, error) {
return &kmsapi.VersionResponse{Version: testProviderAPIVersion, RuntimeName: "testKMS", RuntimeVersion: "0.0.1"}, nil
}
func (s *base64Plugin) Decrypt(ctx context.Context, request *kmsapi.DecryptRequest) (*kmsapi.DecryptResponse, error) {
klog.Infof("Received Decrypt Request for DEK: %s", string(request.Cipher))
s.mu.Lock()
defer s.mu.Unlock()
if s.inFailedState {
return nil, status.Error(codes.FailedPrecondition, "failed precondition - key disabled")
}
buf := make([]byte, base64.StdEncoding.DecodedLen(len(request.Cipher)))
n, err := base64.StdEncoding.Decode(buf, request.Cipher)
if err != nil {
return nil, err
}
return &kmsapi.DecryptResponse{Plain: buf[:n]}, nil
}
func (s *base64Plugin) Encrypt(ctx context.Context, request *kmsapi.EncryptRequest) (*kmsapi.EncryptResponse, error) {
klog.Infof("Received Encrypt Request for DEK: %x", request.Plain)
s.mu.Lock()
defer s.mu.Unlock()
s.lastEncryptRequest = request
if s.inFailedState {
return nil, status.Error(codes.FailedPrecondition, "failed precondition - key disabled")
}
buf := make([]byte, base64.StdEncoding.EncodedLen(len(request.Plain)))
base64.StdEncoding.Encode(buf, request.Plain)
return &kmsapi.EncryptResponse{Cipher: buf}, nil
}

67
test/integration/master/kms_transformation_test.go
View File

@ -23,8 +23,8 @@ import (
"context"
"crypto/aes"
"encoding/binary"
"fmt"
"fmt"
"net/http"
"strings"
"testing"
@ -34,6 +34,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/storage/value"
aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing"
kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
@ -41,6 +42,7 @@ import (
const (
dekKeySizeLen = 2
kmsAPIVersion = "v1beta1"
)
type envelope struct {
@ -112,14 +114,16 @@ resources:
`
providerName := "kms-provider"
pluginMock, err := newBase64Plugin("@kms-provider.sock")
pluginMock, err := mock.NewBase64Plugin("@kms-provider.sock")
if err != nil {
t.Fatalf("failed to create mock of KMS Plugin: %v", err)
}
go pluginMock.grpcServer.Serve(pluginMock.listener)
defer pluginMock.cleanUp()
kmsPluginMustBeUp(t, pluginMock)
go pluginMock.Start()
if err := mock.WaitForBase64PluginToBeUp(pluginMock); err != nil {
t.Fatalf("Failed start plugin, err: %v", err)
}
defer pluginMock.CleanUp()
test, err := newTransformTest(t, encryptionConfig)
if err != nil {
@ -133,10 +137,7 @@ resources:
}
// Since Data Encryption Key (DEK) is randomly generated (per encryption operation), we need to ask KMS Mock for it.
plainTextDEK := pluginMock.lastEncryptRequest.Plain
if err != nil {
t.Fatalf("failed to get DEK from KMS: %v", err)
}
plainTextDEK := pluginMock.LastEncryptRequest()
secretETCDPath := test.getETCDPath()
rawEnvelope, err := test.getRawSecretFromETCD()
@ -197,23 +198,30 @@ resources:
endpoint: unix:///@kms-provider-2.sock
`
pluginMock1, err := newBase64Plugin("@kms-provider-1.sock")
pluginMock1, err := mock.NewBase64Plugin("@kms-provider-1.sock")
if err != nil {
t.Fatalf("failed to create mock of KMS Plugin #1: %v", err)
}
go pluginMock1.grpcServer.Serve(pluginMock1.listener)
defer pluginMock1.cleanUp()
kmsPluginMustBeUp(t, pluginMock1)
pluginMock2, err := newBase64Plugin("@kms-provider-2.sock")
if err != nil {
t.Fatalf("failed to create mock of KMS Plugin #2: %v", err)
if err := pluginMock1.Start(); err != nil {
t.Fatalf("Failed to start kms-plugin, err: %v", err)
}
defer pluginMock1.CleanUp()
if err := mock.WaitForBase64PluginToBeUp(pluginMock1); err != nil {
t.Fatalf("Failed to start plugin #1, err: %v", err)
}
go pluginMock2.grpcServer.Serve(pluginMock2.listener)
defer pluginMock2.cleanUp()
kmsPluginMustBeUp(t, pluginMock2)
pluginMock2, err := mock.NewBase64Plugin("@kms-provider-2.sock")
if err != nil {
t.Fatalf("Failed to create mock of KMS Plugin #2: err: %v", err)
}
if err := pluginMock2.Start(); err != nil {
t.Fatalf("Failed to start kms-plugin, err: %v", err)
}
defer pluginMock2.CleanUp()
if err := mock.WaitForBase64PluginToBeUp(pluginMock2); err != nil {
t.Fatalf("Failed to start KMS Plugin #2: err: %v", err)
}
test, err := newTransformTest(t, encryptionConfig)
if err != nil {
@ -231,32 +239,19 @@ resources:
// Stage 2 - kms-plugin for provider-1 is down. Therefore, expect the health check for provider-1
// to fail, but provider-2 should still be OK
pluginMock1.enterFailedState()
pluginMock1.EnterFailedState()
mustBeUnHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
mustBeHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
pluginMock1.exitFailedState()
pluginMock1.ExitFailedState()
// Stage 3 - kms-plugin for provider-1 is now up. Therefore, expect the health check for provider-1
// to succeed now, but provider-2 is now down.
// Need to sleep since health check chases responses for 3 seconds.
pluginMock2.enterFailedState()
pluginMock2.EnterFailedState()
mustBeHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
mustBeUnHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
}
func kmsPluginMustBeUp(t *testing.T, plugin *base64Plugin) {
t.Helper()
var gRPCErr error
pollErr := wait.PollImmediate(1*time.Second, wait.ForeverTestTimeout, func() (bool, error) {
_, gRPCErr = plugin.Encrypt(context.Background(), &kmsapi.EncryptRequest{Plain: []byte("foo")})
return gRPCErr == nil, nil
})
if pollErr == wait.ErrWaitTimeout {
t.Fatalf("failed to start kms-plugin, error: %v", gRPCErr)
}
}
func mustBeHealthy(t *testing.T, checkName string, clientConfig *rest.Config) {
t.Helper()
var restErr error

1
vendor/modules.txt vendored
View File

@ -1363,6 +1363,7 @@ k8s.io/apiserver/pkg/storage/testing
k8s.io/apiserver/pkg/storage/value
k8s.io/apiserver/pkg/storage/value/encrypt/aes
k8s.io/apiserver/pkg/storage/value/encrypt/envelope
k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing
k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1
k8s.io/apiserver/pkg/storage/value/encrypt/identity
k8s.io/apiserver/pkg/storage/value/encrypt/secretbox