diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/BUILD index 68a3cdceda9..c921c4da7c7 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/BUILD @@ -37,53 +37,43 @@ go_test( ] + select({ "@io_bazel_rules_go//go/platform:android": [ "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", ], "@io_bazel_rules_go//go/platform:darwin": [ "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", ], "@io_bazel_rules_go//go/platform:dragonfly": [ "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", ], "@io_bazel_rules_go//go/platform:freebsd": [ "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", ], "@io_bazel_rules_go//go/platform:linux": [ "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", ], "@io_bazel_rules_go//go/platform:nacl": [ "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", ], "@io_bazel_rules_go//go/platform:netbsd": [ "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", ], "@io_bazel_rules_go//go/platform:openbsd": [ "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", ], "@io_bazel_rules_go//go/platform:plan9": [ "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", ], "@io_bazel_rules_go//go/platform:solaris": [ "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", ], "//conditions:default": [], }), @@ -100,6 +90,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:all-srcs", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:all-srcs", ], tags = ["automanaged"], diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service.go index eae05a7b696..a31f46e8fc7 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service.go @@ -68,6 +68,8 @@ func NewGRPCService(endpoint string, callTimeout time.Duration) (Service, error) c, err := net.DialUnix(unixProtocol, nil, &net.UnixAddr{Name: addr}) if err != nil { klog.Errorf("failed to create connection to unix socket: %s, error: %v", addr, err) + } else { + klog.V(4).Infof("Successfully dialed Unix socket %v", addr) } return c, err })) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service_unix_test.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service_unix_test.go index 9a828931398..8ae057d95ef 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service_unix_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service_unix_test.go @@ -20,44 +20,56 @@ limitations under the License. package envelope import ( - "context" - "encoding/base64" "fmt" - "net" - "os" "reflect" - "runtime" - "strings" "sync" "testing" "time" - "google.golang.org/grpc" + mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing" "k8s.io/apimachinery/pkg/util/uuid" - kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" ) +type testSocket struct { + path string + endpoint string +} + +// newEndpoint constructs a unique name for a Linux Abstract Socket to be used in a test. +// This package uses Linux Domain Sockets to remove the need for clean-up of socket files. +func newEndpoint() *testSocket { + p := fmt.Sprintf("@%s.sock", uuid.NewUUID()) + + return &testSocket{ + path: p, + endpoint: fmt.Sprintf("unix:///%s", p), + } +} + // TestKMSPluginLateStart tests the scenario where kms-plugin pod/container starts after kube-apiserver pod/container. // Since the Dial to kms-plugin is non-blocking we expect the construction of gRPC service to succeed even when // kms-plugin is not yet up - dialing happens in the background. func TestKMSPluginLateStart(t *testing.T) { t.Parallel() callTimeout := 3 * time.Second - endpoint := getSocketName() + s := newEndpoint() - service, err := NewGRPCService(endpoint, callTimeout) + service, err := NewGRPCService(s.endpoint, callTimeout) if err != nil { t.Fatalf("failed to create envelope service, error: %v", err) } defer destroyService(service) time.Sleep(callTimeout / 2) - f, err := startFakeKMSProvider(kmsapiVersion, endpoint) + f, err := mock.NewBase64Plugin(s.path) if err != nil { t.Fatalf("failed to start test KMS provider server, error: %v", err) } - defer f.Stop() + if err := f.Start(); err != nil { + t.Fatalf("Failed to start kms-plugin, err: %v", err) + } + defer f.CleanUp() data := []byte("test data") _, err = service.Encrypt(data) @@ -113,7 +125,7 @@ func TestTimeouts(t *testing.T) { kubeAPIServerWG sync.WaitGroup kmsPluginWG sync.WaitGroup testCompletedWG sync.WaitGroup - socketName = getSocketName() + socketName = newEndpoint() ) testCompletedWG.Add(1) @@ -124,7 +136,7 @@ func TestTimeouts(t *testing.T) { // Simulating late start of kube-apiserver - plugin is up before kube-apiserver, if requested by the testcase. time.Sleep(tt.kubeAPIServerDelay) - service, err = NewGRPCService(socketName, tt.callTimeout) + service, err = NewGRPCService(socketName.endpoint, tt.callTimeout) if err != nil { t.Fatalf("failed to create envelope service, error: %v", err) } @@ -139,11 +151,14 @@ func TestTimeouts(t *testing.T) { // Simulating delayed start of kms-plugin, kube-apiserver is up before the plugin, if requested by the testcase. time.Sleep(tt.pluginDelay) - f, err := startFakeKMSProvider(kmsapiVersion, socketName) + f, err := mock.NewBase64Plugin(socketName.path) if err != nil { - t.Fatalf("failed to start test KMS provider server, error: %v", err) + t.Fatalf("failed to construct test KMS provider server, error: %v", err) } - defer f.Stop() + if err := f.Start(); err != nil { + t.Fatalf("Failed to start test KMS provider server, error: %v", err) + } + defer f.CleanUp() kmsPluginWG.Done() // Keeping plugin up to process requests. testCompletedWG.Wait() @@ -175,16 +190,19 @@ func TestIntermittentConnectionLoss(t *testing.T) { timeout = 30 * time.Second blackOut = 1 * time.Second data = []byte("test data") - endpoint = getSocketName() + endpoint = newEndpoint() ) // Start KMS Plugin - f, err := startFakeKMSProvider(kmsapiVersion, endpoint) + f, err := mock.NewBase64Plugin(endpoint.path) if err != nil { t.Fatalf("failed to start test KMS provider server, error: %v", err) } + if err := f.Start(); err != nil { + t.Fatalf("Failed to start kms-plugin, err: %v", err) + } // connect to kms plugin - service, err := NewGRPCService(endpoint, timeout) + service, err := NewGRPCService(endpoint.endpoint, timeout) if err != nil { t.Fatalf("failed to create envelope service, error: %v", err) } @@ -198,7 +216,7 @@ func TestIntermittentConnectionLoss(t *testing.T) { // Stop KMS Plugin - simulating connection loss t.Log("KMS Plugin is stopping") - f.Stop() + f.CleanUp() time.Sleep(2 * time.Second) wg1.Add(1) @@ -217,11 +235,14 @@ func TestIntermittentConnectionLoss(t *testing.T) { wg1.Wait() time.Sleep(blackOut) // Start KMS Plugin - f, err = startFakeKMSProvider(kmsapiVersion, endpoint) + f, err = mock.NewBase64Plugin(endpoint.path) if err != nil { t.Fatalf("failed to start test KMS provider server, error: %v", err) } - defer f.Stop() + if err := f.Start(); err != nil { + t.Fatalf("Failed to start kms-plugin, err: %v", err) + } + defer f.CleanUp() t.Log("Restarted KMS Plugin") wg2.Wait() @@ -232,15 +253,19 @@ func TestUnsupportedVersion(t *testing.T) { ver := "invalid" data := []byte("test data") wantErr := fmt.Errorf(versionErrorf, ver, kmsapiVersion) - endpoint := getSocketName() + endpoint := newEndpoint() - f, err := startFakeKMSProvider(ver, endpoint) + f, err := mock.NewBase64Plugin(endpoint.path) if err != nil { t.Fatalf("failed to start test KMS provider server, error: %ver", err) } - defer f.Stop() + f.SetVersion(ver) + if err := f.Start(); err != nil { + t.Fatalf("Failed to start kms-plugin, err: %v", err) + } + defer f.CleanUp() - s, err := NewGRPCService(endpoint, 1*time.Second) + s, err := NewGRPCService(endpoint.endpoint, 1*time.Second) if err != nil { t.Fatal(err) } @@ -254,7 +279,7 @@ func TestUnsupportedVersion(t *testing.T) { destroyService(s) - s, err = NewGRPCService(endpoint, 1*time.Second) + s, err = NewGRPCService(endpoint.endpoint, 1*time.Second) if err != nil { t.Fatal(err) } @@ -271,15 +296,18 @@ func TestUnsupportedVersion(t *testing.T) { func TestGRPCService(t *testing.T) { t.Parallel() // Start a test gRPC server. - endpoint := getSocketName() - f, err := startFakeKMSProvider(kmsapiVersion, endpoint) + endpoint := newEndpoint() + f, err := mock.NewBase64Plugin(endpoint.path) if err != nil { - t.Fatalf("failed to start test KMS provider server, error: %v", err) + t.Fatalf("failed to construct test KMS provider server, error: %v", err) } - defer f.Stop() + if err := f.Start(); err != nil { + t.Fatalf("Failed to start kms-plugin, err: %v", err) + } + defer f.CleanUp() // Create the gRPC client service. - service, err := NewGRPCService(endpoint, 1*time.Second) + service, err := NewGRPCService(endpoint.endpoint, 1*time.Second) if err != nil { t.Fatalf("failed to create envelope service, error: %v", err) } @@ -307,15 +335,18 @@ func TestGRPCService(t *testing.T) { func TestGRPCServiceConcurrentAccess(t *testing.T) { t.Parallel() // Start a test gRPC server. - endpoint := getSocketName() - f, err := startFakeKMSProvider(kmsapiVersion, endpoint) + endpoint := newEndpoint() + f, err := mock.NewBase64Plugin(endpoint.path) if err != nil { t.Fatalf("failed to start test KMS provider server, error: %v", err) } - defer f.Stop() + if err := f.Start(); err != nil { + t.Fatalf("Failed to start kms-plugin, err: %v", err) + } + defer f.CleanUp() // Create the gRPC client service. - service, err := NewGRPCService(endpoint, 15*time.Second) + service, err := NewGRPCService(endpoint.endpoint, 15*time.Second) if err != nil { t.Fatalf("failed to create envelope service, error: %v", err) } @@ -356,32 +387,29 @@ func destroyService(service Service) { } } -func getSocketName() string { - return fmt.Sprintf("unix:///@%s.sock", uuid.NewUUID()) -} - // Test all those invalid configuration for KMS provider. func TestInvalidConfiguration(t *testing.T) { t.Parallel() // Start a test gRPC server. - f, err := startFakeKMSProvider(kmsapiVersion, getSocketName()) + f, err := mock.NewBase64Plugin(newEndpoint().path) if err != nil { t.Fatalf("failed to start test KMS provider server, error: %v", err) } - defer f.Stop() + if err := f.Start(); err != nil { + t.Fatalf("Failed to start kms-plugin, err: %v", err) + } + defer f.CleanUp() invalidConfigs := []struct { - name string - apiVersion string - endpoint string + name string + endpoint string }{ - {"emptyConfiguration", kmsapiVersion, ""}, - {"invalidScheme", kmsapiVersion, "tcp://localhost:6060"}, + {"emptyConfiguration", ""}, + {"invalidScheme", "tcp://localhost:6060"}, } for _, testCase := range invalidConfigs { t.Run(testCase.name, func(t *testing.T) { - f.apiVersion = testCase.apiVersion _, err := NewGRPCService(testCase.endpoint, 1*time.Second) if err == nil { t.Fatalf("should fail to create envelope service for %s.", testCase.name) @@ -389,58 +417,3 @@ func TestInvalidConfiguration(t *testing.T) { }) } } - -// Start the gRPC server that listens on unix socket. -func startFakeKMSProvider(version, endpoint string) (*fakeKMSPlugin, error) { - sockFile, err := parseEndpoint(endpoint) - if err != nil { - return nil, fmt.Errorf("failed to parse endpoint:%q, error %v", endpoint, err) - } - listener, err := net.Listen(unixProtocol, sockFile) - if err != nil { - return nil, fmt.Errorf("failed to listen on the unix socket, error: %v", err) - } - - s := grpc.NewServer() - f := &fakeKMSPlugin{apiVersion: version, server: s, sockFile: sockFile} - kmsapi.RegisterKeyManagementServiceServer(s, f) - go s.Serve(listener) - return f, nil -} - -// Fake gRPC sever for remote KMS provider. -// Use base64 to simulate encrypt and decrypt. -type fakeKMSPlugin struct { - apiVersion string - server *grpc.Server - sockFile string -} - -func (s *fakeKMSPlugin) Stop() { - // Stop the server - s.server.Stop() - // If this isn't a Linux abstract namespace socket, or if we're on a non-linux platform, clean up the socket file - if !strings.HasPrefix(s.sockFile, "@") || runtime.GOOS != "linux" { - os.Remove(s.sockFile) - } -} - -func (s *fakeKMSPlugin) Version(ctx context.Context, request *kmsapi.VersionRequest) (*kmsapi.VersionResponse, error) { - return &kmsapi.VersionResponse{Version: s.apiVersion, RuntimeName: "testKMS", RuntimeVersion: "0.0.1"}, nil -} - -func (s *fakeKMSPlugin) Decrypt(ctx context.Context, request *kmsapi.DecryptRequest) (*kmsapi.DecryptResponse, error) { - buf := make([]byte, base64.StdEncoding.DecodedLen(len(request.Cipher))) - n, err := base64.StdEncoding.Decode(buf, request.Cipher) - if err != nil { - return nil, err - } - - return &kmsapi.DecryptResponse{Plain: buf[:n]}, nil -} - -func (s *fakeKMSPlugin) Encrypt(ctx context.Context, request *kmsapi.EncryptRequest) (*kmsapi.EncryptResponse, error) { - buf := make([]byte, base64.StdEncoding.EncodedLen(len(request.Plain))) - base64.StdEncoding.Encode(buf, request.Plain) - return &kmsapi.EncryptResponse{Cipher: buf}, nil -} diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/BUILD new file mode 100644 index 00000000000..d05129d081a --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/BUILD @@ -0,0 +1,106 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["kms_plugin_mock.go"], + importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing", + importpath = "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing", + visibility = ["//visibility:public"], + deps = select({ + "@io_bazel_rules_go//go/platform:android": [ + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/google.golang.org/grpc/codes:go_default_library", + "//vendor/google.golang.org/grpc/status:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], + "@io_bazel_rules_go//go/platform:darwin": [ + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/google.golang.org/grpc/codes:go_default_library", + "//vendor/google.golang.org/grpc/status:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], + "@io_bazel_rules_go//go/platform:dragonfly": [ + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/google.golang.org/grpc/codes:go_default_library", + "//vendor/google.golang.org/grpc/status:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], + "@io_bazel_rules_go//go/platform:freebsd": [ + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/google.golang.org/grpc/codes:go_default_library", + "//vendor/google.golang.org/grpc/status:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/google.golang.org/grpc/codes:go_default_library", + "//vendor/google.golang.org/grpc/status:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], + "@io_bazel_rules_go//go/platform:nacl": [ + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/google.golang.org/grpc/codes:go_default_library", + "//vendor/google.golang.org/grpc/status:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], + "@io_bazel_rules_go//go/platform:netbsd": [ + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/google.golang.org/grpc/codes:go_default_library", + "//vendor/google.golang.org/grpc/status:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], + "@io_bazel_rules_go//go/platform:openbsd": [ + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/google.golang.org/grpc/codes:go_default_library", + "//vendor/google.golang.org/grpc/status:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], + "@io_bazel_rules_go//go/platform:plan9": [ + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/google.golang.org/grpc/codes:go_default_library", + "//vendor/google.golang.org/grpc/status:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], + "@io_bazel_rules_go//go/platform:solaris": [ + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/google.golang.org/grpc/codes:go_default_library", + "//vendor/google.golang.org/grpc/status:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/kms_plugin_mock.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/kms_plugin_mock.go new file mode 100644 index 00000000000..ae348cdf5a4 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/kms_plugin_mock.go @@ -0,0 +1,176 @@ +// +build !windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "context" + "encoding/base64" + "fmt" + "net" + "os" + "runtime" + "strings" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "k8s.io/apimachinery/pkg/util/wait" + kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" + "k8s.io/klog" +) + +const ( + // Now only supported unix domain socket. + unixProtocol = "unix" + + // Current version for the protocol interface definition. + kmsapiVersion = "v1beta1" +) + +// Base64Plugin gRPC sever for a mock KMS provider. +// Uses base64 to simulate encrypt and decrypt. +type Base64Plugin struct { + grpcServer *grpc.Server + listener net.Listener + mu *sync.Mutex + lastEncryptRequest *kmsapi.EncryptRequest + inFailedState bool + ver string + socketPath string +} + +// NewBase64Plugin is a constructor for Base64Plugin. +func NewBase64Plugin(socketPath string) (*Base64Plugin, error) { + server := grpc.NewServer() + result := &Base64Plugin{ + grpcServer: server, + mu: &sync.Mutex{}, + ver: kmsapiVersion, + socketPath: socketPath, + } + + kmsapi.RegisterKeyManagementServiceServer(server, result) + return result, nil +} + +// WaitForBase64PluginToBeUp waits until the plugin is ready to serve requests. +func WaitForBase64PluginToBeUp(plugin *Base64Plugin) error { + var gRPCErr error + pollErr := wait.PollImmediate(1*time.Second, wait.ForeverTestTimeout, func() (bool, error) { + _, gRPCErr = plugin.Encrypt(context.Background(), &kmsapi.EncryptRequest{Plain: []byte("foo")}) + return gRPCErr == nil, nil + }) + + if pollErr == wait.ErrWaitTimeout { + return fmt.Errorf("failed to start kms-plugin, error: %v", gRPCErr) + } + + return nil +} + +// LastEncryptRequest returns the last EncryptRequest.Plain sent to the plugin. +func (s *Base64Plugin) LastEncryptRequest() []byte { + return s.lastEncryptRequest.Plain +} + +// SetVersion sets the version of kms-plugin. +func (s *Base64Plugin) SetVersion(ver string) { + s.ver = ver +} + +// Start starts plugin's gRPC service. +func (s *Base64Plugin) Start() error { + var err error + s.listener, err = net.Listen(unixProtocol, s.socketPath) + if err != nil { + return fmt.Errorf("failed to listen on the unix socket, error: %v", err) + } + klog.Infof("Listening on %s", s.socketPath) + + go s.grpcServer.Serve(s.listener) + return nil +} + +// CleanUp stops gRPC server and the underlying listener. +func (s *Base64Plugin) CleanUp() { + s.grpcServer.Stop() + s.listener.Close() + if !strings.HasPrefix(s.socketPath, "@") || runtime.GOOS != "linux" { + os.Remove(s.socketPath) + } +} + +// EnterFailedState places the plugin into failed state. +func (s *Base64Plugin) EnterFailedState() { + s.mu.Lock() + defer s.mu.Unlock() + s.inFailedState = true +} + +// ExitFailedState removes the plugin from the failed state. +func (s *Base64Plugin) ExitFailedState() { + s.mu.Lock() + defer s.mu.Unlock() + s.inFailedState = false +} + +// Version returns the version of the kms-plugin. +func (s *Base64Plugin) Version(ctx context.Context, request *kmsapi.VersionRequest) (*kmsapi.VersionResponse, error) { + klog.Infof("Received request for Version: %v", request) + return &kmsapi.VersionResponse{Version: s.ver, RuntimeName: "testKMS", RuntimeVersion: "0.0.1"}, nil +} + +// Decrypt performs base64 decoding of the payload of kms.DecryptRequest. +func (s *Base64Plugin) Decrypt(ctx context.Context, request *kmsapi.DecryptRequest) (*kmsapi.DecryptResponse, error) { + klog.V(3).Infof("Received Decrypt Request for DEK: %s", string(request.Cipher)) + + s.mu.Lock() + defer s.mu.Unlock() + if s.inFailedState { + return nil, status.Error(codes.FailedPrecondition, "failed precondition - key disabled") + } + + buf := make([]byte, base64.StdEncoding.DecodedLen(len(request.Cipher))) + n, err := base64.StdEncoding.Decode(buf, request.Cipher) + if err != nil { + return nil, err + } + + return &kmsapi.DecryptResponse{Plain: buf[:n]}, nil +} + +// Encrypt performs base64 encoding of the payload of kms.EncryptRequest. +func (s *Base64Plugin) Encrypt(ctx context.Context, request *kmsapi.EncryptRequest) (*kmsapi.EncryptResponse, error) { + klog.V(3).Infof("Received Encrypt Request for DEK: %x", request.Plain) + s.mu.Lock() + defer s.mu.Unlock() + s.lastEncryptRequest = request + + if s.inFailedState { + return nil, status.Error(codes.FailedPrecondition, "failed precondition - key disabled") + } + + buf := make([]byte, base64.StdEncoding.EncodedLen(len(request.Plain))) + base64.StdEncoding.Encode(buf, request.Plain) + + return &kmsapi.EncryptResponse{Cipher: buf}, nil +} diff --git a/test/integration/master/BUILD b/test/integration/master/BUILD index bf4e4f38e0f..a4be2a028bd 100644 --- a/test/integration/master/BUILD +++ b/test/integration/master/BUILD @@ -74,33 +74,43 @@ go_test( "//vendor/sigs.k8s.io/yaml:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:android": [ + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", ], "@io_bazel_rules_go//go/platform:darwin": [ + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", ], "@io_bazel_rules_go//go/platform:dragonfly": [ + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", ], "@io_bazel_rules_go//go/platform:freebsd": [ + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", ], "@io_bazel_rules_go//go/platform:linux": [ + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", ], "@io_bazel_rules_go//go/platform:nacl": [ + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", ], "@io_bazel_rules_go//go/platform:netbsd": [ + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", ], "@io_bazel_rules_go//go/platform:openbsd": [ + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", ], "@io_bazel_rules_go//go/platform:plan9": [ + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", ], "@io_bazel_rules_go//go/platform:solaris": [ + "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", ], "//conditions:default": [], @@ -122,10 +132,7 @@ filegroup( go_library( name = "go_default_library", - srcs = [ - "kms_plugin_mock.go", - "transformation_testcase.go", - ], + srcs = ["transformation_testcase.go"], importpath = "k8s.io/kubernetes/test/integration/master", deps = [ "//cmd/kube-apiserver/app/testing:go_default_library", @@ -141,67 +148,5 @@ go_library( "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/k8s.io/klog:go_default_library", "//vendor/sigs.k8s.io/yaml:go_default_library", - ] + select({ - "@io_bazel_rules_go//go/platform:android": [ - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", - "//vendor/google.golang.org/grpc/codes:go_default_library", - "//vendor/google.golang.org/grpc/status:go_default_library", - ], - "@io_bazel_rules_go//go/platform:darwin": [ - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", - "//vendor/google.golang.org/grpc/codes:go_default_library", - "//vendor/google.golang.org/grpc/status:go_default_library", - ], - "@io_bazel_rules_go//go/platform:dragonfly": [ - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", - "//vendor/google.golang.org/grpc/codes:go_default_library", - "//vendor/google.golang.org/grpc/status:go_default_library", - ], - "@io_bazel_rules_go//go/platform:freebsd": [ - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", - "//vendor/google.golang.org/grpc/codes:go_default_library", - "//vendor/google.golang.org/grpc/status:go_default_library", - ], - "@io_bazel_rules_go//go/platform:linux": [ - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", - "//vendor/google.golang.org/grpc/codes:go_default_library", - "//vendor/google.golang.org/grpc/status:go_default_library", - ], - "@io_bazel_rules_go//go/platform:nacl": [ - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", - "//vendor/google.golang.org/grpc/codes:go_default_library", - "//vendor/google.golang.org/grpc/status:go_default_library", - ], - "@io_bazel_rules_go//go/platform:netbsd": [ - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", - "//vendor/google.golang.org/grpc/codes:go_default_library", - "//vendor/google.golang.org/grpc/status:go_default_library", - ], - "@io_bazel_rules_go//go/platform:openbsd": [ - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", - "//vendor/google.golang.org/grpc/codes:go_default_library", - "//vendor/google.golang.org/grpc/status:go_default_library", - ], - "@io_bazel_rules_go//go/platform:plan9": [ - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", - "//vendor/google.golang.org/grpc/codes:go_default_library", - "//vendor/google.golang.org/grpc/status:go_default_library", - ], - "@io_bazel_rules_go//go/platform:solaris": [ - "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", - "//vendor/google.golang.org/grpc/codes:go_default_library", - "//vendor/google.golang.org/grpc/status:go_default_library", - ], - "//conditions:default": [], - }), + ], ) diff --git a/test/integration/master/kms_plugin_mock.go b/test/integration/master/kms_plugin_mock.go deleted file mode 100644 index 9d65a5ca915..00000000000 --- a/test/integration/master/kms_plugin_mock.go +++ /dev/null @@ -1,126 +0,0 @@ -// +build !windows - -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package master - -import ( - "context" - "encoding/base64" - "fmt" - "net" - "sync" - - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" - "k8s.io/klog" -) - -const ( - kmsAPIVersion = "v1beta1" - unixProtocol = "unix" -) - -// base64Plugin gRPC sever for a mock KMS provider. -// Uses base64 to simulate encrypt and decrypt. -type base64Plugin struct { - grpcServer *grpc.Server - listener net.Listener - mu *sync.Mutex - lastEncryptRequest *kmsapi.EncryptRequest - inFailedState bool -} - -func newBase64Plugin(socketPath string) (*base64Plugin, error) { - listener, err := net.Listen(unixProtocol, socketPath) - if err != nil { - return nil, fmt.Errorf("failed to listen on the unix socket, error: %v", err) - } - klog.Infof("Listening on %s", socketPath) - - server := grpc.NewServer() - - result := &base64Plugin{ - grpcServer: server, - listener: listener, - mu: &sync.Mutex{}, - } - - kmsapi.RegisterKeyManagementServiceServer(server, result) - - return result, nil -} - -func (s *base64Plugin) cleanUp() { - s.grpcServer.Stop() - s.listener.Close() -} - -var testProviderAPIVersion = kmsAPIVersion - -func (s *base64Plugin) enterFailedState() { - s.mu.Lock() - defer s.mu.Unlock() - s.inFailedState = true -} - -func (s *base64Plugin) exitFailedState() { - s.mu.Lock() - defer s.mu.Unlock() - s.inFailedState = false -} - -func (s *base64Plugin) Version(ctx context.Context, request *kmsapi.VersionRequest) (*kmsapi.VersionResponse, error) { - return &kmsapi.VersionResponse{Version: testProviderAPIVersion, RuntimeName: "testKMS", RuntimeVersion: "0.0.1"}, nil -} - -func (s *base64Plugin) Decrypt(ctx context.Context, request *kmsapi.DecryptRequest) (*kmsapi.DecryptResponse, error) { - klog.Infof("Received Decrypt Request for DEK: %s", string(request.Cipher)) - - s.mu.Lock() - defer s.mu.Unlock() - if s.inFailedState { - return nil, status.Error(codes.FailedPrecondition, "failed precondition - key disabled") - } - - buf := make([]byte, base64.StdEncoding.DecodedLen(len(request.Cipher))) - n, err := base64.StdEncoding.Decode(buf, request.Cipher) - if err != nil { - return nil, err - } - - return &kmsapi.DecryptResponse{Plain: buf[:n]}, nil -} - -func (s *base64Plugin) Encrypt(ctx context.Context, request *kmsapi.EncryptRequest) (*kmsapi.EncryptResponse, error) { - klog.Infof("Received Encrypt Request for DEK: %x", request.Plain) - s.mu.Lock() - defer s.mu.Unlock() - s.lastEncryptRequest = request - - if s.inFailedState { - return nil, status.Error(codes.FailedPrecondition, "failed precondition - key disabled") - } - - buf := make([]byte, base64.StdEncoding.EncodedLen(len(request.Plain))) - base64.StdEncoding.Encode(buf, request.Plain) - - return &kmsapi.EncryptResponse{Cipher: buf}, nil -} diff --git a/test/integration/master/kms_transformation_test.go b/test/integration/master/kms_transformation_test.go index 8c8a1d27b05..b02e3a83672 100644 --- a/test/integration/master/kms_transformation_test.go +++ b/test/integration/master/kms_transformation_test.go @@ -23,8 +23,8 @@ import ( "context" "crypto/aes" "encoding/binary" - "fmt" + "fmt" "net/http" "strings" "testing" @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/storage/value" aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes" + mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing" kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -41,6 +42,7 @@ import ( const ( dekKeySizeLen = 2 + kmsAPIVersion = "v1beta1" ) type envelope struct { @@ -112,14 +114,16 @@ resources: ` providerName := "kms-provider" - pluginMock, err := newBase64Plugin("@kms-provider.sock") + pluginMock, err := mock.NewBase64Plugin("@kms-provider.sock") if err != nil { t.Fatalf("failed to create mock of KMS Plugin: %v", err) } - go pluginMock.grpcServer.Serve(pluginMock.listener) - defer pluginMock.cleanUp() - kmsPluginMustBeUp(t, pluginMock) + go pluginMock.Start() + if err := mock.WaitForBase64PluginToBeUp(pluginMock); err != nil { + t.Fatalf("Failed start plugin, err: %v", err) + } + defer pluginMock.CleanUp() test, err := newTransformTest(t, encryptionConfig) if err != nil { @@ -133,10 +137,7 @@ resources: } // Since Data Encryption Key (DEK) is randomly generated (per encryption operation), we need to ask KMS Mock for it. - plainTextDEK := pluginMock.lastEncryptRequest.Plain - if err != nil { - t.Fatalf("failed to get DEK from KMS: %v", err) - } + plainTextDEK := pluginMock.LastEncryptRequest() secretETCDPath := test.getETCDPath() rawEnvelope, err := test.getRawSecretFromETCD() @@ -197,23 +198,30 @@ resources: endpoint: unix:///@kms-provider-2.sock ` - pluginMock1, err := newBase64Plugin("@kms-provider-1.sock") + pluginMock1, err := mock.NewBase64Plugin("@kms-provider-1.sock") if err != nil { t.Fatalf("failed to create mock of KMS Plugin #1: %v", err) } - go pluginMock1.grpcServer.Serve(pluginMock1.listener) - defer pluginMock1.cleanUp() - kmsPluginMustBeUp(t, pluginMock1) - - pluginMock2, err := newBase64Plugin("@kms-provider-2.sock") - if err != nil { - t.Fatalf("failed to create mock of KMS Plugin #2: %v", err) + if err := pluginMock1.Start(); err != nil { + t.Fatalf("Failed to start kms-plugin, err: %v", err) + } + defer pluginMock1.CleanUp() + if err := mock.WaitForBase64PluginToBeUp(pluginMock1); err != nil { + t.Fatalf("Failed to start plugin #1, err: %v", err) } - go pluginMock2.grpcServer.Serve(pluginMock2.listener) - defer pluginMock2.cleanUp() - kmsPluginMustBeUp(t, pluginMock2) + pluginMock2, err := mock.NewBase64Plugin("@kms-provider-2.sock") + if err != nil { + t.Fatalf("Failed to create mock of KMS Plugin #2: err: %v", err) + } + if err := pluginMock2.Start(); err != nil { + t.Fatalf("Failed to start kms-plugin, err: %v", err) + } + defer pluginMock2.CleanUp() + if err := mock.WaitForBase64PluginToBeUp(pluginMock2); err != nil { + t.Fatalf("Failed to start KMS Plugin #2: err: %v", err) + } test, err := newTransformTest(t, encryptionConfig) if err != nil { @@ -231,32 +239,19 @@ resources: // Stage 2 - kms-plugin for provider-1 is down. Therefore, expect the health check for provider-1 // to fail, but provider-2 should still be OK - pluginMock1.enterFailedState() + pluginMock1.EnterFailedState() mustBeUnHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig) mustBeHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig) - pluginMock1.exitFailedState() + pluginMock1.ExitFailedState() // Stage 3 - kms-plugin for provider-1 is now up. Therefore, expect the health check for provider-1 // to succeed now, but provider-2 is now down. // Need to sleep since health check chases responses for 3 seconds. - pluginMock2.enterFailedState() + pluginMock2.EnterFailedState() mustBeHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig) mustBeUnHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig) } -func kmsPluginMustBeUp(t *testing.T, plugin *base64Plugin) { - t.Helper() - var gRPCErr error - pollErr := wait.PollImmediate(1*time.Second, wait.ForeverTestTimeout, func() (bool, error) { - _, gRPCErr = plugin.Encrypt(context.Background(), &kmsapi.EncryptRequest{Plain: []byte("foo")}) - return gRPCErr == nil, nil - }) - - if pollErr == wait.ErrWaitTimeout { - t.Fatalf("failed to start kms-plugin, error: %v", gRPCErr) - } -} - func mustBeHealthy(t *testing.T, checkName string, clientConfig *rest.Config) { t.Helper() var restErr error diff --git a/vendor/modules.txt b/vendor/modules.txt index cbb663cb23d..611c51b1812 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1363,6 +1363,7 @@ k8s.io/apiserver/pkg/storage/testing k8s.io/apiserver/pkg/storage/value k8s.io/apiserver/pkg/storage/value/encrypt/aes k8s.io/apiserver/pkg/storage/value/encrypt/envelope +k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1 k8s.io/apiserver/pkg/storage/value/encrypt/identity k8s.io/apiserver/pkg/storage/value/encrypt/secretbox