Merge pull request #83811 from immutableT/single-kms-mock

Use single kms-plugin mock in unit and integration tests.

staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/BUILD

@@ -37,53 +37,43 @@ go_test(
     ] + select({
         "@io_bazel_rules_go//go/platform:android": [
             "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:darwin": [
             "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:dragonfly": [
             "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:freebsd": [
             "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:linux": [
             "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:nacl": [
             "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:netbsd": [
             "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:openbsd": [
             "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:plan9": [
             "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:solaris": [
             "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
         ],
         "//conditions:default": [],
     }),
@@ -100,6 +90,7 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:all-srcs",
         "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:all-srcs",
     ],
     tags = ["automanaged"],

staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service.go

@@ -68,6 +68,8 @@ func NewGRPCService(endpoint string, callTimeout time.Duration) (Service, error)
 			c, err := net.DialUnix(unixProtocol, nil, &net.UnixAddr{Name: addr})
 			if err != nil {
 				klog.Errorf("failed to create connection to unix socket: %s, error: %v", addr, err)
+			} else {
+				klog.V(4).Infof("Successfully dialed Unix socket %v", addr)
 			}
 			return c, err
 		}))

staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service_unix_test.go

@@ -20,44 +20,56 @@ limitations under the License.
 package envelope
 
 import (
-	"context"
-	"encoding/base64"
 	"fmt"
-	"net"
-	"os"
 	"reflect"
-	"runtime"
-	"strings"
 	"sync"
 	"testing"
 	"time"
 
-	"google.golang.org/grpc"
+	mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing"
 
 	"k8s.io/apimachinery/pkg/util/uuid"
-	kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
 )
 
+type testSocket struct {
+	path     string
+	endpoint string
+}
+
+// newEndpoint constructs a unique name for a Linux Abstract Socket to be used in a test.
+// This package uses Linux Domain Sockets to remove the need for clean-up of socket files.
+func newEndpoint() *testSocket {
+	p := fmt.Sprintf("@%s.sock", uuid.NewUUID())
+
+	return &testSocket{
+		path:     p,
+		endpoint: fmt.Sprintf("unix:///%s", p),
+	}
+}
+
 // TestKMSPluginLateStart tests the scenario where kms-plugin pod/container starts after kube-apiserver pod/container.
 // Since the Dial to kms-plugin is non-blocking we expect the construction of gRPC service to succeed even when
 // kms-plugin is not yet up - dialing happens in the background.
 func TestKMSPluginLateStart(t *testing.T) {
 	t.Parallel()
 	callTimeout := 3 * time.Second
-	endpoint := getSocketName()
+	s := newEndpoint()
 
-	service, err := NewGRPCService(endpoint, callTimeout)
+	service, err := NewGRPCService(s.endpoint, callTimeout)
 	if err != nil {
 		t.Fatalf("failed to create envelope service, error: %v", err)
 	}
 	defer destroyService(service)
 
 	time.Sleep(callTimeout / 2)
-	f, err := startFakeKMSProvider(kmsapiVersion, endpoint)
+	f, err := mock.NewBase64Plugin(s.path)
 	if err != nil {
 		t.Fatalf("failed to start test KMS provider server, error: %v", err)
 	}
-	defer f.Stop()
+	if err := f.Start(); err != nil {
+		t.Fatalf("Failed to start kms-plugin, err: %v", err)
+	}
+	defer f.CleanUp()
 
 	data := []byte("test data")
 	_, err = service.Encrypt(data)
@@ -113,7 +125,7 @@ func TestTimeouts(t *testing.T) {
 				kubeAPIServerWG sync.WaitGroup
 				kmsPluginWG     sync.WaitGroup
 				testCompletedWG sync.WaitGroup
-				socketName      = getSocketName()
+				socketName      = newEndpoint()
 			)
 
 			testCompletedWG.Add(1)
@@ -124,7 +136,7 @@ func TestTimeouts(t *testing.T) {
 				// Simulating late start of kube-apiserver - plugin is up before kube-apiserver, if requested by the testcase.
 				time.Sleep(tt.kubeAPIServerDelay)
 
-				service, err = NewGRPCService(socketName, tt.callTimeout)
+				service, err = NewGRPCService(socketName.endpoint, tt.callTimeout)
 				if err != nil {
 					t.Fatalf("failed to create envelope service, error: %v", err)
 				}
@@ -139,11 +151,14 @@ func TestTimeouts(t *testing.T) {
 				// Simulating delayed start of kms-plugin, kube-apiserver is up before the plugin, if requested by the testcase.
 				time.Sleep(tt.pluginDelay)
 
-				f, err := startFakeKMSProvider(kmsapiVersion, socketName)
+				f, err := mock.NewBase64Plugin(socketName.path)
 				if err != nil {
-					t.Fatalf("failed to start test KMS provider server, error: %v", err)
+					t.Fatalf("failed to construct test KMS provider server, error: %v", err)
 				}
-				defer f.Stop()
+				if err := f.Start(); err != nil {
+					t.Fatalf("Failed to start test KMS provider server, error: %v", err)
+				}
+				defer f.CleanUp()
 				kmsPluginWG.Done()
 				// Keeping plugin up to process requests.
 				testCompletedWG.Wait()
@@ -175,16 +190,19 @@ func TestIntermittentConnectionLoss(t *testing.T) {
 		timeout  = 30 * time.Second
 		blackOut = 1 * time.Second
 		data     = []byte("test data")
-		endpoint = getSocketName()
+		endpoint = newEndpoint()
 	)
 	// Start KMS Plugin
-	f, err := startFakeKMSProvider(kmsapiVersion, endpoint)
+	f, err := mock.NewBase64Plugin(endpoint.path)
 	if err != nil {
 		t.Fatalf("failed to start test KMS provider server, error: %v", err)
 	}
+	if err := f.Start(); err != nil {
+		t.Fatalf("Failed to start kms-plugin, err: %v", err)
+	}
 
 	//  connect to kms plugin
-	service, err := NewGRPCService(endpoint, timeout)
+	service, err := NewGRPCService(endpoint.endpoint, timeout)
 	if err != nil {
 		t.Fatalf("failed to create envelope service, error: %v", err)
 	}
@@ -198,7 +216,7 @@ func TestIntermittentConnectionLoss(t *testing.T) {
 
 	// Stop KMS Plugin - simulating connection loss
 	t.Log("KMS Plugin is stopping")
-	f.Stop()
+	f.CleanUp()
 	time.Sleep(2 * time.Second)
 
 	wg1.Add(1)
@@ -217,11 +235,14 @@ func TestIntermittentConnectionLoss(t *testing.T) {
 	wg1.Wait()
 	time.Sleep(blackOut)
 	// Start KMS Plugin
-	f, err = startFakeKMSProvider(kmsapiVersion, endpoint)
+	f, err = mock.NewBase64Plugin(endpoint.path)
 	if err != nil {
 		t.Fatalf("failed to start test KMS provider server, error: %v", err)
 	}
-	defer f.Stop()
+	if err := f.Start(); err != nil {
+		t.Fatalf("Failed to start kms-plugin, err: %v", err)
+	}
+	defer f.CleanUp()
 	t.Log("Restarted KMS Plugin")
 
 	wg2.Wait()
@@ -232,15 +253,19 @@ func TestUnsupportedVersion(t *testing.T) {
 	ver := "invalid"
 	data := []byte("test data")
 	wantErr := fmt.Errorf(versionErrorf, ver, kmsapiVersion)
-	endpoint := getSocketName()
+	endpoint := newEndpoint()
 
-	f, err := startFakeKMSProvider(ver, endpoint)
+	f, err := mock.NewBase64Plugin(endpoint.path)
 	if err != nil {
 		t.Fatalf("failed to start test KMS provider server, error: %ver", err)
 	}
-	defer f.Stop()
+	f.SetVersion(ver)
+	if err := f.Start(); err != nil {
+		t.Fatalf("Failed to start kms-plugin, err: %v", err)
+	}
+	defer f.CleanUp()
 
-	s, err := NewGRPCService(endpoint, 1*time.Second)
+	s, err := NewGRPCService(endpoint.endpoint, 1*time.Second)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -254,7 +279,7 @@ func TestUnsupportedVersion(t *testing.T) {
 
 	destroyService(s)
 
-	s, err = NewGRPCService(endpoint, 1*time.Second)
+	s, err = NewGRPCService(endpoint.endpoint, 1*time.Second)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -271,15 +296,18 @@ func TestUnsupportedVersion(t *testing.T) {
 func TestGRPCService(t *testing.T) {
 	t.Parallel()
 	// Start a test gRPC server.
-	endpoint := getSocketName()
+	endpoint := newEndpoint()
-	f, err := startFakeKMSProvider(kmsapiVersion, endpoint)
+	f, err := mock.NewBase64Plugin(endpoint.path)
 	if err != nil {
-		t.Fatalf("failed to start test KMS provider server, error: %v", err)
+		t.Fatalf("failed to construct test KMS provider server, error: %v", err)
 	}
-	defer f.Stop()
+	if err := f.Start(); err != nil {
+		t.Fatalf("Failed to start kms-plugin, err: %v", err)
+	}
+	defer f.CleanUp()
 
 	// Create the gRPC client service.
-	service, err := NewGRPCService(endpoint, 1*time.Second)
+	service, err := NewGRPCService(endpoint.endpoint, 1*time.Second)
 	if err != nil {
 		t.Fatalf("failed to create envelope service, error: %v", err)
 	}
@@ -307,15 +335,18 @@ func TestGRPCService(t *testing.T) {
 func TestGRPCServiceConcurrentAccess(t *testing.T) {
 	t.Parallel()
 	// Start a test gRPC server.
-	endpoint := getSocketName()
+	endpoint := newEndpoint()
-	f, err := startFakeKMSProvider(kmsapiVersion, endpoint)
+	f, err := mock.NewBase64Plugin(endpoint.path)
 	if err != nil {
 		t.Fatalf("failed to start test KMS provider server, error: %v", err)
 	}
-	defer f.Stop()
+	if err := f.Start(); err != nil {
+		t.Fatalf("Failed to start kms-plugin, err: %v", err)
+	}
+	defer f.CleanUp()
 
 	// Create the gRPC client service.
-	service, err := NewGRPCService(endpoint, 15*time.Second)
+	service, err := NewGRPCService(endpoint.endpoint, 15*time.Second)
 	if err != nil {
 		t.Fatalf("failed to create envelope service, error: %v", err)
 	}
@@ -356,32 +387,29 @@ func destroyService(service Service) {
 	}
 }
 
-func getSocketName() string {
-	return fmt.Sprintf("unix:///@%s.sock", uuid.NewUUID())
-}
-
 // Test all those invalid configuration for KMS provider.
 func TestInvalidConfiguration(t *testing.T) {
 	t.Parallel()
 	// Start a test gRPC server.
-	f, err := startFakeKMSProvider(kmsapiVersion, getSocketName())
+	f, err := mock.NewBase64Plugin(newEndpoint().path)
 	if err != nil {
 		t.Fatalf("failed to start test KMS provider server, error: %v", err)
 	}
-	defer f.Stop()
+	if err := f.Start(); err != nil {
+		t.Fatalf("Failed to start kms-plugin, err: %v", err)
+	}
+	defer f.CleanUp()
 
 	invalidConfigs := []struct {
 		name     string
-		apiVersion string
 		endpoint string
 	}{
-		{"emptyConfiguration", kmsapiVersion, ""},
+		{"emptyConfiguration", ""},
-		{"invalidScheme", kmsapiVersion, "tcp://localhost:6060"},
+		{"invalidScheme", "tcp://localhost:6060"},
 	}
 
 	for _, testCase := range invalidConfigs {
 		t.Run(testCase.name, func(t *testing.T) {
-			f.apiVersion = testCase.apiVersion
 			_, err := NewGRPCService(testCase.endpoint, 1*time.Second)
 			if err == nil {
 				t.Fatalf("should fail to create envelope service for %s.", testCase.name)
@@ -389,58 +417,3 @@ func TestInvalidConfiguration(t *testing.T) {
 		})
 	}
 }
-
-// Start the gRPC server that listens on unix socket.
-func startFakeKMSProvider(version, endpoint string) (*fakeKMSPlugin, error) {
-	sockFile, err := parseEndpoint(endpoint)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse endpoint:%q, error %v", endpoint, err)
-	}
-	listener, err := net.Listen(unixProtocol, sockFile)
-	if err != nil {
-		return nil, fmt.Errorf("failed to listen on the unix socket, error: %v", err)
-	}
-
-	s := grpc.NewServer()
-	f := &fakeKMSPlugin{apiVersion: version, server: s, sockFile: sockFile}
-	kmsapi.RegisterKeyManagementServiceServer(s, f)
-	go s.Serve(listener)
-	return f, nil
-}
-
-// Fake gRPC sever for remote KMS provider.
-// Use base64 to simulate encrypt and decrypt.
-type fakeKMSPlugin struct {
-	apiVersion string
-	server     *grpc.Server
-	sockFile   string
-}
-
-func (s *fakeKMSPlugin) Stop() {
-	// Stop the server
-	s.server.Stop()
-	// If this isn't a Linux abstract namespace socket, or if we're on a non-linux platform, clean up the socket file
-	if !strings.HasPrefix(s.sockFile, "@") || runtime.GOOS != "linux" {
-		os.Remove(s.sockFile)
-	}
-}
-
-func (s *fakeKMSPlugin) Version(ctx context.Context, request *kmsapi.VersionRequest) (*kmsapi.VersionResponse, error) {
-	return &kmsapi.VersionResponse{Version: s.apiVersion, RuntimeName: "testKMS", RuntimeVersion: "0.0.1"}, nil
-}
-
-func (s *fakeKMSPlugin) Decrypt(ctx context.Context, request *kmsapi.DecryptRequest) (*kmsapi.DecryptResponse, error) {
-	buf := make([]byte, base64.StdEncoding.DecodedLen(len(request.Cipher)))
-	n, err := base64.StdEncoding.Decode(buf, request.Cipher)
-	if err != nil {
-		return nil, err
-	}
-
-	return &kmsapi.DecryptResponse{Plain: buf[:n]}, nil
-}
-
-func (s *fakeKMSPlugin) Encrypt(ctx context.Context, request *kmsapi.EncryptRequest) (*kmsapi.EncryptResponse, error) {
-	buf := make([]byte, base64.StdEncoding.EncodedLen(len(request.Plain)))
-	base64.StdEncoding.Encode(buf, request.Plain)
-	return &kmsapi.EncryptResponse{Cipher: buf}, nil
-}

staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/BUILD

@@ -0,0 +1,106 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["kms_plugin_mock.go"],
+    importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing",
+    importpath = "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing",
+    visibility = ["//visibility:public"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//vendor/google.golang.org/grpc:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
+            "//vendor/k8s.io/klog:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:darwin": [
+            "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//vendor/google.golang.org/grpc:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
+            "//vendor/k8s.io/klog:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:dragonfly": [
+            "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//vendor/google.golang.org/grpc:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
+            "//vendor/k8s.io/klog:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:freebsd": [
+            "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//vendor/google.golang.org/grpc:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
+            "//vendor/k8s.io/klog:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//vendor/google.golang.org/grpc:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
+            "//vendor/k8s.io/klog:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:nacl": [
+            "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//vendor/google.golang.org/grpc:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
+            "//vendor/k8s.io/klog:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:netbsd": [
+            "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//vendor/google.golang.org/grpc:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
+            "//vendor/k8s.io/klog:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:openbsd": [
+            "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//vendor/google.golang.org/grpc:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
+            "//vendor/k8s.io/klog:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:plan9": [
+            "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//vendor/google.golang.org/grpc:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
+            "//vendor/k8s.io/klog:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:solaris": [
+            "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
+            "//vendor/google.golang.org/grpc:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
+            "//vendor/k8s.io/klog:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/kms_plugin_mock.go

@@ -0,0 +1,176 @@
+// +build !windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"net"
+	"os"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+	kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
+	"k8s.io/klog"
+)
+
+const (
+	// Now only supported unix domain socket.
+	unixProtocol = "unix"
+
+	// Current version for the protocol interface definition.
+	kmsapiVersion = "v1beta1"
+)
+
+// Base64Plugin gRPC sever for a mock KMS provider.
+// Uses base64 to simulate encrypt and decrypt.
+type Base64Plugin struct {
+	grpcServer         *grpc.Server
+	listener           net.Listener
+	mu                 *sync.Mutex
+	lastEncryptRequest *kmsapi.EncryptRequest
+	inFailedState      bool
+	ver                string
+	socketPath         string
+}
+
+// NewBase64Plugin is a constructor for Base64Plugin.
+func NewBase64Plugin(socketPath string) (*Base64Plugin, error) {
+	server := grpc.NewServer()
+	result := &Base64Plugin{
+		grpcServer: server,
+		mu:         &sync.Mutex{},
+		ver:        kmsapiVersion,
+		socketPath: socketPath,
+	}
+
+	kmsapi.RegisterKeyManagementServiceServer(server, result)
+	return result, nil
+}
+
+// WaitForBase64PluginToBeUp waits until the plugin is ready to serve requests.
+func WaitForBase64PluginToBeUp(plugin *Base64Plugin) error {
+	var gRPCErr error
+	pollErr := wait.PollImmediate(1*time.Second, wait.ForeverTestTimeout, func() (bool, error) {
+		_, gRPCErr = plugin.Encrypt(context.Background(), &kmsapi.EncryptRequest{Plain: []byte("foo")})
+		return gRPCErr == nil, nil
+	})
+
+	if pollErr == wait.ErrWaitTimeout {
+		return fmt.Errorf("failed to start kms-plugin, error: %v", gRPCErr)
+	}
+
+	return nil
+}
+
+// LastEncryptRequest returns the last EncryptRequest.Plain sent to the plugin.
+func (s *Base64Plugin) LastEncryptRequest() []byte {
+	return s.lastEncryptRequest.Plain
+}
+
+// SetVersion sets the version of kms-plugin.
+func (s *Base64Plugin) SetVersion(ver string) {
+	s.ver = ver
+}
+
+// Start starts plugin's gRPC service.
+func (s *Base64Plugin) Start() error {
+	var err error
+	s.listener, err = net.Listen(unixProtocol, s.socketPath)
+	if err != nil {
+		return fmt.Errorf("failed to listen on the unix socket, error: %v", err)
+	}
+	klog.Infof("Listening on %s", s.socketPath)
+
+	go s.grpcServer.Serve(s.listener)
+	return nil
+}
+
+// CleanUp stops gRPC server and the underlying listener.
+func (s *Base64Plugin) CleanUp() {
+	s.grpcServer.Stop()
+	s.listener.Close()
+	if !strings.HasPrefix(s.socketPath, "@") || runtime.GOOS != "linux" {
+		os.Remove(s.socketPath)
+	}
+}
+
+// EnterFailedState places the plugin into failed state.
+func (s *Base64Plugin) EnterFailedState() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.inFailedState = true
+}
+
+// ExitFailedState removes the plugin from the failed state.
+func (s *Base64Plugin) ExitFailedState() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.inFailedState = false
+}
+
+// Version returns the version of the kms-plugin.
+func (s *Base64Plugin) Version(ctx context.Context, request *kmsapi.VersionRequest) (*kmsapi.VersionResponse, error) {
+	klog.Infof("Received request for Version: %v", request)
+	return &kmsapi.VersionResponse{Version: s.ver, RuntimeName: "testKMS", RuntimeVersion: "0.0.1"}, nil
+}
+
+// Decrypt performs base64 decoding of the payload of kms.DecryptRequest.
+func (s *Base64Plugin) Decrypt(ctx context.Context, request *kmsapi.DecryptRequest) (*kmsapi.DecryptResponse, error) {
+	klog.V(3).Infof("Received Decrypt Request for DEK: %s", string(request.Cipher))
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.inFailedState {
+		return nil, status.Error(codes.FailedPrecondition, "failed precondition - key disabled")
+	}
+
+	buf := make([]byte, base64.StdEncoding.DecodedLen(len(request.Cipher)))
+	n, err := base64.StdEncoding.Decode(buf, request.Cipher)
+	if err != nil {
+		return nil, err
+	}
+
+	return &kmsapi.DecryptResponse{Plain: buf[:n]}, nil
+}
+
+// Encrypt performs base64 encoding of the payload of kms.EncryptRequest.
+func (s *Base64Plugin) Encrypt(ctx context.Context, request *kmsapi.EncryptRequest) (*kmsapi.EncryptResponse, error) {
+	klog.V(3).Infof("Received Encrypt Request for DEK: %x", request.Plain)
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.lastEncryptRequest = request
+
+	if s.inFailedState {
+		return nil, status.Error(codes.FailedPrecondition, "failed precondition - key disabled")
+	}
+
+	buf := make([]byte, base64.StdEncoding.EncodedLen(len(request.Plain)))
+	base64.StdEncoding.Encode(buf, request.Plain)
+
+	return &kmsapi.EncryptResponse{Cipher: buf}, nil
+}

test/integration/master/BUILD

@@ -74,33 +74,43 @@ go_test(
         "//vendor/sigs.k8s.io/yaml:go_default_library",
     ] + select({
         "@io_bazel_rules_go//go/platform:android": [
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
             "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:darwin": [
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
             "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:dragonfly": [
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
             "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:freebsd": [
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
             "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:linux": [
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
             "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:nacl": [
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
             "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:netbsd": [
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
             "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:openbsd": [
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
             "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:plan9": [
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
             "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:solaris": [
+            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing:go_default_library",
             "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "//conditions:default": [],
@@ -122,10 +132,7 @@ filegroup(
 
 go_library(
     name = "go_default_library",
-    srcs = [
+    srcs = ["transformation_testcase.go"],
-        "kms_plugin_mock.go",
-        "transformation_testcase.go",
-    ],
     importpath = "k8s.io/kubernetes/test/integration/master",
     deps = [
         "//cmd/kube-apiserver/app/testing:go_default_library",
@@ -141,67 +148,5 @@ go_library(
         "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
         "//vendor/sigs.k8s.io/yaml:go_default_library",
-    ] + select({
-        "@io_bazel_rules_go//go/platform:android": [
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/google.golang.org/grpc/codes:go_default_library",
-            "//vendor/google.golang.org/grpc/status:go_default_library",
     ],
-        "@io_bazel_rules_go//go/platform:darwin": [
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/google.golang.org/grpc/codes:go_default_library",
-            "//vendor/google.golang.org/grpc/status:go_default_library",
-        ],
-        "@io_bazel_rules_go//go/platform:dragonfly": [
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/google.golang.org/grpc/codes:go_default_library",
-            "//vendor/google.golang.org/grpc/status:go_default_library",
-        ],
-        "@io_bazel_rules_go//go/platform:freebsd": [
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/google.golang.org/grpc/codes:go_default_library",
-            "//vendor/google.golang.org/grpc/status:go_default_library",
-        ],
-        "@io_bazel_rules_go//go/platform:linux": [
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/google.golang.org/grpc/codes:go_default_library",
-            "//vendor/google.golang.org/grpc/status:go_default_library",
-        ],
-        "@io_bazel_rules_go//go/platform:nacl": [
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/google.golang.org/grpc/codes:go_default_library",
-            "//vendor/google.golang.org/grpc/status:go_default_library",
-        ],
-        "@io_bazel_rules_go//go/platform:netbsd": [
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/google.golang.org/grpc/codes:go_default_library",
-            "//vendor/google.golang.org/grpc/status:go_default_library",
-        ],
-        "@io_bazel_rules_go//go/platform:openbsd": [
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/google.golang.org/grpc/codes:go_default_library",
-            "//vendor/google.golang.org/grpc/status:go_default_library",
-        ],
-        "@io_bazel_rules_go//go/platform:plan9": [
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/google.golang.org/grpc/codes:go_default_library",
-            "//vendor/google.golang.org/grpc/status:go_default_library",
-        ],
-        "@io_bazel_rules_go//go/platform:solaris": [
-            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
-            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/google.golang.org/grpc/codes:go_default_library",
-            "//vendor/google.golang.org/grpc/status:go_default_library",
-        ],
-        "//conditions:default": [],
-    }),
 )

test/integration/master/kms_plugin_mock.go

@@ -1,126 +0,0 @@
-// +build !windows
-
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package master
-
-import (
-	"context"
-	"encoding/base64"
-	"fmt"
-	"net"
-	"sync"
-
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
-
-	kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
-	"k8s.io/klog"
-)
-
-const (
-	kmsAPIVersion = "v1beta1"
-	unixProtocol  = "unix"
-)
-
-// base64Plugin gRPC sever for a mock KMS provider.
-// Uses base64 to simulate encrypt and decrypt.
-type base64Plugin struct {
-	grpcServer         *grpc.Server
-	listener           net.Listener
-	mu                 *sync.Mutex
-	lastEncryptRequest *kmsapi.EncryptRequest
-	inFailedState      bool
-}
-
-func newBase64Plugin(socketPath string) (*base64Plugin, error) {
-	listener, err := net.Listen(unixProtocol, socketPath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to listen on the unix socket, error: %v", err)
-	}
-	klog.Infof("Listening on %s", socketPath)
-
-	server := grpc.NewServer()
-
-	result := &base64Plugin{
-		grpcServer: server,
-		listener:   listener,
-		mu:         &sync.Mutex{},
-	}
-
-	kmsapi.RegisterKeyManagementServiceServer(server, result)
-
-	return result, nil
-}
-
-func (s *base64Plugin) cleanUp() {
-	s.grpcServer.Stop()
-	s.listener.Close()
-}
-
-var testProviderAPIVersion = kmsAPIVersion
-
-func (s *base64Plugin) enterFailedState() {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	s.inFailedState = true
-}
-
-func (s *base64Plugin) exitFailedState() {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	s.inFailedState = false
-}
-
-func (s *base64Plugin) Version(ctx context.Context, request *kmsapi.VersionRequest) (*kmsapi.VersionResponse, error) {
-	return &kmsapi.VersionResponse{Version: testProviderAPIVersion, RuntimeName: "testKMS", RuntimeVersion: "0.0.1"}, nil
-}
-
-func (s *base64Plugin) Decrypt(ctx context.Context, request *kmsapi.DecryptRequest) (*kmsapi.DecryptResponse, error) {
-	klog.Infof("Received Decrypt Request for DEK: %s", string(request.Cipher))
-
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if s.inFailedState {
-		return nil, status.Error(codes.FailedPrecondition, "failed precondition - key disabled")
-	}
-
-	buf := make([]byte, base64.StdEncoding.DecodedLen(len(request.Cipher)))
-	n, err := base64.StdEncoding.Decode(buf, request.Cipher)
-	if err != nil {
-		return nil, err
-	}
-
-	return &kmsapi.DecryptResponse{Plain: buf[:n]}, nil
-}
-
-func (s *base64Plugin) Encrypt(ctx context.Context, request *kmsapi.EncryptRequest) (*kmsapi.EncryptResponse, error) {
-	klog.Infof("Received Encrypt Request for DEK: %x", request.Plain)
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	s.lastEncryptRequest = request
-
-	if s.inFailedState {
-		return nil, status.Error(codes.FailedPrecondition, "failed precondition - key disabled")
-	}
-
-	buf := make([]byte, base64.StdEncoding.EncodedLen(len(request.Plain)))
-	base64.StdEncoding.Encode(buf, request.Plain)
-
-	return &kmsapi.EncryptResponse{Cipher: buf}, nil
-}

test/integration/master/kms_transformation_test.go

@@ -23,8 +23,8 @@ import (
 	"context"
 	"crypto/aes"
 	"encoding/binary"
-	"fmt"
 
+	"fmt"
 	"net/http"
 	"strings"
 	"testing"
@@ -34,6 +34,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/storage/value"
 	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
+	mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing"
 	kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -41,6 +42,7 @@ import (
 
 const (
 	dekKeySizeLen = 2
+	kmsAPIVersion = "v1beta1"
 )
 
 type envelope struct {
@@ -112,14 +114,16 @@ resources:
 `
 
 	providerName := "kms-provider"
-	pluginMock, err := newBase64Plugin("@kms-provider.sock")
+	pluginMock, err := mock.NewBase64Plugin("@kms-provider.sock")
 	if err != nil {
 		t.Fatalf("failed to create mock of KMS Plugin: %v", err)
 	}
 
-	go pluginMock.grpcServer.Serve(pluginMock.listener)
+	go pluginMock.Start()
-	defer pluginMock.cleanUp()
+	if err := mock.WaitForBase64PluginToBeUp(pluginMock); err != nil {
-	kmsPluginMustBeUp(t, pluginMock)
+		t.Fatalf("Failed start plugin, err: %v", err)
+	}
+	defer pluginMock.CleanUp()
 
 	test, err := newTransformTest(t, encryptionConfig)
 	if err != nil {
@@ -133,10 +137,7 @@ resources:
 	}
 
 	// Since Data Encryption Key (DEK) is randomly generated (per encryption operation), we need to ask KMS Mock for it.
-	plainTextDEK := pluginMock.lastEncryptRequest.Plain
+	plainTextDEK := pluginMock.LastEncryptRequest()
-	if err != nil {
-		t.Fatalf("failed to get DEK from KMS: %v", err)
-	}
 
 	secretETCDPath := test.getETCDPath()
 	rawEnvelope, err := test.getRawSecretFromETCD()
@@ -197,23 +198,30 @@ resources:
        endpoint: unix:///@kms-provider-2.sock
 `
 
-	pluginMock1, err := newBase64Plugin("@kms-provider-1.sock")
+	pluginMock1, err := mock.NewBase64Plugin("@kms-provider-1.sock")
 	if err != nil {
 		t.Fatalf("failed to create mock of KMS Plugin #1: %v", err)
 	}
 
-	go pluginMock1.grpcServer.Serve(pluginMock1.listener)
+	if err := pluginMock1.Start(); err != nil {
-	defer pluginMock1.cleanUp()
+		t.Fatalf("Failed to start kms-plugin, err: %v", err)
-	kmsPluginMustBeUp(t, pluginMock1)
+	}
-
+	defer pluginMock1.CleanUp()
-	pluginMock2, err := newBase64Plugin("@kms-provider-2.sock")
+	if err := mock.WaitForBase64PluginToBeUp(pluginMock1); err != nil {
-	if err != nil {
+		t.Fatalf("Failed to start plugin #1, err: %v", err)
-		t.Fatalf("failed to create mock of KMS Plugin #2: %v", err)
 	}
 
-	go pluginMock2.grpcServer.Serve(pluginMock2.listener)
+	pluginMock2, err := mock.NewBase64Plugin("@kms-provider-2.sock")
-	defer pluginMock2.cleanUp()
+	if err != nil {
-	kmsPluginMustBeUp(t, pluginMock2)
+		t.Fatalf("Failed to create mock of KMS Plugin #2: err: %v", err)
+	}
+	if err := pluginMock2.Start(); err != nil {
+		t.Fatalf("Failed to start kms-plugin, err: %v", err)
+	}
+	defer pluginMock2.CleanUp()
+	if err := mock.WaitForBase64PluginToBeUp(pluginMock2); err != nil {
+		t.Fatalf("Failed to start KMS Plugin #2: err: %v", err)
+	}
 
 	test, err := newTransformTest(t, encryptionConfig)
 	if err != nil {
@@ -231,32 +239,19 @@ resources:
 
 	// Stage 2 - kms-plugin for provider-1 is down. Therefore, expect the health check for provider-1
 	// to fail, but provider-2 should still be OK
-	pluginMock1.enterFailedState()
+	pluginMock1.EnterFailedState()
 	mustBeUnHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
 	mustBeHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
-	pluginMock1.exitFailedState()
+	pluginMock1.ExitFailedState()
 
 	// Stage 3 - kms-plugin for provider-1 is now up. Therefore, expect the health check for provider-1
 	// to succeed now, but provider-2 is now down.
 	// Need to sleep since health check chases responses for 3 seconds.
-	pluginMock2.enterFailedState()
+	pluginMock2.EnterFailedState()
 	mustBeHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
 	mustBeUnHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
 }
 
-func kmsPluginMustBeUp(t *testing.T, plugin *base64Plugin) {
-	t.Helper()
-	var gRPCErr error
-	pollErr := wait.PollImmediate(1*time.Second, wait.ForeverTestTimeout, func() (bool, error) {
-		_, gRPCErr = plugin.Encrypt(context.Background(), &kmsapi.EncryptRequest{Plain: []byte("foo")})
-		return gRPCErr == nil, nil
-	})
-
-	if pollErr == wait.ErrWaitTimeout {
-		t.Fatalf("failed to start kms-plugin, error: %v", gRPCErr)
-	}
-}
-
 func mustBeHealthy(t *testing.T, checkName string, clientConfig *rest.Config) {
 	t.Helper()
 	var restErr error

vendor/modules.txt

@@ -1363,6 +1363,7 @@ k8s.io/apiserver/pkg/storage/testing
 k8s.io/apiserver/pkg/storage/value
 k8s.io/apiserver/pkg/storage/value/encrypt/aes
 k8s.io/apiserver/pkg/storage/value/encrypt/envelope
+k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing
 k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1
 k8s.io/apiserver/pkg/storage/value/encrypt/identity
 k8s.io/apiserver/pkg/storage/value/encrypt/secretbox