Merge pull request #54759 from sakshamsharma/remove-google-kms-in-tree

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Remove Google Cloud KMS's in-tree integration

Removes the following introduced by #48574 and others:
* `kms.go` which contained the cloudkms-specific code for Google Cloud KMS service.
* Registering the Google Cloud KMS in the KMS plugin registry.
* Google's `cloudkms` API package from `vendor` folder. 

The following changes are upcoming:
* Removal of KMSPluginRegistry. This would not be needed anymore, since KMS providers will be out-of-tree from now on (so no need of registering them, an address of the process would be enough).
* A service which allows encrypt/decrypt functionality (satisfies `envelope.Service` interface) if initialized with an IP/Port of an out-of-tree process serving KMS requests. Will tentatively use gRPC requests to talk to this external service.

Reference: https://github.com/kubernetes/kubernetes/pull/54439#issuecomment-340062801 and https://github.com/kubernetes/kubernetes/issues/51965#issuecomment-339333937.

```release-note
Google KMS integration was removed from in-tree in favor of a out-of-process extension point that will be used for all KMS providers.
```

Godeps/Godeps.json

@@ -2844,10 +2844,6 @@
 			"ImportPath": "golang.org/x/tools/go/vcs",
 			"Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32"
 		},
-		{
-			"ImportPath": "google.golang.org/api/cloudkms/v1",
-			"Rev": "654f863362977d69086620b5f72f13e911da2410"
-		},
 		{
 			"ImportPath": "google.golang.org/api/cloudmonitoring/v2beta2",
 			"Rev": "654f863362977d69086620b5f72f13e911da2410"

Godeps/LICENSES

@@ -84789,41 +84789,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ================================================================================
 
 
-================================================================================
-= vendor/google.golang.org/api/cloudkms/v1 licensed under: =
-
-Copyright (c) 2011 Google Inc. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-= vendor/google.golang.org/api/LICENSE a651bb3d8b1c412632e28823bb432b40
-================================================================================
-
-
 ================================================================================
 = vendor/google.golang.org/api/cloudmonitoring/v2beta2 licensed under: =
 

cluster/gce/gci/configure-helper.sh

@@ -1520,10 +1520,6 @@ function start-kube-apiserver {
 
   if [[ -n "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then
     local encryption_provider_config_path="/etc/srv/kubernetes/encryption-provider-config.yml"
-    if [[ -n "${GOOGLE_CLOUD_KMS_CONFIG_FILE_NAME:-}" && -n "${GOOGLE_CLOUD_KMS_CONFIG:-}" ]]; then
-        echo "${GOOGLE_CLOUD_KMS_CONFIG}" | base64 --decode > "${GOOGLE_CLOUD_KMS_CONFIG_FILE_NAME}"
-    fi
-
     echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}"
     params+=" --experimental-encryption-provider-config=${encryption_provider_config_path}"
   fi

cmd/kube-apiserver/app/server.go

@@ -265,13 +265,6 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele
 		return nil, nil, nil, nil, nil, utilerrors.NewAggregate(errs)
 	}
 
-	if s.CloudProvider != nil {
-		// Initialize the cloudprovider once, to give it a chance to register KMS plugins, if any.
-		_, err := cloudprovider.InitCloudProvider(s.CloudProvider.CloudProvider, s.CloudProvider.CloudConfigFile)
-		if err != nil {
-			return nil, nil, nil, nil, nil, err
-		}
-	}
 	genericConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, err := BuildGenericConfig(s, proxyTransport)
 	if err != nil {
 		return nil, nil, nil, nil, nil, err

pkg/cloudprovider/providers/gce/BUILD

@@ -40,7 +40,6 @@ go_library(
         "gce_urlmap.go",
         "gce_util.go",
         "gce_zones.go",
-        "kms.go",
         "metrics.go",
         "token_source.go",
     ],
@@ -61,7 +60,6 @@ go_library(
         "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/golang.org/x/oauth2:go_default_library",
         "//vendor/golang.org/x/oauth2/google:go_default_library",
-        "//vendor/google.golang.org/api/cloudkms/v1:go_default_library",
         "//vendor/google.golang.org/api/compute/v0.alpha:go_default_library",
         "//vendor/google.golang.org/api/compute/v0.beta:go_default_library",
         "//vendor/google.golang.org/api/compute/v1:go_default_library",
@@ -77,8 +75,6 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",

pkg/cloudprovider/providers/gce/gce.go

@@ -34,8 +34,6 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/apiserver/pkg/server/options/encryptionconfig"
-	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -48,7 +46,6 @@ import (
 	"github.com/golang/glog"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
-	cloudkms "google.golang.org/api/cloudkms/v1"
 	computealpha "google.golang.org/api/compute/v0.alpha"
 	computebeta "google.golang.org/api/compute/v0.beta"
 	compute "google.golang.org/api/compute/v1"
@@ -106,7 +103,6 @@ type GCECloud struct {
 	serviceBeta              *computebeta.Service
 	serviceAlpha             *computealpha.Service
 	containerService         *container.Service
-	cloudkmsService          *cloudkms.Service
 	client                   clientset.Interface
 	clientBuilder            controller.ControllerClientBuilder
 	eventBroadcaster         record.EventBroadcaster
@@ -195,10 +191,6 @@ type CloudConfig struct {
 	AlphaFeatureGate   *AlphaFeatureGate
 }
 
-// kmsPluginRegisterOnce prevents the cloudprovider from registering its KMS plugin
-// more than once in the KMS plugin registry.
-var kmsPluginRegisterOnce sync.Once
-
 func init() {
 	cloudprovider.RegisterCloudProvider(
 		ProviderName,
@@ -212,11 +204,6 @@ func (g *GCECloud) GetComputeService() *compute.Service {
 	return g.service
 }
 
-// Raw access to the cloudkmsService of GCE cloud. Required for encryption of etcd using Google KMS.
-func (g *GCECloud) GetKMSService() *cloudkms.Service {
-	return g.cloudkmsService
-}
-
 // newGCECloud creates a new instance of GCECloud.
 func newGCECloud(config io.Reader) (gceCloud *GCECloud, err error) {
 	var cloudConfig *CloudConfig
@@ -404,12 +391,6 @@ func CreateGCECloud(config *CloudConfig) (*GCECloud, error) {
 	}
 	containerService.UserAgent = userAgent
 
-	cloudkmsService, err := cloudkms.New(client)
-	if err != nil {
-		return nil, err
-	}
-	cloudkmsService.UserAgent = userAgent
-
 	// ProjectID and.NetworkProjectID may be project number or name.
 	projID, netProjID := tryConvertToProjectNames(config.ProjectID, config.NetworkProjectID, service)
 	onXPN := projID != netProjID
@@ -474,7 +455,6 @@ func CreateGCECloud(config *CloudConfig) (*GCECloud, error) {
 		serviceAlpha:             serviceAlpha,
 		serviceBeta:              serviceBeta,
 		containerService:         containerService,
-		cloudkmsService:          cloudkmsService,
 		projectID:                projID,
 		networkProjectID:         netProjID,
 		onXPN:                    onXPN,
@@ -494,14 +474,6 @@ func CreateGCECloud(config *CloudConfig) (*GCECloud, error) {
 
 	gce.manager = &gceServiceManager{gce}
 
-	// Registering the KMS plugin only the first time.
-	kmsPluginRegisterOnce.Do(func() {
-		// Register the Google Cloud KMS based service in the KMS plugin registry.
-		encryptionconfig.KMSPluginRegistry.Register(KMSServiceName, func(config io.Reader) (envelope.Service, error) {
-			return gce.getGCPCloudKMSService(config)
-		})
-	})
-
 	return gce, nil
 }
 

pkg/cloudprovider/providers/gce/kms.go

@@ -1,167 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package gce
-
-import (
-	"encoding/base64"
-	"fmt"
-	"io"
-
-	"github.com/golang/glog"
-	cloudkms "google.golang.org/api/cloudkms/v1"
-	"google.golang.org/api/googleapi"
-	gcfg "gopkg.in/gcfg.v1"
-	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
-)
-
-const (
-	// KMSServiceName is the name of the cloudkms provider registered by this cloud.
-	KMSServiceName = "gcp-cloudkms"
-
-	defaultGKMSKeyRing         = "google-container-engine"
-	defaultGKMSKeyRingLocation = "global"
-)
-
-// gkmsConfig contains the GCE specific KMS configuration for setting up a KMS connection.
-type gkmsConfig struct {
-	Global struct {
-		// location is the KMS location of the KeyRing to be used for encryption.
-		// It can be found by checking the available KeyRings in the IAM UI.
-		// This is not the same as the GCP location of the project.
-		// +optional
-		Location string `gcfg:"kms-location"`
-		// keyRing is the keyRing of the hosted key to be used. The default value is "google-kubernetes".
-		// +optional
-		KeyRing string `gcfg:"kms-keyring"`
-		// cryptoKey is the name of the key to be used for encryption of Data-Encryption-Keys.
-		CryptoKey string `gcfg:"kms-cryptokey"`
-	}
-}
-
-// readGCPCloudKMSConfig parses and returns the configuration parameters for Google Cloud KMS.
-func readGCPCloudKMSConfig(reader io.Reader) (*gkmsConfig, error) {
-	cfg := &gkmsConfig{}
-	if err := gcfg.FatalOnly(gcfg.ReadInto(cfg, reader)); err != nil {
-		glog.Errorf("Couldn't read Google Cloud KMS config: %v", err)
-		return nil, err
-	}
-	return cfg, nil
-}
-
-// gkmsService provides Encrypt and Decrypt methods which allow cryptographic operations
-// using Google Cloud KMS service.
-type gkmsService struct {
-	parentName      string
-	cloudkmsService *cloudkms.Service
-}
-
-// getGCPCloudKMSService provides a Google Cloud KMS based implementation of envelope.Service.
-func (gce *GCECloud) getGCPCloudKMSService(config io.Reader) (envelope.Service, error) {
-	kmsConfig, err := readGCPCloudKMSConfig(config)
-	if err != nil {
-		return nil, err
-	}
-
-	// Hosting on GCE/GKE with Google KMS encryption provider
-	cloudkmsService := gce.GetKMSService()
-
-	// Set defaults for location and keyRing.
-	location := kmsConfig.Global.Location
-	if len(location) == 0 {
-		location = defaultGKMSKeyRingLocation
-	}
-	keyRing := kmsConfig.Global.KeyRing
-	if len(keyRing) == 0 {
-		keyRing = defaultGKMSKeyRing
-	}
-
-	cryptoKey := kmsConfig.Global.CryptoKey
-	if len(cryptoKey) == 0 {
-		return nil, fmt.Errorf("missing cryptoKey for cloudprovided KMS: " + KMSServiceName)
-	}
-
-	parentName := fmt.Sprintf("projects/%s/locations/%s", gce.projectID, location)
-
-	// Create the keyRing if it does not exist yet
-	_, err = cloudkmsService.Projects.Locations.KeyRings.Create(parentName,
-		&cloudkms.KeyRing{}).KeyRingId(keyRing).Do()
-	if err != nil && unrecoverableCreationError(err) {
-		return nil, err
-	}
-	parentName = parentName + "/keyRings/" + keyRing
-
-	// Create the cryptoKey if it does not exist yet
-	_, err = cloudkmsService.Projects.Locations.KeyRings.CryptoKeys.Create(parentName,
-		&cloudkms.CryptoKey{
-			Purpose: "ENCRYPT_DECRYPT",
-		}).CryptoKeyId(cryptoKey).Do()
-	if err != nil && unrecoverableCreationError(err) {
-		return nil, err
-	}
-	parentName = parentName + "/cryptoKeys/" + cryptoKey
-
-	service := &gkmsService{
-		parentName:      parentName,
-		cloudkmsService: cloudkmsService,
-	}
-
-	// Sanity check before startup. For non-GCP clusters, the user's account may not have permissions to create
-	// the key. We need to verify the existence of the key before apiserver startup.
-	_, err = service.Encrypt([]byte("test"))
-	if err != nil {
-		return nil, fmt.Errorf("failed to encrypt data using Google cloudkms, using key %s. Ensure that the keyRing and cryptoKey exist. Got error: %v", parentName, err)
-	}
-
-	return service, nil
-}
-
-// Decrypt decrypts a base64 representation of encrypted bytes.
-func (t *gkmsService) Decrypt(data string) ([]byte, error) {
-	resp, err := t.cloudkmsService.Projects.Locations.KeyRings.CryptoKeys.
-		Decrypt(t.parentName, &cloudkms.DecryptRequest{
-			Ciphertext: data,
-		}).Do()
-	if err != nil {
-		return nil, err
-	}
-	return base64.StdEncoding.DecodeString(resp.Plaintext)
-}
-
-// Encrypt encrypts bytes, and returns base64 representation of the ciphertext.
-func (t *gkmsService) Encrypt(data []byte) (string, error) {
-	resp, err := t.cloudkmsService.Projects.Locations.KeyRings.CryptoKeys.
-		Encrypt(t.parentName, &cloudkms.EncryptRequest{
-			Plaintext: base64.StdEncoding.EncodeToString(data),
-		}).Do()
-	if err != nil {
-		return "", err
-	}
-	return resp.Ciphertext, nil
-}
-
-// unrecoverableCreationError decides if Kubernetes should ignore the encountered Google KMS
-// error. Only to be used for errors seen while creating a KeyRing or CryptoKey.
-func unrecoverableCreationError(err error) bool {
-	apiError, isAPIError := err.(*googleapi.Error)
-	// 409 means the object exists.
-	// 403 means we do not have permission to create the object, the user must do it.
-	// Else, it is an unrecoverable error.
-	if !isAPIError || (apiError.Code != 409 && apiError.Code != 403) {
-		return true
-	}
-	return false
-}

vendor/BUILD

@@ -370,7 +370,6 @@ filegroup(
         "//vendor/golang.org/x/time/rate:all-srcs",
         "//vendor/golang.org/x/tools/container/intsets:all-srcs",
         "//vendor/golang.org/x/tools/go/vcs:all-srcs",
-        "//vendor/google.golang.org/api/cloudkms/v1:all-srcs",
         "//vendor/google.golang.org/api/cloudmonitoring/v2beta2:all-srcs",
         "//vendor/google.golang.org/api/compute/v0.alpha:all-srcs",
         "//vendor/google.golang.org/api/compute/v0.beta:all-srcs",

vendor/google.golang.org/api/cloudkms/v1/BUILD

@@ -1,28 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
-
-go_library(
-    name = "go_default_library",
-    srcs = ["cloudkms-gen.go"],
-    importpath = "google.golang.org/api/cloudkms/v1",
-    visibility = ["//visibility:public"],
-    deps = [
-        "//vendor/golang.org/x/net/context:go_default_library",
-        "//vendor/golang.org/x/net/context/ctxhttp:go_default_library",
-        "//vendor/google.golang.org/api/gensupport:go_default_library",
-        "//vendor/google.golang.org/api/googleapi:go_default_library",
-    ],
-)
-
-filegroup(
-    name = "package-srcs",
-    srcs = glob(["**"]),
-    tags = ["automanaged"],
-    visibility = ["//visibility:private"],
-)
-
-filegroup(
-    name = "all-srcs",
-    srcs = [":package-srcs"],
-    tags = ["automanaged"],
-    visibility = ["//visibility:public"],
-)