Run kms-plugin in its own POD.

build/lib/release.sh

@@ -409,7 +409,6 @@ function kube::release::package_kube_manifests_tarball() {
   cp "${src_dir}/cluster-autoscaler.manifest" "${dst_dir}/"
   cp "${src_dir}/etcd.manifest" "${dst_dir}"
   cp "${src_dir}/kube-scheduler.manifest" "${dst_dir}"
-  cp "${src_dir}/kms-plugin-container.manifest" "${dst_dir}"
   cp "${src_dir}/kube-apiserver.manifest" "${dst_dir}"
   cp "${src_dir}/abac-authz-policy.jsonl" "${dst_dir}"
   cp "${src_dir}/kube-controller-manager.manifest" "${dst_dir}"

cluster/gce/gci/apiserver_manifest_test.go

@@ -17,11 +17,13 @@ limitations under the License.
 package gci
 
 import (
+	"bytes"
 	"encoding/base64"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"reflect"
 	"strings"
 	"testing"
 
@@ -53,87 +55,93 @@ readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests
 readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty
 readonly LOG_OWNER_USER=$(id -un)
 readonly LOG_OWNER_GROUP=$(id -gn)
+{{if .EncryptionProviderConfig}}
 ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}}
+{{end}}
 ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}}
-readonly ETCD_KMS_KEY_ID={{.ETCDKMSKeyID}}
+{{if .CloudKMSIntegration}}
+readonly CLOUD_KMS_INTEGRATION=true
+{{end}}
 `
 	kubeAPIServerManifestFileName = "kube-apiserver.manifest"
-	kmsPluginManifestFileName     = "kms-plugin-container.manifest"
 	kubeAPIServerStartFuncName    = "start-kube-apiserver"
-
-	// Position of containers within a pod manifest
-	kmsPluginContainerIndex        = 0
-	apiServerContainerIndexNoKMS   = 0
-	apiServerContainerIndexWithKMS = 1
-
-	//	command": [
-	//   "/bin/sh", - Index 0
-	//   "-c",      - Index 1
-	//   "exec /usr/local/bin/kube-apiserver " - Index 2
-	execArgsIndex = 2
-
-	socketVolumeMountIndexKMSPlugin = 1
-	socketVolumeMountIndexAPIServer = 0
 )
 
 type kubeAPIServerEnv struct {
 	KubeHome                     string
-	EncryptionProviderConfig     string
 	EncryptionProviderConfigPath string
-	ETCDKMSKeyID                 string
+	EncryptionProviderConfig     string
+	CloudKMSIntegration          bool
 }
 
 type kubeAPIServerManifestTestCase struct {
 	*ManifestTestCase
-	apiServerContainer v1.Container
-	kmsPluginContainer v1.Container
 }
 
 func newKubeAPIServerManifestTestCase(t *testing.T) *kubeAPIServerManifestTestCase {
 	return &kubeAPIServerManifestTestCase{
-		ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, []string{kmsPluginManifestFileName}),
+		ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, nil),
 	}
 }
 
-func (c *kubeAPIServerManifestTestCase) mustLoadContainers() {
+func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv, kubeEnv string) {
+	c.mustInvokeFunc(kubeEnv, e)
 	c.mustLoadPodFromManifest()
-
-	switch len(c.pod.Spec.Containers) {
-	case 1:
-		c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexNoKMS]
-	case 2:
-		c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexWithKMS]
-		c.kmsPluginContainer = c.pod.Spec.Containers[kmsPluginContainerIndex]
-	default:
-		c.t.Fatalf("got %d containers in apiserver pod, want 1 or 2", len(c.pod.Spec.Containers))
-	}
-}
-
-func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv) {
-	c.mustInvokeFunc(deployHelperEnv, e)
-	c.mustLoadContainers()
-}
-
-func getEncryptionProviderConfigFlag(path string) string {
-	return fmt.Sprintf("--experimental-encryption-provider-config=%s", path)
 }
 
 func TestEncryptionProviderFlag(t *testing.T) {
-	c := newKubeAPIServerManifestTestCase(t)
+	var (
-	defer c.tearDown()
+		//	command": [
+		//   "/bin/sh", - Index 0
+		//   "-c",      - Index 1
+		//   "exec /usr/local/bin/kube-apiserver " - Index 2
+		execArgsIndex        = 2
+		encryptionConfigFlag = "--experimental-encryption-provider-config"
+	)
 
-	e := kubeAPIServerEnv{
+	testCases := []struct {
-		KubeHome:                     c.kubeHome,
+		desc                     string
-		EncryptionProviderConfig:     base64.StdEncoding.EncodeToString([]byte("FOO")),
+		encryptionProviderConfig string
-		EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
+		wantFlag                 bool
+	}{
+		{
+			desc: "ENCRYPTION_PROVIDER_CONFIG is set",
+			encryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("foo")),
+			wantFlag:                 true,
+		},
+		{
+			desc: "ENCRYPTION_PROVIDER_CONFIG is not set",
+			encryptionProviderConfig: "",
+			wantFlag:                 false,
+		},
 	}
 
-	c.invokeTest(e)
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			c := newKubeAPIServerManifestTestCase(t)
+			defer c.tearDown()
 
-	expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
+			e := kubeAPIServerEnv{
-	execArgs := c.apiServerContainer.Command[execArgsIndex]
+				KubeHome:                     c.kubeHome,
-	if !strings.Contains(execArgs, expectedFlag) {
+				EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
-		c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
+				EncryptionProviderConfig:     tc.encryptionProviderConfig,
+			}
+
+			c.invokeTest(e, deployHelperEnv)
+
+			execArgs := c.pod.Spec.Containers[0].Command[execArgsIndex]
+			flagIsInArg := strings.Contains(execArgs, encryptionConfigFlag)
+			flag := fmt.Sprintf("%s=%s", encryptionConfigFlag, e.EncryptionProviderConfigPath)
+
+			switch {
+			case tc.wantFlag && !flagIsInArg:
+				t.Fatalf("Got %q,\n want flags to contain %q", execArgs, flag)
+			case !tc.wantFlag && flagIsInArg:
+				t.Fatalf("Got %q,\n do not want flags to contain %q", execArgs, encryptionConfigFlag)
+			case tc.wantFlag && flagIsInArg && !strings.Contains(execArgs, flag):
+				t.Fatalf("Got flags: %q, want it to contain %q", execArgs, flag)
+			}
+		})
 	}
 }
 
@@ -144,8 +152,8 @@ func TestEncryptionProviderConfig(t *testing.T) {
 	p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
 	e := kubeAPIServerEnv{
 		KubeHome:                     c.kubeHome,
-		EncryptionProviderConfig:     base64.StdEncoding.EncodeToString([]byte("FOO")),
 		EncryptionProviderConfigPath: p,
+		EncryptionProviderConfig:     base64.StdEncoding.EncodeToString([]byte("foo")),
 	}
 
 	c.mustInvokeFunc(deployHelperEnv, e)
@@ -153,60 +161,91 @@ func TestEncryptionProviderConfig(t *testing.T) {
 	if _, err := os.Stat(p); err != nil {
 		c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
 	}
-}
 
-// TestKMSEncryptionProviderConfig asserts that if ETCD_KMS_KEY_ID is set then start-kube-apiserver will produce
+	got, err := ioutil.ReadFile(p)
-// EncryptionProviderConfig file of type KMS and inject experimental-encryption-provider-config startup flag.
-func TestKMSEncryptionProviderConfig(t *testing.T) {
-	c := newKubeAPIServerManifestTestCase(t)
-	defer c.tearDown()
-
-	e := kubeAPIServerEnv{
-		KubeHome:                     c.kubeHome,
-		EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
-		ETCDKMSKeyID:                 "FOO",
-	}
-
-	c.invokeTest(e)
-
-	expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
-	execArgs := c.apiServerContainer.Command[execArgsIndex]
-	if !strings.Contains(execArgs, expectedFlag) {
-		c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
-	}
-
-	p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
-	if _, err := os.Stat(p); err != nil {
-		c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
-	}
-
-	d, err := ioutil.ReadFile(p)
 	if err != nil {
 		c.t.Fatalf("Failed to read encryption provider config %s", p)
 	}
 
-	if !strings.Contains(string(d), "name: grpc-kms-provider") {
+	want := []byte("foo")
-		c.t.Fatalf("Got %s\n, wanted encryption provider config to be of type grpc-kms", string(d))
+	if !bytes.Equal(got, want) {
+		c.t.Fatalf("got encryptionConfig:\n%q\n, want encryptionConfig:\n%q", got, want)
 	}
 }
 
-func TestKMSPluginAndAPIServerSharedVolume(t *testing.T) {
+func TestKMSIntegration(t *testing.T) {
-	c := newKubeAPIServerManifestTestCase(t)
+	var (
-	defer c.tearDown()
+		socketPath  = "/var/run/kmsplugin"
-
+		dirOrCreate = v1.HostPathType(v1.HostPathDirectoryOrCreate)
-	var e = kubeAPIServerEnv{
+		socketName  = "kmssocket"
-		KubeHome:                     c.kubeHome,
+	)
-		EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
+	testCases := []struct {
-		ETCDKMSKeyID:                 "FOO",
+		desc                string
+		cloudKMSIntegration bool
+		wantVolume          v1.Volume
+		wantVolMount        v1.VolumeMount
+	}{
+		{
+			desc:                "CLOUD_KMS_INTEGRATION is set",
+			cloudKMSIntegration: true,
+			wantVolume: v1.Volume{
+				Name: socketName,
+				VolumeSource: v1.VolumeSource{
+					HostPath: &v1.HostPathVolumeSource{
+						Path: socketPath,
+						Type: &dirOrCreate,
+					},
+				},
+			},
+			wantVolMount: v1.VolumeMount{
+				Name:      socketName,
+				MountPath: socketPath,
+			},
+		},
+		{
+			desc:                "CLOUD_KMS_INTEGRATION is not set",
+			cloudKMSIntegration: false,
+		},
 	}
 
-	c.invokeTest(e)
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			c := newKubeAPIServerManifestTestCase(t)
+			defer c.tearDown()
 
-	k := c.kmsPluginContainer.VolumeMounts[socketVolumeMountIndexKMSPlugin].MountPath
+			var e = kubeAPIServerEnv{
-	a := c.apiServerContainer.VolumeMounts[socketVolumeMountIndexAPIServer].MountPath
+				KubeHome:                     c.kubeHome,
+				EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
+				EncryptionProviderConfig:     base64.StdEncoding.EncodeToString([]byte("foo")),
+				CloudKMSIntegration:          tc.cloudKMSIntegration,
+			}
 
-	if k != a {
+			c.invokeTest(e, deployHelperEnv)
-		t.Fatalf("Got %s!=%s, wanted KMSPlugin VolumeMount #1:%s to be equal to kube-apiserver VolumeMount #0:%s",
+			// By this point, we can be sure that kube-apiserver manifest is a valid POD.
-			k, a, k, a)
+
+			var gotVolume v1.Volume
+			for _, v := range c.pod.Spec.Volumes {
+				if v.Name == socketName {
+					gotVolume = v
+					break
+				}
+			}
+
+			if !reflect.DeepEqual(gotVolume, tc.wantVolume) {
+				t.Errorf("got volume %v, want %v", gotVolume, tc.wantVolume)
+			}
+
+			var gotVolumeMount v1.VolumeMount
+			for _, v := range c.pod.Spec.Containers[0].VolumeMounts {
+				if v.Name == socketName {
+					gotVolumeMount = v
+					break
+				}
+			}
+
+			if !reflect.DeepEqual(gotVolumeMount, tc.wantVolMount) {
+				t.Errorf("got volumeMount %v, want %v", gotVolumeMount, tc.wantVolMount)
+			}
+		})
 	}
 }

cluster/gce/gci/configure-helper.sh

@@ -1771,29 +1771,11 @@ function start-kube-apiserver {
     container_env="\"env\":[{${container_env}}],"
   fi
 
-  if [[ -n "${ETCD_KMS_KEY_ID:-}" ]]; then
+  local -r src_file="${src_dir}/kube-apiserver.manifest"
-    ENCRYPTION_PROVIDER_CONFIG=$(cat << EOM | base64 | tr -d '\r\n'
-kind: EncryptionConfig
-apiVersion: v1
-resources:
-  - resources:
-    - secrets
-    providers:
-    - kms:
-       name: grpc-kms-provider
-       cachesize: 1000
-       endpoint: unix:///var/run/kmsplugin/socket.sock
-EOM
-)
-  fi
 
-  if [[ -n "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then
+  # params is passed by reference, so no "$"
-    ENCRYPTION_PROVIDER_CONFIG_PATH="${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml}"
+  setup-etcd-encryption "${src_file}" params
-    echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${ENCRYPTION_PROVIDER_CONFIG_PATH}"
-    params+=" --experimental-encryption-provider-config=${ENCRYPTION_PROVIDER_CONFIG_PATH}"
-  fi
 
-  src_file="${src_dir}/kube-apiserver.manifest"
   # Evaluate variables.
   local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}"
   sed -i -e "s@{{params}}@${params}@g" "${src_file}"
@@ -1822,67 +1804,116 @@ EOM
   sed -i -e "s@{{image_policy_webhook_config_mount}}@${image_policy_webhook_config_mount}@g" "${src_file}"
   sed -i -e "s@{{image_policy_webhook_config_volume}}@${image_policy_webhook_config_volume}@g" "${src_file}"
 
-  if [[ -z "${ETCD_KMS_KEY_ID:-}" ]]; then
+  cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}"
-    # Removing KMS related placeholders.
+}
-    sed -i -e " {
-      s@{{kms_plugin_container}}@@
 
-      s@{{kms_socket_mount}}@@
+# Sets-up etcd encryption.
+# Configuration of etcd level encryption consists of the following steps:
+# 1. Writing encryption provider config to disk
+# 2. Adding experimental-encryption-provider-config flag to kube-apiserver
+# 3. Add kms-socket-vol and kms-socket-vol-mnt to enable communication with kms-plugin (if requested)
+#
+# Expects parameters:
+# $1 - path to kube-apiserver template
+# $2 - kube-apiserver startup flags (must be passed by reference)
+#
+# Assumes vars (supplied via kube-env):
+# ENCRYPTION_PROVIDER_CONFIG
+# CLOUD_KMS_INTEGRATION
+# ENCRYPTION_PROVIDER_CONFIG_PATH (will default to /etc/srv/kubernetes/encryption-provider-config.yml)
+function setup-etcd-encryption {
+  local kube_apiserver_template_path
+  local -n kube_api_server_params
+  local default_encryption_provider_config_vol
+  local default_encryption_provider_config_vol_mnt
+  local encryption_provider_config_vol_mnt
+  local encryption_provider_config_vol
+  local default_kms_socket_dir
+  local default_kms_socket_vol_mnt
+  local default_kms_socket_vol
+  local kms_socket_vol_mnt
+  local kms_socket_vol
+  local encryption_provider_config_path
+
+  kube_apiserver_template_path="$1"
+  if [[ -z "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then
+    sed -i -e " {
       s@{{encryption_provider_mount}}@@
-
-      s@{{kms_socket_volume}}@@
       s@{{encryption_provider_volume}}@@
-    } " "${src_file}"
+      s@{{kms_socket_mount}}@@
-  else
+      s@{{kms_socket_volume}}@@
-    local kms_plugin_src_file="${src_dir}/kms-plugin-container.manifest"
+    } " "${kube_apiserver_template_path}"
-
+    return
-    if [[ ! -f "${kms_plugin_src_file}" ]]; then
-      echo "Error: KMS Integration was requested, but "${kms_plugin_src_file}" is missing."
-      exit 1
-    fi
-
-    if [[ ! -f "${ENCRYPTION_PROVIDER_CONFIG_PATH}" ]]; then
-      echo "Error: KMS Integration was requested, but "${ENCRYPTION_PROVIDER_CONFIG_PATH}" is missing."
-      exit 1
-    fi
-
-    # TODO: Validate that the encryption config is for KMS.
-
-    local kms_socket_dir="/var/run/kmsplugin"
-
-    # kms_socket_mnt is used by both kms_plugin and kube-apiserver - this is how these containers talk.
-    local kms_socket_mnt="{ \"name\": \"kmssocket\", \"mountPath\": \"${kms_socket_dir}\", \"readOnly\": false}"
-
-    local kms_socket_vol="{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}"
-    local kms_path_to_socket="${kms_socket_dir}/socket.sock"
-
-    local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"readOnly\": true}"
-    local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"type\": \"File\"}}"
-
-    # TODO these are used in other places, convert to global.
-    local gce_conf_path="/etc/gce.conf"
-    local cloud_config_mount="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}"
-
-    local kms_plugin_container=$(echo $(sed " {
-      s@{{kms_key_uri}}@${ETCD_KMS_KEY_ID}@
-      s@{{gce_conf_path}}@${gce_conf_path}@
-      s@{{kms_path_to_socket}}@${kms_path_to_socket}@
-      s@{{kms_socket_mount}}@${kms_socket_mnt}@
-      s@{{cloud_config_mount}}@${cloud_config_mount}@
-    } " "${kms_plugin_src_file}") | tr "\n" "\\n")
-
-    sed -i -e " {
-      s@{{kms_plugin_container}}@${kms_plugin_container},@
-
-      s@{{kms_socket_mount}}@${kms_socket_mnt},@
-      s@{{encryption_provider_mount}}@${encryption_provider_mnt},@
-
-      s@{{kms_socket_volume}}@${kms_socket_vol},@
-      s@{{encryption_provider_volume}}@${encryption_provider_vol},@
-    } " "${src_file}"
   fi
 
-  cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}"
+  kube_api_server_params="$2"
+  encryption_provider_config_path=${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml}
+
+  echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}"
+  kube_api_server_params+=" --experimental-encryption-provider-config=${encryption_provider_config_path}"
+
+  default_encryption_provider_config_vol=$(echo "{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${encryption_provider_config_path}\", \"type\": \"File\"}}" | base64 | tr -d '\r\n')
+  default_encryption_provider_config_vol_mnt=$(echo "{ \"name\": \"encryptionconfig\", \"mountPath\": \"${encryption_provider_config_path}\", \"readOnly\": true}" | base64 | tr -d '\r\n')
+
+  encryption_provider_config_vol_mnt=$(echo "${ENCRYPTION_PROVIDER_CONFIG_VOL_MNT:-"${default_encryption_provider_config_vol_mnt}"}" | base64 --decode)
+  encryption_provider_config_vol=$(echo "${ENCRYPTION_PROVIDER_CONFIG_VOL:-"${default_encryption_provider_config_vol}"}" | base64 --decode)
+  sed -i -e " {
+    s@{{encryption_provider_mount}}@${encryption_provider_config_vol_mnt},@
+    s@{{encryption_provider_volume}}@${encryption_provider_config_vol},@
+  } " "${kube_apiserver_template_path}"
+
+  if [[ -n "${CLOUD_KMS_INTEGRATION:-}" ]]; then
+    default_kms_socket_dir="/var/run/kmsplugin"
+    default_kms_socket_vol_mnt=$(echo "{ \"name\": \"kmssocket\", \"mountPath\": \"${default_kms_socket_dir}\", \"readOnly\": false}" | base64 | tr -d '\r\n')
+    default_kms_socket_vol=$(echo "{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${default_kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}" | base64 | tr -d '\r\n')
+
+    kms_socket_vol_mnt=$(echo "${KMS_PLUGIN_SOCKET_VOL_MNT:-"${default_kms_socket_vol_mnt}"}" | base64 --decode)
+    kms_socket_vol=$(echo "${KMS_PLUGIN_SOCKET_VOL:-"${default_kms_socket_vol}"}" | base64 --decode)
+    sed -i -e " {
+      s@{{kms_socket_mount}}@${kms_socket_vol_mnt},@
+      s@{{kms_socket_volume}}@${kms_socket_vol},@
+    } " "${kube_apiserver_template_path}"
+  else
+    sed -i -e " {
+      s@{{kms_socket_mount}}@@
+      s@{{kms_socket_volume}}@@
+    } " "${kube_apiserver_template_path}"
+  fi
+}
+
+
+# Applies encryption provider config.
+# This function may be triggered in two scenarios:
+# 1. Decryption of etcd
+# 2. Encryption of etcd is added after the cluster is deployed
+# Both cases require that the existing secrets in etcd be re-proceeded.
+#
+# Assumes vars (supplied via kube-env):
+# ENCRYPTION_PROVIDER_CONFIG_FORCE
+function apply-encryption-config() {
+  if [[ "${ENCRYPTION_PROVIDER_CONFIG_FORCE:-false}" == "false" ]]; then
+    return
+  fi
+
+  # need kube-apiserver to be ready
+  until kubectl get secret; do
+    sleep ${ENCRYPTION_PROVIDER_CONFIG_FORCE_DELAY:-5}
+  done
+
+  retries=${ENCRYPTION_PROVIDER_CONFIG_FORCE_RETRIES:-5}
+  # The command below may fail when a conflict is detected during an update on a secret (something
+  # else updated the secret in the middle of our update).
+  # TODO: Retry only on errors caused by a conflict.
+  until (( retries == 0 )); do
+    # forces all secrets to be re-written to etcd, and in the process either encrypting or decrypting them
+    # https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/
+    if kubectl get secrets --all-namespaces -o json | kubectl replace -f -; then
+      break
+    fi
+
+    (( retries-- ))
+    sleep "${ENCRYPTION_PROVIDER_CONFIG_FORCE_RETRY_SLEEP:-3}"
+  done
 }
 
 # Starts kubernetes controller manager.
@@ -2778,6 +2809,7 @@ function main() {
     start-kube-addons
     start-cluster-autoscaler
     start-lb-controller
+    apply-encryption-config &
   else
     if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
       start-kube-proxy

cluster/gce/gci/configure_helper_test.go

@@ -143,7 +143,7 @@ func (c *ManifestTestCase) mustLoadPodFromManifest() {
 	}
 
 	if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), json, &c.pod); err != nil {
-		c.t.Fatalf("Failed to decode manifest: %v", err)
+		c.t.Fatalf("Failed to decode manifest:\n%s\nerror: %v", json, err)
 	}
 }
 

cluster/gce/manifests/BUILD

@@ -18,7 +18,6 @@ filegroup(
         "etcd.manifest",
         "etcd-empty-dir-cleanup.yaml",
         "glbc.manifest",
-        "kms-plugin-container.manifest",
         "kube-addon-manager.yaml",
         "kube-apiserver.manifest",
         "kube-controller-manager.manifest",

cluster/gce/manifests/kms-plugin-container.manifest

@@ -1,8 +0,0 @@
-{
-  "name": "kms-plugin",
-  "image": "gcr.io/google-containers/k8s-cloud-kms-plugin:v0.1.1",
-  "command": ["/k8s-cloud-kms-plugin", "--key-uri={{kms_key_uri}}", "--path-to-unix-socket={{kms_path_to_socket}}", "--gce-config={{gce_conf_path}}", "--logtostderr", "2>\&1"],
-  "livenessProbe": { "httpGet": {"host": "127.0.0.1", "port": 8081, "path": "/healthz"}, "initialDelaySeconds": 3, "timeoutSeconds": 3},
-  "ports":[{ "name": "healthz", "containerPort": 8081, "hostPort": 8081}, { "name": "metrics", "containerPort": 8082, "hostPort": 8082}],
-  "volumeMounts": [{{cloud_config_mount}}, {{kms_socket_mount}}]
-}

cluster/gce/manifests/kube-apiserver.manifest

@@ -16,7 +16,6 @@
 "spec":{
 "hostNetwork": true,
 "containers":[
-    {{kms_plugin_container}}
     {
     "name": "kube-apiserver",
     "image": "{{pillar['kube_docker_registry']}}/kube-apiserver:{{pillar['kube-apiserver_docker_tag']}}",