diff --git a/build/lib/release.sh b/build/lib/release.sh index 1275e6e1d96..936067d3966 100644 --- a/build/lib/release.sh +++ b/build/lib/release.sh @@ -409,7 +409,6 @@ function kube::release::package_kube_manifests_tarball() { cp "${src_dir}/cluster-autoscaler.manifest" "${dst_dir}/" cp "${src_dir}/etcd.manifest" "${dst_dir}" cp "${src_dir}/kube-scheduler.manifest" "${dst_dir}" - cp "${src_dir}/kms-plugin-container.manifest" "${dst_dir}" cp "${src_dir}/kube-apiserver.manifest" "${dst_dir}" cp "${src_dir}/abac-authz-policy.jsonl" "${dst_dir}" cp "${src_dir}/kube-controller-manager.manifest" "${dst_dir}" diff --git a/cluster/gce/gci/apiserver_manifest_test.go b/cluster/gce/gci/apiserver_manifest_test.go index 299598aed27..83547360b8e 100644 --- a/cluster/gce/gci/apiserver_manifest_test.go +++ b/cluster/gce/gci/apiserver_manifest_test.go @@ -17,11 +17,13 @@ limitations under the License. package gci import ( + "bytes" "encoding/base64" "fmt" "io/ioutil" "os" "path/filepath" + "reflect" "strings" "testing" @@ -53,87 +55,93 @@ readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty readonly LOG_OWNER_USER=$(id -un) readonly LOG_OWNER_GROUP=$(id -gn) +{{if .EncryptionProviderConfig}} ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}} +{{end}} ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}} -readonly ETCD_KMS_KEY_ID={{.ETCDKMSKeyID}} +{{if .CloudKMSIntegration}} +readonly CLOUD_KMS_INTEGRATION=true +{{end}} ` kubeAPIServerManifestFileName = "kube-apiserver.manifest" - kmsPluginManifestFileName = "kms-plugin-container.manifest" kubeAPIServerStartFuncName = "start-kube-apiserver" - - // Position of containers within a pod manifest - kmsPluginContainerIndex = 0 - apiServerContainerIndexNoKMS = 0 - apiServerContainerIndexWithKMS = 1 - - // command": [ - // "/bin/sh", - Index 0 - // "-c", - Index 1 - // "exec /usr/local/bin/kube-apiserver " - Index 2 - execArgsIndex = 2 - - socketVolumeMountIndexKMSPlugin = 1 - socketVolumeMountIndexAPIServer = 0 ) type kubeAPIServerEnv struct { KubeHome string - EncryptionProviderConfig string EncryptionProviderConfigPath string - ETCDKMSKeyID string + EncryptionProviderConfig string + CloudKMSIntegration bool } type kubeAPIServerManifestTestCase struct { *ManifestTestCase - apiServerContainer v1.Container - kmsPluginContainer v1.Container } func newKubeAPIServerManifestTestCase(t *testing.T) *kubeAPIServerManifestTestCase { return &kubeAPIServerManifestTestCase{ - ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, []string{kmsPluginManifestFileName}), + ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, nil), } } -func (c *kubeAPIServerManifestTestCase) mustLoadContainers() { +func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv, kubeEnv string) { + c.mustInvokeFunc(kubeEnv, e) c.mustLoadPodFromManifest() - - switch len(c.pod.Spec.Containers) { - case 1: - c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexNoKMS] - case 2: - c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexWithKMS] - c.kmsPluginContainer = c.pod.Spec.Containers[kmsPluginContainerIndex] - default: - c.t.Fatalf("got %d containers in apiserver pod, want 1 or 2", len(c.pod.Spec.Containers)) - } -} - -func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv) { - c.mustInvokeFunc(deployHelperEnv, e) - c.mustLoadContainers() -} - -func getEncryptionProviderConfigFlag(path string) string { - return fmt.Sprintf("--experimental-encryption-provider-config=%s", path) } func TestEncryptionProviderFlag(t *testing.T) { - c := newKubeAPIServerManifestTestCase(t) - defer c.tearDown() + var ( + // command": [ + // "/bin/sh", - Index 0 + // "-c", - Index 1 + // "exec /usr/local/bin/kube-apiserver " - Index 2 + execArgsIndex = 2 + encryptionConfigFlag = "--experimental-encryption-provider-config" + ) - e := kubeAPIServerEnv{ - KubeHome: c.kubeHome, - EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("FOO")), - EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"), + testCases := []struct { + desc string + encryptionProviderConfig string + wantFlag bool + }{ + { + desc: "ENCRYPTION_PROVIDER_CONFIG is set", + encryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("foo")), + wantFlag: true, + }, + { + desc: "ENCRYPTION_PROVIDER_CONFIG is not set", + encryptionProviderConfig: "", + wantFlag: false, + }, } - c.invokeTest(e) + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + c := newKubeAPIServerManifestTestCase(t) + defer c.tearDown() - expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath) - execArgs := c.apiServerContainer.Command[execArgsIndex] - if !strings.Contains(execArgs, expectedFlag) { - c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag) + e := kubeAPIServerEnv{ + KubeHome: c.kubeHome, + EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"), + EncryptionProviderConfig: tc.encryptionProviderConfig, + } + + c.invokeTest(e, deployHelperEnv) + + execArgs := c.pod.Spec.Containers[0].Command[execArgsIndex] + flagIsInArg := strings.Contains(execArgs, encryptionConfigFlag) + flag := fmt.Sprintf("%s=%s", encryptionConfigFlag, e.EncryptionProviderConfigPath) + + switch { + case tc.wantFlag && !flagIsInArg: + t.Fatalf("Got %q,\n want flags to contain %q", execArgs, flag) + case !tc.wantFlag && flagIsInArg: + t.Fatalf("Got %q,\n do not want flags to contain %q", execArgs, encryptionConfigFlag) + case tc.wantFlag && flagIsInArg && !strings.Contains(execArgs, flag): + t.Fatalf("Got flags: %q, want it to contain %q", execArgs, flag) + } + }) } } @@ -144,8 +152,8 @@ func TestEncryptionProviderConfig(t *testing.T) { p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml") e := kubeAPIServerEnv{ KubeHome: c.kubeHome, - EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("FOO")), EncryptionProviderConfigPath: p, + EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("foo")), } c.mustInvokeFunc(deployHelperEnv, e) @@ -153,60 +161,91 @@ func TestEncryptionProviderConfig(t *testing.T) { if _, err := os.Stat(p); err != nil { c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err) } -} -// TestKMSEncryptionProviderConfig asserts that if ETCD_KMS_KEY_ID is set then start-kube-apiserver will produce -// EncryptionProviderConfig file of type KMS and inject experimental-encryption-provider-config startup flag. -func TestKMSEncryptionProviderConfig(t *testing.T) { - c := newKubeAPIServerManifestTestCase(t) - defer c.tearDown() - - e := kubeAPIServerEnv{ - KubeHome: c.kubeHome, - EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"), - ETCDKMSKeyID: "FOO", - } - - c.invokeTest(e) - - expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath) - execArgs := c.apiServerContainer.Command[execArgsIndex] - if !strings.Contains(execArgs, expectedFlag) { - c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag) - } - - p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml") - if _, err := os.Stat(p); err != nil { - c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err) - } - - d, err := ioutil.ReadFile(p) + got, err := ioutil.ReadFile(p) if err != nil { c.t.Fatalf("Failed to read encryption provider config %s", p) } - if !strings.Contains(string(d), "name: grpc-kms-provider") { - c.t.Fatalf("Got %s\n, wanted encryption provider config to be of type grpc-kms", string(d)) + want := []byte("foo") + if !bytes.Equal(got, want) { + c.t.Fatalf("got encryptionConfig:\n%q\n, want encryptionConfig:\n%q", got, want) } } -func TestKMSPluginAndAPIServerSharedVolume(t *testing.T) { - c := newKubeAPIServerManifestTestCase(t) - defer c.tearDown() - - var e = kubeAPIServerEnv{ - KubeHome: c.kubeHome, - EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"), - ETCDKMSKeyID: "FOO", +func TestKMSIntegration(t *testing.T) { + var ( + socketPath = "/var/run/kmsplugin" + dirOrCreate = v1.HostPathType(v1.HostPathDirectoryOrCreate) + socketName = "kmssocket" + ) + testCases := []struct { + desc string + cloudKMSIntegration bool + wantVolume v1.Volume + wantVolMount v1.VolumeMount + }{ + { + desc: "CLOUD_KMS_INTEGRATION is set", + cloudKMSIntegration: true, + wantVolume: v1.Volume{ + Name: socketName, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: socketPath, + Type: &dirOrCreate, + }, + }, + }, + wantVolMount: v1.VolumeMount{ + Name: socketName, + MountPath: socketPath, + }, + }, + { + desc: "CLOUD_KMS_INTEGRATION is not set", + cloudKMSIntegration: false, + }, } - c.invokeTest(e) + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + c := newKubeAPIServerManifestTestCase(t) + defer c.tearDown() - k := c.kmsPluginContainer.VolumeMounts[socketVolumeMountIndexKMSPlugin].MountPath - a := c.apiServerContainer.VolumeMounts[socketVolumeMountIndexAPIServer].MountPath + var e = kubeAPIServerEnv{ + KubeHome: c.kubeHome, + EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"), + EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("foo")), + CloudKMSIntegration: tc.cloudKMSIntegration, + } - if k != a { - t.Fatalf("Got %s!=%s, wanted KMSPlugin VolumeMount #1:%s to be equal to kube-apiserver VolumeMount #0:%s", - k, a, k, a) + c.invokeTest(e, deployHelperEnv) + // By this point, we can be sure that kube-apiserver manifest is a valid POD. + + var gotVolume v1.Volume + for _, v := range c.pod.Spec.Volumes { + if v.Name == socketName { + gotVolume = v + break + } + } + + if !reflect.DeepEqual(gotVolume, tc.wantVolume) { + t.Errorf("got volume %v, want %v", gotVolume, tc.wantVolume) + } + + var gotVolumeMount v1.VolumeMount + for _, v := range c.pod.Spec.Containers[0].VolumeMounts { + if v.Name == socketName { + gotVolumeMount = v + break + } + } + + if !reflect.DeepEqual(gotVolumeMount, tc.wantVolMount) { + t.Errorf("got volumeMount %v, want %v", gotVolumeMount, tc.wantVolMount) + } + }) } } diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 3979219806c..057cf69c88e 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1771,29 +1771,11 @@ function start-kube-apiserver { container_env="\"env\":[{${container_env}}]," fi - if [[ -n "${ETCD_KMS_KEY_ID:-}" ]]; then - ENCRYPTION_PROVIDER_CONFIG=$(cat << EOM | base64 | tr -d '\r\n' -kind: EncryptionConfig -apiVersion: v1 -resources: - - resources: - - secrets - providers: - - kms: - name: grpc-kms-provider - cachesize: 1000 - endpoint: unix:///var/run/kmsplugin/socket.sock -EOM -) - fi + local -r src_file="${src_dir}/kube-apiserver.manifest" - if [[ -n "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then - ENCRYPTION_PROVIDER_CONFIG_PATH="${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml}" - echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${ENCRYPTION_PROVIDER_CONFIG_PATH}" - params+=" --experimental-encryption-provider-config=${ENCRYPTION_PROVIDER_CONFIG_PATH}" - fi + # params is passed by reference, so no "$" + setup-etcd-encryption "${src_file}" params - src_file="${src_dir}/kube-apiserver.manifest" # Evaluate variables. local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}" sed -i -e "s@{{params}}@${params}@g" "${src_file}" @@ -1822,67 +1804,116 @@ EOM sed -i -e "s@{{image_policy_webhook_config_mount}}@${image_policy_webhook_config_mount}@g" "${src_file}" sed -i -e "s@{{image_policy_webhook_config_volume}}@${image_policy_webhook_config_volume}@g" "${src_file}" - if [[ -z "${ETCD_KMS_KEY_ID:-}" ]]; then - # Removing KMS related placeholders. - sed -i -e " { - s@{{kms_plugin_container}}@@ + cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}" +} - s@{{kms_socket_mount}}@@ +# Sets-up etcd encryption. +# Configuration of etcd level encryption consists of the following steps: +# 1. Writing encryption provider config to disk +# 2. Adding experimental-encryption-provider-config flag to kube-apiserver +# 3. Add kms-socket-vol and kms-socket-vol-mnt to enable communication with kms-plugin (if requested) +# +# Expects parameters: +# $1 - path to kube-apiserver template +# $2 - kube-apiserver startup flags (must be passed by reference) +# +# Assumes vars (supplied via kube-env): +# ENCRYPTION_PROVIDER_CONFIG +# CLOUD_KMS_INTEGRATION +# ENCRYPTION_PROVIDER_CONFIG_PATH (will default to /etc/srv/kubernetes/encryption-provider-config.yml) +function setup-etcd-encryption { + local kube_apiserver_template_path + local -n kube_api_server_params + local default_encryption_provider_config_vol + local default_encryption_provider_config_vol_mnt + local encryption_provider_config_vol_mnt + local encryption_provider_config_vol + local default_kms_socket_dir + local default_kms_socket_vol_mnt + local default_kms_socket_vol + local kms_socket_vol_mnt + local kms_socket_vol + local encryption_provider_config_path + + kube_apiserver_template_path="$1" + if [[ -z "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then + sed -i -e " { s@{{encryption_provider_mount}}@@ - - s@{{kms_socket_volume}}@@ s@{{encryption_provider_volume}}@@ - } " "${src_file}" - else - local kms_plugin_src_file="${src_dir}/kms-plugin-container.manifest" - - if [[ ! -f "${kms_plugin_src_file}" ]]; then - echo "Error: KMS Integration was requested, but "${kms_plugin_src_file}" is missing." - exit 1 - fi - - if [[ ! -f "${ENCRYPTION_PROVIDER_CONFIG_PATH}" ]]; then - echo "Error: KMS Integration was requested, but "${ENCRYPTION_PROVIDER_CONFIG_PATH}" is missing." - exit 1 - fi - - # TODO: Validate that the encryption config is for KMS. - - local kms_socket_dir="/var/run/kmsplugin" - - # kms_socket_mnt is used by both kms_plugin and kube-apiserver - this is how these containers talk. - local kms_socket_mnt="{ \"name\": \"kmssocket\", \"mountPath\": \"${kms_socket_dir}\", \"readOnly\": false}" - - local kms_socket_vol="{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}" - local kms_path_to_socket="${kms_socket_dir}/socket.sock" - - local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"readOnly\": true}" - local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"type\": \"File\"}}" - - # TODO these are used in other places, convert to global. - local gce_conf_path="/etc/gce.conf" - local cloud_config_mount="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}" - - local kms_plugin_container=$(echo $(sed " { - s@{{kms_key_uri}}@${ETCD_KMS_KEY_ID}@ - s@{{gce_conf_path}}@${gce_conf_path}@ - s@{{kms_path_to_socket}}@${kms_path_to_socket}@ - s@{{kms_socket_mount}}@${kms_socket_mnt}@ - s@{{cloud_config_mount}}@${cloud_config_mount}@ - } " "${kms_plugin_src_file}") | tr "\n" "\\n") - - sed -i -e " { - s@{{kms_plugin_container}}@${kms_plugin_container},@ - - s@{{kms_socket_mount}}@${kms_socket_mnt},@ - s@{{encryption_provider_mount}}@${encryption_provider_mnt},@ - - s@{{kms_socket_volume}}@${kms_socket_vol},@ - s@{{encryption_provider_volume}}@${encryption_provider_vol},@ - } " "${src_file}" + s@{{kms_socket_mount}}@@ + s@{{kms_socket_volume}}@@ + } " "${kube_apiserver_template_path}" + return fi - cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}" + kube_api_server_params="$2" + encryption_provider_config_path=${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml} + + echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}" + kube_api_server_params+=" --experimental-encryption-provider-config=${encryption_provider_config_path}" + + default_encryption_provider_config_vol=$(echo "{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${encryption_provider_config_path}\", \"type\": \"File\"}}" | base64 | tr -d '\r\n') + default_encryption_provider_config_vol_mnt=$(echo "{ \"name\": \"encryptionconfig\", \"mountPath\": \"${encryption_provider_config_path}\", \"readOnly\": true}" | base64 | tr -d '\r\n') + + encryption_provider_config_vol_mnt=$(echo "${ENCRYPTION_PROVIDER_CONFIG_VOL_MNT:-"${default_encryption_provider_config_vol_mnt}"}" | base64 --decode) + encryption_provider_config_vol=$(echo "${ENCRYPTION_PROVIDER_CONFIG_VOL:-"${default_encryption_provider_config_vol}"}" | base64 --decode) + sed -i -e " { + s@{{encryption_provider_mount}}@${encryption_provider_config_vol_mnt},@ + s@{{encryption_provider_volume}}@${encryption_provider_config_vol},@ + } " "${kube_apiserver_template_path}" + + if [[ -n "${CLOUD_KMS_INTEGRATION:-}" ]]; then + default_kms_socket_dir="/var/run/kmsplugin" + default_kms_socket_vol_mnt=$(echo "{ \"name\": \"kmssocket\", \"mountPath\": \"${default_kms_socket_dir}\", \"readOnly\": false}" | base64 | tr -d '\r\n') + default_kms_socket_vol=$(echo "{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${default_kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}" | base64 | tr -d '\r\n') + + kms_socket_vol_mnt=$(echo "${KMS_PLUGIN_SOCKET_VOL_MNT:-"${default_kms_socket_vol_mnt}"}" | base64 --decode) + kms_socket_vol=$(echo "${KMS_PLUGIN_SOCKET_VOL:-"${default_kms_socket_vol}"}" | base64 --decode) + sed -i -e " { + s@{{kms_socket_mount}}@${kms_socket_vol_mnt},@ + s@{{kms_socket_volume}}@${kms_socket_vol},@ + } " "${kube_apiserver_template_path}" + else + sed -i -e " { + s@{{kms_socket_mount}}@@ + s@{{kms_socket_volume}}@@ + } " "${kube_apiserver_template_path}" + fi +} + + +# Applies encryption provider config. +# This function may be triggered in two scenarios: +# 1. Decryption of etcd +# 2. Encryption of etcd is added after the cluster is deployed +# Both cases require that the existing secrets in etcd be re-proceeded. +# +# Assumes vars (supplied via kube-env): +# ENCRYPTION_PROVIDER_CONFIG_FORCE +function apply-encryption-config() { + if [[ "${ENCRYPTION_PROVIDER_CONFIG_FORCE:-false}" == "false" ]]; then + return + fi + + # need kube-apiserver to be ready + until kubectl get secret; do + sleep ${ENCRYPTION_PROVIDER_CONFIG_FORCE_DELAY:-5} + done + + retries=${ENCRYPTION_PROVIDER_CONFIG_FORCE_RETRIES:-5} + # The command below may fail when a conflict is detected during an update on a secret (something + # else updated the secret in the middle of our update). + # TODO: Retry only on errors caused by a conflict. + until (( retries == 0 )); do + # forces all secrets to be re-written to etcd, and in the process either encrypting or decrypting them + # https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/ + if kubectl get secrets --all-namespaces -o json | kubectl replace -f -; then + break + fi + + (( retries-- )) + sleep "${ENCRYPTION_PROVIDER_CONFIG_FORCE_RETRY_SLEEP:-3}" + done } # Starts kubernetes controller manager. @@ -2778,6 +2809,7 @@ function main() { start-kube-addons start-cluster-autoscaler start-lb-controller + apply-encryption-config & else if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then start-kube-proxy diff --git a/cluster/gce/gci/configure_helper_test.go b/cluster/gce/gci/configure_helper_test.go index 0297733acfe..1b1384403a2 100644 --- a/cluster/gce/gci/configure_helper_test.go +++ b/cluster/gce/gci/configure_helper_test.go @@ -143,7 +143,7 @@ func (c *ManifestTestCase) mustLoadPodFromManifest() { } if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), json, &c.pod); err != nil { - c.t.Fatalf("Failed to decode manifest: %v", err) + c.t.Fatalf("Failed to decode manifest:\n%s\nerror: %v", json, err) } } diff --git a/cluster/gce/manifests/BUILD b/cluster/gce/manifests/BUILD index 0530be7531e..011bf7acfaa 100644 --- a/cluster/gce/manifests/BUILD +++ b/cluster/gce/manifests/BUILD @@ -18,7 +18,6 @@ filegroup( "etcd.manifest", "etcd-empty-dir-cleanup.yaml", "glbc.manifest", - "kms-plugin-container.manifest", "kube-addon-manager.yaml", "kube-apiserver.manifest", "kube-controller-manager.manifest", diff --git a/cluster/gce/manifests/kms-plugin-container.manifest b/cluster/gce/manifests/kms-plugin-container.manifest deleted file mode 100644 index cb90c9fef30..00000000000 --- a/cluster/gce/manifests/kms-plugin-container.manifest +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "kms-plugin", - "image": "gcr.io/google-containers/k8s-cloud-kms-plugin:v0.1.1", - "command": ["/k8s-cloud-kms-plugin", "--key-uri={{kms_key_uri}}", "--path-to-unix-socket={{kms_path_to_socket}}", "--gce-config={{gce_conf_path}}", "--logtostderr", "2>\&1"], - "livenessProbe": { "httpGet": {"host": "127.0.0.1", "port": 8081, "path": "/healthz"}, "initialDelaySeconds": 3, "timeoutSeconds": 3}, - "ports":[{ "name": "healthz", "containerPort": 8081, "hostPort": 8081}, { "name": "metrics", "containerPort": 8082, "hostPort": 8082}], - "volumeMounts": [{{cloud_config_mount}}, {{kms_socket_mount}}] -} \ No newline at end of file diff --git a/cluster/gce/manifests/kube-apiserver.manifest b/cluster/gce/manifests/kube-apiserver.manifest index 0a87bf5f5a5..a8a74722ebb 100644 --- a/cluster/gce/manifests/kube-apiserver.manifest +++ b/cluster/gce/manifests/kube-apiserver.manifest @@ -16,7 +16,6 @@ "spec":{ "hostNetwork": true, "containers":[ - {{kms_plugin_container}} { "name": "kube-apiserver", "image": "{{pillar['kube_docker_registry']}}/kube-apiserver:{{pillar['kube-apiserver_docker_tag']}}",