Merge branch 'master' into upgrade_aliases_branch

2025-11-02 14:51:58 +00:00 · 2018-01-18 14:37:51 -08:00
parent 6215d9894f 7dadeee5e8
commit dba2b58202
450 changed files with 5538 additions and 6979 deletions
--- a/cluster/gce/BUILD
+++ b/cluster/gce/BUILD
@@ -49,3 +49,21 @@ release_filegroup(
        "gci/node.yaml",
    ],
 )
+
+pkg_tar(
+    name = "gce-master-manifests",
+    files = [
+        "manifests/abac-authz-policy.jsonl",
+        "manifests/cluster-autoscaler.manifest",
+        "manifests/e2e-image-puller.manifest",
+        "manifests/etcd.manifest",
+        "manifests/glbc.manifest",
+        "manifests/kube-addon-manager.yaml",
+        "manifests/kube-apiserver.manifest",
+        "manifests/kube-controller-manager.manifest",
+        "manifests/kube-proxy.manifest",
+        "manifests/kube-scheduler.manifest",
+        "manifests/rescheduler.manifest",
+    ],
+    mode = "0644",
+)
--- a/cluster/gce/addons/limit-range/limit-range.yaml
+++ b/cluster/gce/addons/limit-range/limit-range.yaml
@@ -0,0 +1,10 @@
+apiVersion: "v1"
+kind: "LimitRange"
+metadata:
+  name: "limits"
+  namespace: default
+spec:
+  limits:
+    - type: "Container"
+      defaultRequest:
+        cpu: "100m"
--- a/cluster/gce/gci/configure-helper.sh
+++ b/cluster/gce/gci/configure-helper.sh
@@ -1334,7 +1334,7 @@ function prepare-kube-proxy-manifest-variables {
 function start-kube-proxy {
  echo "Start kube-proxy static pod"
  prepare-log-file /var/log/kube-proxy.log
-  local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest"
+  local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/kube-proxy.manifest"
  prepare-kube-proxy-manifest-variables "${src_file}"

  cp "${src_file}" /etc/kubernetes/manifests
@@ -2077,7 +2077,7 @@ EOF
  sed -i -e "s@{{ *pillar\['dns_server'\] *}}@${DNS_SERVER_IP}@g" "${kubedns_file}"

  if [[ "${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-}" == "true" ]]; then
-    setup-addon-manifests "addons" "dns-horizontal-autoscaler"
+    setup-addon-manifests "addons" "dns-horizontal-autoscaler" "gce"
  fi
 }

@@ -2225,7 +2225,7 @@ EOF
    setup-addon-manifests "addons" "node-problem-detector/standalone" "node-problem-detector"
  fi
  if echo "${ADMISSION_CONTROL:-}" | grep -q "LimitRanger"; then
-    setup-addon-manifests "admission-controls" "limit-range"
+    setup-addon-manifests "admission-controls" "limit-range" "gce"
  fi
  if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
    setup-addon-manifests "addons" "calico-policy-controller"
@@ -2260,7 +2260,7 @@ function start-image-puller {
 # Starts kube-registry proxy
 function start-kube-registry-proxy {
  echo "Start kube-registry-proxy"
-  cp "${KUBE_HOME}/kube-manifests/kubernetes/kube-registry-proxy.yaml" /etc/kubernetes/manifests
+  cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/kube-registry-proxy.yaml" /etc/kubernetes/manifests
 }

 # Starts a l7 loadbalancing controller for ingress.
--- a/cluster/gce/manifests/abac-authz-policy.jsonl
+++ b/cluster/gce/manifests/abac-authz-policy.jsonl
@@ -0,0 +1,8 @@
+{% set kube_user = grains.kube_user -%}
+{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"admin", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}}
+{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"{{kube_user}}", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}}
+{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"kubelet", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}}
+{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"kube_proxy", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}}
+{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"kubecfg", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}}
+{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"client", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}}
+{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"group":"system:serviceaccounts", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}}
--- a/cluster/gce/manifests/cluster-autoscaler.manifest
+++ b/cluster/gce/manifests/cluster-autoscaler.manifest
@@ -0,0 +1,106 @@
+{% if pillar.get('enable_cluster_autoscaler', '').lower() == 'true' %}
+{% set cloud_config = "" -%}
+{% set cloud_config_mount = "" -%}
+{% set cloud_config_volume = "" -%}
+{% if grains.cloud == 'gce' and grains.cloud_config is defined -%}
+  {% set cloud_config = "--cloud-config=" + grains.cloud_config -%}
+  {% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%}
+  {% set cloud_config_volume = "{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"" + grains.cloud_config + "\", \"type\": \"FileOrCreate\"}}," -%}
+{% endif -%}
+{% set params = pillar['autoscaler_mig_config'] + " " + cloud_config + " " + pillar.get('autoscaler_expander_config', '') -%}
+
+{
+    "kind": "Pod",
+    "apiVersion": "v1",
+    "metadata": {
+        "name": "cluster-autoscaler",
+        "namespace": "kube-system",
+        "labels": {
+            "tier": "cluster-management",
+            "component": "cluster-autoscaler"
+        }
+    },
+    "spec": {
+        "hostNetwork": true,
+        "containers": [
+            {
+                "name": "cluster-autoscaler",
+                "image": "gcr.io/google_containers/cluster-autoscaler:v1.1.0",
+                "livenessProbe": {
+                    "httpGet": {
+                        "path": "/health-check",
+                        "port": 8085
+		    },
+                    "initialDelaySeconds": 600,
+                    "periodSeconds": 60
+		},
+                "command": [
+                    "./run.sh",
+                    "--kubernetes=http://127.0.0.1:8080?inClusterConfig=f",
+                    "--v=4",
+                    "--logtostderr=true",
+                    "--write-status-configmap=true",
+                    "--balance-similar-node-groups=true",
+                    "{{params}}"
+                ],
+                "env": [
+                    {
+                        "name": "LOG_OUTPUT",
+                        "value": "/var/log/cluster-autoscaler.log"
+                    }
+                ],
+                # TODO: Make resource requirements depend on the size of the cluster
+                "resources": {
+                    "requests": {
+                        "cpu": "10m",
+                        "memory": "300Mi"
+                    }
+                },
+                "volumeMounts": [
+                    {{cloud_config_mount}}
+                    {
+                        "name": "ssl-certs",
+                        "readOnly": true,
+                        "mountPath": "/etc/ssl/certs"
+                    },
+                    {
+                        "name": "usrsharecacerts",
+                        "readOnly": true,
+                        "mountPath": "/usr/share/ca-certificates"
+                    },
+                    {
+                        "name": "logfile",
+                        "mountPath": "/var/log/cluster-autoscaler.log",
+                        "readOnly": false
+                    }
+                ],
+                "terminationMessagePath": "/dev/termination-log",
+                "imagePullPolicy": "IfNotPresent"
+            }
+        ],
+        "volumes": [
+            {{cloud_config_volume}}
+            {
+                "name": "ssl-certs",
+                "hostPath": {
+                    "path": "/etc/ssl/certs"
+                }
+            },
+            {
+                "name": "usrsharecacerts",
+                "hostPath": {
+                    "path": "/usr/share/ca-certificates"
+                }
+            },
+            {
+                "name": "logfile",
+                "hostPath": {
+                    "path": "/var/log/cluster-autoscaler.log",
+                    "type": "FileOrCreate"
+                }
+            }
+        ],
+        "restartPolicy": "Always"
+    }
+}
+{% endif %}
--- a/cluster/gce/manifests/e2e-image-puller.manifest
+++ b/cluster/gce/manifests/e2e-image-puller.manifest
@@ -0,0 +1,111 @@
+# e2e-image-puller seeds nodes in an e2e cluster with test images.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: e2e-image-puller
+  namespace: kube-system
+  labels:
+    name: e2e-image-puller
+spec:
+  containers:
+  - name: image-puller
+    resources:
+      requests:
+        cpu: 100m
+      limits:
+        cpu: 100m
+    image: gcr.io/google_containers/busybox:1.24
+    # TODO: Replace this with a go script that pulls in parallel?
+    # Currently it takes ~5m to pull all e2e images, so this is OK, and
+    # fewer moving parts is always better.
+    # TODO: Replace the hardcoded image list with an autogen list; the list is
+    # currently hard-coded for static verification. It was generated via:
+    # grep -Iiroh "gcr.io/google_.*" "${KUBE_ROOT}/test/e2e" | \
+    #     sed -e "s/[,\")}]//g" | awk '{print $1}' | sort | uniq | tr '\n' ' '
+    # We always want the subshell to exit 0 so this pod doesn't end up
+    # blocking tests in an Error state.
+    command:
+    - /bin/sh
+    - -c
+    - >
+      for i in
+      gcr.io/google_containers/alpine-with-bash:1.0
+      gcr.io/google_containers/apparmor-loader:0.1
+      gcr.io/google_containers/busybox:1.24
+      gcr.io/google_containers/dnsutils:e2e
+      gcr.io/google_containers/e2e-net-amd64:1.0
+      gcr.io/google_containers/echoserver:1.6
+      gcr.io/google_containers/eptest:0.1
+      gcr.io/google_containers/fakegitserver:0.1
+      gcr.io/google_containers/galera-install:0.1
+      gcr.io/google_containers/hostexec:1.2
+      gcr.io/google_containers/invalid-image:invalid-tag
+      gcr.io/google_containers/iperf:e2e
+      gcr.io/google_containers/jessie-dnsutils:e2e
+      gcr.io/google_containers/k8s-dns-dnsmasq-amd64:1.14.5
+      gcr.io/google_containers/liveness:e2e
+      gcr.io/google_containers/logs-generator:v0.1.0
+      gcr.io/google_containers/mounttest:0.8
+      gcr.io/google_containers/mounttest-user:0.5
+      gcr.io/google_containers/mysql-galera:e2e
+      gcr.io/google_containers/mysql-healthz:1.0
+      gcr.io/google_containers/netexec:1.4
+      gcr.io/google_containers/netexec:1.5
+      gcr.io/google_containers/netexec:1.7
+      gcr.io/google_containers/nettest:1.7
+      gcr.io/google_containers/nginx:1.7.9
+      gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.1
+      gcr.io/google_containers/nginx-slim:0.7
+      gcr.io/google_containers/nginx-slim:0.8
+      gcr.io/google_containers/node-problem-detector:v0.3.0
+      gcr.io/google_containers/pause
+      gcr.io/google_containers/porter:4524579c0eb935c056c8e75563b4e1eda31587e0
+      gcr.io/google_containers/portforwardtester:1.2
+      gcr.io/google_containers/redis-install-3.2.0:e2e
+      gcr.io/google_containers/resource_consumer:beta4
+      gcr.io/google_containers/resource_consumer/controller:beta4
+      gcr.io/kubernetes-e2e-test-images/serve-hostname-amd64:1.1
+      gcr.io/google_containers/servicelb:0.1
+      gcr.io/google_containers/test-webserver:e2e
+      gcr.io/google_containers/update-demo:kitten
+      gcr.io/google_containers/update-demo:nautilus
+      gcr.io/google_containers/volume-ceph:0.1
+      gcr.io/google_containers/volume-gluster:0.2
+      gcr.io/google_containers/volume-iscsi:0.1
+      gcr.io/google_containers/volume-nfs:0.8
+      gcr.io/google_containers/volume-rbd:0.1
+      gcr.io/google_containers/zookeeper-install-3.5.0-alpha:e2e
+      gcr.io/google_samples/gb-redisslave:nonexistent
+      ; do echo $(date '+%X') pulling $i; docker pull $i 1>/dev/null; done; exit 0;
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - mountPath: /var/run/docker.sock
+      name: socket
+    - mountPath: /usr/bin/docker
+      name: docker
+  # Add a container that runs a health-check
+  - name: nethealth-check
+    resources:
+      requests:
+        cpu: 100m
+      limits:
+        cpu: 100m
+    image:  gcr.io/google_containers/kube-nethealth-amd64:1.0
+    command:
+    - /bin/sh
+    - -c
+    - "/usr/bin/nethealth || true"
+  volumes:
+  - hostPath:
+      path: /var/run/docker.sock
+      type: Socket
+    name: socket
+  - hostPath:
+      path: /usr/bin/docker
+      type: File
+    name: docker
+  # This pod is really fire-and-forget.
+  restartPolicy: OnFailure
+  # This pod needs hostNetworking for true VM perf measurement as well as avoiding cbr0 issues
+  hostNetwork: true
--- a/cluster/gce/manifests/etcd.manifest
+++ b/cluster/gce/manifests/etcd.manifest
@@ -0,0 +1,115 @@
+{% set etcd_protocol = 'http' -%}
+{% set etcd_creds = '' -%}
+{% if pillar.get('etcd_over_ssl', '').lower() == 'true' -%}
+  {% set etcd_protocol = 'https' -%}
+  {% set etcd_creds = '--peer-trusted-ca-file /srv/kubernetes/etcd-ca.crt --peer-cert-file /srv/kubernetes/etcd-peer.crt --peer-key-file /srv/kubernetes/etcd-peer.key -peer-client-cert-auth' -%}
+{% endif -%}
+{% set hostname = pillar.get('hostname', '') -%}
+{% set cluster_state = (pillar.get('initial_etcd_cluster_state') or 'new') -%}
+{% set etcd_cluster_array = (pillar.get('initial_etcd_cluster') or hostname).split(',') -%}
+{% set etcd_cluster = '' -%}
+{# We use vars dictionary to pass variables set inside the for loop, because jinja defines new variables inside the for loop that hide variables from the outside. #}
+{% set vars = {'etcd_cluster': ''} -%}
+{% for host in etcd_cluster_array -%}
+  {% if etcd_cluster != '' -%}
+    {% set etcd_cluster = etcd_cluster ~ ',' -%}
+  {% endif -%}
+  {% set etcd_cluster = etcd_cluster ~ 'etcd-' ~ host ~ '=' ~ etcd_protocol ~'://' ~ host ~ ':' ~ server_port -%}
+  {% do vars.update({'etcd_cluster': etcd_cluster}) -%}
+{% endfor -%}
+{% set etcd_cluster = vars.etcd_cluster -%}
+{% set quota_bytes = '' -%}
+{% if pillar.get('storage_backend', 'etcd3') == 'etcd3' -%}
+  {% set quota_bytes = '--quota-backend-bytes=4294967296' -%}
+{% endif -%}
+{% set liveness_probe_initial_delay = pillar.get('etcd_liveness_probe_initial_delay', 15) -%}
+{% set srv_kube_path = "/srv/kubernetes" -%}
+
+{
+"apiVersion": "v1",
+"kind": "Pod",
+"metadata": {
+  "name":"etcd-server{{ suffix }}",
+  "namespace": "kube-system",
+  "annotations": {
+    "scheduler.alpha.kubernetes.io/critical-pod": ""
+  }
+},
+"spec":{
+"hostNetwork": true,
+"containers":[
+    {
+    "name": "etcd-container",
+    "image": "{{ pillar.get('etcd_docker_repository', 'gcr.io/google_containers/etcd') }}:{{ pillar.get('etcd_docker_tag', '3.1.10') }}",
+    "resources": {
+      "requests": {
+        "cpu": {{ cpulimit }}
+      }
+    },
+    "command": [
+              "/bin/sh",
+              "-c",
+              "if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; exec /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --initial-advertise-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --advertise-client-urls http://127.0.0.1:{{ port }} --listen-client-urls http://127.0.0.1:{{ port }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} {{ etcd_creds }} 1>>/var/log/etcd{{ suffix }}.log 2>&1"
+            ],
+    "env": [
+      { "name": "TARGET_STORAGE",
+        "value": "{{ pillar.get('storage_backend', 'etcd3') }}"
+      },
+      { "name": "TARGET_VERSION",
+        "value": "{{ pillar.get('etcd_version', '3.1.10') }}"
+      },
+      { "name": "DATA_DIRECTORY",
+        "value": "/var/etcd/data{{ suffix }}"
+      }
+        ],
+    "livenessProbe": {
+      "httpGet": {
+        "host": "127.0.0.1",
+        "port": {{ port }},
+        "path": "/health"
+      },
+      "initialDelaySeconds": {{ liveness_probe_initial_delay }},
+      "timeoutSeconds": 15
+    },
+    "ports": [
+      { "name": "serverport",
+        "containerPort": {{ server_port }},
+        "hostPort": {{ server_port }} 
+      },
+      { "name": "clientport",
+        "containerPort": {{ port }},
+        "hostPort": {{ port }}
+      }
+        ],
+    "volumeMounts": [
+      { "name": "varetcd",
+        "mountPath": "/var/etcd",
+        "readOnly": false
+      },
+      { "name": "varlogetcd",
+        "mountPath": "/var/log/etcd{{ suffix }}.log",
+        "readOnly": false
+      },
+      { "name": "etc",
+        "mountPath": "{{ srv_kube_path }}",
+        "readOnly": false
+      }
+    ]
+    }
+],
+"volumes":[
+  { "name": "varetcd",
+    "hostPath": {
+        "path": "/mnt/master-pd/var/etcd"}
+  },
+  { "name": "varlogetcd",
+    "hostPath": {
+        "path": "/var/log/etcd{{ suffix }}.log",
+        "type": "FileOrCreate"}
+  },
+  { "name": "etc",
+    "hostPath": {
+        "path": "{{ srv_kube_path }}"}
+  }
+]
+}}
--- a/cluster/gce/manifests/glbc.manifest
+++ b/cluster/gce/manifests/glbc.manifest
@@ -0,0 +1,56 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: l7-lb-controller-v0.9.7
+  namespace: kube-system
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ''
+  labels:
+    k8s-app: gcp-lb-controller
+    version: v0.9.7
+    kubernetes.io/name: "GLBC"
+spec:
+  terminationGracePeriodSeconds: 600
+  hostNetwork: true
+  containers:
+  - image: gcr.io/google_containers/glbc:0.9.7
+    livenessProbe:
+      httpGet:
+        path: /healthz
+        port: 8086
+        scheme: HTTP
+      initialDelaySeconds: 30
+      # healthz reaches out to GCE
+      periodSeconds: 30
+      timeoutSeconds: 15
+      successThreshold: 1
+      failureThreshold: 5
+    name: l7-lb-controller
+    volumeMounts:
+    - mountPath: /etc/gce.conf
+      name: cloudconfig
+      readOnly: true
+    - mountPath: /var/log/glbc.log
+      name: logfile
+      readOnly: false
+    resources:
+      # Request is set to accomodate this pod alongside the other
+      # master components on a single core master.
+      # TODO: Make resource requirements depend on the size of the cluster
+      requests:
+        cpu: 10m
+        memory: 50Mi
+    command:
+    # TODO: split this out into args when we no longer need to pipe stdout to a file #6428
+    - sh
+    - -c
+    - 'exec /glbc --verbose=true --apiserver-host=http://localhost:8080 --default-backend-service=kube-system/default-http-backend --sync-period=600s --running-in-cluster=false --use-real-cloud=true --config-file-path=/etc/gce.conf --healthz-port=8086 1>>/var/log/glbc.log 2>&1'
+  volumes:
+  - hostPath:
+      path: /etc/gce.conf
+      type: FileOrCreate
+    name: cloudconfig
+  - hostPath:
+      path: /var/log/glbc.log
+      type: FileOrCreate
+    name: logfile
--- a/cluster/gce/manifests/kube-addon-manager.yaml
+++ b/cluster/gce/manifests/kube-addon-manager.yaml
@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-addon-manager
+  namespace: kube-system
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ''
+  labels:
+    component: kube-addon-manager
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-addon-manager
+    # When updating version also bump it in:
+    # - test/kubemark/resources/manifests/kube-addon-manager.yaml
+    image: gcr.io/google-containers/kube-addon-manager:v6.5
+    command:
+    - /bin/bash
+    - -c
+    - exec /opt/kube-addons.sh 1>>/var/log/kube-addon-manager.log 2>&1
+    resources:
+      requests:
+        cpu: 5m
+        memory: 50Mi
+    volumeMounts:
+    - mountPath: /etc/kubernetes/
+      name: addons
+      readOnly: true
+    - mountPath: /var/log
+      name: varlog
+      readOnly: false
+  volumes:
+  - hostPath:
+      path: /etc/kubernetes/
+    name: addons
+  - hostPath:
+      path: /var/log
+    name: varlog
--- a/cluster/gce/manifests/kube-apiserver.manifest
+++ b/cluster/gce/manifests/kube-apiserver.manifest
@@ -0,0 +1,341 @@
+{% set daemon_args = "$DAEMON_ARGS" -%}
+{% if grains['os_family'] == 'RedHat' -%}
+	{% set daemon_args = "" -%}
+{% endif -%}
+
+{% set cloud_provider = "" -%}
+{% set cloud_config = "" -%}
+{% set cloud_config_mount = "" -%}
+{% set cloud_config_volume = "" -%}
+{% set additional_cloud_config_mount = "{\"name\": \"usrsharessl\",\"mountPath\": \"/usr/share/ssl\", \"readOnly\": true}, {\"name\": \"usrssl\",\"mountPath\": \"/usr/ssl\", \"readOnly\": true}, {\"name\": \"usrlibssl\",\"mountPath\": \"/usr/lib/ssl\", \"readOnly\": true}, {\"name\": \"usrlocalopenssl\",\"mountPath\": \"/usr/local/openssl\", \"readOnly\": true}," -%}
+{% set additional_cloud_config_volume = "{\"name\": \"usrsharessl\",\"hostPath\": {\"path\": \"/usr/share/ssl\"}}, {\"name\": \"usrssl\",\"hostPath\": {\"path\": \"/usr/ssl\"}}, {\"name\": \"usrlibssl\",\"hostPath\": {\"path\": \"/usr/lib/ssl\"}}, {\"name\": \"usrlocalopenssl\",\"hostPath\": {\"path\": \"/usr/local/openssl\"}}," -%}
+
+{% set srv_kube_path = "/srv/kubernetes" -%}
+{% set srv_sshproxy_path = "/srv/sshproxy" -%}
+
+{% if grains.cloud is defined -%}
+  {% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
+
+  {% if grains.cloud == 'gce' and grains.cloud_config is defined -%}
+    {% set cloud_config = "--cloud-config=" + grains.cloud_config -%}
+    {% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%}
+    {% set cloud_config_volume = "{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"" + grains.cloud_config + "\", \"type\": \"FileOrCreate\"}}," -%}
+  {% endif -%}
+
+{% endif -%}
+
+{% set advertise_address = "" -%}
+{% if grains.advertise_address is defined -%}
+  {% set advertise_address = "--advertise-address=" + grains.advertise_address -%}
+{% endif -%}
+
+{% set proxy_ssh_options = "" -%}
+{% if grains.proxy_ssh_user is defined -%}
+  {% set proxy_ssh_options = "--ssh-user=" + grains.proxy_ssh_user + " --ssh-keyfile=/srv/sshproxy/.sshkeyfile" -%}
+  {# Append 40 characters onto command to work around #9822. #}
+  {# If mount list changes, this may also need to change. #}
+  {% set proxy_ssh_options = proxy_ssh_options + "                                        " -%}
+{% endif -%}
+
+{% set address = "--address=127.0.0.1" -%}
+
+{% set bind_address = "" -%}
+{% if grains.publicAddressOverride is defined -%}
+  {% set bind_address = "--bind-address=" + grains.publicAddressOverride -%}
+{% endif -%}
+
+{% set storage_backend = "" -%}
+{% if pillar['storage_backend'] is defined -%}
+  {% set storage_backend = "--storage-backend=" + pillar['storage_backend'] -%}
+{% endif -%}
+{% set etcd_servers = "--etcd-servers=http://127.0.0.1:2379" -%}
+{% set etcd_servers_overrides = "--etcd-servers-overrides=/events#http://127.0.0.1:4002" -%}
+
+{% set storage_media_type = "" -%}
+{% if pillar['storage_media_type'] is defined -%}
+  {% set storage_media_type = "--storage-media-type=" + pillar['storage_media_type'] -%}
+{% endif -%}
+
+{% set liveness_probe_initial_delay = pillar.get('kube_apiserver_liveness_probe_initial_delay', 15) -%}
+
+{% set request_timeout = "" -%}
+{% if pillar['kube_apiserver_request_timeout_sec'] is defined -%}
+  {% set request_timeout = "--request-timeout=" + pillar['kube_apiserver_request_timeout_sec'] + "s" -%}
+{% endif -%}
+
+{% set max_requests_inflight = "" -%}
+{% set target_ram_mb = "" -%}
+{% if pillar['num_nodes'] is defined -%}
+  # If the cluster is large, increase max-requests-inflight limit in apiserver.
+  {% if pillar['num_nodes']|int >= 1000 -%}
+    {% set max_requests_inflight = "--max-requests-inflight=1500 --max-mutating-requests-inflight=500" -%}
+  {% endif -%}
+  # Set amount of memory available for apiserver based on number of nodes.
+  # TODO: Once we start setting proper requests and limits for apiserver
+  # we should reuse the same logic here instead of current heuristic.
+  {% set tmp_ram_mb = pillar['num_nodes']|int * 60 %}
+  {% set target_ram_mb = "--target-ram-mb=" + tmp_ram_mb|string -%}
+{% endif -%}
+
+{% set service_cluster_ip_range = "" -%}
+{% if pillar['service_cluster_ip_range'] is defined -%}
+  {% set service_cluster_ip_range = "--service-cluster-ip-range=" + pillar['service_cluster_ip_range'] -%}
+{% endif -%}
+
+{% set cert_file = "--tls-cert-file=/srv/kubernetes/server.cert" -%}
+{% set key_file = "--tls-private-key-file=/srv/kubernetes/server.key" -%}
+{% set kubelet_cert_file = "--kubelet-client-certificate=/srv/kubernetes/kubeapiserver.cert" -%}
+{% set kubelet_key_file = "--kubelet-client-key=/srv/kubernetes/kubeapiserver.key" -%}
+{% set client_ca_file = "" -%}
+
+{% set secure_port = "6443" -%}
+{% if grains['cloud'] is defined and grains.cloud == 'gce' %}
+  {% set secure_port = "443" -%}
+  {% set client_ca_file = "--client-ca-file=/srv/kubernetes/ca.crt" -%}
+{% endif -%}
+
+{% set min_request_timeout = "" -%}
+{% if grains.minRequestTimeout is defined -%}
+  {% set min_request_timeout = "--min-request-timeout=" + grains.minRequestTimeout -%}
+{% endif -%}
+
+{% set token_auth_file = " --token-auth-file=/dev/null" -%}
+{% set basic_auth_file = "" -%}
+{% set authz_mode = "" -%}
+{% set abac_policy_file = "" -%}
+{% if grains['cloud'] is defined and grains.cloud == 'gce' %}
+  {% set token_auth_file = " --token-auth-file=/srv/kubernetes/known_tokens.csv" -%}
+  {% set basic_auth_file = " --basic-auth-file=/srv/kubernetes/basic_auth.csv" -%}
+  {% set authz_mode = " --authorization-mode=ABAC" -%}
+  {% set abac_policy_file = " --authorization-policy-file=/srv/kubernetes/abac-authz-policy.jsonl" -%}
+{% endif -%}
+
+{% set webhook_authentication_config = "" -%}
+{% set webhook_authn_config_mount = "" -%}
+{% set webhook_authn_config_volume = "" -%}
+{% if grains.webhook_authentication_config is defined -%}
+  {% set webhook_authentication_config = " --authentication-token-webhook-config-file=" + grains.webhook_authentication_config -%}
+  {% set webhook_authn_config_mount = "{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"" + grains.webhook_authentication_config + "\", \"readOnly\": false}," -%}
+  {% set webhook_authn_config_volume = "{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"" + grains.webhook_authentication_config + "\", \"type\": \"FileOrCreate\"}}," -%}
+{% endif -%}
+
+{% set webhook_authorization_config = "" -%}
+{% set webhook_config_mount = "" -%}
+{% set webhook_config_volume = "" -%}
+{% if grains.webhook_authorization_config is defined -%}
+  {% set webhook_authorization_config = " --authorization-webhook-config-file=" + grains.webhook_authorization_config -%}
+  {% set webhook_config_mount = "{\"name\": \"webhookconfigmount\",\"mountPath\": \"" + grains.webhook_authorization_config + "\", \"readOnly\": false}," -%}
+  {% set webhook_config_volume = "{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"" + grains.webhook_authorization_config + "\", \"type\": \"FileOrCreate\"}}," -%}
+  {% set authz_mode = authz_mode + ",Webhook" -%}
+{% endif -%}
+
+{% set image_review_config = "" -%}
+{% set admission_controller_config_mount = "" -%}
+{% set admission_controller_config_volume = "" -%}
+{% set image_policy_webhook_config_mount = "" -%}
+{% set image_policy_webhook_config_volume = "" -%}
+{% if grains.image_review_config is defined -%}
+  {% set image_review_config = " --admission-control-config-file=" + grains.image_review_config -%}
+  {% set admission_controller_config_mount = "{\"name\": \"admissioncontrollerconfigmount\",\"mountPath\": \"" + grains.image_review_config + "\", \"readOnly\": false}," -%}
+  {% set admission_controller_config_volume = "{\"name\": \"admissioncontrollerconfigmount\",\"hostPath\": {\"path\": \"" + grains.image_review_config + "\", \"type\": \"FileOrCreate\"}}," -%}
+  {% set image_policy_webhook_config_mount = "{\"name\": \"imagepolicywebhookconfigmount\",\"mountPath\": \"/etc/gcp_image_review.config\", \"readOnly\": false}," -%}
+  {% set image_policy_webhook_config_volume = "{\"name\": \"imagepolicywebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_image_review.config\", \"type\": \"FileOrCreate\"}}," -%}
+{% endif -%}
+
+{% set admission_control = "" -%}
+{% if pillar['admission_control'] is defined -%}
+ {% set admission_control = "--admission-control=" + pillar['admission_control'] -%}
+{% endif -%}
+
+{% set runtime_config = "" -%}
+{% if grains.runtime_config is defined -%}
+ {% set runtime_config = "--runtime-config=" + grains.runtime_config -%}
+{% endif -%}
+
+{% set feature_gates = "" -%}
+{% if grains.feature_gates is defined -%}
+ {% set feature_gates = "--feature-gates=" + grains.feature_gates -%}
+{% endif -%}
+
+{% set log_level = pillar['log_level'] -%}
+{% if pillar['api_server_test_log_level'] is defined -%}
+  {% set log_level = pillar['api_server_test_log_level'] -%}
+{% endif -%}
+
+{% set enable_garbage_collector = "" -%}
+{% if pillar['enable_garbage_collector'] is defined -%}
+  {% set enable_garbage_collector = "--enable-garbage-collector=" + pillar['enable_garbage_collector'] -%}
+{% endif -%}
+
+{% set etcd_quorum_read = "" %}
+{% if pillar['etcd_quorum_read'] is defined -%}
+  {% set etcd_quorum_read = "--etcd_quorum_read=" + pillar['etcd_quorum_read'] -%}
+{% endif -%}
+
+{% set audit_log = "" -%}
+{% set audit_policy_config_mount = "" -%}
+{% set audit_policy_config_volume = "" -%}
+{% set audit_webhook_config_mount = "" -%}
+{% set audit_webhook_config_volume = "" -%}
+{% if pillar['enable_apiserver_basic_audit'] is defined and pillar['enable_apiserver_basic_audit'] in ['true']  -%}
+  {% set audit_log = "--audit-log-path=/var/log/kube-apiserver-audit.log --audit-log-maxage=0 --audit-log-maxbackup=0 --audit-log-maxsize=2000000000" -%}
+{% elif pillar['enable_apiserver_advanced_audit'] is defined and pillar['enable_apiserver_advanced_audit'] in ['true']  -%}
+  {% set audit_log = "--audit-policy-file=/etc/audit_policy.config" -%}
+  {% set audit_policy_config_mount = "{\"name\": \"auditpolicyconfigmount\",\"mountPath\": \"/etc/audit_policy.config\", \"readOnly\": true}," -%}
+  {% set audit_policy_config_volume = "{\"name\": \"auditpolicyconfigmount\",\"hostPath\": {\"path\": \"/etc/audit_policy.config\", \"type\": \"FileOrCreate\"}}," -%}
+  {% if pillar['advanced_audit_backend'] is defined and 'log' in pillar['advanced_audit_backend'] -%}
+    {% set audit_log = audit_log + " --audit-log-path=/var/log/kube-apiserver-audit.log --audit-log-maxage=0 --audit-log-maxbackup=0 --audit-log-maxsize=2000000000" -%}
+  {% endif %}
+  {% if pillar['advanced_audit_backend'] is defined and 'webhook' in pillar['advanced_audit_backend'] -%}
+    {% set audit_log = audit_log + " --audit-webhook-mode=batch" -%}
+    {% set audit_webhook_config_mount = "{\"name\": \"auditwebhookconfigmount\",\"mountPath\": \"/etc/audit_webhook.config\", \"readOnly\": true}," -%}
+    {% set audit_webhook_config_volume = "{\"name\": \"auditwebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/audit_webhook.config\", \"type\": \"FileOrCreate\"}}," -%}
+  {% endif %}
+{% endif -%}
+
+{% set params = address + " " + storage_backend + " " + storage_media_type + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + feature_gates + " " + admission_control + " " + max_requests_inflight + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector + " " + etcd_quorum_read + " " + audit_log + " " + request_timeout -%}
+{% set params = params + " " + cert_file + " " + key_file + " " + kubelet_cert_file + " " + kubelet_key_file + " --secure-port=" + secure_port + token_auth_file + " " + bind_address + " " + log_level + " " + advertise_address  + " " + proxy_ssh_options + authz_mode + abac_policy_file + webhook_authentication_config + webhook_authorization_config + image_review_config -%}
+
+# test_args has to be kept at the end, so they'll overwrite any prior configuration
+{% if pillar['apiserver_test_args'] is defined -%}
+  {% set params = params + " " + pillar['apiserver_test_args'] -%}
+{% endif -%}
+
+{% set container_env = "" -%}
+
+{
+"apiVersion": "v1",
+"kind": "Pod",
+"metadata": {
+  "name":"kube-apiserver",
+  "namespace": "kube-system",
+  "annotations": {
+    "scheduler.alpha.kubernetes.io/critical-pod": ""
+  },
+  "labels": {
+    "tier": "control-plane",
+    "component": "kube-apiserver"
+  }
+},
+"spec":{
+"hostNetwork": true,
+"containers":[
+    {
+    "name": "kube-apiserver",
+    "image": "{{pillar['kube_docker_registry']}}/kube-apiserver:{{pillar['kube-apiserver_docker_tag']}}",
+    "resources": {
+      "requests": {
+        "cpu": "250m"
+      }
+    },
+    "command": [
+                 "/bin/sh",
+                 "-c",
+                 "exec /usr/local/bin/kube-apiserver {{params}} --allow-privileged={{pillar['allow_privileged']}} 1>>/var/log/kube-apiserver.log 2>&1"
+               ],
+    {{container_env}}
+    "livenessProbe": {
+      "httpGet": {
+        "host": "127.0.0.1",
+        "port": 8080,
+        "path": "/healthz"
+      },
+      "initialDelaySeconds": {{liveness_probe_initial_delay}},
+      "timeoutSeconds": 15
+    },
+    "ports":[
+      { "name": "https",
+        "containerPort": {{secure_port}},
+        "hostPort": {{secure_port}}},{
+       "name": "local",
+        "containerPort": 8080,
+        "hostPort": 8080}
+        ],
+    "volumeMounts": [
+        {{cloud_config_mount}}
+        {{additional_cloud_config_mount}}
+        {{webhook_config_mount}}
+        {{webhook_authn_config_mount}}
+        {{audit_policy_config_mount}}
+        {{audit_webhook_config_mount}}
+        {{admission_controller_config_mount}}
+        {{image_policy_webhook_config_mount}}
+        { "name": "srvkube",
+        "mountPath": "{{srv_kube_path}}",
+        "readOnly": true},
+        { "name": "logfile",
+        "mountPath": "/var/log/kube-apiserver.log",
+        "readOnly": false},
+        { "name": "auditlogfile",
+        "mountPath": "/var/log/kube-apiserver-audit.log",
+        "readOnly": false},
+        { "name": "etcssl",
+        "mountPath": "/etc/ssl",
+        "readOnly": true},
+        { "name": "usrsharecacerts",
+        "mountPath": "/usr/share/ca-certificates",
+        "readOnly": true},
+        { "name": "varssl",
+        "mountPath": "/var/ssl",
+        "readOnly": true},
+        { "name": "etcopenssl",
+        "mountPath": "/etc/openssl",
+        "readOnly": true},
+        { "name": "etcpki",
+        "mountPath": "/etc/srv/pki",
+        "readOnly": true},
+        { "name": "srvsshproxy",
+        "mountPath": "{{srv_sshproxy_path}}",
+        "readOnly": false}
+      ]
+    }
+],
+"volumes":[
+  {{cloud_config_volume}}
+  {{additional_cloud_config_volume}}
+  {{webhook_config_volume}}
+  {{webhook_authn_config_volume}}
+  {{audit_policy_config_volume}}
+  {{audit_webhook_config_volume}}
+  {{admission_controller_config_volume}}
+  {{image_policy_webhook_config_volume}}
+  { "name": "srvkube",
+    "hostPath": {
+        "path": "{{srv_kube_path}}"}
+  },
+  { "name": "logfile",
+    "hostPath": {
+        "path": "/var/log/kube-apiserver.log",
+        "type": "FileOrCreate"}
+  },
+  { "name": "auditlogfile",
+    "hostPath": {
+        "path": "/var/log/kube-apiserver-audit.log",
+        "type": "FileOrCreate"}
+  },
+  { "name": "etcssl",
+    "hostPath": {
+        "path": "/etc/ssl"}
+  },
+  { "name": "usrsharecacerts",
+    "hostPath": {
+        "path": "/usr/share/ca-certificates"}
+  },
+  { "name": "varssl",
+    "hostPath": {
+        "path": "/var/ssl"}
+  },
+  { "name": "etcopenssl",
+    "hostPath": {
+        "path": "/etc/openssl"}
+  },
+  { "name": "etcpki",
+    "hostPath": {
+        "path": "/etc/srv/pki"}
+  },
+  { "name": "srvsshproxy",
+    "hostPath": {
+        "path": "{{srv_sshproxy_path}}"}
+  }
+]
+}}
--- a/cluster/gce/manifests/kube-controller-manager.manifest
+++ b/cluster/gce/manifests/kube-controller-manager.manifest
@@ -0,0 +1,188 @@
+{% set cluster_name = "" -%}
+{% set cluster_cidr = "" -%}
+{% set allocate_node_cidrs = "" -%}
+{% set service_cluster_ip_range = "" %}
+{% set terminated_pod_gc = "" -%}
+
+
+{% if pillar['instance_prefix'] is defined -%}
+  {% set cluster_name = "--cluster-name=" + pillar['instance_prefix'] -%}
+{% endif -%}
+{% if pillar['cluster_cidr'] is defined and pillar['cluster_cidr'] != "" -%}
+  {% set cluster_cidr = "--cluster-cidr=" + pillar['cluster_cidr'] -%}
+{% endif -%}
+{% if pillar['service_cluster_ip_range'] is defined and pillar['service_cluster_ip_range'] != "" -%}
+  {% set service_cluster_ip_range = "--service_cluster_ip_range=" + pillar['service_cluster_ip_range'] -%}
+{% endif -%}
+{% if pillar.get('network_provider', '').lower() == 'kubenet' %}
+  {% set allocate_node_cidrs = "--allocate-node-cidrs=true" -%}
+{% elif pillar['allocate_node_cidrs'] is defined -%}
+  {% set allocate_node_cidrs = "--allocate-node-cidrs=" + pillar['allocate_node_cidrs'] -%}
+{% endif -%}
+{% if pillar['terminated_pod_gc_threshold'] is defined -%}
+  {% set terminated_pod_gc = "--terminated-pod-gc-threshold=" + pillar['terminated_pod_gc_threshold'] -%}
+{% endif -%}
+
+{% set enable_garbage_collector = "" -%}
+{% if pillar['enable_garbage_collector'] is defined -%}
+{% set enable_garbage_collector = "--enable-garbage-collector=" + pillar['enable_garbage_collector'] -%}
+{% endif -%}
+
+{% set cloud_provider = "" -%}
+{% set cloud_config = "" -%}
+{% set cloud_config_mount = "" -%}
+{% set cloud_config_volume = "" -%}
+{% set additional_cloud_config_mount = "{\"name\": \"usrsharessl\",\"mountPath\": \"/usr/share/ssl\", \"readOnly\": true}, {\"name\": \"usrssl\",\"mountPath\": \"/usr/ssl\", \"readOnly\": true}, {\"name\": \"usrlibssl\",\"mountPath\": \"/usr/lib/ssl\", \"readOnly\": true}, {\"name\": \"usrlocalopenssl\",\"mountPath\": \"/usr/local/openssl\", \"readOnly\": true}," -%}
+{% set additional_cloud_config_volume = "{\"name\": \"usrsharessl\",\"hostPath\": {\"path\": \"/usr/share/ssl\"}}, {\"name\": \"usrssl\",\"hostPath\": {\"path\": \"/usr/ssl\"}}, {\"name\": \"usrlibssl\",\"hostPath\": {\"path\": \"/usr/lib/ssl\"}}, {\"name\": \"usrlocalopenssl\",\"hostPath\": {\"path\": \"/usr/local/openssl\"}}," -%}
+{% set pv_recycler_mount = "" -%}
+{% set pv_recycler_volume = "" -%}
+{% set srv_kube_path = "/srv/kubernetes" -%}
+{% flex_vol_plugin_dir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec" -%}
+
+{% if grains.cloud is defined -%}
+  {% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
+  {% set service_account_key = "--service-account-private-key-file=/srv/kubernetes/server.key" -%}
+
+  {% if grains.cloud == 'gce' and grains.cloud_config is defined -%}
+    {% set cloud_config = "--cloud-config=" + grains.cloud_config -%}
+    {% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%}
+    {% set cloud_config_volume = "{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"" + grains.cloud_config + "\", \"type\": \"FileOrCreate\"}}," -%}
+  {% endif -%}
+
+{% endif -%}
+
+{% set root_ca_file = "" -%}
+
+{% if grains.cloud is defined and grains.cloud == 'gce' %}
+   {% set root_ca_file = "--root-ca-file=/srv/kubernetes/ca.crt" -%}
+{% endif -%}
+
+{% set log_level = pillar['log_level'] -%}
+{% if pillar['controller_manager_test_log_level'] is defined -%}
+  {% set log_level = pillar['controller_manager_test_log_level'] -%}
+{% endif -%}
+
+{% set feature_gates = "" -%}
+{% if grains.feature_gates is defined -%}
+ {% set feature_gates = "--feature-gates=" + grains.feature_gates -%}
+{% endif -%}
+
+{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + service_cluster_ip_range + " " + terminated_pod_gc + " " + enable_garbage_collector + " " + cloud_provider  + " " + cloud_config + " " + service_account_key + " " + log_level + " " + root_ca_file -%}
+{% set params = params + " " + feature_gates -%}
+
+{% if pillar.get('enable_hostpath_provisioner', '').lower() == 'true' -%}
+{% set params = params + " --enable-hostpath-provisioner" %}
+{% endif -%}
+
+# test_args has to be kept at the end, so they'll overwrite any prior configuration
+{% if pillar['controller_manager_test_args'] is defined -%}
+{% set params = params + " " + pillar['controller_manager_test_args'] -%}
+{% endif -%}
+
+{% set container_env = "" -%}
+
+{
+"apiVersion": "v1",
+"kind": "Pod",
+"metadata": {
+  "name":"kube-controller-manager",
+  "namespace": "kube-system",
+  "annotations": {
+    "scheduler.alpha.kubernetes.io/critical-pod": ""
+  },
+  "labels": {
+    "tier": "control-plane",
+    "component": "kube-controller-manager"
+  }
+},
+"spec":{
+"hostNetwork": true,
+"containers":[
+    {
+    "name": "kube-controller-manager",
+    "image": "{{pillar['kube_docker_registry']}}/kube-controller-manager:{{pillar['kube-controller-manager_docker_tag']}}",
+    "resources": {
+      "requests": {
+        "cpu": "200m"
+      }
+    },
+    "command": [
+                 "/bin/sh",
+                 "-c",
+                 "exec /usr/local/bin/kube-controller-manager {{params}} 1>>/var/log/kube-controller-manager.log 2>&1"
+               ],
+    {{container_env}}
+    "livenessProbe": {
+      "httpGet": {
+        "host": "127.0.0.1",
+        "port": 10252,
+        "path": "/healthz"
+      },
+      "initialDelaySeconds": 15,
+      "timeoutSeconds": 15
+    },
+    "volumeMounts": [
+        {{cloud_config_mount}}
+        {{additional_cloud_config_mount}}
+        {{pv_recycler_mount}}
+        { "name": "srvkube",
+        "mountPath": "{{srv_kube_path}}",
+        "readOnly": true},
+        {{flexvolume_hostpath_mount}}
+        { "name": "logfile",
+        "mountPath": "/var/log/kube-controller-manager.log",
+        "readOnly": false},
+        { "name": "etcssl",
+        "mountPath": "/etc/ssl",
+        "readOnly": true},
+        { "name": "usrsharecacerts",
+        "mountPath": "/usr/share/ca-certificates",
+        "readOnly": true},
+        { "name": "varssl",
+        "mountPath": "/var/ssl",
+        "readOnly": true},
+        { "name": "etcopenssl",
+        "mountPath": "/etc/openssl",
+        "readOnly": true},
+        { "name": "etcpki",
+        "mountPath": "/etc/pki",
+        "readOnly": true}
+      ]
+    }
+],
+"volumes":[
+  {{cloud_config_volume}}
+  {{additional_cloud_config_volume}}
+  {{pv_recycler_volume}}
+  { "name": "srvkube",
+    "hostPath": {
+        "path": "{{srv_kube_path}}"}
+  },
+  {{flexvolume_hostpath}}
+  { "name": "logfile",
+    "hostPath": {
+        "path": "/var/log/kube-controller-manager.log",
+        "type": "FileOrCreate"}
+  },
+  { "name": "etcssl",
+    "hostPath": {
+        "path": "/etc/ssl"}
+  },
+  { "name": "usrsharecacerts",
+    "hostPath": {
+        "path": "/usr/share/ca-certificates"}
+  },
+  { "name": "varssl",
+    "hostPath": {
+        "path": "/var/ssl"}
+  },
+  { "name": "etcopenssl",
+    "hostPath": {
+        "path": "/etc/openssl"}
+  },
+  { "name": "etcpki",
+    "hostPath": {
+        "path": "/etc/pki"}
+  }
+]
+}}
--- a/cluster/gce/manifests/kube-proxy.manifest
+++ b/cluster/gce/manifests/kube-proxy.manifest
@@ -0,0 +1,127 @@
+# Please keep kube-proxy configuration in-sync with:
+# cluster/addons/kube-proxy/kube-proxy-ds.yaml
+
+{% set kubeconfig = "--kubeconfig=/var/lib/kube-proxy/kubeconfig" -%}
+{% if grains.api_servers is defined -%}
+  {% set api_servers = "--master=https://" + grains.api_servers -%}
+{% else -%}
+  {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
+  {% set api_servers = "--master=https://" + ips[0][0] -%}
+{% endif -%}
+{% if grains['cloud'] is defined and grains.cloud == 'gce'  %}
+  {% set api_servers_with_port = api_servers -%}
+{% else -%}
+  {% set api_servers_with_port = api_servers + ":6443" -%}
+{% endif -%}
+{% set test_args = "" -%}
+{% if pillar['kubeproxy_test_args'] is defined -%}
+  {% set test_args=pillar['kubeproxy_test_args'] %}
+{% endif -%}
+{% set cluster_cidr = "" -%}
+{% if pillar['cluster_cidr'] is defined -%}
+  {% set cluster_cidr=" --cluster-cidr=" + pillar['cluster_cidr'] %}
+{% endif -%}
+
+{% set log_level = pillar['log_level'] -%}
+{% if pillar['kubeproxy_test_log_level'] is defined -%}
+  {% set log_level = pillar['kubeproxy_test_log_level'] -%}
+{% endif -%}
+
+{% set feature_gates = "" -%}
+{% if grains.feature_gates is defined -%}
+ {% set feature_gates = "--feature-gates=" + grains.feature_gates -%}
+{% endif -%}
+
+{% set throttles = "--iptables-sync-period=1m --iptables-min-sync-period=10s --ipvs-sync-period=1m --ipvs-min-sync-period=10s" -%}
+
+{% set pod_priority = "" -%}
+{% if pillar.get('enable_pod_priority', '').lower() == 'true' -%}
+  {% set pod_priority = "priorityClassName: system-node-critical" -%}
+{% endif -%}
+
+# test_args should always go last to overwrite prior configuration
+{% set params = log_level + " " + throttles + " " + feature_gates + " " + test_args -%}
+
+{% set container_env = "" -%}
+{% set kube_cache_mutation_detector_env_name = "" -%}
+{% set kube_cache_mutation_detector_env_value = "" -%}
+
+# kube-proxy podspec
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-proxy
+  namespace: kube-system
+  # This annotation ensures that kube-proxy does not get evicted if the node
+  # supports critical pod annotation based priority scheme.
+  # Note that kube-proxy runs as a static pod so this annotation does NOT have
+  # any effect on rescheduler (default scheduler and rescheduler are not
+  # involved in scheduling kube-proxy).
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ''
+  labels:
+    tier: node
+    component: kube-proxy
+spec:
+  {{pod_priority}}
+  hostNetwork: true
+  tolerations:
+  - operator: "Exists"
+    effect: "NoExecute"
+  - operator: "Exists"
+    effect: "NoSchedule"
+  containers:
+  - name: kube-proxy
+    image: {{pillar['kube_docker_registry']}}/kube-proxy:{{pillar['kube-proxy_docker_tag']}}
+    resources:
+      requests:
+        cpu: {{ cpurequest }}
+    command:
+    - /bin/sh
+    - -c
+    - exec kube-proxy {{api_servers_with_port}} {{kubeconfig}} {{cluster_cidr}} --resource-container="" --oom-score-adj=-998 {{params}} 1>>/var/log/kube-proxy.log 2>&1
+    {{container_env}}
+    {{kube_cache_mutation_detector_env_name}}
+      {{kube_cache_mutation_detector_env_value}}
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - mountPath: /etc/ssl/certs
+      name: etc-ssl-certs
+      readOnly: true
+    - mountPath: /usr/share/ca-certificates
+      name: usr-ca-certs
+      readOnly: true
+    - mountPath: /var/log
+      name: varlog
+      readOnly: false
+    - mountPath: /var/lib/kube-proxy/kubeconfig
+      name: kubeconfig
+      readOnly: false
+    - mountPath: /run/xtables.lock
+      name: iptableslock
+      readOnly: false
+    - mountPath: /lib/modules
+      name: lib-modules
+      readOnly: true
+  volumes:
+  - hostPath:
+      path: /usr/share/ca-certificates
+    name: usr-ca-certs
+  - hostPath:
+      path: /etc/ssl/certs
+    name: etc-ssl-certs
+  - hostPath:
+      path: /var/lib/kube-proxy/kubeconfig
+      type: FileOrCreate
+    name: kubeconfig
+  - hostPath:
+      path: /var/log
+    name: varlog
+  - hostPath:
+      path: /run/xtables.lock
+      type: FileOrCreate
+    name: iptableslock
+  - name: lib-modules
+    hostPath:
+      path: /lib/modules
--- a/cluster/gce/manifests/kube-registry-proxy.yaml
+++ b/cluster/gce/manifests/kube-registry-proxy.yaml
@@ -0,0 +1,35 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-registry-proxy
+  namespace: kube-system
+  labels:
+    k8s-app: kube-registry
+    kubernetes.io/cluster-service: "true"
+    version: v0.4
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-registry
+        kubernetes.io/name: "kube-registry-proxy"
+        kubernetes.io/cluster-service: "true"
+        version: v0.4
+    spec:
+      containers:
+      - name: kube-registry-proxy
+        image: gcr.io/google_containers/kube-registry-proxy:0.4
+        resources:
+          limits:
+            cpu: 100m
+            memory: 50Mi
+        env:
+        - name: REGISTRY_HOST
+          value: kube-registry.kube-system.svc.cluster.local
+        - name: REGISTRY_PORT
+          value: "5000"
+        ports:
+        - name: registry
+          containerPort: 80
+          hostPort: 5000
+
--- a/cluster/gce/manifests/kube-scheduler.manifest
+++ b/cluster/gce/manifests/kube-scheduler.manifest
@@ -0,0 +1,89 @@
+{% set params = "--master=127.0.0.1:8080" -%}
+{% set srv_kube_path = "/srv/kubernetes" -%}
+
+{% set log_level = pillar['log_level'] -%}
+{% if pillar['scheduler_test_log_level'] is defined -%}
+  {% set log_level = pillar['scheduler_test_log_level'] -%}
+{% endif -%}
+
+{% set feature_gates = "" -%}
+{% if grains.feature_gates is defined -%}
+  {% set feature_gates = "--feature-gates=" + grains.feature_gates -%}
+{% endif -%}
+
+{% set scheduling_algorithm_provider = "" -%}
+{% if grains.scheduling_algorithm_provider is defined -%}
+  {% set scheduling_algorithm_provider = "--algorithm-provider=" + grains.scheduling_algorithm_provider -%}
+{% endif -%}
+
+{% set params = params + log_level + " " + feature_gates + " " + scheduling_algorithm_provider -%}
+
+# test_args has to be kept at the end, so they'll overwrite any prior configuration
+{% if pillar['scheduler_test_args'] is defined -%}
+{% set params = params + " " + pillar['scheduler_test_args'] -%}
+{% endif -%}
+
+{
+"apiVersion": "v1",
+"kind": "Pod",
+"metadata": {
+  "name":"kube-scheduler",
+  "namespace": "kube-system",
+  "annotations": {
+    "scheduler.alpha.kubernetes.io/critical-pod": ""
+  },
+  "labels": {
+    "tier": "control-plane",
+    "component": "kube-scheduler"
+  }
+},
+"spec":{
+"hostNetwork": true,
+"containers":[
+    {
+    "name": "kube-scheduler",
+    "image": "{{pillar['kube_docker_registry']}}/kube-scheduler:{{pillar['kube-scheduler_docker_tag']}}",
+    "resources": {
+      "requests": {
+        "cpu": "75m"
+      }
+    },
+    "command": [
+                 "/bin/sh",
+                 "-c",
+                 "exec /usr/local/bin/kube-scheduler {{params}} 1>>/var/log/kube-scheduler.log 2>&1"
+               ],
+    "livenessProbe": {
+      "httpGet": {
+        "host": "127.0.0.1",
+        "port": 10251,
+        "path": "/healthz"
+      },
+      "initialDelaySeconds": 15,
+      "timeoutSeconds": 15
+    },
+    "volumeMounts": [
+        {
+          "name": "logfile",
+          "mountPath": "/var/log/kube-scheduler.log",
+          "readOnly": false
+        },
+        {
+          "name": "srvkube",
+          "mountPath": "{{srv_kube_path}}",
+          "readOnly": true
+        }
+      ]
+    }
+],
+"volumes":[
+  {
+    "name": "srvkube",
+    "hostPath": {"path": "{{srv_kube_path}}"}
+  },
+  {
+    "name": "logfile",
+    "hostPath": {"path": "/var/log/kube-scheduler.log", "type": "FileOrCreate"}
+  }
+]
+}}
--- a/cluster/gce/manifests/rescheduler.manifest
+++ b/cluster/gce/manifests/rescheduler.manifest
@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: rescheduler-v0.3.1
+  namespace: kube-system
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ''
+  labels:
+    k8s-app: rescheduler
+    version: v0.3.1
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "Rescheduler"
+spec:
+  hostNetwork: true
+  containers:
+  - image: gcr.io/google-containers/rescheduler:v0.3.1
+    name: rescheduler
+    volumeMounts:
+    - mountPath: /var/log/rescheduler.log
+      name: logfile
+      readOnly: false
+    # TODO: Make resource requirements depend on the size of the cluster
+    resources:
+      requests:
+        cpu: 10m
+        memory: 100Mi
+    command:
+    # TODO: split this out into args when we no longer need to pipe stdout to a file #6428
+    - sh
+    - -c
+    - 'exec /rescheduler --running-in-cluster=false 1>>/var/log/rescheduler.log 2>&1'
+  volumes:
+  - hostPath:
+      path: /var/log/rescheduler.log
+      type: FileOrCreate
+    name: logfile
--- a/cluster/gce/util.sh
+++ b/cluster/gce/util.sh
@@ -255,21 +255,16 @@ fi
 # Assumed vars:
 #   PROJECT
 #   SERVER_BINARY_TAR
-#   SALT_TAR
 #   KUBE_MANIFESTS_TAR
 #   ZONE
 # Vars set:
 #   SERVER_BINARY_TAR_URL
 #   SERVER_BINARY_TAR_HASH
-#   SALT_TAR_URL
-#   SALT_TAR_HASH
 #   KUBE_MANIFESTS_TAR_URL
 #   KUBE_MANIFESTS_TAR_HASH
 function upload-server-tars() {
  SERVER_BINARY_TAR_URL=
  SERVER_BINARY_TAR_HASH=
-  SALT_TAR_URL=
-  SALT_TAR_HASH=
  KUBE_MANIFESTS_TAR_URL=
  KUBE_MANIFESTS_TAR_HASH=

@@ -287,13 +282,11 @@ function upload-server-tars() {
  set-preferred-region

  SERVER_BINARY_TAR_HASH=$(sha1sum-file "${SERVER_BINARY_TAR}")
-  SALT_TAR_HASH=$(sha1sum-file "${SALT_TAR}")
  if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then
    KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${KUBE_MANIFESTS_TAR}")
  fi

  local server_binary_tar_urls=()
-  local salt_tar_urls=()
  local kube_manifest_tar_urls=()

  for region in "${PREFERRED_REGION[@]}"; do
@@ -313,13 +306,10 @@ function upload-server-tars() {

    echo "+++ Staging server tars to Google Storage: ${staging_path}"
    local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}"
-    local salt_gs_url="${staging_path}/${SALT_TAR##*/}"
    copy-to-staging "${staging_path}" "${server_binary_gs_url}" "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_HASH}"
-    copy-to-staging "${staging_path}" "${salt_gs_url}" "${SALT_TAR}" "${SALT_TAR_HASH}"

    # Convert from gs:// URL to an https:// URL
    server_binary_tar_urls+=("${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}")
-    salt_tar_urls+=("${salt_gs_url/gs:\/\//https://storage.googleapis.com/}")
    if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then
      local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}"
      copy-to-staging "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}"
@@ -329,7 +319,6 @@ function upload-server-tars() {
  done

  SERVER_BINARY_TAR_URL=$(join_csv "${server_binary_tar_urls[@]}")
-  SALT_TAR_URL=$(join_csv "${salt_tar_urls[@]}")
  if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then
    KUBE_MANIFESTS_TAR_URL=$(join_csv "${kube_manifests_tar_urls[@]}")
  fi