Remove manifest copies from Trusty support

This change revises the way to provide kube-system manifests for clusters on Trusty. Originally, we maintained copies of some manifests under cluster/gce/trusty/kube-manifests, which is not scalable and hard to maintain. With this change, clusters on Trusty will use the same source of manifests as ContainerVM. This change also fixes some minor problems such as shell variables and comments to meet the style guidance better.

build/common.sh

@@ -925,12 +925,9 @@ function kube::release::package_salt_tarball() {
 }
 
 # This will pack kube-system manifests files for distros without using salt
-# such as Ubuntu Trusty.
+# such as Ubuntu Trusty. For Trusty, we directly copy manifests from cluster/addons
-#
+# and cluster/saltbase/salt. The script of cluster initialization will remove
-# There are two sources of manifests files: (1) some manifests in the directory
+# the salt configuration and evaluate the variables in the manifests.
-# cluster/saltbase/salt and cluster/addons can be used directly or after minor
-# revision, so we copy them from there; (2) otherwise, we will maintain separate
-# copies in cluster/gce/<distro>/kube-manifests.
 function kube::release::package_kube_manifests_tarball() {
   kube::log::status "Building tarball: manifests"
 
@@ -938,7 +935,6 @@ function kube::release::package_kube_manifests_tarball() {
   rm -rf "${release_stage}"
   mkdir -p "${release_stage}/trusty"
 
-  # Source 1: manifests from cluster/saltbase/salt and cluster/addons
   local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt"
   cp "${salt_dir}/fluentd-es/fluentd-es.yaml" "${release_stage}/"
   cp "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${release_stage}/"
@@ -946,6 +942,8 @@ function kube::release::package_kube_manifests_tarball() {
   cp "${salt_dir}/kube-proxy/kube-proxy.manifest" "${release_stage}/"
   cp "${salt_dir}/etcd/etcd.manifest" "${release_stage}/trusty"
   cp "${salt_dir}/kube-scheduler/kube-scheduler.manifest" "${release_stage}/trusty"
+  cp "${salt_dir}/kube-apiserver/kube-apiserver.manifest" "${release_stage}/trusty"
+  cp "${salt_dir}/kube-controller-manager/kube-controller-manager.manifest" "${release_stage}/trusty"
   cp "${salt_dir}/kube-addons/namespace.yaml" "${release_stage}/trusty"
   cp "${salt_dir}/kube-addons/kube-addons.sh" "${release_stage}/trusty"
   cp "${salt_dir}/kube-addons/kube-addon-update.sh" "${release_stage}/trusty"
@@ -954,10 +952,7 @@ function kube::release::package_kube_manifests_tarball() {
   objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . \( -name \*.yaml -or -name \*.yaml.in -or -name \*.json \) | grep -v demo)
   tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${release_stage}/trusty"
 
-  # Source 2: manifests from cluster/gce/<distro>/kube-manifests.
+  # This is for coreos only. ContainerVM or Trusty does not use it.
-  # TODO(andyzheng0831): Avoid using separate copies for trusty. We should use whatever
-  # from cluster/saltbase/salt to minimize maintenance cost.
-  cp "${KUBE_ROOT}/cluster/gce/trusty/kube-manifests/"* "${release_stage}/trusty"
   cp -r "${KUBE_ROOT}/cluster/gce/coreos/kube-manifests"/* "${release_stage}/"
 
   kube::release::clean_cruft

cluster/gce/trusty/configure.sh

@@ -23,7 +23,7 @@
 config_hostname() {
   # Set the hostname to the short version.
   short_hostname=$(hostname -s)
-  hostname $short_hostname
+  hostname "${short_hostname}"
 }
 
 config_ip_firewall() {
@@ -54,7 +54,7 @@ create_dirs() {
 download_kube_env() {
   # Fetch kube-env from GCE metadata server.
   readonly tmp_install_dir="/var/cache/kubernetes-install"
-  mkdir -p ${tmp_install_dir}
+  mkdir -p "${tmp_install_dir}"
   curl --fail --silent --show-error \
     -H "X-Google-Metadata-Request: True" \
     -o "${tmp_install_dir}/kube_env.yaml" \
@@ -78,12 +78,12 @@ kind: Config
 users:
 - name: kubelet
   user:
-    client-certificate-data: ${KUBELET_CERT}
+    client-certificate-data: "${KUBELET_CERT}"
-    client-key-data: ${KUBELET_KEY}
+    client-key-data: "${KUBELET_KEY}"
 clusters:
 - name: local
   cluster:
-    certificate-authority-data: ${KUBELET_CA_CERT}
+    certificate-authority-data: "${KUBELET_CA_CERT}"
 contexts:
 - context:
     cluster: local
@@ -101,11 +101,11 @@ kind: Config
 users:
 - name: kube-proxy
   user:
-    token: ${KUBE_PROXY_TOKEN}
+    token: "${KUBE_PROXY_TOKEN}"
 clusters:
 - name: local
   cluster:
-    certificate-authority-data: ${CA_CERT}
+    certificate-authority-data: "${CA_CERT}"
 contexts:
 - context:
     cluster: local
@@ -171,7 +171,7 @@ install_kube_binary_config() {
   echo "Downloading k8s tar file ${k8s_tar}"
   download_or_bust "${k8s_tar}" "${SERVER_BINARY_TAR_URL}"
   # Validate hash.
-  actual=$(sha1sum ${k8s_tar} | awk '{ print $1 }') || true
+  actual=$(sha1sum "${k8s_tar}" | awk '{ print $1 }') || true
   if [ "${actual}" != "${SERVER_BINARY_TAR_HASH}" ]; then
     echo "== ${k8s_tar} corrupted, sha1 ${actual} doesn't match expected ${SERVER_BINARY_TAR_HASH} =="
   else
@@ -198,8 +198,8 @@ install_kube_binary_config() {
     BINARY_PATH="/usr/local/bin/"
   fi
   if ! which kubelet > /dev/null || ! which kube-proxy > /dev/null || [ "${TEST_CLUSTER:-}" = "true" ]; then
-    cp /tmp/kubernetes/server/bin/kubelet ${BINARY_PATH}
+    cp /tmp/kubernetes/server/bin/kubelet "${BINARY_PATH}"
-    cp /tmp/kubernetes/server/bin/kubectl ${BINARY_PATH}
+    cp /tmp/kubernetes/server/bin/kubectl "${BINARY_PATH}"
   fi
   # Clean up.
   rm -rf "/tmp/kubernetes"
@@ -216,7 +216,7 @@ install_kube_binary_config() {
   echo "Downloading kube-manifest tar file ${manifests_tar}"
   download_or_bust "${manifests_tar}" "${KUBE_MANIFESTS_TAR_URL}"
   # Validate hash.
-  actual=$(sha1sum ${manifests_tar} | awk '{ print $1 }') || true
+  actual=$(sha1sum "${manifests_tar}" | awk '{ print $1 }') || true
   if [ "${actual}" != "${KUBE_MANIFESTS_TAR_HASH}" ]; then
     echo "== ${manifests_tar} corrupted, sha1 ${actual} doesn't match expected ${KUBE_MANIFESTS_TAR_HASH} =="
   else
@@ -306,12 +306,11 @@ health_monitoring() {
       echo "Docker daemon failed!"
       pkill docker
     fi
-    if ! curl --insecure -m ${max_seconds} -f -s https://127.0.0.1:${KUBELET_PORT:-10250}/healthz > /dev/null; then
+    if ! curl --insecure -m "${max_seconds}" -f -s https://127.0.0.1:${KUBELET_PORT:-10250}/healthz > /dev/null; then
       echo "Kubelet is unhealthy!"
       pkill kubelet
     fi
-    # TODO(andyzheng0831): Add master side health monitoring.
+    sleep "${sleep_seconds}"
-    sleep ${sleep_seconds}
   done
 }
 
@@ -327,29 +326,29 @@ mount_master_pd() {
   readonly mount_point="/mnt/disks/master-pd"
 
   # TODO(zmerlynn): GKE is still lagging in master-pd creation
-  if [ ! -e ${pd_path} ]; then
+  if [ ! -e "${pd_path}" ]; then
     return
   fi
   # Format and mount the disk, create directories on it for all of the master's
   # persistent data, and link them to where they're used.
-  mkdir -p ${mount_point}
+  mkdir -p "${mount_point}"
-  /usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" ${pd_path} ${mount_point} >/var/log/master-pd-mount.log || \
+  /usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" "${pd_path}" "${mount_point}" >/var/log/master-pd-mount.log || \
     { echo "!!! master-pd mount failed, review /var/log/master-pd-mount.log !!!"; return 1; }
   # Contains all the data stored in etcd
   mkdir -m 700 -p "${mount_point}/var/etcd"
   # Contains the dynamically generated apiserver auth certs and keys
   mkdir -p "${mount_point}/etc/srv/kubernetes"
   # Directory for kube-apiserver to store SSH key (if necessary)
-  mkdir -p /"${mount_point}/etc/srv/sshproxy"
+  mkdir -p "${mount_point}/etc/srv/sshproxy"
   ln -s -f "${mount_point}/var/etcd" /var/etcd
   mkdir -p /etc/srv
-  ln -s -f /"${mount_point}/etc/srv/kubernetes" /etc/srv/kubernetes
+  ln -s -f "${mount_point}/etc/srv/kubernetes" /etc/srv/kubernetes
-  ln -s -f /"${mount_point}/etc/srv/sshproxy" /etc/srv/sshproxy
+  ln -s -f "${mount_point}/etc/srv/sshproxy" /etc/srv/sshproxy
 
   if ! id etcd &>/dev/null; then
     useradd -s /sbin/nologin -d /var/etcd etcd
   fi
-  chown -R etcd /"${mount_point}/var/etcd"
+  chown -R etcd "${mount_point}/var/etcd"
   chgrp -R etcd "${mount_point}/var/etcd"
 }
 
@@ -393,20 +392,20 @@ create_master_auth() {
     # NB: If this list ever changes, this script actually has to
     # change to detect the existence of this file, kill any deleted
     # old tokens and add any new tokens (to handle the upgrade case).
-    add_token_entry "system:scheduler" ${known_tokens_csv}
+    add_token_entry "system:scheduler" "${known_tokens_csv}"
-    add_token_entry "system:controller_manager" ${known_tokens_csv}
+    add_token_entry "system:controller_manager" "${known_tokens_csv}"
-    add_token_entry "system:logging" ${known_tokens_csv}
+    add_token_entry "system:logging" "${known_tokens_csv}"
-    add_token_entry "system:monitoring" ${known_tokens_csv}
+    add_token_entry "system:monitoring" "${known_tokens_csv}"
-    add_token_entry "system:dns" ${known_tokens_csv}
+    add_token_entry "system:dns" "${known_tokens_csv}"
   fi
 
   if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then
     cat <<EOF >/etc/gce.conf
 [global]
-token-url = ${TOKEN_URL}
+token-url = "${TOKEN_URL}"
-token-body = ${TOKEN_BODY}
+token-body = "${TOKEN_BODY}"
-project-id = ${PROJECT_ID}
+project-id = "${PROJECT_ID}"
-network-name = ${NODE_NETWORK}
+network-name = "${NODE_NETWORK}"
 EOF
   fi
 }
@@ -477,10 +476,20 @@ compute_master_manifest_variables() {
   fi
   DOCKER_REGISTRY="gcr.io/google_containers"
   if [ -n "${KUBE_DOCKER_REGISTRY:-}" ]; then
-    DOCKER_REGISTRY=${KUBE_DOCKER_REGISTRY}
+    DOCKER_REGISTRY="${KUBE_DOCKER_REGISTRY}"
   fi
 }
 
+# A helper function for removing salt configuration and comments from a file.
+# This is mainly for preparing a manifest file.
+# $1: Full path of the file to manipulate
+remove_salt_config_comments() {
+  # Remove salt configuration
+  sed -i "/^[ |\t]*{[#|%]/d" $1
+  # Remove comments
+  sed -i "/^[ |\t]*#/d" $1
+}
+
 # Starts k8s apiserver.
 # It prepares the log file, loads the docker image, calculates variables, sets them
 # in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
@@ -526,12 +535,21 @@ start_kube_apiserver() {
   readonly kube_apiserver_docker_tag=$(cat /run/kube-docker-files/kube-apiserver.docker_tag)
 
   src_file="/run/kube-manifests/kubernetes/trusty/kube-apiserver.manifest"
-  sed -i -e "s@{{params}}@${params}@g" ${src_file}
+  remove_salt_config_comments "${src_file}"
-  sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" ${src_file}
+  # Evaluate variables
-  sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" ${src_file}
+  sed -i -e "s@{{params}}@${params}@g" "${src_file}"
-  sed -i -e "s@{{kube_docker_registry}}@${DOCKER_REGISTRY}@g" ${src_file}
+  sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
-  sed -i -e "s@{{kube-apiserver_docker_tag}}@${kube_apiserver_docker_tag}@g" ${src_file}
+  sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}"
-  cp ${src_file} /etc/kubernetes/manifests
+  sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}"
+  sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}"
+  sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
+  sed -i -e "s@{{pillar\['kube-apiserver_docker_tag'\]}}@${kube_apiserver_docker_tag}@g" "${src_file}"
+  sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}"
+  sed -i -e "s@{{secure_port}}@443@g" "${src_file}"
+  sed -i -e "s@{{secure_port}}@8080@g" "${src_file}"
+  sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}"
+  sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}"
+  cp "${src_file}" /etc/kubernetes/manifests
 }
 
 # Starts k8s controller manager.
@@ -550,6 +568,9 @@ start_kube_controller_manager() {
 
   # Calculate variables and assemble the command line.
   params="--master=127.0.0.1:8080 --cloud-provider=gce --root-ca-file=/etc/srv/kubernetes/ca.crt --service-account-private-key-file=/etc/srv/kubernetes/server.key"
+  if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then
+    params="${params} --cloud-config=/etc/gce.conf"
+  fi
   if [ -n "${INSTANCE_PREFIX:-}" ]; then
     params="${params} --cluster-name=${INSTANCE_PREFIX}"
   fi
@@ -573,15 +594,20 @@ start_kube_controller_manager() {
   readonly kube_rc_docker_tag=$(cat /run/kube-docker-files/kube-controller-manager.docker_tag)
 
   src_file="/run/kube-manifests/kubernetes/trusty/kube-controller-manager.manifest"
-  sed -i -e "s@{{params}}@${params}@g" ${src_file}
+  remove_salt_config_comments "${src_file}"
-  sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" ${src_file}
+  # Evaluate variables
-  sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" ${src_file}
+  sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
-  sed -i -e "s@{{kube_docker_registry}}@${DOCKER_REGISTRY}@g" ${src_file}
+  sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
-  sed -i -e "s@{{kube-controller-manager_docker_tag}}@${kube_rc_docker_tag}@g" ${src_file}
+  sed -i -e "s@{{pillar\['kube-controller-manager_docker_tag'\]}}@${kube_rc_docker_tag}@g" "${src_file}"
-  cp ${src_file} /etc/kubernetes/manifests
+  sed -i -e "s@{{params}}@${params}@g" "${src_file}"
+  sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}"
+  sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}"
+  sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}"
+  sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}"
+  cp "${src_file}" /etc/kubernetes/manifests
 }
 
-# Start k8s scheduler.
+# Starts k8s scheduler.
 # It prepares the log file, loads the docker image, calculates variables, sets them
 # in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
 #
@@ -607,14 +633,14 @@ start_kube_scheduler() {
 
   # Remove salt comments and replace variables with values
   src_file="/run/kube-manifests/kubernetes/trusty/kube-scheduler.manifest"
-  sed -i "/^ *{%/d" ${src_file}
+  remove_salt_config_comments "${src_file}"
-  sed -i -e "s@{{params}}@${params}@g" ${src_file}
+  sed -i -e "s@{{params}}@${params}@g" "${src_file}"
-  sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" ${src_file}
+  sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
-  sed -i -e "s@{{pillar\['kube-scheduler_docker_tag'\]}}@${kube_scheduler_docker_tag}@g" ${src_file}
+  sed -i -e "s@{{pillar\['kube-scheduler_docker_tag'\]}}@${kube_scheduler_docker_tag}@g" "${src_file}"
-  cp ${src_file} /etc/kubernetes/manifests
+  cp "${src_file}" /etc/kubernetes/manifests
 }
 
-# Start a fluentd static pod for logging.
+# Starts a fluentd static pod for logging.
 start_fluentd() {
   if [ "${ENABLE_NODE_LOGGING:-}" = "true" ]; then
     if [ "${LOGGING_DESTINATION:-}" = "gcp" ]; then
@@ -652,7 +678,7 @@ setup_addon_manifests() {
   chmod 644 "${dst_dir}"/*
 }
 
-# Start k8s addons static pods.
+# Prepares the manifests of k8s addons static pods.
 prepare_kube_addons() {
   addon_src_dir="/run/kube-manifests/kubernetes/trusty"
   addon_dst_dir="/etc/kubernetes/addons"
@@ -674,7 +700,7 @@ prepare_kube_addons() {
     else
       controller_yaml="${controller_yaml}/heapster-controller.yaml"
     fi
-    sed -i "/^ *{%/d" "${controller_yaml}"
+    remove_salt_config_comments "${controller_yaml}"
     sed -i -e "s@{{ *heapster_memory *}}@${heapster_memory}@g" "${controller_yaml}"
   fi
   cp "${addon_src_dir}/namespace.yaml" "${addon_dst_dir}"
@@ -699,7 +725,7 @@ prepare_kube_addons() {
     mv "${addon_dst_dir}/registry/registry-pv.yaml.in" "${registry_pv_file}"
     mv "${addon_dst_dir}/registry/registry-pvc.yaml.in" "${registry_pvc_file}"
     # Replace the salt configurations with variable values.
-    sed -i "/^ *{%/d" "${registry_pv_file}"
+    remove_salt_config_comments "${controller_yaml}"
     sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pv_file}"
     sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pvc_file}"
     sed -i -e "s@{{ *pillar\['cluster_registry_disk_name'\] *}}@${CLUSTER_REGISTRY_DISK}@g" "${registry_pvc_file}"

cluster/gce/trusty/kube-manifests/kube-apiserver.manifest

@@ -1,98 +0,0 @@
-{
-"apiVersion": "v1",
-"kind": "Pod",
-"metadata": {
-  "name":"kube-apiserver",
-  "namespace": "kube-system"
-},
-"spec":{
-"hostNetwork": true,
-"containers":[
-    {
-    "name": "kube-apiserver",
-    "image": "{{kube_docker_registry}}/kube-apiserver:{{kube-apiserver_docker_tag}}",
-    "resources": {
-      "requests": {
-        "cpu": "250m"
-      }
-    },
-    "command": [
-                 "/bin/sh",
-                 "-c",
-                 "/usr/local/bin/kube-apiserver {{params}} 1>>/var/log/kube-apiserver.log 2>&1"
-               ],
-    "livenessProbe": {
-      "httpGet": {
-        "host": "127.0.0.1",
-        "port": 8080,
-        "path": "/healthz"
-      },
-      "initialDelaySeconds": 15,
-      "timeoutSeconds": 15
-    },
-    "ports":[
-      { "name": "https",
-        "containerPort": 443,
-        "hostPort": 443},{
-       "name": "local",
-        "containerPort": 8080,
-        "hostPort": 8080}
-        ],
-    "volumeMounts": [
-        {{cloud_config_mount}}
-        { "name": "srvkube",
-        "mountPath": "/etc/srv/kubernetes",
-        "readOnly": true},
-        { "name": "logfile",
-        "mountPath": "/var/log/kube-apiserver.log",
-        "readOnly": false},
-        { "name": "etcssl",
-        "mountPath": "/etc/ssl",
-        "readOnly": true},
-        { "name": "varssl",
-        "mountPath": "/var/ssl",
-        "readOnly": true},
-        { "name": "etcopenssl",
-        "mountPath": "/etc/openssl",
-        "readOnly": true},
-        { "name": "etcpkitls",
-        "mountPath": "/etc/pki/tls",
-        "readOnly": true},
-        { "name": "srvsshproxy",
-        "mountPath": "/etc/srv/sshproxy",
-        "readOnly": false}
-      ]
-    }
-],
-"volumes":[
-  {{cloud_config_volume}}
-  { "name": "srvkube",
-    "hostPath": {
-        "path": "/etc/srv/kubernetes"}
-  },
-  { "name": "logfile",
-    "hostPath": {
-        "path": "/var/log/kube-apiserver.log"}
-  },
-  { "name": "etcssl",
-    "hostPath": {
-        "path": "/etc/ssl"}
-  },
-  { "name": "varssl",
-    "hostPath": {
-        "path": "/var/ssl"}
-  },
-  { "name": "etcopenssl",
-    "hostPath": {
-        "path": "/etc/openssl"}
-  },
-  { "name": "etcpkitls",
-    "hostPath": {
-        "path": "/etc/pki/tls"}
-  },
-  { "name": "srvsshproxy",
-    "hostPath": {
-        "path": "/etc/srv/sshproxy"}
-  }
-]
-}}

cluster/gce/trusty/kube-manifests/kube-controller-manager.manifest

@@ -1,83 +0,0 @@
-{
-"apiVersion": "v1",
-"kind": "Pod",
-"metadata": {
-  "name":"kube-controller-manager",
-  "namespace": "kube-system"
-},
-"spec":{
-"hostNetwork": true,
-"containers":[
-    {
-    "name": "kube-controller-manager",
-    "image": "{{kube_docker_registry}}/kube-controller-manager:{{kube-controller-manager_docker_tag}}",
-    "resources": {
-      "requests": {
-        "cpu": "200m"
-      }
-    },
-    "command": [
-                 "/bin/sh",
-                 "-c",
-                 "/usr/local/bin/kube-controller-manager {{params}} 1>>/var/log/kube-controller-manager.log 2>&1"
-               ],
-    "livenessProbe": {
-      "httpGet": {
-        "host": "127.0.0.1",
-        "port": 10252,
-        "path": "/healthz"
-      },
-      "initialDelaySeconds": 15,
-      "timeoutSeconds": 15
-    },
-    "volumeMounts": [
-        {{cloud_config_mount}}
-        { "name": "srvkube",
-        "mountPath": "/etc/srv/kubernetes",
-        "readOnly": true},
-        { "name": "logfile",
-        "mountPath": "/var/log/kube-controller-manager.log",
-        "readOnly": false},
-        { "name": "etcssl",
-        "mountPath": "/etc/ssl",
-        "readOnly": true},
-        { "name": "varssl",
-        "mountPath": "/var/ssl",
-        "readOnly": true},
-        { "name": "etcopenssl",
-        "mountPath": "/etc/openssl",
-        "readOnly": true},
-        { "name": "etcpkitls",
-        "mountPath": "/etc/pki/tls",
-        "readOnly": true}
-      ]
-    }
-],
-"volumes":[
-  {{cloud_config_volume}}
-  { "name": "srvkube",
-    "hostPath": {
-        "path": "/etc/srv/kubernetes"}
-  },
-  { "name": "logfile",
-    "hostPath": {
-        "path": "/var/log/kube-controller-manager.log"}
-  },
-  { "name": "etcssl",
-    "hostPath": {
-        "path": "/etc/ssl"}
-  },
-  { "name": "varssl",
-    "hostPath": {
-        "path": "/var/ssl"}
-  },
-  { "name": "etcopenssl",
-    "hostPath": {
-        "path": "/etc/openssl"}
-  },
-  { "name": "etcpkitls",
-    "hostPath": {
-        "path": "/etc/pki/tls"}
-  }
-]
-}}

cluster/gce/trusty/node.yaml

@@ -264,8 +264,7 @@ Content-Disposition: attachment; filename="kube-node-health-monitoring.conf"
 
 description "Kubenetes node health monitoring"
 
-# The termination of kube-proxy upstart job indicates that kubelet and docker are ready.
+start on stopped kube-docker
-start on stopped kube-proxy
 
 respawn
 

cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest

@@ -7,6 +7,11 @@
 {% set cloud_config = "" -%}
 {% set cloud_config_mount = "" -%}
 {% set cloud_config_volume = "" -%}
+{% set additional_cloud_config_mount = "{\"name\": \"usrsharessl\",\"mountPath\": \"/usr/share/ssl\", \"readOnly\": true}, {\"name\": \"usrssl\",\"mountPath\": \"/usr/ssl\", \"readOnly\": true}, {\"name\": \"usrlibssl\",\"mountPath\": \"/usr/lib/ssl\", \"readOnly\": true}, {\"name\": \"usrlocalopenssl\",\"mountPath\": \"/usr/local/openssl\", \"readOnly\": true}," -%}
+{% set additional_cloud_config_volume = "{\"name\": \"usrsharessl\",\"hostPath\": {\"path\": \"/usr/share/ssl\"}}, {\"name\": \"usrssl\",\"hostPath\": {\"path\": \"/usr/ssl\"}}, {\"name\": \"usrlibssl\",\"hostPath\": {\"path\": \"/usr/lib/ssl\"}}, {\"name\": \"usrlocalopenssl\",\"hostPath\": {\"path\": \"/usr/local/openssl\"}}," -%}
+
+{% set srv_kube_path = "/srv/kubernetes" -%}
+{% set srv_sshproxy_path = "/srv/sshproxy" -%}
 
 {% if grains.cloud is defined -%}
   {% if grains.cloud not in ['vagrant', 'vsphere'] -%}
@@ -141,8 +146,9 @@
         ],
     "volumeMounts": [
         {{cloud_config_mount}}
+        {{additional_cloud_config_mount}}
         { "name": "srvkube",
-        "mountPath": "/srv/kubernetes",
+        "mountPath": "{{srv_kube_path}}",
         "readOnly": true},
         { "name": "logfile",
         "mountPath": "/var/log/kube-apiserver.log",
@@ -150,21 +156,9 @@
         { "name": "etcssl",
         "mountPath": "/etc/ssl",
         "readOnly": true},
-        { "name": "usrsharessl",
-        "mountPath": "/usr/share/ssl",
-        "readOnly": true},
         { "name": "varssl",
         "mountPath": "/var/ssl",
         "readOnly": true},
-        { "name": "usrssl",
-        "mountPath": "/usr/ssl",
-        "readOnly": true},
-        { "name": "usrlibssl",
-        "mountPath": "/usr/lib/ssl",
-        "readOnly": true},
-        { "name": "usrlocalopenssl",
-        "mountPath": "/usr/local/openssl",
-        "readOnly": true},
         { "name": "etcopenssl",
         "mountPath": "/etc/openssl",
         "readOnly": true},
@@ -172,16 +166,17 @@
         "mountPath": "/etc/pki/tls",
         "readOnly": true},
         { "name": "srvsshproxy",
-        "mountPath": "/srv/sshproxy",
+        "mountPath": "{{srv_sshproxy_path}}",
         "readOnly": false}
       ]
     }
 ],
 "volumes":[
   {{cloud_config_volume}}
+  {{additional_cloud_config_volume}}
   { "name": "srvkube",
     "hostPath": {
-        "path": "/srv/kubernetes"}
+        "path": "{{srv_kube_path}}"}
   },
   { "name": "logfile",
     "hostPath": {
@@ -191,26 +186,10 @@
     "hostPath": {
         "path": "/etc/ssl"}
   },
-  { "name": "usrsharessl",
-    "hostPath": {
-        "path": "/usr/share/ssl"}
-  },
   { "name": "varssl",
     "hostPath": {
         "path": "/var/ssl"}
   },
-  { "name": "usrssl",
-    "hostPath": {
-        "path": "/usr/ssl"}
-  },
-  { "name": "usrlibssl",
-    "hostPath": {
-        "path": "/usr/lib/ssl"}
-  },
-  { "name": "usrlocalopenssl",
-    "hostPath": {
-        "path": "/usr/local/openssl"}
-  },
   { "name": "etcopenssl",
     "hostPath": {
         "path": "/etc/openssl"}
@@ -221,7 +200,7 @@
   },
   { "name": "srvsshproxy",
     "hostPath": {
-        "path": "/srv/sshproxy"}
+        "path": "{{srv_sshproxy_path}}"}
   }
 ]
 }}

cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest

@@ -27,6 +27,9 @@
 {% set cloud_config = "" -%}
 {% set cloud_config_mount = "" -%}
 {% set cloud_config_volume = "" -%}
+{% set additional_cloud_config_mount = "{\"name\": \"usrsharessl\",\"mountPath\": \"/usr/share/ssl\", \"readOnly\": true}, {\"name\": \"usrssl\",\"mountPath\": \"/usr/ssl\", \"readOnly\": true}, {\"name\": \"usrlibssl\",\"mountPath\": \"/usr/lib/ssl\", \"readOnly\": true}, {\"name\": \"usrlocalopenssl\",\"mountPath\": \"/usr/local/openssl\", \"readOnly\": true}," -%}
+{% set additional_cloud_config_volume = "{\"name\": \"usrsharessl\",\"hostPath\": {\"path\": \"/usr/share/ssl\"}}, {\"name\": \"usrssl\",\"hostPath\": {\"path\": \"/usr/ssl\"}}, {\"name\": \"usrlibssl\",\"hostPath\": {\"path\": \"/usr/lib/ssl\"}}, {\"name\": \"usrlocalopenssl\",\"hostPath\": {\"path\": \"/usr/local/openssl\"}}," -%}
+{% set srv_kube_path = "/srv/kubernetes" -%}
 
 {% if grains.cloud is defined -%}
   {% if grains.cloud not in ['vagrant', 'vsphere'] -%}
@@ -94,8 +97,9 @@
     },
     "volumeMounts": [
         {{cloud_config_mount}}
+        {{additional_cloud_config_mount}}
         { "name": "srvkube",
-        "mountPath": "/srv/kubernetes",
+        "mountPath": "{{srv_kube_path}}",
         "readOnly": true},
         { "name": "logfile",
         "mountPath": "/var/log/kube-controller-manager.log",
@@ -103,21 +107,9 @@
         { "name": "etcssl",
         "mountPath": "/etc/ssl",
         "readOnly": true},
-        { "name": "usrsharessl",
-        "mountPath": "/usr/share/ssl",
-        "readOnly": true},
         { "name": "varssl",
         "mountPath": "/var/ssl",
         "readOnly": true},
-        { "name": "usrssl",
-        "mountPath": "/usr/ssl",
-        "readOnly": true},
-        { "name": "usrlibssl",
-        "mountPath": "/usr/lib/ssl",
-        "readOnly": true},
-        { "name": "usrlocalopenssl",
-        "mountPath": "/usr/local/openssl",
-        "readOnly": true},
         { "name": "etcopenssl",
         "mountPath": "/etc/openssl",
         "readOnly": true},
@@ -129,9 +121,10 @@
 ],
 "volumes":[
   {{cloud_config_volume}}
+  {{additional_cloud_config_volume}}
   { "name": "srvkube",
     "hostPath": {
-        "path": "/srv/kubernetes"}
+        "path": "{{srv_kube_path}}"}
   },
   { "name": "logfile",
     "hostPath": {
@@ -141,26 +134,10 @@
     "hostPath": {
         "path": "/etc/ssl"}
   },
-  { "name": "usrsharessl",
-    "hostPath": {
-        "path": "/usr/share/ssl"}
-  },
   { "name": "varssl",
     "hostPath": {
         "path": "/var/ssl"}
   },
-  { "name": "usrssl",
-    "hostPath": {
-        "path": "/usr/ssl"}
-  },
-  { "name": "usrlibssl",
-    "hostPath": {
-        "path": "/usr/lib/ssl"}
-  },
-  { "name": "usrlocalopenssl",
-    "hostPath": {
-        "path": "/usr/local/openssl"}
-  },
   { "name": "etcopenssl",
     "hostPath": {
         "path": "/etc/openssl"}

hack/verify-flags/exceptions.txt

@@ -13,6 +13,7 @@ cluster/gce/configure-vm.sh:  cloud_config: ${CLOUD_CONFIG}
 cluster/gce/configure-vm.sh:  env-to-grains "runtime_config"
 cluster/gce/configure-vm.sh:  kubelet_api_servers: '${KUBELET_APISERVER}'
 cluster/gce/coreos/helper.sh:# cloud_config yaml file should be passed
+cluster/gce/trusty/configure.sh:  sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}"
 cluster/gce/util.sh:    local node_ip=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \
 cluster/juju/charms/trusty/kubernetes-master/files/controller-manager.upstart.tmpl:     --address=%(bind_address)s \
 cluster/juju/charms/trusty/kubernetes-master/files/scheduler.upstart.tmpl:     --address=%(bind_address)s \