Prepare kube-system pods manifest for trusty nodes.

This change refactors the code of preparing kube-system manifests
for trusty based cluster. The manifests used by nodes do not contain
salt configuration, so we can simply copy them from the directory
cluster/saltbase/salt, make a tarball, and upload to Google Storage.

cluster/gce/trusty/configure.sh

@@ -144,8 +144,19 @@ install_additional_packages() {
   fi
 }
 
-# Downloads kubernetes binaries and salt tarball, unpacks them, and places them
+# Retry a download until we get it.
-# to suitable directories.
+#
+# $1 is the file to create
+# $2 is the URL to download
+download_or_bust() {
+  rm -f $1 > /dev/null
+  until curl --ipv4 -Lo "$1" --connect-timeout 20 --retry 6 --retry-delay 10 "$2"; do
+    echo "Failed to download file ($2). Retrying."
+  done
+}
+
+# Downloads kubernetes binaries and kube-system manifest tarball, unpacks them,
+# and places them into suitable directories.
 install_kube_binary_config() {
   . /etc/kube-env
   # For a testing cluster, we pull kubelet, kube-proxy, and kubectl binaries,
@@ -160,10 +171,10 @@ install_kube_binary_config() {
     cd /tmp
     k8s_sha1="${SERVER_BINARY_TAR_URL##*/}.sha1"
     echo "Downloading k8s tar sha1 file ${k8s_sha1}"
-    curl -Lo "${k8s_sha1}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SERVER_BINARY_TAR_URL}.sha1"
+    download_or_bust "${k8s_sha1}" "${SERVER_BINARY_TAR_URL}.sha1"
     k8s_tar="${SERVER_BINARY_TAR_URL##*/}"
     echo "Downloading k8s tar file ${k8s_tar}"
-    curl -Lo "${k8s_tar}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SERVER_BINARY_TAR_URL}"
+    download_or_bust "${k8s_tar}" "${SERVER_BINARY_TAR_URL}"
     # Validate hash.
     actual=$(sha1sum ${k8s_tar} | awk '{ print $1 }') || true
     if [ "${actual}" != "${SERVER_BINARY_TAR_HASH}" ]; then
@@ -180,25 +191,24 @@ install_kube_binary_config() {
     rm "/tmp/${k8s_sha1}"
   fi
 
-  # Put saltbase configuration files in /etc/saltbase. We will use the add-on yaml files.
+  # Put kube-system pods manifests in /etc/kube-manifests/.
-  mkdir -p /etc/saltbase
+  cd /etc
-  cd /etc/saltbase
+  manifests_sha1="${KUBE_MANIFESTS_TAR_URL##*/}.sha1"
-  salt_sha1="${SALT_TAR_URL##*/}.sha1"
+  echo "Downloading kube-manifests tar sha1 file ${manifests_sha1}"
-  echo "Downloading Salt tar sha1 file ${salt_sha1}"
+  download_or_bust "${manifests_sha1}" "${KUBE_MANIFESTS_TAR_URL}.sha1"
-  curl -Lo "${salt_sha1}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SALT_TAR_URL}.sha1"
+  manifests_tar="${KUBE_MANIFESTS_TAR_URL##*/}"
-  salt_tar="${SALT_TAR_URL##*/}"
+  echo "Downloading kube-manifest tar file ${manifests_tar}"
-  echo "Downloading Salt tar file ${salt_tar}"
+  download_or_bust "${manifests_tar}" "${KUBE_MANIFESTS_TAR_URL}"
-  curl -Lo "${salt_tar}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SALT_TAR_URL}"
   # Validate hash.
-  actual=$(sha1sum ${salt_tar} | awk '{ print $1 }') || true
+  actual=$(sha1sum ${manifests_tar} | awk '{ print $1 }') || true
-  if [ "${actual}" != "${SALT_TAR_HASH}" ]; then
+  if [ "${actual}" != "${KUBE_MANIFESTS_TAR_HASH}" ]; then
-    echo "== ${salt_tar} corrupted, sha1 ${actual} doesn't match expected ${SALT_TAR_HASH} =="
+    echo "== ${manifests_tar} corrupted, sha1 ${actual} doesn't match expected ${KUBE_MANIFESTS_TAR_HASH} =="
   else
-    echo "Validated ${SALT_TAR_URL} SHA1 = ${SALT_TAR_HASH}"
+    echo "Validated ${KUBE_MANIFESTS_TAR_URL} SHA1 = ${KUBE_MANIFESTS_TAR_HASH}"
   fi
-  tar xzf "/etc/saltbase/${salt_tar}" -C /etc/saltbase/ --overwrite
+  tar xzf "/etc/${manifests_tar}" -C /etc/ --overwrite
-  rm "/etc/saltbase/${salt_sha1}"
+  rm "/etc/${manifests_sha1}"
-  rm "/etc/saltbase/${salt_tar}"
+  rm "/etc/${manifests_tar}"
 }
 
 restart_docker_daemon() {
@@ -208,7 +218,7 @@ restart_docker_daemon() {
   if [ "${TEST_CLUSTER:-}" = "true" ]; then
     DOCKER_OPTS="${DOCKER_OPTS} --log-level=debug"
   fi
-  echo "DOCKER_OPTS=\"${DOCKER_OPTS} ${EXTRA_DOCKER_OPTS}\"" > /etc/default/docker
+  echo "DOCKER_OPTS=\"${DOCKER_OPTS} ${EXTRA_DOCKER_OPTS:-}\"" > /etc/default/docker
   # Make sure the network interface cbr0 is created before restarting docker daemon
   while ! [ -L /sys/class/net/cbr0 ]; do
     echo "Sleep 1 second to wait for cbr0"

cluster/gce/trusty/node.yaml

@@ -218,19 +218,19 @@ script
 	set -o errexit
 	set -o nounset
 
-	# Configuration files are located at /etc/saltbase.
+	# Kube-system pod manifest files are located at /etc/kube-manifests.
 	. /etc/kube-env
 	# Fluentd
 	if [ "${ENABLE_NODE_LOGGING:-}" = "true" ]; then
 		if [ "${LOGGING_DESTINATION:-}" = "gcp" ]; then
-			cp /etc/saltbase/kubernetes/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml /etc/kubernetes/manifests/
+			cp /etc/kube-manifests/fluentd-gcp.yaml /etc/kubernetes/manifests/
 		elif [ "${LOGGING_DESTINATION:-}" = "elasticsearch" ]; then
-			cp /etc/saltbase/kubernetes/saltbase/salt/fluentd-es/fluentd-es.yaml /etc/kubernetes/manifests/
+			cp /etc/kube-manifests/fluentd-es.yaml /etc/kubernetes/manifests/
 		fi
 	fi
 	# Kube-registry-proxy
 	if [ "${ENABLE_CLUSTER_REGISTRY:-}" = "true" ]; then
-		cp /etc/saltbase/kubernetes/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml /etc/kubernetes/manifests/
+		cp /etc/kube-manifests/kube-registry-proxy.yaml /etc/kubernetes/manifests/
 	fi
 end script
 

cluster/gce/util.sh

@@ -156,8 +156,40 @@ function copy-if-not-staged() {
   fi
 }
 
+# Prepare a tarball of kube-system manifests for trusty based cluster.
+#
+# Vars set:
+#   KUBE_MANIFESTS_TAR_URL
+#   KUBE_MANIFESTS_TAR_HASH
+function prepare-manifests-tar() {
+  KUBE_MANIFESTS_TAR_URL=
+  KUBE_MANIFESTS_TAR_HASH=
+  if [[ "${OS_DISTRIBUTION}" != "trusty" ]]; then
+    return
+  fi
+  local tmp_dir="${KUBE_TEMP}/kube-manifests"
+  mkdir -p ${tmp_dir}
+  # The manifests used by nodes can be directly used on non-salt system.
+  # We simply copy them from cluster/saltbase/salt.
+  local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt"
+  cp -f "${salt_dir}/fluentd-es/fluentd-es.yaml" "${tmp_dir}"
+  cp -f "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${tmp_dir}"
+  cp -f "${salt_dir}/kube-registry-proxy/kube-registry-proxy.yaml" "${tmp_dir}"
+
+  local kube_manifests_tar="${KUBE_TEMP}/kube-manifests.tar.gz"
+  tar czf "${kube_manifests_tar}" -C "${KUBE_TEMP}" kube-manifests
+  KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${kube_manifests_tar}")
+  local kube_manifests_gs_url="${staging_path}/${kube_manifests_tar##*/}"
+  copy-if-not-staged "${staging_path}" "${kube_manifests_gs_url}" "${kube_manifests_tar}" "${KUBE_MANIFESTS_TAR_HASH}"
+  # Convert from gs:// URL to an https:// URL
+  KUBE_MANIFESTS_TAR_URL="${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}"
+}
+
+
 # Take the local tar files and upload them to Google Storage.  They will then be
 # downloaded by the master as part of the start up script for the master.
+# If running on Ubuntu trusty, we also pack the dir cluster/gce/trusty/kube-manifest
+# and upload it to Google Storage.
 #
 # Assumed vars:
 #   PROJECT
@@ -207,6 +239,12 @@ function upload-server-tars() {
   # Convert from gs:// URL to an https:// URL
   SERVER_BINARY_TAR_URL="${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}"
   SALT_TAR_URL="${salt_gs_url/gs:\/\//https://storage.googleapis.com/}"
+
+  # Create a tar for kube-system manifests files and stage it.
+  # TODO(andyzheng0831): After finishing k8s master on trusty (issue #16702),
+  # we will not need to stage the salt tar for trusty anymore.
+  # TODO(andyzheng0831): Add release support for this tar, in case GKE will it.
+  prepare-manifests-tar
 }
 
 # Detect minions created in the minion group
@@ -1311,6 +1349,12 @@ EOF
   if [ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]; then
     cat >>$file <<EOF
 TERMINATED_POD_GC_THRESHOLD: $(yaml-quote ${TERMINATED_POD_GC_THRESHOLD})
+EOF
+  fi
+  if [[ "${OS_DISTRIBUTION}" == "trusty" ]]; then
+    cat >>$file <<EOF
+KUBE_MANIFESTS_TAR_URL: $(yaml-quote ${KUBE_MANIFESTS_TAR_URL})
+KUBE_MANIFESTS_TAR_HASH: $(yaml-quote ${KUBE_MANIFESTS_TAR_HASH})
 EOF
   fi
   if [ -n "${TEST_CLUSTER:-}" ]; then