Merge pull request #23558 from andyzheng0831/enhance

Automatic merge from submit-queue

Trusty: Regional release .tar.gz support

@zmerlynn and @roberthbailey please review it. This change is to support the feature added in PR #22234. The entire logic is pretty much the same as in #22234, with only few minor changes in implementation.

I had manually run e2e tests with "export RELEASE_REGION_FALLBACK=true" on two clusters: (1) Trusty on master nodes on ContainerVM; (2) Master and nodes all on trusty. All tests are green. I don't figure out a way to simulate regional fallback. But I did test the function download_or_bust() out-of-box.

cc/ @wonderfly @dchen1107 @fabioy FYI.

cluster/gce/trusty/configure.sh

@@ -36,37 +36,73 @@ for k,v in yaml.load(sys.stdin).iteritems():
 ''' < "${tmp_install_dir}/kube_env.yaml" > /etc/kube-env)
 }
 
-# Retry a download until we get it.
+validate_hash() {
+  file="$1"
+  expected="$2"
+
+  actual=$(sha1sum ${file} | awk '{ print $1 }') || true
+  if [ "${actual}" != "${expected}" ]; then
+    echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} =="
+    return 1
+  fi
+}
+
+# Retry a download until we get it. Takes a hash and a set of URLs.
 #
-# $1 is the file to create
+# $1: The sha1 of the URL. Can be "" if the sha1 is unknown, which means
-# $2 is the URL to download
+#     we are downloading a hash file.
+# $2: The temp file containing a list of urls to download.
 download_or_bust() {
-  rm -f $1 > /dev/null
+  file_hash="$1"
-  until curl --ipv4 -Lo "$1" --connect-timeout 20 --retry 6 --retry-delay 10 "$2"; do
+  tmpfile_urls="$2"
-    echo "Failed to download file ($2). Retrying."
+
+  while true; do
+    # Read urls from the file one-by-one.
+    while read -r url; do
+      if [ ! -n "${file_hash}" ]; then
+        url="${url/.tar.gz/.tar.gz.sha1}"
+      fi
+      file="${url##*/}"
+      rm -f "${file}"
+      if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then
+        echo "== Failed to download ${url}. Retrying. =="
+      elif [ -n "${file_hash}" ] && ! validate_hash "${file}" "${file_hash}"; then
+        echo "== Hash validation of ${url} failed. Retrying. =="
+      else
+        if [ -n "${file_hash}" ]; then
+          echo "== Downloaded ${url} (SHA1 = ${file_hash}) =="
+        else
+          echo "== Downloaded ${url} =="
+        fi
+        return
+      fi
+    done < "${tmpfile_urls}"
   done
 }
 
 # Downloads kubernetes binaries and kube-system manifest tarball, unpacks them,
 # and places them into suitable directories.
 install_kube_binary_config() {
-  # In anyway we have to download the release tarball as docker_tag files and
+  # Upstart does not support shell array well. Put urls in a temp file with one
-  # kube-proxy image file are there.
+  # url at a line, and we will use 'read' command to get them one-by-one.
+  tmp_binary_urls=$(mktemp /tmp/kube-temp.XXXXXX)
+  echo "${SERVER_BINARY_TAR_URL}" | tr "," "\n" > "${tmp_binary_urls}"
+  tmp_manifests_urls=$(mktemp /tmp/kube-temp.XXXXXX)
+  echo "${KUBE_MANIFESTS_TAR_URL}" | tr "," "\n" > "${tmp_manifests_urls}"
+
   cd /tmp
-  k8s_sha1="${SERVER_BINARY_TAR_URL##*/}.sha1"
+  read -r server_binary_tar_url < "${tmp_binary_urls}"
-  echo "Downloading k8s tar sha1 file ${k8s_sha1}"
+  readonly server_binary_tar="${server_binary_tar_url##*/}"
-  download_or_bust "${k8s_sha1}" "${SERVER_BINARY_TAR_URL}.sha1"
+  if [ -n "${SERVER_BINARY_TAR_HASH:-}" ]; then
-  k8s_tar="${SERVER_BINARY_TAR_URL##*/}"
+    readonly server_binary_tar_hash="${SERVER_BINARY_TAR_HASH}"
-  echo "Downloading k8s tar file ${k8s_tar}"
-  download_or_bust "${k8s_tar}" "${SERVER_BINARY_TAR_URL}"
-  # Validate hash.
-  actual=$(sha1sum "${k8s_tar}" | awk '{ print $1 }') || true
-  if [ "${actual}" != "${SERVER_BINARY_TAR_HASH}" ]; then
-    echo "== ${k8s_tar} corrupted, sha1 ${actual} doesn't match expected ${SERVER_BINARY_TAR_HASH} =="
   else
-    echo "Validated ${SERVER_BINARY_TAR_URL} SHA1 = ${SERVER_BINARY_TAR_HASH}"
+    echo "Downloading binary release sha1 (not found in env)"
+    download_or_bust "" "${tmp_binary_urls}"
+    readonly server_binary_tar_hash=$(cat "${server_binary_tar}.sha1")
   fi
-  tar xzf "/tmp/${k8s_tar}" -C /tmp/ --overwrite
+  echo "Downloading binary release tar"
+  download_or_bust "${server_binary_tar_hash}" "${tmp_binary_urls}"
+  tar xzf "/tmp/${server_binary_tar}" -C /tmp/ --overwrite
   # Copy docker_tag and image files to /run/kube-docker-files.
   mkdir -p /run/kube-docker-files
   cp /tmp/kubernetes/server/bin/*.docker_tag /run/kube-docker-files/
@@ -93,27 +129,21 @@ install_kube_binary_config() {
     mount --bind /home/kubernetes/bin/kubectl "${BIN_PATH}/kubectl"
     mount --bind -o remount,ro,^noexec "${BIN_PATH}/kubectl" "${BIN_PATH}/kubectl"
   fi
-  # Clean up.
-  rm -rf /tmp/kubernetes
-  rm "/tmp/${k8s_tar}"
-  rm "/tmp/${k8s_sha1}"
 
   # Put kube-system pods manifests in /etc/kube-manifests/.
   mkdir -p /run/kube-manifests
   cd /run/kube-manifests
-  manifests_sha1="${KUBE_MANIFESTS_TAR_URL##*/}.sha1"
+  read -r manifests_tar_url < "${tmp_manifests_urls}"
-  echo "Downloading kube-system manifests tar sha1 file ${manifests_sha1}"
+  readonly manifests_tar="${manifests_tar_url##*/}"
-  download_or_bust "${manifests_sha1}" "${KUBE_MANIFESTS_TAR_URL}.sha1"
+  if [ -n "${KUBE_MANIFESTS_TAR_HASH:-}" ]; then
-  manifests_tar="${KUBE_MANIFESTS_TAR_URL##*/}"
+    readonly manifests_tar_hash="${KUBE_MANIFESTS_TAR_HASH}"
-  echo "Downloading kube-manifest tar file ${manifests_tar}"
-  download_or_bust "${manifests_tar}" "${KUBE_MANIFESTS_TAR_URL}"
-  # Validate hash.
-  actual=$(sha1sum "${manifests_tar}" | awk '{ print $1 }') || true
-  if [ "${actual}" != "${KUBE_MANIFESTS_TAR_HASH}" ]; then
-    echo "== ${manifests_tar} corrupted, sha1 ${actual} doesn't match expected ${KUBE_MANIFESTS_TAR_HASH} =="
   else
-    echo "Validated ${KUBE_MANIFESTS_TAR_URL} SHA1 = ${KUBE_MANIFESTS_TAR_HASH}"
+    echo "Downloading k8s manifests sha1 (not found in env)"
+    download_or_bust "" "${tmp_manifests_urls}"
+    readonly manifests_tar_hash=$(cat "${manifests_tar}.sha1")
   fi
+  echo "Downloading k8s manifests tar"
+  download_or_bust "${manifests_tar_hash}" "${tmp_manifests_urls}"
   tar xzf "/run/kube-manifests/${manifests_tar}" -C /run/kube-manifests/ --overwrite
   readonly kube_addon_registry="${KUBE_ADDON_REGISTRY:-gcr.io/google_containers}"
   if [ "${kube_addon_registry}" != "gcr.io/google_containers" ]; then
@@ -123,6 +153,13 @@ install_kube_binary_config() {
       xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@"
   fi
   cp /run/kube-manifests/kubernetes/trusty/configure-helper.sh /etc/kube-configure-helper.sh
-  rm "/run/kube-manifests/${manifests_sha1}"
+
-  rm "/run/kube-manifests/${manifests_tar}"
+  # Clean up.
+  rm -rf /tmp/kubernetes
+  rm -f "/tmp/${server_binary_tar}"
+  rm -f "/tmp/${server_binary_tar}.sha1"
+  rm -f "/run/kube-manifests/${manifests_tar}"
+  rm -f "/run/kube-manifests/${manifests_tar}.sha1"
+  rm -f "${tmp_binary_urls}"
+  rm -f "${tmp_manifests_urls}"
 }

cluster/gce/util.sh

@@ -249,14 +249,17 @@ function upload-server-tars() {
     fi
   done
 
-  if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
+  if [[ "${OS_DISTRIBUTION}" == "coreos" ]]; then
-    # TODO: Support fallback .tar.gz settings on CoreOS/Trusty
+    # TODO: Support fallback .tar.gz settings on CoreOS
     SERVER_BINARY_TAR_URL="${server_binary_tar_urls[0]}"
     SALT_TAR_URL="${salt_tar_urls[0]}"
     KUBE_MANIFESTS_TAR_URL="${kube_manifests_tar_urls[0]}"
   else
     SERVER_BINARY_TAR_URL=$(join_csv "${server_binary_tar_urls[@]}")
     SALT_TAR_URL=$(join_csv "${salt_tar_urls[@]}")
+    if [[ "${OS_DISTRIBUTION}" == "trusty" ]]; then
+      KUBE_MANIFESTS_TAR_URL=$(join_csv "${kube_manifests_tar_urls[@]}")
+    fi
   fi
 }