diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 311d7f1a0bc..2d96fffcaa7 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -148,19 +148,50 @@ function detect-project() { fi } +# Use gsutil to get the md5 hash for a particular tar +function gsutil_get_tar_md5() { + # location_tar could be local or in the cloud + # local tar_location example ./_output/release-tars/kubernetes-server-linux-amd64.tar.gz + # cloud tar_location example gs://kubernetes-staging-PROJECT/kubernetes-devel/kubernetes-server-linux-amd64.tar.gz + local -r tar_location=$1 + #parse the output and return the md5 hash + #the sed command at the end removes whitespace + local -r tar_md5=$(gsutil hash -h -m ${tar_location} 2>/dev/null | grep "Hash (md5):" | awk -F ':' '{print $2}' | sed 's/^[[:space:]]*//g') + echo "${tar_md5}" +} + # Copy a release tar and its accompanying hash. function copy-to-staging() { local -r staging_path=$1 local -r gs_url=$2 local -r tar=$3 local -r hash=$4 + local -r basename_tar=$(basename ${tar}) + + #check whether this tar alread exists and has the same hash + #if it matches, then don't bother uploading it again + + #remote_tar_md5 checks the remote location for the existing tarball and its md5 + #staging_path example gs://kubernetes-staging-PROJECT/kubernetes-devel + #basename_tar example kubernetes-server-linux-amd64.tar.gz + local -r remote_tar_md5=$(gsutil_get_tar_md5 "${staging_path}/${basename_tar}") + if [[ -n ${remote_tar_md5} ]]; then + #local_tar_md5 checks the remote location for the existing tarball and its md5 hash + #tar example ./_output/release-tars/kubernetes-server-linux-amd64.tar.gz + local -r local_tar_md5=$(gsutil_get_tar_md5 "${tar}") + if [[ "${remote_tar_md5}" == "${local_tar_md5}" ]]; then + echo "+++ ${basename_tar} uploaded earlier, cloud and local file md5 match (md5 = ${local_tar_md5})" + return 0 + fi + fi echo "${hash}" > "${tar}.sha1" gsutil -m -q -h "Cache-Control:private, max-age=0" cp "${tar}" "${tar}.sha1" "${staging_path}" gsutil -m acl ch -g all:R "${gs_url}" "${gs_url}.sha1" >/dev/null 2>&1 - echo "+++ $(basename ${tar}) uploaded (sha1 = ${hash})" + echo "+++ ${basename_tar} uploaded (sha1 = ${hash})" } + # Given the cluster zone, return the list of regional GCS release # bucket suffixes for the release in preference order. GCS doesn't # give us an API for this, so we hardcode it.