From 2ee8983dafbb047871d673089f8b29758a07ba06 Mon Sep 17 00:00:00 2001 From: Christoph Blecker Date: Thu, 22 Jun 2017 17:37:38 -0700 Subject: [PATCH] Dockerize update-staging-godeps --- build/common.sh | 1 + hack/lib/util.sh | 12 +++ hack/update-staging-godeps-dockerized.sh | 115 +++++++++++++++++++++++ hack/update-staging-godeps.sh | 85 ++--------------- hack/verify-staging-godeps.sh | 2 +- 5 files changed, 136 insertions(+), 79 deletions(-) create mode 100755 hack/update-staging-godeps-dockerized.sh diff --git a/build/common.sh b/build/common.sh index 1223510dd04..a2b5d21659a 100755 --- a/build/common.sh +++ b/build/common.sh @@ -714,6 +714,7 @@ function kube::build::copy_output() { --filter='- /_temp/' \ --filter='+ /vendor/' \ --filter='+ /Godeps/' \ + --filter='+ /staging/***/Godeps/**' \ --filter='+ /_output/dockerized/bin/**' \ --filter='+ zz_generated.*' \ --filter='+ generated.proto' \ diff --git a/hack/lib/util.sh b/hack/lib/util.sh index 6424b1f1248..bd8fd0398c2 100755 --- a/hack/lib/util.sh +++ b/hack/lib/util.sh @@ -824,6 +824,18 @@ function kube::util::ensure-cfssl { popd > /dev/null } +# kube::util::ensure_dockerized +# Confirms that the script is being run inside a kube-build image +# +function kube::util::ensure_dockerized { + if [[ -f /kube-build-image ]]; then + return 0 + else + echo "ERROR: This script is designed to be run inside a kube-build container" + exit 1 + fi +} + # Some useful colors. if [[ -z "${color_start-}" ]]; then declare -r color_start="\033[" diff --git a/hack/update-staging-godeps-dockerized.sh b/hack/update-staging-godeps-dockerized.sh new file mode 100755 index 00000000000..f1b48e5f611 --- /dev/null +++ b/hack/update-staging-godeps-dockerized.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# updates the godeps.json file in the staging folders to allow clean vendoring +# based on kubernetes levels. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" + +# Sets default values +DRY_RUN=false +FAIL_ON_DIFF=false +GODEP_OPTS="" + +# Controls verbosity of the script output and logging. +KUBE_VERBOSE="${KUBE_VERBOSE:-5}" +if (( ${KUBE_VERBOSE} >= 6 )); then + GODEP_OPTS+=("-v") +fi + +while getopts ":df" opt; do + case $opt in + d) # do not godep-restore into a temporary directory, but use the existing GOPATH + DRY_RUN=true + ;; + f) # fail if something in the Godeps.json files changed + FAIL_ON_DIFF=true + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + esac +done + +kube::util::ensure_dockerized +kube::golang::setup_env +kube::util::ensure_single_dir_gopath +kube::util::ensure_no_staging_repos_in_gopath +kube::util::ensure_godep_version v79 + +kube::log::status "Checking whether godeps are restored" +if ! kube::util::godep_restored 2>&1 | sed 's/^/ /'; then + ${KUBE_ROOT}/hack/godep-restore.sh +fi + +kube::util::ensure-temp-dir +TMP_GOPATH="${KUBE_TEMP}/go" + +function updateGodepManifest() { + pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null + kube::log::status "Updating godeps for k8s.io/${repo}" + rm -rf Godeps # remove the current Godeps.json so we always rebuild it + GOPATH="${TMP_GOPATH}:${GOPATH}:${GOPATH}/src/k8s.io/kubernetes/staging" godep save ${GODEP_OPTS} ./... 2>&1 | sed 's/^/ /' + + # Rewriting Godeps.json to remove commits that don't really exist because we haven't pushed the prereqs yet + go run "${KUBE_ROOT}/staging/godeps-json-updater.go" --godeps-file="${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" --override-import-path="k8s.io/${repo}" + + # commit so that following repos do not see this repo as dirty + git add vendor >/dev/null + git commit -a -m "Updated Godeps.json" >/dev/null + popd >/dev/null +} + +function diffGodepManifest() { + local ret=0 + diff --ignore-matching-lines='^\s*\"GoVersion\":' --ignore-matching-line='^\s*\"GodepVersion\":' --ignore-matching-lines='^\s*\"Comment\"' -u "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json" "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" || ret=$? + if [[ "${ret}" != "0" && "${FAIL_ON_DIFF}" == true ]]; then + exit ${ret} + fi +} + +# move into staging and save the dependencies for everything in order +mkdir -p "${TMP_GOPATH}/src/k8s.io" +for repo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do + cp -a "${KUBE_ROOT}/staging/src/k8s.io/${repo}" "${TMP_GOPATH}/src/k8s.io/" + + pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null + git init >/dev/null + git config --local user.email "nobody@k8s.io" + git config --local user.name "$0" + git add . >/dev/null + git commit -q -m "Snapshot" >/dev/null + popd >/dev/null + + updateGodepManifest + diffGodepManifest + + if [ "${DRY_RUN}" != true ]; then + cp "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json" + # Assume Godeps.json is not updated, as the working tree needs to be clean + # Without this, the script would pause after each staging repo to prompt the + # user to commit all changes (difficult inside a container). It's safe to + # ignore this file, as we know it's going to be changing, and we don't copy + # the git tree back out from the container. + git update-index --assume-unchanged "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json" + fi +done diff --git a/hack/update-staging-godeps.sh b/hack/update-staging-godeps.sh index 553cddb5ca9..0b5c11b1d5b 100755 --- a/hack/update-staging-godeps.sh +++ b/hack/update-staging-godeps.sh @@ -14,91 +14,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -# updates the godeps.json file in the staging folders to allow clean vendoring -# based on kubernetes levels. -# TODO this does not address client-go, since it takes a different approach to vendoring -# TODO client-go should probably be made consistent - set -o errexit set -o nounset set -o pipefail -V="" -DRY_RUN=false -FAIL_ON_DIFF=false -while getopts ":vdf" opt; do - case $opt in - v) # increase verbosity - V="-v" - ;; - d) # do not godep-restore into a temporary directory, but use the existing GOPATH - DRY_RUN=true - ;; - f) # fail if something in the Godeps.json files changed - FAIL_ON_DIFF=true - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - exit 1 - ;; - esac -done -readonly V IN_PLACE - KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${KUBE_ROOT}/hack/lib/init.sh" -source "${KUBE_ROOT}/hack/lib/util.sh" -echo "Checking whether godeps are restored" -if ! kube::util::godep_restored 2>&1 | sed 's/^/ /'; then - echo -e '\nExecute script 'hack/godep-restore.sh' to download dependencies.' 1>&2 - exit 1 -fi +# Ensure you have a clean working directory before starting +kube::util::ensure_clean_working_dir -kube::util::ensure-temp-dir -kube::util::ensure_single_dir_gopath -kube::util::ensure_godep_version v79 -kube::util::ensure_no_staging_repos_in_gopath +# NOTE: All output from this script needs to be copied back to the calling +# source tree. This is managed in kube::build::copy_output in build/common.sh. +# If the output set is changed update that function. -TMP_GOPATH="${KUBE_TEMP}/go" +${KUBE_ROOT}/build/run.sh hack/update-staging-godeps-dockerized.sh "$@" -function updateGodepManifest() { - local repo="${1}" - pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null - echo "Updating godeps for k8s.io/${repo}" - rm -rf Godeps # remove the current Godeps.json so we always rebuild it - GOPATH="${TMP_GOPATH}:${GOPATH}:${KUBE_ROOT}/staging" godep save ${V} ./... 2>&1 | sed 's/^/ /' - - echo "Rewriting Godeps.json to remove commits that don't really exist because we haven't pushed the prereqs yet" - go run "${KUBE_ROOT}/staging/godeps-json-updater.go" --godeps-file="${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" --override-import-path="k8s.io/${repo}" - - # commit so that following repos do not see this repo as dirty - git add vendor >/dev/null - git commit -a -m "Updated Godeps.json" >/dev/null - popd >/dev/null -} - -# move into staging and save the dependencies for everything in order -mkdir -p "${TMP_GOPATH}/src/k8s.io" -for repo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do - kube::util::ensure_clean_working_dir - - cp -a "${KUBE_ROOT}/staging/src/k8s.io/${repo}" "${TMP_GOPATH}/src/k8s.io/" - - pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null - git init >/dev/null - git config --local user.email "nobody@k8s.io" - git config --local user.name "$0" - git add . >/dev/null - git commit -q -m "Snapshot" >/dev/null - popd >/dev/null - - updateGodepManifest "${repo}" - - if [ "${FAIL_ON_DIFF}" == true ]; then - diff --ignore-matching-lines='^\s*\"GoVersion\":' --ignore-matching-line='^\s*\"GodepVersion\":' --ignore-matching-lines='^\s*\"Comment\"' -u "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json" "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" - fi - if [ "${DRY_RUN}" != true ]; then - cp "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json" - fi -done +# ex: ts=2 sw=2 et filetype=sh diff --git a/hack/verify-staging-godeps.sh b/hack/verify-staging-godeps.sh index a81ffa53491..7a8c5f9977f 100755 --- a/hack/verify-staging-godeps.sh +++ b/hack/verify-staging-godeps.sh @@ -19,4 +19,4 @@ set -o nounset set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. -${KUBE_ROOT}/hack/update-staging-godeps.sh -d -f "$@" +KUBE_VERBOSE=3 KUBE_RUN_COPY_OUTPUT=N ${KUBE_ROOT}/hack/update-staging-godeps.sh -d -f "$@"