diff --git a/build/build-image/Dockerfile b/build/build-image/Dockerfile index e37a241de86..59be59e4091 100644 --- a/build/build-image/Dockerfile +++ b/build/build-image/Dockerfile @@ -36,6 +36,13 @@ ENV KUBE_OUTPUT_SUBPATH _output/dockerized # Pick up version stuff here as we don't copy our .git over. ENV KUBE_GIT_VERSION_FILE ${HOME}/.dockerized-kube-version-defs +# Add system-wide git user information +RUN git config --system user.email "nobody@k8s.io" \ + && git config --system user.name "kube-build-image" + +# Fix permissions on gopath +RUN chmod -R a+rwx $GOPATH + # Make log messages use the right timezone ADD localtime /etc/localtime RUN chmod a+r /etc/localtime diff --git a/build/build-image/cross/Dockerfile b/build/build-image/cross/Dockerfile index d3ec13ebf7e..6b7db4fc10d 100644 --- a/build/build-image/cross/Dockerfile +++ b/build/build-image/cross/Dockerfile @@ -37,7 +37,7 @@ RUN for platform in ${KUBE_CROSSPLATFORMS}; do GOOS=${platform%/*} GOARCH=${plat # Install g++, then download and install protoc for generating protobuf output RUN apt-get update \ - && apt-get install -y g++ rsync apt-utils file patch \ + && apt-get install -y g++ rsync jq apt-utils file patch \ && apt-get clean && rm -rf /var/lib/apt/lists/* RUN mkdir -p /usr/local/src/protobuf \ @@ -62,14 +62,13 @@ RUN echo "deb http://archive.ubuntu.com/ubuntu xenial main universe" > /etc/apt/ # work around 64MB tmpfs size in Docker 1.6 ENV TMPDIR /tmp.k8s - -# Get the code coverage tool, goimports, and godep RUN mkdir $TMPDIR \ && chmod a+rwx $TMPDIR \ - && chmod o+t $TMPDIR \ - && go get golang.org/x/tools/cmd/cover \ - golang.org/x/tools/cmd/goimports \ - github.com/tools/godep + && chmod o+t $TMPDIR + +# Get the code coverage tool and goimports +RUN go get golang.org/x/tools/cmd/cover \ + golang.org/x/tools/cmd/goimports # Download and symlink etcd. We need this for our integration tests. RUN export ETCD_VERSION=v3.0.17; \ diff --git a/build/build-image/cross/VERSION b/build/build-image/cross/VERSION index 490e779dc9b..25308d3ee1e 100644 --- a/build/build-image/cross/VERSION +++ b/build/build-image/cross/VERSION @@ -1 +1 @@ -v1.8.3-1 +v1.8.3-2 diff --git a/build/build-image/rsyncd.sh b/build/build-image/rsyncd.sh index 33aba4e3c02..a265e56e204 100755 --- a/build/build-image/rsyncd.sh +++ b/build/build-image/rsyncd.sh @@ -77,7 +77,7 @@ port = 8730 secrets file = ${SECRETS} read only = false path = ${VOLUME} - filter = - /.make/ - /.git/ - /_tmp/ + filter = - /.make/ - /_tmp/ EOF exec /usr/bin/rsync --no-detach --daemon --config="${CONFFILE}" "$@" diff --git a/build/common.sh b/build/common.sh index f6def8f3e3b..a2b5d21659a 100755 --- a/build/common.sh +++ b/build/common.sh @@ -682,14 +682,10 @@ function kube::build::sync_to_container() { # they will not be re-generated by 'make'. kube::build::rsync \ --delete \ - --filter='+ /staging/**' \ - --filter='- /.git/' \ --filter='- /.make/' \ --filter='- /_tmp/' \ --filter='- /_output/' \ --filter='- /' \ - --filter='- zz_generated.*' \ - --filter='- generated.proto' \ "${KUBE_ROOT}/" "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/" kube::build::stop_rsyncd_container @@ -715,8 +711,10 @@ function kube::build::copy_output() { # We are looking to copy out all of the built binaries along with various # generated files. kube::build::rsync \ - --filter='- /vendor/' \ --filter='- /_temp/' \ + --filter='+ /vendor/' \ + --filter='+ /Godeps/' \ + --filter='+ /staging/***/Godeps/**' \ --filter='+ /_output/dockerized/bin/**' \ --filter='+ zz_generated.*' \ --filter='+ generated.proto' \ diff --git a/hack/godep-restore.sh b/hack/godep-restore.sh index a982597f59b..ae9caf07442 100755 --- a/hack/godep-restore.sh +++ b/hack/godep-restore.sh @@ -24,6 +24,6 @@ source "${KUBE_ROOT}/hack/lib/util.sh" kube::util::ensure_godep_version v79 -echo "Starting to download all kubernetes godeps. This takes a while" +kube::log::status "Starting to download all kubernetes godeps. This takes a while" GOPATH=${GOPATH}:${KUBE_ROOT}/staging godep restore "$@" -echo "Download finished" +kube::log::status "Download finished" diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index f7e3a632bde..005ee328205 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -334,11 +334,13 @@ EOF local go_version go_version=($(go version)) - if [[ "${go_version[2]}" < "go1.8" && "${go_version[2]}" != "devel" ]]; then + local minimum_go_version + minimum_go_version=go1.8.3 + if [[ "${go_version[2]}" < "${minimum_go_version}" && "${go_version[2]}" != "devel" ]]; then kube::log::usage_from_stdin </dev/null)" == *"godep ${GODEP_VERSION}"* ]]; then return fi @@ -505,7 +505,7 @@ kube::util::ensure_godep_version() { GOPATH="${KUBE_TEMP}/go" go get -d -u github.com/tools/godep 2>/dev/null pushd "${KUBE_TEMP}/go/src/github.com/tools/godep" >/dev/null - git checkout "${GODEP_VERSION}" + git checkout -q "${GODEP_VERSION}" GOPATH="${KUBE_TEMP}/go" go install . popd >/dev/null @@ -825,6 +825,18 @@ function kube::util::ensure-cfssl { popd > /dev/null } +# kube::util::ensure_dockerized +# Confirms that the script is being run inside a kube-build image +# +function kube::util::ensure_dockerized { + if [[ -f /kube-build-image ]]; then + return 0 + else + echo "ERROR: This script is designed to be run inside a kube-build container" + exit 1 + fi +} + # Some useful colors. if [[ -z "${color_start-}" ]]; then declare -r color_start="\033[" diff --git a/hack/update-staging-godeps-dockerized.sh b/hack/update-staging-godeps-dockerized.sh new file mode 100755 index 00000000000..f1b48e5f611 --- /dev/null +++ b/hack/update-staging-godeps-dockerized.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# updates the godeps.json file in the staging folders to allow clean vendoring +# based on kubernetes levels. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" + +# Sets default values +DRY_RUN=false +FAIL_ON_DIFF=false +GODEP_OPTS="" + +# Controls verbosity of the script output and logging. +KUBE_VERBOSE="${KUBE_VERBOSE:-5}" +if (( ${KUBE_VERBOSE} >= 6 )); then + GODEP_OPTS+=("-v") +fi + +while getopts ":df" opt; do + case $opt in + d) # do not godep-restore into a temporary directory, but use the existing GOPATH + DRY_RUN=true + ;; + f) # fail if something in the Godeps.json files changed + FAIL_ON_DIFF=true + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + esac +done + +kube::util::ensure_dockerized +kube::golang::setup_env +kube::util::ensure_single_dir_gopath +kube::util::ensure_no_staging_repos_in_gopath +kube::util::ensure_godep_version v79 + +kube::log::status "Checking whether godeps are restored" +if ! kube::util::godep_restored 2>&1 | sed 's/^/ /'; then + ${KUBE_ROOT}/hack/godep-restore.sh +fi + +kube::util::ensure-temp-dir +TMP_GOPATH="${KUBE_TEMP}/go" + +function updateGodepManifest() { + pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null + kube::log::status "Updating godeps for k8s.io/${repo}" + rm -rf Godeps # remove the current Godeps.json so we always rebuild it + GOPATH="${TMP_GOPATH}:${GOPATH}:${GOPATH}/src/k8s.io/kubernetes/staging" godep save ${GODEP_OPTS} ./... 2>&1 | sed 's/^/ /' + + # Rewriting Godeps.json to remove commits that don't really exist because we haven't pushed the prereqs yet + go run "${KUBE_ROOT}/staging/godeps-json-updater.go" --godeps-file="${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" --override-import-path="k8s.io/${repo}" + + # commit so that following repos do not see this repo as dirty + git add vendor >/dev/null + git commit -a -m "Updated Godeps.json" >/dev/null + popd >/dev/null +} + +function diffGodepManifest() { + local ret=0 + diff --ignore-matching-lines='^\s*\"GoVersion\":' --ignore-matching-line='^\s*\"GodepVersion\":' --ignore-matching-lines='^\s*\"Comment\"' -u "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json" "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" || ret=$? + if [[ "${ret}" != "0" && "${FAIL_ON_DIFF}" == true ]]; then + exit ${ret} + fi +} + +# move into staging and save the dependencies for everything in order +mkdir -p "${TMP_GOPATH}/src/k8s.io" +for repo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do + cp -a "${KUBE_ROOT}/staging/src/k8s.io/${repo}" "${TMP_GOPATH}/src/k8s.io/" + + pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null + git init >/dev/null + git config --local user.email "nobody@k8s.io" + git config --local user.name "$0" + git add . >/dev/null + git commit -q -m "Snapshot" >/dev/null + popd >/dev/null + + updateGodepManifest + diffGodepManifest + + if [ "${DRY_RUN}" != true ]; then + cp "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json" + # Assume Godeps.json is not updated, as the working tree needs to be clean + # Without this, the script would pause after each staging repo to prompt the + # user to commit all changes (difficult inside a container). It's safe to + # ignore this file, as we know it's going to be changing, and we don't copy + # the git tree back out from the container. + git update-index --assume-unchanged "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json" + fi +done diff --git a/hack/update-staging-godeps.sh b/hack/update-staging-godeps.sh index 553cddb5ca9..0b5c11b1d5b 100755 --- a/hack/update-staging-godeps.sh +++ b/hack/update-staging-godeps.sh @@ -14,91 +14,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -# updates the godeps.json file in the staging folders to allow clean vendoring -# based on kubernetes levels. -# TODO this does not address client-go, since it takes a different approach to vendoring -# TODO client-go should probably be made consistent - set -o errexit set -o nounset set -o pipefail -V="" -DRY_RUN=false -FAIL_ON_DIFF=false -while getopts ":vdf" opt; do - case $opt in - v) # increase verbosity - V="-v" - ;; - d) # do not godep-restore into a temporary directory, but use the existing GOPATH - DRY_RUN=true - ;; - f) # fail if something in the Godeps.json files changed - FAIL_ON_DIFF=true - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - exit 1 - ;; - esac -done -readonly V IN_PLACE - KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${KUBE_ROOT}/hack/lib/init.sh" -source "${KUBE_ROOT}/hack/lib/util.sh" -echo "Checking whether godeps are restored" -if ! kube::util::godep_restored 2>&1 | sed 's/^/ /'; then - echo -e '\nExecute script 'hack/godep-restore.sh' to download dependencies.' 1>&2 - exit 1 -fi +# Ensure you have a clean working directory before starting +kube::util::ensure_clean_working_dir -kube::util::ensure-temp-dir -kube::util::ensure_single_dir_gopath -kube::util::ensure_godep_version v79 -kube::util::ensure_no_staging_repos_in_gopath +# NOTE: All output from this script needs to be copied back to the calling +# source tree. This is managed in kube::build::copy_output in build/common.sh. +# If the output set is changed update that function. -TMP_GOPATH="${KUBE_TEMP}/go" +${KUBE_ROOT}/build/run.sh hack/update-staging-godeps-dockerized.sh "$@" -function updateGodepManifest() { - local repo="${1}" - pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null - echo "Updating godeps for k8s.io/${repo}" - rm -rf Godeps # remove the current Godeps.json so we always rebuild it - GOPATH="${TMP_GOPATH}:${GOPATH}:${KUBE_ROOT}/staging" godep save ${V} ./... 2>&1 | sed 's/^/ /' - - echo "Rewriting Godeps.json to remove commits that don't really exist because we haven't pushed the prereqs yet" - go run "${KUBE_ROOT}/staging/godeps-json-updater.go" --godeps-file="${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" --override-import-path="k8s.io/${repo}" - - # commit so that following repos do not see this repo as dirty - git add vendor >/dev/null - git commit -a -m "Updated Godeps.json" >/dev/null - popd >/dev/null -} - -# move into staging and save the dependencies for everything in order -mkdir -p "${TMP_GOPATH}/src/k8s.io" -for repo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do - kube::util::ensure_clean_working_dir - - cp -a "${KUBE_ROOT}/staging/src/k8s.io/${repo}" "${TMP_GOPATH}/src/k8s.io/" - - pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null - git init >/dev/null - git config --local user.email "nobody@k8s.io" - git config --local user.name "$0" - git add . >/dev/null - git commit -q -m "Snapshot" >/dev/null - popd >/dev/null - - updateGodepManifest "${repo}" - - if [ "${FAIL_ON_DIFF}" == true ]; then - diff --ignore-matching-lines='^\s*\"GoVersion\":' --ignore-matching-line='^\s*\"GodepVersion\":' --ignore-matching-lines='^\s*\"Comment\"' -u "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json" "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" - fi - if [ "${DRY_RUN}" != true ]; then - cp "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json" - fi -done +# ex: ts=2 sw=2 et filetype=sh diff --git a/hack/verify-staging-godeps.sh b/hack/verify-staging-godeps.sh index a81ffa53491..7a8c5f9977f 100755 --- a/hack/verify-staging-godeps.sh +++ b/hack/verify-staging-godeps.sh @@ -19,4 +19,4 @@ set -o nounset set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. -${KUBE_ROOT}/hack/update-staging-godeps.sh -d -f "$@" +KUBE_VERBOSE=3 KUBE_RUN_COPY_OUTPUT=N ${KUBE_ROOT}/hack/update-staging-godeps.sh -d -f "$@"