diff --git a/.gitignore b/.gitignore index 88675afaea9..bf7b19e9c66 100644 --- a/.gitignore +++ b/.gitignore @@ -130,5 +130,5 @@ zz_generated_*_test.go /bazel-* *.pyc -# generated by verify-godeps.sh +# generated by verify-vendor.sh vendordiff.patch diff --git a/build/root/Makefile b/build/root/Makefile index 57501d0768a..3664bb000bf 100644 --- a/build/root/Makefile +++ b/build/root/Makefile @@ -111,7 +111,7 @@ define VERIFY_HELP_INFO # Runs all the presubmission verifications. # # Args: -# BRANCH: Branch to be passed to verify-godeps.sh script. +# BRANCH: Branch to be passed to verify-vendor.sh script. # WHAT: List of checks to run # # Example: diff --git a/hack/.shellcheck_failures b/hack/.shellcheck_failures index bb1edb4ac76..c0ed815ac5f 100644 --- a/hack/.shellcheck_failures +++ b/hack/.shellcheck_failures @@ -53,12 +53,11 @@ ./hack/update-generated-kms-dockerized.sh ./hack/update-generated-protobuf-dockerized.sh ./hack/update-generated-runtime-dockerized.sh -./hack/update-godep-licenses.sh ./hack/update-gofmt.sh ./hack/update-openapi-spec.sh -./hack/update-staging-godeps-dockerized.sh -./hack/update-staging-godeps.sh ./hack/update-translations.sh +./hack/update-vendor-licenses.sh +./hack/update-vendor.sh ./hack/update-workspace-mirror.sh ./hack/verify-api-groups.sh ./hack/verify-boilerplate.sh diff --git a/hack/godep-restore.sh b/hack/godep-restore.sh deleted file mode 100755 index 2f1b10784f4..00000000000 --- a/hack/godep-restore.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -source "${KUBE_ROOT}/hack/lib/init.sh" - -kube::log::status "Restoring kubernetes godeps" - -if kube::util::godep_restored 2>&1; then - kube::log::status "Dependencies appear to be current - skipping download" - exit 0 -fi - -kube::util::ensure_godep_version - -kube::log::status "Downloading dependencies - this might take a while" -GOPATH="${GOPATH}:${KUBE_ROOT}/staging" ${KUBE_GODEP:?} restore "$@" -kube::log::status "Done" diff --git a/hack/godep-save.sh b/hack/godep-save.sh deleted file mode 100755 index 6430b6e6b3d..00000000000 --- a/hack/godep-save.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -source "${KUBE_ROOT}/hack/lib/init.sh" - -kube::log::status "Ensuring prereqs" -kube::util::ensure_single_dir_gopath -kube::util::ensure_no_staging_repos_in_gopath - -kube::util::ensure_godep_version - -BACKUP=_tmp/godep-save.$RANDOM -mkdir -p "${BACKUP}" - -function kube::godep_save::cleanup() { - if [[ -d "${BACKUP}/vendor" ]]; then - kube::log::error "${BACKUP}/vendor exists, restoring it" - rm -rf vendor - mv "${BACKUP}/vendor" vendor - fi - if [[ -d "${BACKUP}/Godeps" ]]; then - kube::log::error "${BACKUP}/Godeps exists, restoring it" - rm -rf Godeps - mv "${BACKUP}/Godeps" Godeps - fi -} -kube::util::trap_add kube::godep_save::cleanup EXIT - -# Clear old state, but save it in case of error -if [[ -d vendor ]]; then - mv vendor "${BACKUP}/vendor" -fi -if [[ -d Godeps ]]; then - mv Godeps "${BACKUP}/Godeps" -fi - -## Workaround "httplex" was dropped from golang.org/x/net repo and the code -## was moved to the "golang.org/x/net/http/httpguts" directory, we do not use -## this directly, however many packages we vendor are still using the older -## golang.org/x/net and we need to keep this until all those dependencies -## are switched to newer golang.org/x/net. -IGNORED_PACKAGES=( - "golang.org/x/net/lex/httplex" -) -REQUIRED_BINS=( - "golang.org/x/net/internal/nettest" - "golang.org/x/net/internal/socks" - "golang.org/x/net/internal/sockstest" -) - -# Some things we want in godeps aren't code dependencies, so ./... -# won't pick them up. -REQUIRED_BINS+=( - "github.com/bazelbuild/bazel-gazelle/cmd/gazelle" - "github.com/bazelbuild/buildtools/buildozer" - "github.com/cespare/prettybench" - "github.com/client9/misspell/cmd/misspell" - "github.com/cloudflare/cfssl/cmd/cfssl" - "github.com/cloudflare/cfssl/cmd/cfssljson" - "github.com/jstemmer/go-junit-report" - "github.com/jteeuwen/go-bindata/go-bindata" - "github.com/onsi/ginkgo/ginkgo" - "golang.org/x/lint/golint" - "k8s.io/kube-openapi/cmd/openapi-gen" - "k8s.io/repo-infra/kazel" - "./..." -) - -kube::log::status "Running godep save - this might take a while" -# This uses $(pwd) rather than ${KUBE_ROOT} because KUBE_ROOT will be -# realpath'ed, and godep barfs ("... is not using a known version control -# system") on our staging dirs. -GOPATH="${GOPATH}:$(pwd)/staging" "${KUBE_GODEP:?}" save -i "$(IFS=,; echo "${IGNORED_PACKAGES[*]}")" "${REQUIRED_BINS[@]}" - -# create a symlink in vendor directory pointing to the staging client. This -# let other packages use the staging client as if it were vendored. -for repo in staging/src/k8s.io/*; do - repo="${repo#staging/src/k8s.io/}" - if [ ! -e "vendor/k8s.io/${repo}" ]; then - ln -s "../../staging/src/k8s.io/${repo}" "vendor/k8s.io/${repo}" - fi -done - -# Workaround broken symlink in docker repo because godep copies the link, but -# not the target -rm -rf vendor/github.com/docker/docker/project/ - -kube::log::status "Updating BUILD files" -# Assume that anything imported through godep doesn't need Bazel to build. -# Prune out any Bazel build files, since these can break the build due to -# missing dependencies that aren't included by godep. -find vendor/ -type f \( -name BUILD -o -name BUILD.bazel -o -name WORKSPACE \) \ - -exec rm -f {} \; -hack/update-bazel.sh >/dev/null - -kube::log::status "Updating LICENSES file" -hack/update-godep-licenses.sh >/dev/null - -kube::log::status "Creating OWNERS file" -rm -f "Godeps/OWNERS" "vendor/OWNERS" -cat <<__EOF__ > "Godeps/OWNERS" -# See the OWNERS docs at https://go.k8s.io/owners - -approvers: -- dep-approvers -__EOF__ -cp "Godeps/OWNERS" "vendor/OWNERS" - -# Clean up -rm -rf "${BACKUP}" diff --git a/hack/jenkins/verify.sh b/hack/jenkins/verify.sh index 8de1e490364..2da874fde2c 100755 --- a/hack/jenkins/verify.sh +++ b/hack/jenkins/verify.sh @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Sets up the environment (e.g. installing godep and etcd if necessary) +# Sets up the environment (e.g. installing etcd if necessary) # and then runs all of the verification checks. set -o errexit diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index 22a3bc2be5f..36f9f437e84 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -437,7 +437,7 @@ kube::golang::setup_env() { # Unset GOBIN in case it already exists in the current session. unset GOBIN - # This seems to matter to some tools (godep, ginkgo...) + # This seems to matter to some tools export GO15VENDOREXPERIMENT=1 } diff --git a/hack/lib/init.sh b/hack/lib/init.sh index f169a609b55..42f34567db9 100755 --- a/hack/lib/init.sh +++ b/hack/lib/init.sh @@ -22,6 +22,12 @@ set -o pipefail # https://github.com/kubernetes/kubernetes/issues/52255 unset CDPATH +# Until all GOPATH references are removed from all build scripts as well, +# explicitly reset to auto mode to avoid picking up user-set GO111MODULE preferences. +# As individual scripts (like hack/update-vendor.sh) make use of go modules, +# they can explicitly set GO111MODULE=on +export GO111MODULE=auto + # The root of the build/dist directory KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE}")/../.." && pwd -P)" diff --git a/hack/lib/util.sh b/hack/lib/util.sh index 8552db40515..2bd68348e0d 100755 --- a/hack/lib/util.sh +++ b/hack/lib/util.sh @@ -330,64 +330,6 @@ kube::util::git_upstream_remote_name() { head -n 1 | awk '{print $1}' } -# Ensures the current directory is a git tree for doing things like restoring or -# validating godeps -kube::util::create-fake-git-tree() { - local -r target_dir=${1:-$(pwd)} - - pushd "${target_dir}" >/dev/null || return 1 - git init >/dev/null - git config --local user.email "nobody@k8s.io" - git config --local user.name "$0" - git add . >/dev/null - git commit -q -m "Snapshot" >/dev/null - if (( ${KUBE_VERBOSE:-5} >= 6 )); then - kube::log::status "${target_dir} is now a git tree." - fi - popd >/dev/null || return 1 -} - -# Checks whether godep restore was run in the current GOPATH, i.e. that all referenced repos exist -# and are checked out to the referenced rev. -kube::util::godep_restored() { - local -r godeps_json=${1:-Godeps/Godeps.json} - local -r gopath=${2:-${GOPATH%:*}} - - kube::util::require-jq - - local root - local old_rev="" - while read -r path rev; do - rev="${rev//[\'\"]}" # remove quotes which are around revs sometimes - - if [[ "${rev}" == "${old_rev}" ]] && [[ "${path}" == "${root}"* ]]; then - # avoid checking the same git/hg root again - continue - fi - - root="${path}" - while [ "${root}" != "." ] && [ ! -d "${gopath}/src/${root}/.git" ] && [ ! -d "${gopath}/src/${root}/.hg" ]; do - root=$(dirname "${root}") - done - if [ "${root}" == "." ]; then - echo "No checkout of ${path} found in GOPATH \"${gopath}\"." 1>&2 - return 1 - fi - local head - if [ -d "${gopath}/src/${root}/.git" ]; then - head="$(cd "${gopath}/src/${root}" && git rev-parse HEAD)" - else - head="$(cd "${gopath}/src/${root}" && hg parent --template '{node}')" - fi - if [ "${head}" != "${rev}" ]; then - echo "Unexpected HEAD '${head}' at ${gopath}/src/${root}, expected '${rev}'." 1>&2 - return 1 - fi - old_rev="${rev}" - done < <(jq '.Deps|.[]|.ImportPath + " " + .Rev' -r < "${godeps_json}") - return 0 -} - # Exits script if working directory is dirty. If it's run interactively in the terminal # the user can commit changes in a second terminal. This script will wait. kube::util::ensure_clean_working_dir() { @@ -404,35 +346,6 @@ kube::util::ensure_clean_working_dir() { done 1>&2 } -# Ensure that none of the staging repos is checked out in the GOPATH because this -# easily confused godep. -kube::util::ensure_no_staging_repos_in_gopath() { - kube::util::ensure_single_dir_gopath - local error=0 - for repo_file in "${KUBE_ROOT}"/staging/src/k8s.io/*; do - if [[ ! -d "${repo_file}" ]]; then - # not a directory or there were no files - continue; - fi - repo="$(basename "${repo_file}")" - if [ -e "${GOPATH}/src/k8s.io/${repo}" ]; then - echo "k8s.io/${repo} exists in GOPATH. Remove before running godep-save.sh." 1>&2 - error=1 - fi - done - if [ "${error}" = "1" ]; then - exit 1 - fi -} - -# Checks that the GOPATH is simple, i.e. consists only of one directory, not multiple. -kube::util::ensure_single_dir_gopath() { - if [[ "${GOPATH}" == *:* ]]; then - echo "GOPATH must consist of a single directory." 1>&2 - exit 1 - fi -} - # Find the base commit using: # $PULL_BASE_SHA if set (from Prow) # current ref from the remote upstream branch diff --git a/hack/make-rules/update.sh b/hack/make-rules/update.sh index 035af3a15a2..8b86a7131b4 100755 --- a/hack/make-rules/update.sh +++ b/hack/make-rules/update.sh @@ -47,8 +47,6 @@ if ! ${ALL} ; then echo "Running in short-circuit mode; run with FORCE_ALL=true to force all scripts to run." fi -"${KUBE_ROOT}/hack/godep-restore.sh" ${V} - BASH_TARGETS=" update-generated-protobuf update-codegen @@ -57,7 +55,6 @@ BASH_TARGETS=" update-generated-docs update-generated-swagger-docs update-openapi-spec - update-staging-godeps update-bazel" for t in ${BASH_TARGETS}; do diff --git a/hack/make-rules/verify.sh b/hack/make-rules/verify.sh index 3eb6ff7c9fa..a3e80423317 100755 --- a/hack/make-rules/verify.sh +++ b/hack/make-rules/verify.sh @@ -45,12 +45,13 @@ if [[ ${EXCLUDE_TYPECHECK:-} =~ ^[yY]$ ]]; then fi -# Exclude godep checks in certain cases, if they're running in a separate job. +# Exclude vendor checks in certain cases, if they're running in a separate job. if [[ ${EXCLUDE_GODEP:-} =~ ^[yY]$ ]]; then EXCLUDED_PATTERNS+=( "verify-godeps.sh" # runs in separate godeps job - "verify-staging-godeps.sh" # runs in separate godeps job - "verify-godep-licenses.sh" # runs in separate godeps job + "verify-godeps-licenses.sh" # runs in separate godeps job + "verify-vendor.sh" # runs in separate godeps job + "verify-vendor-licenses.sh" # runs in separate godeps job ) fi @@ -63,12 +64,12 @@ fi # Only run whitelisted fast checks in quick mode. # These run in <10s each on enisoc's workstation, assuming that -# `make` and `hack/godep-restore.sh` had already been run. +# `make` had already been run. QUICK_PATTERNS+=( "verify-api-groups.sh" "verify-bazel.sh" "verify-boilerplate.sh" - "verify-godep-licenses.sh" + "verify-vendor-licenses.sh" "verify-gofmt.sh" "verify-imports.sh" "verify-pkg-names.sh" diff --git a/hack/pin-dependency.sh b/hack/pin-dependency.sh new file mode 100755 index 00000000000..b2aac35002d --- /dev/null +++ b/hack/pin-dependency.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" + +# Usage: +# hack/pin-dependency.sh $MODULE $SHA-OR-TAG +# +# Example: +# hack/pin-dependency.sh github.com/docker/docker 501cb131a7b7 + +# Explicitly opt into go modules, even though we're inside a GOPATH directory +export GO111MODULE=on +# Explicitly clear GOPATH, to ensure nothing this script calls makes use of that path info +export GOPATH= +# Explicitly clear GOFLAGS, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor +export GOFLAGS= +# Detect problematic GOPROXY settings that prevent lookup of dependencies +if [[ "${GOPROXY:-}" == "off" ]]; then + kube::log::error "Cannot run hack/pin-dependency.sh with \$GOPROXY=off" + exit 1 +fi + +kube::golang::verify_go_version +kube::util::require-jq + +dep="${1:-}" +sha="${2:-}" +if [[ -z "${dep}" || -z "${sha}" ]]; then + echo "Usage:" + echo " hack/pin-dependency.sh \$MODULE \$SHA-OR-TAG" + echo "" + echo "Example:" + echo " hack/pin-dependency.sh github.com/docker/docker 501cb131a7b7" + echo "" + exit 1 +fi + +# Add the require directive +echo "Running: go get ${dep}@${sha}" +go get "${dep}@${sha}" + +# Find the resolved version +rev=$(go mod edit -json | jq -r '.Require[] | select(.Path == "github.com/docker/docker") | .Version') +echo "Resolved to ${dep}@${rev}" + +# Add the replace directive +echo "Running: go mod edit -replace ${dep}=${dep}@${rev}" +go mod edit -replace "${dep}=${dep}@${rev}" + +echo "" +echo "Run hack/update-vendor.sh to rebuild the vendor directory" diff --git a/hack/update-bazel.sh b/hack/update-bazel.sh index 7f5c9328a22..f126e41d919 100755 --- a/hack/update-bazel.sh +++ b/hack/update-bazel.sh @@ -25,9 +25,11 @@ export GOBIN="${KUBE_OUTPUT_BINPATH}" PATH="${GOBIN}:${PATH}" # Install tools we need, but only from vendor/... -go install k8s.io/kubernetes/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle -go install k8s.io/kubernetes/vendor/github.com/bazelbuild/buildtools/buildozer -go install k8s.io/kubernetes/vendor/k8s.io/repo-infra/kazel +pushd "${KUBE_ROOT}/vendor" + go install ./github.com/bazelbuild/bazel-gazelle/cmd/gazelle + go install ./github.com/bazelbuild/buildtools/buildozer + go install ./k8s.io/repo-infra/kazel +popd # Find all of the staging repos. while IFS='' read -r repo; do staging_repos+=("${repo}"); done <\ diff --git a/hack/update-staging-godeps-dockerized.sh b/hack/update-staging-godeps-dockerized.sh deleted file mode 100755 index 790ba66a8cb..00000000000 --- a/hack/update-staging-godeps-dockerized.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# updates the godeps.json file in the staging folders to allow clean vendoring -# based on kubernetes levels. - -set -o errexit -set -o nounset -set -o pipefail - -KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. -source "${KUBE_ROOT}/hack/lib/init.sh" - -# Sets default values -DRY_RUN=false -FAIL_ON_DIFF=false -GODEP_OPTS="" - -# Controls verbosity of the script output and logging. -KUBE_VERBOSE="${KUBE_VERBOSE:-5}" -if (( ${KUBE_VERBOSE} >= 6 )); then - GODEP_OPTS+=("-v") -fi - -while getopts ":df" opt; do - case ${opt} in - d) # do not godep-restore into a temporary directory, but use the existing GOPATH - DRY_RUN=true - ;; - f) # fail if something in the Godeps.json files changed - FAIL_ON_DIFF=true - ;; - \?) - echo "Invalid option: -${OPTARG}" >&2 - exit 1 - ;; - esac -done - -# Confirm this is running inside a docker container, as this will modify the git tree (unsafe to run outside of container) -kube::util::ensure_dockerized -kube::golang::setup_env -# Ensure we have a simple gopath so that we can modify it, and that no staging repos have made their way in -kube::util::ensure_single_dir_gopath -kube::util::ensure_no_staging_repos_in_gopath -# Confirm we have the right godep version installed -kube::util::ensure_godep_version -# Create a fake git repo the root of the repo to prevent godeps from complaining -kube::util::create-fake-git-tree "${KUBE_ROOT}" - -"${KUBE_ROOT}/hack/godep-restore.sh" - -kube::util::ensure-temp-dir -TMP_GOPATH="${KUBE_TEMP}/go" - -function updateGodepManifest() { - pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null - kube::log::status "Updating godeps for k8s.io/${repo}" - rm -rf Godeps # remove the current Godeps.json so we always rebuild it - GOPATH="${TMP_GOPATH}:${GOPATH}:${GOPATH}/src/k8s.io/kubernetes/staging" ${KUBE_GODEP:?} save ${GODEP_OPTS} ./... 2>&1 | sed 's/^/ /' - - # Rewriting Godeps.json to cross-out commits that don't really exist because we haven't pushed the prereqs yet - local repo - for repo in $(ls -1 ${KUBE_ROOT}/staging/src/k8s.io); do - # remove staging prefix - jq '.Deps |= map(.ImportPath |= ltrimstr("k8s.io/kubernetes/staging/src/"))' Godeps/Godeps.json | - - # x-out staging repo revisions. They will only be known when the publisher bot has created the final export. - # We keep the staging dependencies in here though to give the publisher bot a way to detect when the staging - # dependencies changed. If they have changed, the bot will run a complete godep restore+save. If they didn't - # it will avoid that step, which takes quite some time. - jq '.Deps |= map((select(.ImportPath | (startswith("k8s.io/'${repo}'/") or . == "k8s.io/'${repo}'")) | .Rev |= "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx") // .)' | - - # remove comments - jq 'del(.Deps[].Comment)' | - - # format with tabs - unexpand --first-only --tabs=2 > Godeps/Godeps.json.out - - mv Godeps/Godeps.json.out Godeps/Godeps.json - done - - # commit so that following repos do not see this repo as dirty - git add vendor >/dev/null - git commit -a -m "Updated Godeps.json" >/dev/null - popd >/dev/null -} - -function diffGodepManifest() { - local ret=0 - diff --ignore-matching-lines='^\s*\"GoVersion\":' --ignore-matching-line='^\s*\"GodepVersion\":' --ignore-matching-lines='^\s*\"Comment\"' -u "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json" "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" || ret=$? - if [[ "${ret}" != "0" && "${FAIL_ON_DIFF}" == true ]]; then - exit ${ret} - fi -} - -# move into staging and save the dependencies for everything in order -mkdir -p "${TMP_GOPATH}/src/k8s.io" -for repo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do - cp -a "${KUBE_ROOT}/staging/src/k8s.io/${repo}" "${TMP_GOPATH}/src/k8s.io/" - - # Create a fake git tree for the staging repo to prevent godeps from complaining - kube::util::create-fake-git-tree "${TMP_GOPATH}/src/k8s.io/${repo}" - - updateGodepManifest - diffGodepManifest - - if [ "${DRY_RUN}" != true ]; then - cp "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json" - cp "${KUBE_ROOT}/Godeps/OWNERS" "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/OWNERS" - # Assume Godeps.json is not updated, as the working tree needs to be clean - # Without this, the script would pause after each staging repo to prompt the - # user to commit all changes (difficult inside a container). It's safe to - # ignore this file, as we know it's going to be changing, and we don't copy - # the git tree back out from the container. - git update-index --assume-unchanged "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json" - fi -done diff --git a/hack/update-staging-godeps.sh b/hack/update-staging-godeps.sh deleted file mode 100755 index 75da7a411a5..00000000000 --- a/hack/update-staging-godeps.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. -source "${KUBE_ROOT}/hack/lib/init.sh" - -# Ensure you have a clean working directory before starting -kube::util::ensure_clean_working_dir - -# NOTE: All output from this script needs to be copied back to the calling -# source tree. This is managed in kube::build::copy_output in build/common.sh. -# If the output set is changed update that function. - -${KUBE_ROOT}/build/run.sh hack/update-staging-godeps-dockerized.sh "$@" - -# ex: ts=2 sw=2 et filetype=sh diff --git a/hack/update-godep-licenses.sh b/hack/update-vendor-licenses.sh similarity index 90% rename from hack/update-godep-licenses.sh rename to hack/update-vendor-licenses.sh index 6385f1be1df..b823cb8125f 100755 --- a/hack/update-godep-licenses.sh +++ b/hack/update-vendor-licenses.sh @@ -21,9 +21,9 @@ # # --create-missing will write the files that only exist upstream, locally. # This option is mostly used for testing as we cannot check-in any of the -# additionally created files into the godep auto-generated tree. +# additionally created files into the vendor auto-generated tree. # -# Run every time a license file is added/modified within /Godeps to +# Run every time a license file is added/modified within /vendor to # update /Godeps/LICENSES set -o errexit @@ -129,6 +129,8 @@ process_content () { KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${KUBE_ROOT}/hack/lib/init.sh" +export GO111MODULE=on + # Check bash version if ((${BASH_VERSINFO[0]}<4)); then echo @@ -147,7 +149,7 @@ fi LICENSE_ROOT="${LICENSE_ROOT:-${KUBE_ROOT}}" cd "${LICENSE_ROOT}" -GODEPS_LICENSE_FILE="Godeps/LICENSES" +VENDOR_LICENSE_FILE="Godeps/LICENSES" TMP_LICENSE_FILE="/tmp/Godeps.LICENSES.$$" DEPS_DIR="vendor" declare -Ag CONTENT @@ -163,8 +165,17 @@ echo "= LICENSE $(cat "${LICENSE_ROOT}/LICENSE" | md5sum | awk '{print $1}')" echo "================================================================================" ) > ${TMP_LICENSE_FILE} -# Loop through every package in Godeps.json -for PACKAGE in $(jq -r ".Deps[].ImportPath" < Godeps/Godeps.json | sort -f); do +# Loop through every vendored package +for PACKAGE in $(go list -m -json all | jq -r .Path | sort -f); do + if [[ -e "staging/src/${PACKAGE}" ]]; then + echo "$PACKAGE is a staging package, skipping" > /dev/stderr + continue + fi + if [[ ! -e "${DEPS_DIR}/${PACKAGE}" ]]; then + echo "$PACKAGE doesn't exist in vendor, skipping" > /dev/stderr + continue + fi + process_content "${PACKAGE}" LICENSE process_content "${PACKAGE}" COPYRIGHT process_content "${PACKAGE}" COPYING @@ -204,4 +215,4 @@ __EOF__ echo done >> ${TMP_LICENSE_FILE} -cat ${TMP_LICENSE_FILE} > ${GODEPS_LICENSE_FILE} +cat ${TMP_LICENSE_FILE} > ${VENDOR_LICENSE_FILE} diff --git a/hack/update-vendor.sh b/hack/update-vendor.sh new file mode 100755 index 00000000000..25b81fa75b0 --- /dev/null +++ b/hack/update-vendor.sh @@ -0,0 +1,295 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" + +# Explicitly opt into go modules, even though we're inside a GOPATH directory +export GO111MODULE=on +# Explicitly clear GOPATH, to ensure nothing this script calls makes use of that path info +export GOPATH= +# Explicitly clear GOFLAGS, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor +export GOFLAGS= +# Ensure sort order doesn't depend on locale +export LANG=C +export LC_ALL=C +# Detect problematic GOPROXY settings that prevent lookup of dependencies +if [[ "${GOPROXY:-}" == "off" ]]; then + kube::log::error "Cannot run hack/update-vendor.sh with \$GOPROXY=off" + exit 1 +fi + +kube::golang::verify_go_version +kube::util::require-jq + +TMP_DIR="${TMP_DIR:-$(mktemp -d /tmp/update-vendor.XXXX)}" +LOG_FILE="${LOG_FILE:-${TMP_DIR}/update-vendor.log}" +kube::log::status "logfile at ${LOG_FILE}" + +if [ -z "${BASH_XTRACEFD:-}" ]; then + exec 19> "${LOG_FILE}" + export BASH_XTRACEFD="19" + set -x +fi + +# ensure_require_replace_directives_for_all_dependencies: +# - ensures all existing 'require' directives have an associated 'replace' directive pinning a version +# - adds explicit 'require' directives for all transitive dependencies +# - adds explicit 'replace' directives for all require directives (existing 'replace' directives take precedence) +function ensure_require_replace_directives_for_all_dependencies() { + local local_tmp_dir + local_tmp_dir=$(mktemp -d "${TMP_DIR}/pin_replace.XXXX") + + # collect 'require' directives that actually specify a version + local require_filter='(.Version != null) and (.Version != "v0.0.0") and (.Version != "v0.0.0-00010101000000-000000000000")' + # collect 'replace' directives that unconditionally pin versions (old=new@version) + local replace_filter='(.Old.Version == null) and (.New.Version != null)' + + # Capture local require/replace directives before running any go commands that can modify the go.mod file + local require_json="${local_tmp_dir}/require.json" + local replace_json="${local_tmp_dir}/replace.json" + go mod edit -json | jq -r ".Require // [] | sort | .[] | select(${require_filter})" > "${require_json}" + go mod edit -json | jq -r ".Replace // [] | sort | .[] | select(${replace_filter})" > "${replace_json}" + + # 1. Ensure require directives have a corresponding replace directive pinning a version + cat "${require_json}" | jq -r '"-replace \(.Path)=\(.Path)@\(.Version)"' | xargs -L 100 go mod edit -fmt + cat "${replace_json}" | jq -r '"-replace \(.Old.Path)=\(.New.Path)@\(.New.Version)"'| xargs -L 100 go mod edit -fmt + + # 2. Add explicit require directives for indirect dependencies + go list -m -json all | jq -r 'select(.Main != true) | select(.Indirect == true) | "-require \(.Path)@\(.Version)"' | xargs -L 100 go mod edit -fmt + + # 3. Add explicit replace directives pinning dependencies that aren't pinned yet + go list -m -json all | jq -r 'select(.Main != true) | select(.Replace == null) | "-replace \(.Path)=\(.Path)@\(.Version)"' | xargs -L 100 go mod edit -fmt +} + +function group_replace_directives() { + local local_tmp_dir + local_tmp_dir=$(mktemp -d "${TMP_DIR}/group_replace.XXXX") + local go_mod_replace="${local_tmp_dir}/go.mod.replace.tmp" + local go_mod_noreplace="${local_tmp_dir}/go.mod.noreplace.tmp" + # separate replace and non-replace directives + cat go.mod | awk " + # print lines between 'replace (' ... ')' lines + /^replace [(]/ { inreplace=1; next } + inreplace && /^[)]/ { inreplace=0; next } + inreplace { print > \"${go_mod_replace}\"; next } + + # print ungrouped replace directives with the replace directive trimmed + /^replace [^(]/ { sub(/^replace /,\"\"); print > \"${go_mod_replace}\"; next } + + # otherwise print to the noreplace file + { print > \"${go_mod_noreplace}\" } + " + cat "${go_mod_noreplace}" > go.mod + echo "replace (" >> go.mod + cat "${go_mod_replace}" >> go.mod + echo ")" >> go.mod + + go mod edit -fmt +} + +function add_generated_comments() { + local local_tmp_dir + local_tmp_dir=$(mktemp -d "${TMP_DIR}/add_generated_comments.XXXX") + local go_mod_nocomments="${local_tmp_dir}/go.mod.nocomments.tmp" + + # drop comments before the module directive + cat go.mod | awk " + BEGIN { dropcomments=1 } + /^module / { dropcomments=0 } + dropcomments && /^\/\// { next } + { print } + " > "${go_mod_nocomments}" + + # Add the specified comments + local comments="${1}" + echo "${comments}" > go.mod + echo "" >> go.mod + cat "${go_mod_nocomments}" >> go.mod + + # Format + go mod edit -fmt +} + + +# Phase 1: ensure go.mod files for staging modules and main module + +for repo in $(ls staging/src/k8s.io); do + pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1 + if [[ ! -f go.mod ]]; then + kube::log::status "go.mod: initialize ${repo}" + rm -f Godeps/Godeps.json # remove before initializing, staging Godeps are not authoritative + go mod init "k8s.io/${repo}" + go mod edit -fmt + fi + popd >/dev/null 2>&1 +done + +if [[ ! -f go.mod ]]; then + kube::log::status "go.mod: initialize k8s.io/kubernetes" + go mod init "k8s.io/kubernetes" + rm -f Godeps/Godeps.json # remove after initializing +fi + + +# Phase 2: ensure staging repo require/replace directives + +kube::log::status "go.mod: update staging references" +# Prune +go mod edit -json | jq -r '.Require[]? | select(.Version == "v0.0.0") | "-droprequire \(.Path)"' | xargs -L 100 go mod edit -fmt +go mod edit -json | jq -r '.Replace[]? | select(.New.Path | startswith("./staging/")) | "-dropreplace \(.Old.Path)"' | xargs -L 100 go mod edit -fmt +# Readd +ls staging/src/k8s.io | sort | xargs -n 1 -I {} echo "-require k8s.io/{}@v0.0.0" | xargs -L 100 go mod edit -fmt +ls staging/src/k8s.io | sort | xargs -n 1 -I {} echo "-replace k8s.io/{}=./staging/src/k8s.io/{}" | xargs -L 100 go mod edit -fmt + + +# Phase 3: capture required (minimum) versions from all modules, and replaced (pinned) versions from the root module + +# pin referenced versions +ensure_require_replace_directives_for_all_dependencies +# resolves/expands references in the root go.mod (if needed) +go mod tidy >>"${LOG_FILE}" 2>&1 +# pin expanded versions +ensure_require_replace_directives_for_all_dependencies +# group replace directives +group_replace_directives + +# Phase 4: copy root go.mod to staging dirs and rewrite + +kube::log::status "go.mod: propagate to staging modules" +for repo in $(ls staging/src/k8s.io | sort); do + pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1 + echo "=== propagating to ${repo}" >> "${LOG_FILE}" + # copy root go.mod, changing module name + cat "${KUBE_ROOT}/go.mod" | sed "s#module k8s.io/kubernetes#module k8s.io/${repo}#" > "${KUBE_ROOT}/staging/src/k8s.io/${repo}/go.mod" + # remove `require` directives for staging components (will get re-added as needed by `go list`) + ls "${KUBE_ROOT}/staging/src/k8s.io" | sort | xargs -n 1 -I {} echo "-droprequire k8s.io/{}" | xargs -L 100 go mod edit + # rewrite `replace` directives for staging components to point to peer directories + ls "${KUBE_ROOT}/staging/src/k8s.io" | sort | xargs -n 1 -I {} echo "-replace k8s.io/{}=../{}" | xargs -L 100 go mod edit + popd >/dev/null 2>&1 +done + + +# Phase 5: sort and tidy staging components + +kube::log::status "go.mod: sorting staging modules" +# tidy staging repos in reverse dependency order. +# the content of dependencies' go.mod files affects what `go mod tidy` chooses to record in a go.mod file. +ls staging/src/k8s.io | sort | xargs -I {} echo "k8s.io/{}" > "${TMP_DIR}/tidy_unordered.txt" +rm -f "${TMP_DIR}/tidy_deps.txt" +for repo in $(cat "${TMP_DIR}/tidy_unordered.txt"); do + # record existence of the repo to ensure modules with no peer relationships still get included in the order + echo "${repo} ${repo}" >> "${TMP_DIR}/tidy_deps.txt" + + pushd "${KUBE_ROOT}/staging/src/${repo}" >/dev/null 2>&1 + # save the original go.mod, since go list doesn't just add missing entries, it also removes specific required versions from it + tmp_go_mod="${TMP_DIR}/tidy_${repo/\//_}_go.mod.original" + tmp_go_deps="${TMP_DIR}/tidy_${repo/\//_}_deps.txt" + cp go.mod "${tmp_go_mod}" + + echo "=== sorting ${repo}" >> "${LOG_FILE}" + # 'go list' calculates direct imports and updates go.mod so that go list -m lists our module dependencies + echo "=== computing imports for ${repo}" >> "${LOG_FILE}" + go list all >>"${LOG_FILE}" 2>&1 + echo "=== computing tools imports for ${repo}" >> "${LOG_FILE}" + go list -tags=tools all >>"${LOG_FILE}" 2>&1 + + # capture module dependencies + go list -m -f '{{if not .Main}}{{.Path}}{{end}}' all > "${tmp_go_deps}" + + # restore the original go.mod file + cp "${tmp_go_mod}" go.mod + + # list all module dependencies + for dep in $(join "${TMP_DIR}/tidy_unordered.txt" "${tmp_go_deps}"); do + # record the relationship (put dep first, because we want to sort leaves first) + echo "${dep} ${repo}" >> "${TMP_DIR}/tidy_deps.txt" + # switch the required version to an explicit v0.0.0 (rather than an unknown v0.0.0-00010101000000-000000000000) + go mod edit -require "${dep}@v0.0.0" + done + popd >/dev/null 2>&1 +done + +kube::log::status "go.mod: tidying" +for repo in $(tsort "${TMP_DIR}/tidy_deps.txt"); do + pushd "${KUBE_ROOT}/staging/src/${repo}" >/dev/null 2>&1 + echo "=== tidying ${repo}" >> "${LOG_FILE}" + go mod tidy >>"${LOG_FILE}" 2>&1 + popd >/dev/null 2>&1 +done +echo "=== tidying root" >> "${LOG_FILE}" +go mod tidy >>"${LOG_FILE}" 2>&1 + + +# Phase 6: add generated comments to go.mod files +kube::log::status "go.mod: adding generated comments" +add_generated_comments " +// This is a generated file. +// Run hack/pin-dependency.sh to change pinned dependency versions. +// Run hack/update-vendor.sh to update go.mod files and the vendor directory. +" +for repo in $(ls staging/src/k8s.io | sort); do + pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1 + add_generated_comments " +// This is a generated file. Do not edit directly. +// Run hack/pin-dependency.sh to change pinned dependency versions. +// Run hack/update-vendor.sh to update go.mod files and the vendor directory. +" + popd >/dev/null 2>&1 +done + + +# Phase 6: rebuild vendor directory + +kube::log::status "vendor: running 'go mod vendor'" +go mod vendor >>"${LOG_FILE}" 2>&1 + +# sort recorded packages for a given vendored dependency in modules.txt. +# `go mod vendor` outputs in imported order, which means slight go changes (or different platforms) can result in a differently ordered modules.txt. +# scan | prefix comment lines with the module name | sort field 1 | strip leading text on comment lines +cat vendor/modules.txt | awk '{if($1=="#") print $2 " " $0; else print}' | sort -k1,1 -s | sed 's/.*#/#/' > "${TMP_DIR}/modules.txt.tmp" +mv "${TMP_DIR}/modules.txt.tmp" vendor/modules.txt + +# create a symlink in vendor directory pointing to the staging components. +# This lets other packages and tools use the local staging components as if they were vendored. +for repo in $(ls staging/src/k8s.io); do + rm -fr "${KUBE_ROOT}/vendor/k8s.io/${repo}" + ln -s "../../staging/src/k8s.io/${repo}" "${KUBE_ROOT}/vendor/k8s.io/${repo}" +done + +kube::log::status "vendor: updating BUILD files" +# Assume that anything imported through vendor doesn't need Bazel to build. +# Prune out any Bazel build files, since these can break the build due to +# missing dependencies that aren't included by go mod vendor. +find vendor/ -type f \( -name BUILD -o -name BUILD.bazel -o -name WORKSPACE \) -exec rm -f {} \; +hack/update-bazel.sh >>"${LOG_FILE}" 2>&1 + +kube::log::status "vendor: updating LICENSES file" +hack/update-vendor-licenses.sh >>"${LOG_FILE}" 2>&1 + +kube::log::status "vendor: creating OWNERS file" +rm -f "Godeps/OWNERS" "vendor/OWNERS" +cat <<__EOF__ > "Godeps/OWNERS" +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: +- dep-approvers +__EOF__ +cp "Godeps/OWNERS" "vendor/OWNERS" diff --git a/hack/verify-godep-licenses.sh b/hack/verify-godep-licenses.sh index 76f33662f11..609eba4fab5 100755 --- a/hack/verify-godep-licenses.sh +++ b/hack/verify-godep-licenses.sh @@ -19,37 +19,5 @@ set -o nounset set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -source "${KUBE_ROOT}/hack/lib/init.sh" - -readonly branch=${1:-${KUBE_VERIFY_GIT_BRANCH:-master}} -if ! [[ ${KUBE_FORCE_VERIFY_CHECKS:-} =~ ^[yY]$ ]] && \ - ! kube::util::has_changes "${branch}" 'Godeps/' && \ - ! kube::util::has_changes "${branch}" 'vendor/'; then - exit 0 -fi - -# create a nice clean place to put our new godeps -# must be in the user dir (e.g. KUBE_ROOT) in order for the docker volume mount -# to work with docker-machine on macs -mkdir -p "${KUBE_ROOT}/_tmp" -_tmpdir="$(mktemp -d "${KUBE_ROOT}/_tmp/kube-godep-licenses.XXXXXX")" -#echo "Created workspace: ${_tmpdir}" -function cleanup { - #echo "Removing workspace: ${_tmpdir}" - rm -rf "${_tmpdir}" -} -trap cleanup EXIT - -cp -r "${KUBE_ROOT}/Godeps" "${_tmpdir}/Godeps" -ln -s "${KUBE_ROOT}/LICENSE" "${_tmpdir}" -ln -s "${KUBE_ROOT}/vendor" "${_tmpdir}" - -# Update Godep Licenses -LICENSE_ROOT="${_tmpdir}" "${KUBE_ROOT}/hack/update-godep-licenses.sh" - -# Compare Godep Licenses -if ! _out="$(diff -Naupr "${KUBE_ROOT}/Godeps/LICENSES" "${_tmpdir}/Godeps/LICENSES")"; then - echo "Your godep licenses file is out of date. Run hack/update-godep-licenses.sh and commit the results." >&2 - echo "${_out}" >&2 - exit 1 -fi +echo "Deprecated, use verify-vendor-licenses.sh instead" +"${KUBE_ROOT}/hack/verify-vendor-licenses.sh" "$@" diff --git a/hack/verify-godeps.sh b/hack/verify-godeps.sh index b565274a1d5..f016302cee7 100755 --- a/hack/verify-godeps.sh +++ b/hack/verify-godeps.sh @@ -19,109 +19,5 @@ set -o nounset set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -source "${KUBE_ROOT}/hack/lib/init.sh" - -readonly branch=${1:-${KUBE_VERIFY_GIT_BRANCH:-master}} -if ! [[ ${KUBE_FORCE_VERIFY_CHECKS:-} =~ ^[yY]$ ]] && \ - ! kube::util::has_changes "${branch}" 'Godeps/' && \ - ! kube::util::has_changes "${branch}" 'vendor/' && \ - ! kube::util::has_changes "${branch}" 'hack/lib/' && \ - ! kube::util::has_changes "${branch}" 'hack/.*godep'; then - exit 0 -fi - -# Ensure we have the right godep version available -kube::util::ensure_godep_version - -if [[ -z ${TMP_GOPATH:-} ]]; then - # Create a nice clean place to put our new godeps - _tmpdir="$(kube::realpath "$(mktemp -d -t gopath.XXXXXX)")" -else - # reuse what we might have saved previously - _tmpdir="${TMP_GOPATH}" -fi - -if [[ -z ${KEEP_TMP:-} ]]; then - KEEP_TMP=false -fi -function cleanup { - if [ "${KEEP_TMP}" == "true" ]; then - echo "Leaving ${_tmpdir} for you to examine or copy. Please delete it manually when finished. (rm -rf ${_tmpdir})" - else - echo "Removing ${_tmpdir}" - rm -rf "${_tmpdir}" - fi -} -trap cleanup EXIT - -# Copy the contents of the kube directory into the nice clean place -_kubetmp="${_tmpdir}/src/k8s.io" -mkdir -p "${_kubetmp}" -# should create ${_kubectmp}/kubernetes -git archive --format=tar --prefix=kubernetes/ "$(git write-tree)" | (cd "${_kubetmp}" && tar xf -) -_kubetmp="${_kubetmp}/kubernetes" - -# Do all our work in the new GOPATH -export GOPATH="${_tmpdir}" -export PATH="${GOPATH}/bin:${PATH}" - -pushd "${_kubetmp}" > /dev/null 2>&1 - # Restore the Godeps into our temp directory - hack/godep-restore.sh - - # Destroy deps in the copy of the kube tree - rm -rf ./Godeps ./vendor - - # For some reason the kube tree needs to be a git repo for the godep tool to - # run. Doesn't make sense. - git init > /dev/null 2>&1 - - # Recreate the Godeps using the nice clean set we just downloaded - hack/godep-save.sh -popd > /dev/null 2>&1 - -ret=0 - -pushd "${KUBE_ROOT}" > /dev/null 2>&1 - # Test for diffs - if ! _out="$(diff -Naupr --ignore-matching-lines='^\s*\"GoVersion\":' Godeps/Godeps.json "${_kubetmp}/Godeps/Godeps.json")"; then - echo "Your Godeps.json is different:" >&2 - echo "${_out}" >&2 - echo "Godeps Verify failed." >&2 - echo "${_out}" > godepdiff.patch - echo "If you're seeing this locally, run the below command to fix your Godeps.json:" >&2 - echo "patch -p0 < godepdiff.patch" >&2 - echo "(The above output can be saved as godepdiff.patch if you're not running this locally)" >&2 - echo "(The patch file should also be exported as a build artifact if run through CI)" >&2 - KEEP_TMP=true - if [[ -f godepdiff.patch && -d "${ARTIFACTS:-}" ]]; then - echo "Copying patch to artifacts.." - cp godepdiff.patch "${ARTIFACTS:-}/" - fi - ret=1 - fi - - if ! _out="$(diff -Naupr -x "BUILD" -x "AUTHORS*" -x "CONTRIBUTORS*" vendor "${_kubetmp}/vendor")"; then - echo "Your vendored results are different:" >&2 - echo "${_out}" >&2 - echo "Godeps Verify failed." >&2 - echo "${_out}" > vendordiff.patch - echo "If you're seeing this locally, run the below command to fix your directories:" >&2 - echo "patch -p0 < vendordiff.patch" >&2 - echo "(The above output can be saved as godepdiff.patch if you're not running this locally)" >&2 - echo "(The patch file should also be exported as a build artifact if run through CI)" >&2 - KEEP_TMP=true - if [[ -f vendordiff.patch && -d "${ARTIFACTS:-}" ]]; then - echo "Copying patch to artifacts.." - cp vendordiff.patch "${ARTIFACTS:-}/" - fi - ret=1 - fi -popd > /dev/null 2>&1 - -if [[ ${ret} -gt 0 ]]; then - exit ${ret} -fi - -echo "Godeps Verified." -# ex: ts=2 sw=2 et filetype=sh +echo "Deprecated, use verify-vendor.sh instead" +"${KUBE_ROOT}/hack/verify-vendor.sh" "$@" diff --git a/hack/verify-no-vendor-cycles.sh b/hack/verify-no-vendor-cycles.sh index e245f76954f..84d1a210d9b 100755 --- a/hack/verify-no-vendor-cycles.sh +++ b/hack/verify-no-vendor-cycles.sh @@ -20,6 +20,8 @@ set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +export GO111MODULE=auto + staging_repos=($(ls "${KUBE_ROOT}/staging/src/k8s.io/")) staging_repos_pattern=$(IFS="|"; echo "${staging_repos[*]}") diff --git a/hack/verify-publishing-bot.py b/hack/verify-publishing-bot.py index 7a4c4bdff3d..e457ca259d9 100755 --- a/hack/verify-publishing-bot.py +++ b/hack/verify-publishing-bot.py @@ -23,18 +23,23 @@ import json import yaml -def get_godeps_dependencies(godeps): +def get_godeps_dependencies(rootdir, components): all_dependencies = {} - for arg in godeps: - with open(arg) as f: - data = json.load(f) - dependencies = [] - for dep in data["Deps"]: - if dep['Rev'].startswith('xxxx') and dep['ImportPath'].startswith("k8s.io"): - package = "/".join(dep['ImportPath'].split('/')[0:2]) - if package not in dependencies: - dependencies.append(package.split('/')[1]) - all_dependencies[(data["ImportPath"].split('/')[1])] = dependencies + for component in components: + with open(os.path.join(rootdir, component, "go.mod")) as f: + print(component + " dependencies") + all_dependencies[component] = [] + lines = list(set(f)) + lines.sort() + for line in lines: + for dep in components: + if dep == component: + continue + if ("k8s.io/" + dep + " v0.0.0") not in line: + continue + print("\t"+dep) + if dep not in all_dependencies[component]: + all_dependencies[component].append(dep) return all_dependencies @@ -48,12 +53,12 @@ def main(): rootdir = os.path.dirname(__file__) + "/../" rootdir = os.path.abspath(rootdir) - godeps = [] - for root, dirnames, filenames in os.walk(rootdir + '/staging/'): - for filename in fnmatch.filter(filenames, 'Godeps.json'): - godeps.append(os.path.join(root, filename)) + components = [] + for component in os.listdir(rootdir + '/staging/src/k8s.io/'): + components.append(component) + components.sort() - godep_dependencies = get_godeps_dependencies(godeps) + godep_dependencies = get_godeps_dependencies(rootdir + '/staging/src/k8s.io/', components) rules_dependencies = get_rules_dependencies(rootdir + "/staging/publishing/rules.yaml") processed_repos = [] diff --git a/hack/verify-staging-godeps.sh b/hack/verify-staging-godeps.sh deleted file mode 100755 index 7e40cec2943..00000000000 --- a/hack/verify-staging-godeps.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -source "${KUBE_ROOT}/hack/lib/init.sh" - -readonly branch=${1:-${KUBE_VERIFY_GIT_BRANCH:-master}} -if ! [[ ${KUBE_FORCE_VERIFY_CHECKS:-} =~ ^[yY]$ ]] && \ - ! kube::util::has_changes "${branch}" 'staging/' && \ - ! kube::util::has_changes "${branch}" 'build/' && \ - ! kube::util::has_changes "${branch}" 'Godeps/' && \ - ! kube::util::has_changes "${branch}" 'vendor/' && \ - ! kube::util::has_changes "${branch}" 'hack/lib/' && \ - ! kube::util::has_changes "${branch}" 'hack/.*godep'; then - exit 0 -fi - -KUBE_VERBOSE="${KUBE_VERBOSE:-3}" KUBE_RUN_COPY_OUTPUT=N "${KUBE_ROOT}/hack/update-staging-godeps.sh" -d -f "$@" diff --git a/hack/verify-vendor-licenses.sh b/hack/verify-vendor-licenses.sh new file mode 100755 index 00000000000..8aea7de80df --- /dev/null +++ b/hack/verify-vendor-licenses.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" + +readonly branch=${1:-${KUBE_VERIFY_GIT_BRANCH:-master}} +if ! [[ ${KUBE_FORCE_VERIFY_CHECKS:-} =~ ^[yY]$ ]] && \ + ! kube::util::has_changes "${branch}" 'Godeps/' && \ + ! kube::util::has_changes "${branch}" 'go.mod' && \ + ! kube::util::has_changes "${branch}" 'go.sum' && \ + ! kube::util::has_changes "${branch}" 'vendor/'; then + exit 0 +fi + +# create a nice clean place to put our new licenses +# must be in the user dir (e.g. KUBE_ROOT) in order for the docker volume mount +# to work with docker-machine on macs +mkdir -p "${KUBE_ROOT}/_tmp" +_tmpdir="$(mktemp -d "${KUBE_ROOT}/_tmp/kube-vendor-licenses.XXXXXX")" +#echo "Created workspace: ${_tmpdir}" +function cleanup { + #echo "Removing workspace: ${_tmpdir}" + rm -rf "${_tmpdir}" +} +trap cleanup EXIT + +cp -r "${KUBE_ROOT}/Godeps" "${_tmpdir}/Godeps" +ln -s "${KUBE_ROOT}/LICENSE" "${_tmpdir}" +ln -s "${KUBE_ROOT}/vendor" "${_tmpdir}" +ln -s "${KUBE_ROOT}/staging" "${_tmpdir}" + +# Update vendor Licenses +LICENSE_ROOT="${_tmpdir}" "${KUBE_ROOT}/hack/update-vendor-licenses.sh" + +# Compare vendor Licenses +if ! _out="$(diff -Naupr "${KUBE_ROOT}/Godeps/LICENSES" "${_tmpdir}/Godeps/LICENSES")"; then + echo "Your vendor licenses file is out of date. Run hack/update-vendor-licenses.sh and commit the results." >&2 + echo "${_out}" >&2 + exit 1 +fi diff --git a/hack/verify-vendor.sh b/hack/verify-vendor.sh new file mode 100755 index 00000000000..5595c8b7aa6 --- /dev/null +++ b/hack/verify-vendor.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" + +readonly branch=${1:-${KUBE_VERIFY_GIT_BRANCH:-master}} +if ! [[ ${KUBE_FORCE_VERIFY_CHECKS:-} =~ ^[yY]$ ]] && \ + ! kube::util::has_changes "${branch}" 'Godeps/' && \ + ! kube::util::has_changes "${branch}" 'go.mod' && \ + ! kube::util::has_changes "${branch}" 'go.sum' && \ + ! kube::util::has_changes "${branch}" 'vendor/' && \ + ! kube::util::has_changes "${branch}" 'staging/' && \ + ! kube::util::has_changes "${branch}" 'hack/lib/' && \ + ! kube::util::has_changes "${branch}" 'hack/.*vendor'; then + exit 0 +fi + +if [[ -z ${TMP_GOPATH:-} ]]; then + # Create a nice clean place to put our new vendor + _tmpdir="$(kube::realpath "$(mktemp -d -t verifyvendor.XXXXXX)")" +else + # reuse what we might have saved previously + _tmpdir="${TMP_GOPATH}" +fi + +if [[ -z ${KEEP_TMP:-} ]]; then + KEEP_TMP=false +fi +function cleanup { + # make go module dirs writeable + chmod -R +w "${_tmpdir}" + if [ "${KEEP_TMP}" == "true" ]; then + echo "Leaving ${_tmpdir} for you to examine or copy. Please delete it manually when finished. (rm -rf ${_tmpdir})" + else + echo "Removing ${_tmpdir}" + rm -rf "${_tmpdir}" + fi +} +trap cleanup EXIT + +# Copy the contents of the kube directory into the nice clean place (which is NOT shaped like a GOPATH) +_kubetmp="${_tmpdir}" +mkdir -p "${_kubetmp}" +# should create ${_kubectmp}/kubernetes +git archive --format=tar --prefix=kubernetes/ "$(git write-tree)" | (cd "${_kubetmp}" && tar xf -) +_kubetmp="${_kubetmp}/kubernetes" + +# Do all our work with an unset GOPATH +export GOPATH= + +pushd "${_kubetmp}" > /dev/null 2>&1 + # Destroy deps in the copy of the kube tree + rm -rf ./Godeps/LICENSES ./vendor + + # Recreate the vendor tree using the nice clean set we just downloaded + hack/update-vendor.sh +popd > /dev/null 2>&1 + +ret=0 + +pushd "${KUBE_ROOT}" > /dev/null 2>&1 + # Test for diffs + if ! _out="$(diff -Naupr --ignore-matching-lines='^\s*\"GoVersion\":' go.mod "${_kubetmp}/go.mod")"; then + echo "Your go.mod file is different:" >&2 + echo "${_out}" >&2 + echo "Vendor Verify failed." >&2 + echo "If you're seeing this locally, run the below command to fix your go.mod:" >&2 + echo "hack/update-vendor.sh" >&2 + ret=1 + fi + + if ! _out="$(diff -Naupr -x "BUILD" -x "AUTHORS*" -x "CONTRIBUTORS*" vendor "${_kubetmp}/vendor")"; then + echo "Your vendored results are different:" >&2 + echo "${_out}" >&2 + echo "Vendor Verify failed." >&2 + echo "${_out}" > vendordiff.patch + echo "If you're seeing this locally, run the below command to fix your directories:" >&2 + echo "hack/update-vendor.sh" >&2 + ret=1 + fi +popd > /dev/null 2>&1 + +if [[ ${ret} -gt 0 ]]; then + exit ${ret} +fi + +echo "Vendor Verified." +# ex: ts=2 sw=2 et filetype=sh