From ada932a6e200d33f79f66f632d26d5f8264a98a2 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 1 Mar 2024 17:34:04 -0500 Subject: [PATCH] Simplify update-vendor.sh to use go work sync --- hack/make-rules/update.sh | 1 - hack/update-go-workspace.sh | 41 ----- hack/update-kustomize.sh | 1 - hack/update-vendor.sh | 319 +++++++++++++----------------------- hack/verify-go-workspace.sh | 30 ---- hack/verify-vendor.sh | 4 +- 6 files changed, 118 insertions(+), 278 deletions(-) delete mode 100755 hack/update-go-workspace.sh delete mode 100755 hack/verify-go-workspace.sh diff --git a/hack/make-rules/update.sh b/hack/make-rules/update.sh index 4037cd3b86c..69684b5d785 100755 --- a/hack/make-rules/update.sh +++ b/hack/make-rules/update.sh @@ -36,7 +36,6 @@ if ! ${ALL} ; then fi BASH_TARGETS=( - update-go-workspace update-codegen update-generated-api-compatibility-data update-generated-docs diff --git a/hack/update-go-workspace.sh b/hack/update-go-workspace.sh deleted file mode 100755 index 6dcbbffe999..00000000000 --- a/hack/update-go-workspace.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2022 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script generates go.work so that it includes all Go packages -# in this repo, with a few exceptions. - -set -o errexit -set -o nounset -set -o pipefail - -# Go tools really don't like it if you have a symlink in `pwd`. -cd "$(pwd -P)" - -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -source "${KUBE_ROOT}/hack/lib/init.sh" - -# This sets up the environment, like GOCACHE, which keeps the worktree cleaner. -kube::golang::setup_env - -cd "${KUBE_ROOT}" - -# Ensure all modules are included in go.work -go work edit -use . -git ls-files -z ':(glob)./staging/src/k8s.io/*/go.mod' \ - | xargs -0 -n1 dirname \ - | tr '\n' '\0' \ - | xargs -0 -n1 go work edit -use -go mod download # generate go.work.sum diff --git a/hack/update-kustomize.sh b/hack/update-kustomize.sh index 3f6b3f2ea21..19cdaeeb0bf 100755 --- a/hack/update-kustomize.sh +++ b/hack/update-kustomize.sh @@ -55,7 +55,6 @@ fi ./hack/update-vendor.sh ./hack/update-internal-modules.sh -./hack/update-go-workspace.sh ./hack/lint-dependencies.sh sed -i'' -e "s/const kustomizeVersion.*$/const kustomizeVersion = \"${LATEST_KUSTOMIZE}\"/" staging/src/k8s.io/kubectl/pkg/cmd/version/version.go diff --git a/hack/update-vendor.sh b/hack/update-vendor.sh index fd9f742321e..312eb3c32d0 100755 --- a/hack/update-vendor.sh +++ b/hack/update-vendor.sh @@ -66,44 +66,6 @@ function finish { } trap finish EXIT -# ensure_require_replace_directives_for_all_dependencies: -# - ensures all existing 'require' directives have an associated 'replace' directive pinning a version -# - adds explicit 'require' directives for all transitive dependencies -# - adds explicit 'replace' directives for all require directives (existing 'replace' directives take precedence) -function ensure_require_replace_directives_for_all_dependencies() { - local local_tmp_dir - local_tmp_dir=$(mktemp -d "${TMP_DIR}/pin_replace.XXXX") - - # collect 'require' directives that actually specify a version - local require_filter='(.Version != null) and (.Version != "v0.0.0") and (.Version != "v0.0.0-00010101000000-000000000000")' - # collect 'replace' directives that unconditionally pin versions (old=new@version) - local replace_filter='(.Old.Version == null) and (.New.Version != null)' - - # Capture local require/replace directives before running any go commands that can modify the go.mod file - local require_json="${local_tmp_dir}/require.json" - local replace_json="${local_tmp_dir}/replace.json" - go mod edit -json \ - | jq -r ".Require // [] | sort | .[] | select(${require_filter})" \ - > "${require_json}" - go mod edit -json \ - | jq -r ".Replace // [] | sort | .[] | select(${replace_filter})" \ - > "${replace_json}" - - # Propagate root replace/require directives into staging modules, in case we are downgrading, so they don't bump the root required version back up - for repo in $(kube::util::list_staging_repos); do - ( - cd "staging/src/k8s.io/${repo}" - jq -r '"-require \(.Path)@\(.Version)"' < "${require_json}" \ - | xargs -L 100 go mod edit -fmt - jq -r '"-replace \(.Old.Path)=\(.New.Path)@\(.New.Version)"' < "${replace_json}" \ - | xargs -L 100 go mod edit -fmt - ) - done - - # tidy to ensure require directives are added for indirect dependencies - go mod tidy -} - function print_go_mod_section() { local directive="$1" local file="$2" @@ -180,210 +142,134 @@ function add_generated_comments() { go mod edit -fmt } +function add_staging_replace_directives() { + local path_to_staging_k8s_io="$1" + # Prune + go mod edit -json \ + | jq -r '.Require[]? | select(.Version == "v0.0.0") | "-droprequire \(.Path)"' \ + | xargs -L 100 go mod edit -fmt + go mod edit -json \ + | jq -r '.Replace[]? | select(.New.Path | startswith("'"${path_to_staging_k8s_io}"'")) | "-dropreplace \(.Old.Path)"' \ + | xargs -L 100 go mod edit -fmt + # Re-add + kube::util::list_staging_repos \ + | while read -r X; do echo "-require k8s.io/${X}@v0.0.0"; done \ + | xargs -L 100 go mod edit -fmt + kube::util::list_staging_repos \ + | while read -r X; do echo "-replace k8s.io/${X}=${path_to_staging_k8s_io}/${X}"; done \ + | xargs -L 100 go mod edit -fmt +} -# Phase 1: ensure go.mod files for staging modules and main module +# === Capture go / godebug directives from root go.mod +go_directive_value=$(grep '^go 1.' go.mod | awk '{print $2}' || true) +if [[ -z "${go_directive_value}" ]]; then + kube::log::error "root go.mod must have 'go 1.x.y' directive" >&22 2>&1 + exit 1 +fi +godebug_directive_value=$(grep 'godebug default=go' go.mod | awk '{print $2}' || true) +if [[ -z "${godebug_directive_value}" ]]; then + kube::log::error "root go.mod must have 'godebug default=go1.x' directive" >&22 2>&1 + exit 1 +fi +# === Ensure staging go.mod files exist for repo in $(kube::util::list_staging_repos); do ( cd "staging/src/k8s.io/${repo}" if [[ ! -f go.mod ]]; then kube::log::status "go.mod: initialize ${repo}" >&11 - rm -f Godeps/Godeps.json # remove before initializing, staging Godeps are not authoritative go mod init "k8s.io/${repo}" - go mod edit -fmt fi + go mod edit -go "${go_directive_value}" -godebug "${godebug_directive_value}" ) done -if [[ ! -f go.mod ]]; then - kube::log::status "go.mod: initialize k8s.io/kubernetes" >&11 - go mod init "k8s.io/kubernetes" - rm -f Godeps/Godeps.json # remove after initializing -fi - - -# Phase 2: ensure staging repo require/replace directives - -kube::log::status "go.mod: update staging references" >&11 -# Prune -go mod edit -json \ - | jq -r '.Require[]? | select(.Version == "v0.0.0") | "-droprequire \(.Path)"' \ - | xargs -L 100 go mod edit -fmt -go mod edit -json \ - | jq -r '.Replace[]? | select(.New.Path | startswith("./staging/")) | "-dropreplace \(.Old.Path)"' \ - | xargs -L 100 go mod edit -fmt -# Re-add -kube::util::list_staging_repos \ - | while read -r X; do echo "-require k8s.io/${X}@v0.0.0"; done \ - | xargs -L 100 go mod edit -fmt -kube::util::list_staging_repos \ - | while read -r X; do echo "-replace k8s.io/${X}=./staging/src/k8s.io/${X}"; done \ - | xargs -L 100 go mod edit -fmt - - -# Phase 3: capture required (minimum) versions from all modules, and replaced (pinned) versions from the root module - -# pin referenced versions -ensure_require_replace_directives_for_all_dependencies -# resolves/expands references in the root go.mod (if needed) -go mod tidy -# pin expanded versions -ensure_require_replace_directives_for_all_dependencies -# group require/replace directives -group_directives - -# Phase 4: copy root go.mod to staging dirs and rewrite - -kube::log::status "go.mod: propagate to staging modules" >&11 +# === Ensure root and staging go.mod files refer to each other using v0.0.0 and local path replaces +kube::log::status "go.mod: update staging module references" >&11 +add_staging_replace_directives "./staging/src/k8s.io" for repo in $(kube::util::list_staging_repos); do ( cd "staging/src/k8s.io/${repo}" - - echo "=== propagating to ${repo}" - # copy root go.mod, changing module name - sed "s#module k8s.io/kubernetes#module k8s.io/${repo}#" \ - < "${KUBE_ROOT}/go.mod" \ - > "${KUBE_ROOT}/staging/src/k8s.io/${repo}/go.mod" - # remove `require` directives for staging components (will get re-added as needed by `go list`) - kube::util::list_staging_repos \ - | while read -r X; do echo "-droprequire k8s.io/${X}"; done \ - | xargs -L 100 go mod edit - # rewrite `replace` directives for staging components to point to peer directories - kube::util::list_staging_repos \ - | while read -r X; do echo "-replace k8s.io/${X}=../${X}"; done \ - | xargs -L 100 go mod edit + add_staging_replace_directives ".." ) done +# === Ensure all root and staging modules are included in go.work +kube::log::status "go.mod: go work use" >&11 +( + cd "${KUBE_ROOT}" + unset GOWORK + unset GOFLAGS + if [[ ! -f go.work ]]; then + kube::log::status "go.work: initialize" >&11 + go work init + fi + # Prune use directives + go work edit -json \ + | jq -r '.Use[]? | "-dropuse \(.DiskPath)"' \ + | xargs -L 100 go work edit -fmt + # Ensure go and godebug directives + go work edit -go "${go_directive_value}" -godebug "${godebug_directive_value}" + # Re-add use directives + go work use . + for repo in $(kube::util::list_staging_repos); do + go work use "./staging/src/k8s.io/${repo}" + done +) -# Phase 5: sort and tidy staging components - -kube::log::status "go.mod: sorting staging modules" >&11 -# tidy staging repos in reverse dependency order. -# the content of dependencies' go.mod files affects what `go mod tidy` chooses to record in a go.mod file. -tidy_unordered="${TMP_DIR}/tidy_unordered.txt" -kube::util::list_staging_repos \ - | xargs -I {} echo "k8s.io/{}" > "${tidy_unordered}" -rm -f "${TMP_DIR}/tidy_deps.txt" -# SC2094 checks that you do not read and write to the same file in a pipeline. -# We do read from ${tidy_unordered} in the pipeline and mention it within the -# pipeline (but only ready it again) so we disable the lint to assure shellcheck -# that :this-is-fine: -# shellcheck disable=SC2094 -while IFS= read -r repo; do - # record existence of the repo to ensure modules with no peer relationships still get included in the order - echo "${repo} ${repo}" >> "${TMP_DIR}/tidy_deps.txt" +# === Propagate MVS across all root / staging modules (calculated by `go work`) back into root / staging modules +kube::log::status "go.mod: go work sync" >&11 +( + cd "${KUBE_ROOT}" + unset GOWORK + unset GOFLAGS + go work sync +) +# === Tidy +kube::log::status "go.mod: tidy" >&11 +for repo in $(kube::util::list_staging_repos); do ( - cd "${KUBE_ROOT}/staging/src/${repo}" - - # save the original go.mod, since go list doesn't just add missing entries, it also removes specific required versions from it - tmp_go_mod="${TMP_DIR}/tidy_${repo/\//_}_go.mod.original" - tmp_go_deps="${TMP_DIR}/tidy_${repo/\//_}_deps.txt" - cp go.mod "${tmp_go_mod}" - - echo "=== sorting ${repo}" - # 'go list' calculates direct imports and updates go.mod so that go list -m lists our module dependencies - echo "=== computing imports for ${repo}" - go list all - # ignore errors related to importing `package main` packages, but catch - # other errors (https://github.com/golang/go/issues/59186) - errs=() - kube::util::read-array errs < <( - go list -e -tags=tools -json all | jq -r '.Error.Err | select( . != null )' \ - | grep -v "is a program, not an importable package" - ) - if (( "${#errs[@]}" != 0 )); then - for err in "${errs[@]}"; do - echo "${err}" >&2 - done - exit 1 - fi - - # capture module dependencies - go list -m -f '{{if not .Main}}{{.Path}}{{end}}' all > "${tmp_go_deps}" - - # restore the original go.mod file - cp "${tmp_go_mod}" go.mod - - # list all module dependencies - for dep in $(join "${tidy_unordered}" "${tmp_go_deps}"); do - # record the relationship (put dep first, because we want to sort leaves first) - echo "${dep} ${repo}" >> "${TMP_DIR}/tidy_deps.txt" - # switch the required version to an explicit v0.0.0 (rather than an unknown v0.0.0-00010101000000-000000000000) - go mod edit -require "${dep}@v0.0.0" - done - ) -done < "${tidy_unordered}" - -kube::log::status "go.mod: tidying" >&11 -for repo in $(tsort "${TMP_DIR}/tidy_deps.txt"); do - ( - cd "${KUBE_ROOT}/staging/src/${repo}" - echo "=== tidying ${repo}" - - # prune replace directives that pin to the naturally selected version. - # do this before tidying, since tidy removes unused modules that - # don't provide any relevant packages, which forgets which version of the - # unused transitive dependency we had a require directive for, - # and prevents pruning the matching replace directive after tidying. - go list -m -json all | - jq -r 'select(.Replace != null) | - select(.Path == .Replace.Path) | - select(.Version == .Replace.Version) | - "-dropreplace \(.Replace.Path)"' | - xargs -L 100 go mod edit -fmt - + echo "=== tidying k8s.io/${repo}" + cd "staging/src/k8s.io/${repo}" go mod tidy -v + group_directives + ) +done +echo "=== tidying root" +go mod tidy -v +group_directives - # disallow transitive dependencies on k8s.io/kubernetes - loopback_deps=() - kube::util::read-array loopback_deps < <(go list all 2>/dev/null | grep k8s.io/kubernetes/ || true) - if (( "${#loopback_deps[@]}" > 0 )); then - kube::log::error "${#loopback_deps[@]} disallowed ${repo} -> k8s.io/kubernetes dependencies exist via the following imports: $(go mod why "${loopback_deps[@]}")" >&22 2>&1 - exit 1 - fi +# === Prune unused replace directives, format modules +kube::log::status "go.mod: prune" >&11 +for repo in $(kube::util::list_staging_repos); do + ( + echo "=== pruning k8s.io/${repo}" + cd "staging/src/k8s.io/${repo}" - # prune unused pinned replace directives + # drop all unused replace directives comm -23 \ <(go mod edit -json | jq -r '.Replace[] | .Old.Path' | sort) \ <(go list -m -json all | jq -r .Path | sort) | while read -r X; do echo "-dropreplace=${X}"; done | xargs -L 100 go mod edit -fmt - # prune replace directives that pin to the naturally selected version - go list -m -json all | - jq -r 'select(.Replace != null) | - select(.Path == .Replace.Path) | - select(.Version == .Replace.Version) | - "-dropreplace \(.Replace.Path)"' | - xargs -L 100 go mod edit -fmt - - # group require/replace directives group_directives ) done -echo "=== tidying root" -go mod tidy -# prune unused pinned non-local replace directives +echo "=== pruning root" +# drop unused replace directives other than to local paths comm -23 \ <(go mod edit -json | jq -r '.Replace[] | select(.New.Path | startswith("./") | not) | .Old.Path' | sort) \ <(go list -m -json all | jq -r .Path | sort) | while read -r X; do echo "-dropreplace=${X}"; done | xargs -L 100 go mod edit -fmt -# disallow transitive dependencies on k8s.io/kubernetes -loopback_deps=() -kube::util::read-array loopback_deps < <(go mod graph | grep ' k8s.io/kubernetes' || true) -if (( "${#loopback_deps[@]}" > 0 )); then - kube::log::error "${#loopback_deps[@]} disallowed transitive k8s.io/kubernetes dependencies exist via the following imports:" >&22 2>&1 - kube::log::error "${loopback_deps[@]}" >&22 2>&1 - exit 1 -fi +group_directives -# Phase 6: add generated comments to go.mod files +# === Add generated comments to go.mod files kube::log::status "go.mod: adding generated comments" >&11 add_generated_comments " // This is a generated file. Do not edit directly. @@ -399,17 +285,20 @@ for repo in $(kube::util::list_staging_repos); do ) done - -# Phase 7: update internal modules +# === Update internal modules kube::log::status "vendor: updating internal modules" >&11 hack/update-internal-modules.sh -# Phase 8: rebuild vendor directory +# === Rebuild vendor directory ( kube::log::status "vendor: running 'go work vendor'" >&11 unset GOWORK unset GOFLAGS + # rebuild go.work.sum + rm -f go.work.sum + go mod download + # rebuild vendor go work vendor ) @@ -430,4 +319,28 @@ reviewers: - dep-reviewers __EOF__ +# === Disallow transitive dependencies on k8s.io/kubernetes +kube::log::status "go.mod: prevent staging --> k8s.io/kubernetes dep" >&11 +for repo in $(kube::util::list_staging_repos); do + ( + echo "=== checking k8s.io/${repo}" + cd "staging/src/k8s.io/${repo}" + loopback_deps=() + kube::util::read-array loopback_deps < <(go list all 2>/dev/null | grep k8s.io/kubernetes/ || true) + if (( "${#loopback_deps[@]}" > 0 )); then + kube::log::error "${#loopback_deps[@]} disallowed ${repo} -> k8s.io/kubernetes dependencies exist via the following imports: $(go mod why "${loopback_deps[@]}")" >&22 2>&1 + exit 1 + fi + ) +done + +kube::log::status "go.mod: prevent k8s.io/kubernetes --> * --> k8s.io/kubernetes dep" >&11 +loopback_deps=() +kube::util::read-array loopback_deps < <(go mod graph | grep ' k8s.io/kubernetes' || true) +if (( "${#loopback_deps[@]}" > 0 )); then + kube::log::error "${#loopback_deps[@]} disallowed transitive k8s.io/kubernetes dependencies exist via the following imports:" >&22 2>&1 + kube::log::error "${loopback_deps[@]}" >&22 2>&1 + exit 1 +fi + kube::log::status "NOTE: don't forget to handle vendor/* and LICENSE/* files that were added or removed" >&11 diff --git a/hack/verify-go-workspace.sh b/hack/verify-go-workspace.sh deleted file mode 100755 index 334c892ad45..00000000000 --- a/hack/verify-go-workspace.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2021 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script checks whether the OWNERS files need to be formatted or not by -# `yamlfmt`. Run `hack/update-yamlfmt.sh` to actually format sources. -# -# Usage: `hack/verify-go-workspace.sh`. - - -set -o errexit -set -o nounset -set -o pipefail - -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -source "${KUBE_ROOT}/hack/lib/verify-generated.sh" - -kube::verify::generated "Go workspace files need to be updated" "Please run 'hack/update-go-workspace.sh'" hack/update-go-workspace.sh diff --git a/hack/verify-vendor.sh b/hack/verify-vendor.sh index dbdefedae39..c68fbbbc840 100755 --- a/hack/verify-vendor.sh +++ b/hack/verify-vendor.sh @@ -62,8 +62,8 @@ popd > /dev/null 2>&1 ret=0 pushd "${KUBE_ROOT}" > /dev/null 2>&1 - # Test for diffs in go.mod / go.sum / go.work - for file in $(git ls-files -cmo --exclude-standard -- ':!:vendor/*' ':(glob)**/go.mod' ':(glob)**/go.sum' ':(glob)**/go.work'); do + # Test for diffs in go.mod / go.sum / go.work / go.work.sum + for file in $(git ls-files -cmo --exclude-standard -- ':!:vendor/*' ':(glob)**/go.mod' ':(glob)**/go.sum' ':(glob)**/go.work' ':(glob)**/go.work.sum'); do if ! _out="$(diff -Naupr "${file}" "${_kubetmp}/${file}")"; then echo "Your ${file} file is different:" >&2 echo "${_out}" >&2