Simplify update-vendor.sh to use go work sync

This commit is contained in:
Jordan Liggitt 2024-03-01 17:34:04 -05:00
parent daf76e6ead
commit ada932a6e2
No known key found for this signature in database
6 changed files with 118 additions and 278 deletions

View File

@ -36,7 +36,6 @@ if ! ${ALL} ; then
fi
BASH_TARGETS=(
update-go-workspace
update-codegen
update-generated-api-compatibility-data
update-generated-docs

View File

@ -1,41 +0,0 @@
#!/usr/bin/env bash
# Copyright 2022 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script generates go.work so that it includes all Go packages
# in this repo, with a few exceptions.
set -o errexit
set -o nounset
set -o pipefail
# Go tools really don't like it if you have a symlink in `pwd`.
cd "$(pwd -P)"
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
# This sets up the environment, like GOCACHE, which keeps the worktree cleaner.
kube::golang::setup_env
cd "${KUBE_ROOT}"
# Ensure all modules are included in go.work
go work edit -use .
git ls-files -z ':(glob)./staging/src/k8s.io/*/go.mod' \
| xargs -0 -n1 dirname \
| tr '\n' '\0' \
| xargs -0 -n1 go work edit -use
go mod download # generate go.work.sum

View File

@ -55,7 +55,6 @@ fi
./hack/update-vendor.sh
./hack/update-internal-modules.sh
./hack/update-go-workspace.sh
./hack/lint-dependencies.sh
sed -i'' -e "s/const kustomizeVersion.*$/const kustomizeVersion = \"${LATEST_KUSTOMIZE}\"/" staging/src/k8s.io/kubectl/pkg/cmd/version/version.go

View File

@ -66,44 +66,6 @@ function finish {
}
trap finish EXIT
# ensure_require_replace_directives_for_all_dependencies:
# - ensures all existing 'require' directives have an associated 'replace' directive pinning a version
# - adds explicit 'require' directives for all transitive dependencies
# - adds explicit 'replace' directives for all require directives (existing 'replace' directives take precedence)
function ensure_require_replace_directives_for_all_dependencies() {
local local_tmp_dir
local_tmp_dir=$(mktemp -d "${TMP_DIR}/pin_replace.XXXX")
# collect 'require' directives that actually specify a version
local require_filter='(.Version != null) and (.Version != "v0.0.0") and (.Version != "v0.0.0-00010101000000-000000000000")'
# collect 'replace' directives that unconditionally pin versions (old=new@version)
local replace_filter='(.Old.Version == null) and (.New.Version != null)'
# Capture local require/replace directives before running any go commands that can modify the go.mod file
local require_json="${local_tmp_dir}/require.json"
local replace_json="${local_tmp_dir}/replace.json"
go mod edit -json \
| jq -r ".Require // [] | sort | .[] | select(${require_filter})" \
> "${require_json}"
go mod edit -json \
| jq -r ".Replace // [] | sort | .[] | select(${replace_filter})" \
> "${replace_json}"
# Propagate root replace/require directives into staging modules, in case we are downgrading, so they don't bump the root required version back up
for repo in $(kube::util::list_staging_repos); do
(
cd "staging/src/k8s.io/${repo}"
jq -r '"-require \(.Path)@\(.Version)"' < "${require_json}" \
| xargs -L 100 go mod edit -fmt
jq -r '"-replace \(.Old.Path)=\(.New.Path)@\(.New.Version)"' < "${replace_json}" \
| xargs -L 100 go mod edit -fmt
)
done
# tidy to ensure require directives are added for indirect dependencies
go mod tidy
}
function print_go_mod_section() {
local directive="$1"
local file="$2"
@ -180,210 +142,134 @@ function add_generated_comments() {
go mod edit -fmt
}
function add_staging_replace_directives() {
local path_to_staging_k8s_io="$1"
# Prune
go mod edit -json \
| jq -r '.Require[]? | select(.Version == "v0.0.0") | "-droprequire \(.Path)"' \
| xargs -L 100 go mod edit -fmt
go mod edit -json \
| jq -r '.Replace[]? | select(.New.Path | startswith("'"${path_to_staging_k8s_io}"'")) | "-dropreplace \(.Old.Path)"' \
| xargs -L 100 go mod edit -fmt
# Re-add
kube::util::list_staging_repos \
| while read -r X; do echo "-require k8s.io/${X}@v0.0.0"; done \
| xargs -L 100 go mod edit -fmt
kube::util::list_staging_repos \
| while read -r X; do echo "-replace k8s.io/${X}=${path_to_staging_k8s_io}/${X}"; done \
| xargs -L 100 go mod edit -fmt
}
# Phase 1: ensure go.mod files for staging modules and main module
# === Capture go / godebug directives from root go.mod
go_directive_value=$(grep '^go 1.' go.mod | awk '{print $2}' || true)
if [[ -z "${go_directive_value}" ]]; then
kube::log::error "root go.mod must have 'go 1.x.y' directive" >&22 2>&1
exit 1
fi
godebug_directive_value=$(grep 'godebug default=go' go.mod | awk '{print $2}' || true)
if [[ -z "${godebug_directive_value}" ]]; then
kube::log::error "root go.mod must have 'godebug default=go1.x' directive" >&22 2>&1
exit 1
fi
# === Ensure staging go.mod files exist
for repo in $(kube::util::list_staging_repos); do
(
cd "staging/src/k8s.io/${repo}"
if [[ ! -f go.mod ]]; then
kube::log::status "go.mod: initialize ${repo}" >&11
rm -f Godeps/Godeps.json # remove before initializing, staging Godeps are not authoritative
go mod init "k8s.io/${repo}"
go mod edit -fmt
fi
go mod edit -go "${go_directive_value}" -godebug "${godebug_directive_value}"
)
done
if [[ ! -f go.mod ]]; then
kube::log::status "go.mod: initialize k8s.io/kubernetes" >&11
go mod init "k8s.io/kubernetes"
rm -f Godeps/Godeps.json # remove after initializing
fi
# Phase 2: ensure staging repo require/replace directives
kube::log::status "go.mod: update staging references" >&11
# Prune
go mod edit -json \
| jq -r '.Require[]? | select(.Version == "v0.0.0") | "-droprequire \(.Path)"' \
| xargs -L 100 go mod edit -fmt
go mod edit -json \
| jq -r '.Replace[]? | select(.New.Path | startswith("./staging/")) | "-dropreplace \(.Old.Path)"' \
| xargs -L 100 go mod edit -fmt
# Re-add
kube::util::list_staging_repos \
| while read -r X; do echo "-require k8s.io/${X}@v0.0.0"; done \
| xargs -L 100 go mod edit -fmt
kube::util::list_staging_repos \
| while read -r X; do echo "-replace k8s.io/${X}=./staging/src/k8s.io/${X}"; done \
| xargs -L 100 go mod edit -fmt
# Phase 3: capture required (minimum) versions from all modules, and replaced (pinned) versions from the root module
# pin referenced versions
ensure_require_replace_directives_for_all_dependencies
# resolves/expands references in the root go.mod (if needed)
go mod tidy
# pin expanded versions
ensure_require_replace_directives_for_all_dependencies
# group require/replace directives
group_directives
# Phase 4: copy root go.mod to staging dirs and rewrite
kube::log::status "go.mod: propagate to staging modules" >&11
# === Ensure root and staging go.mod files refer to each other using v0.0.0 and local path replaces
kube::log::status "go.mod: update staging module references" >&11
add_staging_replace_directives "./staging/src/k8s.io"
for repo in $(kube::util::list_staging_repos); do
(
cd "staging/src/k8s.io/${repo}"
echo "=== propagating to ${repo}"
# copy root go.mod, changing module name
sed "s#module k8s.io/kubernetes#module k8s.io/${repo}#" \
< "${KUBE_ROOT}/go.mod" \
> "${KUBE_ROOT}/staging/src/k8s.io/${repo}/go.mod"
# remove `require` directives for staging components (will get re-added as needed by `go list`)
kube::util::list_staging_repos \
| while read -r X; do echo "-droprequire k8s.io/${X}"; done \
| xargs -L 100 go mod edit
# rewrite `replace` directives for staging components to point to peer directories
kube::util::list_staging_repos \
| while read -r X; do echo "-replace k8s.io/${X}=../${X}"; done \
| xargs -L 100 go mod edit
add_staging_replace_directives ".."
)
done
# === Ensure all root and staging modules are included in go.work
kube::log::status "go.mod: go work use" >&11
(
cd "${KUBE_ROOT}"
unset GOWORK
unset GOFLAGS
if [[ ! -f go.work ]]; then
kube::log::status "go.work: initialize" >&11
go work init
fi
# Prune use directives
go work edit -json \
| jq -r '.Use[]? | "-dropuse \(.DiskPath)"' \
| xargs -L 100 go work edit -fmt
# Ensure go and godebug directives
go work edit -go "${go_directive_value}" -godebug "${godebug_directive_value}"
# Re-add use directives
go work use .
for repo in $(kube::util::list_staging_repos); do
go work use "./staging/src/k8s.io/${repo}"
done
)
# Phase 5: sort and tidy staging components
kube::log::status "go.mod: sorting staging modules" >&11
# tidy staging repos in reverse dependency order.
# the content of dependencies' go.mod files affects what `go mod tidy` chooses to record in a go.mod file.
tidy_unordered="${TMP_DIR}/tidy_unordered.txt"
kube::util::list_staging_repos \
| xargs -I {} echo "k8s.io/{}" > "${tidy_unordered}"
rm -f "${TMP_DIR}/tidy_deps.txt"
# SC2094 checks that you do not read and write to the same file in a pipeline.
# We do read from ${tidy_unordered} in the pipeline and mention it within the
# pipeline (but only ready it again) so we disable the lint to assure shellcheck
# that :this-is-fine:
# shellcheck disable=SC2094
while IFS= read -r repo; do
# record existence of the repo to ensure modules with no peer relationships still get included in the order
echo "${repo} ${repo}" >> "${TMP_DIR}/tidy_deps.txt"
# === Propagate MVS across all root / staging modules (calculated by `go work`) back into root / staging modules
kube::log::status "go.mod: go work sync" >&11
(
cd "${KUBE_ROOT}"
unset GOWORK
unset GOFLAGS
go work sync
)
# === Tidy
kube::log::status "go.mod: tidy" >&11
for repo in $(kube::util::list_staging_repos); do
(
cd "${KUBE_ROOT}/staging/src/${repo}"
# save the original go.mod, since go list doesn't just add missing entries, it also removes specific required versions from it
tmp_go_mod="${TMP_DIR}/tidy_${repo/\//_}_go.mod.original"
tmp_go_deps="${TMP_DIR}/tidy_${repo/\//_}_deps.txt"
cp go.mod "${tmp_go_mod}"
echo "=== sorting ${repo}"
# 'go list' calculates direct imports and updates go.mod so that go list -m lists our module dependencies
echo "=== computing imports for ${repo}"
go list all
# ignore errors related to importing `package main` packages, but catch
# other errors (https://github.com/golang/go/issues/59186)
errs=()
kube::util::read-array errs < <(
go list -e -tags=tools -json all | jq -r '.Error.Err | select( . != null )' \
| grep -v "is a program, not an importable package"
)
if (( "${#errs[@]}" != 0 )); then
for err in "${errs[@]}"; do
echo "${err}" >&2
done
exit 1
fi
# capture module dependencies
go list -m -f '{{if not .Main}}{{.Path}}{{end}}' all > "${tmp_go_deps}"
# restore the original go.mod file
cp "${tmp_go_mod}" go.mod
# list all module dependencies
for dep in $(join "${tidy_unordered}" "${tmp_go_deps}"); do
# record the relationship (put dep first, because we want to sort leaves first)
echo "${dep} ${repo}" >> "${TMP_DIR}/tidy_deps.txt"
# switch the required version to an explicit v0.0.0 (rather than an unknown v0.0.0-00010101000000-000000000000)
go mod edit -require "${dep}@v0.0.0"
done
)
done < "${tidy_unordered}"
kube::log::status "go.mod: tidying" >&11
for repo in $(tsort "${TMP_DIR}/tidy_deps.txt"); do
(
cd "${KUBE_ROOT}/staging/src/${repo}"
echo "=== tidying ${repo}"
# prune replace directives that pin to the naturally selected version.
# do this before tidying, since tidy removes unused modules that
# don't provide any relevant packages, which forgets which version of the
# unused transitive dependency we had a require directive for,
# and prevents pruning the matching replace directive after tidying.
go list -m -json all |
jq -r 'select(.Replace != null) |
select(.Path == .Replace.Path) |
select(.Version == .Replace.Version) |
"-dropreplace \(.Replace.Path)"' |
xargs -L 100 go mod edit -fmt
echo "=== tidying k8s.io/${repo}"
cd "staging/src/k8s.io/${repo}"
go mod tidy -v
group_directives
)
done
echo "=== tidying root"
go mod tidy -v
group_directives
# disallow transitive dependencies on k8s.io/kubernetes
loopback_deps=()
kube::util::read-array loopback_deps < <(go list all 2>/dev/null | grep k8s.io/kubernetes/ || true)
if (( "${#loopback_deps[@]}" > 0 )); then
kube::log::error "${#loopback_deps[@]} disallowed ${repo} -> k8s.io/kubernetes dependencies exist via the following imports: $(go mod why "${loopback_deps[@]}")" >&22 2>&1
exit 1
fi
# === Prune unused replace directives, format modules
kube::log::status "go.mod: prune" >&11
for repo in $(kube::util::list_staging_repos); do
(
echo "=== pruning k8s.io/${repo}"
cd "staging/src/k8s.io/${repo}"
# prune unused pinned replace directives
# drop all unused replace directives
comm -23 \
<(go mod edit -json | jq -r '.Replace[] | .Old.Path' | sort) \
<(go list -m -json all | jq -r .Path | sort) |
while read -r X; do echo "-dropreplace=${X}"; done |
xargs -L 100 go mod edit -fmt
# prune replace directives that pin to the naturally selected version
go list -m -json all |
jq -r 'select(.Replace != null) |
select(.Path == .Replace.Path) |
select(.Version == .Replace.Version) |
"-dropreplace \(.Replace.Path)"' |
xargs -L 100 go mod edit -fmt
# group require/replace directives
group_directives
)
done
echo "=== tidying root"
go mod tidy
# prune unused pinned non-local replace directives
echo "=== pruning root"
# drop unused replace directives other than to local paths
comm -23 \
<(go mod edit -json | jq -r '.Replace[] | select(.New.Path | startswith("./") | not) | .Old.Path' | sort) \
<(go list -m -json all | jq -r .Path | sort) |
while read -r X; do echo "-dropreplace=${X}"; done |
xargs -L 100 go mod edit -fmt
# disallow transitive dependencies on k8s.io/kubernetes
loopback_deps=()
kube::util::read-array loopback_deps < <(go mod graph | grep ' k8s.io/kubernetes' || true)
if (( "${#loopback_deps[@]}" > 0 )); then
kube::log::error "${#loopback_deps[@]} disallowed transitive k8s.io/kubernetes dependencies exist via the following imports:" >&22 2>&1
kube::log::error "${loopback_deps[@]}" >&22 2>&1
exit 1
fi
group_directives
# Phase 6: add generated comments to go.mod files
# === Add generated comments to go.mod files
kube::log::status "go.mod: adding generated comments" >&11
add_generated_comments "
// This is a generated file. Do not edit directly.
@ -399,17 +285,20 @@ for repo in $(kube::util::list_staging_repos); do
)
done
# Phase 7: update internal modules
# === Update internal modules
kube::log::status "vendor: updating internal modules" >&11
hack/update-internal-modules.sh
# Phase 8: rebuild vendor directory
# === Rebuild vendor directory
(
kube::log::status "vendor: running 'go work vendor'" >&11
unset GOWORK
unset GOFLAGS
# rebuild go.work.sum
rm -f go.work.sum
go mod download
# rebuild vendor
go work vendor
)
@ -430,4 +319,28 @@ reviewers:
- dep-reviewers
__EOF__
# === Disallow transitive dependencies on k8s.io/kubernetes
kube::log::status "go.mod: prevent staging --> k8s.io/kubernetes dep" >&11
for repo in $(kube::util::list_staging_repos); do
(
echo "=== checking k8s.io/${repo}"
cd "staging/src/k8s.io/${repo}"
loopback_deps=()
kube::util::read-array loopback_deps < <(go list all 2>/dev/null | grep k8s.io/kubernetes/ || true)
if (( "${#loopback_deps[@]}" > 0 )); then
kube::log::error "${#loopback_deps[@]} disallowed ${repo} -> k8s.io/kubernetes dependencies exist via the following imports: $(go mod why "${loopback_deps[@]}")" >&22 2>&1
exit 1
fi
)
done
kube::log::status "go.mod: prevent k8s.io/kubernetes --> * --> k8s.io/kubernetes dep" >&11
loopback_deps=()
kube::util::read-array loopback_deps < <(go mod graph | grep ' k8s.io/kubernetes' || true)
if (( "${#loopback_deps[@]}" > 0 )); then
kube::log::error "${#loopback_deps[@]} disallowed transitive k8s.io/kubernetes dependencies exist via the following imports:" >&22 2>&1
kube::log::error "${loopback_deps[@]}" >&22 2>&1
exit 1
fi
kube::log::status "NOTE: don't forget to handle vendor/* and LICENSE/* files that were added or removed" >&11

View File

@ -1,30 +0,0 @@
#!/usr/bin/env bash
# Copyright 2021 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script checks whether the OWNERS files need to be formatted or not by
# `yamlfmt`. Run `hack/update-yamlfmt.sh` to actually format sources.
#
# Usage: `hack/verify-go-workspace.sh`.
set -o errexit
set -o nounset
set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/verify-generated.sh"
kube::verify::generated "Go workspace files need to be updated" "Please run 'hack/update-go-workspace.sh'" hack/update-go-workspace.sh

View File

@ -62,8 +62,8 @@ popd > /dev/null 2>&1
ret=0
pushd "${KUBE_ROOT}" > /dev/null 2>&1
# Test for diffs in go.mod / go.sum / go.work
for file in $(git ls-files -cmo --exclude-standard -- ':!:vendor/*' ':(glob)**/go.mod' ':(glob)**/go.sum' ':(glob)**/go.work'); do
# Test for diffs in go.mod / go.sum / go.work / go.work.sum
for file in $(git ls-files -cmo --exclude-standard -- ':!:vendor/*' ':(glob)**/go.mod' ':(glob)**/go.sum' ':(glob)**/go.work' ':(glob)**/go.work.sum'); do
if ! _out="$(diff -Naupr "${file}" "${_kubetmp}/${file}")"; then
echo "Your ${file} file is different:" >&2
echo "${_out}" >&2