Merge pull request #79316 from BenTheElder/shellll

fix vendor scripts shellcheck failures
This commit is contained in:
Kubernetes Prow Robot 2019-06-25 01:15:16 -07:00 committed by GitHub
commit dac0bd9029
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 84 additions and 52 deletions

View File

@ -22,10 +22,7 @@
./hack/lib/test.sh ./hack/lib/test.sh
./hack/lib/version.sh ./hack/lib/version.sh
./hack/make-rules/make-help.sh ./hack/make-rules/make-help.sh
./hack/pin-dependency.sh
./hack/test-integration.sh ./hack/test-integration.sh
./hack/update-vendor.sh
./hack/verify-no-vendor-cycles.sh
./hack/verify-test-featuregates.sh ./hack/verify-test-featuregates.sh
./test/cmd/diff.sh ./test/cmd/diff.sh
./test/cmd/discovery.sh ./test/cmd/discovery.sh

View File

@ -171,6 +171,8 @@ kube::util::host_platform() {
echo "$(kube::util::host_os)/$(kube::util::host_arch)" echo "$(kube::util::host_os)/$(kube::util::host_arch)"
} }
# looks for $1 in well-known output locations for the platform ($2)
# $KUBE_ROOT must be set
kube::util::find-binary-for-platform() { kube::util::find-binary-for-platform() {
local -r lookfor="$1" local -r lookfor="$1"
local -r platform="$2" local -r platform="$2"
@ -194,6 +196,8 @@ kube::util::find-binary-for-platform() {
echo -n "${bin}" echo -n "${bin}"
} }
# looks for $1 in well-known output locations for the host platform
# $KUBE_ROOT must be set
kube::util::find-binary() { kube::util::find-binary() {
kube::util::find-binary-for-platform "$1" "$(kube::util::host_platform)" kube::util::find-binary-for-platform "$1" "$(kube::util::host_platform)"
} }
@ -265,6 +269,8 @@ kube::util::remove-gen-docs() {
# * Special handling for empty group: v1 -> api/v1, unversioned -> api/unversioned # * Special handling for empty group: v1 -> api/v1, unversioned -> api/unversioned
# * Special handling for groups suffixed with ".k8s.io": foo.k8s.io/v1 -> apis/foo/v1 # * Special handling for groups suffixed with ".k8s.io": foo.k8s.io/v1 -> apis/foo/v1
# * Very special handling for when both group and version are "": / -> api # * Very special handling for when both group and version are "": / -> api
#
# $KUBE_ROOT must be set.
kube::util::group-version-to-pkg-path() { kube::util::group-version-to-pkg-path() {
local group_version="$1" local group_version="$1"
@ -531,6 +537,17 @@ EOF
EOF EOF
} }
# list_staging_repos outputs a sorted list of repos in staging/src/k8s.io
# each entry will just be the $repo portion of staging/src/k8s.io/$repo/...
# $KUBE_ROOT must be set.
function kube::util::list_staging_repos() {
(
cd "${KUBE_ROOT}/staging/src/k8s.io" && \
find . -mindepth 1 -maxdepth 1 -type d | cut -c 3- | sort
)
}
# Determines if docker can be run, failures may simply require that the user be added to the docker group. # Determines if docker can be run, failures may simply require that the user be added to the docker group.
function kube::util::ensure_docker_daemon_connectivity { function kube::util::ensure_docker_daemon_connectivity {
IFS=" " read -ra DOCKER <<< "${DOCKER_OPTS}" IFS=" " read -ra DOCKER <<< "${DOCKER_OPTS}"

View File

@ -90,7 +90,7 @@ echo "Running: go mod edit -replace ${dep}=${dep}@${rev}"
go mod edit -replace "${dep}=${dep}@${rev}" go mod edit -replace "${dep}=${dep}@${rev}"
# Propagate pinned version to staging repos that also have that dependency # Propagate pinned version to staging repos that also have that dependency
for repo in $(ls staging/src/k8s.io | sort); do for repo in $(kube::util::list_staging_repos); do
pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1 pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1
if go mod edit -json | jq -e -r ".Require[] | select(.Path == \"${dep}\")" > /dev/null 2>&1; then if go mod edit -json | jq -e -r ".Require[] | select(.Path == \"${dep}\")" > /dev/null 2>&1; then
go mod edit -require "${dep}@${rev}" go mod edit -require "${dep}@${rev}"

View File

@ -67,17 +67,17 @@ function ensure_require_replace_directives_for_all_dependencies() {
go mod edit -json | jq -r ".Replace // [] | sort | .[] | select(${replace_filter})" > "${replace_json}" go mod edit -json | jq -r ".Replace // [] | sort | .[] | select(${replace_filter})" > "${replace_json}"
# 1a. Ensure replace directives have an explicit require directive # 1a. Ensure replace directives have an explicit require directive
cat "${replace_json}" | jq -r '"-require \(.Old.Path)@\(.New.Version)"' | xargs -L 100 go mod edit -fmt jq -r '"-require \(.Old.Path)@\(.New.Version)"' < "${replace_json}" | xargs -L 100 go mod edit -fmt
# 1b. Ensure require directives have a corresponding replace directive pinning a version # 1b. Ensure require directives have a corresponding replace directive pinning a version
cat "${require_json}" | jq -r '"-replace \(.Path)=\(.Path)@\(.Version)"' | xargs -L 100 go mod edit -fmt jq -r '"-replace \(.Path)=\(.Path)@\(.Version)"' < "${require_json}" | xargs -L 100 go mod edit -fmt
cat "${replace_json}" | jq -r '"-replace \(.Old.Path)=\(.New.Path)@\(.New.Version)"'| xargs -L 100 go mod edit -fmt jq -r '"-replace \(.Old.Path)=\(.New.Path)@\(.New.Version)"' < "${replace_json}" | xargs -L 100 go mod edit -fmt
# 2. Propagate root replace/require directives into staging modules, in case we are downgrading, so they don't bump the root required version back up # 2. Propagate root replace/require directives into staging modules, in case we are downgrading, so they don't bump the root required version back up
for repo in $(ls staging/src/k8s.io); do for repo in $(kube::util::list_staging_repos); do
pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1 pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1
cat "${require_json}" | jq -r '"-require \(.Path)@\(.Version)"' | xargs -L 100 go mod edit -fmt jq -r '"-require \(.Path)@\(.Version)"' < "${require_json}" | xargs -L 100 go mod edit -fmt
cat "${require_json}" | jq -r '"-replace \(.Path)=\(.Path)@\(.Version)"' | xargs -L 100 go mod edit -fmt jq -r '"-replace \(.Path)=\(.Path)@\(.Version)"' < "${require_json}" | xargs -L 100 go mod edit -fmt
cat "${replace_json}" | jq -r '"-replace \(.Old.Path)=\(.New.Path)@\(.New.Version)"'| xargs -L 100 go mod edit -fmt jq -r '"-replace \(.Old.Path)=\(.New.Path)@\(.New.Version)"' < "${replace_json}" | xargs -L 100 go mod edit -fmt
popd >/dev/null 2>&1 popd >/dev/null 2>&1
done done
@ -94,7 +94,7 @@ function group_replace_directives() {
local go_mod_replace="${local_tmp_dir}/go.mod.replace.tmp" local go_mod_replace="${local_tmp_dir}/go.mod.replace.tmp"
local go_mod_noreplace="${local_tmp_dir}/go.mod.noreplace.tmp" local go_mod_noreplace="${local_tmp_dir}/go.mod.noreplace.tmp"
# separate replace and non-replace directives # separate replace and non-replace directives
cat go.mod | awk " awk "
# print lines between 'replace (' ... ')' lines # print lines between 'replace (' ... ')' lines
/^replace [(]/ { inreplace=1; next } /^replace [(]/ { inreplace=1; next }
inreplace && /^[)]/ { inreplace=0; next } inreplace && /^[)]/ { inreplace=0; next }
@ -105,11 +105,13 @@ function group_replace_directives() {
# otherwise print to the noreplace file # otherwise print to the noreplace file
{ print > \"${go_mod_noreplace}\" } { print > \"${go_mod_noreplace}\" }
" " < go.mod
cat "${go_mod_noreplace}" > go.mod {
echo "replace (" >> go.mod cat "${go_mod_noreplace}";
cat "${go_mod_replace}" >> go.mod echo "replace (";
echo ")" >> go.mod cat "${go_mod_replace}";
echo ")";
} > go.mod
go mod edit -fmt go mod edit -fmt
} }
@ -120,18 +122,20 @@ function add_generated_comments() {
local go_mod_nocomments="${local_tmp_dir}/go.mod.nocomments.tmp" local go_mod_nocomments="${local_tmp_dir}/go.mod.nocomments.tmp"
# drop comments before the module directive # drop comments before the module directive
cat go.mod | awk " awk "
BEGIN { dropcomments=1 } BEGIN { dropcomments=1 }
/^module / { dropcomments=0 } /^module / { dropcomments=0 }
dropcomments && /^\/\// { next } dropcomments && /^\/\// { next }
{ print } { print }
" > "${go_mod_nocomments}" " < go.mod > "${go_mod_nocomments}"
# Add the specified comments # Add the specified comments
local comments="${1}" local comments="${1}"
echo "${comments}" > go.mod {
echo "" >> go.mod echo "${comments}"
cat "${go_mod_nocomments}" >> go.mod echo ""
cat "${go_mod_nocomments}"
} > go.mod
# Format # Format
go mod edit -fmt go mod edit -fmt
@ -140,7 +144,7 @@ function add_generated_comments() {
# Phase 1: ensure go.mod files for staging modules and main module # Phase 1: ensure go.mod files for staging modules and main module
for repo in $(ls staging/src/k8s.io); do for repo in $(kube::util::list_staging_repos); do
pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1 pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1
if [[ ! -f go.mod ]]; then if [[ ! -f go.mod ]]; then
kube::log::status "go.mod: initialize ${repo}" kube::log::status "go.mod: initialize ${repo}"
@ -165,8 +169,8 @@ kube::log::status "go.mod: update staging references"
go mod edit -json | jq -r '.Require[]? | select(.Version == "v0.0.0") | "-droprequire \(.Path)"' | xargs -L 100 go mod edit -fmt go mod edit -json | jq -r '.Require[]? | select(.Version == "v0.0.0") | "-droprequire \(.Path)"' | xargs -L 100 go mod edit -fmt
go mod edit -json | jq -r '.Replace[]? | select(.New.Path | startswith("./staging/")) | "-dropreplace \(.Old.Path)"' | xargs -L 100 go mod edit -fmt go mod edit -json | jq -r '.Replace[]? | select(.New.Path | startswith("./staging/")) | "-dropreplace \(.Old.Path)"' | xargs -L 100 go mod edit -fmt
# Readd # Readd
ls staging/src/k8s.io | sort | xargs -n 1 -I {} echo "-require k8s.io/{}@v0.0.0" | xargs -L 100 go mod edit -fmt kube::util::list_staging_repos | xargs -n 1 -I {} echo "-require k8s.io/{}@v0.0.0" | xargs -L 100 go mod edit -fmt
ls staging/src/k8s.io | sort | xargs -n 1 -I {} echo "-replace k8s.io/{}=./staging/src/k8s.io/{}" | xargs -L 100 go mod edit -fmt kube::util::list_staging_repos | xargs -n 1 -I {} echo "-replace k8s.io/{}=./staging/src/k8s.io/{}" | xargs -L 100 go mod edit -fmt
# Phase 3: capture required (minimum) versions from all modules, and replaced (pinned) versions from the root module # Phase 3: capture required (minimum) versions from all modules, and replaced (pinned) versions from the root module
@ -183,15 +187,15 @@ group_replace_directives
# Phase 4: copy root go.mod to staging dirs and rewrite # Phase 4: copy root go.mod to staging dirs and rewrite
kube::log::status "go.mod: propagate to staging modules" kube::log::status "go.mod: propagate to staging modules"
for repo in $(ls staging/src/k8s.io | sort); do for repo in $(kube::util::list_staging_repos); do
pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1 pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1
echo "=== propagating to ${repo}" >> "${LOG_FILE}" echo "=== propagating to ${repo}" >> "${LOG_FILE}"
# copy root go.mod, changing module name # copy root go.mod, changing module name
cat "${KUBE_ROOT}/go.mod" | sed "s#module k8s.io/kubernetes#module k8s.io/${repo}#" > "${KUBE_ROOT}/staging/src/k8s.io/${repo}/go.mod" sed "s#module k8s.io/kubernetes#module k8s.io/${repo}#" < "${KUBE_ROOT}/go.mod" > "${KUBE_ROOT}/staging/src/k8s.io/${repo}/go.mod"
# remove `require` directives for staging components (will get re-added as needed by `go list`) # remove `require` directives for staging components (will get re-added as needed by `go list`)
ls "${KUBE_ROOT}/staging/src/k8s.io" | sort | xargs -n 1 -I {} echo "-droprequire k8s.io/{}" | xargs -L 100 go mod edit kube::util::list_staging_repos | xargs -n 1 -I {} echo "-droprequire k8s.io/{}" | xargs -L 100 go mod edit
# rewrite `replace` directives for staging components to point to peer directories # rewrite `replace` directives for staging components to point to peer directories
ls "${KUBE_ROOT}/staging/src/k8s.io" | sort | xargs -n 1 -I {} echo "-replace k8s.io/{}=../{}" | xargs -L 100 go mod edit kube::util::list_staging_repos | xargs -n 1 -I {} echo "-replace k8s.io/{}=../{}" | xargs -L 100 go mod edit
popd >/dev/null 2>&1 popd >/dev/null 2>&1
done done
@ -201,9 +205,15 @@ done
kube::log::status "go.mod: sorting staging modules" kube::log::status "go.mod: sorting staging modules"
# tidy staging repos in reverse dependency order. # tidy staging repos in reverse dependency order.
# the content of dependencies' go.mod files affects what `go mod tidy` chooses to record in a go.mod file. # the content of dependencies' go.mod files affects what `go mod tidy` chooses to record in a go.mod file.
ls staging/src/k8s.io | sort | xargs -I {} echo "k8s.io/{}" > "${TMP_DIR}/tidy_unordered.txt" tidy_unordered="${TMP_DIR}/tidy_unordered.txt"
kube::util::list_staging_repos | xargs -I {} echo "k8s.io/{}" > "${tidy_unordered}"
rm -f "${TMP_DIR}/tidy_deps.txt" rm -f "${TMP_DIR}/tidy_deps.txt"
for repo in $(cat "${TMP_DIR}/tidy_unordered.txt"); do # SC2094 checks that you do not read and write to the same file in a pipeline.
# We do read from ${tidy_unordered} in the pipeline and mention it within the
# pipeline (but only ready it again) so we disable the lint to assure shellcheck
# that :this-is-fine:
# shellcheck disable=SC2094
while IFS= read -r repo; do
# record existence of the repo to ensure modules with no peer relationships still get included in the order # record existence of the repo to ensure modules with no peer relationships still get included in the order
echo "${repo} ${repo}" >> "${TMP_DIR}/tidy_deps.txt" echo "${repo} ${repo}" >> "${TMP_DIR}/tidy_deps.txt"
@ -213,12 +223,14 @@ for repo in $(cat "${TMP_DIR}/tidy_unordered.txt"); do
tmp_go_deps="${TMP_DIR}/tidy_${repo/\//_}_deps.txt" tmp_go_deps="${TMP_DIR}/tidy_${repo/\//_}_deps.txt"
cp go.mod "${tmp_go_mod}" cp go.mod "${tmp_go_mod}"
echo "=== sorting ${repo}" >> "${LOG_FILE}" {
# 'go list' calculates direct imports and updates go.mod so that go list -m lists our module dependencies echo "=== sorting ${repo}"
echo "=== computing imports for ${repo}" >> "${LOG_FILE}" # 'go list' calculates direct imports and updates go.mod so that go list -m lists our module dependencies
go list all >>"${LOG_FILE}" 2>&1 echo "=== computing imports for ${repo}"
echo "=== computing tools imports for ${repo}" >> "${LOG_FILE}" go list all
go list -tags=tools all >>"${LOG_FILE}" 2>&1 echo "=== computing tools imports for ${repo}"
go list -tags=tools all
} >> "${LOG_FILE}" 2>&1
# capture module dependencies # capture module dependencies
go list -m -f '{{if not .Main}}{{.Path}}{{end}}' all > "${tmp_go_deps}" go list -m -f '{{if not .Main}}{{.Path}}{{end}}' all > "${tmp_go_deps}"
@ -227,14 +239,14 @@ for repo in $(cat "${TMP_DIR}/tidy_unordered.txt"); do
cp "${tmp_go_mod}" go.mod cp "${tmp_go_mod}" go.mod
# list all module dependencies # list all module dependencies
for dep in $(join "${TMP_DIR}/tidy_unordered.txt" "${tmp_go_deps}"); do for dep in $(join "${tidy_unordered}" "${tmp_go_deps}"); do
# record the relationship (put dep first, because we want to sort leaves first) # record the relationship (put dep first, because we want to sort leaves first)
echo "${dep} ${repo}" >> "${TMP_DIR}/tidy_deps.txt" echo "${dep} ${repo}" >> "${TMP_DIR}/tidy_deps.txt"
# switch the required version to an explicit v0.0.0 (rather than an unknown v0.0.0-00010101000000-000000000000) # switch the required version to an explicit v0.0.0 (rather than an unknown v0.0.0-00010101000000-000000000000)
go mod edit -require "${dep}@v0.0.0" go mod edit -require "${dep}@v0.0.0"
done done
popd >/dev/null 2>&1 popd >/dev/null 2>&1
done done < "${tidy_unordered}"
kube::log::status "go.mod: tidying" kube::log::status "go.mod: tidying"
for repo in $(tsort "${TMP_DIR}/tidy_deps.txt"); do for repo in $(tsort "${TMP_DIR}/tidy_deps.txt"); do
@ -256,10 +268,11 @@ for repo in $(tsort "${TMP_DIR}/tidy_deps.txt"); do
go mod tidy -v >>"${LOG_FILE}" 2>&1 go mod tidy -v >>"${LOG_FILE}" 2>&1
# disallow transitive dependencies on k8s.io/kubernetes # disallow transitive dependencies on k8s.io/kubernetes
loopback_deps="$(go list all 2>/dev/null | grep k8s.io/kubernetes/ || true)" loopback_deps=()
if [[ ! -z "${loopback_deps}" ]]; then kube::util::read-array loopback_deps < <(go list all 2>/dev/null | grep k8s.io/kubernetes/ || true)
if [[ -n "${loopback_deps[*]}" ]]; then
kube::log::error "Disallowed ${repo} -> k8s.io/kubernetes dependencies exist via the following imports: kube::log::error "Disallowed ${repo} -> k8s.io/kubernetes dependencies exist via the following imports:
$(go mod why ${loopback_deps})" $(go mod why "${loopback_deps[@]}")"
exit 1 exit 1
fi fi
@ -291,7 +304,7 @@ add_generated_comments "
// Run hack/pin-dependency.sh to change pinned dependency versions. // Run hack/pin-dependency.sh to change pinned dependency versions.
// Run hack/update-vendor.sh to update go.mod files and the vendor directory. // Run hack/update-vendor.sh to update go.mod files and the vendor directory.
" "
for repo in $(ls staging/src/k8s.io | sort); do for repo in $(kube::util::list_staging_repos); do
pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1 pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1
add_generated_comments "// This is a generated file. Do not edit directly." add_generated_comments "// This is a generated file. Do not edit directly."
popd >/dev/null 2>&1 popd >/dev/null 2>&1
@ -306,12 +319,12 @@ go mod vendor >>"${LOG_FILE}" 2>&1
# sort recorded packages for a given vendored dependency in modules.txt. # sort recorded packages for a given vendored dependency in modules.txt.
# `go mod vendor` outputs in imported order, which means slight go changes (or different platforms) can result in a differently ordered modules.txt. # `go mod vendor` outputs in imported order, which means slight go changes (or different platforms) can result in a differently ordered modules.txt.
# scan | prefix comment lines with the module name | sort field 1 | strip leading text on comment lines # scan | prefix comment lines with the module name | sort field 1 | strip leading text on comment lines
cat vendor/modules.txt | awk '{if($1=="#") print $2 " " $0; else print}' | sort -k1,1 -s | sed 's/.*#/#/' > "${TMP_DIR}/modules.txt.tmp" awk '{if($1=="#") print $2 " " $0; else print}' < vendor/modules.txt | sort -k1,1 -s | sed 's/.*#/#/' > "${TMP_DIR}/modules.txt.tmp"
mv "${TMP_DIR}/modules.txt.tmp" vendor/modules.txt mv "${TMP_DIR}/modules.txt.tmp" vendor/modules.txt
# create a symlink in vendor directory pointing to the staging components. # create a symlink in vendor directory pointing to the staging components.
# This lets other packages and tools use the local staging components as if they were vendored. # This lets other packages and tools use the local staging components as if they were vendored.
for repo in $(ls staging/src/k8s.io); do for repo in $(kube::util::list_staging_repos); do
rm -fr "${KUBE_ROOT}/vendor/k8s.io/${repo}" rm -fr "${KUBE_ROOT}/vendor/k8s.io/${repo}"
ln -s "../../staging/src/k8s.io/${repo}" "${KUBE_ROOT}/vendor/k8s.io/${repo}" ln -s "../../staging/src/k8s.io/${repo}" "${KUBE_ROOT}/vendor/k8s.io/${repo}"
done done

View File

@ -17,28 +17,33 @@
set -o errexit set -o errexit
set -o nounset set -o nounset
set -o pipefail set -o pipefail
set -x;
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
export GO111MODULE=auto export GO111MODULE=auto
staging_repos=($(ls "${KUBE_ROOT}/staging/src/k8s.io/")) staging_repos=()
kube::util::read-array staging_repos < <(kube::util::list_staging_repos)
staging_repos_pattern=$(IFS="|"; echo "${staging_repos[*]}") staging_repos_pattern=$(IFS="|"; echo "${staging_repos[*]}")
cd "${KUBE_ROOT}"
failed=false failed=false
for i in $(find vendor/ -type d); do while IFS= read -r dir; do
deps=$(go list -f '{{range .Deps}}{{.}}{{"\n"}}{{end}}' ./$i 2> /dev/null || echo "") deps=$(go list -f '{{range .Deps}}{{.}}{{"\n"}}{{end}}' "${dir}" 2> /dev/null || echo "")
deps_on_main=$(echo "${deps}" | grep -v "k8s.io/kubernetes/vendor/" | grep "k8s.io/kubernetes" || echo "") deps_on_main=$(echo "${deps}" | grep -v "k8s.io/kubernetes/vendor/" | grep "k8s.io/kubernetes" || echo "")
if [ -n "${deps_on_main}" ]; then if [ -n "${deps_on_main}" ]; then
echo "Package ${i} has a cyclic dependency on the main repository." echo "Package ${dir} has a cyclic dependency on the main repository."
failed=true failed=true
fi fi
deps_on_staging=$(echo "${deps}" | grep "k8s.io/kubernetes/vendor/k8s.io" | grep -E "k8s.io\/${staging_repos_pattern}\>" || echo "") deps_on_staging=$(echo "${deps}" | grep "k8s.io/kubernetes/vendor/k8s.io" | grep -E "k8s.io\/${staging_repos_pattern}\>" || echo "")
if [ -n "${deps_on_staging}" ]; then if [ -n "${deps_on_staging}" ]; then
echo "Package ${i} has a cyclic dependency on staging repository packages: ${deps_on_staging}" echo "Package ${dir} has a cyclic dependency on staging repository packages: ${deps_on_staging}"
failed=true failed=true
fi fi
done done < <(find ./vendor -type d)
if [[ "${failed}" == "true" ]]; then if [[ "${failed}" == "true" ]]; then
exit 1 exit 1