From cf280dd6c2a4549bd0cc6608ef2519b42c154a88 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Tue, 24 Sep 2024 12:02:54 -0700 Subject: [PATCH] Skip Go target normalization in integration tests --- hack/install-etcd.sh | 2 ++ hack/lib/etcd.sh | 2 +- hack/lib/golang.sh | 21 +++++++++++++++++---- hack/make-rules/test-integration.sh | 29 ++++++++++++++++++++--------- hack/make-rules/test.sh | 2 +- 5 files changed, 41 insertions(+), 15 deletions(-) diff --git a/hack/install-etcd.sh b/hack/install-etcd.sh index 897707460e8..fb4bed85364 100755 --- a/hack/install-etcd.sh +++ b/hack/install-etcd.sh @@ -25,4 +25,6 @@ set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. source "${KUBE_ROOT}/hack/lib/init.sh" +FOUND="$(echo "${PATH}" | sed 's/:/\n/g' | grep -q "^${KUBE_ROOT}/third_party/etcd$" || true)" kube::etcd::install +test -n "${FOUND}" || echo " PATH=\"\$PATH:${KUBE_ROOT}/third_party/etcd\"" diff --git a/hack/lib/etcd.sh b/hack/lib/etcd.sh index 8261c04f1cc..8fb03c61a7c 100755 --- a/hack/lib/etcd.sh +++ b/hack/lib/etcd.sh @@ -159,7 +159,7 @@ kube::etcd::install() { cd "${KUBE_ROOT}/third_party" || return 1 if [[ $(readlink etcd) == etcd-v${ETCD_VERSION}-${os}-* ]]; then - V=3 kube::log::info "etcd v${ETCD_VERSION} is already installed" + V=4 kube::log::info "etcd v${ETCD_VERSION} is already installed" return 0 # already installed fi diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index 908208100a0..6b673a12925 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -383,7 +383,7 @@ kube::golang::best_guess_go_targets() { continue fi - if [[ "${target}" =~ ^([[:alnum:]]+".")+[[:alnum:]]+"/" ]]; then + if [[ "${target}" =~ ^([[:alnum:]]+".")+[[:alnum:]]+"/".+ ]]; then # If the target starts with what looks like a domain name, assume it has a # fully-qualified Go package name. echo "${target}" @@ -413,6 +413,19 @@ kube::golang::best_guess_go_targets() { done } +kube::golang::internal::lazy_normalize() { + target="$1" + + if [[ "${target}" =~ ^([[:alnum:]]+".")+[[:alnum:]]+"/".+ ]]; then + # If the target starts with what looks like a domain name, assume it has a + # fully-qualified Go package name. + echo "${target}" + return + fi + + go list -find -e "${target}" +} + # kube::golang::normalize_go_targets takes a list of build targets, which might # be Go-style names (e.g. example.com/foo/bar or ./foo/bar) or just local paths # (e.g. foo/bar) and produces a respective list (on stdout) of Go package @@ -433,7 +446,7 @@ kube::golang::normalize_go_targets() { local tst tst="$(basename "${target}")" local pkg - pkg="$(go list -find -e "${dir}")" + pkg="$(kube::golang::internal::lazy_normalize "${dir}")" echo "${pkg}/${tst}" continue fi @@ -441,11 +454,11 @@ kube::golang::normalize_go_targets() { local dir dir="$(dirname "${target}")" local pkg - pkg="$(go list -find -e "${dir}")" + pkg="$(kube::golang::internal::lazy_normalize "${dir}")" echo "${pkg}/..." continue fi - go list -find -e "${target}" + kube::golang::internal::lazy_normalize "${target}" done } diff --git a/hack/make-rules/test-integration.sh b/hack/make-rules/test-integration.sh index ad9cd90034c..dc2386ed83b 100755 --- a/hack/make-rules/test-integration.sh +++ b/hack/make-rules/test-integration.sh @@ -21,6 +21,10 @@ set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. source "${KUBE_ROOT}/hack/lib/init.sh" +kube::golang::setup_env +kube::golang::setup_gomaxprocs +kube::util::require-jq + # start the cache mutation detector by default so that cache mutators will be found KUBE_CACHE_MUTATION_DETECTOR="${KUBE_CACHE_MUTATION_DETECTOR:-true}" export KUBE_CACHE_MUTATION_DETECTOR @@ -43,16 +47,23 @@ KUBE_TEST_ARGS=${KUBE_TEST_ARGS:-} # Default glog module settings. KUBE_TEST_VMODULE=${KUBE_TEST_VMODULE:-""} -kube::test::find_integration_test_dirs() { +kube::test::find_integration_test_pkgs() { ( cd "${KUBE_ROOT}" - # The "./" syntax here produces Go-compatible package names. - find ./test/integration/ -name '*_test.go' -print0 \ - | xargs -0n1 dirname \ - | LC_ALL=C sort -u - find ./staging/src/k8s.io/apiextensions-apiserver/test/integration/ -name '*_test.go' -print0 \ - | xargs -0n1 dirname \ - | LC_ALL=C sort -u + + # Get a list of all the modules in this workspace. + local -a workspace_module_patterns + kube::util::read-array workspace_module_patterns < <( + go list -m -json | jq -r '.Dir' \ + | while read -r D; do + SUB="${D}/test/integration"; + test -d "${SUB}" && echo "${SUB}/..."; + done) + + # Get a list of all packages which have test files. + go list -find \ + -f '{{if or (gt (len .TestGoFiles) 0) (gt (len .XTestGoFiles) 0)}}{{.ImportPath}}{{end}}' \ + "${workspace_module_patterns[@]}" ) } @@ -81,7 +92,7 @@ runTests() { # empty here to ensure that we aren't unintentionally consuming them from the # previous make invocation. KUBE_TEST_ARGS="${SHORT:--short=true} --vmodule=${KUBE_TEST_VMODULE} ${KUBE_TEST_ARGS}" \ - WHAT="${WHAT:-$(kube::test::find_integration_test_dirs | paste -sd' ' -)}" \ + WHAT="${WHAT:-$(kube::test::find_integration_test_pkgs | paste -sd' ' -)}" \ GOFLAGS="${GOFLAGS:-}" \ KUBE_TIMEOUT="${KUBE_TIMEOUT}" \ KUBE_RACE="" \ diff --git a/hack/make-rules/test.sh b/hack/make-rules/test.sh index 0842ffb4f22..75fa127c7e8 100755 --- a/hack/make-rules/test.sh +++ b/hack/make-rules/test.sh @@ -158,7 +158,7 @@ if [[ ${#testcases[@]} -eq 0 ]]; then kube::util::read-array testcases < <(kube::test::find_go_packages) else # If the user passed targets, we should normalize them. - # This can be slow! + # This can be slow for large numbers of inputs. kube::log::status "Normalizing Go targets" kube::util::read-array testcases < <(kube::golang::normalize_go_targets "${testcases[@]}") fi