diff --git a/hack/boilerplate/boilerplate.py b/hack/boilerplate/boilerplate.py index 89224186aca..eaf758b9fe5 100755 --- a/hack/boilerplate/boilerplate.py +++ b/hack/boilerplate/boilerplate.py @@ -62,10 +62,6 @@ def get_refs(): def is_generated_file(filename, data, regexs): - for d in skipped_ungenerated_files: - if d in filename: - return False - p = regexs["generated"] return p.search(data) @@ -151,21 +147,15 @@ def file_extension(filename): return os.path.splitext(filename)[1].split(".")[-1].lower() -skipped_dirs = ['third_party', '_gopath', '_output', '.git', 'cluster/env.sh', - "vendor", "test/e2e/generated/bindata.go", "hack/boilerplate/test", - "staging/src/k8s.io/kubectl/pkg/generated/bindata.go"] - -# list all the files contain 'DO NOT EDIT', but are not generated -skipped_ungenerated_files = [ - 'hack/update-generated-swagger-docs.sh', - 'hack/boilerplate/boilerplate.py' -] +skipped_names = ['third_party', '_gopath', '_output', '.git', 'cluster/env.sh', + "vendor", "test/e2e/generated/bindata.go", "hack/boilerplate/test", + "staging/src/k8s.io/kubectl/pkg/generated/bindata.go"] def normalize_files(files): newfiles = [] for pathname in files: - if any(x in pathname for x in skipped_dirs): + if any(x in pathname for x in skipped_names): continue newfiles.append(pathname) for i, pathname in enumerate(newfiles): @@ -184,9 +174,13 @@ def get_files(extensions): # as we would prune these later in normalize_files(). But doing it # cuts down the amount of filesystem walking we do and cuts down # the size of the file list - for d in skipped_dirs: + for d in skipped_names: if d in dirs: dirs.remove(d) + for d in dirs: + # dirs that start with __ are ignored + if re.match("^__", d): + dirs.remove(d) for name in walkfiles: pathname = os.path.join(root, name) @@ -222,7 +216,7 @@ def get_regexs(): # strip #!.* from scripts regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) # Search for generated files - regexs["generated"] = re.compile('DO NOT EDIT') + regexs["generated"] = re.compile(r"^[/*#]+ +.* DO NOT EDIT\.$", re.MULTILINE) return regexs diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index e74a090ccac..806da41142f 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -521,7 +521,10 @@ kube::golang::setup_env() { kube::golang::create_gopath_tree export GOPATH="${KUBE_GOPATH}" - export GOCACHE="${KUBE_GOPATH}/cache" + # If these are not set, set them now. This ensures that any subsequent + # scripts we run (which may call this function again) use the same values. + export GOCACHE="${GOCACHE:-"${KUBE_GOPATH}/cache/build"}" + export GOMODCACHE="${GOMODCACHE:-"${KUBE_GOPATH}/cache/mod"}" # Make sure our own Go binaries are in PATH. export PATH="${KUBE_GOPATH}/bin:${PATH}" diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 537a0ccd583..69d620255f2 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -41,65 +41,40 @@ if [[ "${DBG_CODEGEN}" == 1 ]]; then kube::log::status "DBG: starting generated_files" fi -# This is a partial 'find' command. The caller is expected to pass the -# remaining arguments. -# -# Example: -# kfind -type f -name foobar.go -function kfind() { - # We want to include the "special" vendor directories which are actually - # part of the Kubernetes source tree (./staging/*) but we need them to be - # named as their ./vendor/* equivalents. Also, we do not want all of - # ./vendor nor ./hack/tools/vendor nor even all of ./vendor/k8s.io. - find -H . \ - \( \ - -not \( \ - \( \ - -name '_*' -o \ - -name '.[^.]*' -o \ - \( \ - -name 'vendor' \ - -type d \ - \) -o \ - \( \ - -name 'testdata' \ - -type d \ - \) \ - \) -prune \ - \) \ - \) \ - "$@" \ - | sed 's|^./staging/src|vendor|' +function git_find() { + # Similar to find but faster and easier to understand. We want to include + # modified and untracked files because this might be running against code + # which is not tracked by git yet. + git ls-files -cmo --exclude-standard "$@" } -function find_all_go_dirs() { - kfind -type f -name \*.go \ - | sed 's|/[^/]*$||' \ - | sed 's|^./||' \ - | LC_ALL=C sort -u +function git_grep() { + # We want to include modified and untracked files because this might be + # running against code which is not tracked by git yet. + git grep --untracked "$@" } -# This variable holds a list of every directory that contains Go files in this -# project. Other rules and variables can use this as a starting point to -# reduce filesystem accesses. -if [[ "${DBG_CODEGEN}" == 1 ]]; then - kube::log::status "DBG: finding all *.go dirs" -fi -ALL_GO_DIRS=() -kube::util::read-array ALL_GO_DIRS < <(find_all_go_dirs) -if [[ "${DBG_CODEGEN}" == 1 ]]; then - kube::log::status "DBG: found ${#ALL_GO_DIRS[@]} *.go dirs" -fi - # Generate a list of all files that have a `+k8s:` comment-tag. This will be # used to derive lists of files/dirs for generation tools. +# +# We want to include the "special" vendor directories which are actually part +# of the Kubernetes source tree (staging/*) but we need them to be named as +# their vendor/* equivalents. We do not want all of vendor nor +# hack/tools/vendor nor even all of vendor/k8s.io - just the subset that lives +# in staging. if [[ "${DBG_CODEGEN}" == 1 ]]; then kube::log::status "DBG: finding all +k8s: tags" fi ALL_K8S_TAG_FILES=() kube::util::read-array ALL_K8S_TAG_FILES < <( - find "${ALL_GO_DIRS[@]}" -maxdepth 1 -type f -name \*.go -print0 \ - | xargs -0 grep --color=never -l '^// *+k8s:') + git_grep -l \ + -e '^// *+k8s:' `# match +k8s: tags` \ + -- \ + ':!:vendor/*' `# not under vendor` \ + ':!:*/testdata/*' `# not under any testdata` \ + ':(glob)**/*.go' `# in any *.go file` \ + | sed 's|^staging/src|vendor|' `# see comments above` \ + ) if [[ "${DBG_CODEGEN}" == 1 ]]; then kube::log::status "DBG: found ${#ALL_K8S_TAG_FILES[@]} +k8s: tagged files" fi @@ -151,6 +126,8 @@ function codegen::prerelease() { done fi + git_find -z ':(glob)**'/"${output_base}.go" | xargs -0 rm -f + ./hack/run-in-gopath.sh "${gen_prerelease_bin}" \ --v "${KUBE_VERBOSE}" \ --logtostderr \ @@ -212,6 +189,8 @@ function codegen::deepcopy() { done fi + git_find -z ':(glob)**'/"${output_base}.go" | xargs -0 rm -f + ./hack/run-in-gopath.sh "${gen_deepcopy_bin}" \ --v "${KUBE_VERBOSE}" \ --logtostderr \ @@ -280,6 +259,8 @@ function codegen::defaults() { done fi + git_find -z ':(glob)**'/"${output_base}.go" | xargs -0 rm -f + ./hack/run-in-gopath.sh "${gen_defaulter_bin}" \ --v "${KUBE_VERBOSE}" \ --logtostderr \ @@ -359,6 +340,8 @@ function codegen::conversions() { done fi + git_find -z ':(glob)**'/"${output_base}.go" | xargs -0 rm -f + ./hack/run-in-gopath.sh "${gen_conversion_bin}" \ --v "${KUBE_VERBOSE}" \ --logtostderr \ @@ -527,6 +510,8 @@ function codegen::openapi() { "${apimachinery_dirs[@]}" ) + git_find -z ':(glob)**'/"${output_base}.go" | xargs -0 rm -f + for prefix in "${targets[@]}"; do local report_file="${OUT_DIR}/${prefix}_violations.report" # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write @@ -602,14 +587,12 @@ function codegen::applyconfigs() { local applyconfigurationgen applyconfigurationgen=$(kube::util::find-binary "applyconfiguration-gen") - # because client-gen doesn't do policy/v1alpha1, we have to skip it too local ext_apis=() kube::util::read-array ext_apis < <( - cd "${KUBE_ROOT}/staging/src" - find k8s.io/api -name types.go -print0 \ - | xargs -0 -n1 dirname \ - | grep -v pkg.apis.policy.v1alpha1 \ - | LC_ALL=C sort -u) + cd "${KUBE_ROOT}/staging/src" + git_find -z ':(glob)k8s.io/api/**/types.go' \ + | xargs -0 -n1 dirname \ + | LC_ALL=C sort -u) ext_apis+=("k8s.io/apimachinery/pkg/apis/meta/v1") kube::log::status "Generating apply-config code for ${#ext_apis[@]} targets" @@ -620,6 +603,12 @@ function codegen::applyconfigs() { done fi + git_grep -l --null \ + -e '^// Code generated by applyconfiguration-gen. DO NOT EDIT.$' \ + -- \ + ':(glob)staging/src/k8s.io/client-go/**/*.go' \ + | xargs -0 rm -f + "${applyconfigurationgen}" \ --openapi-schema <("${modelsschema}") \ --go-header-file "${BOILERPLATE_FILENAME}" \ @@ -666,6 +655,12 @@ function codegen::clients() { done fi + git_grep -l --null \ + -e '^// Code generated by client-gen. DO NOT EDIT.$' \ + -- \ + ':(glob)staging/src/k8s.io/client-go/**/*.go' \ + | xargs -0 rm -f + "${clientgen}" \ --go-header-file "${BOILERPLATE_FILENAME}" \ --output-base "${KUBE_ROOT}/vendor" \ @@ -691,7 +686,7 @@ function codegen::listers() { local ext_apis=() kube::util::read-array ext_apis < <( cd "${KUBE_ROOT}/staging/src" - find k8s.io/api -name types.go -print0 \ + git_find -z ':(glob)k8s.io/api/**/types.go' \ | xargs -0 -n1 dirname \ | LC_ALL=C sort -u) @@ -703,6 +698,12 @@ function codegen::listers() { done fi + git_grep -l --null \ + -e '^// Code generated by lister-gen. DO NOT EDIT.$' \ + -- \ + ':(glob)staging/src/k8s.io/client-go/**/*.go' \ + | xargs -0 rm -f + "${listergen}" \ --go-header-file "${BOILERPLATE_FILENAME}" \ --output-base "${KUBE_ROOT}/vendor" \ @@ -722,13 +723,11 @@ function codegen::informers() { local informergen informergen=$(kube::util::find-binary "informer-gen") - # because client-gen doesn't do policy/v1alpha1, we have to skip it too local ext_apis=() kube::util::read-array ext_apis < <( cd "${KUBE_ROOT}/staging/src" - find k8s.io/api -name types.go -print0 \ + git_find -z ':(glob)k8s.io/api/**/types.go' \ | xargs -0 -n1 dirname \ - | grep -v pkg.apis.policy.v1alpha1 \ | LC_ALL=C sort -u) kube::log::status "Generating informer code for ${#ext_apis[@]} targets" @@ -739,6 +738,12 @@ function codegen::informers() { done fi + git_grep -l --null \ + -e '^// Code generated by informer-gen. DO NOT EDIT.$' \ + -- \ + ':(glob)staging/src/k8s.io/client-go/**/*.go' \ + | xargs -0 rm -f + "${informergen}" \ --go-header-file "${BOILERPLATE_FILENAME}" \ --output-base "${KUBE_ROOT}/vendor" \ diff --git a/hack/update-generated-protobuf.sh b/hack/update-generated-protobuf.sh index 01b826ab2a0..76ba550dcad 100755 --- a/hack/update-generated-protobuf.sh +++ b/hack/update-generated-protobuf.sh @@ -27,12 +27,24 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. # source tree. This is managed in kube::build::copy_output in build/common.sh. # If the output set is changed update that function. -APIROOTS=${APIROOTS:-$(git grep --files-with-matches -e '// +k8s:protobuf-gen=package' cmd pkg staging | \ - xargs -n 1 dirname | \ - sed 's,^,k8s.io/kubernetes/,;s,k8s.io/kubernetes/staging/src/,,' | \ - sort | uniq +APIROOTS=${APIROOTS:-$( \ + git grep --untracked --null -l \ + -e '// +k8s:protobuf-gen=package' \ + -- \ + cmd pkg staging \ + | xargs -0 -n1 dirname \ + | sed 's,^,k8s.io/kubernetes/,;s,k8s.io/kubernetes/staging/src/,,' \ + | sort -u )} -"${KUBE_ROOT}/build/run.sh" hack/update-generated-protobuf-dockerized.sh "${APIROOTS}" "$@" +function git_find() { + # Similar to find but faster and easier to understand. We want to include + # modified and untracked files because this might be running against code + # which is not tracked by git yet. + git ls-files -cmo --exclude-standard ':!:vendor/*' "$@" +} -# ex: ts=2 sw=2 et filetype=sh +git_find -z ':(glob)**/generated.proto' | xargs -0 rm -f +git_find -z ':(glob)**/generated.pb.go' | xargs -0 rm -f + +"${KUBE_ROOT}/build/run.sh" hack/update-generated-protobuf-dockerized.sh "${APIROOTS}" "$@" diff --git a/hack/update-generated-swagger-docs.sh b/hack/update-generated-swagger-docs.sh index 41bc65a4d63..17f6e6502cb 100755 --- a/hack/update-generated-swagger-docs.sh +++ b/hack/update-generated-swagger-docs.sh @@ -43,19 +43,21 @@ gen_types_swagger_doc() { { echo -e "$(cat hack/boilerplate/boilerplate.generatego.txt)\n" echo "package ${group_version##*/}" + # Indenting here prevents the boilerplate checker from thinking this file + # is generated - gofmt will fix the indents anyway. cat < "${TMPFILE}" diff --git a/hack/verify-codegen.sh b/hack/verify-codegen.sh index 7fd0a19a7ad..2442e2a6d3a 100755 --- a/hack/verify-codegen.sh +++ b/hack/verify-codegen.sh @@ -26,21 +26,27 @@ set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. source "${KUBE_ROOT}/hack/lib/init.sh" +kube::util::ensure_clean_working_dir + +# This sets up the environment, like GOCACHE, which keeps the worktree cleaner. kube::golang::setup_env -# call verify on sub-project for now -# -# Note: these must be before the main script call because the later calls the sub-project's -# update-codegen.sh scripts. We wouldn't see any error on changes then. -export CODEGEN_PKG=./vendor/k8s.io/code-generator -vendor/k8s.io/code-generator/hack/verify-codegen.sh -vendor/k8s.io/kube-aggregator/hack/verify-codegen.sh -vendor/k8s.io/sample-apiserver/hack/verify-codegen.sh -vendor/k8s.io/sample-controller/hack/verify-codegen.sh -vendor/k8s.io/apiextensions-apiserver/hack/verify-codegen.sh -vendor/k8s.io/metrics/hack/verify-codegen.sh +_tmpdir="$(kube::realpath "$(mktemp -d -t "$(basename "$0").XXXXXX")")" +git worktree add -f -q "${_tmpdir}" HEAD +kube::util::trap_add "git worktree remove -f ${_tmpdir}" EXIT +cd "${_tmpdir}" -# This won't actually update anything because of --verify-only, but it tells -# the openapi tool to verify against the real filenames. +# Update generated code export UPDATE_API_KNOWN_VIOLATIONS=true -"${KUBE_ROOT}/hack/update-codegen.sh" --verify-only "$@" +hack/update-codegen.sh "$@" + +# Test for diffs +diffs=$(git status --porcelain | wc -l) +if [[ ${diffs} -gt 0 ]]; then + git status >&2 + git diff >&2 + echo "Generated files need to be updated" >&2 + echo "Please run 'hack/update-codegen.sh'" >&2 + exit 1 +fi +echo "Generated files are up to date"