Merge pull request #115198 from thockin/codegen-7-remove-genfiles-part1

Codegen: remove generated files before regenerating

hack/boilerplate/boilerplate.py

@@ -62,10 +62,6 @@ def get_refs():
 
 
 def is_generated_file(filename, data, regexs):
-    for d in skipped_ungenerated_files:
-        if d in filename:
-            return False
-
     p = regexs["generated"]
     return p.search(data)
 
@@ -151,21 +147,15 @@ def file_extension(filename):
     return os.path.splitext(filename)[1].split(".")[-1].lower()
 
 
-skipped_dirs = ['third_party', '_gopath', '_output', '.git', 'cluster/env.sh',
+skipped_names = ['third_party', '_gopath', '_output', '.git', 'cluster/env.sh',
-                "vendor", "test/e2e/generated/bindata.go", "hack/boilerplate/test",
+                 "vendor", "test/e2e/generated/bindata.go", "hack/boilerplate/test",
-                "staging/src/k8s.io/kubectl/pkg/generated/bindata.go"]
+                 "staging/src/k8s.io/kubectl/pkg/generated/bindata.go"]
-
-# list all the files contain 'DO NOT EDIT', but are not generated
-skipped_ungenerated_files = [
-    'hack/update-generated-swagger-docs.sh',
-    'hack/boilerplate/boilerplate.py'
-]
 
 
 def normalize_files(files):
     newfiles = []
     for pathname in files:
-        if any(x in pathname for x in skipped_dirs):
+        if any(x in pathname for x in skipped_names):
             continue
         newfiles.append(pathname)
     for i, pathname in enumerate(newfiles):
@@ -184,9 +174,13 @@ def get_files(extensions):
             # as we would prune these later in normalize_files(). But doing it
             # cuts down the amount of filesystem walking we do and cuts down
             # the size of the file list
-            for d in skipped_dirs:
+            for d in skipped_names:
                 if d in dirs:
                     dirs.remove(d)
+            for d in dirs:
+                # dirs that start with __ are ignored
+                if re.match("^__", d):
+                    dirs.remove(d)
 
             for name in walkfiles:
                 pathname = os.path.join(root, name)
@@ -222,7 +216,7 @@ def get_regexs():
     # strip #!.* from scripts
     regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE)
     # Search for generated files
-    regexs["generated"] = re.compile('DO NOT EDIT')
+    regexs["generated"] = re.compile(r"^[/*#]+ +.* DO NOT EDIT\.$", re.MULTILINE)
     return regexs
 
 

hack/lib/golang.sh

@@ -521,7 +521,10 @@ kube::golang::setup_env() {
   kube::golang::create_gopath_tree
   export GOPATH="${KUBE_GOPATH}"
 
-  export GOCACHE="${KUBE_GOPATH}/cache"
+  # If these are not set, set them now.  This ensures that any subsequent
+  # scripts we run (which may call this function again) use the same values.
+  export GOCACHE="${GOCACHE:-"${KUBE_GOPATH}/cache/build"}"
+  export GOMODCACHE="${GOMODCACHE:-"${KUBE_GOPATH}/cache/mod"}"
 
   # Make sure our own Go binaries are in PATH.
   export PATH="${KUBE_GOPATH}/bin:${PATH}"

hack/update-codegen.sh

@@ -41,65 +41,40 @@ if [[ "${DBG_CODEGEN}" == 1 ]]; then
     kube::log::status "DBG: starting generated_files"
 fi
 
-# This is a partial 'find' command.  The caller is expected to pass the
+function git_find() {
-# remaining arguments.
+    # Similar to find but faster and easier to understand.  We want to include
-#
+    # modified and untracked files because this might be running against code
-# Example:
+    # which is not tracked by git yet.
-#   kfind -type f -name foobar.go
+    git ls-files -cmo --exclude-standard "$@"
-function kfind() {
-    # We want to include the "special" vendor directories which are actually
-    # part of the Kubernetes source tree (./staging/*) but we need them to be
-    # named as their ./vendor/* equivalents.  Also, we do not want all of
-    # ./vendor nor ./hack/tools/vendor nor even all of ./vendor/k8s.io.
-    find -H .                      \
-        \(                         \
-        -not \(                    \
-            \(                     \
-                -name '_*' -o      \
-                -name '.[^.]*' -o  \
-                \(                 \
-                  -name 'vendor'   \
-                  -type d          \
-                \) -o              \
-                \(                 \
-                  -name 'testdata' \
-                  -type d          \
-                \)                 \
-            \) -prune              \
-        \)                         \
-        \)                         \
-        "$@"                       \
-        | sed 's|^./staging/src|vendor|'
 }
 
-function find_all_go_dirs() {
+function git_grep() {
-    kfind -type f -name \*.go  \
+    # We want to include modified and untracked files because this might be
-        | sed 's|/[^/]*$||'    \
+    # running against code which is not tracked by git yet.
-        | sed 's|^./||'        \
+    git grep --untracked "$@"
-        | LC_ALL=C sort -u
 }
 
-# This variable holds a list of every directory that contains Go files in this
-# project.  Other rules and variables can use this as a starting point to
-# reduce filesystem accesses.
-if [[ "${DBG_CODEGEN}" == 1 ]]; then
-    kube::log::status "DBG: finding all *.go dirs"
-fi
-ALL_GO_DIRS=()
-kube::util::read-array ALL_GO_DIRS < <(find_all_go_dirs)
-if [[ "${DBG_CODEGEN}" == 1 ]]; then
-    kube::log::status "DBG: found ${#ALL_GO_DIRS[@]} *.go dirs"
-fi
-
 # Generate a list of all files that have a `+k8s:` comment-tag.  This will be
 # used to derive lists of files/dirs for generation tools.
+#
+# We want to include the "special" vendor directories which are actually part
+# of the Kubernetes source tree (staging/*) but we need them to be named as
+# their vendor/* equivalents.  We do not want all of vendor nor
+# hack/tools/vendor nor even all of vendor/k8s.io - just the subset that lives
+# in staging.
 if [[ "${DBG_CODEGEN}" == 1 ]]; then
     kube::log::status "DBG: finding all +k8s: tags"
 fi
 ALL_K8S_TAG_FILES=()
 kube::util::read-array ALL_K8S_TAG_FILES < <(
-    find "${ALL_GO_DIRS[@]}" -maxdepth 1 -type f -name \*.go -print0 \
+    git_grep -l \
-        | xargs -0 grep --color=never -l '^// *+k8s:')
+        -e '^// *+k8s:'                `# match +k8s: tags` \
+        -- \
+        ':!:vendor/*'                  `# not under vendor` \
+        ':!:*/testdata/*'              `# not under any testdata` \
+        ':(glob)**/*.go'               `# in any *.go file` \
+        | sed 's|^staging/src|vendor|' `# see comments above` \
+    )
 if [[ "${DBG_CODEGEN}" == 1 ]]; then
     kube::log::status "DBG: found ${#ALL_K8S_TAG_FILES[@]} +k8s: tagged files"
 fi
@@ -151,6 +126,8 @@ function codegen::prerelease() {
         done
     fi
 
+    git_find -z ':(glob)**'/"${output_base}.go" | xargs -0 rm -f
+
     ./hack/run-in-gopath.sh "${gen_prerelease_bin}" \
         --v "${KUBE_VERBOSE}" \
         --logtostderr \
@@ -212,6 +189,8 @@ function codegen::deepcopy() {
         done
     fi
 
+    git_find -z ':(glob)**'/"${output_base}.go" | xargs -0 rm -f
+
     ./hack/run-in-gopath.sh "${gen_deepcopy_bin}" \
         --v "${KUBE_VERBOSE}" \
         --logtostderr \
@@ -280,6 +259,8 @@ function codegen::defaults() {
         done
     fi
 
+    git_find -z ':(glob)**'/"${output_base}.go" | xargs -0 rm -f
+
     ./hack/run-in-gopath.sh "${gen_defaulter_bin}" \
         --v "${KUBE_VERBOSE}" \
         --logtostderr \
@@ -359,6 +340,8 @@ function codegen::conversions() {
         done
     fi
 
+    git_find -z ':(glob)**'/"${output_base}.go" | xargs -0 rm -f
+
     ./hack/run-in-gopath.sh "${gen_conversion_bin}" \
         --v "${KUBE_VERBOSE}" \
         --logtostderr \
@@ -527,6 +510,8 @@ function codegen::openapi() {
             "${apimachinery_dirs[@]}"
         )
 
+    git_find -z ':(glob)**'/"${output_base}.go" | xargs -0 rm -f
+
     for prefix in "${targets[@]}"; do
         local report_file="${OUT_DIR}/${prefix}_violations.report"
         # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write
@@ -602,14 +587,12 @@ function codegen::applyconfigs() {
     local applyconfigurationgen
     applyconfigurationgen=$(kube::util::find-binary "applyconfiguration-gen")
 
-    # because client-gen doesn't do policy/v1alpha1, we have to skip it too
     local ext_apis=()
     kube::util::read-array ext_apis < <(
-      cd "${KUBE_ROOT}/staging/src"
+        cd "${KUBE_ROOT}/staging/src"
-      find k8s.io/api -name types.go -print0 \
+        git_find -z ':(glob)k8s.io/api/**/types.go' \
-          | xargs -0 -n1 dirname \
+            | xargs -0 -n1 dirname \
-          | grep -v pkg.apis.policy.v1alpha1 \
+            | LC_ALL=C sort -u)
-          | LC_ALL=C sort -u)
     ext_apis+=("k8s.io/apimachinery/pkg/apis/meta/v1")
 
     kube::log::status "Generating apply-config code for ${#ext_apis[@]} targets"
@@ -620,6 +603,12 @@ function codegen::applyconfigs() {
         done
     fi
 
+    git_grep -l --null \
+        -e '^// Code generated by applyconfiguration-gen. DO NOT EDIT.$' \
+        -- \
+        ':(glob)staging/src/k8s.io/client-go/**/*.go' \
+        | xargs -0 rm -f
+
     "${applyconfigurationgen}" \
         --openapi-schema <("${modelsschema}") \
         --go-header-file "${BOILERPLATE_FILENAME}" \
@@ -666,6 +655,12 @@ function codegen::clients() {
         done
     fi
 
+    git_grep -l --null \
+        -e '^// Code generated by client-gen. DO NOT EDIT.$' \
+        -- \
+        ':(glob)staging/src/k8s.io/client-go/**/*.go' \
+        | xargs -0 rm -f
+
     "${clientgen}" \
         --go-header-file "${BOILERPLATE_FILENAME}" \
         --output-base "${KUBE_ROOT}/vendor" \
@@ -691,7 +686,7 @@ function codegen::listers() {
     local ext_apis=()
     kube::util::read-array ext_apis < <(
         cd "${KUBE_ROOT}/staging/src"
-        find k8s.io/api -name types.go -print0 \
+        git_find -z ':(glob)k8s.io/api/**/types.go' \
             | xargs -0 -n1 dirname \
             | LC_ALL=C sort -u)
 
@@ -703,6 +698,12 @@ function codegen::listers() {
         done
     fi
 
+    git_grep -l --null \
+        -e '^// Code generated by lister-gen. DO NOT EDIT.$' \
+        -- \
+        ':(glob)staging/src/k8s.io/client-go/**/*.go' \
+        | xargs -0 rm -f
+
     "${listergen}" \
         --go-header-file "${BOILERPLATE_FILENAME}" \
         --output-base "${KUBE_ROOT}/vendor" \
@@ -722,13 +723,11 @@ function codegen::informers() {
     local informergen
     informergen=$(kube::util::find-binary "informer-gen")
 
-    # because client-gen doesn't do policy/v1alpha1, we have to skip it too
     local ext_apis=()
     kube::util::read-array ext_apis < <(
         cd "${KUBE_ROOT}/staging/src"
-        find k8s.io/api -name types.go -print0 \
+        git_find -z ':(glob)k8s.io/api/**/types.go' \
             | xargs -0 -n1 dirname \
-            | grep -v pkg.apis.policy.v1alpha1 \
             | LC_ALL=C sort -u)
 
     kube::log::status "Generating informer code for ${#ext_apis[@]} targets"
@@ -739,6 +738,12 @@ function codegen::informers() {
         done
     fi
 
+    git_grep -l --null \
+        -e '^// Code generated by informer-gen. DO NOT EDIT.$' \
+        -- \
+        ':(glob)staging/src/k8s.io/client-go/**/*.go' \
+        | xargs -0 rm -f
+
     "${informergen}" \
         --go-header-file "${BOILERPLATE_FILENAME}" \
         --output-base "${KUBE_ROOT}/vendor" \

hack/update-generated-protobuf.sh

@@ -27,12 +27,24 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 # source tree.  This is managed in kube::build::copy_output in build/common.sh.
 # If the output set is changed update that function.
 
-APIROOTS=${APIROOTS:-$(git grep --files-with-matches -e '// +k8s:protobuf-gen=package' cmd pkg staging | \
+APIROOTS=${APIROOTS:-$( \
-	xargs -n 1 dirname | \
+    git grep --untracked --null -l \
-	sed 's,^,k8s.io/kubernetes/,;s,k8s.io/kubernetes/staging/src/,,' | \
+        -e '// +k8s:protobuf-gen=package' \
-	sort | uniq
+        -- \
+        cmd pkg staging \
+        | xargs -0 -n1 dirname \
+        | sed 's,^,k8s.io/kubernetes/,;s,k8s.io/kubernetes/staging/src/,,' \
+        | sort -u
 )}
 
-"${KUBE_ROOT}/build/run.sh" hack/update-generated-protobuf-dockerized.sh "${APIROOTS}" "$@"
+function git_find() {
+    # Similar to find but faster and easier to understand.  We want to include
+    # modified and untracked files because this might be running against code
+    # which is not tracked by git yet.
+    git ls-files -cmo --exclude-standard ':!:vendor/*' "$@"
+}
 
-# ex: ts=2 sw=2 et filetype=sh
+git_find -z ':(glob)**/generated.proto' | xargs -0 rm -f
+git_find -z ':(glob)**/generated.pb.go' | xargs -0 rm -f
+
+"${KUBE_ROOT}/build/run.sh" hack/update-generated-protobuf-dockerized.sh "${APIROOTS}" "$@"

hack/update-generated-swagger-docs.sh

@@ -43,19 +43,21 @@ gen_types_swagger_doc() {
   {
     echo -e "$(cat hack/boilerplate/boilerplate.generatego.txt)\n"
     echo "package ${group_version##*/}"
+    # Indenting here prevents the boilerplate checker from thinking this file
+    # is generated - gofmt will fix the indents anyway.
     cat <<EOF
 
-// This file contains a collection of methods that can be used from go-restful to
+      // This file contains a collection of methods that can be used from go-restful to
-// generate Swagger API documentation for its models. Please read this PR for more
+      // generate Swagger API documentation for its models. Please read this PR for more
-// information on the implementation: https://github.com/emicklei/go-restful/pull/215
+      // information on the implementation: https://github.com/emicklei/go-restful/pull/215
-//
+      //
-// TODOs are ignored from the parser (e.g. TODO(andronat):... || TODO:...) if and only if
+      // TODOs are ignored from the parser (e.g. TODO(andronat):... || TODO:...) if and only if
-// they are on one line! For multiple line or blocks that you want to ignore use ---.
+      // they are on one line! For multiple line or blocks that you want to ignore use ---.
-// Any context after a --- is ignored.
+      // Any context after a --- is ignored.
-//
+      //
-// Those methods can be generated by using hack/update-generated-swagger-docs.sh
+      // Those methods can be generated by using hack/update-generated-swagger-docs.sh
 
-// AUTO-GENERATED FUNCTIONS START HERE. DO NOT EDIT.
+      // AUTO-GENERATED FUNCTIONS START HERE. DO NOT EDIT.
 EOF
   } > "${TMPFILE}"
 

hack/verify-codegen.sh

@@ -26,21 +26,27 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
+kube::util::ensure_clean_working_dir
+
+# This sets up the environment, like GOCACHE, which keeps the worktree cleaner.
 kube::golang::setup_env
 
-# call verify on sub-project for now
+_tmpdir="$(kube::realpath "$(mktemp -d -t "$(basename "$0").XXXXXX")")"
-#
+git worktree add -f -q "${_tmpdir}" HEAD
-# Note: these must be before the main script call because the later calls the sub-project's
+kube::util::trap_add "git worktree remove -f ${_tmpdir}" EXIT
-#       update-codegen.sh scripts. We wouldn't see any error on changes then.
+cd "${_tmpdir}"
-export CODEGEN_PKG=./vendor/k8s.io/code-generator
-vendor/k8s.io/code-generator/hack/verify-codegen.sh
-vendor/k8s.io/kube-aggregator/hack/verify-codegen.sh
-vendor/k8s.io/sample-apiserver/hack/verify-codegen.sh
-vendor/k8s.io/sample-controller/hack/verify-codegen.sh
-vendor/k8s.io/apiextensions-apiserver/hack/verify-codegen.sh
-vendor/k8s.io/metrics/hack/verify-codegen.sh
 
-# This won't actually update anything because of --verify-only, but it tells
+# Update generated code
-# the openapi tool to verify against the real filenames.
 export UPDATE_API_KNOWN_VIOLATIONS=true
-"${KUBE_ROOT}/hack/update-codegen.sh" --verify-only "$@"
+hack/update-codegen.sh "$@"
+
+# Test for diffs
+diffs=$(git status --porcelain | wc -l)
+if [[ ${diffs} -gt 0 ]]; then
+  git status >&2
+  git diff >&2
+  echo "Generated files need to be updated" >&2
+  echo "Please run 'hack/update-codegen.sh'" >&2
+  exit 1
+fi
+echo "Generated files are up to date"