Copy make's codegen logic to update-codegen.sh

The `make` rules which auto-generate some of our API stuff are
incredibly baroque, and hard to maintain.  They were originally added on
the assumption that we would stop checking generated files into git.
Since then we have moved away from that goal, and the worst problems
with generated files have been resolved.

Reasons to kill this:
* It is slow on every build, as opposed to just being slow when running
  the generators.  It is even slow to calculate that there's nothing to
  update.
* Most development work doesn't involve changing APIs.
* It only covers about half (or less) of the generated code, and making
  it cover more would be even slower.
* Approximately 1 person knows how this all works.
* We have CI to make sure changes do not get merged without updating
  this code.
* We have corner cases where this does the WRONG thing and tracking
  those down is ugly and hard in perpetuity.

So this commit puts all the same logic that WAS in the
Makefile.generated_files into update-codegen.sh.

I do not love this script, especially WRT sub-packages, but I am trying
not to boil the ocean.  I hope to follow up with some more cleanups over
time.

I have tested this manually and with the scripts and it still seems to
catch errors properly.

This includes a change to kube::util::read-array to make it not unset
variables and not over-write non-array variables.

hack/lib/util.sh

@@ -798,7 +798,9 @@ function kube::util::md5() {
 
 # kube::util::read-array
 # Reads in stdin and adds it line by line to the array provided. This can be
-# used instead of "mapfile -t", and is bash 3 compatible.
+# used instead of "mapfile -t", and is bash 3 compatible.  If the named array
+# exists and is an array, it will be used.  Otherwise it will be unset and
+# recreated.
 #
 # Assumed vars:
 #   $1 (name of array to create/modify)
@@ -806,11 +808,44 @@ function kube::util::md5() {
 # Example usage:
 #   kube::util::read-array files < <(ls -1)
 #
+# When in doubt:
+#  $ W=abc         # a string
+#  $ X=(a b c)     # an array
+#  $ declare -A Y  # an associative array
+#  $ unset Z       # not set at all
+#  $ declare -p W X Y Z
+#  declare -- W="abc"
+#  declare -a X=([0]="a" [1]="b" [2]="c")
+#  declare -A Y
+#  bash: line 26: declare: Z: not found
+#  $ kube::util::read-array W < <(echo -ne "1 1\n2 2\n3 3\n")
+#  bash: W is defined but isn't an array
+#  $ kube::util::read-array X < <(echo -ne "1 1\n2 2\n3 3\n")
+#  $ kube::util::read-array Y < <(echo -ne "1 1\n2 2\n3 3\n")
+#  bash: Y is defined but isn't an array
+#  $ kube::util::read-array Z < <(echo -ne "1 1\n2 2\n3 3\n")
+#  $ declare -p W X Y Z
+#  declare -- W="abc"
+#  declare -a X=([0]="1 1" [1]="2 2" [2]="3 3")
+#  declare -A Y
+#  declare -a Z=([0]="1 1" [1]="2 2" [2]="3 3")
 function kube::util::read-array {
-  local i=0
+  if [[ -z "$1" ]]; then
-  unset -v "$1"
+    echo "usage: ${FUNCNAME[0]} <varname>" >&2
-  while IFS= read -r "$1[i++]"; do :; done
+    return 1
-  eval "[[ \${$1[--i]} ]]" || unset "$1[i]" # ensures last element isn't empty
+  fi
+  if [[ -n $(declare -p "$1" 2>/dev/null) ]]; then
+    if ! declare -p "$1" 2>/dev/null | grep -q '^declare -a'; then
+      echo "${FUNCNAME[0]}: $1 is defined but isn't an array" >&2
+      return 2
+    fi
+  fi
+  # shellcheck disable=SC2034 # this variable _is_ used
+  local __read_array_i=0
+  while IFS= read -r "$1[__read_array_i++]"; do :; done
+  if ! eval "[[ \${$1[--__read_array_i]} ]]"; then
+    unset "$1[__read_array_i]" # ensures last element isn't empty
+  fi
 }
 
 # Some useful colors.

hack/update-codegen.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-
 # Copyright 2014 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,117 +13,783 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Please do not add any logic to this shell script. Add logic to the go code
+# shellcheck disable=2046 # printf word-splitting is intentional
-# that generates the set-gen program.
 
 set -o errexit
 set -o nounset
 set -o pipefail
 
+# This tool wants a different default than usual.
+KUBE_VERBOSE="${KUBE_VERBOSE:-1}"
+
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
 kube::golang::setup_env
 
-GO111MODULE=on GOPROXY=off go install k8s.io/kubernetes/pkg/generated/openapi/cmd/models-schema
+DBG_CODEGEN="${DBG_CODEGEN:-0}"
-GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/applyconfiguration-gen
+GENERATED_FILE_PREFIX="${GENERATED_FILE_PREFIX:-zz_generated.}"
-GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/client-gen
+UPDATE_API_KNOWN_VIOLATIONS="${UPDATE_API_KNOWN_VIOLATIONS:-}"
-GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/lister-gen
-GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/informer-gen
 
+OUT_DIR="_output"
+BIN_DIR="${OUT_DIR}/bin"
+PRJ_SRC_PATH="k8s.io/kubernetes"
+BOILERPLATE_FILENAME="vendor/k8s.io/code-generator/hack/boilerplate.go.txt"
+APPLYCONFIG_PKG="k8s.io/client-go/applyconfigurations"
+
+if [[ "${DBG_CODEGEN}" == 1 ]]; then
+    kube::log::status "DBG: starting generated_files"
+fi
+
+# This variable holds a list of every directory that contains Go files in this
+# project.  Other rules and variables can use this as a starting point to
+# reduce filesystem accesses.
+if [[ "${DBG_CODEGEN}" == 1 ]]; then
+    kube::log::status "DBG: finding all *.go dirs"
+fi
+ALL_GO_DIRS=()
+kube::util::read-array ALL_GO_DIRS < \
+    <(hack/make-rules/helpers/cache_go_dirs.sh .make/all_go_dirs)
+if [[ "${DBG_CODEGEN}" == 1 ]]; then
+    kube::log::status "DBG: found ${#ALL_GO_DIRS[@]} *.go dirs"
+fi
+
+# Generate a list of all files that have a `+k8s:` comment-tag.  This will be
+# used to derive lists of files/dirs for generation tools.
+if [[ "${DBG_CODEGEN}" == 1 ]]; then
+    kube::log::status "DBG: finding all +k8s: tags"
+fi
+ALL_K8S_TAG_FILES=()
+kube::util::read-array ALL_K8S_TAG_FILES < <(
+    find "${ALL_GO_DIRS[@]}" -maxdepth 1 -type f -name \*.go -print0 \
+        | xargs -0 grep --color=never -l '^// *+k8s:')
+if [[ "${DBG_CODEGEN}" == 1 ]]; then
+    kube::log::status "DBG: found ${#ALL_K8S_TAG_FILES[@]} +k8s: tagged files"
+fi
+
+#
+# Code generation logic.
+#
+
+# prerelease-lifecycle generation
+#
+# Any package that wants prerelease-lifecycle functions generated must include a
+# comment-tag in column 0 of one file of the form:
+#     // +k8s:prerelease-lifecycle-gen=true
+function codegen::prerelease() {
+    # Build the tool.
+    hack/make-rules/build.sh k8s.io/code-generator/cmd/prerelease-lifecycle-gen
+
+    # The result file, in each pkg, of prerelease-lifecycle generation.
+    local output_base="${GENERATED_FILE_PREFIX}prerelease-lifecycle"
+
+    # The tool used to generate prerelease-lifecycle code.
+    local gen_prerelease_bin="${BIN_DIR}/prerelease-lifecycle-gen"
+
+    # Find all the directories that request prerelease-lifecycle generation.
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: finding all +k8s:prerelease-lifecycle-gen tags"
+    fi
+    local tag_dirs=()
+    kube::util::read-array tag_dirs < <( \
+        grep --color=never -l '+k8s:prerelease-lifecycle-gen=true' "${ALL_K8S_TAG_FILES[@]}" \
+            | xargs -n1 dirname \
+            | LC_ALL=C sort -u)
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:prerelease-lifecycle-gen tagged dirs"
+    fi
+
+    local tag_pkgs=()
+    for dir in "${tag_dirs[@]}"; do
+        tag_pkgs+=("${PRJ_SRC_PATH}/$dir")
+    done
+
+    kube::log::status "Generating prerelease-lifecycle code for ${#tag_pkgs[@]} targets"
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: running ${gen_prerelease_bin} for:"
+        for dir in "${tag_dirs[@]}"; do
+            kube::log::status "DBG:     $dir"
+        done
+    fi
+
+    ./hack/run-in-gopath.sh "${gen_prerelease_bin}" \
+        --v "${KUBE_VERBOSE}" \
+        --logtostderr \
+        -h "${BOILERPLATE_FILENAME}" \
+        -O "${output_base}" \
+        $(printf -- " -i %s" "${tag_pkgs[@]}") \
+        "$@"
+
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "Generated prerelease-lifecycle code"
+    fi
+}
+
+# Deep-copy generation
+#
+# Any package that wants deep-copy functions generated must include a
+# comment-tag in column 0 of one file of the form:
+#     // +k8s:deepcopy-gen=<VALUE>
+#
+# The <VALUE> may be one of:
+#     generate: generate deep-copy functions into the package
+#     register: generate deep-copy functions and register them with a
+#               scheme
+function codegen::deepcopy() {
+    # Build the tool.
+    hack/make-rules/build.sh k8s.io/code-generator/cmd/deepcopy-gen
+
+    # The result file, in each pkg, of deep-copy generation.
+    local output_base="${GENERATED_FILE_PREFIX}deepcopy"
+
+    # The tool used to generate deep copies.
+    local gen_deepcopy_bin="${BIN_DIR}/deepcopy-gen"
+
+    # Find all the directories that request deep-copy generation.
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: finding all +k8s:deepcopy-gen tags"
+    fi
+    local tag_dirs=()
+    kube::util::read-array tag_dirs < <( \
+        grep --color=never -l '+k8s:deepcopy-gen=' "${ALL_K8S_TAG_FILES[@]}" \
+            | xargs -n1 dirname \
+            | LC_ALL=C sort -u)
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:deepcopy-gen tagged dirs"
+    fi
+
+    local tag_pkgs=()
+    for dir in "${tag_dirs[@]}"; do
+        tag_pkgs+=("${PRJ_SRC_PATH}/$dir")
+    done
+
+    kube::log::status "Generating deepcopy code for ${#tag_pkgs[@]} targets"
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: running ${gen_deepcopy_bin} for:"
+        for dir in "${tag_dirs[@]}"; do
+            kube::log::status "DBG:     $dir"
+        done
+    fi
+
+    ./hack/run-in-gopath.sh "${gen_deepcopy_bin}" \
+        --v "${KUBE_VERBOSE}" \
+        --logtostderr \
+        -h "${BOILERPLATE_FILENAME}" \
+        -O "${output_base}" \
+        --bounding-dirs "${PRJ_SRC_PATH},k8s.io/api" \
+        $(printf -- " -i %s" "${tag_pkgs[@]}") \
+        "$@"
+
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "Generated deepcopy code"
+    fi
+}
+
+# Defaulter generation
+#
+# Any package that wants defaulter functions generated must include a
+# comment-tag in column 0 of one file of the form:
+#     // +k8s:defaulter-gen=<VALUE>
+#
+# The <VALUE> depends on context:
+#     on types:
+#       true:  always generate a defaulter for this type
+#       false: never generate a defaulter for this type
+#     on functions:
+#       covers: if the function name matches SetDefault_NAME, instructs
+#               the generator not to recurse
+#     on packages:
+#       FIELDNAME: any object with a field of this name is a candidate
+#                  for having a defaulter generated
+function codegen::defaults() {
+    # Build the tool.
+    hack/make-rules/build.sh k8s.io/code-generator/cmd/defaulter-gen
+
+    # The result file, in each pkg, of defaulter generation.
+    local output_base="${GENERATED_FILE_PREFIX}defaults"
+
+    # The tool used to generate defaulters.
+    local gen_defaulter_bin="${BIN_DIR}/defaulter-gen"
+
+    # All directories that request any form of defaulter generation.
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: finding all +k8s:defaulter-gen tags"
+    fi
+    local tag_dirs=()
+    kube::util::read-array tag_dirs < <( \
+        grep --color=never -l '+k8s:defaulter-gen=' "${ALL_K8S_TAG_FILES[@]}" \
+            | xargs -n1 dirname \
+            | LC_ALL=C sort -u)
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:defaulter-gen tagged dirs"
+    fi
+
+    local tag_pkgs=()
+    for dir in "${tag_dirs[@]}"; do
+        tag_pkgs+=("${PRJ_SRC_PATH}/$dir")
+    done
+
+    kube::log::status "Generating defaulter code for ${#tag_pkgs[@]} targets"
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: running ${gen_defaulter_bin} for:"
+        for dir in "${tag_dirs[@]}"; do
+            kube::log::status "DBG:     $dir"
+        done
+    fi
+
+    ./hack/run-in-gopath.sh "${gen_defaulter_bin}" \
+        --v "${KUBE_VERBOSE}" \
+        --logtostderr \
+        -h "${BOILERPLATE_FILENAME}" \
+        -O "${output_base}" \
+        $(printf -- " --extra-peer-dirs %s" "${tag_pkgs[@]}") \
+        $(printf -- " -i %s" "${tag_pkgs[@]}") \
+        "$@"
+
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "Generated defaulter code"
+    fi
+}
+
+# Conversion generation
+
+# Any package that wants conversion functions generated into it must
+# include one or more comment-tags in its `doc.go` file, of the form:
+#     // +k8s:conversion-gen=<INTERNAL_TYPES_DIR>
+#
+# The INTERNAL_TYPES_DIR is a project-local path to another directory
+# which should be considered when evaluating peer types for
+# conversions.  An optional additional comment of the form
+#     // +k8s:conversion-gen-external-types=<EXTERNAL_TYPES_DIR>
+#
+# identifies where to find the external types; if there is no such
+# comment then the external types are sought in the package where the
+# `k8s:conversion` tag is found.
+#
+# Conversions, in both directions, are generated for every type name
+# that is defined in both an internal types package and the external
+# types package.
+#
+# TODO: it might be better in the long term to make peer-types explicit in the
+# IDL.
+function codegen::conversions() {
+    # Build the tool.
+    hack/make-rules/build.sh k8s.io/code-generator/cmd/conversion-gen
+
+    # The result file, in each pkg, of conversion generation.
+    local output_base="${GENERATED_FILE_PREFIX}conversion"
+
+    # The tool used to generate conversions.
+    local gen_conversion_bin="${BIN_DIR}/conversion-gen"
+
+    # All directories that request any form of conversion generation.
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: finding all +k8s:conversion-gen tags"
+    fi
+    local tag_dirs=()
+    kube::util::read-array tag_dirs < <(\
+        grep --color=never -l '^// *+k8s:conversion-gen=' "${ALL_K8S_TAG_FILES[@]}" \
+            | xargs -n1 dirname \
+            | LC_ALL=C sort -u)
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:conversion-gen tagged dirs"
+    fi
+
+    local tag_pkgs=()
+    for dir in "${tag_dirs[@]}"; do
+        tag_pkgs+=("${PRJ_SRC_PATH}/$dir")
+    done
+
+    local extra_peer_pkgs=(
+        k8s.io/kubernetes/pkg/apis/core
+        k8s.io/kubernetes/pkg/apis/core/v1
+        k8s.io/api/core/v1
+    )
+
+    kube::log::status "Generating conversion code for ${#tag_pkgs[@]} targets"
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: running ${gen_conversion_bin} for:"
+        for dir in "${tag_dirs[@]}"; do
+            kube::log::status "DBG:     $dir"
+        done
+    fi
+
+    ./hack/run-in-gopath.sh "${gen_conversion_bin}" \
+        --v "${KUBE_VERBOSE}" \
+        --logtostderr \
+        -h "${BOILERPLATE_FILENAME}" \
+        -O "${output_base}" \
+        $(printf -- " --extra-peer-dirs %s" "${extra_peer_pkgs[@]}") \
+        $(printf -- " --extra-dirs %s" "${tag_pkgs[@]}") \
+        $(printf -- " -i %s" "${tag_pkgs[@]}") \
+        "$@"
+
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "Generated conversion code"
+    fi
+}
+
+# $@: directories to exclude
+# example:
+#    k8s_tag_files_except foo bat/qux
+function k8s_tag_files_except() {
+    for f in "${ALL_K8S_TAG_FILES[@]}"; do
+        local excl=""
+        for x in "$@"; do
+            if [[ "$f" =~ "$x"/.* ]]; then
+                excl="true"
+                break
+            fi
+        done
+        if [[ "${excl}" != true ]]; then
+            echo "$f"
+        fi
+    done
+}
+
+# $@: directories to exclude
+# example:
+#    k8s_tag_files_matching foo bat/qux
+function k8s_tag_files_matching() {
+    for f in "${ALL_K8S_TAG_FILES[@]}"; do
+        for x in "$@"; do
+            if [[ "$f" =~ "${x}"/.* ]]; then
+                echo "$f"
+                break
+            fi
+        done
+    done
+}
+
+# $1: the name of a scalar variable to read
+# example:
+#    FOO_VAR="foo value"
+#    BAR_VAR="bar value"
+#    x=FOO
+#    indirect "${x}_VAR" # -> "foo value\n"
+function indirect() {
+    # This is a trick to get bash to indirectly read a variable.
+    # Thanks StackOverflow!
+    local var="$1"
+    echo "${!var}"
+}
+
+# $1: the name of an array variable to read
+#    FOO_ARR=(a b c)
+#    BAR_ARR=(1 2 3)
+#    x=FOO
+#    indirect_array "${x}_ARR" # -> "a\nb\nc\n"
+function indirect_array() {
+    # This is a trick to get bash to indirectly read an array.
+    # Thanks StackOverflow!
+    local arrayname="$1"
+    # shellcheck disable=SC1087 # intentional
+    local tmp="$arrayname[@]"
+    printf -- "%s\n" "${!tmp}"
+}
+
+# OpenAPI generation
+#
+# Any package that wants open-api functions generated must include a
+# comment-tag in column 0 of one file of the form:
+#     // +k8s:openapi-gen=true
+function codegen::openapi() {
+    # Build the tool.
+    hack/make-rules/build.sh k8s.io/kube-openapi/cmd/openapi-gen
+
+    # The result file, in each pkg, of open-api generation.
+    local output_base="${GENERATED_FILE_PREFIX}openapi"
+
+    # The tool used to generate open apis.
+    local gen_openapi_bin="${BIN_DIR}/openapi-gen"
+
+    # Standard dirs which all targets need.
+    local apimachinery_dirs=(
+        vendor/k8s.io/apimachinery/pkg/apis/meta/v1
+        vendor/k8s.io/apimachinery/pkg/runtime
+        vendor/k8s.io/apimachinery/pkg/version
+    )
+
+    # These should probably be configured by tags in code-files somewhere.
+    local targets=(
+        KUBE
+        AGGREGATOR
+        APIEXTENSIONS
+        CODEGEN
+        SAMPLEAPISERVER
+    )
+
+    # shellcheck disable=SC2034 # used indirectly
+    local KUBE_output_dir="pkg/generated/openapi"
+    # shellcheck disable=SC2034 # used indirectly
+    local KUBE_known_violations_file="api/api-rules/violation_exceptions.list"
+    # shellcheck disable=SC2034 # used indirectly
+    local KUBE_tag_files=()
+    kube::util::read-array KUBE_tag_files < <(
+        k8s_tag_files_except \
+            vendor/k8s.io/code-generator \
+            vendor/k8s.io/sample-apiserver
+        )
+
+    # shellcheck disable=SC2034 # used indirectly
+    local AGGREGATOR_output_dir="staging/src/k8s.io/kube-aggregator/pkg/generated/openapi"
+    # shellcheck disable=SC2034 # used indirectly
+    local AGGREGATOR_known_violations_file="api/api-rules/aggregator_violation_exceptions.list"
+    # shellcheck disable=SC2034 # used indirectly
+    local AGGREGATOR_tag_files=()
+    kube::util::read-array AGGREGATOR_tag_files < <(
+        k8s_tag_files_matching \
+            vendor/k8s.io/kube-aggregator \
+            "${apimachinery_dirs[@]}"
+        )
+
+    # shellcheck disable=SC2034 # used indirectly
+    local APIEXTENSIONS_output_dir="staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi"
+    # shellcheck disable=SC2034 # used indirectly
+    local APIEXTENSIONS_known_violations_file="api/api-rules/apiextensions_violation_exceptions.list"
+    # shellcheck disable=SC2034 # used indirectly
+    local APIEXTENSIONS_tag_files=()
+    kube::util::read-array APIEXTENSIONS_tag_files < <(
+        k8s_tag_files_matching \
+            vendor/k8s.io/apiextensions \
+            vendor/k8s.io/api/autoscaling/v1 \
+            "${apimachinery_dirs[@]}"
+        )
+
+    # shellcheck disable=SC2034 # used indirectly
+    local CODEGEN_output_dir="staging/src/k8s.io/code-generator/examples/apiserver/openapi"
+    # shellcheck disable=SC2034 # used indirectly
+    local CODEGEN_known_violations_file="api/api-rules/codegen_violation_exceptions.list"
+    # shellcheck disable=SC2034 # used indirectly
+    local CODEGEN_tag_files=()
+    kube::util::read-array CODEGEN_tag_files < <(
+        k8s_tag_files_matching \
+            vendor/k8s.io/code-generator \
+            "${apimachinery_dirs[@]}"
+        )
+
+    # shellcheck disable=SC2034 # used indirectly
+    local SAMPLEAPISERVER_output_dir="staging/src/k8s.io/sample-apiserver/pkg/generated/openapi"
+    # shellcheck disable=SC2034 # used indirectly
+    local SAMPLEAPISERVER_known_violations_file="api/api-rules/sample_apiserver_violation_exceptions.list"
+    # shellcheck disable=SC2034 # used indirectly
+    local SAMPLEAPISERVER_tag_files=()
+    kube::util::read-array SAMPLEAPISERVER_tag_files < <(
+        k8s_tag_files_matching \
+            vendor/k8s.io/sample-apiserver \
+            "${apimachinery_dirs[@]}"
+        )
+
+    for prefix in "${targets[@]}"; do
+        local report_file="${OUT_DIR}/${prefix}_violations.report"
+        # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write
+        # updated API violations to the known API violation exceptions list.
+        if [[ "${UPDATE_API_KNOWN_VIOLATIONS}" == true ]]; then
+            report_file=$(indirect "${prefix}_known_violations_file")
+        fi
+
+        # 2 lines because shellcheck
+        local output_dir
+        output_dir=$(indirect "${prefix}_output_dir")
+
+        if [[ "${DBG_CODEGEN}" == 1 ]]; then
+            kube::log::status "DBG: finding all +k8s:openapi-gen tags for ${prefix}"
+        fi
+
+        local tag_dirs=()
+        kube::util::read-array tag_dirs < <(
+            grep --color=never -l '+k8s:openapi-gen=' $(indirect_array "${prefix}_tag_files") \
+                | xargs -n1 dirname \
+                | LC_ALL=C sort -u
+            )
+
+        if [[ "${DBG_CODEGEN}" == 1 ]]; then
+            kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:openapi-gen tagged dirs for ${prefix}"
+        fi
+
+        local tag_pkgs=()
+        for dir in "${tag_dirs[@]}"; do
+            tag_pkgs+=("${PRJ_SRC_PATH}/$dir")
+        done
+
+        kube::log::status "Generating openapi code for ${prefix}"
+        if [[ "${DBG_CODEGEN}" == 1 ]]; then
+            kube::log::status "DBG: running ${gen_openapi_bin} for:"
+            for dir in "${tag_dirs[@]}"; do
+                kube::log::status "DBG:     $dir"
+            done
+        fi
+
+        ./hack/run-in-gopath.sh ${gen_openapi_bin} \
+            --v "${KUBE_VERBOSE}" \
+            --logtostderr \
+            -h "${BOILERPLATE_FILENAME}" \
+            -O "${output_base}" \
+            -p "${PRJ_SRC_PATH}/${output_dir}" \
+            -r "${report_file}" \
+            $(printf -- " -i %s" "${tag_pkgs[@]}") \
+            "$@"
+
+        touch "${report_file}"
+        # 2 lines because shellcheck
+        local known_filename
+        known_filename=$(indirect "${prefix}_known_violations_file")
+        if ! diff -u "${known_filename}" "${report_file}"; then
+            echo -e "ERROR:"
+            echo -e "\t'${prefix}' API rule check failed - reported violations differ from known violations"
+            echo -e "\tPlease read api/api-rules/README.md to resolve the failure in ${known_filename}"
+        fi
+
+        if [[ "${DBG_CODEGEN}" == 1 ]]; then
+            kube::log::status "Generated openapi code"
+        fi
+    done # for each prefix
+}
+
+function codegen::applyconfigs() {
+    GO111MODULE=on GOPROXY=off go install \
+        k8s.io/kubernetes/pkg/generated/openapi/cmd/models-schema \
+        k8s.io/code-generator/cmd/applyconfiguration-gen
+
+    local modelsschema
     modelsschema=$(kube::util::find-binary "models-schema")
+    local applyconfigurationgen
     applyconfigurationgen=$(kube::util::find-binary "applyconfiguration-gen")
+
+    # because client-gen doesn't do policy/v1alpha1, we have to skip it too
+    local ext_apis=()
+    kube::util::read-array ext_apis < <(
+      cd "${KUBE_ROOT}/staging/src"
+      find k8s.io/api -name types.go -print0 | xargs -0 -n1 dirname | sort | grep -v pkg.apis.policy.v1alpha1
+    )
+    ext_apis+=("k8s.io/apimachinery/pkg/apis/meta/v1")
+
+    kube::log::status "Generating apply-config code for ${#ext_apis[@]} targets"
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: running ${applyconfigurationgen} for:"
+        for api in "${ext_apis[@]}"; do
+            kube::log::status "DBG:     $api"
+        done
+    fi
+
+    "${applyconfigurationgen}" \
+        --openapi-schema <("${modelsschema}") \
+        --go-header-file "${BOILERPLATE_FILENAME}" \
+        --output-base "${KUBE_ROOT}/vendor" \
+        --output-package "${APPLYCONFIG_PKG}" \
+        $(printf -- " --input-dirs %s" "${ext_apis[@]}") \
+        "$@"
+
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "Generated apply-config code"
+    fi
+}
+
+function codegen::clients() {
+    GO111MODULE=on GOPROXY=off go install \
+        k8s.io/code-generator/cmd/client-gen
+
+    local clientgen
     clientgen=$(kube::util::find-binary "client-gen")
-listergen=$(kube::util::find-binary "lister-gen")
-informergen=$(kube::util::find-binary "informer-gen")
 
-IFS=" " read -r -a GROUP_VERSIONS <<< "${KUBE_AVAILABLE_GROUP_VERSIONS}"
+    IFS=" " read -r -a group_versions <<< "${KUBE_AVAILABLE_GROUP_VERSIONS}"
-GV_DIRS=()
+    local gv_dirs=()
-for gv in "${GROUP_VERSIONS[@]}"; do
+    for gv in "${group_versions[@]}"; do
         # add items, but strip off any leading apis/ you find to match command expectations
+        local api_dir
         api_dir=$(kube::util::group-version-to-pkg-path "${gv}")
-  nopkg_dir=${api_dir#pkg/}
+        local nopkg_dir=${api_dir#pkg/}
         nopkg_dir=${nopkg_dir#vendor/k8s.io/api/}
-  pkg_dir=${nopkg_dir#apis/}
+        local pkg_dir=${nopkg_dir#apis/}
-
 
         # skip groups that aren't being served, clients for these don't matter
         if [[ " ${KUBE_NONSERVER_GROUP_VERSIONS} " == *" ${gv} "* ]]; then
           continue
         fi
 
-  GV_DIRS+=("${pkg_dir}")
+        gv_dirs+=("${pkg_dir}")
     done
-# delimit by commas for the command
-GV_DIRS_CSV=$(IFS=',';echo "${GV_DIRS[*]// /,}";IFS=$)
 
-applyconfigurationgen_external_apis=()
+    kube::log::status "Generating client code for ${#gv_dirs[@]} targets"
-# because client-gen doesn't do policy/v1alpha1, we have to skip it too
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
-kube::util::read-array applyconfigurationgen_external_apis < <(
+        kube::log::status "DBG: running ${clientgen} for:"
-  cd "${KUBE_ROOT}/staging/src"
+        for dir in "${gv_dirs[@]}"; do
-  find k8s.io/api -name types.go -print0 | xargs -0 -n1 dirname | sort | grep -v pkg.apis.policy.v1alpha1
+            kube::log::status "DBG:     $dir"
-)
+        done
-applyconfigurationgen_external_apis+=("k8s.io/apimachinery/pkg/apis/meta/v1")
+    fi
-applyconfigurationgen_external_apis_csv=$(IFS=,; echo "${applyconfigurationgen_external_apis[*]}")
-applyconfigurations_package="k8s.io/client-go/applyconfigurations"
-${applyconfigurationgen} \
-  --openapi-schema <(${modelsschema}) \
-  --output-base "${KUBE_ROOT}/vendor" \
-  --output-package "${applyconfigurations_package}" \
-  --input-dirs "${applyconfigurationgen_external_apis_csv}" \
-  --go-header-file "${KUBE_ROOT}/hack/boilerplate/boilerplate.generatego.txt" \
-  "$@"
 
-# This can be called with one flag, --verify-only, so it works for both the
+    "${clientgen}" \
-# update- and verify- scripts.
+        --go-header-file "${BOILERPLATE_FILENAME}" \
-${clientgen} \
         --output-base "${KUBE_ROOT}/vendor" \
         --output-package="k8s.io/client-go" \
         --clientset-name="kubernetes" \
         --input-base="k8s.io/api" \
-  --input="${GV_DIRS_CSV}" \
+        --apply-configuration-package "${APPLYCONFIG_PKG}" \
-  --apply-configuration-package "${applyconfigurations_package}" \
+        $(printf -- " --input %s" "${gv_dirs[@]}") \
-  --go-header-file "${KUBE_ROOT}/hack/boilerplate/boilerplate.generatego.txt" \
         "$@"
 
-listergen_external_apis=()
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
-kube::util::read-array listergen_external_apis < <(
+        kube::log::status "Generated client code"
+    fi
+}
+
+function codegen::listers() {
+    GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/lister-gen
+
+    local listergen
+    listergen=$(kube::util::find-binary "lister-gen")
+
+    local ext_apis=()
+    kube::util::read-array ext_apis < <(
         cd "${KUBE_ROOT}/staging/src"
         find k8s.io/api -name types.go -print0 | xargs -0 -n1 dirname | sort
     )
-listergen_external_apis_csv=$(IFS=,; echo "${listergen_external_apis[*]}")
+
-${listergen} \
+    kube::log::status "Generating lister code for ${#ext_apis[@]} targets"
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: running ${listergen} for:"
+        for api in "${ext_apis[@]}"; do
+            kube::log::status "DBG:     $api"
+        done
+    fi
+
+    "${listergen}" \
+        --go-header-file "${BOILERPLATE_FILENAME}" \
         --output-base "${KUBE_ROOT}/vendor" \
         --output-package "k8s.io/client-go/listers" \
-  --input-dirs "${listergen_external_apis_csv}" \
+        $(printf -- " --input-dirs %s" "${ext_apis[@]}") \
-  --go-header-file "${KUBE_ROOT}/hack/boilerplate/boilerplate.generatego.txt" \
         "$@"
 
-informergen_external_apis=()
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "Generated lister code"
+    fi
+}
+
+function codegen::informers() {
+    GO111MODULE=on GOPROXY=off go install \
+        k8s.io/code-generator/cmd/informer-gen
+
+    local informergen
+    informergen=$(kube::util::find-binary "informer-gen")
+
     # because client-gen doesn't do policy/v1alpha1, we have to skip it too
-kube::util::read-array informergen_external_apis < <(
+    local ext_apis=()
+    kube::util::read-array ext_apis < <(
         cd "${KUBE_ROOT}/staging/src"
         find k8s.io/api -name types.go -print0 | xargs -0 -n1 dirname | sort | grep -v pkg.apis.policy.v1alpha1
     )
-informergen_external_apis_csv=$(IFS=,; echo "${informergen_external_apis[*]}")
 
-${informergen} \
+    kube::log::status "Generating informer code for ${#ext_apis[@]} targets"
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: running ${informergen} for:"
+        for api in "${ext_apis[@]}"; do
+            kube::log::status "DBG:     $api"
+        done
+    fi
+
+    "${informergen}" \
+        --go-header-file "${BOILERPLATE_FILENAME}" \
         --output-base "${KUBE_ROOT}/vendor" \
         --output-package "k8s.io/client-go/informers" \
         --single-directory \
-  --input-dirs "${informergen_external_apis_csv}" \
         --versioned-clientset-package k8s.io/client-go/kubernetes \
         --listers-package k8s.io/client-go/listers \
-  --go-header-file "${KUBE_ROOT}/hack/boilerplate/boilerplate.generatego.txt" \
+        $(printf -- " --input-dirs %s" "${ext_apis[@]}") \
         "$@"
 
-# You may add additional calls of code generators like set-gen above.
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "Generated informer code"
+    fi
+}
 
-# call generation on sub-project for now
+function codegen::subprojects() {
-CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/code-generator/hack/update-codegen.sh
+    # Call generation on sub-projects.
-CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/kube-aggregator/hack/update-codegen.sh
+    # TODO(thockin): make these take a list of codegens and flags
-CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/sample-apiserver/hack/update-codegen.sh
+    local subs=(
-CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/sample-controller/hack/update-codegen.sh
+        vendor/k8s.io/code-generator/hack/update-codegen.sh
-CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/apiextensions-apiserver/hack/update-codegen.sh
+        vendor/k8s.io/kube-aggregator/hack/update-codegen.sh
-CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/metrics/hack/update-codegen.sh
+        vendor/k8s.io/sample-apiserver/hack/update-codegen.sh
-CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/apiextensions-apiserver/examples/client-go/hack/update-codegen.sh
+        vendor/k8s.io/sample-controller/hack/update-codegen.sh
+        vendor/k8s.io/apiextensions-apiserver/hack/update-codegen.sh
+        vendor/k8s.io/metrics/hack/update-codegen.sh
+        vendor/k8s.io/apiextensions-apiserver/examples/client-go/hack/update-codegen.sh
+    )
+
+    for s in "${subs[@]}"; do 
+        CODEGEN_PKG=./vendor/k8s.io/code-generator "$s"
+    done
+}
+
+
+#
+# main
+#
+
+function list_codegens() {
+    (
+        shopt -s extdebug
+        declare -F \
+            | cut -f3 -d' ' \
+            | grep "^codegen::" \
+            | while read -r fn; do declare -F "$fn"; done \
+            | sort -n -k2 \
+            | cut -f1 -d' ' \
+            | sed 's/^codegen:://'
+    )
+}
+
+# shellcheck disable=SC2207 # safe, no functions have spaces
+all_codegens=($(list_codegens))
+
+function print_codegens() {
+    echo "available codegens:"
+    for g in "${all_codegens[@]}"; do
+        echo "    $g"
+    done
+}
+
+# Validate and accumulate flags to pass thru and codegens to run if args are
+# specified.
+flags_to_pass=()
+codegens_to_run=()
+for arg; do
+    # Use -? to list known codegens.
+    if [[ "${arg}" == "-?" ]]; then
+        print_codegens
+        exit 0
+    fi
+    if [[ "${arg}" =~ ^- ]]; then
+        flags_to_pass+=("${arg}")
+        continue
+    fi
+    # Make sure each non-flag arg matches at least one codegen.
+    nmatches=0
+    for t in "${all_codegens[@]}"; do
+        if [[ "$t" =~ ${arg} ]]; then
+            nmatches=$((nmatches+1))
+            # Don't run codegens twice, just keep the first match.
+            # shellcheck disable=SC2076 # we want literal matching
+            if [[ " ${codegens_to_run[*]} " =~ " $t " ]]; then
+                continue
+            fi
+            codegens_to_run+=("$t")
+            continue
+        fi
+    done
+    if [[ ${nmatches} == 0 ]]; then
+        echo "ERROR: no codegens match pattern '${arg}'"
+        echo
+        print_codegens
+        exit 1
+    fi
+    codegens_to_run+=("${matches[@]}")
+done
+
+# If no codegens were specified, run them all.
+if [[ "${#codegens_to_run[@]}" == 0 ]]; then
+    codegens_to_run=("${all_codegens[@]}")
+fi
+
+for g in "${codegens_to_run[@]}"; do
+    "codegen::${g}" "${flags_to_pass[@]}"
+done

hack/verify-codegen.sh

@@ -32,11 +32,15 @@ kube::golang::setup_env
 #
 # Note: these must be before the main script call because the later calls the sub-project's
 #       update-codegen.sh scripts. We wouldn't see any error on changes then.
-CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/code-generator/hack/verify-codegen.sh
+export CODEGEN_PKG=./vendor/k8s.io/code-generator
-CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/kube-aggregator/hack/verify-codegen.sh
+vendor/k8s.io/code-generator/hack/verify-codegen.sh
-CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/sample-apiserver/hack/verify-codegen.sh
+vendor/k8s.io/kube-aggregator/hack/verify-codegen.sh
-CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/sample-controller/hack/verify-codegen.sh
+vendor/k8s.io/sample-apiserver/hack/verify-codegen.sh
-CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/apiextensions-apiserver/hack/verify-codegen.sh
+vendor/k8s.io/sample-controller/hack/verify-codegen.sh
-CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/metrics/hack/verify-codegen.sh
+vendor/k8s.io/apiextensions-apiserver/hack/verify-codegen.sh
+vendor/k8s.io/metrics/hack/verify-codegen.sh
 
-"${KUBE_ROOT}/hack/update-codegen.sh" --verify-only
+# This won't actually update anything because of --verify-only, but it tells
+# the openapi tool to verify against the real filenames.
+export UPDATE_API_KNOWN_VIOLATIONS=true
+"${KUBE_ROOT}/hack/update-codegen.sh" --verify-only "$@"