Merge pull request #117899 from thockin/codegen_purge_openapi_shell_indirection

Simpler openapi gen - subprojects do themselves

hack/update-codegen.sh

@@ -32,6 +32,7 @@ kube::golang::setup_env
 DBG_CODEGEN="${DBG_CODEGEN:-0}"
 GENERATED_FILE_PREFIX="${GENERATED_FILE_PREFIX:-zz_generated.}"
 UPDATE_API_KNOWN_VIOLATIONS="${UPDATE_API_KNOWN_VIOLATIONS:-}"
+API_KNOWN_VIOLATIONS_DIR="${API_KNOWN_VIOLATIONS_DIR:-"${KUBE_ROOT}/api/api-rules"}"
 
 OUT_DIR="_output"
 PRJ_SRC_PATH="k8s.io/kubernetes"
@@ -497,47 +498,6 @@ function k8s_tag_files_except() {
     done
 }
 
-# $@: directories to exclude
-# example:
-#    k8s_tag_files_matching foo bat/qux
-function k8s_tag_files_matching() {
-    for f in "${ALL_K8S_TAG_FILES[@]}"; do
-        for x in "$@"; do
-            if [[ "$f" =~ "${x}"/.* ]]; then
-                echo "$f"
-                break
-            fi
-        done
-    done
-}
-
-# $1: the name of a scalar variable to read
-# example:
-#    FOO_VAR="foo value"
-#    BAR_VAR="bar value"
-#    x=FOO
-#    indirect "${x}_VAR" # -> "foo value\n"
-function indirect() {
-    # This is a trick to get bash to indirectly read a variable.
-    # Thanks StackOverflow!
-    local var="$1"
-    echo "${!var}"
-}
-
-# $1: the name of an array variable to read
-#    FOO_ARR=(a b c)
-#    BAR_ARR=(1 2 3)
-#    x=FOO
-#    indirect_array "${x}_ARR" # -> "a\nb\nc\n"
-function indirect_array() {
-    # This is a trick to get bash to indirectly read an array.
-    # Thanks StackOverflow!
-    local arrayname="$1"
-    # shellcheck disable=SC1087 # intentional
-    local tmp="$arrayname[@]"
-    printf -- "%s\n" "${!tmp}"
-}
-
 # OpenAPI generation
 #
 # Any package that wants open-api functions generated must include a
@@ -555,148 +515,73 @@ function codegen::openapi() {
     local gen_openapi_bin
     gen_openapi_bin="$(kube::util::find-binary "openapi-gen")"
 
-    # Standard dirs which all targets need.
+    local output_dir="pkg/generated/openapi"
-    local apimachinery_dirs=(
+    local known_violations_file="${API_KNOWN_VIOLATIONS_DIR}/violation_exceptions.list"
-        vendor/k8s.io/apimachinery/pkg/apis/meta/v1
-        vendor/k8s.io/apimachinery/pkg/runtime
-        vendor/k8s.io/apimachinery/pkg/version
-    )
 
-    # These should probably be configured by tags in code-files somewhere.
+    local report_file="${OUT_DIR}/api_violations.report"
-    local targets=(
+    # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write
-        KUBE
+    # updated API violations to the known API violation exceptions list.
-        AGGREGATOR
+    if [[ "${UPDATE_API_KNOWN_VIOLATIONS}" == true ]]; then
-        APIEXTENSIONS
+        report_file="${known_violations_file}"
-        CODEGEN
+    fi
-        SAMPLEAPISERVER
-    )
 
-    # shellcheck disable=SC2034 # used indirectly
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
-    local KUBE_output_dir="pkg/generated/openapi"
+        kube::log::status "DBG: finding all +k8s:openapi-gen tags"
-    # shellcheck disable=SC2034 # used indirectly
+    fi
-    local KUBE_known_violations_file="api/api-rules/violation_exceptions.list"
+
-    # shellcheck disable=SC2034 # used indirectly
+    local tag_files=()
-    local KUBE_tag_files=()
+    kube::util::read-array tag_files < <(
-    kube::util::read-array KUBE_tag_files < <(
         k8s_tag_files_except \
             vendor/k8s.io/code-generator \
             vendor/k8s.io/sample-apiserver
         )
 
-    # shellcheck disable=SC2034 # used indirectly
+    local tag_dirs=()
-    local AGGREGATOR_output_dir="staging/src/k8s.io/kube-aggregator/pkg/generated/openapi"
+    kube::util::read-array tag_dirs < <(
-    # shellcheck disable=SC2034 # used indirectly
+        grep -l --null '+k8s:openapi-gen=' "${tag_files[@]}" \
-    local AGGREGATOR_known_violations_file="api/api-rules/aggregator_violation_exceptions.list"
+            | xargs -0 -n1 dirname \
-    # shellcheck disable=SC2034 # used indirectly
+            | sort -u)
-    local AGGREGATOR_tag_files=()
-    kube::util::read-array AGGREGATOR_tag_files < <(
-        k8s_tag_files_matching \
-            vendor/k8s.io/kube-aggregator \
-            "${apimachinery_dirs[@]}"
-        )
 
-    # shellcheck disable=SC2034 # used indirectly
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
-    local APIEXTENSIONS_output_dir="staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi"
+        kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:openapi-gen tagged dirs"
-    # shellcheck disable=SC2034 # used indirectly
+    fi
-    local APIEXTENSIONS_known_violations_file="api/api-rules/apiextensions_violation_exceptions.list"
-    # shellcheck disable=SC2034 # used indirectly
-    local APIEXTENSIONS_tag_files=()
-    kube::util::read-array APIEXTENSIONS_tag_files < <(
-        k8s_tag_files_matching \
-            vendor/k8s.io/apiextensions-apiserver \
-            vendor/k8s.io/api/autoscaling/v1 \
-            "${apimachinery_dirs[@]}"
-        )
 
-    # shellcheck disable=SC2034 # used indirectly
+    local tag_pkgs=()
-    local CODEGEN_output_dir="staging/src/k8s.io/code-generator/examples/apiserver/openapi"
+    for dir in "${tag_dirs[@]}"; do
-    # shellcheck disable=SC2034 # used indirectly
+        tag_pkgs+=("${PRJ_SRC_PATH}/$dir")
-    local CODEGEN_known_violations_file="api/api-rules/codegen_violation_exceptions.list"
+    done
-    # shellcheck disable=SC2034 # used indirectly
-    local CODEGEN_tag_files=()
-    kube::util::read-array CODEGEN_tag_files < <(
-        k8s_tag_files_matching \
-            vendor/k8s.io/code-generator \
-            "${apimachinery_dirs[@]}"
-        )
 
-    # shellcheck disable=SC2034 # used indirectly
+    kube::log::status "Generating openapi code"
-    local SAMPLEAPISERVER_output_dir="staging/src/k8s.io/sample-apiserver/pkg/generated/openapi"
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
-    # shellcheck disable=SC2034 # used indirectly
+        kube::log::status "DBG: running ${gen_openapi_bin} for:"
-    local SAMPLEAPISERVER_known_violations_file="api/api-rules/sample_apiserver_violation_exceptions.list"
-    # shellcheck disable=SC2034 # used indirectly
-    local SAMPLEAPISERVER_tag_files=()
-    kube::util::read-array SAMPLEAPISERVER_tag_files < <(
-        k8s_tag_files_matching \
-            vendor/k8s.io/sample-apiserver \
-            "${apimachinery_dirs[@]}"
-        )
-
-    git_find -z ':(glob)**'/"${output_base}.go" | xargs -0 rm -f
-
-    for prefix in "${targets[@]}"; do
-        local report_file="${OUT_DIR}/${prefix}_violations.report"
-        # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write
-        # updated API violations to the known API violation exceptions list.
-        if [[ "${UPDATE_API_KNOWN_VIOLATIONS}" == true ]]; then
-            report_file=$(indirect "${prefix}_known_violations_file")
-        fi
-
-        # 2 lines because shellcheck
-        local output_dir
-        output_dir=$(indirect "${prefix}_output_dir")
-
-        if [[ "${DBG_CODEGEN}" == 1 ]]; then
-            kube::log::status "DBG: finding all +k8s:openapi-gen tags for ${prefix}"
-        fi
-
-        local tag_dirs=()
-        kube::util::read-array tag_dirs < <(
-            grep -l --null '+k8s:openapi-gen=' $(indirect_array "${prefix}_tag_files") \
-                | xargs -0 -n1 dirname \
-                | sort -u)
-
-        if [[ "${DBG_CODEGEN}" == 1 ]]; then
-            kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:openapi-gen tagged dirs for ${prefix}"
-        fi
-
-        local tag_pkgs=()
         for dir in "${tag_dirs[@]}"; do
-            tag_pkgs+=("${PRJ_SRC_PATH}/$dir")
+            kube::log::status "DBG:     $dir"
         done
+    fi
 
-        kube::log::status "Generating openapi code for ${prefix}"
+    git_find -z ':(glob)pkg/generated/**'/"${output_base}.go" | xargs -0 rm -f
-        if [[ "${DBG_CODEGEN}" == 1 ]]; then
-            kube::log::status "DBG: running ${gen_openapi_bin} for:"
-            for dir in "${tag_dirs[@]}"; do
-                kube::log::status "DBG:     $dir"
-            done
-        fi
 
-        ./hack/run-in-gopath.sh "${gen_openapi_bin}" \
+    ./hack/run-in-gopath.sh "${gen_openapi_bin}" \
-            --v "${KUBE_VERBOSE}" \
+        --v "${KUBE_VERBOSE}" \
-            --logtostderr \
+        --logtostderr \
-            -h "${BOILERPLATE_FILENAME}" \
+        -h "${BOILERPLATE_FILENAME}" \
-            -O "${output_base}" \
+        -O "${output_base}" \
-            -p "${PRJ_SRC_PATH}/${output_dir}" \
+        -p "${PRJ_SRC_PATH}/${output_dir}" \
-            -r "${report_file}" \
+        -r "${report_file}" \
-            $(printf -- " -i %s" "${tag_pkgs[@]}") \
+        $(printf -- " -i %s" "${tag_pkgs[@]}") \
-            "$@"
+        "$@"
 
-        touch "${report_file}"
+    touch "${report_file}"
-        # 2 lines because shellcheck
+    local known_filename="${known_violations_file}"
-        local known_filename
+    if ! diff -u "${known_filename}" "${report_file}"; then
-        known_filename=$(indirect "${prefix}_known_violations_file")
+        echo -e "ERROR:"
-        if ! diff -u "${known_filename}" "${report_file}"; then
+        echo -e "\tAPI rule check failed - reported violations differ from known violations"
-            echo -e "ERROR:"
+        echo -e "\tPlease read api/api-rules/README.md to resolve the failure in ${known_filename}"
-            echo -e "\t'${prefix}' API rule check failed - reported violations differ from known violations"
+    fi
-            echo -e "\tPlease read api/api-rules/README.md to resolve the failure in ${known_filename}"
-        fi
 
-        if [[ "${DBG_CODEGEN}" == 1 ]]; then
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
-            kube::log::status "Generated openapi code"
+        kube::log::status "Generated openapi code"
-        fi
+    fi
-    done # for each prefix
 }
 
 function codegen::applyconfigs() {
@@ -898,8 +783,8 @@ function codegen::subprojects() {
         vendor/k8s.io/kube-aggregator
         vendor/k8s.io/sample-apiserver
         vendor/k8s.io/sample-controller
-        vendor/k8s.io/apiextensions-apiserver
         vendor/k8s.io/metrics
+        vendor/k8s.io/apiextensions-apiserver
         vendor/k8s.io/apiextensions-apiserver/examples/client-go
     )
 
@@ -908,7 +793,10 @@ function codegen::subprojects() {
     for sub in "${subs[@]}"; do
         kube::log::status "Generating code for subproject ${sub}"
         pushd "${sub}" >/dev/null
-        CODEGEN_PKG="${codegen}" ./hack/update-codegen.sh > >(indent) 2> >(indent >&2)
+        CODEGEN_PKG="${codegen}" \
+        UPDATE_API_KNOWN_VIOLATIONS="${UPDATE_API_KNOWN_VIOLATIONS}" \
+        API_KNOWN_VIOLATIONS_DIR="${API_KNOWN_VIOLATIONS_DIR}" \
+            ./hack/update-codegen.sh > >(indent) 2> >(indent >&2)
         popd >/dev/null
     done
 }

staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh

@@ -28,6 +28,22 @@ kube::codegen::gen_helpers \
     --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
     --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
 
+if [[ -n "${API_KNOWN_VIOLATIONS_DIR:-}" ]]; then
+    report_filename="${API_KNOWN_VIOLATIONS_DIR}/apiextensions_violation_exceptions.list"
+    if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then
+        update_report="--update-report"
+    fi
+fi
+
+kube::codegen::gen_openapi \
+    --input-pkg-root k8s.io/apiextensions-apiserver/pkg \
+    --extra-pkgs k8s.io/api/autoscaling/v1 `# needed for Scale type` \
+    --output-pkg-root k8s.io/apiextensions-apiserver/pkg/generated \
+    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
+    --report-filename "${report_filename:-"/dev/null"}" \
+    ${update_report:+"${update_report}"} \
+    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
+
 kube::codegen::gen_client \
     --with-watch \
     --input-pkg-root k8s.io/apiextensions-apiserver/pkg/apis \

staging/src/k8s.io/code-generator/examples/hack/update-codegen.sh

@@ -34,10 +34,19 @@ kube::codegen::gen_helpers \
     --output-base "$(dirname "${BASH_SOURCE[0]}")/../../../.." \
     --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
 
+if [[ -n "${API_KNOWN_VIOLATIONS_DIR:-}" ]]; then
+    report_filename="${API_KNOWN_VIOLATIONS_DIR}/codegen_violation_exceptions.list"
+    if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then
+        update_report="--update-report"
+    fi
+fi
+
 kube::codegen::gen_openapi \
     --input-pkg-root k8s.io/code-generator/examples/apiserver/apis \
     --output-pkg-root k8s.io/code-generator/examples/apiserver \
     --output-base "$(dirname "${BASH_SOURCE[0]}")/../../../.." \
+    --report-filename "${report_filename:-"/dev/null"}" \
+    ${update_report:+"${update_report}"} \
     --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
 
 kube::codegen::gen_client \

staging/src/k8s.io/code-generator/kube_codegen.sh

@@ -237,6 +237,9 @@ function kube::codegen::gen_helpers() {
 #     An optional path at which to write an API violations report.  "-" means
 #     stdout.
 #
+#   --update-report
+#     If specified, update the report file in place, rather than diffing it.
+#
 #   --boilerplate <string = path_to_kube_codegen_boilerplate>
 #     An optional override for the header file to insert into generated files.
 #
@@ -247,10 +250,12 @@ function kube::codegen::gen_openapi() {
     local openapi_subdir="openapi"
     local extra_pkgs=()
     local report="/dev/null"
+    local update_report=""
     local boilerplate="${KUBE_CODEGEN_ROOT}/hack/boilerplate.go.txt"
     local v="${KUBE_VERBOSE:-0}"
 
     while [ "$#" -gt 0 ]; do
+        echo "$1"
         case "$1" in
             "--input-pkg-root")
                 in_pkg_root="$2"
@@ -276,6 +281,10 @@ function kube::codegen::gen_openapi() {
                 report="$2"
                 shift 2
                 ;;
+            "--update-report")
+                update_report="true"
+                shift
+                ;;
             "--boilerplate")
                 boilerplate="$2"
                 shift 2
@@ -300,6 +309,13 @@ function kube::codegen::gen_openapi() {
         return 1
     fi
 
+    set -x
+    local new_report
+    new_report="$(mktemp -t "$(basename "$0").api_violations.XXXXXX")"
+    if [ -n "${update_report}" ]; then
+        new_report="${report}"
+    fi
+
     (
         # To support running this from anywhere, first cd into this directory,
         # and then install with forced module mode on and fully qualified name.
@@ -348,12 +364,20 @@ function kube::codegen::gen_openapi() {
             --go-header-file "${boilerplate}" \
             --output-base "${out_base}" \
             --output-package "${out_pkg_root}/${openapi_subdir}" \
-            --report-filename "${report}" \
+            --report-filename "${new_report}" \
             --input-dirs "k8s.io/apimachinery/pkg/apis/meta/v1" \
             --input-dirs "k8s.io/apimachinery/pkg/runtime" \
             --input-dirs "k8s.io/apimachinery/pkg/version" \
             "${inputs[@]}"
     fi
+
+    touch "${report}" # in case it doesn't exist yet
+    if ! diff -u "${report}" "${new_report}"; then
+        echo -e "ERROR:"
+        echo -e "\tAPI rule check failed for ${report}: new reported violations"
+        echo -e "\tPlease read api/api-rules/README.md"
+        return 1
+    fi
 }
 
 # Generate client code

staging/src/k8s.io/kube-aggregator/hack/update-codegen.sh

@@ -28,6 +28,21 @@ kube::codegen::gen_helpers \
     --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
     --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
 
+if [[ -n "${API_KNOWN_VIOLATIONS_DIR:-}" ]]; then
+    report_filename="${API_KNOWN_VIOLATIONS_DIR}/aggregator_violation_exceptions.list"
+    if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then
+        update_report="--update-report"
+    fi
+fi
+
+kube::codegen::gen_openapi \
+    --input-pkg-root k8s.io/kube-aggregator/pkg/apis \
+    --output-pkg-root k8s.io/kube-aggregator/pkg/generated \
+    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
+    --report-filename "${report_filename:-"/dev/null"}" \
+    ${update_report:+"${update_report}"} \
+    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
+
 kube::codegen::gen_client \
     --with-watch \
     --input-pkg-root k8s.io/kube-aggregator/pkg/apis \

staging/src/k8s.io/sample-apiserver/hack/update-codegen.sh

@@ -33,10 +33,19 @@ kube::codegen::gen_helpers \
     --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
     --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
 
+if [[ -n "${API_KNOWN_VIOLATIONS_DIR:-}" ]]; then
+    report_filename="${API_KNOWN_VIOLATIONS_DIR}/sample_apiserver_violation_exceptions.list"
+    if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then
+        update_report="--update-report"
+    fi
+fi
+
 kube::codegen::gen_openapi \
     --input-pkg-root k8s.io/sample-apiserver/pkg/apis \
     --output-pkg-root k8s.io/sample-apiserver/pkg/generated \
     --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
+    --report-filename "${report_filename:-"/dev/null"}" \
+    ${update_report:+"${update_report}"} \
     --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
 
 kube::codegen::gen_client \