Merge pull request #117899 from thockin/codegen_purge_openapi_shell_indirection

Simpler openapi gen - subprojects do themselves
2025-08-09 12:07:47 +00:00 · 2023-05-10 09:35:07 -07:00 · 2023-05-10 09:35:07 -07:00 · f319dab8d9
commit f319dab8d9
parent 7aea5a9b25 6607834207
6 changed files with 131 additions and 170 deletions
--- a/hack/update-codegen.sh
+++ b/hack/update-codegen.sh
@ -32,6 +32,7 @@ kube::golang::setup_env
 DBG_CODEGEN="${DBG_CODEGEN:-0}"
 GENERATED_FILE_PREFIX="${GENERATED_FILE_PREFIX:-zz_generated.}"
 UPDATE_API_KNOWN_VIOLATIONS="${UPDATE_API_KNOWN_VIOLATIONS:-}"
+API_KNOWN_VIOLATIONS_DIR="${API_KNOWN_VIOLATIONS_DIR:-"${KUBE_ROOT}/api/api-rules"}"

 OUT_DIR="_output"
 PRJ_SRC_PATH="k8s.io/kubernetes"
@ -497,47 +498,6 @@ function k8s_tag_files_except() {
    done
 }

-# $@: directories to exclude
-# example:
-#    k8s_tag_files_matching foo bat/qux
-function k8s_tag_files_matching() {
-    for f in "${ALL_K8S_TAG_FILES[@]}"; do
-        for x in "$@"; do
-            if [[ "$f" =~ "${x}"/.* ]]; then
-                echo "$f"
-                break
-            fi
-        done
-    done
-}
-
-# $1: the name of a scalar variable to read
-# example:
-#    FOO_VAR="foo value"
-#    BAR_VAR="bar value"
-#    x=FOO
-#    indirect "${x}_VAR" # -> "foo value\n"
-function indirect() {
-    # This is a trick to get bash to indirectly read a variable.
-    # Thanks StackOverflow!
-    local var="$1"
-    echo "${!var}"
-}
-
-# $1: the name of an array variable to read
-#    FOO_ARR=(a b c)
-#    BAR_ARR=(1 2 3)
-#    x=FOO
-#    indirect_array "${x}_ARR" # -> "a\nb\nc\n"
-function indirect_array() {
-    # This is a trick to get bash to indirectly read an array.
-    # Thanks StackOverflow!
-    local arrayname="$1"
-    # shellcheck disable=SC1087 # intentional
-    local tmp="$arrayname[@]"
-    printf -- "%s\n" "${!tmp}"
-}
-
 # OpenAPI generation
 #
 # Any package that wants open-api functions generated must include a
@ -555,109 +515,35 @@ function codegen::openapi() {
    local gen_openapi_bin
    gen_openapi_bin="$(kube::util::find-binary "openapi-gen")"

-    # Standard dirs which all targets need.
-    local apimachinery_dirs=(
-        vendor/k8s.io/apimachinery/pkg/apis/meta/v1
-        vendor/k8s.io/apimachinery/pkg/runtime
-        vendor/k8s.io/apimachinery/pkg/version
-    )
+    local output_dir="pkg/generated/openapi"
+    local known_violations_file="${API_KNOWN_VIOLATIONS_DIR}/violation_exceptions.list"

-    # These should probably be configured by tags in code-files somewhere.
-    local targets=(
-        KUBE
-        AGGREGATOR
-        APIEXTENSIONS
-        CODEGEN
-        SAMPLEAPISERVER
-    )
+    local report_file="${OUT_DIR}/api_violations.report"
+    # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write
+    # updated API violations to the known API violation exceptions list.
+    if [[ "${UPDATE_API_KNOWN_VIOLATIONS}" == true ]]; then
+        report_file="${known_violations_file}"
+    fi

-    # shellcheck disable=SC2034 # used indirectly
-    local KUBE_output_dir="pkg/generated/openapi"
-    # shellcheck disable=SC2034 # used indirectly
-    local KUBE_known_violations_file="api/api-rules/violation_exceptions.list"
-    # shellcheck disable=SC2034 # used indirectly
-    local KUBE_tag_files=()
-    kube::util::read-array KUBE_tag_files < <(
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: finding all +k8s:openapi-gen tags"
+    fi
+
+    local tag_files=()
+    kube::util::read-array tag_files < <(
        k8s_tag_files_except \
            vendor/k8s.io/code-generator \
            vendor/k8s.io/sample-apiserver
        )

-    # shellcheck disable=SC2034 # used indirectly
-    local AGGREGATOR_output_dir="staging/src/k8s.io/kube-aggregator/pkg/generated/openapi"
-    # shellcheck disable=SC2034 # used indirectly
-    local AGGREGATOR_known_violations_file="api/api-rules/aggregator_violation_exceptions.list"
-    # shellcheck disable=SC2034 # used indirectly
-    local AGGREGATOR_tag_files=()
-    kube::util::read-array AGGREGATOR_tag_files < <(
-        k8s_tag_files_matching \
-            vendor/k8s.io/kube-aggregator \
-            "${apimachinery_dirs[@]}"
-        )
-
-    # shellcheck disable=SC2034 # used indirectly
-    local APIEXTENSIONS_output_dir="staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi"
-    # shellcheck disable=SC2034 # used indirectly
-    local APIEXTENSIONS_known_violations_file="api/api-rules/apiextensions_violation_exceptions.list"
-    # shellcheck disable=SC2034 # used indirectly
-    local APIEXTENSIONS_tag_files=()
-    kube::util::read-array APIEXTENSIONS_tag_files < <(
-        k8s_tag_files_matching \
-            vendor/k8s.io/apiextensions-apiserver \
-            vendor/k8s.io/api/autoscaling/v1 \
-            "${apimachinery_dirs[@]}"
-        )
-
-    # shellcheck disable=SC2034 # used indirectly
-    local CODEGEN_output_dir="staging/src/k8s.io/code-generator/examples/apiserver/openapi"
-    # shellcheck disable=SC2034 # used indirectly
-    local CODEGEN_known_violations_file="api/api-rules/codegen_violation_exceptions.list"
-    # shellcheck disable=SC2034 # used indirectly
-    local CODEGEN_tag_files=()
-    kube::util::read-array CODEGEN_tag_files < <(
-        k8s_tag_files_matching \
-            vendor/k8s.io/code-generator \
-            "${apimachinery_dirs[@]}"
-        )
-
-    # shellcheck disable=SC2034 # used indirectly
-    local SAMPLEAPISERVER_output_dir="staging/src/k8s.io/sample-apiserver/pkg/generated/openapi"
-    # shellcheck disable=SC2034 # used indirectly
-    local SAMPLEAPISERVER_known_violations_file="api/api-rules/sample_apiserver_violation_exceptions.list"
-    # shellcheck disable=SC2034 # used indirectly
-    local SAMPLEAPISERVER_tag_files=()
-    kube::util::read-array SAMPLEAPISERVER_tag_files < <(
-        k8s_tag_files_matching \
-            vendor/k8s.io/sample-apiserver \
-            "${apimachinery_dirs[@]}"
-        )
-
-    git_find -z ':(glob)**'/"${output_base}.go" | xargs -0 rm -f
-
-    for prefix in "${targets[@]}"; do
-        local report_file="${OUT_DIR}/${prefix}_violations.report"
-        # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write
-        # updated API violations to the known API violation exceptions list.
-        if [[ "${UPDATE_API_KNOWN_VIOLATIONS}" == true ]]; then
-            report_file=$(indirect "${prefix}_known_violations_file")
-        fi
-
-        # 2 lines because shellcheck
-        local output_dir
-        output_dir=$(indirect "${prefix}_output_dir")
-
-        if [[ "${DBG_CODEGEN}" == 1 ]]; then
-            kube::log::status "DBG: finding all +k8s:openapi-gen tags for ${prefix}"
-        fi
-
    local tag_dirs=()
    kube::util::read-array tag_dirs < <(
-            grep -l --null '+k8s:openapi-gen=' $(indirect_array "${prefix}_tag_files") \
+        grep -l --null '+k8s:openapi-gen=' "${tag_files[@]}" \
            | xargs -0 -n1 dirname \
            | sort -u)

    if [[ "${DBG_CODEGEN}" == 1 ]]; then
-            kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:openapi-gen tagged dirs for ${prefix}"
+        kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:openapi-gen tagged dirs"
    fi

    local tag_pkgs=()
@ -665,7 +551,7 @@ function codegen::openapi() {
        tag_pkgs+=("${PRJ_SRC_PATH}/$dir")
    done

-        kube::log::status "Generating openapi code for ${prefix}"
+    kube::log::status "Generating openapi code"
    if [[ "${DBG_CODEGEN}" == 1 ]]; then
        kube::log::status "DBG: running ${gen_openapi_bin} for:"
        for dir in "${tag_dirs[@]}"; do
@ -673,6 +559,8 @@ function codegen::openapi() {
        done
    fi

+    git_find -z ':(glob)pkg/generated/**'/"${output_base}.go" | xargs -0 rm -f
+
    ./hack/run-in-gopath.sh "${gen_openapi_bin}" \
        --v "${KUBE_VERBOSE}" \
        --logtostderr \
@ -684,19 +572,16 @@ function codegen::openapi() {
        "$@"

    touch "${report_file}"
-        # 2 lines because shellcheck
-        local known_filename
-        known_filename=$(indirect "${prefix}_known_violations_file")
+    local known_filename="${known_violations_file}"
    if ! diff -u "${known_filename}" "${report_file}"; then
        echo -e "ERROR:"
-            echo -e "\t'${prefix}' API rule check failed - reported violations differ from known violations"
+        echo -e "\tAPI rule check failed - reported violations differ from known violations"
        echo -e "\tPlease read api/api-rules/README.md to resolve the failure in ${known_filename}"
    fi

    if [[ "${DBG_CODEGEN}" == 1 ]]; then
        kube::log::status "Generated openapi code"
    fi
-    done # for each prefix
 }

 function codegen::applyconfigs() {
@ -898,8 +783,8 @@ function codegen::subprojects() {
        vendor/k8s.io/kube-aggregator
        vendor/k8s.io/sample-apiserver
        vendor/k8s.io/sample-controller
-        vendor/k8s.io/apiextensions-apiserver
        vendor/k8s.io/metrics
+        vendor/k8s.io/apiextensions-apiserver
        vendor/k8s.io/apiextensions-apiserver/examples/client-go
    )

@ -908,7 +793,10 @@ function codegen::subprojects() {
    for sub in "${subs[@]}"; do
        kube::log::status "Generating code for subproject ${sub}"
        pushd "${sub}" >/dev/null
-        CODEGEN_PKG="${codegen}" ./hack/update-codegen.sh > >(indent) 2> >(indent >&2)
+        CODEGEN_PKG="${codegen}" \
+        UPDATE_API_KNOWN_VIOLATIONS="${UPDATE_API_KNOWN_VIOLATIONS}" \
+        API_KNOWN_VIOLATIONS_DIR="${API_KNOWN_VIOLATIONS_DIR}" \
+            ./hack/update-codegen.sh > >(indent) 2> >(indent >&2)
        popd >/dev/null
    done
 }
--- a/staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh
+++ b/staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh
@ -28,6 +28,22 @@ kube::codegen::gen_helpers \
    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"

+if [[ -n "${API_KNOWN_VIOLATIONS_DIR:-}" ]]; then
+    report_filename="${API_KNOWN_VIOLATIONS_DIR}/apiextensions_violation_exceptions.list"
+    if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then
+        update_report="--update-report"
+    fi
+fi
+
+kube::codegen::gen_openapi \
+    --input-pkg-root k8s.io/apiextensions-apiserver/pkg \
+    --extra-pkgs k8s.io/api/autoscaling/v1 `# needed for Scale type` \
+    --output-pkg-root k8s.io/apiextensions-apiserver/pkg/generated \
+    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
+    --report-filename "${report_filename:-"/dev/null"}" \
+    ${update_report:+"${update_report}"} \
+    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
+
 kube::codegen::gen_client \
    --with-watch \
    --input-pkg-root k8s.io/apiextensions-apiserver/pkg/apis \
--- a/staging/src/k8s.io/code-generator/examples/hack/update-codegen.sh
+++ b/staging/src/k8s.io/code-generator/examples/hack/update-codegen.sh
@ -34,10 +34,19 @@ kube::codegen::gen_helpers \
    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../../.." \
    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"

+if [[ -n "${API_KNOWN_VIOLATIONS_DIR:-}" ]]; then
+    report_filename="${API_KNOWN_VIOLATIONS_DIR}/codegen_violation_exceptions.list"
+    if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then
+        update_report="--update-report"
+    fi
+fi
+
 kube::codegen::gen_openapi \
    --input-pkg-root k8s.io/code-generator/examples/apiserver/apis \
    --output-pkg-root k8s.io/code-generator/examples/apiserver \
    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../../.." \
+    --report-filename "${report_filename:-"/dev/null"}" \
+    ${update_report:+"${update_report}"} \
    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"

 kube::codegen::gen_client \
--- a/staging/src/k8s.io/code-generator/kube_codegen.sh
+++ b/staging/src/k8s.io/code-generator/kube_codegen.sh
@ -237,6 +237,9 @@ function kube::codegen::gen_helpers() {
 #     An optional path at which to write an API violations report.  "-" means
 #     stdout.
 #
+#   --update-report
+#     If specified, update the report file in place, rather than diffing it.
+#
 #   --boilerplate <string = path_to_kube_codegen_boilerplate>
 #     An optional override for the header file to insert into generated files.
 #
@ -247,10 +250,12 @@ function kube::codegen::gen_openapi() {
    local openapi_subdir="openapi"
    local extra_pkgs=()
    local report="/dev/null"
+    local update_report=""
    local boilerplate="${KUBE_CODEGEN_ROOT}/hack/boilerplate.go.txt"
    local v="${KUBE_VERBOSE:-0}"

    while [ "$#" -gt 0 ]; do
+        echo "$1"
        case "$1" in
            "--input-pkg-root")
                in_pkg_root="$2"
@ -276,6 +281,10 @@ function kube::codegen::gen_openapi() {
                report="$2"
                shift 2
                ;;
+            "--update-report")
+                update_report="true"
+                shift
+                ;;
            "--boilerplate")
                boilerplate="$2"
                shift 2
@ -300,6 +309,13 @@ function kube::codegen::gen_openapi() {
        return 1
    fi

+    set -x
+    local new_report
+    new_report="$(mktemp -t "$(basename "$0").api_violations.XXXXXX")"
+    if [ -n "${update_report}" ]; then
+        new_report="${report}"
+    fi
+
    (
        # To support running this from anywhere, first cd into this directory,
        # and then install with forced module mode on and fully qualified name.
@ -348,12 +364,20 @@ function kube::codegen::gen_openapi() {
            --go-header-file "${boilerplate}" \
            --output-base "${out_base}" \
            --output-package "${out_pkg_root}/${openapi_subdir}" \
-            --report-filename "${report}" \
+            --report-filename "${new_report}" \
            --input-dirs "k8s.io/apimachinery/pkg/apis/meta/v1" \
            --input-dirs "k8s.io/apimachinery/pkg/runtime" \
            --input-dirs "k8s.io/apimachinery/pkg/version" \
            "${inputs[@]}"
    fi
+
+    touch "${report}" # in case it doesn't exist yet
+    if ! diff -u "${report}" "${new_report}"; then
+        echo -e "ERROR:"
+        echo -e "\tAPI rule check failed for ${report}: new reported violations"
+        echo -e "\tPlease read api/api-rules/README.md"
+        return 1
+    fi
 }

 # Generate client code
--- a/staging/src/k8s.io/kube-aggregator/hack/update-codegen.sh
+++ b/staging/src/k8s.io/kube-aggregator/hack/update-codegen.sh
@ -28,6 +28,21 @@ kube::codegen::gen_helpers \
    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"

+if [[ -n "${API_KNOWN_VIOLATIONS_DIR:-}" ]]; then
+    report_filename="${API_KNOWN_VIOLATIONS_DIR}/aggregator_violation_exceptions.list"
+    if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then
+        update_report="--update-report"
+    fi
+fi
+
+kube::codegen::gen_openapi \
+    --input-pkg-root k8s.io/kube-aggregator/pkg/apis \
+    --output-pkg-root k8s.io/kube-aggregator/pkg/generated \
+    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
+    --report-filename "${report_filename:-"/dev/null"}" \
+    ${update_report:+"${update_report}"} \
+    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
+
 kube::codegen::gen_client \
    --with-watch \
    --input-pkg-root k8s.io/kube-aggregator/pkg/apis \
--- a/staging/src/k8s.io/sample-apiserver/hack/update-codegen.sh
+++ b/staging/src/k8s.io/sample-apiserver/hack/update-codegen.sh
@ -33,10 +33,19 @@ kube::codegen::gen_helpers \
    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"

+if [[ -n "${API_KNOWN_VIOLATIONS_DIR:-}" ]]; then
+    report_filename="${API_KNOWN_VIOLATIONS_DIR}/sample_apiserver_violation_exceptions.list"
+    if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then
+        update_report="--update-report"
+    fi
+fi
+
 kube::codegen::gen_openapi \
    --input-pkg-root k8s.io/sample-apiserver/pkg/apis \
    --output-pkg-root k8s.io/sample-apiserver/pkg/generated \
    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
+    --report-filename "${report_filename:-"/dev/null"}" \
+    ${update_report:+"${update_report}"} \
    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"

 kube::codegen::gen_client \