Simpler openapi gen - subprojects do themselves

This involves moving the report files, but it allows me to delete the indirect variable and indirect array code in update-codgen. As proud as I was of figuring that out, I am also ashamed of myself for doing it. This is my atonement.
2025-08-14 14:23:37 +00:00 · 2023-03-12 21:07:02 -07:00 · 2023-03-12 21:07:02 -07:00 · ef796dca9b
commit ef796dca9b
parent 93b2845bec
11 changed files with 103 additions and 171 deletions
--- a/api/api-rules/README.md
+++ b/api/api-rules/README.md
@ -1,8 +1,10 @@
 # Existing API Rule Violations

-This folder contains the checked-in report file of known API rule violations.
-The file violation\_exceptions.list is used by Make rule during OpenAPI spec generation to make
-sure that no new API rule violation is introduced into our code base.
+This folder contains the checked-in report file of known API rule violations
+for the main Kubernetes repository.  Reports for staging repositories are
+checked in to those repositories.  These files are used during OpenAPI spec
+generation to make sure that no new API rule violation is introduced into our
+code base.

 ## API Rule Violation Format

--- a/hack/update-codegen.sh
+++ b/hack/update-codegen.sh
@ -497,47 +497,6 @@ function k8s_tag_files_except() {
    done
 }

-# $@: directories to exclude
-# example:
-#    k8s_tag_files_matching foo bat/qux
-function k8s_tag_files_matching() {
-    for f in "${ALL_K8S_TAG_FILES[@]}"; do
-        for x in "$@"; do
-            if [[ "$f" =~ "${x}"/.* ]]; then
-                echo "$f"
-                break
-            fi
-        done
-    done
-}
-
-# $1: the name of a scalar variable to read
-# example:
-#    FOO_VAR="foo value"
-#    BAR_VAR="bar value"
-#    x=FOO
-#    indirect "${x}_VAR" # -> "foo value\n"
-function indirect() {
-    # This is a trick to get bash to indirectly read a variable.
-    # Thanks StackOverflow!
-    local var="$1"
-    echo "${!var}"
-}
-
-# $1: the name of an array variable to read
-#    FOO_ARR=(a b c)
-#    BAR_ARR=(1 2 3)
-#    x=FOO
-#    indirect_array "${x}_ARR" # -> "a\nb\nc\n"
-function indirect_array() {
-    # This is a trick to get bash to indirectly read an array.
-    # Thanks StackOverflow!
-    local arrayname="$1"
-    # shellcheck disable=SC1087 # intentional
-    local tmp="$arrayname[@]"
-    printf -- "%s\n" "${!tmp}"
-}
-
 # OpenAPI generation
 #
 # Any package that wants open-api functions generated must include a
@ -555,148 +514,73 @@ function codegen::openapi() {
    local gen_openapi_bin
    gen_openapi_bin="$(kube::util::find-binary "openapi-gen")"

-    # Standard dirs which all targets need.
-    local apimachinery_dirs=(
-        vendor/k8s.io/apimachinery/pkg/apis/meta/v1
-        vendor/k8s.io/apimachinery/pkg/runtime
-        vendor/k8s.io/apimachinery/pkg/version
-    )
+    local output_dir="pkg/generated/openapi"
+    local known_violations_file="api/api-rules/violation_exceptions.list"

-    # These should probably be configured by tags in code-files somewhere.
-    local targets=(
-        KUBE
-        AGGREGATOR
-        APIEXTENSIONS
-        CODEGEN
-        SAMPLEAPISERVER
-    )
+    local report_file="${OUT_DIR}/api_violations.report"
+    # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write
+    # updated API violations to the known API violation exceptions list.
+    if [[ "${UPDATE_API_KNOWN_VIOLATIONS}" == true ]]; then
+        report_file="${known_violations_file}"
+    fi

-    # shellcheck disable=SC2034 # used indirectly
-    local KUBE_output_dir="pkg/generated/openapi"
-    # shellcheck disable=SC2034 # used indirectly
-    local KUBE_known_violations_file="api/api-rules/violation_exceptions.list"
-    # shellcheck disable=SC2034 # used indirectly
-    local KUBE_tag_files=()
-    kube::util::read-array KUBE_tag_files < <(
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: finding all +k8s:openapi-gen tags"
+    fi
+
+    local tag_files=()
+    kube::util::read-array tag_files < <(
        k8s_tag_files_except \
            vendor/k8s.io/code-generator \
            vendor/k8s.io/sample-apiserver
        )

-    # shellcheck disable=SC2034 # used indirectly
-    local AGGREGATOR_output_dir="staging/src/k8s.io/kube-aggregator/pkg/generated/openapi"
-    # shellcheck disable=SC2034 # used indirectly
-    local AGGREGATOR_known_violations_file="api/api-rules/aggregator_violation_exceptions.list"
-    # shellcheck disable=SC2034 # used indirectly
-    local AGGREGATOR_tag_files=()
-    kube::util::read-array AGGREGATOR_tag_files < <(
-        k8s_tag_files_matching \
-            vendor/k8s.io/kube-aggregator \
-            "${apimachinery_dirs[@]}"
-        )
+    local tag_dirs=()
+    kube::util::read-array tag_dirs < <(
+        grep -l --null '+k8s:openapi-gen=' "${tag_files[@]}" \
+            | xargs -0 -n1 dirname \
+            | sort -u)

-    # shellcheck disable=SC2034 # used indirectly
-    local APIEXTENSIONS_output_dir="staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi"
-    # shellcheck disable=SC2034 # used indirectly
-    local APIEXTENSIONS_known_violations_file="api/api-rules/apiextensions_violation_exceptions.list"
-    # shellcheck disable=SC2034 # used indirectly
-    local APIEXTENSIONS_tag_files=()
-    kube::util::read-array APIEXTENSIONS_tag_files < <(
-        k8s_tag_files_matching \
-            vendor/k8s.io/apiextensions-apiserver \
-            vendor/k8s.io/api/autoscaling/v1 \
-            "${apimachinery_dirs[@]}"
-        )
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:openapi-gen tagged dirs"
+    fi

-    # shellcheck disable=SC2034 # used indirectly
-    local CODEGEN_output_dir="staging/src/k8s.io/code-generator/examples/apiserver/openapi"
-    # shellcheck disable=SC2034 # used indirectly
-    local CODEGEN_known_violations_file="api/api-rules/codegen_violation_exceptions.list"
-    # shellcheck disable=SC2034 # used indirectly
-    local CODEGEN_tag_files=()
-    kube::util::read-array CODEGEN_tag_files < <(
-        k8s_tag_files_matching \
-            vendor/k8s.io/code-generator \
-            "${apimachinery_dirs[@]}"
-        )
+    local tag_pkgs=()
+    for dir in "${tag_dirs[@]}"; do
+        tag_pkgs+=("${PRJ_SRC_PATH}/$dir")
+    done

-    # shellcheck disable=SC2034 # used indirectly
-    local SAMPLEAPISERVER_output_dir="staging/src/k8s.io/sample-apiserver/pkg/generated/openapi"
-    # shellcheck disable=SC2034 # used indirectly
-    local SAMPLEAPISERVER_known_violations_file="api/api-rules/sample_apiserver_violation_exceptions.list"
-    # shellcheck disable=SC2034 # used indirectly
-    local SAMPLEAPISERVER_tag_files=()
-    kube::util::read-array SAMPLEAPISERVER_tag_files < <(
-        k8s_tag_files_matching \
-            vendor/k8s.io/sample-apiserver \
-            "${apimachinery_dirs[@]}"
-        )
-
-    git_find -z ':(glob)**'/"${output_base}.go" | xargs -0 rm -f
-
-    for prefix in "${targets[@]}"; do
-        local report_file="${OUT_DIR}/${prefix}_violations.report"
-        # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write
-        # updated API violations to the known API violation exceptions list.
-        if [[ "${UPDATE_API_KNOWN_VIOLATIONS}" == true ]]; then
-            report_file=$(indirect "${prefix}_known_violations_file")
-        fi
-
-        # 2 lines because shellcheck
-        local output_dir
-        output_dir=$(indirect "${prefix}_output_dir")
-
-        if [[ "${DBG_CODEGEN}" == 1 ]]; then
-            kube::log::status "DBG: finding all +k8s:openapi-gen tags for ${prefix}"
-        fi
-
-        local tag_dirs=()
-        kube::util::read-array tag_dirs < <(
-            grep -l --null '+k8s:openapi-gen=' $(indirect_array "${prefix}_tag_files") \
-                | xargs -0 -n1 dirname \
-                | sort -u)
-
-        if [[ "${DBG_CODEGEN}" == 1 ]]; then
-            kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:openapi-gen tagged dirs for ${prefix}"
-        fi
-
-        local tag_pkgs=()
+    kube::log::status "Generating openapi code"
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: running ${gen_openapi_bin} for:"
        for dir in "${tag_dirs[@]}"; do
-            tag_pkgs+=("${PRJ_SRC_PATH}/$dir")
+            kube::log::status "DBG:     $dir"
        done
+    fi

-        kube::log::status "Generating openapi code for ${prefix}"
-        if [[ "${DBG_CODEGEN}" == 1 ]]; then
-            kube::log::status "DBG: running ${gen_openapi_bin} for:"
-            for dir in "${tag_dirs[@]}"; do
-                kube::log::status "DBG:     $dir"
-            done
-        fi
+    git_find -z ':(glob)pkg/generated/**'/"${output_base}.go" | xargs -0 rm -f

-        ./hack/run-in-gopath.sh "${gen_openapi_bin}" \
-            --v "${KUBE_VERBOSE}" \
-            --logtostderr \
-            -h "${BOILERPLATE_FILENAME}" \
-            -O "${output_base}" \
-            -p "${PRJ_SRC_PATH}/${output_dir}" \
-            -r "${report_file}" \
-            $(printf -- " -i %s" "${tag_pkgs[@]}") \
-            "$@"
+    ./hack/run-in-gopath.sh "${gen_openapi_bin}" \
+        --v "${KUBE_VERBOSE}" \
+        --logtostderr \
+        -h "${BOILERPLATE_FILENAME}" \
+        -O "${output_base}" \
+        -p "${PRJ_SRC_PATH}/${output_dir}" \
+        -r "${report_file}" \
+        $(printf -- " -i %s" "${tag_pkgs[@]}") \
+        "$@"

-        touch "${report_file}"
-        # 2 lines because shellcheck
-        local known_filename
-        known_filename=$(indirect "${prefix}_known_violations_file")
-        if ! diff -u "${known_filename}" "${report_file}"; then
-            echo -e "ERROR:"
-            echo -e "\t'${prefix}' API rule check failed - reported violations differ from known violations"
-            echo -e "\tPlease read api/api-rules/README.md to resolve the failure in ${known_filename}"
-        fi
+    touch "${report_file}"
+    local known_filename="${known_violations_file}"
+    if ! diff -u "${known_filename}" "${report_file}"; then
+        echo -e "ERROR:"
+        echo -e "\tAPI rule check failed - reported violations differ from known violations"
+        echo -e "\tPlease read api/api-rules/README.md to resolve the failure in ${known_filename}"
+    fi

-        if [[ "${DBG_CODEGEN}" == 1 ]]; then
-            kube::log::status "Generated openapi code"
-        fi
-    done # for each prefix
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "Generated openapi code"
+    fi
 }

 function codegen::applyconfigs() {
--- a/staging/src/k8s.io/apiextensions-apiserver/api_violation_exceptions.list
+++ b/staging/src/k8s.io/apiextensions-apiserver/api_violation_exceptions.list
--- a/staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh
+++ b/staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh
@ -28,11 +28,17 @@ kube::codegen::gen_helpers \
    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"

+if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then
+    update_report="--update-report"
+fi
+
 kube::codegen::gen_openapi \
    --input-pkg-root k8s.io/apiextensions-apiserver/pkg \
    --extra-pkgs k8s.io/api/autoscaling/v1 `# needed for Scale type` \
    --output-pkg-root k8s.io/apiextensions-apiserver/pkg/generated \
    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
+    --report-filename "${SCRIPT_ROOT}/api_violation_exceptions.list" \
+    ${update_report:+"${update_report}"} \
    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"

 kube::codegen::gen_client \
--- a/staging/src/k8s.io/code-generator/examples/api_violation_exceptions.list
+++ b/staging/src/k8s.io/code-generator/examples/api_violation_exceptions.list
--- a/staging/src/k8s.io/code-generator/examples/hack/update-codegen.sh
+++ b/staging/src/k8s.io/code-generator/examples/hack/update-codegen.sh
@ -34,10 +34,16 @@ kube::codegen::gen_helpers \
    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../../.." \
    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"

+if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then
+    update_report="--update-report"
+fi
+
 kube::codegen::gen_openapi \
    --input-pkg-root k8s.io/code-generator/examples/apiserver/apis \
    --output-pkg-root k8s.io/code-generator/examples/apiserver \
    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../../.." \
+    --report-filename "${SCRIPT_ROOT}/api_violation_exceptions.list" \
+    ${update_report:+"${update_report}"} \
    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"

 kube::codegen::gen_client \
--- a/staging/src/k8s.io/code-generator/kube_codegen.sh
+++ b/staging/src/k8s.io/code-generator/kube_codegen.sh
@ -237,6 +237,9 @@ function kube::codegen::gen_helpers() {
 #     An optional path at which to write an API violations report.  "-" means
 #     stdout.
 #
+#   --update-report
+#     If specified, update the report file in place, rather than diffing it.
+#
 #   --boilerplate <string = path_to_kube_codegen_boilerplate>
 #     An optional override for the header file to insert into generated files.
 #
@ -247,6 +250,7 @@ function kube::codegen::gen_openapi() {
    local openapi_subdir="openapi"
    local extra_pkgs=()
    local report="/dev/null"
+    local update_report=""
    local boilerplate="${KUBE_CODEGEN_ROOT}/hack/boilerplate.go.txt"
    local v="${KUBE_VERBOSE:-0}"

@ -276,6 +280,10 @@ function kube::codegen::gen_openapi() {
                report="$2"
                shift 2
                ;;
+            "--update-report")
+                update_report="true"
+                shift
+                ;;
            "--boilerplate")
                boilerplate="$2"
                shift 2
@ -300,6 +308,12 @@ function kube::codegen::gen_openapi() {
        return 1
    fi

+    local new_report
+    new_report="$(mktemp -t "$(basename "$0").api_violations.XXXXXX")"
+    if [ -n "${update_report}" ]; then
+        new_report="${report}"
+    fi
+
    (
        # To support running this from anywhere, first cd into this directory,
        # and then install with forced module mode on and fully qualified name.
@ -348,12 +362,20 @@ function kube::codegen::gen_openapi() {
            --go-header-file "${boilerplate}" \
            --output-base "${out_base}" \
            --output-package "${out_pkg_root}/${openapi_subdir}" \
-            --report-filename "${report}" \
+            --report-filename "${new_report}" \
            --input-dirs "k8s.io/apimachinery/pkg/apis/meta/v1" \
            --input-dirs "k8s.io/apimachinery/pkg/runtime" \
            --input-dirs "k8s.io/apimachinery/pkg/version" \
            "${inputs[@]}"
    fi
+
+    touch "${report}" # in case it doesn't exist yet
+    if ! diff -u "${report}" "${new_report}"; then
+        echo -e "ERROR:"
+        echo -e "\tAPI rule check failed for ${report}: new reported violations"
+        echo -e "\tPlease read api/api-rules/README.md"
+        return 1
+    fi
 }

 # Generate client code
--- a/staging/src/k8s.io/kube-aggregator/api_violation_exceptions.list
+++ b/staging/src/k8s.io/kube-aggregator/api_violation_exceptions.list
--- a/staging/src/k8s.io/kube-aggregator/hack/update-codegen.sh
+++ b/staging/src/k8s.io/kube-aggregator/hack/update-codegen.sh
@ -28,10 +28,16 @@ kube::codegen::gen_helpers \
    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"

+if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then
+    update_report="--update-report"
+fi
+
 kube::codegen::gen_openapi \
    --input-pkg-root k8s.io/kube-aggregator/pkg/apis \
    --output-pkg-root k8s.io/kube-aggregator/pkg/generated \
    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
+    --report-filename "${SCRIPT_ROOT}/api_violation_exceptions.list" \
+    ${update_report:+"${update_report}"} \
    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"

 kube::codegen::gen_client \
--- a/staging/src/k8s.io/sample-apiserver/api_violation_exceptions.list
+++ b/staging/src/k8s.io/sample-apiserver/api_violation_exceptions.list
--- a/staging/src/k8s.io/sample-apiserver/hack/update-codegen.sh
+++ b/staging/src/k8s.io/sample-apiserver/hack/update-codegen.sh
@ -33,10 +33,16 @@ kube::codegen::gen_helpers \
    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"

+if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then
+    update_report="--update-report"
+fi
+
 kube::codegen::gen_openapi \
    --input-pkg-root k8s.io/sample-apiserver/pkg/apis \
    --output-pkg-root k8s.io/sample-apiserver/pkg/generated \
    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
+    --report-filename "${SCRIPT_ROOT}/api_violation_exceptions.list" \
+    ${update_report:+"${update_report}"} \
    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"

 kube::codegen::gen_client \