diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index e5cfef5634d..201aca0b004 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -32,6 +32,7 @@ kube::golang::setup_env DBG_CODEGEN="${DBG_CODEGEN:-0}" GENERATED_FILE_PREFIX="${GENERATED_FILE_PREFIX:-zz_generated.}" UPDATE_API_KNOWN_VIOLATIONS="${UPDATE_API_KNOWN_VIOLATIONS:-}" +API_KNOWN_VIOLATIONS_DIR="${API_KNOWN_VIOLATIONS_DIR:-"${KUBE_ROOT}/api/api-rules"}" OUT_DIR="_output" PRJ_SRC_PATH="k8s.io/kubernetes" @@ -497,47 +498,6 @@ function k8s_tag_files_except() { done } -# $@: directories to exclude -# example: -# k8s_tag_files_matching foo bat/qux -function k8s_tag_files_matching() { - for f in "${ALL_K8S_TAG_FILES[@]}"; do - for x in "$@"; do - if [[ "$f" =~ "${x}"/.* ]]; then - echo "$f" - break - fi - done - done -} - -# $1: the name of a scalar variable to read -# example: -# FOO_VAR="foo value" -# BAR_VAR="bar value" -# x=FOO -# indirect "${x}_VAR" # -> "foo value\n" -function indirect() { - # This is a trick to get bash to indirectly read a variable. - # Thanks StackOverflow! - local var="$1" - echo "${!var}" -} - -# $1: the name of an array variable to read -# FOO_ARR=(a b c) -# BAR_ARR=(1 2 3) -# x=FOO -# indirect_array "${x}_ARR" # -> "a\nb\nc\n" -function indirect_array() { - # This is a trick to get bash to indirectly read an array. - # Thanks StackOverflow! - local arrayname="$1" - # shellcheck disable=SC1087 # intentional - local tmp="$arrayname[@]" - printf -- "%s\n" "${!tmp}" -} - # OpenAPI generation # # Any package that wants open-api functions generated must include a @@ -555,148 +515,73 @@ function codegen::openapi() { local gen_openapi_bin gen_openapi_bin="$(kube::util::find-binary "openapi-gen")" - # Standard dirs which all targets need. - local apimachinery_dirs=( - vendor/k8s.io/apimachinery/pkg/apis/meta/v1 - vendor/k8s.io/apimachinery/pkg/runtime - vendor/k8s.io/apimachinery/pkg/version - ) + local output_dir="pkg/generated/openapi" + local known_violations_file="${API_KNOWN_VIOLATIONS_DIR}/violation_exceptions.list" - # These should probably be configured by tags in code-files somewhere. - local targets=( - KUBE - AGGREGATOR - APIEXTENSIONS - CODEGEN - SAMPLEAPISERVER - ) + local report_file="${OUT_DIR}/api_violations.report" + # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write + # updated API violations to the known API violation exceptions list. + if [[ "${UPDATE_API_KNOWN_VIOLATIONS}" == true ]]; then + report_file="${known_violations_file}" + fi - # shellcheck disable=SC2034 # used indirectly - local KUBE_output_dir="pkg/generated/openapi" - # shellcheck disable=SC2034 # used indirectly - local KUBE_known_violations_file="api/api-rules/violation_exceptions.list" - # shellcheck disable=SC2034 # used indirectly - local KUBE_tag_files=() - kube::util::read-array KUBE_tag_files < <( + if [[ "${DBG_CODEGEN}" == 1 ]]; then + kube::log::status "DBG: finding all +k8s:openapi-gen tags" + fi + + local tag_files=() + kube::util::read-array tag_files < <( k8s_tag_files_except \ vendor/k8s.io/code-generator \ vendor/k8s.io/sample-apiserver ) - # shellcheck disable=SC2034 # used indirectly - local AGGREGATOR_output_dir="staging/src/k8s.io/kube-aggregator/pkg/generated/openapi" - # shellcheck disable=SC2034 # used indirectly - local AGGREGATOR_known_violations_file="api/api-rules/aggregator_violation_exceptions.list" - # shellcheck disable=SC2034 # used indirectly - local AGGREGATOR_tag_files=() - kube::util::read-array AGGREGATOR_tag_files < <( - k8s_tag_files_matching \ - vendor/k8s.io/kube-aggregator \ - "${apimachinery_dirs[@]}" - ) + local tag_dirs=() + kube::util::read-array tag_dirs < <( + grep -l --null '+k8s:openapi-gen=' "${tag_files[@]}" \ + | xargs -0 -n1 dirname \ + | sort -u) - # shellcheck disable=SC2034 # used indirectly - local APIEXTENSIONS_output_dir="staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi" - # shellcheck disable=SC2034 # used indirectly - local APIEXTENSIONS_known_violations_file="api/api-rules/apiextensions_violation_exceptions.list" - # shellcheck disable=SC2034 # used indirectly - local APIEXTENSIONS_tag_files=() - kube::util::read-array APIEXTENSIONS_tag_files < <( - k8s_tag_files_matching \ - vendor/k8s.io/apiextensions-apiserver \ - vendor/k8s.io/api/autoscaling/v1 \ - "${apimachinery_dirs[@]}" - ) + if [[ "${DBG_CODEGEN}" == 1 ]]; then + kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:openapi-gen tagged dirs" + fi - # shellcheck disable=SC2034 # used indirectly - local CODEGEN_output_dir="staging/src/k8s.io/code-generator/examples/apiserver/openapi" - # shellcheck disable=SC2034 # used indirectly - local CODEGEN_known_violations_file="api/api-rules/codegen_violation_exceptions.list" - # shellcheck disable=SC2034 # used indirectly - local CODEGEN_tag_files=() - kube::util::read-array CODEGEN_tag_files < <( - k8s_tag_files_matching \ - vendor/k8s.io/code-generator \ - "${apimachinery_dirs[@]}" - ) + local tag_pkgs=() + for dir in "${tag_dirs[@]}"; do + tag_pkgs+=("${PRJ_SRC_PATH}/$dir") + done - # shellcheck disable=SC2034 # used indirectly - local SAMPLEAPISERVER_output_dir="staging/src/k8s.io/sample-apiserver/pkg/generated/openapi" - # shellcheck disable=SC2034 # used indirectly - local SAMPLEAPISERVER_known_violations_file="api/api-rules/sample_apiserver_violation_exceptions.list" - # shellcheck disable=SC2034 # used indirectly - local SAMPLEAPISERVER_tag_files=() - kube::util::read-array SAMPLEAPISERVER_tag_files < <( - k8s_tag_files_matching \ - vendor/k8s.io/sample-apiserver \ - "${apimachinery_dirs[@]}" - ) - - git_find -z ':(glob)**'/"${output_base}.go" | xargs -0 rm -f - - for prefix in "${targets[@]}"; do - local report_file="${OUT_DIR}/${prefix}_violations.report" - # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write - # updated API violations to the known API violation exceptions list. - if [[ "${UPDATE_API_KNOWN_VIOLATIONS}" == true ]]; then - report_file=$(indirect "${prefix}_known_violations_file") - fi - - # 2 lines because shellcheck - local output_dir - output_dir=$(indirect "${prefix}_output_dir") - - if [[ "${DBG_CODEGEN}" == 1 ]]; then - kube::log::status "DBG: finding all +k8s:openapi-gen tags for ${prefix}" - fi - - local tag_dirs=() - kube::util::read-array tag_dirs < <( - grep -l --null '+k8s:openapi-gen=' $(indirect_array "${prefix}_tag_files") \ - | xargs -0 -n1 dirname \ - | sort -u) - - if [[ "${DBG_CODEGEN}" == 1 ]]; then - kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:openapi-gen tagged dirs for ${prefix}" - fi - - local tag_pkgs=() + kube::log::status "Generating openapi code" + if [[ "${DBG_CODEGEN}" == 1 ]]; then + kube::log::status "DBG: running ${gen_openapi_bin} for:" for dir in "${tag_dirs[@]}"; do - tag_pkgs+=("${PRJ_SRC_PATH}/$dir") + kube::log::status "DBG: $dir" done + fi - kube::log::status "Generating openapi code for ${prefix}" - if [[ "${DBG_CODEGEN}" == 1 ]]; then - kube::log::status "DBG: running ${gen_openapi_bin} for:" - for dir in "${tag_dirs[@]}"; do - kube::log::status "DBG: $dir" - done - fi + git_find -z ':(glob)pkg/generated/**'/"${output_base}.go" | xargs -0 rm -f - ./hack/run-in-gopath.sh "${gen_openapi_bin}" \ - --v "${KUBE_VERBOSE}" \ - --logtostderr \ - -h "${BOILERPLATE_FILENAME}" \ - -O "${output_base}" \ - -p "${PRJ_SRC_PATH}/${output_dir}" \ - -r "${report_file}" \ - $(printf -- " -i %s" "${tag_pkgs[@]}") \ - "$@" + ./hack/run-in-gopath.sh "${gen_openapi_bin}" \ + --v "${KUBE_VERBOSE}" \ + --logtostderr \ + -h "${BOILERPLATE_FILENAME}" \ + -O "${output_base}" \ + -p "${PRJ_SRC_PATH}/${output_dir}" \ + -r "${report_file}" \ + $(printf -- " -i %s" "${tag_pkgs[@]}") \ + "$@" - touch "${report_file}" - # 2 lines because shellcheck - local known_filename - known_filename=$(indirect "${prefix}_known_violations_file") - if ! diff -u "${known_filename}" "${report_file}"; then - echo -e "ERROR:" - echo -e "\t'${prefix}' API rule check failed - reported violations differ from known violations" - echo -e "\tPlease read api/api-rules/README.md to resolve the failure in ${known_filename}" - fi + touch "${report_file}" + local known_filename="${known_violations_file}" + if ! diff -u "${known_filename}" "${report_file}"; then + echo -e "ERROR:" + echo -e "\tAPI rule check failed - reported violations differ from known violations" + echo -e "\tPlease read api/api-rules/README.md to resolve the failure in ${known_filename}" + fi - if [[ "${DBG_CODEGEN}" == 1 ]]; then - kube::log::status "Generated openapi code" - fi - done # for each prefix + if [[ "${DBG_CODEGEN}" == 1 ]]; then + kube::log::status "Generated openapi code" + fi } function codegen::applyconfigs() { @@ -898,8 +783,8 @@ function codegen::subprojects() { vendor/k8s.io/kube-aggregator vendor/k8s.io/sample-apiserver vendor/k8s.io/sample-controller - vendor/k8s.io/apiextensions-apiserver vendor/k8s.io/metrics + vendor/k8s.io/apiextensions-apiserver vendor/k8s.io/apiextensions-apiserver/examples/client-go ) @@ -908,7 +793,10 @@ function codegen::subprojects() { for sub in "${subs[@]}"; do kube::log::status "Generating code for subproject ${sub}" pushd "${sub}" >/dev/null - CODEGEN_PKG="${codegen}" ./hack/update-codegen.sh > >(indent) 2> >(indent >&2) + CODEGEN_PKG="${codegen}" \ + UPDATE_API_KNOWN_VIOLATIONS="${UPDATE_API_KNOWN_VIOLATIONS}" \ + API_KNOWN_VIOLATIONS_DIR="${API_KNOWN_VIOLATIONS_DIR}" \ + ./hack/update-codegen.sh > >(indent) 2> >(indent >&2) popd >/dev/null done } diff --git a/staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh b/staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh index 44aca6f1ec2..37676e2e963 100755 --- a/staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh +++ b/staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh @@ -28,6 +28,22 @@ kube::codegen::gen_helpers \ --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \ --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" +if [[ -n "${API_KNOWN_VIOLATIONS_DIR:-}" ]]; then + report_filename="${API_KNOWN_VIOLATIONS_DIR}/apiextensions_violation_exceptions.list" + if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then + update_report="--update-report" + fi +fi + +kube::codegen::gen_openapi \ + --input-pkg-root k8s.io/apiextensions-apiserver/pkg \ + --extra-pkgs k8s.io/api/autoscaling/v1 `# needed for Scale type` \ + --output-pkg-root k8s.io/apiextensions-apiserver/pkg/generated \ + --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \ + --report-filename "${report_filename:-"/dev/null"}" \ + ${update_report:+"${update_report}"} \ + --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" + kube::codegen::gen_client \ --with-watch \ --input-pkg-root k8s.io/apiextensions-apiserver/pkg/apis \ diff --git a/staging/src/k8s.io/code-generator/examples/hack/update-codegen.sh b/staging/src/k8s.io/code-generator/examples/hack/update-codegen.sh index 3d3a4f3a3bf..7c7b251cde2 100755 --- a/staging/src/k8s.io/code-generator/examples/hack/update-codegen.sh +++ b/staging/src/k8s.io/code-generator/examples/hack/update-codegen.sh @@ -34,10 +34,19 @@ kube::codegen::gen_helpers \ --output-base "$(dirname "${BASH_SOURCE[0]}")/../../../.." \ --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" +if [[ -n "${API_KNOWN_VIOLATIONS_DIR:-}" ]]; then + report_filename="${API_KNOWN_VIOLATIONS_DIR}/codegen_violation_exceptions.list" + if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then + update_report="--update-report" + fi +fi + kube::codegen::gen_openapi \ --input-pkg-root k8s.io/code-generator/examples/apiserver/apis \ --output-pkg-root k8s.io/code-generator/examples/apiserver \ --output-base "$(dirname "${BASH_SOURCE[0]}")/../../../.." \ + --report-filename "${report_filename:-"/dev/null"}" \ + ${update_report:+"${update_report}"} \ --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" kube::codegen::gen_client \ diff --git a/staging/src/k8s.io/code-generator/kube_codegen.sh b/staging/src/k8s.io/code-generator/kube_codegen.sh index 59e807929dc..5d30fccb058 100755 --- a/staging/src/k8s.io/code-generator/kube_codegen.sh +++ b/staging/src/k8s.io/code-generator/kube_codegen.sh @@ -237,6 +237,9 @@ function kube::codegen::gen_helpers() { # An optional path at which to write an API violations report. "-" means # stdout. # +# --update-report +# If specified, update the report file in place, rather than diffing it. +# # --boilerplate # An optional override for the header file to insert into generated files. # @@ -247,10 +250,12 @@ function kube::codegen::gen_openapi() { local openapi_subdir="openapi" local extra_pkgs=() local report="/dev/null" + local update_report="" local boilerplate="${KUBE_CODEGEN_ROOT}/hack/boilerplate.go.txt" local v="${KUBE_VERBOSE:-0}" while [ "$#" -gt 0 ]; do + echo "$1" case "$1" in "--input-pkg-root") in_pkg_root="$2" @@ -276,6 +281,10 @@ function kube::codegen::gen_openapi() { report="$2" shift 2 ;; + "--update-report") + update_report="true" + shift + ;; "--boilerplate") boilerplate="$2" shift 2 @@ -300,6 +309,13 @@ function kube::codegen::gen_openapi() { return 1 fi + set -x + local new_report + new_report="$(mktemp -t "$(basename "$0").api_violations.XXXXXX")" + if [ -n "${update_report}" ]; then + new_report="${report}" + fi + ( # To support running this from anywhere, first cd into this directory, # and then install with forced module mode on and fully qualified name. @@ -348,12 +364,20 @@ function kube::codegen::gen_openapi() { --go-header-file "${boilerplate}" \ --output-base "${out_base}" \ --output-package "${out_pkg_root}/${openapi_subdir}" \ - --report-filename "${report}" \ + --report-filename "${new_report}" \ --input-dirs "k8s.io/apimachinery/pkg/apis/meta/v1" \ --input-dirs "k8s.io/apimachinery/pkg/runtime" \ --input-dirs "k8s.io/apimachinery/pkg/version" \ "${inputs[@]}" fi + + touch "${report}" # in case it doesn't exist yet + if ! diff -u "${report}" "${new_report}"; then + echo -e "ERROR:" + echo -e "\tAPI rule check failed for ${report}: new reported violations" + echo -e "\tPlease read api/api-rules/README.md" + return 1 + fi } # Generate client code diff --git a/staging/src/k8s.io/kube-aggregator/hack/update-codegen.sh b/staging/src/k8s.io/kube-aggregator/hack/update-codegen.sh index 85815696fbe..c64782c821c 100755 --- a/staging/src/k8s.io/kube-aggregator/hack/update-codegen.sh +++ b/staging/src/k8s.io/kube-aggregator/hack/update-codegen.sh @@ -28,6 +28,21 @@ kube::codegen::gen_helpers \ --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \ --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" +if [[ -n "${API_KNOWN_VIOLATIONS_DIR:-}" ]]; then + report_filename="${API_KNOWN_VIOLATIONS_DIR}/aggregator_violation_exceptions.list" + if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then + update_report="--update-report" + fi +fi + +kube::codegen::gen_openapi \ + --input-pkg-root k8s.io/kube-aggregator/pkg/apis \ + --output-pkg-root k8s.io/kube-aggregator/pkg/generated \ + --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \ + --report-filename "${report_filename:-"/dev/null"}" \ + ${update_report:+"${update_report}"} \ + --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" + kube::codegen::gen_client \ --with-watch \ --input-pkg-root k8s.io/kube-aggregator/pkg/apis \ diff --git a/staging/src/k8s.io/sample-apiserver/hack/update-codegen.sh b/staging/src/k8s.io/sample-apiserver/hack/update-codegen.sh index be72c4f3694..996bab05b09 100755 --- a/staging/src/k8s.io/sample-apiserver/hack/update-codegen.sh +++ b/staging/src/k8s.io/sample-apiserver/hack/update-codegen.sh @@ -33,10 +33,19 @@ kube::codegen::gen_helpers \ --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \ --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" +if [[ -n "${API_KNOWN_VIOLATIONS_DIR:-}" ]]; then + report_filename="${API_KNOWN_VIOLATIONS_DIR}/sample_apiserver_violation_exceptions.list" + if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then + update_report="--update-report" + fi +fi + kube::codegen::gen_openapi \ --input-pkg-root k8s.io/sample-apiserver/pkg/apis \ --output-pkg-root k8s.io/sample-apiserver/pkg/generated \ --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \ + --report-filename "${report_filename:-"/dev/null"}" \ + ${update_report:+"${update_report}"} \ --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" kube::codegen::gen_client \