diff --git a/api/api-rules/README.md b/api/api-rules/README.md index 5edeaa77a5d..306ac93b8c9 100644 --- a/api/api-rules/README.md +++ b/api/api-rules/README.md @@ -1,8 +1,10 @@ # Existing API Rule Violations -This folder contains the checked-in report file of known API rule violations. -The file violation\_exceptions.list is used by Make rule during OpenAPI spec generation to make -sure that no new API rule violation is introduced into our code base. +This folder contains the checked-in report file of known API rule violations +for the main Kubernetes repository. Reports for staging repositories are +checked in to those repositories. These files are used during OpenAPI spec +generation to make sure that no new API rule violation is introduced into our +code base. ## API Rule Violation Format diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 30ede5fea8a..89f05332b5f 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -497,47 +497,6 @@ function k8s_tag_files_except() { done } -# $@: directories to exclude -# example: -# k8s_tag_files_matching foo bat/qux -function k8s_tag_files_matching() { - for f in "${ALL_K8S_TAG_FILES[@]}"; do - for x in "$@"; do - if [[ "$f" =~ "${x}"/.* ]]; then - echo "$f" - break - fi - done - done -} - -# $1: the name of a scalar variable to read -# example: -# FOO_VAR="foo value" -# BAR_VAR="bar value" -# x=FOO -# indirect "${x}_VAR" # -> "foo value\n" -function indirect() { - # This is a trick to get bash to indirectly read a variable. - # Thanks StackOverflow! - local var="$1" - echo "${!var}" -} - -# $1: the name of an array variable to read -# FOO_ARR=(a b c) -# BAR_ARR=(1 2 3) -# x=FOO -# indirect_array "${x}_ARR" # -> "a\nb\nc\n" -function indirect_array() { - # This is a trick to get bash to indirectly read an array. - # Thanks StackOverflow! - local arrayname="$1" - # shellcheck disable=SC1087 # intentional - local tmp="$arrayname[@]" - printf -- "%s\n" "${!tmp}" -} - # OpenAPI generation # # Any package that wants open-api functions generated must include a @@ -555,148 +514,73 @@ function codegen::openapi() { local gen_openapi_bin gen_openapi_bin="$(kube::util::find-binary "openapi-gen")" - # Standard dirs which all targets need. - local apimachinery_dirs=( - vendor/k8s.io/apimachinery/pkg/apis/meta/v1 - vendor/k8s.io/apimachinery/pkg/runtime - vendor/k8s.io/apimachinery/pkg/version - ) + local output_dir="pkg/generated/openapi" + local known_violations_file="api/api-rules/violation_exceptions.list" - # These should probably be configured by tags in code-files somewhere. - local targets=( - KUBE - AGGREGATOR - APIEXTENSIONS - CODEGEN - SAMPLEAPISERVER - ) + local report_file="${OUT_DIR}/api_violations.report" + # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write + # updated API violations to the known API violation exceptions list. + if [[ "${UPDATE_API_KNOWN_VIOLATIONS}" == true ]]; then + report_file="${known_violations_file}" + fi - # shellcheck disable=SC2034 # used indirectly - local KUBE_output_dir="pkg/generated/openapi" - # shellcheck disable=SC2034 # used indirectly - local KUBE_known_violations_file="api/api-rules/violation_exceptions.list" - # shellcheck disable=SC2034 # used indirectly - local KUBE_tag_files=() - kube::util::read-array KUBE_tag_files < <( + if [[ "${DBG_CODEGEN}" == 1 ]]; then + kube::log::status "DBG: finding all +k8s:openapi-gen tags" + fi + + local tag_files=() + kube::util::read-array tag_files < <( k8s_tag_files_except \ vendor/k8s.io/code-generator \ vendor/k8s.io/sample-apiserver ) - # shellcheck disable=SC2034 # used indirectly - local AGGREGATOR_output_dir="staging/src/k8s.io/kube-aggregator/pkg/generated/openapi" - # shellcheck disable=SC2034 # used indirectly - local AGGREGATOR_known_violations_file="api/api-rules/aggregator_violation_exceptions.list" - # shellcheck disable=SC2034 # used indirectly - local AGGREGATOR_tag_files=() - kube::util::read-array AGGREGATOR_tag_files < <( - k8s_tag_files_matching \ - vendor/k8s.io/kube-aggregator \ - "${apimachinery_dirs[@]}" - ) + local tag_dirs=() + kube::util::read-array tag_dirs < <( + grep -l --null '+k8s:openapi-gen=' "${tag_files[@]}" \ + | xargs -0 -n1 dirname \ + | sort -u) - # shellcheck disable=SC2034 # used indirectly - local APIEXTENSIONS_output_dir="staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi" - # shellcheck disable=SC2034 # used indirectly - local APIEXTENSIONS_known_violations_file="api/api-rules/apiextensions_violation_exceptions.list" - # shellcheck disable=SC2034 # used indirectly - local APIEXTENSIONS_tag_files=() - kube::util::read-array APIEXTENSIONS_tag_files < <( - k8s_tag_files_matching \ - vendor/k8s.io/apiextensions-apiserver \ - vendor/k8s.io/api/autoscaling/v1 \ - "${apimachinery_dirs[@]}" - ) + if [[ "${DBG_CODEGEN}" == 1 ]]; then + kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:openapi-gen tagged dirs" + fi - # shellcheck disable=SC2034 # used indirectly - local CODEGEN_output_dir="staging/src/k8s.io/code-generator/examples/apiserver/openapi" - # shellcheck disable=SC2034 # used indirectly - local CODEGEN_known_violations_file="api/api-rules/codegen_violation_exceptions.list" - # shellcheck disable=SC2034 # used indirectly - local CODEGEN_tag_files=() - kube::util::read-array CODEGEN_tag_files < <( - k8s_tag_files_matching \ - vendor/k8s.io/code-generator \ - "${apimachinery_dirs[@]}" - ) + local tag_pkgs=() + for dir in "${tag_dirs[@]}"; do + tag_pkgs+=("${PRJ_SRC_PATH}/$dir") + done - # shellcheck disable=SC2034 # used indirectly - local SAMPLEAPISERVER_output_dir="staging/src/k8s.io/sample-apiserver/pkg/generated/openapi" - # shellcheck disable=SC2034 # used indirectly - local SAMPLEAPISERVER_known_violations_file="api/api-rules/sample_apiserver_violation_exceptions.list" - # shellcheck disable=SC2034 # used indirectly - local SAMPLEAPISERVER_tag_files=() - kube::util::read-array SAMPLEAPISERVER_tag_files < <( - k8s_tag_files_matching \ - vendor/k8s.io/sample-apiserver \ - "${apimachinery_dirs[@]}" - ) - - git_find -z ':(glob)**'/"${output_base}.go" | xargs -0 rm -f - - for prefix in "${targets[@]}"; do - local report_file="${OUT_DIR}/${prefix}_violations.report" - # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write - # updated API violations to the known API violation exceptions list. - if [[ "${UPDATE_API_KNOWN_VIOLATIONS}" == true ]]; then - report_file=$(indirect "${prefix}_known_violations_file") - fi - - # 2 lines because shellcheck - local output_dir - output_dir=$(indirect "${prefix}_output_dir") - - if [[ "${DBG_CODEGEN}" == 1 ]]; then - kube::log::status "DBG: finding all +k8s:openapi-gen tags for ${prefix}" - fi - - local tag_dirs=() - kube::util::read-array tag_dirs < <( - grep -l --null '+k8s:openapi-gen=' $(indirect_array "${prefix}_tag_files") \ - | xargs -0 -n1 dirname \ - | sort -u) - - if [[ "${DBG_CODEGEN}" == 1 ]]; then - kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:openapi-gen tagged dirs for ${prefix}" - fi - - local tag_pkgs=() + kube::log::status "Generating openapi code" + if [[ "${DBG_CODEGEN}" == 1 ]]; then + kube::log::status "DBG: running ${gen_openapi_bin} for:" for dir in "${tag_dirs[@]}"; do - tag_pkgs+=("${PRJ_SRC_PATH}/$dir") + kube::log::status "DBG: $dir" done + fi - kube::log::status "Generating openapi code for ${prefix}" - if [[ "${DBG_CODEGEN}" == 1 ]]; then - kube::log::status "DBG: running ${gen_openapi_bin} for:" - for dir in "${tag_dirs[@]}"; do - kube::log::status "DBG: $dir" - done - fi + git_find -z ':(glob)pkg/generated/**'/"${output_base}.go" | xargs -0 rm -f - ./hack/run-in-gopath.sh "${gen_openapi_bin}" \ - --v "${KUBE_VERBOSE}" \ - --logtostderr \ - -h "${BOILERPLATE_FILENAME}" \ - -O "${output_base}" \ - -p "${PRJ_SRC_PATH}/${output_dir}" \ - -r "${report_file}" \ - $(printf -- " -i %s" "${tag_pkgs[@]}") \ - "$@" + ./hack/run-in-gopath.sh "${gen_openapi_bin}" \ + --v "${KUBE_VERBOSE}" \ + --logtostderr \ + -h "${BOILERPLATE_FILENAME}" \ + -O "${output_base}" \ + -p "${PRJ_SRC_PATH}/${output_dir}" \ + -r "${report_file}" \ + $(printf -- " -i %s" "${tag_pkgs[@]}") \ + "$@" - touch "${report_file}" - # 2 lines because shellcheck - local known_filename - known_filename=$(indirect "${prefix}_known_violations_file") - if ! diff -u "${known_filename}" "${report_file}"; then - echo -e "ERROR:" - echo -e "\t'${prefix}' API rule check failed - reported violations differ from known violations" - echo -e "\tPlease read api/api-rules/README.md to resolve the failure in ${known_filename}" - fi + touch "${report_file}" + local known_filename="${known_violations_file}" + if ! diff -u "${known_filename}" "${report_file}"; then + echo -e "ERROR:" + echo -e "\tAPI rule check failed - reported violations differ from known violations" + echo -e "\tPlease read api/api-rules/README.md to resolve the failure in ${known_filename}" + fi - if [[ "${DBG_CODEGEN}" == 1 ]]; then - kube::log::status "Generated openapi code" - fi - done # for each prefix + if [[ "${DBG_CODEGEN}" == 1 ]]; then + kube::log::status "Generated openapi code" + fi } function codegen::applyconfigs() { diff --git a/api/api-rules/apiextensions_violation_exceptions.list b/staging/src/k8s.io/apiextensions-apiserver/api_violation_exceptions.list similarity index 100% rename from api/api-rules/apiextensions_violation_exceptions.list rename to staging/src/k8s.io/apiextensions-apiserver/api_violation_exceptions.list diff --git a/staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh b/staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh index a532fff837a..6ab4ee161e5 100755 --- a/staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh +++ b/staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh @@ -28,11 +28,17 @@ kube::codegen::gen_helpers \ --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \ --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" +if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then + update_report="--update-report" +fi + kube::codegen::gen_openapi \ --input-pkg-root k8s.io/apiextensions-apiserver/pkg \ --extra-pkgs k8s.io/api/autoscaling/v1 `# needed for Scale type` \ --output-pkg-root k8s.io/apiextensions-apiserver/pkg/generated \ --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \ + --report-filename "${SCRIPT_ROOT}/api_violation_exceptions.list" \ + ${update_report:+"${update_report}"} \ --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" kube::codegen::gen_client \ diff --git a/api/api-rules/codegen_violation_exceptions.list b/staging/src/k8s.io/code-generator/examples/api_violation_exceptions.list similarity index 100% rename from api/api-rules/codegen_violation_exceptions.list rename to staging/src/k8s.io/code-generator/examples/api_violation_exceptions.list diff --git a/staging/src/k8s.io/code-generator/examples/hack/update-codegen.sh b/staging/src/k8s.io/code-generator/examples/hack/update-codegen.sh index 3d3a4f3a3bf..e9c65267065 100755 --- a/staging/src/k8s.io/code-generator/examples/hack/update-codegen.sh +++ b/staging/src/k8s.io/code-generator/examples/hack/update-codegen.sh @@ -34,10 +34,16 @@ kube::codegen::gen_helpers \ --output-base "$(dirname "${BASH_SOURCE[0]}")/../../../.." \ --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" +if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then + update_report="--update-report" +fi + kube::codegen::gen_openapi \ --input-pkg-root k8s.io/code-generator/examples/apiserver/apis \ --output-pkg-root k8s.io/code-generator/examples/apiserver \ --output-base "$(dirname "${BASH_SOURCE[0]}")/../../../.." \ + --report-filename "${SCRIPT_ROOT}/api_violation_exceptions.list" \ + ${update_report:+"${update_report}"} \ --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" kube::codegen::gen_client \ diff --git a/staging/src/k8s.io/code-generator/kube_codegen.sh b/staging/src/k8s.io/code-generator/kube_codegen.sh index 59e807929dc..3342b9dcaeb 100755 --- a/staging/src/k8s.io/code-generator/kube_codegen.sh +++ b/staging/src/k8s.io/code-generator/kube_codegen.sh @@ -237,6 +237,9 @@ function kube::codegen::gen_helpers() { # An optional path at which to write an API violations report. "-" means # stdout. # +# --update-report +# If specified, update the report file in place, rather than diffing it. +# # --boilerplate # An optional override for the header file to insert into generated files. # @@ -247,6 +250,7 @@ function kube::codegen::gen_openapi() { local openapi_subdir="openapi" local extra_pkgs=() local report="/dev/null" + local update_report="" local boilerplate="${KUBE_CODEGEN_ROOT}/hack/boilerplate.go.txt" local v="${KUBE_VERBOSE:-0}" @@ -276,6 +280,10 @@ function kube::codegen::gen_openapi() { report="$2" shift 2 ;; + "--update-report") + update_report="true" + shift + ;; "--boilerplate") boilerplate="$2" shift 2 @@ -300,6 +308,12 @@ function kube::codegen::gen_openapi() { return 1 fi + local new_report + new_report="$(mktemp -t "$(basename "$0").api_violations.XXXXXX")" + if [ -n "${update_report}" ]; then + new_report="${report}" + fi + ( # To support running this from anywhere, first cd into this directory, # and then install with forced module mode on and fully qualified name. @@ -348,12 +362,20 @@ function kube::codegen::gen_openapi() { --go-header-file "${boilerplate}" \ --output-base "${out_base}" \ --output-package "${out_pkg_root}/${openapi_subdir}" \ - --report-filename "${report}" \ + --report-filename "${new_report}" \ --input-dirs "k8s.io/apimachinery/pkg/apis/meta/v1" \ --input-dirs "k8s.io/apimachinery/pkg/runtime" \ --input-dirs "k8s.io/apimachinery/pkg/version" \ "${inputs[@]}" fi + + touch "${report}" # in case it doesn't exist yet + if ! diff -u "${report}" "${new_report}"; then + echo -e "ERROR:" + echo -e "\tAPI rule check failed for ${report}: new reported violations" + echo -e "\tPlease read api/api-rules/README.md" + return 1 + fi } # Generate client code diff --git a/api/api-rules/aggregator_violation_exceptions.list b/staging/src/k8s.io/kube-aggregator/api_violation_exceptions.list similarity index 100% rename from api/api-rules/aggregator_violation_exceptions.list rename to staging/src/k8s.io/kube-aggregator/api_violation_exceptions.list diff --git a/staging/src/k8s.io/kube-aggregator/hack/update-codegen.sh b/staging/src/k8s.io/kube-aggregator/hack/update-codegen.sh index 8e77aa5f25b..2fc6d555c6b 100755 --- a/staging/src/k8s.io/kube-aggregator/hack/update-codegen.sh +++ b/staging/src/k8s.io/kube-aggregator/hack/update-codegen.sh @@ -28,10 +28,16 @@ kube::codegen::gen_helpers \ --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \ --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" +if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then + update_report="--update-report" +fi + kube::codegen::gen_openapi \ --input-pkg-root k8s.io/kube-aggregator/pkg/apis \ --output-pkg-root k8s.io/kube-aggregator/pkg/generated \ --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \ + --report-filename "${SCRIPT_ROOT}/api_violation_exceptions.list" \ + ${update_report:+"${update_report}"} \ --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" kube::codegen::gen_client \ diff --git a/api/api-rules/sample_apiserver_violation_exceptions.list b/staging/src/k8s.io/sample-apiserver/api_violation_exceptions.list similarity index 100% rename from api/api-rules/sample_apiserver_violation_exceptions.list rename to staging/src/k8s.io/sample-apiserver/api_violation_exceptions.list diff --git a/staging/src/k8s.io/sample-apiserver/hack/update-codegen.sh b/staging/src/k8s.io/sample-apiserver/hack/update-codegen.sh index be72c4f3694..25b548b1757 100755 --- a/staging/src/k8s.io/sample-apiserver/hack/update-codegen.sh +++ b/staging/src/k8s.io/sample-apiserver/hack/update-codegen.sh @@ -33,10 +33,16 @@ kube::codegen::gen_helpers \ --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \ --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" +if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then + update_report="--update-report" +fi + kube::codegen::gen_openapi \ --input-pkg-root k8s.io/sample-apiserver/pkg/apis \ --output-pkg-root k8s.io/sample-apiserver/pkg/generated \ --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \ + --report-filename "${SCRIPT_ROOT}/api_violation_exceptions.list" \ + ${update_report:+"${update_report}"} \ --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" kube::codegen::gen_client \