Copy make's codegen logic to update-codegen.sh

The `make` rules which auto-generate some of our API stuff are
incredibly baroque, and hard to maintain.  They were originally added on
the assumption that we would stop checking generated files into git.
Since then we have moved away from that goal, and the worst problems
with generated files have been resolved.

Reasons to kill this:
* It is slow on every build, as opposed to just being slow when running
  the generators.  It is even slow to calculate that there's nothing to
  update.
* Most development work doesn't involve changing APIs.
* It only covers about half (or less) of the generated code, and making
  it cover more would be even slower.
* Approximately 1 person knows how this all works.
* We have CI to make sure changes do not get merged without updating
  this code.
* We have corner cases where this does the WRONG thing and tracking
  those down is ugly and hard in perpetuity.

So this commit puts all the same logic that WAS in the
Makefile.generated_files into update-codegen.sh.

I do not love this script, especially WRT sub-packages, but I am trying
not to boil the ocean.  I hope to follow up with some more cleanups over
time.

I have tested this manually and with the scripts and it still seems to
catch errors properly.

This includes a change to kube::util::read-array to make it not unset
variables and not over-write non-array variables.
This commit is contained in:
Tim Hockin 2022-09-23 13:06:35 -07:00
parent b52b50bcea
commit 7afaf23b01
No known key found for this signature in database
3 changed files with 814 additions and 110 deletions

View File

@ -798,7 +798,9 @@ function kube::util::md5() {
# kube::util::read-array
# Reads in stdin and adds it line by line to the array provided. This can be
# used instead of "mapfile -t", and is bash 3 compatible.
# used instead of "mapfile -t", and is bash 3 compatible. If the named array
# exists and is an array, it will be used. Otherwise it will be unset and
# recreated.
#
# Assumed vars:
# $1 (name of array to create/modify)
@ -806,11 +808,44 @@ function kube::util::md5() {
# Example usage:
# kube::util::read-array files < <(ls -1)
#
# When in doubt:
# $ W=abc # a string
# $ X=(a b c) # an array
# $ declare -A Y # an associative array
# $ unset Z # not set at all
# $ declare -p W X Y Z
# declare -- W="abc"
# declare -a X=([0]="a" [1]="b" [2]="c")
# declare -A Y
# bash: line 26: declare: Z: not found
# $ kube::util::read-array W < <(echo -ne "1 1\n2 2\n3 3\n")
# bash: W is defined but isn't an array
# $ kube::util::read-array X < <(echo -ne "1 1\n2 2\n3 3\n")
# $ kube::util::read-array Y < <(echo -ne "1 1\n2 2\n3 3\n")
# bash: Y is defined but isn't an array
# $ kube::util::read-array Z < <(echo -ne "1 1\n2 2\n3 3\n")
# $ declare -p W X Y Z
# declare -- W="abc"
# declare -a X=([0]="1 1" [1]="2 2" [2]="3 3")
# declare -A Y
# declare -a Z=([0]="1 1" [1]="2 2" [2]="3 3")
function kube::util::read-array {
local i=0
unset -v "$1"
while IFS= read -r "$1[i++]"; do :; done
eval "[[ \${$1[--i]} ]]" || unset "$1[i]" # ensures last element isn't empty
if [[ -z "$1" ]]; then
echo "usage: ${FUNCNAME[0]} <varname>" >&2
return 1
fi
if [[ -n $(declare -p "$1" 2>/dev/null) ]]; then
if ! declare -p "$1" 2>/dev/null | grep -q '^declare -a'; then
echo "${FUNCNAME[0]}: $1 is defined but isn't an array" >&2
return 2
fi
fi
# shellcheck disable=SC2034 # this variable _is_ used
local __read_array_i=0
while IFS= read -r "$1[__read_array_i++]"; do :; done
if ! eval "[[ \${$1[--__read_array_i]} ]]"; then
unset "$1[__read_array_i]" # ensures last element isn't empty
fi
}
# Some useful colors.

View File

@ -1,5 +1,4 @@
#!/usr/bin/env bash
# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@ -14,117 +13,783 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Please do not add any logic to this shell script. Add logic to the go code
# that generates the set-gen program.
# shellcheck disable=2046 # printf word-splitting is intentional
set -o errexit
set -o nounset
set -o pipefail
# This tool wants a different default than usual.
KUBE_VERBOSE="${KUBE_VERBOSE:-1}"
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
kube::golang::setup_env
GO111MODULE=on GOPROXY=off go install k8s.io/kubernetes/pkg/generated/openapi/cmd/models-schema
GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/applyconfiguration-gen
GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/client-gen
GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/lister-gen
GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/informer-gen
DBG_CODEGEN="${DBG_CODEGEN:-0}"
GENERATED_FILE_PREFIX="${GENERATED_FILE_PREFIX:-zz_generated.}"
UPDATE_API_KNOWN_VIOLATIONS="${UPDATE_API_KNOWN_VIOLATIONS:-}"
modelsschema=$(kube::util::find-binary "models-schema")
applyconfigurationgen=$(kube::util::find-binary "applyconfiguration-gen")
clientgen=$(kube::util::find-binary "client-gen")
listergen=$(kube::util::find-binary "lister-gen")
informergen=$(kube::util::find-binary "informer-gen")
OUT_DIR="_output"
BIN_DIR="${OUT_DIR}/bin"
PRJ_SRC_PATH="k8s.io/kubernetes"
BOILERPLATE_FILENAME="vendor/k8s.io/code-generator/hack/boilerplate.go.txt"
APPLYCONFIG_PKG="k8s.io/client-go/applyconfigurations"
IFS=" " read -r -a GROUP_VERSIONS <<< "${KUBE_AVAILABLE_GROUP_VERSIONS}"
GV_DIRS=()
for gv in "${GROUP_VERSIONS[@]}"; do
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: starting generated_files"
fi
# This variable holds a list of every directory that contains Go files in this
# project. Other rules and variables can use this as a starting point to
# reduce filesystem accesses.
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: finding all *.go dirs"
fi
ALL_GO_DIRS=()
kube::util::read-array ALL_GO_DIRS < \
<(hack/make-rules/helpers/cache_go_dirs.sh .make/all_go_dirs)
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: found ${#ALL_GO_DIRS[@]} *.go dirs"
fi
# Generate a list of all files that have a `+k8s:` comment-tag. This will be
# used to derive lists of files/dirs for generation tools.
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: finding all +k8s: tags"
fi
ALL_K8S_TAG_FILES=()
kube::util::read-array ALL_K8S_TAG_FILES < <(
find "${ALL_GO_DIRS[@]}" -maxdepth 1 -type f -name \*.go -print0 \
| xargs -0 grep --color=never -l '^// *+k8s:')
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: found ${#ALL_K8S_TAG_FILES[@]} +k8s: tagged files"
fi
#
# Code generation logic.
#
# prerelease-lifecycle generation
#
# Any package that wants prerelease-lifecycle functions generated must include a
# comment-tag in column 0 of one file of the form:
# // +k8s:prerelease-lifecycle-gen=true
function codegen::prerelease() {
# Build the tool.
hack/make-rules/build.sh k8s.io/code-generator/cmd/prerelease-lifecycle-gen
# The result file, in each pkg, of prerelease-lifecycle generation.
local output_base="${GENERATED_FILE_PREFIX}prerelease-lifecycle"
# The tool used to generate prerelease-lifecycle code.
local gen_prerelease_bin="${BIN_DIR}/prerelease-lifecycle-gen"
# Find all the directories that request prerelease-lifecycle generation.
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: finding all +k8s:prerelease-lifecycle-gen tags"
fi
local tag_dirs=()
kube::util::read-array tag_dirs < <( \
grep --color=never -l '+k8s:prerelease-lifecycle-gen=true' "${ALL_K8S_TAG_FILES[@]}" \
| xargs -n1 dirname \
| LC_ALL=C sort -u)
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:prerelease-lifecycle-gen tagged dirs"
fi
local tag_pkgs=()
for dir in "${tag_dirs[@]}"; do
tag_pkgs+=("${PRJ_SRC_PATH}/$dir")
done
kube::log::status "Generating prerelease-lifecycle code for ${#tag_pkgs[@]} targets"
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: running ${gen_prerelease_bin} for:"
for dir in "${tag_dirs[@]}"; do
kube::log::status "DBG: $dir"
done
fi
./hack/run-in-gopath.sh "${gen_prerelease_bin}" \
--v "${KUBE_VERBOSE}" \
--logtostderr \
-h "${BOILERPLATE_FILENAME}" \
-O "${output_base}" \
$(printf -- " -i %s" "${tag_pkgs[@]}") \
"$@"
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "Generated prerelease-lifecycle code"
fi
}
# Deep-copy generation
#
# Any package that wants deep-copy functions generated must include a
# comment-tag in column 0 of one file of the form:
# // +k8s:deepcopy-gen=<VALUE>
#
# The <VALUE> may be one of:
# generate: generate deep-copy functions into the package
# register: generate deep-copy functions and register them with a
# scheme
function codegen::deepcopy() {
# Build the tool.
hack/make-rules/build.sh k8s.io/code-generator/cmd/deepcopy-gen
# The result file, in each pkg, of deep-copy generation.
local output_base="${GENERATED_FILE_PREFIX}deepcopy"
# The tool used to generate deep copies.
local gen_deepcopy_bin="${BIN_DIR}/deepcopy-gen"
# Find all the directories that request deep-copy generation.
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: finding all +k8s:deepcopy-gen tags"
fi
local tag_dirs=()
kube::util::read-array tag_dirs < <( \
grep --color=never -l '+k8s:deepcopy-gen=' "${ALL_K8S_TAG_FILES[@]}" \
| xargs -n1 dirname \
| LC_ALL=C sort -u)
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:deepcopy-gen tagged dirs"
fi
local tag_pkgs=()
for dir in "${tag_dirs[@]}"; do
tag_pkgs+=("${PRJ_SRC_PATH}/$dir")
done
kube::log::status "Generating deepcopy code for ${#tag_pkgs[@]} targets"
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: running ${gen_deepcopy_bin} for:"
for dir in "${tag_dirs[@]}"; do
kube::log::status "DBG: $dir"
done
fi
./hack/run-in-gopath.sh "${gen_deepcopy_bin}" \
--v "${KUBE_VERBOSE}" \
--logtostderr \
-h "${BOILERPLATE_FILENAME}" \
-O "${output_base}" \
--bounding-dirs "${PRJ_SRC_PATH},k8s.io/api" \
$(printf -- " -i %s" "${tag_pkgs[@]}") \
"$@"
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "Generated deepcopy code"
fi
}
# Defaulter generation
#
# Any package that wants defaulter functions generated must include a
# comment-tag in column 0 of one file of the form:
# // +k8s:defaulter-gen=<VALUE>
#
# The <VALUE> depends on context:
# on types:
# true: always generate a defaulter for this type
# false: never generate a defaulter for this type
# on functions:
# covers: if the function name matches SetDefault_NAME, instructs
# the generator not to recurse
# on packages:
# FIELDNAME: any object with a field of this name is a candidate
# for having a defaulter generated
function codegen::defaults() {
# Build the tool.
hack/make-rules/build.sh k8s.io/code-generator/cmd/defaulter-gen
# The result file, in each pkg, of defaulter generation.
local output_base="${GENERATED_FILE_PREFIX}defaults"
# The tool used to generate defaulters.
local gen_defaulter_bin="${BIN_DIR}/defaulter-gen"
# All directories that request any form of defaulter generation.
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: finding all +k8s:defaulter-gen tags"
fi
local tag_dirs=()
kube::util::read-array tag_dirs < <( \
grep --color=never -l '+k8s:defaulter-gen=' "${ALL_K8S_TAG_FILES[@]}" \
| xargs -n1 dirname \
| LC_ALL=C sort -u)
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:defaulter-gen tagged dirs"
fi
local tag_pkgs=()
for dir in "${tag_dirs[@]}"; do
tag_pkgs+=("${PRJ_SRC_PATH}/$dir")
done
kube::log::status "Generating defaulter code for ${#tag_pkgs[@]} targets"
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: running ${gen_defaulter_bin} for:"
for dir in "${tag_dirs[@]}"; do
kube::log::status "DBG: $dir"
done
fi
./hack/run-in-gopath.sh "${gen_defaulter_bin}" \
--v "${KUBE_VERBOSE}" \
--logtostderr \
-h "${BOILERPLATE_FILENAME}" \
-O "${output_base}" \
$(printf -- " --extra-peer-dirs %s" "${tag_pkgs[@]}") \
$(printf -- " -i %s" "${tag_pkgs[@]}") \
"$@"
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "Generated defaulter code"
fi
}
# Conversion generation
# Any package that wants conversion functions generated into it must
# include one or more comment-tags in its `doc.go` file, of the form:
# // +k8s:conversion-gen=<INTERNAL_TYPES_DIR>
#
# The INTERNAL_TYPES_DIR is a project-local path to another directory
# which should be considered when evaluating peer types for
# conversions. An optional additional comment of the form
# // +k8s:conversion-gen-external-types=<EXTERNAL_TYPES_DIR>
#
# identifies where to find the external types; if there is no such
# comment then the external types are sought in the package where the
# `k8s:conversion` tag is found.
#
# Conversions, in both directions, are generated for every type name
# that is defined in both an internal types package and the external
# types package.
#
# TODO: it might be better in the long term to make peer-types explicit in the
# IDL.
function codegen::conversions() {
# Build the tool.
hack/make-rules/build.sh k8s.io/code-generator/cmd/conversion-gen
# The result file, in each pkg, of conversion generation.
local output_base="${GENERATED_FILE_PREFIX}conversion"
# The tool used to generate conversions.
local gen_conversion_bin="${BIN_DIR}/conversion-gen"
# All directories that request any form of conversion generation.
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: finding all +k8s:conversion-gen tags"
fi
local tag_dirs=()
kube::util::read-array tag_dirs < <(\
grep --color=never -l '^// *+k8s:conversion-gen=' "${ALL_K8S_TAG_FILES[@]}" \
| xargs -n1 dirname \
| LC_ALL=C sort -u)
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:conversion-gen tagged dirs"
fi
local tag_pkgs=()
for dir in "${tag_dirs[@]}"; do
tag_pkgs+=("${PRJ_SRC_PATH}/$dir")
done
local extra_peer_pkgs=(
k8s.io/kubernetes/pkg/apis/core
k8s.io/kubernetes/pkg/apis/core/v1
k8s.io/api/core/v1
)
kube::log::status "Generating conversion code for ${#tag_pkgs[@]} targets"
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: running ${gen_conversion_bin} for:"
for dir in "${tag_dirs[@]}"; do
kube::log::status "DBG: $dir"
done
fi
./hack/run-in-gopath.sh "${gen_conversion_bin}" \
--v "${KUBE_VERBOSE}" \
--logtostderr \
-h "${BOILERPLATE_FILENAME}" \
-O "${output_base}" \
$(printf -- " --extra-peer-dirs %s" "${extra_peer_pkgs[@]}") \
$(printf -- " --extra-dirs %s" "${tag_pkgs[@]}") \
$(printf -- " -i %s" "${tag_pkgs[@]}") \
"$@"
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "Generated conversion code"
fi
}
# $@: directories to exclude
# example:
# k8s_tag_files_except foo bat/qux
function k8s_tag_files_except() {
for f in "${ALL_K8S_TAG_FILES[@]}"; do
local excl=""
for x in "$@"; do
if [[ "$f" =~ "$x"/.* ]]; then
excl="true"
break
fi
done
if [[ "${excl}" != true ]]; then
echo "$f"
fi
done
}
# $@: directories to exclude
# example:
# k8s_tag_files_matching foo bat/qux
function k8s_tag_files_matching() {
for f in "${ALL_K8S_TAG_FILES[@]}"; do
for x in "$@"; do
if [[ "$f" =~ "${x}"/.* ]]; then
echo "$f"
break
fi
done
done
}
# $1: the name of a scalar variable to read
# example:
# FOO_VAR="foo value"
# BAR_VAR="bar value"
# x=FOO
# indirect "${x}_VAR" # -> "foo value\n"
function indirect() {
# This is a trick to get bash to indirectly read a variable.
# Thanks StackOverflow!
local var="$1"
echo "${!var}"
}
# $1: the name of an array variable to read
# FOO_ARR=(a b c)
# BAR_ARR=(1 2 3)
# x=FOO
# indirect_array "${x}_ARR" # -> "a\nb\nc\n"
function indirect_array() {
# This is a trick to get bash to indirectly read an array.
# Thanks StackOverflow!
local arrayname="$1"
# shellcheck disable=SC1087 # intentional
local tmp="$arrayname[@]"
printf -- "%s\n" "${!tmp}"
}
# OpenAPI generation
#
# Any package that wants open-api functions generated must include a
# comment-tag in column 0 of one file of the form:
# // +k8s:openapi-gen=true
function codegen::openapi() {
# Build the tool.
hack/make-rules/build.sh k8s.io/kube-openapi/cmd/openapi-gen
# The result file, in each pkg, of open-api generation.
local output_base="${GENERATED_FILE_PREFIX}openapi"
# The tool used to generate open apis.
local gen_openapi_bin="${BIN_DIR}/openapi-gen"
# Standard dirs which all targets need.
local apimachinery_dirs=(
vendor/k8s.io/apimachinery/pkg/apis/meta/v1
vendor/k8s.io/apimachinery/pkg/runtime
vendor/k8s.io/apimachinery/pkg/version
)
# These should probably be configured by tags in code-files somewhere.
local targets=(
KUBE
AGGREGATOR
APIEXTENSIONS
CODEGEN
SAMPLEAPISERVER
)
# shellcheck disable=SC2034 # used indirectly
local KUBE_output_dir="pkg/generated/openapi"
# shellcheck disable=SC2034 # used indirectly
local KUBE_known_violations_file="api/api-rules/violation_exceptions.list"
# shellcheck disable=SC2034 # used indirectly
local KUBE_tag_files=()
kube::util::read-array KUBE_tag_files < <(
k8s_tag_files_except \
vendor/k8s.io/code-generator \
vendor/k8s.io/sample-apiserver
)
# shellcheck disable=SC2034 # used indirectly
local AGGREGATOR_output_dir="staging/src/k8s.io/kube-aggregator/pkg/generated/openapi"
# shellcheck disable=SC2034 # used indirectly
local AGGREGATOR_known_violations_file="api/api-rules/aggregator_violation_exceptions.list"
# shellcheck disable=SC2034 # used indirectly
local AGGREGATOR_tag_files=()
kube::util::read-array AGGREGATOR_tag_files < <(
k8s_tag_files_matching \
vendor/k8s.io/kube-aggregator \
"${apimachinery_dirs[@]}"
)
# shellcheck disable=SC2034 # used indirectly
local APIEXTENSIONS_output_dir="staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi"
# shellcheck disable=SC2034 # used indirectly
local APIEXTENSIONS_known_violations_file="api/api-rules/apiextensions_violation_exceptions.list"
# shellcheck disable=SC2034 # used indirectly
local APIEXTENSIONS_tag_files=()
kube::util::read-array APIEXTENSIONS_tag_files < <(
k8s_tag_files_matching \
vendor/k8s.io/apiextensions \
vendor/k8s.io/api/autoscaling/v1 \
"${apimachinery_dirs[@]}"
)
# shellcheck disable=SC2034 # used indirectly
local CODEGEN_output_dir="staging/src/k8s.io/code-generator/examples/apiserver/openapi"
# shellcheck disable=SC2034 # used indirectly
local CODEGEN_known_violations_file="api/api-rules/codegen_violation_exceptions.list"
# shellcheck disable=SC2034 # used indirectly
local CODEGEN_tag_files=()
kube::util::read-array CODEGEN_tag_files < <(
k8s_tag_files_matching \
vendor/k8s.io/code-generator \
"${apimachinery_dirs[@]}"
)
# shellcheck disable=SC2034 # used indirectly
local SAMPLEAPISERVER_output_dir="staging/src/k8s.io/sample-apiserver/pkg/generated/openapi"
# shellcheck disable=SC2034 # used indirectly
local SAMPLEAPISERVER_known_violations_file="api/api-rules/sample_apiserver_violation_exceptions.list"
# shellcheck disable=SC2034 # used indirectly
local SAMPLEAPISERVER_tag_files=()
kube::util::read-array SAMPLEAPISERVER_tag_files < <(
k8s_tag_files_matching \
vendor/k8s.io/sample-apiserver \
"${apimachinery_dirs[@]}"
)
for prefix in "${targets[@]}"; do
local report_file="${OUT_DIR}/${prefix}_violations.report"
# When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write
# updated API violations to the known API violation exceptions list.
if [[ "${UPDATE_API_KNOWN_VIOLATIONS}" == true ]]; then
report_file=$(indirect "${prefix}_known_violations_file")
fi
# 2 lines because shellcheck
local output_dir
output_dir=$(indirect "${prefix}_output_dir")
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: finding all +k8s:openapi-gen tags for ${prefix}"
fi
local tag_dirs=()
kube::util::read-array tag_dirs < <(
grep --color=never -l '+k8s:openapi-gen=' $(indirect_array "${prefix}_tag_files") \
| xargs -n1 dirname \
| LC_ALL=C sort -u
)
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:openapi-gen tagged dirs for ${prefix}"
fi
local tag_pkgs=()
for dir in "${tag_dirs[@]}"; do
tag_pkgs+=("${PRJ_SRC_PATH}/$dir")
done
kube::log::status "Generating openapi code for ${prefix}"
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: running ${gen_openapi_bin} for:"
for dir in "${tag_dirs[@]}"; do
kube::log::status "DBG: $dir"
done
fi
./hack/run-in-gopath.sh ${gen_openapi_bin} \
--v "${KUBE_VERBOSE}" \
--logtostderr \
-h "${BOILERPLATE_FILENAME}" \
-O "${output_base}" \
-p "${PRJ_SRC_PATH}/${output_dir}" \
-r "${report_file}" \
$(printf -- " -i %s" "${tag_pkgs[@]}") \
"$@"
touch "${report_file}"
# 2 lines because shellcheck
local known_filename
known_filename=$(indirect "${prefix}_known_violations_file")
if ! diff -u "${known_filename}" "${report_file}"; then
echo -e "ERROR:"
echo -e "\t'${prefix}' API rule check failed - reported violations differ from known violations"
echo -e "\tPlease read api/api-rules/README.md to resolve the failure in ${known_filename}"
fi
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "Generated openapi code"
fi
done # for each prefix
}
function codegen::applyconfigs() {
GO111MODULE=on GOPROXY=off go install \
k8s.io/kubernetes/pkg/generated/openapi/cmd/models-schema \
k8s.io/code-generator/cmd/applyconfiguration-gen
local modelsschema
modelsschema=$(kube::util::find-binary "models-schema")
local applyconfigurationgen
applyconfigurationgen=$(kube::util::find-binary "applyconfiguration-gen")
# because client-gen doesn't do policy/v1alpha1, we have to skip it too
local ext_apis=()
kube::util::read-array ext_apis < <(
cd "${KUBE_ROOT}/staging/src"
find k8s.io/api -name types.go -print0 | xargs -0 -n1 dirname | sort | grep -v pkg.apis.policy.v1alpha1
)
ext_apis+=("k8s.io/apimachinery/pkg/apis/meta/v1")
kube::log::status "Generating apply-config code for ${#ext_apis[@]} targets"
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: running ${applyconfigurationgen} for:"
for api in "${ext_apis[@]}"; do
kube::log::status "DBG: $api"
done
fi
"${applyconfigurationgen}" \
--openapi-schema <("${modelsschema}") \
--go-header-file "${BOILERPLATE_FILENAME}" \
--output-base "${KUBE_ROOT}/vendor" \
--output-package "${APPLYCONFIG_PKG}" \
$(printf -- " --input-dirs %s" "${ext_apis[@]}") \
"$@"
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "Generated apply-config code"
fi
}
function codegen::clients() {
GO111MODULE=on GOPROXY=off go install \
k8s.io/code-generator/cmd/client-gen
local clientgen
clientgen=$(kube::util::find-binary "client-gen")
IFS=" " read -r -a group_versions <<< "${KUBE_AVAILABLE_GROUP_VERSIONS}"
local gv_dirs=()
for gv in "${group_versions[@]}"; do
# add items, but strip off any leading apis/ you find to match command expectations
local api_dir
api_dir=$(kube::util::group-version-to-pkg-path "${gv}")
nopkg_dir=${api_dir#pkg/}
local nopkg_dir=${api_dir#pkg/}
nopkg_dir=${nopkg_dir#vendor/k8s.io/api/}
pkg_dir=${nopkg_dir#apis/}
local pkg_dir=${nopkg_dir#apis/}
# skip groups that aren't being served, clients for these don't matter
if [[ " ${KUBE_NONSERVER_GROUP_VERSIONS} " == *" ${gv} "* ]]; then
continue
fi
GV_DIRS+=("${pkg_dir}")
done
# delimit by commas for the command
GV_DIRS_CSV=$(IFS=',';echo "${GV_DIRS[*]// /,}";IFS=$)
gv_dirs+=("${pkg_dir}")
done
applyconfigurationgen_external_apis=()
# because client-gen doesn't do policy/v1alpha1, we have to skip it too
kube::util::read-array applyconfigurationgen_external_apis < <(
cd "${KUBE_ROOT}/staging/src"
find k8s.io/api -name types.go -print0 | xargs -0 -n1 dirname | sort | grep -v pkg.apis.policy.v1alpha1
)
applyconfigurationgen_external_apis+=("k8s.io/apimachinery/pkg/apis/meta/v1")
applyconfigurationgen_external_apis_csv=$(IFS=,; echo "${applyconfigurationgen_external_apis[*]}")
applyconfigurations_package="k8s.io/client-go/applyconfigurations"
${applyconfigurationgen} \
--openapi-schema <(${modelsschema}) \
--output-base "${KUBE_ROOT}/vendor" \
--output-package "${applyconfigurations_package}" \
--input-dirs "${applyconfigurationgen_external_apis_csv}" \
--go-header-file "${KUBE_ROOT}/hack/boilerplate/boilerplate.generatego.txt" \
"$@"
kube::log::status "Generating client code for ${#gv_dirs[@]} targets"
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: running ${clientgen} for:"
for dir in "${gv_dirs[@]}"; do
kube::log::status "DBG: $dir"
done
fi
# This can be called with one flag, --verify-only, so it works for both the
# update- and verify- scripts.
${clientgen} \
"${clientgen}" \
--go-header-file "${BOILERPLATE_FILENAME}" \
--output-base "${KUBE_ROOT}/vendor" \
--output-package="k8s.io/client-go" \
--clientset-name="kubernetes" \
--input-base="k8s.io/api" \
--input="${GV_DIRS_CSV}" \
--apply-configuration-package "${applyconfigurations_package}" \
--go-header-file "${KUBE_ROOT}/hack/boilerplate/boilerplate.generatego.txt" \
--apply-configuration-package "${APPLYCONFIG_PKG}" \
$(printf -- " --input %s" "${gv_dirs[@]}") \
"$@"
listergen_external_apis=()
kube::util::read-array listergen_external_apis < <(
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "Generated client code"
fi
}
function codegen::listers() {
GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/lister-gen
local listergen
listergen=$(kube::util::find-binary "lister-gen")
local ext_apis=()
kube::util::read-array ext_apis < <(
cd "${KUBE_ROOT}/staging/src"
find k8s.io/api -name types.go -print0 | xargs -0 -n1 dirname | sort
)
listergen_external_apis_csv=$(IFS=,; echo "${listergen_external_apis[*]}")
${listergen} \
)
kube::log::status "Generating lister code for ${#ext_apis[@]} targets"
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: running ${listergen} for:"
for api in "${ext_apis[@]}"; do
kube::log::status "DBG: $api"
done
fi
"${listergen}" \
--go-header-file "${BOILERPLATE_FILENAME}" \
--output-base "${KUBE_ROOT}/vendor" \
--output-package "k8s.io/client-go/listers" \
--input-dirs "${listergen_external_apis_csv}" \
--go-header-file "${KUBE_ROOT}/hack/boilerplate/boilerplate.generatego.txt" \
$(printf -- " --input-dirs %s" "${ext_apis[@]}") \
"$@"
informergen_external_apis=()
# because client-gen doesn't do policy/v1alpha1, we have to skip it too
kube::util::read-array informergen_external_apis < <(
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "Generated lister code"
fi
}
function codegen::informers() {
GO111MODULE=on GOPROXY=off go install \
k8s.io/code-generator/cmd/informer-gen
local informergen
informergen=$(kube::util::find-binary "informer-gen")
# because client-gen doesn't do policy/v1alpha1, we have to skip it too
local ext_apis=()
kube::util::read-array ext_apis < <(
cd "${KUBE_ROOT}/staging/src"
find k8s.io/api -name types.go -print0 | xargs -0 -n1 dirname | sort | grep -v pkg.apis.policy.v1alpha1
)
informergen_external_apis_csv=$(IFS=,; echo "${informergen_external_apis[*]}")
)
${informergen} \
kube::log::status "Generating informer code for ${#ext_apis[@]} targets"
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "DBG: running ${informergen} for:"
for api in "${ext_apis[@]}"; do
kube::log::status "DBG: $api"
done
fi
"${informergen}" \
--go-header-file "${BOILERPLATE_FILENAME}" \
--output-base "${KUBE_ROOT}/vendor" \
--output-package "k8s.io/client-go/informers" \
--single-directory \
--input-dirs "${informergen_external_apis_csv}" \
--versioned-clientset-package k8s.io/client-go/kubernetes \
--listers-package k8s.io/client-go/listers \
--go-header-file "${KUBE_ROOT}/hack/boilerplate/boilerplate.generatego.txt" \
$(printf -- " --input-dirs %s" "${ext_apis[@]}") \
"$@"
# You may add additional calls of code generators like set-gen above.
if [[ "${DBG_CODEGEN}" == 1 ]]; then
kube::log::status "Generated informer code"
fi
}
# call generation on sub-project for now
CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/code-generator/hack/update-codegen.sh
CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/kube-aggregator/hack/update-codegen.sh
CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/sample-apiserver/hack/update-codegen.sh
CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/sample-controller/hack/update-codegen.sh
CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/apiextensions-apiserver/hack/update-codegen.sh
CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/metrics/hack/update-codegen.sh
CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/apiextensions-apiserver/examples/client-go/hack/update-codegen.sh
function codegen::subprojects() {
# Call generation on sub-projects.
# TODO(thockin): make these take a list of codegens and flags
local subs=(
vendor/k8s.io/code-generator/hack/update-codegen.sh
vendor/k8s.io/kube-aggregator/hack/update-codegen.sh
vendor/k8s.io/sample-apiserver/hack/update-codegen.sh
vendor/k8s.io/sample-controller/hack/update-codegen.sh
vendor/k8s.io/apiextensions-apiserver/hack/update-codegen.sh
vendor/k8s.io/metrics/hack/update-codegen.sh
vendor/k8s.io/apiextensions-apiserver/examples/client-go/hack/update-codegen.sh
)
for s in "${subs[@]}"; do
CODEGEN_PKG=./vendor/k8s.io/code-generator "$s"
done
}
#
# main
#
function list_codegens() {
(
shopt -s extdebug
declare -F \
| cut -f3 -d' ' \
| grep "^codegen::" \
| while read -r fn; do declare -F "$fn"; done \
| sort -n -k2 \
| cut -f1 -d' ' \
| sed 's/^codegen:://'
)
}
# shellcheck disable=SC2207 # safe, no functions have spaces
all_codegens=($(list_codegens))
function print_codegens() {
echo "available codegens:"
for g in "${all_codegens[@]}"; do
echo " $g"
done
}
# Validate and accumulate flags to pass thru and codegens to run if args are
# specified.
flags_to_pass=()
codegens_to_run=()
for arg; do
# Use -? to list known codegens.
if [[ "${arg}" == "-?" ]]; then
print_codegens
exit 0
fi
if [[ "${arg}" =~ ^- ]]; then
flags_to_pass+=("${arg}")
continue
fi
# Make sure each non-flag arg matches at least one codegen.
nmatches=0
for t in "${all_codegens[@]}"; do
if [[ "$t" =~ ${arg} ]]; then
nmatches=$((nmatches+1))
# Don't run codegens twice, just keep the first match.
# shellcheck disable=SC2076 # we want literal matching
if [[ " ${codegens_to_run[*]} " =~ " $t " ]]; then
continue
fi
codegens_to_run+=("$t")
continue
fi
done
if [[ ${nmatches} == 0 ]]; then
echo "ERROR: no codegens match pattern '${arg}'"
echo
print_codegens
exit 1
fi
codegens_to_run+=("${matches[@]}")
done
# If no codegens were specified, run them all.
if [[ "${#codegens_to_run[@]}" == 0 ]]; then
codegens_to_run=("${all_codegens[@]}")
fi
for g in "${codegens_to_run[@]}"; do
"codegen::${g}" "${flags_to_pass[@]}"
done

View File

@ -32,11 +32,15 @@ kube::golang::setup_env
#
# Note: these must be before the main script call because the later calls the sub-project's
# update-codegen.sh scripts. We wouldn't see any error on changes then.
CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/code-generator/hack/verify-codegen.sh
CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/kube-aggregator/hack/verify-codegen.sh
CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/sample-apiserver/hack/verify-codegen.sh
CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/sample-controller/hack/verify-codegen.sh
CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/apiextensions-apiserver/hack/verify-codegen.sh
CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/metrics/hack/verify-codegen.sh
export CODEGEN_PKG=./vendor/k8s.io/code-generator
vendor/k8s.io/code-generator/hack/verify-codegen.sh
vendor/k8s.io/kube-aggregator/hack/verify-codegen.sh
vendor/k8s.io/sample-apiserver/hack/verify-codegen.sh
vendor/k8s.io/sample-controller/hack/verify-codegen.sh
vendor/k8s.io/apiextensions-apiserver/hack/verify-codegen.sh
vendor/k8s.io/metrics/hack/verify-codegen.sh
"${KUBE_ROOT}/hack/update-codegen.sh" --verify-only
# This won't actually update anything because of --verify-only, but it tells
# the openapi tool to verify against the real filenames.
export UPDATE_API_KNOWN_VIOLATIONS=true
"${KUBE_ROOT}/hack/update-codegen.sh" --verify-only "$@"