diff --git a/api/api-rules/apiextensions_violation_exceptions.list b/api/api-rules/apiextensions_violation_exceptions.list new file mode 100644 index 00000000000..791d6593f98 --- /dev/null +++ b/api/api-rules/apiextensions_violation_exceptions.list @@ -0,0 +1,20 @@ +API rule violation: names_match,k8s.io/apimachinery/pkg/api/resource,Quantity,Format +API rule violation: names_match,k8s.io/apimachinery/pkg/api/resource,Quantity,d +API rule violation: names_match,k8s.io/apimachinery/pkg/api/resource,Quantity,i +API rule violation: names_match,k8s.io/apimachinery/pkg/api/resource,Quantity,s +API rule violation: names_match,k8s.io/apimachinery/pkg/api/resource,int64Amount,scale +API rule violation: names_match,k8s.io/apimachinery/pkg/api/resource,int64Amount,value +API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,APIResourceList,APIResources +API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,Duration,Duration +API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,InternalEvent,Object +API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,InternalEvent,Type +API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,MicroTime,Time +API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,StatusCause,Type +API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,Time,Time +API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,RawExtension,Raw +API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,ContentEncoding +API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,ContentType +API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,Raw +API rule violation: names_match,k8s.io/apimachinery/pkg/util/intstr,IntOrString,IntVal +API rule violation: names_match,k8s.io/apimachinery/pkg/util/intstr,IntOrString,StrVal +API rule violation: names_match,k8s.io/apimachinery/pkg/util/intstr,IntOrString,Type diff --git a/api/api-rules/codegen_violation_exceptions.list b/api/api-rules/codegen_violation_exceptions.list new file mode 100644 index 00000000000..9618d9169e1 --- /dev/null +++ b/api/api-rules/codegen_violation_exceptions.list @@ -0,0 +1,13 @@ +API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,APIResourceList,APIResources +API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,Duration,Duration +API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,InternalEvent,Object +API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,InternalEvent,Type +API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,MicroTime,Time +API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,StatusCause,Type +API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,Time,Time +API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,RawExtension,Raw +API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,ContentEncoding +API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,ContentType +API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,Raw +API rule violation: names_match,k8s.io/code-generator/_examples/apiserver/apis/example/v1,TestTypeStatus,Blah +API rule violation: names_match,k8s.io/code-generator/_examples/apiserver/apis/example2/v1,TestTypeStatus,Blah diff --git a/api/api-rules/sample_apiserver_violation_exceptions.list b/api/api-rules/sample_apiserver_violation_exceptions.list index c211037baf7..3bfae5c07ae 100644 --- a/api/api-rules/sample_apiserver_violation_exceptions.list +++ b/api/api-rules/sample_apiserver_violation_exceptions.list @@ -5,3 +5,7 @@ API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,InternalEve API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,MicroTime,Time API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,StatusCause,Type API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,Time,Time +API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,RawExtension,Raw +API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,ContentEncoding +API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,ContentType +API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,Raw diff --git a/build/root/Makefile.generated_files b/build/root/Makefile.generated_files index 4892325569d..abe198b2716 100644 --- a/build/root/Makefile.generated_files +++ b/build/root/Makefile.generated_files @@ -408,81 +408,102 @@ $(CONVERSION_GEN): $(k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/conversi # The result file, in each pkg, of open-api generation. OPENAPI_BASENAME := $(GENERATED_FILE_PREFIX)openapi OPENAPI_FILENAME := $(OPENAPI_BASENAME).go -OPENAPI_OUTPUT_PKG := pkg/generated/openapi -CRD_OPENAPI_OUTPUT_PKG := staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi BOILERPLATE_FILENAME := vendor/k8s.io/code-generator/hack/boilerplate.go.txt -REPORT_FILENAME := $(OUT_DIR)/violations.report IGNORED_REPORT_FILENAME := $(OUT_DIR)/ignored_violations.report -KNOWN_VIOLATION_FILENAME := api/api-rules/violation_exceptions.list -# When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write -# updated API violations to the known API violation exceptions list. -ifeq ($(UPDATE_API_KNOWN_VIOLATIONS),true) - REPORT_FILENAME:=$(KNOWN_VIOLATION_FILENAME) - # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, touch the exceptions - # list so that the OPENAPI_OUTFILE target re-run instead of being cached. - $(shell touch $(KNOWN_VIOLATION_FILENAME)) -endif -API_RULE_CHECK_FAILURE_MESSAGE := "ERROR: \n\t API rule check failed. Reported violations differ from known violations. Please read api/api-rules/README.md to resolve the failure. \n" + +API_RULE_CHECK_FAILURE_MESSAGE = "ERROR: \n\t $(1) API rule check failed. Reported violations differ from known violations. Please read api/api-rules/README.md to resolve the failure in $(2). \n" # The tool used to generate open apis. OPENAPI_GEN := $(BIN_DIR)/openapi-gen -# Find all the directories that request open-api generation. -ifeq ($(DBG_MAKEFILE),1) - $(warning ***** finding all +k8s:openapi-gen tags) +KUBE_KNOWN_VIOLATION_FILENAME := api/api-rules/violation_exceptions.list +APIEXTENSIONS_KNOWN_VIOLATION_FILENAME := api/api-rules/apiextensions_violation_exceptions.list +CODEGEN_KNOWN_VIOLATION_FILENAME := api/api-rules/codegen_violation_exceptions.list +SAMPLEAPISERVER_KNOWN_VIOLATION_FILENAME := api/api-rules/sample_apiserver_violation_exceptions.list + +APIMACHINERY_DEFAULT_TAG_FILES := vendor/k8s.io/apimachinery/pkg/apis/meta/v1/% vendor/k8s.io/apimachinery/pkg/runtime/% vendor/k8s.io/apimachinery/pkg/version/% + +KUBE_OPENAPI_TAG_FILES := $(filter-out vendor/k8s.io/code-generator/% vendor/k8s.io/sample-apiserver/%, $(ALL_K8S_TAG_FILES)) +APIEXTENSIONS_OPENAPI_TAG_FILES := $(filter $(APIMACHINERY_DEFAULT_TAG_FILES) vendor/k8s.io/apiextensions/% vendor/k8s.io/api/autoscaling/v1/%, $(ALL_K8S_TAG_FILES)) +CODEGEN_OPENAPI_TAG_FILES := $(filter $(APIMACHINERY_DEFAULT_TAG_FILES) vendor/k8s.io/code-generator/%, $(ALL_K8S_TAG_FILES)) +SAMPLEAPISERVER_OPENAPI_TAG_FILES := $(filter $(APIMACHINERY_DEFAULT_TAG_FILES) vendor/k8s.io/sample-apiserver/%, $(ALL_K8S_TAG_FILES)) + +KUBE_OPENAPI_OUTPUT_PKG := pkg/generated/openapi +APIEXTENSIONS_OPENAPI_OUTPUT_PKG := staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi +CODEGEN_OPENAPI_OUTPUT_PKG := staging/src/k8s.io/code-generator/_examples/apiserver/openapi +SAMPLEAPISERVER_OPENAPI_OUTPUT_PKG := staging/src/k8s.io/sample-apiserver/pkg/generated/openapi + +OPENAPI_TARGETS := KUBE APIEXTENSIONS CODEGEN SAMPLEAPISERVER + +# Find all the directories that request openapi generation. +define OPENAPI_DIR_DEF +ifeq ($$(DBG_MAKEFILE),1) + $$(warning ***** finding all +k8s:openapi-gen tags for $(prefix)) endif -OPENAPI_DIRS := $(shell \ - grep --color=never -l '+k8s:openapi-gen=' $(ALL_K8S_TAG_FILES) \ - | xargs -n1 dirname \ - | LC_ALL=C sort -u \ +$(prefix)_OPENAPI_DIRS := $(shell \ + grep --color=never -l '+k8s:openapi-gen=' $($(prefix)_OPENAPI_TAG_FILES) \ + | xargs -n1 dirname \ + | LC_ALL=C sort -u \ ) +endef +$(foreach prefix, $(OPENAPI_TARGETS), $(eval $(OPENAPI_DIR_DEF))) -OPENAPI_OUTFILE := $(OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME) -CRD_OPENAPI_OUTFILE := $(CRD_OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME) +# Compute all openapi output file names +$(foreach prefix, $(OPENAPI_TARGETS), $(eval \ + $(prefix)_OPENAPI_OUTFILE := $($(prefix)_OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME) \ +)) -# This rule is the user-friendly entrypoint for openapi generation. -.PHONY: gen_openapi -gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN) $(CRD_OPENAPI_OUTFILE) - -# For each dir in OPENAPI_DIRS, this establishes a dependency between the -# output file and the input files that should trigger a rebuild. +# For each openapi target compute the spec +define OPENAPI_TARGETS_DEF +# For each dir in $(prefix)_OPENAPI_DIRS, this establishes a dependency +# between the output file and the input files that should trigger a rebuild. # -# Note that this is a deps-only statement, not a full rule (see below for that). +# Note that this is a deps-only statement, not a full rule +# (see below for that). # # The '$(eval)' is needed because this has a different RHS for each LHS, and # would otherwise produce results that make can't parse. -$(foreach dir, $(OPENAPI_DIRS), $(eval \ - $(OPENAPI_OUTFILE): $($(PRJ_SRC_PATH)/$(dir)) \ +$(foreach dir, $($(prefix)_OPENAPI_DIRS), $(eval \ + $($(prefix)_OPENAPI_OUTFILE): $($(PRJ_SRC_PATH)/$(dir)) \ )) -# How to regenerate open-api code. This emits a single file for all results. -# The Make rule fails if generated API rule violation report differs from the checked-in -# violation file, and prints error message to request developer to fix either the API -# source code, or the known API rule violation file. -$(OPENAPI_OUTFILE): $(OPENAPI_GEN) $(KNOWN_VIOLATION_FILENAME) - ./hack/run-in-gopath.sh $(OPENAPI_GEN) \ - --v $(KUBE_VERBOSE) \ - --logtostderr \ - -i $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(OPENAPI_DIRS)) | sed 's/ /,/g') \ - -p $(PRJ_SRC_PATH)/$(OPENAPI_OUTPUT_PKG) \ - -O $(OPENAPI_BASENAME) \ - -h $(BOILERPLATE_FILENAME) \ - -r $(REPORT_FILENAME) \ - "$$@"; \ - diff $(REPORT_FILENAME) $(KNOWN_VIOLATION_FILENAME) || \ - (echo -e $(API_RULE_CHECK_FAILURE_MESSAGE); exit 1) +# Rebuild spec if the violation file changed. +$($(prefix)_OPENAPI_OUTFILE): $($(prefix)_KNOWN_VIOLATION_FILENAME) -# TODO(roycaihw): move the automation to apiextensions-apiserver -$(CRD_OPENAPI_OUTFILE): $(OPENAPI_GEN) - ./hack/run-in-gopath.sh $(OPENAPI_GEN) \ - --v $(KUBE_VERBOSE) \ - --logtostderr \ - -i "k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/api/autoscaling/v1" \ - -p $(PRJ_SRC_PATH)/$(CRD_OPENAPI_OUTPUT_PKG) \ - -O $(OPENAPI_BASENAME) \ - -h $(BOILERPLATE_FILENAME) \ - -r $(IGNORED_REPORT_FILENAME) \ - "$$@" +# When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write +# updated API violations to the known API violation exceptions list. +ifeq ($(UPDATE_API_KNOWN_VIOLATIONS),true) + $(prefix)_REPORT_FILENAME := $($(prefix)_KNOWN_VIOLATION_FILENAME) + # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, touch the exceptions + # list so that the $(prefix)_OPENAPI_OUTFILE target re-run instead of being cached. + $$(shell touch $($(prefix)_KNOWN_VIOLATION_FILENAME)) +else + $(prefix)_REPORT_FILENAME := $(OUT_DIR)/$(prefix)_violations.report +endif + +# How to regenerate open-api code. This emits a single file for all results. +# The Make rule fails if generated API rule violation report differs from the +# checked-in violation file, and prints error message to request developer to +# fix either the API source code, or the known API rule violation file. +$$($(prefix)_OPENAPI_OUTFILE): $(OPENAPI_GEN) $($(prefix)_KNOWN_VIOLATION_FILENAME:-) + ./hack/run-in-gopath.sh $(OPENAPI_GEN) \ + --v $(KUBE_VERBOSE) \ + --logtostderr \ + -i $$$$(echo $(addprefix $(PRJ_SRC_PATH)/, $($(prefix)_OPENAPI_DIRS)) | sed 's/ /,/g') \ + -p $(PRJ_SRC_PATH)/$($(prefix)_OPENAPI_OUTPUT_PKG) \ + -O $(OPENAPI_BASENAME) \ + -h $(BOILERPLATE_FILENAME) \ + -r $$($(prefix)_REPORT_FILENAME) \ + "$$$$@" + test -f $($(prefix)_KNOWN_VIOLATION_FILENAME) || touch $($(prefix)_KNOWN_VIOLATION_FILENAME) + diff $$($(prefix)_REPORT_FILENAME) $($(prefix)_KNOWN_VIOLATION_FILENAME) || \ + (echo -e $(call API_RULE_CHECK_FAILURE_MESSAGE,$(prefix),$($(prefix)_KNOWN_VIOLATION_FILENAME)); exit 1) +endef +$(foreach prefix, $(OPENAPI_TARGETS), $(eval $(OPENAPI_TARGETS_DEF))) + +# This rule is the user-friendly entrypoint for openapi generation. +.PHONY: gen_openapi +gen_openapi: $(OPENAPI_GEN) $(KUBE_OPENAPI_OUTFILE) $(APIEXTENSIONS_OPENAPI_OUTFILE) $(CODEGEN_OPENAPI_OUTFILE) $(SAMPLEAPISERVER_OPENAPI_OUTFILE) # How to build the generator tool. The deps for this are defined in # the $(GO_PKGDEPS_FILE), above. diff --git a/hack/verify-generated-files-remake.sh b/hack/verify-generated-files-remake.sh index ea08b741565..d5c750a364b 100755 --- a/hack/verify-generated-files-remake.sh +++ b/hack/verify-generated-files-remake.sh @@ -56,7 +56,7 @@ function newer() { if [[ "${F}" -nt "$2" ]]; then echo "${F}" fi - done + done | LC_ALL=C sort } # $1 = filename pattern as in "zz_generated.$1.go" @@ -66,7 +66,7 @@ function older() { if [[ "$2" -nt "${F}" ]]; then echo "${F}" fi - done + done | LC_ALL=C sort } function assert_clean() { @@ -286,12 +286,32 @@ if [[ -n "${X}" ]]; then fi # -# Test when we touch a file in a package that needs codegen. +# Test when we touch a file in a package that needs codegen for all openapi specs. # assert_clean -touch "staging/src/k8s.io/api/core/v1/types.go" +touch "staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go" +touch "${STAMP}" +make generated_files >/dev/null +X="$(newer openapi "${STAMP}")" +if [[ -z "${X}" || ${X} != "./pkg/generated/openapi/zz_generated.openapi.go +./staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi/zz_generated.openapi.go +./staging/src/k8s.io/code-generator/_examples/apiserver/openapi/zz_generated.openapi.go +./staging/src/k8s.io/sample-apiserver/pkg/generated/openapi/zz_generated.openapi.go" ]]; then + echo "Wrong generated openapi files changed after touching src file:" + echo " ${X:-(none)}" | tr '\n' ' ' + echo "" + exit 1 +fi + +# +# Test when we touch a file in a package that needs codegen for only the main openapi spec. +# + +assert_clean + +touch "staging/src/k8s.io/api/apps/v1/types.go" touch "${STAMP}" make generated_files >/dev/null X="$(newer openapi "${STAMP}")" @@ -302,6 +322,40 @@ if [[ -z "${X}" || ${X} != "./pkg/generated/openapi/zz_generated.openapi.go" ]]; exit 1 fi +# +# Test when we touch a file, modify the violation file it should fail, and UPDATE_API_KNOWN_VIOLATIONS=true updates it. +# + +assert_clean + +touch "staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go" +echo > api/api-rules/violation_exceptions.list +echo > api/api-rules/codegen_violation_exceptions.list +if make generated_files >/dev/null; then + echo "Expected make generated_files to fail with API violations." + echo "" + exit 1 +fi +touch "${STAMP}" +make generated_files UPDATE_API_KNOWN_VIOLATIONS=true >/dev/null +X="$(newer openapi "${STAMP}")" +if [[ -z "${X}" || ${X} != "./pkg/generated/openapi/zz_generated.openapi.go +./staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi/zz_generated.openapi.go +./staging/src/k8s.io/code-generator/_examples/apiserver/openapi/zz_generated.openapi.go +./staging/src/k8s.io/sample-apiserver/pkg/generated/openapi/zz_generated.openapi.go" ]]; then + echo "Wrong generated openapi files changed after updating violation files:" + echo " ${X:-(none)}" | tr '\n' ' ' + echo "" + exit 1 +fi +for f in api/api-rules/violation_exceptions.list api/api-rules/codegen_violation_exceptions.list; do + if ! git diff --quiet "$f"; then + echo "Violation file \"$f\" was not updated with UPDATE_API_KNOWN_VIOLATIONS=true." + echo "" + exit 1 + fi +done + # # Test when the codegen tool itself changes: openapi #