Makefile.generated_files: build code-generator+sample-apiserver OpenAPI schemas

api/api-rules/apiextensions_violation_exceptions.list

@@ -0,0 +1,20 @@
+API rule violation: names_match,k8s.io/apimachinery/pkg/api/resource,Quantity,Format
+API rule violation: names_match,k8s.io/apimachinery/pkg/api/resource,Quantity,d
+API rule violation: names_match,k8s.io/apimachinery/pkg/api/resource,Quantity,i
+API rule violation: names_match,k8s.io/apimachinery/pkg/api/resource,Quantity,s
+API rule violation: names_match,k8s.io/apimachinery/pkg/api/resource,int64Amount,scale
+API rule violation: names_match,k8s.io/apimachinery/pkg/api/resource,int64Amount,value
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,APIResourceList,APIResources
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,Duration,Duration
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,InternalEvent,Object
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,InternalEvent,Type
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,MicroTime,Time
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,StatusCause,Type
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,Time,Time
+API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,RawExtension,Raw
+API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,ContentEncoding
+API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,ContentType
+API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,Raw
+API rule violation: names_match,k8s.io/apimachinery/pkg/util/intstr,IntOrString,IntVal
+API rule violation: names_match,k8s.io/apimachinery/pkg/util/intstr,IntOrString,StrVal
+API rule violation: names_match,k8s.io/apimachinery/pkg/util/intstr,IntOrString,Type

api/api-rules/codegen_violation_exceptions.list

@@ -0,0 +1,13 @@
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,APIResourceList,APIResources
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,Duration,Duration
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,InternalEvent,Object
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,InternalEvent,Type
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,MicroTime,Time
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,StatusCause,Type
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,Time,Time
+API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,RawExtension,Raw
+API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,ContentEncoding
+API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,ContentType
+API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,Raw
+API rule violation: names_match,k8s.io/code-generator/_examples/apiserver/apis/example/v1,TestTypeStatus,Blah
+API rule violation: names_match,k8s.io/code-generator/_examples/apiserver/apis/example2/v1,TestTypeStatus,Blah

api/api-rules/sample_apiserver_violation_exceptions.list

@@ -5,3 +5,7 @@ API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,InternalEve
 API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,MicroTime,Time
 API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,StatusCause,Type
 API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,Time,Time
+API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,RawExtension,Raw
+API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,ContentEncoding
+API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,ContentType
+API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,Raw

build/root/Makefile.generated_files

@@ -408,81 +408,102 @@ $(CONVERSION_GEN): $(k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/conversi
 # The result file, in each pkg, of open-api generation.
 OPENAPI_BASENAME := $(GENERATED_FILE_PREFIX)openapi
 OPENAPI_FILENAME := $(OPENAPI_BASENAME).go
-OPENAPI_OUTPUT_PKG := pkg/generated/openapi
-CRD_OPENAPI_OUTPUT_PKG := staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi
 BOILERPLATE_FILENAME := vendor/k8s.io/code-generator/hack/boilerplate.go.txt
-REPORT_FILENAME := $(OUT_DIR)/violations.report
 IGNORED_REPORT_FILENAME := $(OUT_DIR)/ignored_violations.report
-KNOWN_VIOLATION_FILENAME := api/api-rules/violation_exceptions.list
+
-# When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write
+API_RULE_CHECK_FAILURE_MESSAGE = "ERROR: \n\t $(1) API rule check failed. Reported violations differ from known violations. Please read api/api-rules/README.md to resolve the failure in $(2). \n"
-# updated API violations to the known API violation exceptions list.
-ifeq ($(UPDATE_API_KNOWN_VIOLATIONS),true)
-    REPORT_FILENAME:=$(KNOWN_VIOLATION_FILENAME)
-    # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, touch the exceptions
-    # list so that the OPENAPI_OUTFILE target re-run instead of being cached.
-    $(shell touch $(KNOWN_VIOLATION_FILENAME))
-endif
-API_RULE_CHECK_FAILURE_MESSAGE := "ERROR: \n\t API rule check failed. Reported violations differ from known violations. Please read api/api-rules/README.md to resolve the failure. \n"
 
 # The tool used to generate open apis.
 OPENAPI_GEN := $(BIN_DIR)/openapi-gen
 
-# Find all the directories that request open-api generation.
+KUBE_KNOWN_VIOLATION_FILENAME := api/api-rules/violation_exceptions.list
-ifeq ($(DBG_MAKEFILE),1)
+APIEXTENSIONS_KNOWN_VIOLATION_FILENAME := api/api-rules/apiextensions_violation_exceptions.list
-    $(warning ***** finding all +k8s:openapi-gen tags)
+CODEGEN_KNOWN_VIOLATION_FILENAME := api/api-rules/codegen_violation_exceptions.list
+SAMPLEAPISERVER_KNOWN_VIOLATION_FILENAME := api/api-rules/sample_apiserver_violation_exceptions.list
+
+APIMACHINERY_DEFAULT_TAG_FILES := vendor/k8s.io/apimachinery/pkg/apis/meta/v1/% vendor/k8s.io/apimachinery/pkg/runtime/% vendor/k8s.io/apimachinery/pkg/version/%
+
+KUBE_OPENAPI_TAG_FILES :=            $(filter-out vendor/k8s.io/code-generator/% vendor/k8s.io/sample-apiserver/%, $(ALL_K8S_TAG_FILES))
+APIEXTENSIONS_OPENAPI_TAG_FILES :=   $(filter $(APIMACHINERY_DEFAULT_TAG_FILES) vendor/k8s.io/apiextensions/% vendor/k8s.io/api/autoscaling/v1/%, $(ALL_K8S_TAG_FILES))
+CODEGEN_OPENAPI_TAG_FILES :=         $(filter $(APIMACHINERY_DEFAULT_TAG_FILES) vendor/k8s.io/code-generator/%, $(ALL_K8S_TAG_FILES))
+SAMPLEAPISERVER_OPENAPI_TAG_FILES := $(filter $(APIMACHINERY_DEFAULT_TAG_FILES) vendor/k8s.io/sample-apiserver/%, $(ALL_K8S_TAG_FILES))
+
+KUBE_OPENAPI_OUTPUT_PKG := pkg/generated/openapi
+APIEXTENSIONS_OPENAPI_OUTPUT_PKG := staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi
+CODEGEN_OPENAPI_OUTPUT_PKG := staging/src/k8s.io/code-generator/_examples/apiserver/openapi
+SAMPLEAPISERVER_OPENAPI_OUTPUT_PKG := staging/src/k8s.io/sample-apiserver/pkg/generated/openapi
+
+OPENAPI_TARGETS := KUBE APIEXTENSIONS CODEGEN SAMPLEAPISERVER
+
+# Find all the directories that request openapi generation.
+define OPENAPI_DIR_DEF
+ifeq ($$(DBG_MAKEFILE),1)
+    $$(warning ***** finding all +k8s:openapi-gen tags for $(prefix))
 endif
-OPENAPI_DIRS := $(shell                                             \
+$(prefix)_OPENAPI_DIRS := $(shell                                            \
-    grep --color=never -l '+k8s:openapi-gen=' $(ALL_K8S_TAG_FILES)  \
+    grep --color=never -l '+k8s:openapi-gen=' $($(prefix)_OPENAPI_TAG_FILES) \
-        | xargs -n1 dirname                                         \
+        | xargs -n1 dirname                                                  \
-        | LC_ALL=C sort -u                                          \
+        | LC_ALL=C sort -u                                                   \
 )
+endef
+$(foreach prefix, $(OPENAPI_TARGETS), $(eval $(OPENAPI_DIR_DEF)))
 
-OPENAPI_OUTFILE := $(OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME)
+# Compute all openapi output file names
-CRD_OPENAPI_OUTFILE := $(CRD_OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME)
+$(foreach prefix, $(OPENAPI_TARGETS), $(eval                                         \
+    $(prefix)_OPENAPI_OUTFILE := $($(prefix)_OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME) \
+))
 
-# This rule is the user-friendly entrypoint for openapi generation.
+# For each openapi target compute the spec
-.PHONY: gen_openapi
+define OPENAPI_TARGETS_DEF
-gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN) $(CRD_OPENAPI_OUTFILE)
+# For each dir in $(prefix)_OPENAPI_DIRS, this establishes a dependency
-
+# between the output file and the input files that should trigger a rebuild.
-# For each dir in OPENAPI_DIRS, this establishes a dependency between the
-# output file and the input files that should trigger a rebuild.
 #
-# Note that this is a deps-only statement, not a full rule (see below for that).
+# Note that this is a deps-only statement, not a full rule
+# (see below for that).
 #
 # The '$(eval)' is needed because this has a different RHS for each LHS, and
 # would otherwise produce results that make can't parse.
-$(foreach dir, $(OPENAPI_DIRS), $(eval             \
+$(foreach dir, $($(prefix)_OPENAPI_DIRS), $(eval            \
-    $(OPENAPI_OUTFILE): $($(PRJ_SRC_PATH)/$(dir))  \
+    $($(prefix)_OPENAPI_OUTFILE): $($(PRJ_SRC_PATH)/$(dir)) \
 ))
 
-# How to regenerate open-api code.  This emits a single file for all results.
+# Rebuild spec if the violation file changed.
-# The Make rule fails if generated API rule violation report differs from the checked-in
+$($(prefix)_OPENAPI_OUTFILE): $($(prefix)_KNOWN_VIOLATION_FILENAME)
-# violation file, and prints error message to request developer to fix either the API
-# source code, or the known API rule violation file.
-$(OPENAPI_OUTFILE): $(OPENAPI_GEN) $(KNOWN_VIOLATION_FILENAME)
-	./hack/run-in-gopath.sh $(OPENAPI_GEN)                                          \
-	    --v $(KUBE_VERBOSE)                                                         \
-	    --logtostderr                                                               \
-	    -i $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(OPENAPI_DIRS)) | sed 's/ /,/g')  \
-	    -p $(PRJ_SRC_PATH)/$(OPENAPI_OUTPUT_PKG)                                    \
-	    -O $(OPENAPI_BASENAME)                                                      \
-	    -h $(BOILERPLATE_FILENAME)                                                  \
-	    -r $(REPORT_FILENAME)                                                       \
-	    "$$@";                                                                      \
-	diff $(REPORT_FILENAME) $(KNOWN_VIOLATION_FILENAME) ||                          \
-	(echo -e $(API_RULE_CHECK_FAILURE_MESSAGE); exit 1)
 
-# TODO(roycaihw): move the automation to apiextensions-apiserver
+# When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write
-$(CRD_OPENAPI_OUTFILE): $(OPENAPI_GEN)
+# updated API violations to the known API violation exceptions list.
-	./hack/run-in-gopath.sh $(OPENAPI_GEN)                                          \
+ifeq ($(UPDATE_API_KNOWN_VIOLATIONS),true)
-	    --v $(KUBE_VERBOSE)                                                         \
+    $(prefix)_REPORT_FILENAME := $($(prefix)_KNOWN_VIOLATION_FILENAME)
-	    --logtostderr                                                               \
+    # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, touch the exceptions
-	    -i "k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/api/autoscaling/v1"         \
+    # list so that the $(prefix)_OPENAPI_OUTFILE target re-run instead of being cached.
-	    -p $(PRJ_SRC_PATH)/$(CRD_OPENAPI_OUTPUT_PKG)                                \
+    $$(shell touch $($(prefix)_KNOWN_VIOLATION_FILENAME))
-	    -O $(OPENAPI_BASENAME)                                                      \
+else
-	    -h $(BOILERPLATE_FILENAME)                                                  \
+    $(prefix)_REPORT_FILENAME := $(OUT_DIR)/$(prefix)_violations.report
-	    -r $(IGNORED_REPORT_FILENAME)                                               \
+endif
-	    "$$@"
+
+# How to regenerate open-api code.  This emits a single file for all results.
+# The Make rule fails if generated API rule violation report differs from the
+# checked-in violation file, and prints error message to request developer to
+# fix either the API source code, or the known API rule violation file.
+$$($(prefix)_OPENAPI_OUTFILE): $(OPENAPI_GEN) $($(prefix)_KNOWN_VIOLATION_FILENAME:-)
+	./hack/run-in-gopath.sh $(OPENAPI_GEN)                                                     \
+	    --v $(KUBE_VERBOSE)                                                                    \
+	    --logtostderr                                                                          \
+	    -i $$$$(echo $(addprefix $(PRJ_SRC_PATH)/, $($(prefix)_OPENAPI_DIRS)) | sed 's/ /,/g') \
+	    -p $(PRJ_SRC_PATH)/$($(prefix)_OPENAPI_OUTPUT_PKG)                                     \
+	    -O $(OPENAPI_BASENAME)                                                                 \
+	    -h $(BOILERPLATE_FILENAME)                                                             \
+	    -r $$($(prefix)_REPORT_FILENAME)                                                       \
+	    "$$$$@"
+	test -f $($(prefix)_KNOWN_VIOLATION_FILENAME) || touch $($(prefix)_KNOWN_VIOLATION_FILENAME)
+	diff $$($(prefix)_REPORT_FILENAME) $($(prefix)_KNOWN_VIOLATION_FILENAME) || \
+	(echo -e $(call API_RULE_CHECK_FAILURE_MESSAGE,$(prefix),$($(prefix)_KNOWN_VIOLATION_FILENAME)); exit 1)
+endef
+$(foreach prefix, $(OPENAPI_TARGETS), $(eval $(OPENAPI_TARGETS_DEF)))
+
+# This rule is the user-friendly entrypoint for openapi generation.
+.PHONY: gen_openapi
+gen_openapi: $(OPENAPI_GEN) $(KUBE_OPENAPI_OUTFILE) $(APIEXTENSIONS_OPENAPI_OUTFILE) $(CODEGEN_OPENAPI_OUTFILE) $(SAMPLEAPISERVER_OPENAPI_OUTFILE)
 
 # How to build the generator tool.  The deps for this are defined in
 # the $(GO_PKGDEPS_FILE), above.

hack/verify-generated-files-remake.sh

@@ -56,7 +56,7 @@ function newer() {
         if [[ "${F}" -nt "$2" ]]; then
             echo "${F}"
         fi
-    done
+    done | LC_ALL=C sort
 }
 
 # $1 = filename pattern as in "zz_generated.$1.go"
@@ -66,7 +66,7 @@ function older() {
         if [[ "$2" -nt "${F}" ]]; then
             echo "${F}"
         fi
-    done
+    done | LC_ALL=C sort
 }
 
 function assert_clean() {
@@ -286,12 +286,32 @@ if [[ -n "${X}" ]]; then
 fi
 
 #
-# Test when we touch a file in a package that needs codegen.
+# Test when we touch a file in a package that needs codegen for all openapi specs.
 #
 
 assert_clean
 
-touch "staging/src/k8s.io/api/core/v1/types.go"
+touch "staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go"
+touch "${STAMP}"
+make generated_files >/dev/null
+X="$(newer openapi "${STAMP}")"
+if [[ -z "${X}" || ${X} != "./pkg/generated/openapi/zz_generated.openapi.go
+./staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi/zz_generated.openapi.go
+./staging/src/k8s.io/code-generator/_examples/apiserver/openapi/zz_generated.openapi.go
+./staging/src/k8s.io/sample-apiserver/pkg/generated/openapi/zz_generated.openapi.go" ]]; then
+    echo "Wrong generated openapi files changed after touching src file:"
+    echo "  ${X:-(none)}" | tr '\n' ' '
+    echo ""
+    exit 1
+fi
+
+#
+# Test when we touch a file in a package that needs codegen for only the main openapi spec.
+#
+
+assert_clean
+
+touch "staging/src/k8s.io/api/apps/v1/types.go"
 touch "${STAMP}"
 make generated_files >/dev/null
 X="$(newer openapi "${STAMP}")"
@@ -302,6 +322,40 @@ if [[ -z "${X}" || ${X} != "./pkg/generated/openapi/zz_generated.openapi.go" ]];
     exit 1
 fi
 
+#
+# Test when we touch a file, modify the violation file it should fail, and UPDATE_API_KNOWN_VIOLATIONS=true updates it.
+#
+
+assert_clean
+
+touch "staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go"
+echo > api/api-rules/violation_exceptions.list
+echo > api/api-rules/codegen_violation_exceptions.list
+if make generated_files >/dev/null; then
+    echo "Expected make generated_files to fail with API violations."
+    echo ""
+    exit 1
+fi
+touch "${STAMP}"
+make generated_files UPDATE_API_KNOWN_VIOLATIONS=true >/dev/null
+X="$(newer openapi "${STAMP}")"
+if [[ -z "${X}" || ${X} != "./pkg/generated/openapi/zz_generated.openapi.go
+./staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi/zz_generated.openapi.go
+./staging/src/k8s.io/code-generator/_examples/apiserver/openapi/zz_generated.openapi.go
+./staging/src/k8s.io/sample-apiserver/pkg/generated/openapi/zz_generated.openapi.go" ]]; then
+    echo "Wrong generated openapi files changed after updating violation files:"
+    echo "  ${X:-(none)}" | tr '\n' ' '
+    echo ""
+    exit 1
+fi
+for f in api/api-rules/violation_exceptions.list api/api-rules/codegen_violation_exceptions.list; do
+    if ! git diff --quiet "$f"; then
+        echo "Violation file \"$f\" was not updated with UPDATE_API_KNOWN_VIOLATIONS=true."
+        echo ""
+        exit 1
+    fi
+done
+
 #
 # Test when the codegen tool itself changes: openapi
 #