From cb1332db06ad8283bdf45fb693659fc668f2dd3d Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Sun, 29 Apr 2018 18:27:27 +0100 Subject: [PATCH 01/11] Get rid of verify_generated_files It doesn't work right, we autogenerate on-demand, and we want to stop checking them in. --- build/root/Makefile | 19 ++----------------- build/root/Makefile.generated_files | 17 ----------------- 2 files changed, 2 insertions(+), 34 deletions(-) diff --git a/build/root/Makefile b/build/root/Makefile index c7e16e90e6b..32ddf4fda04 100644 --- a/build/root/Makefile +++ b/build/root/Makefile @@ -124,7 +124,7 @@ ifeq ($(PRINT_HELP),y) verify: @echo "$$VERIFY_HELP_INFO" else -verify: verify_generated_files +verify: KUBE_VERIFY_GIT_BRANCH=$(BRANCH) hack/make-rules/verify.sh endif @@ -139,7 +139,7 @@ ifeq ($(PRINT_HELP),y) quick-verify: @echo "$$QUICK_VERIFY_HELP_INFO" else -quick-verify: verify_generated_files +quick-verify: QUICK=true SILENT=false hack/make-rules/verify.sh endif @@ -483,21 +483,6 @@ generated_files: $(MAKE) -f Makefile.generated_files $@ CALLED_FROM_MAIN_MAKEFILE=1 endif -define VERIFY_GENERATED_FILES_HELP_INFO -# Verify auto-generated files needed for the build. -# -# Example: -# make verify_generated_files -endef -.PHONY: verify_generated_files -ifeq ($(PRINT_HELP),y) -verify_generated_files: - @echo "$$VERIFY_GENERATED_FILES_HELP_INFO" -else -verify_generated_files: - $(MAKE) -f Makefile.generated_files $@ CALLED_FROM_MAIN_MAKEFILE=1 -endif - define HELP_INFO # Print make targets and help info # diff --git a/build/root/Makefile.generated_files b/build/root/Makefile.generated_files index 59372a0c305..d373c470d5b 100644 --- a/build/root/Makefile.generated_files +++ b/build/root/Makefile.generated_files @@ -37,11 +37,6 @@ SHELL := /bin/bash .PHONY: generated_files generated_files: gen_deepcopy gen_defaulter gen_conversion gen_openapi gen_bindata -.PHONY: verify_generated_files -verify_generated_files: verify_gen_deepcopy \ - verify_gen_defaulter \ - verify_gen_conversion - # Code-generation logic. # # This stuff can be pretty tricky, and there's probably some corner cases that @@ -245,10 +240,6 @@ RUN_GEN_DEEPCOPY = \ gen_deepcopy: $(DEEPCOPY_FILES) $(DEEPCOPY_GEN) $(RUN_GEN_DEEPCOPY) -.PHONY: verify_gen_deepcopy -verify_gen_deepcopy: $(DEEPCOPY_GEN) - $(RUN_GEN_DEEPCOPY) --verify-only - # For each dir in DEEPCOPY_DIRS, this establishes a dependency between the # output file and the input files that should trigger a rebuild. # @@ -383,10 +374,6 @@ RUN_GEN_DEFAULTER := \ gen_defaulter: $(DEFAULTER_FILES) $(DEFAULTER_GEN) $(RUN_GEN_DEFAULTER) -.PHONY: verify_gen_deepcopy -verify_gen_defaulter: $(DEFAULTER_GEN) - $(RUN_GEN_DEFAULTER) --verify-only - # For each dir in DEFAULTER_DIRS, this establishes a dependency between the # output file and the input files that should trigger a rebuild. # @@ -551,10 +538,6 @@ RUN_GEN_CONVERSION = \ gen_conversion: $(CONVERSION_FILES) $(CONVERSION_GEN) $(RUN_GEN_CONVERSION) -.PHONY: verify_gen_conversion -verify_gen_conversion: $(CONVERSION_GEN) - $(RUN_GEN_CONVERSION) --verify-only - # Establish a dependency between the deps file and the dir. Whenever a dir # changes (files added or removed) the deps file will be considered stale. # From fba394ce8ca8ef9a67570e0738537524cecbebc5 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Sun, 29 Apr 2018 18:32:51 +0100 Subject: [PATCH 02/11] Build: Test if .todo file exists AND is empty --- build/root/Makefile.generated_files | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/root/Makefile.generated_files b/build/root/Makefile.generated_files index d373c470d5b..9bd8676f13b 100644 --- a/build/root/Makefile.generated_files +++ b/build/root/Makefile.generated_files @@ -218,7 +218,7 @@ DEEPCOPY_FILES := $(addsuffix /$(DEEPCOPY_FILENAME), $(DEEPCOPY_DIRS)) # Shell function for reuse in rules. RUN_GEN_DEEPCOPY = \ function run_gen_deepcopy() { \ - if [[ -f $(META_DIR)/$(DEEPCOPY_GEN).todo ]]; then \ + if [[ -s $(META_DIR)/$(DEEPCOPY_GEN).todo ]]; then \ pkgs=$$(cat $(META_DIR)/$(DEEPCOPY_GEN).todo | paste -sd, -); \ if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ echo "DBG: running $(DEEPCOPY_GEN) for $$pkgs"; \ @@ -352,7 +352,7 @@ DEFAULTER_FILES := $(addsuffix /$(DEFAULTER_FILENAME), $(DEFAULTER_DIRS)) RUN_GEN_DEFAULTER := \ function run_gen_defaulter() { \ - if [[ -f $(META_DIR)/$(DEFAULTER_GEN).todo ]]; then \ + if [[ -s $(META_DIR)/$(DEFAULTER_GEN).todo ]]; then \ pkgs=$$(cat $(META_DIR)/$(DEFAULTER_GEN).todo | paste -sd, -); \ if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ echo "DBG: running $(DEFAULTER_GEN) for $$pkgs"; \ @@ -516,7 +516,7 @@ CONVERSION_EXTRA_PEER_DIRS := k8s.io/kubernetes/pkg/apis/core,k8s.io/kubernetes/ # Shell function for reuse in rules. RUN_GEN_CONVERSION = \ function run_gen_conversion() { \ - if [[ -f $(META_DIR)/$(CONVERSION_GEN).todo ]]; then \ + if [[ -s $(META_DIR)/$(CONVERSION_GEN).todo ]]; then \ pkgs=$$(cat $(META_DIR)/$(CONVERSION_GEN).todo | paste -sd, -); \ if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ echo "DBG: running $(CONVERSION_GEN) for $$pkgs"; \ From 90e0ac54b903117bf5f9810c7b94782113ffb6e2 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Sun, 29 Apr 2018 18:39:01 +0100 Subject: [PATCH 03/11] Simplify Makefile for genfiles a bit --- build/root/Makefile.generated_files | 98 ++++++++++++----------------- 1 file changed, 39 insertions(+), 59 deletions(-) diff --git a/build/root/Makefile.generated_files b/build/root/Makefile.generated_files index 9bd8676f13b..68e45dfa573 100644 --- a/build/root/Makefile.generated_files +++ b/build/root/Makefile.generated_files @@ -215,30 +215,23 @@ DEEPCOPY_DIRS := $(shell \ ) DEEPCOPY_FILES := $(addsuffix /$(DEEPCOPY_FILENAME), $(DEEPCOPY_DIRS)) -# Shell function for reuse in rules. -RUN_GEN_DEEPCOPY = \ - function run_gen_deepcopy() { \ - if [[ -s $(META_DIR)/$(DEEPCOPY_GEN).todo ]]; then \ - pkgs=$$(cat $(META_DIR)/$(DEEPCOPY_GEN).todo | paste -sd, -); \ - if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ - echo "DBG: running $(DEEPCOPY_GEN) for $$pkgs"; \ - fi; \ - ./hack/run-in-gopath.sh $(DEEPCOPY_GEN) \ - --v $(KUBE_VERBOSE) \ - --logtostderr \ - -i "$$pkgs" \ - --bounding-dirs $(PRJ_SRC_PATH),"k8s.io/api" \ - -O $(DEEPCOPY_BASENAME) \ - "$$@"; \ - fi \ - }; \ - run_gen_deepcopy - # This rule aggregates the set of files to generate and then generates them all # in a single run of the tool. .PHONY: gen_deepcopy gen_deepcopy: $(DEEPCOPY_FILES) $(DEEPCOPY_GEN) - $(RUN_GEN_DEEPCOPY) + if [[ -s $(META_DIR)/$(DEEPCOPY_GEN).todo ]]; then \ + pkgs=$$(cat $(META_DIR)/$(DEEPCOPY_GEN).todo | paste -sd, -); \ + if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ + echo "DBG: running $(DEEPCOPY_GEN) for $$pkgs"; \ + fi; \ + ./hack/run-in-gopath.sh $(DEEPCOPY_GEN) \ + --v $(KUBE_VERBOSE) \ + --logtostderr \ + -i "$$pkgs" \ + --bounding-dirs $(PRJ_SRC_PATH),"k8s.io/api" \ + -O $(DEEPCOPY_BASENAME) \ + "$$@"; \ + fi \ # For each dir in DEEPCOPY_DIRS, this establishes a dependency between the # output file and the input files that should trigger a rebuild. @@ -350,29 +343,23 @@ DEFAULTER_DIRS := $(shell \ DEFAULTER_FILES := $(addsuffix /$(DEFAULTER_FILENAME), $(DEFAULTER_DIRS)) -RUN_GEN_DEFAULTER := \ - function run_gen_defaulter() { \ - if [[ -s $(META_DIR)/$(DEFAULTER_GEN).todo ]]; then \ - pkgs=$$(cat $(META_DIR)/$(DEFAULTER_GEN).todo | paste -sd, -); \ - if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ - echo "DBG: running $(DEFAULTER_GEN) for $$pkgs"; \ - fi; \ - ./hack/run-in-gopath.sh $(DEFAULTER_GEN) \ - --v $(KUBE_VERBOSE) \ - --logtostderr \ - -i "$$pkgs" \ - --extra-peer-dirs $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(DEFAULTER_DIRS)) | sed 's/ /,/g') \ - -O $(DEFAULTER_BASENAME) \ - "$$@"; \ - fi \ - }; \ - run_gen_defaulter - # This rule aggregates the set of files to generate and then generates them all # in a single run of the tool. .PHONY: gen_defaulter gen_defaulter: $(DEFAULTER_FILES) $(DEFAULTER_GEN) - $(RUN_GEN_DEFAULTER) + if [[ -s $(META_DIR)/$(DEFAULTER_GEN).todo ]]; then \ + pkgs=$$(cat $(META_DIR)/$(DEFAULTER_GEN).todo | paste -sd, -); \ + if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ + echo "DBG: running $(DEFAULTER_GEN) for $$pkgs"; \ + fi; \ + ./hack/run-in-gopath.sh $(DEFAULTER_GEN) \ + --v $(KUBE_VERBOSE) \ + --logtostderr \ + -i "$$pkgs" \ + --extra-peer-dirs $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(DEFAULTER_DIRS)) | sed 's/ /,/g') \ + -O $(DEFAULTER_BASENAME) \ + "$$@"; \ + fi # For each dir in DEFAULTER_DIRS, this establishes a dependency between the # output file and the input files that should trigger a rebuild. @@ -513,30 +500,23 @@ CONVERSION_DIRS := $(shell \ CONVERSION_FILES := $(addsuffix /$(CONVERSION_FILENAME), $(CONVERSION_DIRS)) CONVERSION_EXTRA_PEER_DIRS := k8s.io/kubernetes/pkg/apis/core,k8s.io/kubernetes/pkg/apis/core/v1,k8s.io/api/core/v1 -# Shell function for reuse in rules. -RUN_GEN_CONVERSION = \ - function run_gen_conversion() { \ - if [[ -s $(META_DIR)/$(CONVERSION_GEN).todo ]]; then \ - pkgs=$$(cat $(META_DIR)/$(CONVERSION_GEN).todo | paste -sd, -); \ - if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ - echo "DBG: running $(CONVERSION_GEN) for $$pkgs"; \ - fi; \ - ./hack/run-in-gopath.sh $(CONVERSION_GEN) \ - --extra-peer-dirs $(CONVERSION_EXTRA_PEER_DIRS) \ - --v $(KUBE_VERBOSE) \ - --logtostderr \ - -i "$$pkgs" \ - -O $(CONVERSION_BASENAME) \ - "$$@"; \ - fi \ - }; \ - run_gen_conversion - # This rule aggregates the set of files to generate and then generates them all # in a single run of the tool. .PHONY: gen_conversion gen_conversion: $(CONVERSION_FILES) $(CONVERSION_GEN) - $(RUN_GEN_CONVERSION) + if [[ -s $(META_DIR)/$(CONVERSION_GEN).todo ]]; then \ + pkgs=$$(cat $(META_DIR)/$(CONVERSION_GEN).todo | paste -sd, -); \ + if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ + echo "DBG: running $(CONVERSION_GEN) for $$pkgs"; \ + fi; \ + ./hack/run-in-gopath.sh $(CONVERSION_GEN) \ + --extra-peer-dirs $(CONVERSION_EXTRA_PEER_DIRS) \ + --v $(KUBE_VERBOSE) \ + --logtostderr \ + -i "$$pkgs" \ + -O $(CONVERSION_BASENAME) \ + "$$@"; \ + fi # Establish a dependency between the deps file and the dir. Whenever a dir # changes (files added or removed) the deps file will be considered stale. From 46aecdaa0aac971ab1db4620b0c9bd36dcdfc608 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Sun, 29 Apr 2018 19:16:20 +0100 Subject: [PATCH 04/11] Better test for generated file rebuilds --- hack/verify-generated-files-remake.sh | 303 +++++++++++++++++++++++++- 1 file changed, 298 insertions(+), 5 deletions(-) mode change 100644 => 100755 hack/verify-generated-files-remake.sh diff --git a/hack/verify-generated-files-remake.sh b/hack/verify-generated-files-remake.sh old mode 100644 new mode 100755 index 9c80a6bc013..5cc44616a26 --- a/hack/verify-generated-files-remake.sh +++ b/hack/verify-generated-files-remake.sh @@ -21,10 +21,303 @@ set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${KUBE_ROOT}/hack/lib/init.sh" -make generated_files -X=$(make generated_files DBG_CODEGEN=1) -if [[ -n "${X}" ]]; then - echo "Back-to-back 'make' runs are not clean for codegen" - echo "${X}" +# $1 = filename pattern as in "zz_generated.$1.go" +function find_genfiles() { + find . \ + \( \ + -not \( \ + \( \ + -path ./_\* -o \ + -path ./.\* \ + \) -prune \ + \) \ + \) -name "zz_generated.$1.go" +} + +# $1 = filename pattern as in "zz_generated.$1.go" +# $2 timestamp file +function newer() { + find_genfiles "$1" | while read F; do + if [[ "${F}" -nt "$2" ]]; then + echo "${F}" + fi + done +} + +# $1 = filename pattern as in "zz_generated.$1.go" +# $2 timestamp file +function older() { + find_genfiles "$1" | while read F; do + if [[ "$2" -nt "${F}" ]]; then + echo "${F}" + fi + done +} + +function assert_clean() { + make generated_files >/dev/null + touch "${STAMP}" + make generated_files >/dev/null + X=($(newer deepcopy "${STAMP}")) + if [[ "${#X[*]}" != 0 ]]; then + echo "Generated files changed on back-to-back 'make' runs:" + echo " ${X[@]:-(none)}" + return 1 + fi + true +} + +STAMP=/tmp/stamp.$RANDOM + +# +# Test when we touch a file in a package that needs codegen. +# + +assert_clean + +DIR=staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1 +touch "$DIR/types.go" +touch "${STAMP}" +make generated_files >/dev/null +X=($(newer deepcopy "${STAMP}")) +if [[ "${#X[*]}" != 1 || ! ( "${X[0]}" =~ "${DIR}/zz_generated.deepcopy.go" ) ]]; then + echo "Wrong generated deepcopy files changed after touching src file:" + echo " ${X[@]:-(none)}" + exit 1 +fi +X=($(newer defaults "${STAMP}")) +if [[ "${#X[*]}" != 1 || ! ( "${X[0]}" =~ "${DIR}/zz_generated.defaults.go" ) ]]; then + echo "Wrong generated defaults files changed after touching src file:" + echo " ${X[@]:-(none)}" + exit 1 +fi +X=($(newer conversion "${STAMP}")) +if [[ "${#X[*]}" != 1 || ! ( "${X[0]}" =~ "${DIR}/zz_generated.conversion.go" ) ]]; then + echo "Wrong generated conversion files changed after touching src file:" + echo " ${X[@]:-(none)}" + exit 1 +fi + +# +# Test when the codegen tool itself changes: deepcopy +# + +assert_clean + +touch staging/src/k8s.io/code-generator/cmd/deepcopy-gen/main.go +touch "${STAMP}" +make generated_files >/dev/null +X=($(older deepcopy "${STAMP}")) +if [[ "${#X[*]}" != 0 ]]; then + echo "Generated deepcopy files did not change after touching code-generator file:" + echo " ${X[@]:-(none)}" + exit 1 +fi + +assert_clean + +touch staging/src/k8s.io/code-generator/cmd/deepcopy-gen/ +touch "${STAMP}" +make generated_files >/dev/null +X=($(older deepcopy "${STAMP}")) +if [[ "${#X[*]}" != 0 ]]; then + echo "Generated deepcopy files did not change after touching code-generator dir:" + echo " ${X[@]:-(none)}" + exit 1 +fi + +assert_clean + +touch vendor/k8s.io/gengo/examples/deepcopy-gen/generators/deepcopy.go +touch "${STAMP}" +make generated_files >/dev/null +X=($(older deepcopy "${STAMP}")) +if [[ "${#X[*]}" != 0 ]]; then + echo "Generated deepcopy files did not change after touching code-generator dep file:" + echo " ${X[@]:-(none)}" + exit 1 +fi + +assert_clean + +touch vendor/k8s.io/gengo/examples/deepcopy-gen/generators/ +touch "${STAMP}" +make generated_files >/dev/null +X=($(older deepcopy "${STAMP}")) +if [[ "${#X[*]}" != 0 ]]; then + echo "Generated deepcopy files did not change after touching code-generator dep dir:" + echo " ${X[@]:-(none)}" + exit 1 +fi + +# +# Test when the codegen tool itself changes: defaults +# + +assert_clean + +touch staging/src/k8s.io/code-generator/cmd/defaulter-gen/main.go +touch "${STAMP}" +make generated_files >/dev/null +X=($(older defaults "${STAMP}")) +if [[ "${#X[*]}" != 0 ]]; then + echo "Generated defaults files did not change after touching code-generator file:" + echo " ${X[@]:-(none)}" + exit 1 +fi + +assert_clean + +touch staging/src/k8s.io/code-generator/cmd/defaulter-gen/ +touch "${STAMP}" +make generated_files >/dev/null +X=($(older defaults "${STAMP}")) +if [[ "${#X[*]}" != 0 ]]; then + echo "Generated defaults files did not change after touching code-generator dir:" + echo " ${X[@]:-(none)}" + exit 1 +fi + +assert_clean + +touch vendor/k8s.io/gengo/examples/defaulter-gen/generators/defaulter.go +touch "${STAMP}" +make generated_files >/dev/null +X=($(older defaults "${STAMP}")) +if [[ "${#X[*]}" != 0 ]]; then + echo "Generated defaults files did not change after touching code-generator dep file:" + echo " ${X[@]:-(none)}" + exit 1 +fi + +assert_clean + +touch vendor/k8s.io/gengo/examples/defaulter-gen/generators/ +touch "${STAMP}" +make generated_files >/dev/null +X=($(older defaults "${STAMP}")) +if [[ "${#X[*]}" != 0 ]]; then + echo "Generated defaults files did not change after touching code-generator dep dir:" + echo " ${X[@]:-(none)}" + exit 1 +fi + +# +# Test when the codegen tool itself changes: conversion +# + +assert_clean + +touch staging/src/k8s.io/code-generator/cmd/conversion-gen/main.go +touch "${STAMP}" +make generated_files >/dev/null +X=($(older conversion "${STAMP}")) +if [[ "${#X[*]}" != 0 ]]; then + echo "Generated conversion files did not change after touching code-generator file:" + echo " ${X[@]:-(none)}" + exit 1 +fi + +assert_clean + +touch staging/src/k8s.io/code-generator/cmd/conversion-gen/ +touch "${STAMP}" +make generated_files >/dev/null +X=($(older conversion "${STAMP}")) +if [[ "${#X[*]}" != 0 ]]; then + echo "Generated conversion files did not change after touching code-generator dir:" + echo " ${X[@]:-(none)}" + exit 1 +fi + +assert_clean + +touch vendor/k8s.io/code-generator/cmd/conversion-gen/generators/conversion.go +touch "${STAMP}" +make generated_files >/dev/null +X=($(older conversion "${STAMP}")) +if [[ "${#X[*]}" != 0 ]]; then + echo "Generated conversion files did not change after touching code-generator dep file:" + echo " ${X[@]:-(none)}" + exit 1 +fi + +assert_clean + +touch vendor/k8s.io/code-generator/cmd/conversion-gen/generators/ +touch "${STAMP}" +make generated_files >/dev/null +X=($(older conversion "${STAMP}")) +if [[ "${#X[*]}" != 0 ]]; then + echo "Generated conversion files did not change after touching code-generator dep dir:" + echo " ${X[@]:-(none)}" + exit 1 +fi + +# +# Test when we touch a file in a package that needs codegen. +# + +assert_clean + +touch "staging/src/k8s.io/api/core/v1/types.go" +touch "${STAMP}" +make generated_files >/dev/null +X=($(newer openapi "${STAMP}")) +if [[ "${#X[*]}" != 1 || ! ( "${X[0]}" =~ "pkg/generated/openapi/zz_generated.openapi.go" ) ]]; then + echo "Wrong generated openapi files changed after touching src file:" + echo "${X[@]:-(none)}" + exit 1 +fi + +# +# Test when the codegen tool itself changes: openapi +# + +assert_clean + +touch staging/src/k8s.io/code-generator/cmd/openapi-gen/main.go +touch "${STAMP}" +make generated_files >/dev/null +X=($(older openapi "${STAMP}")) +if [[ "${#X[*]}" != 0 ]]; then + echo "Generated openapi files did not change after touching code-generator file:" + echo " ${X[@]:-(none)}" + exit 1 +fi + +assert_clean + +touch staging/src/k8s.io/code-generator/cmd/openapi-gen/ +touch "${STAMP}" +make generated_files >/dev/null +X=($(older openapi "${STAMP}")) +if [[ "${#X[*]}" != 0 ]]; then + echo "Generated openapi files did not change after touching code-generator dir:" + echo " ${X[@]:-(none)}" + exit 1 +fi + +assert_clean + +touch vendor/k8s.io/kube-openapi/pkg/generators/openapi.go +touch "${STAMP}" +make generated_files >/dev/null +X=($(older openapi "${STAMP}")) +if [[ "${#X[*]}" != 0 ]]; then + echo "Generated openapi files did not change after touching code-generator dep file:" + echo " ${X[@]:-(none)}" + exit 1 +fi + +assert_clean + +touch vendor/k8s.io/kube-openapi/pkg/generators +touch "${STAMP}" +make generated_files >/dev/null +X=($(older openapi "${STAMP}")) +if [[ "${#X[*]}" != 0 ]]; then + echo "Generated openapi files did not change after touching code-generator dep dir:" + echo " ${X[@]:-(none)}" exit 1 fi From 4bb2118fb85909ab11dd5238815cc83237f6d591 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Sun, 29 Apr 2018 21:34:55 +0100 Subject: [PATCH 05/11] Simplify find in cache_go_dirs.sh --- hack/make-rules/helpers/cache_go_dirs.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/hack/make-rules/helpers/cache_go_dirs.sh b/hack/make-rules/helpers/cache_go_dirs.sh index 390f8a21bc0..6587fecf371 100755 --- a/hack/make-rules/helpers/cache_go_dirs.sh +++ b/hack/make-rules/helpers/cache_go_dirs.sh @@ -36,22 +36,23 @@ trap "rm -f '${CACHE}'" HUP INT TERM ERR # Example: # kfind -type f -name foobar.go function kfind() { - # include the "special" vendor directories which are actually part - # of the Kubernetes source tree - generators will use these for - # including certain core API concepts. - find -H . ./vendor/k8s.io/apimachinery ./vendor/k8s.io/apiserver ./vendor/k8s.io/kube-aggregator ./vendor/k8s.io/apiextensions-apiserver ./vendor/k8s.io/metrics ./vendor/k8s.io/sample-apiserver ./vendor/k8s.io/api ./vendor/k8s.io/client-go ./vendor/k8s.io/code-generator ./vendor/k8s.io/sample-controller \ + # We want to include the "special" vendor directories which are actually + # part of the Kubernetes source tree (./staging/*) but we need them to be + # named as their ./vendor/* equivalents. Also, we do not want all of + # ./vendor or even all of ./vendor/k8s.io. + find -H . \ \( \ -not \( \ \( \ -path ./vendor -o \ - -path ./staging -o \ -path ./_\* -o \ -path ./.\* -o \ -path ./docs \ \) -prune \ \) \ \) \ - "$@" + "$@" \ + | sed 's|^./staging/src|vendor|' } NEED_FIND=true From f75ffe50a1bf724efc62de1fa2b9f26736214cfc Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Sun, 29 Apr 2018 23:01:59 +0100 Subject: [PATCH 06/11] Add a helper tool to emit Makefile snippets This loads the whole go universe one time, so should be faster that repeated calls to `go list` --- hack/make-rules/BUILD | 5 +- hack/make-rules/helpers/go2make/BUILD | 41 ++++ hack/make-rules/helpers/go2make/go2make.go | 215 +++++++++++++++++ hack/make-rules/helpers/go2make/pkgwalk.go | 117 +++++++++ .../helpers/go2make/pkgwalk_test.go | 222 ++++++++++++++++++ .../testdata/dir-with-gofiles/.dot/README | 1 + .../dir-with-gofiles/.dot/subsubdir/README | 1 + .../go2make/testdata/dir-with-gofiles/README | 1 + .../dir-with-gofiles/_underscore/README | 1 + .../_underscore/subsubdir/README | 1 + .../go2make/testdata/dir-with-gofiles/bar.go | 20 ++ .../go2make/testdata/dir-with-gofiles/foo.go | 20 ++ .../testdata/dir-with-gofiles/skipme/README | 1 + .../dir-with-gofiles/skipme/subsubdir/README | 1 + .../testdata/dir-with-gofiles/subdir/README | 1 + .../testdata/dir-with-gofiles/testdata/README | 1 + .../testdata/subsubdir/README | 1 + .../testdata/dir-without-gofiles/README | 1 + 18 files changed, 650 insertions(+), 1 deletion(-) create mode 100644 hack/make-rules/helpers/go2make/BUILD create mode 100644 hack/make-rules/helpers/go2make/go2make.go create mode 100644 hack/make-rules/helpers/go2make/pkgwalk.go create mode 100644 hack/make-rules/helpers/go2make/pkgwalk_test.go create mode 100644 hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/.dot/README create mode 100644 hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/.dot/subsubdir/README create mode 100644 hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/README create mode 100644 hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/_underscore/README create mode 100644 hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/_underscore/subsubdir/README create mode 100644 hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/bar.go create mode 100644 hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/foo.go create mode 100644 hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/skipme/README create mode 100644 hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/skipme/subsubdir/README create mode 100644 hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/subdir/README create mode 100644 hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/testdata/README create mode 100644 hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/testdata/subsubdir/README create mode 100644 hack/make-rules/helpers/go2make/testdata/dir-without-gofiles/README diff --git a/hack/make-rules/BUILD b/hack/make-rules/BUILD index d05b2a98ba1..2ff9bb3bfac 100644 --- a/hack/make-rules/BUILD +++ b/hack/make-rules/BUILD @@ -106,7 +106,10 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//hack/make-rules/helpers/go2make:all-srcs", + ], tags = ["automanaged"], visibility = ["//visibility:public"], ) diff --git a/hack/make-rules/helpers/go2make/BUILD b/hack/make-rules/helpers/go2make/BUILD new file mode 100644 index 00000000000..ab5678ffd32 --- /dev/null +++ b/hack/make-rules/helpers/go2make/BUILD @@ -0,0 +1,41 @@ +# gazelle:exclude testdata + +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "go2make.go", + "pkgwalk.go", + ], + importpath = "k8s.io/kubernetes/hack/make-rules/helpers/go2make", + visibility = ["//visibility:private"], + deps = ["//vendor/github.com/spf13/pflag:go_default_library"], +) + +go_binary( + name = "go2make", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = ["pkgwalk_test.go"], + data = glob(["testdata/**"]), + embed = [":go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/hack/make-rules/helpers/go2make/go2make.go b/hack/make-rules/helpers/go2make/go2make.go new file mode 100644 index 00000000000..0ffa585bc71 --- /dev/null +++ b/hack/make-rules/helpers/go2make/go2make.go @@ -0,0 +1,215 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + goflag "flag" + "fmt" + "go/build" + "io" + "os" + "sort" + "strings" + + "github.com/spf13/pflag" +) + +var flPrune = pflag.StringSlice("prune", nil, "sub-packages to prune (recursive, may be specified multiple times)") +var flDebug = pflag.BoolP("debug", "d", false, "enable debugging output") +var flHelp = pflag.BoolP("help", "h", false, "print help and exit") + +func main() { + pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) + pflag.Usage = func() { help(os.Stderr) } + pflag.Parse() + + debug("PWD", getwd()) + + build.Default.BuildTags = []string{"ignore_autogenerated"} + build.Default.UseAllFiles = false + + if *flHelp { + help(os.Stdout) + os.Exit(0) + } + if len(pflag.Args()) == 0 { + help(os.Stderr) + os.Exit(1) + } + for _, in := range pflag.Args() { + if strings.HasSuffix(in, "/...") { + // Recurse. + debug("starting", in) + pkgName := strings.TrimSuffix(in, "/...") + if err := WalkPkg(pkgName, visitPkg); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + } else { + // Import one package. + if err := saveImport(in); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + } + } +} + +func help(out io.Writer) { + fmt.Fprintf(out, "Usage: %s [FLAG...] \n", os.Args[0]) + fmt.Fprintf(out, "\n") + fmt.Fprintf(out, "go2make calculates all of the dependencies of a set of Go packages and prints\n") + fmt.Fprintf(out, "them as variable definitions suitable for use as a Makefile.\n") + fmt.Fprintf(out, "\n") + fmt.Fprintf(out, "Package specifications may be simple (e.g. 'example.com/txt/color') or\n") + fmt.Fprintf(out, "recursive (e.g. 'example.com/txt/...')\n") + fmt.Fprintf(out, " Example:\n") + fmt.Fprintf(out, " $ %s ./example.com/pretty\n", os.Args[0]) + fmt.Fprintf(out, " example.com/txt/split := \\\n") + fmt.Fprintf(out, " /go/src/example.com/txt/split/ \\\n") + fmt.Fprintf(out, " /go/src/example.com/txt/split/split.go \\\n") + fmt.Fprintf(out, " ./example.com/pretty := \\\n") + fmt.Fprintf(out, " /go/src/example.com/pretty/ \\\n") + fmt.Fprintf(out, " /go/src/example.com/pretty/print.go \\\n") + fmt.Fprintf(out, " /go/src/example.com/txt/split/ \\\n") + fmt.Fprintf(out, " /go/src/example.com/txt/split/split.go\n") + fmt.Fprintf(out, "\n") + fmt.Fprintf(out, " Flags:\n") + + pflag.PrintDefaults() +} + +func debug(items ...interface{}) { + if *flDebug { + x := []interface{}{"DBG:"} + x = append(x, items...) + fmt.Println(x...) + } +} + +func visitPkg(importPath, absPath string) error { + debug("visit", importPath) + return saveImport(importPath) +} + +func prune(pkgName string) bool { + for _, pr := range *flPrune { + if pr == pkgName { + return true + } + } + return false +} + +// cache keeps track of which packages we have already loaded. +var cache = map[string]*build.Package{} + +func saveImport(pkgName string) error { + if cache[pkgName] != nil { + return nil + } + if prune(pkgName) { + debug("prune", pkgName) + return ErrSkipPkg + } + pkg, err := loadPackage(pkgName) + if err != nil { + return err + } + debug("save", pkgName) + cache[pkgName] = pkg + + debug("recurse", pkgName) + defer func() { debug("done ", pkgName) }() + if !pkg.Goroot && (len(pkg.GoFiles)+len(pkg.Imports) > 0) { + // Process deps of this package before the package itself. + for _, impName := range pkg.Imports { + if impName == "C" { + continue + } + debug("depends on", impName) + saveImport(impName) + } + + // Emit a variable for each package. + var buf bytes.Buffer + buf.WriteString(pkgName) + buf.WriteString(" := ") + + // Packages depend on their own directories, their own files, and + // transitive list of all deps' directories and files. + all := map[string]struct{}{} + all[pkg.Dir+"/"] = struct{}{} + filesForPkg(pkg, all) + for _, imp := range pkg.Imports { + pkg := cache[imp] + if pkg == nil || pkg.Goroot { + continue + } + all[pkg.Dir+"/"] = struct{}{} + filesForPkg(pkg, all) + } + // Sort and de-dup them. + files := flatten(all) + for _, f := range files { + buf.WriteString(" \\\n ") + buf.WriteString(f) + } + + fmt.Println(buf.String()) + } + return nil +} + +func filesForPkg(pkg *build.Package, all map[string]struct{}) { + for _, file := range pkg.GoFiles { + if pkg.Dir != "." { + file = pkg.Dir + "/" + file + } + all[file] = struct{}{} + } +} + +func flatten(all map[string]struct{}) []string { + list := make([]string, 0, len(all)) + for k := range all { + list = append(list, k) + } + sort.Strings(list) + return list +} + +func loadPackage(pkgName string) (*build.Package, error) { + debug("load", pkgName) + pkg, err := build.Import(pkgName, getwd(), 0) + if err != nil { + // We can ignore NoGoError. Anything else is real. + if _, ok := err.(*build.NoGoError); !ok { + return nil, err + } + } + return pkg, nil +} + +func getwd() string { + pwd, err := os.Getwd() + if err != nil { + panic(fmt.Sprintf("can't get working directory: %v", err)) + } + return pwd +} diff --git a/hack/make-rules/helpers/go2make/pkgwalk.go b/hack/make-rules/helpers/go2make/pkgwalk.go new file mode 100644 index 00000000000..6d8dfdb0f7a --- /dev/null +++ b/hack/make-rules/helpers/go2make/pkgwalk.go @@ -0,0 +1,117 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "go/build" + "os" + "path" + "sort" +) + +// VisitFunc is a function called by WalkPkg to examine a single package. +type VisitFunc func(importPath string, absPath string) error + +// ErrSkipPkg can be returned by a VisitFunc to indicate that the package in +// question should not be walked any further. +var ErrSkipPkg = fmt.Errorf("package skipped") + +// WalkPkg recursively visits all packages under pkgName. This is similar +// to filepath.Walk, except that it follows symlinks. A package is always +// visited before the children of that package. If visit returns ErrSkipPkg, +// pkgName will not be walked. +func WalkPkg(pkgName string, visit VisitFunc) error { + // Visit the package itself. + pkg, err := findPackage(pkgName) + if err != nil { + return err + } + if err := visit(pkg.ImportPath, pkg.Dir); err == ErrSkipPkg { + return nil + } else if err != nil { + return err + } + + // Read all of the child dirents and find sub-packages. + infos, err := readDirInfos(pkg.Dir) + if err != nil { + return err + } + for _, info := range infos { + if !info.IsDir() { + continue + } + name := info.Name() + if name[0] == '_' || (len(name) > 1 && name[0] == '.') || name == "testdata" { + continue + } + // Don't use path.Join() because it drops leading `./` via path.Clean(). + err := WalkPkg(pkgName+"/"+name, visit) + if err != nil { + return err + } + } + return nil +} + +// findPackage finds a Go package. +func findPackage(pkgName string) (*build.Package, error) { + debug("find", pkgName) + pkg, err := build.Import(pkgName, getwd(), build.FindOnly) + if err != nil { + return nil, err + } + return pkg, nil +} + +// readDirInfos returns a list of os.FileInfo structures for the dirents under +// dirPath. The result list is sorted by name. This is very similar to +// ioutil.ReadDir, except that it follows symlinks. +func readDirInfos(dirPath string) ([]os.FileInfo, error) { + names, err := readDirNames(dirPath) + if err != nil { + return nil, err + } + sort.Strings(names) + + infos := make([]os.FileInfo, 0, len(names)) + for _, n := range names { + info, err := os.Stat(path.Join(dirPath, n)) + if err != nil { + return nil, err + } + infos = append(infos, info) + } + return infos, nil +} + +// readDirNames returns a list of all dirents in dirPath. The result list is +// not sorted or filtered. +func readDirNames(dirPath string) ([]string, error) { + d, err := os.Open(dirPath) + if err != nil { + return nil, err + } + defer d.Close() + + names, err := d.Readdirnames(-1) + if err != nil { + return nil, err + } + return names, nil +} diff --git a/hack/make-rules/helpers/go2make/pkgwalk_test.go b/hack/make-rules/helpers/go2make/pkgwalk_test.go new file mode 100644 index 00000000000..140f5121558 --- /dev/null +++ b/hack/make-rules/helpers/go2make/pkgwalk_test.go @@ -0,0 +1,222 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "path" + "reflect" + "sort" + "testing" +) + +func Test_WalkPkg(t *testing.T) { + testCases := []struct { + pkg string + fail bool + expected []string + }{ + { + pkg: "./testdata/nonexistent-dir", + fail: true, + }, + { + pkg: "./testdata/dir-without-gofiles", + expected: []string{"./testdata/dir-without-gofiles"}, + }, + { + pkg: "./testdata/dir-with-gofiles", + expected: []string{"./testdata/dir-with-gofiles", "./testdata/dir-with-gofiles/subdir"}, + }, + } + + for i, tc := range testCases { + visited := []string{} + err := WalkPkg(tc.pkg, func(imp, abs string) error { + if _, base := path.Split(imp); base == "skipme" { + return ErrSkipPkg + } + visited = append(visited, imp) + return nil + }) + if err != nil && tc.fail { + continue + } + if err != nil { + t.Errorf("[%d] unexpected error: %v", i, err) + continue + } + if tc.fail { + t.Errorf("[%d] expected error", i) + continue + } + if !reflect.DeepEqual(visited, tc.expected) { + t.Errorf("[%d] unexpected results: %v", i, visited) + } + } +} +func Test_findPackage(t *testing.T) { + testCases := []struct { + pkg string + fail bool + }{ + { + pkg: "./testdata/nonexistent-dir", + fail: true, + }, + { + pkg: "./testdata/dir-without-gofiles", + }, + { + pkg: "./testdata/dir-with-gofiles", + }, + } + + for i, tc := range testCases { + _, err := findPackage(tc.pkg) + if err != nil && tc.fail { + continue + } + if err != nil { + t.Errorf("[%d] unexpected error: %v", i, err) + continue + } + if tc.fail { + t.Errorf("[%d] expected error", i) + continue + } + } +} + +func Test_readDirInfos(t *testing.T) { + testCases := []struct { + dir string + fail bool + expected map[string]bool + }{ + { + dir: "./testdata/nonexistent-dir", + fail: true, + }, + { + dir: "./testdata/dir-without-gofiles", + expected: map[string]bool{"README": true}, + }, + { + dir: "./testdata/dir-with-gofiles", + expected: map[string]bool{ + "README": true, + "foo.go": true, + "bar.go": true, + "subdir": true, + "testdata": true, + "_underscore": true, + ".dot": true, + "skipme": true, + }, + }, + } + + for i, tc := range testCases { + infos, err := readDirInfos(tc.dir) + if err != nil && tc.fail { + continue + } + if err != nil { + t.Errorf("[%d] unexpected error: %v", i, err) + continue + } + if tc.fail { + t.Errorf("[%d] expected error", i) + continue + } + result := make([]string, len(infos)) + sorted := make([]string, len(infos)) + for i, inf := range infos { + result[i] = inf.Name() + sorted[i] = inf.Name() + } + sort.Strings(sorted) + if !reflect.DeepEqual(result, sorted) { + t.Errorf("[%d] result was not sorted: %v", i, result) + } + for _, r := range result { + if !tc.expected[r] { + t.Errorf("[%d] got unexpected result: %s", i, r) + } else { + delete(tc.expected, r) + } + } + for r := range tc.expected { + t.Errorf("[%d] missing expected result: %s", i, r) + } + } +} + +func Test_readDirNames(t *testing.T) { + testCases := []struct { + dir string + fail bool + expected map[string]bool + }{ + { + dir: "./testdata/nonexistent-dir", + fail: true, + }, + { + dir: "./testdata/dir-without-gofiles", + expected: map[string]bool{"README": true}, + }, + { + dir: "./testdata/dir-with-gofiles", + expected: map[string]bool{ + "README": true, + "foo.go": true, + "bar.go": true, + "subdir": true, + "testdata": true, + "_underscore": true, + ".dot": true, + "skipme": true, + }, + }, + } + + for i, tc := range testCases { + result, err := readDirNames(tc.dir) + if err != nil && tc.fail { + continue + } + if err != nil { + t.Errorf("[%d] unexpected error: %v", i, err) + continue + } + if tc.fail { + t.Errorf("[%d] expected error", i) + continue + } + for _, r := range result { + if !tc.expected[r] { + t.Errorf("[%d] got unexpected result: %s", i, r) + } else { + delete(tc.expected, r) + } + } + for r := range tc.expected { + t.Errorf("[%d] missing expected result: %s", i, r) + } + } +} diff --git a/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/.dot/README b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/.dot/README new file mode 100644 index 00000000000..e845566c06f --- /dev/null +++ b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/.dot/README @@ -0,0 +1 @@ +README diff --git a/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/.dot/subsubdir/README b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/.dot/subsubdir/README new file mode 100644 index 00000000000..e845566c06f --- /dev/null +++ b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/.dot/subsubdir/README @@ -0,0 +1 @@ +README diff --git a/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/README b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/README new file mode 100644 index 00000000000..e845566c06f --- /dev/null +++ b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/README @@ -0,0 +1 @@ +README diff --git a/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/_underscore/README b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/_underscore/README new file mode 100644 index 00000000000..e845566c06f --- /dev/null +++ b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/_underscore/README @@ -0,0 +1 @@ +README diff --git a/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/_underscore/subsubdir/README b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/_underscore/subsubdir/README new file mode 100644 index 00000000000..e845566c06f --- /dev/null +++ b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/_underscore/subsubdir/README @@ -0,0 +1 @@ +README diff --git a/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/bar.go b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/bar.go new file mode 100644 index 00000000000..ce91c95ae93 --- /dev/null +++ b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/bar.go @@ -0,0 +1,20 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gofiles + +func bar() { +} diff --git a/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/foo.go b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/foo.go new file mode 100644 index 00000000000..57690637d17 --- /dev/null +++ b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/foo.go @@ -0,0 +1,20 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gofiles + +func foo() { +} diff --git a/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/skipme/README b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/skipme/README new file mode 100644 index 00000000000..e845566c06f --- /dev/null +++ b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/skipme/README @@ -0,0 +1 @@ +README diff --git a/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/skipme/subsubdir/README b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/skipme/subsubdir/README new file mode 100644 index 00000000000..e845566c06f --- /dev/null +++ b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/skipme/subsubdir/README @@ -0,0 +1 @@ +README diff --git a/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/subdir/README b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/subdir/README new file mode 100644 index 00000000000..e845566c06f --- /dev/null +++ b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/subdir/README @@ -0,0 +1 @@ +README diff --git a/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/testdata/README b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/testdata/README new file mode 100644 index 00000000000..e845566c06f --- /dev/null +++ b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/testdata/README @@ -0,0 +1 @@ +README diff --git a/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/testdata/subsubdir/README b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/testdata/subsubdir/README new file mode 100644 index 00000000000..e845566c06f --- /dev/null +++ b/hack/make-rules/helpers/go2make/testdata/dir-with-gofiles/testdata/subsubdir/README @@ -0,0 +1 @@ +README diff --git a/hack/make-rules/helpers/go2make/testdata/dir-without-gofiles/README b/hack/make-rules/helpers/go2make/testdata/dir-without-gofiles/README new file mode 100644 index 00000000000..e845566c06f --- /dev/null +++ b/hack/make-rules/helpers/go2make/testdata/dir-without-gofiles/README @@ -0,0 +1 @@ +README From 6d621ea5494d356f57881e99e07e44961ddf2223 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Sun, 29 Apr 2018 23:27:27 +0100 Subject: [PATCH 07/11] Simplify build for deepcopy --- build/root/Makefile.generated_files | 1276 ++++++++++++--------------- 1 file changed, 585 insertions(+), 691 deletions(-) diff --git a/build/root/Makefile.generated_files b/build/root/Makefile.generated_files index 68e45dfa573..4ce6c804a29 100644 --- a/build/root/Makefile.generated_files +++ b/build/root/Makefile.generated_files @@ -35,78 +35,58 @@ SHELL := /bin/bash # This rule collects all the generated file sets into a single rule. Other # rules should depend on this to ensure generated files are rebuilt. .PHONY: generated_files -generated_files: gen_deepcopy gen_defaulter gen_conversion gen_openapi gen_bindata +generated_files: gen_deepcopy + +##TH##FIXMEgen_defaulter gen_conversion gen_openapi gen_bindata + -# Code-generation logic. # -# This stuff can be pretty tricky, and there's probably some corner cases that -# we don't handle well. That said, here's a straightforward test to prove that -# the most common cases work. Sadly, it is manual. +# Helper logic to calculate Go's dependency DAG ourselves. # -# make clean -# find . -name .make\* | xargs rm -f -# find . -name zz_generated\* | xargs rm -f -# # verify `find . -name zz_generated.deepcopy.go | wc -l` is 0 -# # verify `find . -name .make | wc -l` is 0 + +# This is a file that will be emitted by the go2make tool, containing a +# variable for each Go package in the project (including deps) which lists all +# of the transitive deps of that package. Each variable is named the same as +# the package - for example the variable for `k8s.io/kubernetes/pkg/api` is +# $(k8s.io/kubernetes/pkg/api). This is roughly the same DAG that the Go +# compiler uses. These variables can be used to figure out if, for example, +# generated code needs to be regenerated. +GO_PKGDEPS_FILE = go-pkgdeps.mk + +# Include the Go package dependencies file. This will cause the rule of +# the same name to be considered and if it is updated, make will restart and +# reload the updated deps. +sinclude $(META_DIR)/$(GO_PKGDEPS_FILE) + +# Update the set of Go deps for our project. This will let us determine if +# we really need to do expensive codegen. We use FORCE because it is not a +# PHONY file, but we do want it to be re-evaluated every time make is run. The +# file will only be touched if it actually changes. +$(META_DIR)/$(GO_PKGDEPS_FILE): FORCE + if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ + echo "DBG: calculating Go dependencies"; \ + fi + hack/run-in-gopath.sh go install ./hack/make-rules/helpers/go2make + hack/run-in-gopath.sh go2make \ + k8s.io/kubernetes/... \ + --prune k8s.io/kubernetes/staging \ + --prune k8s.io/kubernetes/vendor \ + k8s.io/kubernetes/vendor/k8s.io/... \ + > $@.tmp + if ! cmp -s $@.tmp $@; then \ + if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ + echo "DBG: $(GO_PKGDEPS_FILE) changed"; \ + fi; \ + cat $@.tmp > $@; \ + fi + rm -f $@.tmp + +.PHONY: FORCE +FORCE: + # -# make nonexistent -# # expect "No rule to make target" -# # verify `find .make/ -type f | wc -l` has many files +# Helper logic to find which directories need codegen as quickly as possible. # -# make gen_deepcopy -# # expect deepcopy-gen is built exactly once -# # expect many files to be regenerated -# # verify `find . -name zz_generated.deepcopy.go | wc -l` has files -# make gen_deepcopy -# # expect nothing to be rebuilt, finish in O(seconds) -# touch pkg/api/types.go -# make gen_deepcopy -# # expect one file to be regenerated -# make gen_deepcopy -# # expect nothing to be rebuilt, finish in O(seconds) -# touch vendor/k8s.io/code-generator/cmd/deepcopy-gen/main.go -# make gen_deepcopy -# # expect deepcopy-gen is built exactly once -# # expect many files to be regenerated -# # verify `find . -name zz_generated.deepcopy.go | wc -l` has files -# make gen_deepcopy -# # expect nothing to be rebuilt, finish in O(seconds) -# -# make gen_conversion -# # expect conversion-gen is built exactly once -# # expect many files to be regenerated -# # verify `find . -name zz_generated.conversion.go | wc -l` has files -# make gen_conversion -# # expect nothing to be rebuilt, finish in O(seconds) -# touch pkg/api/types.go -# make gen_conversion -# # expect one file to be regenerated -# make gen_conversion -# # expect nothing to be rebuilt, finish in O(seconds) -# touch vendor/k8s.io/code-generator/cmd/conversion-gen/main.go -# make gen_conversion -# # expect conversion-gen is built exactly once -# # expect many files to be regenerated -# # verify `find . -name zz_generated.conversion.go | wc -l` has files -# make gen_conversion -# # expect nothing to be rebuilt, finish in O(seconds) -# -# make all -# # expect it to build -# -# make test -# # expect it to pass -# -# make clean -# # verify `find . -name zz_generated.deepcopy.go | wc -l` is 0 -# # verify `find . -name .make | wc -l` is 0 -# -# make all WHAT=cmd/kube-proxy -# # expect it to build -# -# make clean -# make test WHAT=cmd/kube-proxy -# # expect it to pass # This variable holds a list of every directory that contains Go files in this # project. Other rules and variables can use this as a starting point to @@ -118,63 +98,6 @@ ALL_GO_DIRS := $(shell \ hack/make-rules/helpers/cache_go_dirs.sh $(META_DIR)/all_go_dirs.mk \ ) -# The name of the metadata file which lists *.go files in each pkg. -GOFILES_META := gofiles.mk - -# Establish a dependency between the deps file and the dir. Whenever a dir -# changes (files added or removed) the deps file will be considered stale. -# -# The variable value was set in $(GOFILES_META) and included as part of the -# dependency management logic. -# -# This is looser than we really need (e.g. we don't really care about non *.go -# files or even *_test.go files), but this is much easier to represent. -# -# Because we 'sinclude' the deps file, it is considered for rebuilding, as part -# of make's normal evaluation. If it gets rebuilt, make will restart. -# -# The '$(eval)' is needed because this has a different RHS for each LHS, and -# would otherwise produce results that make can't parse. -$(foreach dir, $(ALL_GO_DIRS), $(eval \ - $(META_DIR)/$(dir)/$(GOFILES_META): $(dir) \ -)) - -# How to rebuild a deps file. When make determines that the deps file is stale -# (see above), it executes this rule, and then re-loads the deps file. -# -# This is looser than we really need (e.g. we don't really care about test -# files), but this is MUCH faster than calling `go list`. -# -# We regenerate the output file in order to satisfy make's "newer than" rules, -# but we only need to rebuild targets if the contents actually changed. That -# is what the .stamp file represents. -$(foreach dir, $(ALL_GO_DIRS), \ - $(META_DIR)/$(dir)/$(GOFILES_META)): - FILES=$$(ls $$@.tmp; \ - if ! cmp -s $@.tmp $@; then \ - if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ - echo "DBG: gofiles changed for $@"; \ - fi; \ - touch $@.stamp; \ - fi; \ - mv $@.tmp $@ - -# This is required to fill in the DAG, since some cases (e.g. 'make clean all') -# will reference the .stamp file when it doesn't exist. We don't need to -# rebuild it in that case, just keep make happy. -$(foreach dir, $(ALL_GO_DIRS), \ - $(META_DIR)/$(dir)/$(GOFILES_META).stamp): - -# Include any deps files as additional Makefile rules. This triggers make to -# consider the deps files for rebuild, which makes the whole -# dependency-management logic work. 'sinclude' is "silent include" which does -# not fail if the file does not exist. -$(foreach dir, $(ALL_GO_DIRS), $(eval \ - sinclude $(META_DIR)/$(dir)/$(GOFILES_META) \ -)) - # Generate a list of all files that have a `+k8s:` comment-tag. This will be # used to derive lists of files/dirs for generation tools. ifeq ($(DBG_MAKEFILE),1) @@ -185,7 +108,12 @@ ALL_K8S_TAG_FILES := $(shell \ | xargs grep --color=never -l '^// *+k8s:' \ ) + # +# Code generation logic. +# + + # Deep-copy generation # # Any package that wants deep-copy functions generated must include a @@ -215,10 +143,14 @@ DEEPCOPY_DIRS := $(shell \ ) DEEPCOPY_FILES := $(addsuffix /$(DEEPCOPY_FILENAME), $(DEEPCOPY_DIRS)) +# Reset the list of packages that need generation. +$(shell mkdir -p $$(dirname $(META_DIR)/$(DEEPCOPY_GEN))) +$(shell rm -f $(META_DIR)/$(DEEPCOPY_GEN).todo) + # This rule aggregates the set of files to generate and then generates them all # in a single run of the tool. .PHONY: gen_deepcopy -gen_deepcopy: $(DEEPCOPY_FILES) $(DEEPCOPY_GEN) +gen_deepcopy: $(DEEPCOPY_GEN) $(META_DIR)/$(DEEPCOPY_GEN).todo if [[ -s $(META_DIR)/$(DEEPCOPY_GEN).todo ]]; then \ pkgs=$$(cat $(META_DIR)/$(DEEPCOPY_GEN).todo | paste -sd, -); \ if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ @@ -242,590 +174,552 @@ gen_deepcopy: $(DEEPCOPY_FILES) $(DEEPCOPY_GEN) # # The '$(eval)' is needed because this has a different RHS for each LHS, and # would otherwise produce results that make can't parse. -# -# We depend on the $(GOFILES_META).stamp to detect when the set of input files -# has changed. This allows us to detect deleted input files. -$(foreach dir, $(DEEPCOPY_DIRS), $(eval \ - $(dir)/$(DEEPCOPY_FILENAME): $(META_DIR)/$(dir)/$(GOFILES_META).stamp \ - $(gofiles__$(dir)) \ +$(foreach dir, $(DEEPCOPY_DIRS), $(eval \ + $(dir)/$(DEEPCOPY_FILENAME): $($(PRJ_SRC_PATH)/$(dir)) \ )) -# Unilaterally remove any leftovers from previous runs. -$(shell rm -f $(META_DIR)/$(DEEPCOPY_GEN)*.todo) - # How to regenerate deep-copy code. This is a little slow to run, so we batch # it up and trigger the batch from the 'generated_files' target. +$(META_DIR)/$(DEEPCOPY_GEN).todo: $(DEEPCOPY_FILES) + $(DEEPCOPY_FILES): $(DEEPCOPY_GEN) - mkdir -p $$(dirname $(META_DIR)/$(DEEPCOPY_GEN)) if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ echo "DBG: deepcopy needed $(@D): $?"; \ ls -lf --full-time $@ $? || true; \ fi echo $(PRJ_SRC_PATH)/$(@D) >> $(META_DIR)/$(DEEPCOPY_GEN).todo -# This calculates the dependencies for the generator tool, so we only rebuild -# it when needed. It is PHONY so that it always runs, but it only updates the -# file if the contents have actually changed. We 'sinclude' this later. -.PHONY: $(META_DIR)/$(DEEPCOPY_GEN).mk -$(META_DIR)/$(DEEPCOPY_GEN).mk: - mkdir -p $(@D); \ - (echo -n "$(DEEPCOPY_GEN): "; \ - ./hack/run-in-gopath.sh go list \ - -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ - ./vendor/k8s.io/code-generator/cmd/deepcopy-gen \ - | grep --color=never "^$(PRJ_SRC_PATH)/" \ - | xargs ./hack/run-in-gopath.sh go list \ - -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ - | paste -sd' ' - \ - | sed 's/ / \\=,/g' \ - | tr '=,' '\n\t' \ - | sed "s|$$(pwd -P)/||"; \ - ) > $@.tmp; \ - if ! cmp -s $@.tmp $@; then \ - if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ - echo "DBG: $(DEEPCOPY_GEN).mk changed"; \ - fi; \ - cat $@.tmp > $@; \ - rm -f $@.tmp; \ - fi - -# Include dependency info for the generator tool. This will cause the rule of -# the same name to be considered and if it is updated, make will restart. -sinclude $(META_DIR)/$(DEEPCOPY_GEN).mk - # How to build the generator tool. The deps for this are defined in -# the $(DEEPCOPY_GEN).mk, above. +# the $(GO_PKGDEPS_FILE), above. # # A word on the need to touch: This rule might trigger if, for example, a # non-Go file was added or deleted from a directory on which this depends. # This target needs to be reconsidered, but Go realizes it doesn't actually # have to be rebuilt. In that case, make will forever see the dependency as -# newer than the binary, and try to rebuild it over and over. So we touch it, -# and make is happy. -$(DEEPCOPY_GEN): +# newer than the binary, and try to "rebuild" it over and over. So we touch +# it, and make is happy. +$(DEEPCOPY_GEN): $(k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/deepcopy-gen) hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/deepcopy-gen touch $@ -# -# Defaulter generation -# -# Any package that wants defaulter functions generated must include a -# comment-tag in column 0 of one file of the form: -# // +k8s:defaulter-gen= -# -# The depends on context: -# on types: -# true: always generate a defaulter for this type -# false: never generate a defaulter for this type -# on functions: -# covers: if the function name matches SetDefault_NAME, instructs -# the generator not to recurse -# on packages: -# FIELDNAME: any object with a field of this name is a candidate -# for having a defaulter generated -# The result file, in each pkg, of defaulter generation. -DEFAULTER_BASENAME := $(GENERATED_FILE_PREFIX)defaults -DEFAULTER_FILENAME := $(DEFAULTER_BASENAME).go - -# The tool used to generate defaulters. -DEFAULTER_GEN := $(BIN_DIR)/defaulter-gen - -# All directories that request any form of defaulter generation. -ifeq ($(DBG_MAKEFILE),1) - $(warning ***** finding all +k8s:defaulter-gen tags) -endif -DEFAULTER_DIRS := $(shell \ - grep --color=never -l '+k8s:defaulter-gen=' $(ALL_K8S_TAG_FILES) \ - | xargs -n1 dirname \ - | LC_ALL=C sort -u \ -) - -DEFAULTER_FILES := $(addsuffix /$(DEFAULTER_FILENAME), $(DEFAULTER_DIRS)) - -# This rule aggregates the set of files to generate and then generates them all -# in a single run of the tool. -.PHONY: gen_defaulter -gen_defaulter: $(DEFAULTER_FILES) $(DEFAULTER_GEN) - if [[ -s $(META_DIR)/$(DEFAULTER_GEN).todo ]]; then \ - pkgs=$$(cat $(META_DIR)/$(DEFAULTER_GEN).todo | paste -sd, -); \ - if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ - echo "DBG: running $(DEFAULTER_GEN) for $$pkgs"; \ - fi; \ - ./hack/run-in-gopath.sh $(DEFAULTER_GEN) \ - --v $(KUBE_VERBOSE) \ - --logtostderr \ - -i "$$pkgs" \ - --extra-peer-dirs $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(DEFAULTER_DIRS)) | sed 's/ /,/g') \ - -O $(DEFAULTER_BASENAME) \ - "$$@"; \ - fi - -# For each dir in DEFAULTER_DIRS, this establishes a dependency between the -# output file and the input files that should trigger a rebuild. -# -# The variable value was set in $(GOFILES_META) and included as part of the -# dependency management logic. -# -# Note that this is a deps-only statement, not a full rule (see below). This -# has to be done in a distinct step because wildcards don't work in static -# pattern rules. -# -# The '$(eval)' is needed because this has a different RHS for each LHS, and -# would otherwise produce results that make can't parse. -# -# We depend on the $(GOFILES_META).stamp to detect when the set of input files -# has changed. This allows us to detect deleted input files. -$(foreach dir, $(DEFAULTER_DIRS), $(eval \ - $(dir)/$(DEFAULTER_FILENAME): $(META_DIR)/$(dir)/$(GOFILES_META).stamp \ - $(gofiles__$(dir)) \ -)) - -# For each dir in DEFAULTER_DIRS, for each target in $(defaulters__$(dir)), -# this establishes a dependency between the output file and the input files -# that should trigger a rebuild. -# -# The variable value was set in $(GOFILES_META) and included as part of the -# dependency management logic. -# -# Note that this is a deps-only statement, not a full rule (see below). This -# has to be done in a distinct step because wildcards don't work in static -# pattern rules. -# -# The '$(eval)' is needed because this has a different RHS for each LHS, and -# would otherwise produce results that make can't parse. -# -# We depend on the $(GOFILES_META).stamp to detect when the set of input files -# has changed. This allows us to detect deleted input files. -$(foreach dir, $(DEFAULTER_DIRS), \ - $(foreach tgt, $(defaulters__$(dir)), $(eval \ - $(dir)/$(DEFAULTER_FILENAME): $(META_DIR)/$(tgt)/$(GOFILES_META).stamp \ - $(gofiles__$(tgt)) \ - )) \ -) - -# Unilaterally remove any leftovers from previous runs. -$(shell rm -f $(META_DIR)/$(DEFAULTER_GEN)*.todo) - -# How to regenerate defaulter code. This is a little slow to run, so we batch -# it up and trigger the batch from the 'generated_files' target. -$(DEFAULTER_FILES): $(DEFAULTER_GEN) - mkdir -p $$(dirname $(META_DIR)/$(DEFAULTER_GEN)) - if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ - echo "DBG: defaulter needed $(@D): $?"; \ - ls -lf --full-time $@ $? || true; \ - fi - echo $(PRJ_SRC_PATH)/$(@D) >> $(META_DIR)/$(DEFAULTER_GEN).todo - -# This calculates the dependencies for the generator tool, so we only rebuild -# it when needed. It is PHONY so that it always runs, but it only updates the -# file if the contents have actually changed. We 'sinclude' this later. -.PHONY: $(META_DIR)/$(DEFAULTER_GEN).mk -$(META_DIR)/$(DEFAULTER_GEN).mk: - mkdir -p $(@D); \ - (echo -n "$(DEFAULTER_GEN): "; \ - ./hack/run-in-gopath.sh go list \ - -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ - ./vendor/k8s.io/code-generator/cmd/defaulter-gen \ - | grep --color=never "^$(PRJ_SRC_PATH)/" \ - | xargs ./hack/run-in-gopath.sh go list \ - -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ - | paste -sd' ' - \ - | sed 's/ / \\=,/g' \ - | tr '=,' '\n\t' \ - | sed "s|$$(pwd -P)/||"; \ - ) > $@.tmp; \ - if ! cmp -s $@.tmp $@; then \ - if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ - echo "DBG: $(DEFAULTER_GEN).mk changed"; \ - fi; \ - cat $@.tmp > $@; \ - rm -f $@.tmp; \ - fi - -# Include dependency info for the generator tool. This will cause the rule of -# the same name to be considered and if it is updated, make will restart. -sinclude $(META_DIR)/$(DEFAULTER_GEN).mk - -# How to build the generator tool. The deps for this are defined in -# the $(DEFAULTER_GEN).mk, above. -# -# A word on the need to touch: This rule might trigger if, for example, a -# non-Go file was added or deleted from a directory on which this depends. -# This target needs to be reconsidered, but Go realizes it doesn't actually -# have to be rebuilt. In that case, make will forever see the dependency as -# newer than the binary, and try to rebuild it over and over. So we touch it, -# and make is happy. -$(DEFAULTER_GEN): - hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/defaulter-gen - touch $@ - -# -# Conversion generation -# -# Any package that wants conversion functions generated must include one or -# more comment-tags in any .go file, in column 0, of the form: -# // +k8s:conversion-gen= -# -# The CONVERSION_TARGET_DIR is a project-local path to another directory which -# should be considered when evaluating peer types for conversions. Types which -# are found in the source package (where conversions are being generated) -# but do not have a peer in one of the target directories will not have -# conversions generated. -# -# TODO: it might be better in the long term to make peer-types explicit in the -# IDL. - -# The result file, in each pkg, of conversion generation. -CONVERSION_BASENAME := $(GENERATED_FILE_PREFIX)conversion -CONVERSION_FILENAME := $(CONVERSION_BASENAME).go - -# The tool used to generate conversions. -CONVERSION_GEN := $(BIN_DIR)/conversion-gen - -# The name of the metadata file listing conversion peers for each pkg. -CONVERSIONS_META := conversions.mk - -# All directories that request any form of conversion generation. -ifeq ($(DBG_MAKEFILE),1) - $(warning ***** finding all +k8s:conversion-gen tags) -endif -CONVERSION_DIRS := $(shell \ - grep --color=never '^// *+k8s:conversion-gen=' $(ALL_K8S_TAG_FILES) \ - | cut -f1 -d: \ - | xargs -n1 dirname \ - | LC_ALL=C sort -u \ -) - -CONVERSION_FILES := $(addsuffix /$(CONVERSION_FILENAME), $(CONVERSION_DIRS)) -CONVERSION_EXTRA_PEER_DIRS := k8s.io/kubernetes/pkg/apis/core,k8s.io/kubernetes/pkg/apis/core/v1,k8s.io/api/core/v1 - -# This rule aggregates the set of files to generate and then generates them all -# in a single run of the tool. -.PHONY: gen_conversion -gen_conversion: $(CONVERSION_FILES) $(CONVERSION_GEN) - if [[ -s $(META_DIR)/$(CONVERSION_GEN).todo ]]; then \ - pkgs=$$(cat $(META_DIR)/$(CONVERSION_GEN).todo | paste -sd, -); \ - if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ - echo "DBG: running $(CONVERSION_GEN) for $$pkgs"; \ - fi; \ - ./hack/run-in-gopath.sh $(CONVERSION_GEN) \ - --extra-peer-dirs $(CONVERSION_EXTRA_PEER_DIRS) \ - --v $(KUBE_VERBOSE) \ - --logtostderr \ - -i "$$pkgs" \ - -O $(CONVERSION_BASENAME) \ - "$$@"; \ - fi - -# Establish a dependency between the deps file and the dir. Whenever a dir -# changes (files added or removed) the deps file will be considered stale. -# -# This is looser than we really need (e.g. we don't really care about non *.go -# files or even *_test.go files), but this is much easier to represent. -# -# Because we 'sinclude' the deps file, it is considered for rebuilding, as part -# of make's normal evaluation. If it gets rebuilt, make will restart. -# -# The '$(eval)' is needed because this has a different RHS for each LHS, and -# would otherwise produce results that make can't parse. -$(foreach dir, $(CONVERSION_DIRS), $(eval \ - $(META_DIR)/$(dir)/$(CONVERSIONS_META): $(dir) \ -)) - -# How to rebuild a deps file. When make determines that the deps file is stale -# (see above), it executes this rule, and then re-loads the deps file. -# -# This is looser than we really need (e.g. we don't really care about test -# files), but this is MUCH faster than calling `go list`. -# -# We regenerate the output file in order to satisfy make's "newer than" rules, -# but we only need to rebuild targets if the contents actually changed. That -# is what the .stamp file represents. -$(foreach dir, $(CONVERSION_DIRS), \ - $(META_DIR)/$(dir)/$(CONVERSIONS_META)): - TAGS=$$(grep --color=never -h '^// *+k8s:conversion-gen=' $$@.tmp; \ - if ! cmp -s $@.tmp $@; then \ - if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ - echo "DBG: conversions changed for $@"; \ - fi; \ - touch $@.stamp; \ - fi; \ - mv $@.tmp $@ - -# Include any deps files as additional Makefile rules. This triggers make to -# consider the deps files for rebuild, which makes the whole -# dependency-management logic work. 'sinclude' is "silent include" which does -# not fail if the file does not exist. -$(foreach dir, $(CONVERSION_DIRS), $(eval \ - sinclude $(META_DIR)/$(dir)/$(CONVERSIONS_META) \ -)) - -# For each dir in CONVERSION_DIRS, this establishes a dependency between the -# output file and the input files that should trigger a rebuild. -# -# The variable value was set in $(GOFILES_META) and included as part of the -# dependency management logic. -# -# Note that this is a deps-only statement, not a full rule (see below). This -# has to be done in a distinct step because wildcards don't work in static -# pattern rules. -# -# The '$(eval)' is needed because this has a different RHS for each LHS, and -# would otherwise produce results that make can't parse. -# -# We depend on the $(GOFILES_META).stamp to detect when the set of input files -# has changed. This allows us to detect deleted input files. -$(foreach dir, $(CONVERSION_DIRS), $(eval \ - $(dir)/$(CONVERSION_FILENAME): $(META_DIR)/$(dir)/$(GOFILES_META).stamp \ - $(gofiles__$(dir)) \ -)) - -# For each dir in CONVERSION_DIRS, for each target in $(conversions__$(dir)), -# this establishes a dependency between the output file and the input files -# that should trigger a rebuild. -# -# The variable value was set in $(GOFILES_META) and included as part of the -# dependency management logic. -# -# Note that this is a deps-only statement, not a full rule (see below). This -# has to be done in a distinct step because wildcards don't work in static -# pattern rules. -# -# The '$(eval)' is needed because this has a different RHS for each LHS, and -# would otherwise produce results that make can't parse. -# -# We depend on the $(GOFILES_META).stamp to detect when the set of input files -# has changed. This allows us to detect deleted input files. -$(foreach dir, $(CONVERSION_DIRS), \ - $(foreach tgt, $(conversions__$(dir)), $(eval \ - $(dir)/$(CONVERSION_FILENAME): $(META_DIR)/$(tgt)/$(GOFILES_META).stamp \ - $(gofiles__$(tgt)) \ - )) \ -) - -# Unilaterally remove any leftovers from previous runs. -$(shell rm -f $(META_DIR)/$(CONVERSION_GEN)*.todo) - -# How to regenerate conversion code. This is a little slow to run, so we batch -# it up and trigger the batch from the 'generated_files' target. -$(CONVERSION_FILES): $(CONVERSION_GEN) - mkdir -p $$(dirname $(META_DIR)/$(CONVERSION_GEN)) - if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ - echo "DBG: conversion needed $(@D): $?"; \ - ls -lf --full-time $@ $? || true; \ - fi - echo $(PRJ_SRC_PATH)/$(@D) >> $(META_DIR)/$(CONVERSION_GEN).todo - -# This calculates the dependencies for the generator tool, so we only rebuild -# it when needed. It is PHONY so that it always runs, but it only updates the -# file if the contents have actually changed. We 'sinclude' this later. -.PHONY: $(META_DIR)/$(CONVERSION_GEN).mk -$(META_DIR)/$(CONVERSION_GEN).mk: - mkdir -p $(@D); \ - (echo -n "$(CONVERSION_GEN): "; \ - ./hack/run-in-gopath.sh go list \ - -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ - ./vendor/k8s.io/code-generator/cmd/conversion-gen \ - | grep --color=never "^$(PRJ_SRC_PATH)/" \ - | xargs ./hack/run-in-gopath.sh go list \ - -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ - | paste -sd' ' - \ - | sed 's/ / \\=,/g' \ - | tr '=,' '\n\t' \ - | sed "s|$$(pwd -P)/||"; \ - ) > $@.tmp; \ - if ! cmp -s $@.tmp $@; then \ - if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ - echo "DBG: $(CONVERSION_GEN).mk changed"; \ - fi; \ - cat $@.tmp > $@; \ - rm -f $@.tmp; \ - fi - -# Include dependency info for the generator tool. This will cause the rule of -# the same name to be considered and if it is updated, make will restart. -sinclude $(META_DIR)/$(CONVERSION_GEN).mk - -# How to build the generator tool. The deps for this are defined in -# the $(CONVERSION_GEN).mk, above. -# -# A word on the need to touch: This rule might trigger if, for example, a -# non-Go file was added or deleted from a directory on which this depends. -# This target needs to be reconsidered, but Go realizes it doesn't actually -# have to be rebuilt. In that case, make will forever see the dependency as -# newer than the binary, and try to rebuild it over and over. So we touch it, -# and make is happy. -$(CONVERSION_GEN): - hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/conversion-gen - touch $@ - -# -# Open-api generation -# -# Any package that wants open-api functions generated must include a -# comment-tag in column 0 of one file of the form: -# // +k8s:openapi-gen=true -# -# The result file, in each pkg, of open-api generation. -OPENAPI_BASENAME := $(GENERATED_FILE_PREFIX)openapi -OPENAPI_FILENAME := $(OPENAPI_BASENAME).go -OPENAPI_OUTPUT_PKG := pkg/generated/openapi - -# The tool used to generate open apis. -OPENAPI_GEN := $(BIN_DIR)/openapi-gen - -# Find all the directories that request open-api generation. -ifeq ($(DBG_MAKEFILE),1) - $(warning ***** finding all +k8s:openapi-gen tags) -endif -OPENAPI_DIRS := $(shell \ - grep --color=never -l '+k8s:openapi-gen=' $(ALL_K8S_TAG_FILES) \ - | xargs -n1 dirname \ - | LC_ALL=C sort -u \ -) - -OPENAPI_OUTFILE := $(OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME) - -# This rule is the user-friendly entrypoint for openapi generation. -.PHONY: gen_openapi -gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN) - -# For each dir in OPENAPI_DIRS, this establishes a dependency between the -# output file and the input files that should trigger a rebuild. -# -# Note that this is a deps-only statement, not a full rule (see below). This -# has to be done in a distinct step because wildcards don't work in static -# pattern rules. -# -# The '$(eval)' is needed because this has a different RHS for each LHS, and -# would otherwise produce results that make can't parse. -# -# We depend on the $(GOFILES_META).stamp to detect when the set of input files -# has changed. This allows us to detect deleted input files. -$(foreach dir, $(OPENAPI_DIRS), $(eval \ - $(OPENAPI_OUTFILE): $(META_DIR)/$(dir)/$(GOFILES_META).stamp \ - $(gofiles__$(dir)) \ -)) - -# How to regenerate open-api code. This emits a single file for all results. -$(OPENAPI_OUTFILE): $(OPENAPI_GEN) $(OPENAPI_GEN) - function run_gen_openapi() { \ - ./hack/run-in-gopath.sh $(OPENAPI_GEN) \ - --v $(KUBE_VERBOSE) \ - --logtostderr \ - -i $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(OPENAPI_DIRS)) | sed 's/ /,/g') \ - -p $(PRJ_SRC_PATH)/$(OPENAPI_OUTPUT_PKG) \ - -O $(OPENAPI_BASENAME) \ - "$$@"; \ - }; \ - run_gen_openapi - -# This calculates the dependencies for the generator tool, so we only rebuild -# it when needed. It is PHONY so that it always runs, but it only updates the -# file if the contents have actually changed. We 'sinclude' this later. -.PHONY: $(META_DIR)/$(OPENAPI_GEN).mk -$(META_DIR)/$(OPENAPI_GEN).mk: - mkdir -p $(@D); \ - (echo -n "$(OPENAPI_GEN): "; \ - ./hack/run-in-gopath.sh go list \ - -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ - ./vendor/k8s.io/code-generator/cmd/openapi-gen \ - | grep --color=never "^$(PRJ_SRC_PATH)/" \ - | xargs ./hack/run-in-gopath.sh go list \ - -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ - | paste -sd' ' - \ - | sed 's/ / \\=,/g' \ - | tr '=,' '\n\t' \ - | sed "s|$$(pwd -P)/||"; \ - ) > $@.tmp; \ - if ! cmp -s $@.tmp $@; then \ - if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ - echo "DBG: $(OPENAPI_GEN).mk changed"; \ - fi; \ - cat $@.tmp > $@; \ - rm -f $@.tmp; \ - fi - -# Include dependency info for the generator tool. This will cause the rule of -# the same name to be considered and if it is updated, make will restart. -sinclude $(META_DIR)/$(OPENAPI_GEN).mk - -# How to build the generator tool. The deps for this are defined in -# the $(OPENAPI_GEN).mk, above. -# -# A word on the need to touch: This rule might trigger if, for example, a -# non-Go file was added or deleted from a directory on which this depends. -# This target needs to be reconsidered, but Go realizes it doesn't actually -# have to be rebuilt. In that case, make will forever see the dependency as -# newer than the binary, and try to rebuild it over and over. So we touch it, -# and make is happy. -$(OPENAPI_GEN): - hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/openapi-gen - touch $@ - -# -# bindata generation -# - -# The tool used to generate bindata files. -BINDATA_GEN := $(BIN_DIR)/go-bindata - -# A wrapper script that generates all bindata files. It is fast enough that we -# don't care. -BINDATA_SCRIPT := hack/generate-bindata.sh - -# This rule is the user-friendly entrypoint for bindata generation. -.PHONY: gen_bindata -gen_bindata: $(BINDATA_GEN) FORCE - ./hack/run-in-gopath.sh $(BINDATA_SCRIPT) - -FORCE: - -# This calculates the dependencies for the generator tool, so we only rebuild -# it when needed. It is PHONY so that it always runs, but it only updates the -# file if the contents have actually changed. We 'sinclude' this later. -.PHONY: $(META_DIR)/$(BINDATA_GEN).mk -$(META_DIR)/$(BINDATA_GEN).mk: - mkdir -p $(@D); \ - (echo -n "$(BINDATA_GEN): "; \ - ./hack/run-in-gopath.sh go list \ - -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ - ./vendor/github.com/jteeuwen/go-bindata/go-bindata \ - | grep --color=never "^$(PRJ_SRC_PATH)/" \ - | xargs ./hack/run-in-gopath.sh go list \ - -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ - | paste -sd' ' - \ - | sed 's/ / \\=,/g' \ - | tr '=,' '\n\t' \ - | sed "s|$$(pwd -P)/||"; \ - ) > $@.tmp; \ - if ! cmp -s $@.tmp $@; then \ - if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ - echo "DBG: $(BINDATA_GEN).mk changed"; \ - fi; \ - cat $@.tmp > $@; \ - rm -f $@.tmp; \ - fi - -# Include dependency info for the generator tool. This will cause the rule of -# the same name to be considered and if it is updated, make will restart. -sinclude $(META_DIR)/$(BINDATA_GEN).mk - -# How to build the generator tool. The deps for this are defined in -# the $(BINDATA_GEN).mk, above. -# -# A word on the need to touch: This rule might trigger if, for example, a -# non-Go file was added or deleted from a directory on which this depends. -# This target needs to be reconsidered, but Go realizes it doesn't actually -# have to be rebuilt. In that case, make will forever see the dependency as -# newer than the binary, and try to rebuild it over and over. So we touch it, -# and make is happy. -$(BINDATA_GEN): - hack/make-rules/build.sh ./vendor/github.com/jteeuwen/go-bindata/go-bindata - touch $@ +##TH### Defaulter generation +##TH### +##TH### Any package that wants defaulter functions generated must include a +##TH### comment-tag in column 0 of one file of the form: +##TH### // +k8s:defaulter-gen= +##TH### +##TH### The depends on context: +##TH### on types: +##TH### true: always generate a defaulter for this type +##TH### false: never generate a defaulter for this type +##TH### on functions: +##TH### covers: if the function name matches SetDefault_NAME, instructs +##TH### the generator not to recurse +##TH### on packages: +##TH### FIELDNAME: any object with a field of this name is a candidate +##TH### for having a defaulter generated +##TH## +##TH### The result file, in each pkg, of defaulter generation. +##TH##DEFAULTER_BASENAME := $(GENERATED_FILE_PREFIX)defaults +##TH##DEFAULTER_FILENAME := $(DEFAULTER_BASENAME).go +##TH## +##TH### The tool used to generate defaulters. +##TH##DEFAULTER_GEN := $(BIN_DIR)/defaulter-gen +##TH## +##TH### All directories that request any form of defaulter generation. +##TH##ifeq ($(DBG_MAKEFILE),1) +##TH## $(warning ***** finding all +k8s:defaulter-gen tags) +##TH##endif +##TH##DEFAULTER_DIRS := $(shell \ +##TH## grep --color=never -l '+k8s:defaulter-gen=' $(ALL_K8S_TAG_FILES) \ +##TH## | xargs -n1 dirname \ +##TH## | LC_ALL=C sort -u \ +##TH##) +##TH## +##TH##DEFAULTER_FILES := $(addsuffix /$(DEFAULTER_FILENAME), $(DEFAULTER_DIRS)) +##TH## +##TH### This rule aggregates the set of files to generate and then generates them all +##TH### in a single run of the tool. +##TH##.PHONY: gen_defaulter +##TH##gen_defaulter: $(DEFAULTER_FILES) $(DEFAULTER_GEN) +##TH## if [[ -s $(META_DIR)/$(DEFAULTER_GEN).todo ]]; then \ +##TH## pkgs=$$(cat $(META_DIR)/$(DEFAULTER_GEN).todo | paste -sd, -); \ +##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ +##TH## echo "DBG: running $(DEFAULTER_GEN) for $$pkgs"; \ +##TH## fi; \ +##TH## ./hack/run-in-gopath.sh $(DEFAULTER_GEN) \ +##TH## --v $(KUBE_VERBOSE) \ +##TH## --logtostderr \ +##TH## -i "$$pkgs" \ +##TH## --extra-peer-dirs $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(DEFAULTER_DIRS)) | sed 's/ /,/g') \ +##TH## -O $(DEFAULTER_BASENAME) \ +##TH## "$$@"; \ +##TH## fi +##TH## +##TH### For each dir in DEFAULTER_DIRS, this establishes a dependency between the +##TH### output file and the input files that should trigger a rebuild. +##TH### +##TH### The variable value was set in $(GOFILES_META) and included as part of the +##TH### dependency management logic. +##TH### +##TH### Note that this is a deps-only statement, not a full rule (see below). This +##TH### has to be done in a distinct step because wildcards don't work in static +##TH### pattern rules. +##TH### +##TH### The '$(eval)' is needed because this has a different RHS for each LHS, and +##TH### would otherwise produce results that make can't parse. +##TH### +##TH### We depend on the $(GOFILES_META).stamp to detect when the set of input files +##TH### has changed. This allows us to detect deleted input files. +##TH##$(foreach dir, $(DEFAULTER_DIRS), $(eval \ +##TH## $(dir)/$(DEFAULTER_FILENAME): $(META_DIR)/$(dir)/$(GOFILES_META).stamp \ +##TH## $(gofiles__$(dir)) \ +##TH##)) +##TH## +##TH### For each dir in DEFAULTER_DIRS, for each target in $(defaulters__$(dir)), +##TH### this establishes a dependency between the output file and the input files +##TH### that should trigger a rebuild. +##TH### +##TH### The variable value was set in $(GOFILES_META) and included as part of the +##TH### dependency management logic. +##TH### +##TH### Note that this is a deps-only statement, not a full rule (see below). This +##TH### has to be done in a distinct step because wildcards don't work in static +##TH### pattern rules. +##TH### +##TH### The '$(eval)' is needed because this has a different RHS for each LHS, and +##TH### would otherwise produce results that make can't parse. +##TH### +##TH### We depend on the $(GOFILES_META).stamp to detect when the set of input files +##TH### has changed. This allows us to detect deleted input files. +##TH##$(foreach dir, $(DEFAULTER_DIRS), \ +##TH## $(foreach tgt, $(defaulters__$(dir)), $(eval \ +##TH## $(dir)/$(DEFAULTER_FILENAME): $(META_DIR)/$(tgt)/$(GOFILES_META).stamp \ +##TH## $(gofiles__$(tgt)) \ +##TH## )) \ +##TH##) +##TH## +##TH### Unilaterally remove any leftovers from previous runs. +##TH##$(shell rm -f $(META_DIR)/$(DEFAULTER_GEN)*.todo) +##TH## +##TH### How to regenerate defaulter code. This is a little slow to run, so we batch +##TH### it up and trigger the batch from the 'generated_files' target. +##TH##$(DEFAULTER_FILES): $(DEFAULTER_GEN) +##TH## mkdir -p $$(dirname $(META_DIR)/$(DEFAULTER_GEN)) +##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ +##TH## echo "DBG: defaulter needed $(@D): $?"; \ +##TH## ls -lf --full-time $@ $? || true; \ +##TH## fi +##TH## echo $(PRJ_SRC_PATH)/$(@D) >> $(META_DIR)/$(DEFAULTER_GEN).todo +##TH## +##TH### This calculates the dependencies for the generator tool, so we only rebuild +##TH### it when needed. It is PHONY so that it always runs, but it only updates the +##TH### file if the contents have actually changed. We 'sinclude' this later. +##TH##.PHONY: $(META_DIR)/$(DEFAULTER_GEN).mk +##TH##$(META_DIR)/$(DEFAULTER_GEN).mk: +##TH## mkdir -p $(@D); \ +##TH## (echo -n "$(DEFAULTER_GEN): "; \ +##TH## ./hack/run-in-gopath.sh go list \ +##TH## -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ +##TH## ./vendor/k8s.io/code-generator/cmd/defaulter-gen \ +##TH## | grep --color=never "^$(PRJ_SRC_PATH)/" \ +##TH## | xargs ./hack/run-in-gopath.sh go list \ +##TH## -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ +##TH## | paste -sd' ' - \ +##TH## | sed 's/ / \\=,/g' \ +##TH## | tr '=,' '\n\t' \ +##TH## | sed "s|$$(pwd -P)/||"; \ +##TH## ) > $@.tmp; \ +##TH## if ! cmp -s $@.tmp $@; then \ +##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ +##TH## echo "DBG: $(DEFAULTER_GEN).mk changed"; \ +##TH## fi; \ +##TH## cat $@.tmp > $@; \ +##TH## rm -f $@.tmp; \ +##TH## fi +##TH## +##TH### Include dependency info for the generator tool. This will cause the rule of +##TH### the same name to be considered and if it is updated, make will restart. +##TH##sinclude $(META_DIR)/$(DEFAULTER_GEN).mk +##TH## +##TH### How to build the generator tool. The deps for this are defined in +##TH### the $(DEFAULTER_GEN).mk, above. +##TH### +##TH### A word on the need to touch: This rule might trigger if, for example, a +##TH### non-Go file was added or deleted from a directory on which this depends. +##TH### This target needs to be reconsidered, but Go realizes it doesn't actually +##TH### have to be rebuilt. In that case, make will forever see the dependency as +##TH### newer than the binary, and try to rebuild it over and over. So we touch it, +##TH### and make is happy. +##TH##$(DEFAULTER_GEN): +##TH## hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/defaulter-gen +##TH## touch $@ +##TH## +##TH### +##TH### Conversion generation +##TH### +##TH### Any package that wants conversion functions generated must include one or +##TH### more comment-tags in any .go file, in column 0, of the form: +##TH### // +k8s:conversion-gen= +##TH### +##TH### The CONVERSION_TARGET_DIR is a project-local path to another directory which +##TH### should be considered when evaluating peer types for conversions. Types which +##TH### are found in the source package (where conversions are being generated) +##TH### but do not have a peer in one of the target directories will not have +##TH### conversions generated. +##TH### +##TH### TODO: it might be better in the long term to make peer-types explicit in the +##TH### IDL. +##TH## +##TH### The result file, in each pkg, of conversion generation. +##TH##CONVERSION_BASENAME := $(GENERATED_FILE_PREFIX)conversion +##TH##CONVERSION_FILENAME := $(CONVERSION_BASENAME).go +##TH## +##TH### The tool used to generate conversions. +##TH##CONVERSION_GEN := $(BIN_DIR)/conversion-gen +##TH## +##TH### The name of the metadata file listing conversion peers for each pkg. +##TH##CONVERSIONS_META := conversions.mk +##TH## +##TH### All directories that request any form of conversion generation. +##TH##ifeq ($(DBG_MAKEFILE),1) +##TH## $(warning ***** finding all +k8s:conversion-gen tags) +##TH##endif +##TH##CONVERSION_DIRS := $(shell \ +##TH## grep --color=never '^// *+k8s:conversion-gen=' $(ALL_K8S_TAG_FILES) \ +##TH## | cut -f1 -d: \ +##TH## | xargs -n1 dirname \ +##TH## | LC_ALL=C sort -u \ +##TH##) +##TH## +##TH##CONVERSION_FILES := $(addsuffix /$(CONVERSION_FILENAME), $(CONVERSION_DIRS)) +##TH##CONVERSION_EXTRA_PEER_DIRS := k8s.io/kubernetes/pkg/apis/core,k8s.io/kubernetes/pkg/apis/core/v1,k8s.io/api/core/v1 +##TH## +##TH### This rule aggregates the set of files to generate and then generates them all +##TH### in a single run of the tool. +##TH##.PHONY: gen_conversion +##TH##gen_conversion: $(CONVERSION_FILES) $(CONVERSION_GEN) +##TH## if [[ -s $(META_DIR)/$(CONVERSION_GEN).todo ]]; then \ +##TH## pkgs=$$(cat $(META_DIR)/$(CONVERSION_GEN).todo | paste -sd, -); \ +##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ +##TH## echo "DBG: running $(CONVERSION_GEN) for $$pkgs"; \ +##TH## fi; \ +##TH## ./hack/run-in-gopath.sh $(CONVERSION_GEN) \ +##TH## --extra-peer-dirs $(CONVERSION_EXTRA_PEER_DIRS) \ +##TH## --v $(KUBE_VERBOSE) \ +##TH## --logtostderr \ +##TH## -i "$$pkgs" \ +##TH## -O $(CONVERSION_BASENAME) \ +##TH## "$$@"; \ +##TH## fi +##TH## +##TH### Establish a dependency between the deps file and the dir. Whenever a dir +##TH### changes (files added or removed) the deps file will be considered stale. +##TH### +##TH### This is looser than we really need (e.g. we don't really care about non *.go +##TH### files or even *_test.go files), but this is much easier to represent. +##TH### +##TH### Because we 'sinclude' the deps file, it is considered for rebuilding, as part +##TH### of make's normal evaluation. If it gets rebuilt, make will restart. +##TH### +##TH### The '$(eval)' is needed because this has a different RHS for each LHS, and +##TH### would otherwise produce results that make can't parse. +##TH##$(foreach dir, $(CONVERSION_DIRS), $(eval \ +##TH## $(META_DIR)/$(dir)/$(CONVERSIONS_META): $(dir) \ +##TH##)) +##TH## +##TH### How to rebuild a deps file. When make determines that the deps file is stale +##TH### (see above), it executes this rule, and then re-loads the deps file. +##TH### +##TH### This is looser than we really need (e.g. we don't really care about test +##TH### files), but this is MUCH faster than calling `go list`. +##TH### +##TH### We regenerate the output file in order to satisfy make's "newer than" rules, +##TH### but we only need to rebuild targets if the contents actually changed. That +##TH### is what the .stamp file represents. +##TH##$(foreach dir, $(CONVERSION_DIRS), \ +##TH## $(META_DIR)/$(dir)/$(CONVERSIONS_META)): +##TH## TAGS=$$(grep --color=never -h '^// *+k8s:conversion-gen=' $$@.tmp; \ +##TH## if ! cmp -s $@.tmp $@; then \ +##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ +##TH## echo "DBG: conversions changed for $@"; \ +##TH## fi; \ +##TH## touch $@.stamp; \ +##TH## fi; \ +##TH## mv $@.tmp $@ +##TH## +##TH### Include any deps files as additional Makefile rules. This triggers make to +##TH### consider the deps files for rebuild, which makes the whole +##TH### dependency-management logic work. 'sinclude' is "silent include" which does +##TH### not fail if the file does not exist. +##TH##$(foreach dir, $(CONVERSION_DIRS), $(eval \ +##TH## sinclude $(META_DIR)/$(dir)/$(CONVERSIONS_META) \ +##TH##)) +##TH## +##TH### For each dir in CONVERSION_DIRS, this establishes a dependency between the +##TH### output file and the input files that should trigger a rebuild. +##TH### +##TH### The variable value was set in $(GOFILES_META) and included as part of the +##TH### dependency management logic. +##TH### +##TH### Note that this is a deps-only statement, not a full rule (see below). This +##TH### has to be done in a distinct step because wildcards don't work in static +##TH### pattern rules. +##TH### +##TH### The '$(eval)' is needed because this has a different RHS for each LHS, and +##TH### would otherwise produce results that make can't parse. +##TH### +##TH### We depend on the $(GOFILES_META).stamp to detect when the set of input files +##TH### has changed. This allows us to detect deleted input files. +##TH##$(foreach dir, $(CONVERSION_DIRS), $(eval \ +##TH## $(dir)/$(CONVERSION_FILENAME): $(META_DIR)/$(dir)/$(GOFILES_META).stamp \ +##TH## $(gofiles__$(dir)) \ +##TH##)) +##TH## +##TH### For each dir in CONVERSION_DIRS, for each target in $(conversions__$(dir)), +##TH### this establishes a dependency between the output file and the input files +##TH### that should trigger a rebuild. +##TH### +##TH### The variable value was set in $(GOFILES_META) and included as part of the +##TH### dependency management logic. +##TH### +##TH### Note that this is a deps-only statement, not a full rule (see below). This +##TH### has to be done in a distinct step because wildcards don't work in static +##TH### pattern rules. +##TH### +##TH### The '$(eval)' is needed because this has a different RHS for each LHS, and +##TH### would otherwise produce results that make can't parse. +##TH### +##TH### We depend on the $(GOFILES_META).stamp to detect when the set of input files +##TH### has changed. This allows us to detect deleted input files. +##TH##$(foreach dir, $(CONVERSION_DIRS), \ +##TH## $(foreach tgt, $(conversions__$(dir)), $(eval \ +##TH## $(dir)/$(CONVERSION_FILENAME): $(META_DIR)/$(tgt)/$(GOFILES_META).stamp \ +##TH## $(gofiles__$(tgt)) \ +##TH## )) \ +##TH##) +##TH## +##TH### Unilaterally remove any leftovers from previous runs. +##TH##$(shell rm -f $(META_DIR)/$(CONVERSION_GEN)*.todo) +##TH## +##TH### How to regenerate conversion code. This is a little slow to run, so we batch +##TH### it up and trigger the batch from the 'generated_files' target. +##TH##$(CONVERSION_FILES): $(CONVERSION_GEN) +##TH## mkdir -p $$(dirname $(META_DIR)/$(CONVERSION_GEN)) +##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ +##TH## echo "DBG: conversion needed $(@D): $?"; \ +##TH## ls -lf --full-time $@ $? || true; \ +##TH## fi +##TH## echo $(PRJ_SRC_PATH)/$(@D) >> $(META_DIR)/$(CONVERSION_GEN).todo +##TH## +##TH### This calculates the dependencies for the generator tool, so we only rebuild +##TH### it when needed. It is PHONY so that it always runs, but it only updates the +##TH### file if the contents have actually changed. We 'sinclude' this later. +##TH##.PHONY: $(META_DIR)/$(CONVERSION_GEN).mk +##TH##$(META_DIR)/$(CONVERSION_GEN).mk: +##TH## mkdir -p $(@D); \ +##TH## (echo -n "$(CONVERSION_GEN): "; \ +##TH## ./hack/run-in-gopath.sh go list \ +##TH## -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ +##TH## ./vendor/k8s.io/code-generator/cmd/conversion-gen \ +##TH## | grep --color=never "^$(PRJ_SRC_PATH)/" \ +##TH## | xargs ./hack/run-in-gopath.sh go list \ +##TH## -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ +##TH## | paste -sd' ' - \ +##TH## | sed 's/ / \\=,/g' \ +##TH## | tr '=,' '\n\t' \ +##TH## | sed "s|$$(pwd -P)/||"; \ +##TH## ) > $@.tmp; \ +##TH## if ! cmp -s $@.tmp $@; then \ +##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ +##TH## echo "DBG: $(CONVERSION_GEN).mk changed"; \ +##TH## fi; \ +##TH## cat $@.tmp > $@; \ +##TH## rm -f $@.tmp; \ +##TH## fi +##TH## +##TH### Include dependency info for the generator tool. This will cause the rule of +##TH### the same name to be considered and if it is updated, make will restart. +##TH##sinclude $(META_DIR)/$(CONVERSION_GEN).mk +##TH## +##TH### How to build the generator tool. The deps for this are defined in +##TH### the $(CONVERSION_GEN).mk, above. +##TH### +##TH### A word on the need to touch: This rule might trigger if, for example, a +##TH### non-Go file was added or deleted from a directory on which this depends. +##TH### This target needs to be reconsidered, but Go realizes it doesn't actually +##TH### have to be rebuilt. In that case, make will forever see the dependency as +##TH### newer than the binary, and try to rebuild it over and over. So we touch it, +##TH### and make is happy. +##TH##$(CONVERSION_GEN): +##TH## hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/conversion-gen +##TH## touch $@ +##TH## +##TH### +##TH### Open-api generation +##TH### +##TH### Any package that wants open-api functions generated must include a +##TH### comment-tag in column 0 of one file of the form: +##TH### // +k8s:openapi-gen=true +##TH### +##TH### The result file, in each pkg, of open-api generation. +##TH##OPENAPI_BASENAME := $(GENERATED_FILE_PREFIX)openapi +##TH##OPENAPI_FILENAME := $(OPENAPI_BASENAME).go +##TH##OPENAPI_OUTPUT_PKG := pkg/generated/openapi +##TH## +##TH### The tool used to generate open apis. +##TH##OPENAPI_GEN := $(BIN_DIR)/openapi-gen +##TH## +##TH### Find all the directories that request open-api generation. +##TH##ifeq ($(DBG_MAKEFILE),1) +##TH## $(warning ***** finding all +k8s:openapi-gen tags) +##TH##endif +##TH##OPENAPI_DIRS := $(shell \ +##TH## grep --color=never -l '+k8s:openapi-gen=' $(ALL_K8S_TAG_FILES) \ +##TH## | xargs -n1 dirname \ +##TH## | LC_ALL=C sort -u \ +##TH##) +##TH## +##TH##OPENAPI_OUTFILE := $(OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME) +##TH## +##TH### This rule is the user-friendly entrypoint for openapi generation. +##TH##.PHONY: gen_openapi +##TH##gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN) +##TH## +##TH### For each dir in OPENAPI_DIRS, this establishes a dependency between the +##TH### output file and the input files that should trigger a rebuild. +##TH### +##TH### Note that this is a deps-only statement, not a full rule (see below). This +##TH### has to be done in a distinct step because wildcards don't work in static +##TH### pattern rules. +##TH### +##TH### The '$(eval)' is needed because this has a different RHS for each LHS, and +##TH### would otherwise produce results that make can't parse. +##TH### +##TH### We depend on the $(GOFILES_META).stamp to detect when the set of input files +##TH### has changed. This allows us to detect deleted input files. +##TH##$(foreach dir, $(OPENAPI_DIRS), $(eval \ +##TH## $(OPENAPI_OUTFILE): $(META_DIR)/$(dir)/$(GOFILES_META).stamp \ +##TH## $(gofiles__$(dir)) \ +##TH##)) +##TH## +##TH### How to regenerate open-api code. This emits a single file for all results. +##TH##$(OPENAPI_OUTFILE): $(OPENAPI_GEN) $(OPENAPI_GEN) +##TH## function run_gen_openapi() { \ +##TH## ./hack/run-in-gopath.sh $(OPENAPI_GEN) \ +##TH## --v $(KUBE_VERBOSE) \ +##TH## --logtostderr \ +##TH## -i $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(OPENAPI_DIRS)) | sed 's/ /,/g') \ +##TH## -p $(PRJ_SRC_PATH)/$(OPENAPI_OUTPUT_PKG) \ +##TH## -O $(OPENAPI_BASENAME) \ +##TH## "$$@"; \ +##TH## }; \ +##TH## run_gen_openapi +##TH## +##TH### This calculates the dependencies for the generator tool, so we only rebuild +##TH### it when needed. It is PHONY so that it always runs, but it only updates the +##TH### file if the contents have actually changed. We 'sinclude' this later. +##TH##.PHONY: $(META_DIR)/$(OPENAPI_GEN).mk +##TH##$(META_DIR)/$(OPENAPI_GEN).mk: +##TH## mkdir -p $(@D); \ +##TH## (echo -n "$(OPENAPI_GEN): "; \ +##TH## ./hack/run-in-gopath.sh go list \ +##TH## -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ +##TH## ./vendor/k8s.io/code-generator/cmd/openapi-gen \ +##TH## | grep --color=never "^$(PRJ_SRC_PATH)/" \ +##TH## | xargs ./hack/run-in-gopath.sh go list \ +##TH## -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ +##TH## | paste -sd' ' - \ +##TH## | sed 's/ / \\=,/g' \ +##TH## | tr '=,' '\n\t' \ +##TH## | sed "s|$$(pwd -P)/||"; \ +##TH## ) > $@.tmp; \ +##TH## if ! cmp -s $@.tmp $@; then \ +##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ +##TH## echo "DBG: $(OPENAPI_GEN).mk changed"; \ +##TH## fi; \ +##TH## cat $@.tmp > $@; \ +##TH## rm -f $@.tmp; \ +##TH## fi +##TH## +##TH### Include dependency info for the generator tool. This will cause the rule of +##TH### the same name to be considered and if it is updated, make will restart. +##TH##sinclude $(META_DIR)/$(OPENAPI_GEN).mk +##TH## +##TH### How to build the generator tool. The deps for this are defined in +##TH### the $(OPENAPI_GEN).mk, above. +##TH### +##TH### A word on the need to touch: This rule might trigger if, for example, a +##TH### non-Go file was added or deleted from a directory on which this depends. +##TH### This target needs to be reconsidered, but Go realizes it doesn't actually +##TH### have to be rebuilt. In that case, make will forever see the dependency as +##TH### newer than the binary, and try to rebuild it over and over. So we touch it, +##TH### and make is happy. +##TH##$(OPENAPI_GEN): +##TH## hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/openapi-gen +##TH## touch $@ +##TH## +##TH### +##TH### bindata generation +##TH### +##TH## +##TH### The tool used to generate bindata files. +##TH##BINDATA_GEN := $(BIN_DIR)/go-bindata +##TH## +##TH### A wrapper script that generates all bindata files. It is fast enough that we +##TH### don't care. +##TH##BINDATA_SCRIPT := hack/generate-bindata.sh +##TH## +##TH### This rule is the user-friendly entrypoint for bindata generation. +##TH##.PHONY: gen_bindata +##TH##gen_bindata: $(BINDATA_GEN) FORCE +##TH## ./hack/run-in-gopath.sh $(BINDATA_SCRIPT) +##TH## +##TH### This calculates the dependencies for the generator tool, so we only rebuild +##TH### it when needed. It is PHONY so that it always runs, but it only updates the +##TH### file if the contents have actually changed. We 'sinclude' this later. +##TH##.PHONY: $(META_DIR)/$(BINDATA_GEN).mk +##TH##$(META_DIR)/$(BINDATA_GEN).mk: +##TH## mkdir -p $(@D); \ +##TH## (echo -n "$(BINDATA_GEN): "; \ +##TH## ./hack/run-in-gopath.sh go list \ +##TH## -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ +##TH## ./vendor/github.com/jteeuwen/go-bindata/go-bindata \ +##TH## | grep --color=never "^$(PRJ_SRC_PATH)/" \ +##TH## | xargs ./hack/run-in-gopath.sh go list \ +##TH## -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ +##TH## | paste -sd' ' - \ +##TH## | sed 's/ / \\=,/g' \ +##TH## | tr '=,' '\n\t' \ +##TH## | sed "s|$$(pwd -P)/||"; \ +##TH## ) > $@.tmp; \ +##TH## if ! cmp -s $@.tmp $@; then \ +##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ +##TH## echo "DBG: $(BINDATA_GEN).mk changed"; \ +##TH## fi; \ +##TH## cat $@.tmp > $@; \ +##TH## rm -f $@.tmp; \ +##TH## fi +##TH## +##TH### Include dependency info for the generator tool. This will cause the rule of +##TH### the same name to be considered and if it is updated, make will restart. +##TH##sinclude $(META_DIR)/$(BINDATA_GEN).mk +##TH## +##TH### How to build the generator tool. The deps for this are defined in +##TH### the $(BINDATA_GEN).mk, above. +##TH### +##TH### A word on the need to touch: This rule might trigger if, for example, a +##TH### non-Go file was added or deleted from a directory on which this depends. +##TH### This target needs to be reconsidered, but Go realizes it doesn't actually +##TH### have to be rebuilt. In that case, make will forever see the dependency as +##TH### newer than the binary, and try to rebuild it over and over. So we touch it, +##TH### and make is happy. +##TH##$(BINDATA_GEN): +##TH## hack/make-rules/build.sh ./vendor/github.com/jteeuwen/go-bindata/go-bindata +##TH## touch $@ From 0419d11403d8a0e66876c0da0a6f9cc4d24a3b87 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Mon, 30 Apr 2018 00:36:27 +0100 Subject: [PATCH 08/11] Simplify build for defaulter --- build/root/Makefile.generated_files | 252 +++++++++++----------------- 1 file changed, 96 insertions(+), 156 deletions(-) diff --git a/build/root/Makefile.generated_files b/build/root/Makefile.generated_files index 4ce6c804a29..7fa7deedf28 100644 --- a/build/root/Makefile.generated_files +++ b/build/root/Makefile.generated_files @@ -35,9 +35,9 @@ SHELL := /bin/bash # This rule collects all the generated file sets into a single rule. Other # rules should depend on this to ensure generated files are rebuilt. .PHONY: generated_files -generated_files: gen_deepcopy +generated_files: gen_deepcopy gen_defaulter -##TH##FIXMEgen_defaulter gen_conversion gen_openapi gen_bindata +##TH##FIXME gen_conversion gen_openapi gen_bindata # @@ -203,160 +203,100 @@ $(DEEPCOPY_GEN): $(k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/deepcopy-g touch $@ -##TH### Defaulter generation -##TH### -##TH### Any package that wants defaulter functions generated must include a -##TH### comment-tag in column 0 of one file of the form: -##TH### // +k8s:defaulter-gen= -##TH### -##TH### The depends on context: -##TH### on types: -##TH### true: always generate a defaulter for this type -##TH### false: never generate a defaulter for this type -##TH### on functions: -##TH### covers: if the function name matches SetDefault_NAME, instructs -##TH### the generator not to recurse -##TH### on packages: -##TH### FIELDNAME: any object with a field of this name is a candidate -##TH### for having a defaulter generated -##TH## -##TH### The result file, in each pkg, of defaulter generation. -##TH##DEFAULTER_BASENAME := $(GENERATED_FILE_PREFIX)defaults -##TH##DEFAULTER_FILENAME := $(DEFAULTER_BASENAME).go -##TH## -##TH### The tool used to generate defaulters. -##TH##DEFAULTER_GEN := $(BIN_DIR)/defaulter-gen -##TH## -##TH### All directories that request any form of defaulter generation. -##TH##ifeq ($(DBG_MAKEFILE),1) -##TH## $(warning ***** finding all +k8s:defaulter-gen tags) -##TH##endif -##TH##DEFAULTER_DIRS := $(shell \ -##TH## grep --color=never -l '+k8s:defaulter-gen=' $(ALL_K8S_TAG_FILES) \ -##TH## | xargs -n1 dirname \ -##TH## | LC_ALL=C sort -u \ -##TH##) -##TH## -##TH##DEFAULTER_FILES := $(addsuffix /$(DEFAULTER_FILENAME), $(DEFAULTER_DIRS)) -##TH## -##TH### This rule aggregates the set of files to generate and then generates them all -##TH### in a single run of the tool. -##TH##.PHONY: gen_defaulter -##TH##gen_defaulter: $(DEFAULTER_FILES) $(DEFAULTER_GEN) -##TH## if [[ -s $(META_DIR)/$(DEFAULTER_GEN).todo ]]; then \ -##TH## pkgs=$$(cat $(META_DIR)/$(DEFAULTER_GEN).todo | paste -sd, -); \ -##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ -##TH## echo "DBG: running $(DEFAULTER_GEN) for $$pkgs"; \ -##TH## fi; \ -##TH## ./hack/run-in-gopath.sh $(DEFAULTER_GEN) \ -##TH## --v $(KUBE_VERBOSE) \ -##TH## --logtostderr \ -##TH## -i "$$pkgs" \ -##TH## --extra-peer-dirs $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(DEFAULTER_DIRS)) | sed 's/ /,/g') \ -##TH## -O $(DEFAULTER_BASENAME) \ -##TH## "$$@"; \ -##TH## fi -##TH## -##TH### For each dir in DEFAULTER_DIRS, this establishes a dependency between the -##TH### output file and the input files that should trigger a rebuild. -##TH### -##TH### The variable value was set in $(GOFILES_META) and included as part of the -##TH### dependency management logic. -##TH### -##TH### Note that this is a deps-only statement, not a full rule (see below). This -##TH### has to be done in a distinct step because wildcards don't work in static -##TH### pattern rules. -##TH### -##TH### The '$(eval)' is needed because this has a different RHS for each LHS, and -##TH### would otherwise produce results that make can't parse. -##TH### -##TH### We depend on the $(GOFILES_META).stamp to detect when the set of input files -##TH### has changed. This allows us to detect deleted input files. -##TH##$(foreach dir, $(DEFAULTER_DIRS), $(eval \ -##TH## $(dir)/$(DEFAULTER_FILENAME): $(META_DIR)/$(dir)/$(GOFILES_META).stamp \ -##TH## $(gofiles__$(dir)) \ -##TH##)) -##TH## -##TH### For each dir in DEFAULTER_DIRS, for each target in $(defaulters__$(dir)), -##TH### this establishes a dependency between the output file and the input files -##TH### that should trigger a rebuild. -##TH### -##TH### The variable value was set in $(GOFILES_META) and included as part of the -##TH### dependency management logic. -##TH### -##TH### Note that this is a deps-only statement, not a full rule (see below). This -##TH### has to be done in a distinct step because wildcards don't work in static -##TH### pattern rules. -##TH### -##TH### The '$(eval)' is needed because this has a different RHS for each LHS, and -##TH### would otherwise produce results that make can't parse. -##TH### -##TH### We depend on the $(GOFILES_META).stamp to detect when the set of input files -##TH### has changed. This allows us to detect deleted input files. -##TH##$(foreach dir, $(DEFAULTER_DIRS), \ -##TH## $(foreach tgt, $(defaulters__$(dir)), $(eval \ -##TH## $(dir)/$(DEFAULTER_FILENAME): $(META_DIR)/$(tgt)/$(GOFILES_META).stamp \ -##TH## $(gofiles__$(tgt)) \ -##TH## )) \ -##TH##) -##TH## -##TH### Unilaterally remove any leftovers from previous runs. -##TH##$(shell rm -f $(META_DIR)/$(DEFAULTER_GEN)*.todo) -##TH## -##TH### How to regenerate defaulter code. This is a little slow to run, so we batch -##TH### it up and trigger the batch from the 'generated_files' target. -##TH##$(DEFAULTER_FILES): $(DEFAULTER_GEN) -##TH## mkdir -p $$(dirname $(META_DIR)/$(DEFAULTER_GEN)) -##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ -##TH## echo "DBG: defaulter needed $(@D): $?"; \ -##TH## ls -lf --full-time $@ $? || true; \ -##TH## fi -##TH## echo $(PRJ_SRC_PATH)/$(@D) >> $(META_DIR)/$(DEFAULTER_GEN).todo -##TH## -##TH### This calculates the dependencies for the generator tool, so we only rebuild -##TH### it when needed. It is PHONY so that it always runs, but it only updates the -##TH### file if the contents have actually changed. We 'sinclude' this later. -##TH##.PHONY: $(META_DIR)/$(DEFAULTER_GEN).mk -##TH##$(META_DIR)/$(DEFAULTER_GEN).mk: -##TH## mkdir -p $(@D); \ -##TH## (echo -n "$(DEFAULTER_GEN): "; \ -##TH## ./hack/run-in-gopath.sh go list \ -##TH## -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ -##TH## ./vendor/k8s.io/code-generator/cmd/defaulter-gen \ -##TH## | grep --color=never "^$(PRJ_SRC_PATH)/" \ -##TH## | xargs ./hack/run-in-gopath.sh go list \ -##TH## -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ -##TH## | paste -sd' ' - \ -##TH## | sed 's/ / \\=,/g' \ -##TH## | tr '=,' '\n\t' \ -##TH## | sed "s|$$(pwd -P)/||"; \ -##TH## ) > $@.tmp; \ -##TH## if ! cmp -s $@.tmp $@; then \ -##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ -##TH## echo "DBG: $(DEFAULTER_GEN).mk changed"; \ -##TH## fi; \ -##TH## cat $@.tmp > $@; \ -##TH## rm -f $@.tmp; \ -##TH## fi -##TH## -##TH### Include dependency info for the generator tool. This will cause the rule of -##TH### the same name to be considered and if it is updated, make will restart. -##TH##sinclude $(META_DIR)/$(DEFAULTER_GEN).mk -##TH## -##TH### How to build the generator tool. The deps for this are defined in -##TH### the $(DEFAULTER_GEN).mk, above. -##TH### -##TH### A word on the need to touch: This rule might trigger if, for example, a -##TH### non-Go file was added or deleted from a directory on which this depends. -##TH### This target needs to be reconsidered, but Go realizes it doesn't actually -##TH### have to be rebuilt. In that case, make will forever see the dependency as -##TH### newer than the binary, and try to rebuild it over and over. So we touch it, -##TH### and make is happy. -##TH##$(DEFAULTER_GEN): -##TH## hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/defaulter-gen -##TH## touch $@ -##TH## -##TH### +# Defaulter generation +# +# Any package that wants defaulter functions generated must include a +# comment-tag in column 0 of one file of the form: +# // +k8s:defaulter-gen= +# +# The depends on context: +# on types: +# true: always generate a defaulter for this type +# false: never generate a defaulter for this type +# on functions: +# covers: if the function name matches SetDefault_NAME, instructs +# the generator not to recurse +# on packages: +# FIELDNAME: any object with a field of this name is a candidate +# for having a defaulter generated + +# The result file, in each pkg, of defaulter generation. +DEFAULTER_BASENAME := $(GENERATED_FILE_PREFIX)defaults +DEFAULTER_FILENAME := $(DEFAULTER_BASENAME).go + +# The tool used to generate defaulters. +DEFAULTER_GEN := $(BIN_DIR)/defaulter-gen + +# All directories that request any form of defaulter generation. +ifeq ($(DBG_MAKEFILE),1) + $(warning ***** finding all +k8s:defaulter-gen tags) +endif +DEFAULTER_DIRS := $(shell \ + grep --color=never -l '+k8s:defaulter-gen=' $(ALL_K8S_TAG_FILES) \ + | xargs -n1 dirname \ + | LC_ALL=C sort -u \ +) + +DEFAULTER_FILES := $(addsuffix /$(DEFAULTER_FILENAME), $(DEFAULTER_DIRS)) + +# Reset the list of packages that need generation. +$(shell mkdir -p $$(dirname $(META_DIR)/$(DEFAULTER_GEN))) +$(shell rm -f $(META_DIR)/$(DEFAULTER_GEN).todo) + +# This rule aggregates the set of files to generate and then generates them all +# in a single run of the tool. +.PHONY: gen_defaulter +gen_defaulter: $(DEFAULTER_GEN) $(META_DIR)/$(DEFAULTER_GEN).todo + if [[ -s $(META_DIR)/$(DEFAULTER_GEN).todo ]]; then \ + pkgs=$$(cat $(META_DIR)/$(DEFAULTER_GEN).todo | paste -sd, -); \ + if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ + echo "DBG: running $(DEFAULTER_GEN) for $$pkgs"; \ + fi; \ + ./hack/run-in-gopath.sh $(DEFAULTER_GEN) \ + --v $(KUBE_VERBOSE) \ + --logtostderr \ + -i "$$pkgs" \ + --extra-peer-dirs $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(DEFAULTER_DIRS)) | sed 's/ /,/g') \ + -O $(DEFAULTER_BASENAME) \ + "$$@"; \ + fi + +# For each dir in DEFAULTER_DIRS, this establishes a dependency between the +# output file and the input files that should trigger a rebuild. +# +# Note that this is a deps-only statement, not a full rule (see below for that). +# +# The '$(eval)' is needed because this has a different RHS for each LHS, and +# would otherwise produce results that make can't parse. +$(foreach dir, $(DEFAULTER_DIRS), $(eval \ + $(dir)/$(DEFAULTER_FILENAME): $($(PRJ_SRC_PATH)/$(dir)) \ +)) + +# How to regenerate defaulter code. This is a little slow to run, so we batch +# it up and trigger the batch from the 'generated_files' target. +$(META_DIR)/$(DEFAULTER_GEN).todo: $(DEFAULTER_FILES) + +$(DEFAULTER_FILES): $(DEFAULTER_GEN) + if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ + echo "DBG: defaulter needed $(@D): $?"; \ + ls -lf --full-time $@ $? || true; \ + fi + echo $(PRJ_SRC_PATH)/$(@D) >> $(META_DIR)/$(DEFAULTER_GEN).todo + +# How to build the generator tool. The deps for this are defined in +# the $(GO_PKGDEPS_FILE), above. +# +# A word on the need to touch: This rule might trigger if, for example, a +# non-Go file was added or deleted from a directory on which this depends. +# This target needs to be reconsidered, but Go realizes it doesn't actually +# have to be rebuilt. In that case, make will forever see the dependency as +# newer than the binary, and try to "rebuild" it over and over. So we touch +# it, and make is happy. +$(DEFAULTER_GEN): $(k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/defaulter-gen) + hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/defaulter-gen + touch $@ + + ##TH### Conversion generation ##TH### ##TH### Any package that wants conversion functions generated must include one or From bc1a6ee0713e9914d957c131d32aed5d5768f4f5 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Mon, 30 Apr 2018 02:02:57 +0100 Subject: [PATCH 09/11] Simplify build for conversion --- build/root/Makefile.generated_files | 306 +++++++++------------------- 1 file changed, 99 insertions(+), 207 deletions(-) diff --git a/build/root/Makefile.generated_files b/build/root/Makefile.generated_files index 7fa7deedf28..79e94037e9a 100644 --- a/build/root/Makefile.generated_files +++ b/build/root/Makefile.generated_files @@ -35,9 +35,9 @@ SHELL := /bin/bash # This rule collects all the generated file sets into a single rule. Other # rules should depend on this to ensure generated files are rebuilt. .PHONY: generated_files -generated_files: gen_deepcopy gen_defaulter +generated_files: gen_deepcopy gen_defaulter gen_conversion -##TH##FIXME gen_conversion gen_openapi gen_bindata +##TH##FIXME gen_openapi gen_bindata # @@ -297,211 +297,103 @@ $(DEFAULTER_GEN): $(k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/defaulter touch $@ -##TH### Conversion generation -##TH### -##TH### Any package that wants conversion functions generated must include one or -##TH### more comment-tags in any .go file, in column 0, of the form: -##TH### // +k8s:conversion-gen= -##TH### -##TH### The CONVERSION_TARGET_DIR is a project-local path to another directory which -##TH### should be considered when evaluating peer types for conversions. Types which -##TH### are found in the source package (where conversions are being generated) -##TH### but do not have a peer in one of the target directories will not have -##TH### conversions generated. -##TH### -##TH### TODO: it might be better in the long term to make peer-types explicit in the -##TH### IDL. -##TH## -##TH### The result file, in each pkg, of conversion generation. -##TH##CONVERSION_BASENAME := $(GENERATED_FILE_PREFIX)conversion -##TH##CONVERSION_FILENAME := $(CONVERSION_BASENAME).go -##TH## -##TH### The tool used to generate conversions. -##TH##CONVERSION_GEN := $(BIN_DIR)/conversion-gen -##TH## -##TH### The name of the metadata file listing conversion peers for each pkg. -##TH##CONVERSIONS_META := conversions.mk -##TH## -##TH### All directories that request any form of conversion generation. -##TH##ifeq ($(DBG_MAKEFILE),1) -##TH## $(warning ***** finding all +k8s:conversion-gen tags) -##TH##endif -##TH##CONVERSION_DIRS := $(shell \ -##TH## grep --color=never '^// *+k8s:conversion-gen=' $(ALL_K8S_TAG_FILES) \ -##TH## | cut -f1 -d: \ -##TH## | xargs -n1 dirname \ -##TH## | LC_ALL=C sort -u \ -##TH##) -##TH## -##TH##CONVERSION_FILES := $(addsuffix /$(CONVERSION_FILENAME), $(CONVERSION_DIRS)) -##TH##CONVERSION_EXTRA_PEER_DIRS := k8s.io/kubernetes/pkg/apis/core,k8s.io/kubernetes/pkg/apis/core/v1,k8s.io/api/core/v1 -##TH## -##TH### This rule aggregates the set of files to generate and then generates them all -##TH### in a single run of the tool. -##TH##.PHONY: gen_conversion -##TH##gen_conversion: $(CONVERSION_FILES) $(CONVERSION_GEN) -##TH## if [[ -s $(META_DIR)/$(CONVERSION_GEN).todo ]]; then \ -##TH## pkgs=$$(cat $(META_DIR)/$(CONVERSION_GEN).todo | paste -sd, -); \ -##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ -##TH## echo "DBG: running $(CONVERSION_GEN) for $$pkgs"; \ -##TH## fi; \ -##TH## ./hack/run-in-gopath.sh $(CONVERSION_GEN) \ -##TH## --extra-peer-dirs $(CONVERSION_EXTRA_PEER_DIRS) \ -##TH## --v $(KUBE_VERBOSE) \ -##TH## --logtostderr \ -##TH## -i "$$pkgs" \ -##TH## -O $(CONVERSION_BASENAME) \ -##TH## "$$@"; \ -##TH## fi -##TH## -##TH### Establish a dependency between the deps file and the dir. Whenever a dir -##TH### changes (files added or removed) the deps file will be considered stale. -##TH### -##TH### This is looser than we really need (e.g. we don't really care about non *.go -##TH### files or even *_test.go files), but this is much easier to represent. -##TH### -##TH### Because we 'sinclude' the deps file, it is considered for rebuilding, as part -##TH### of make's normal evaluation. If it gets rebuilt, make will restart. -##TH### -##TH### The '$(eval)' is needed because this has a different RHS for each LHS, and -##TH### would otherwise produce results that make can't parse. -##TH##$(foreach dir, $(CONVERSION_DIRS), $(eval \ -##TH## $(META_DIR)/$(dir)/$(CONVERSIONS_META): $(dir) \ -##TH##)) -##TH## -##TH### How to rebuild a deps file. When make determines that the deps file is stale -##TH### (see above), it executes this rule, and then re-loads the deps file. -##TH### -##TH### This is looser than we really need (e.g. we don't really care about test -##TH### files), but this is MUCH faster than calling `go list`. -##TH### -##TH### We regenerate the output file in order to satisfy make's "newer than" rules, -##TH### but we only need to rebuild targets if the contents actually changed. That -##TH### is what the .stamp file represents. -##TH##$(foreach dir, $(CONVERSION_DIRS), \ -##TH## $(META_DIR)/$(dir)/$(CONVERSIONS_META)): -##TH## TAGS=$$(grep --color=never -h '^// *+k8s:conversion-gen=' $$@.tmp; \ -##TH## if ! cmp -s $@.tmp $@; then \ -##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ -##TH## echo "DBG: conversions changed for $@"; \ -##TH## fi; \ -##TH## touch $@.stamp; \ -##TH## fi; \ -##TH## mv $@.tmp $@ -##TH## -##TH### Include any deps files as additional Makefile rules. This triggers make to -##TH### consider the deps files for rebuild, which makes the whole -##TH### dependency-management logic work. 'sinclude' is "silent include" which does -##TH### not fail if the file does not exist. -##TH##$(foreach dir, $(CONVERSION_DIRS), $(eval \ -##TH## sinclude $(META_DIR)/$(dir)/$(CONVERSIONS_META) \ -##TH##)) -##TH## -##TH### For each dir in CONVERSION_DIRS, this establishes a dependency between the -##TH### output file and the input files that should trigger a rebuild. -##TH### -##TH### The variable value was set in $(GOFILES_META) and included as part of the -##TH### dependency management logic. -##TH### -##TH### Note that this is a deps-only statement, not a full rule (see below). This -##TH### has to be done in a distinct step because wildcards don't work in static -##TH### pattern rules. -##TH### -##TH### The '$(eval)' is needed because this has a different RHS for each LHS, and -##TH### would otherwise produce results that make can't parse. -##TH### -##TH### We depend on the $(GOFILES_META).stamp to detect when the set of input files -##TH### has changed. This allows us to detect deleted input files. -##TH##$(foreach dir, $(CONVERSION_DIRS), $(eval \ -##TH## $(dir)/$(CONVERSION_FILENAME): $(META_DIR)/$(dir)/$(GOFILES_META).stamp \ -##TH## $(gofiles__$(dir)) \ -##TH##)) -##TH## -##TH### For each dir in CONVERSION_DIRS, for each target in $(conversions__$(dir)), -##TH### this establishes a dependency between the output file and the input files -##TH### that should trigger a rebuild. -##TH### -##TH### The variable value was set in $(GOFILES_META) and included as part of the -##TH### dependency management logic. -##TH### -##TH### Note that this is a deps-only statement, not a full rule (see below). This -##TH### has to be done in a distinct step because wildcards don't work in static -##TH### pattern rules. -##TH### -##TH### The '$(eval)' is needed because this has a different RHS for each LHS, and -##TH### would otherwise produce results that make can't parse. -##TH### -##TH### We depend on the $(GOFILES_META).stamp to detect when the set of input files -##TH### has changed. This allows us to detect deleted input files. -##TH##$(foreach dir, $(CONVERSION_DIRS), \ -##TH## $(foreach tgt, $(conversions__$(dir)), $(eval \ -##TH## $(dir)/$(CONVERSION_FILENAME): $(META_DIR)/$(tgt)/$(GOFILES_META).stamp \ -##TH## $(gofiles__$(tgt)) \ -##TH## )) \ -##TH##) -##TH## -##TH### Unilaterally remove any leftovers from previous runs. -##TH##$(shell rm -f $(META_DIR)/$(CONVERSION_GEN)*.todo) -##TH## -##TH### How to regenerate conversion code. This is a little slow to run, so we batch -##TH### it up and trigger the batch from the 'generated_files' target. -##TH##$(CONVERSION_FILES): $(CONVERSION_GEN) -##TH## mkdir -p $$(dirname $(META_DIR)/$(CONVERSION_GEN)) -##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ -##TH## echo "DBG: conversion needed $(@D): $?"; \ -##TH## ls -lf --full-time $@ $? || true; \ -##TH## fi -##TH## echo $(PRJ_SRC_PATH)/$(@D) >> $(META_DIR)/$(CONVERSION_GEN).todo -##TH## -##TH### This calculates the dependencies for the generator tool, so we only rebuild -##TH### it when needed. It is PHONY so that it always runs, but it only updates the -##TH### file if the contents have actually changed. We 'sinclude' this later. -##TH##.PHONY: $(META_DIR)/$(CONVERSION_GEN).mk -##TH##$(META_DIR)/$(CONVERSION_GEN).mk: -##TH## mkdir -p $(@D); \ -##TH## (echo -n "$(CONVERSION_GEN): "; \ -##TH## ./hack/run-in-gopath.sh go list \ -##TH## -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ -##TH## ./vendor/k8s.io/code-generator/cmd/conversion-gen \ -##TH## | grep --color=never "^$(PRJ_SRC_PATH)/" \ -##TH## | xargs ./hack/run-in-gopath.sh go list \ -##TH## -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ -##TH## | paste -sd' ' - \ -##TH## | sed 's/ / \\=,/g' \ -##TH## | tr '=,' '\n\t' \ -##TH## | sed "s|$$(pwd -P)/||"; \ -##TH## ) > $@.tmp; \ -##TH## if ! cmp -s $@.tmp $@; then \ -##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ -##TH## echo "DBG: $(CONVERSION_GEN).mk changed"; \ -##TH## fi; \ -##TH## cat $@.tmp > $@; \ -##TH## rm -f $@.tmp; \ -##TH## fi -##TH## -##TH### Include dependency info for the generator tool. This will cause the rule of -##TH### the same name to be considered and if it is updated, make will restart. -##TH##sinclude $(META_DIR)/$(CONVERSION_GEN).mk -##TH## -##TH### How to build the generator tool. The deps for this are defined in -##TH### the $(CONVERSION_GEN).mk, above. -##TH### -##TH### A word on the need to touch: This rule might trigger if, for example, a -##TH### non-Go file was added or deleted from a directory on which this depends. -##TH### This target needs to be reconsidered, but Go realizes it doesn't actually -##TH### have to be rebuilt. In that case, make will forever see the dependency as -##TH### newer than the binary, and try to rebuild it over and over. So we touch it, -##TH### and make is happy. -##TH##$(CONVERSION_GEN): -##TH## hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/conversion-gen -##TH## touch $@ -##TH## -##TH### +# Conversion generation +# +# Any package that wants conversion functions generated must include one or +# more comment-tags in any .go file, in column 0, of the form: +# // +k8s:conversion-gen= +# +# The CONVERSION_TARGET_DIR is a project-local path to another directory which +# should be considered when evaluating peer types for conversions. Types which +# are found in the source package (where conversions are being generated) +# but do not have a peer in one of the target directories will not have +# conversions generated. +# +# TODO: it might be better in the long term to make peer-types explicit in the +# IDL. + +# The result file, in each pkg, of conversion generation. +CONVERSION_BASENAME := $(GENERATED_FILE_PREFIX)conversion +CONVERSION_FILENAME := $(CONVERSION_BASENAME).go + +# The tool used to generate conversions. +CONVERSION_GEN := $(BIN_DIR)/conversion-gen + +# The name of the metadata file listing conversion peers for each pkg. +CONVERSIONS_META := conversions.mk + +# All directories that request any form of conversion generation. +ifeq ($(DBG_MAKEFILE),1) + $(warning ***** finding all +k8s:conversion-gen tags) +endif +CONVERSION_DIRS := $(shell \ + grep --color=never '^// *+k8s:conversion-gen=' $(ALL_K8S_TAG_FILES) \ + | cut -f1 -d: \ + | xargs -n1 dirname \ + | LC_ALL=C sort -u \ +) + +CONVERSION_FILES := $(addsuffix /$(CONVERSION_FILENAME), $(CONVERSION_DIRS)) +CONVERSION_EXTRA_PEER_DIRS := k8s.io/kubernetes/pkg/apis/core,k8s.io/kubernetes/pkg/apis/core/v1,k8s.io/api/core/v1 + +# Reset the list of packages that need generation. +$(shell mkdir -p $$(dirname $(META_DIR)/$(CONVERSION_GEN))) +$(shell rm -f $(META_DIR)/$(CONVERSION_GEN).todo) + +# This rule aggregates the set of files to generate and then generates them all +# in a single run of the tool. +.PHONY: gen_conversion +gen_conversion: $(CONVERSION_GEN) $(META_DIR)/$(CONVERSION_GEN).todo + if [[ -s $(META_DIR)/$(CONVERSION_GEN).todo ]]; then \ + pkgs=$$(cat $(META_DIR)/$(CONVERSION_GEN).todo | paste -sd, -); \ + if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ + echo "DBG: running $(CONVERSION_GEN) for $$pkgs"; \ + fi; \ + ./hack/run-in-gopath.sh $(CONVERSION_GEN) \ + --extra-peer-dirs $(CONVERSION_EXTRA_PEER_DIRS) \ + --v $(KUBE_VERBOSE) \ + --logtostderr \ + -i "$$pkgs" \ + -O $(CONVERSION_BASENAME) \ + "$$@"; \ + fi + +# For each dir in CONVERSION_DIRS, this establishes a dependency between the +# output file and the input files that should trigger a rebuild. +# +# Note that this is a deps-only statement, not a full rule (see below for that). +# +# The '$(eval)' is needed because this has a different RHS for each LHS, and +# would otherwise produce results that make can't parse. +$(foreach dir, $(CONVERSION_DIRS), $(eval \ + $(dir)/$(CONVERSION_FILENAME): $($(PRJ_SRC_PATH)/$(dir)) \ +)) + +# How to regenerate conversion code. This is a little slow to run, so we batch +# it up and trigger the batch from the 'generated_files' target. +$(META_DIR)/$(CONVERSION_GEN).todo: $(CONVERSION_FILES) + +$(CONVERSION_FILES): $(CONVERSION_GEN) + if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ + echo "DBG: conversion needed $(@D): $?"; \ + ls -lf --full-time $@ $? || true; \ + fi + echo $(PRJ_SRC_PATH)/$(@D) >> $(META_DIR)/$(CONVERSION_GEN).todo + +# How to build the generator tool. The deps for this are defined in +# the $(GO_PKGDEPS_FILE), above. +# +# A word on the need to touch: This rule might trigger if, for example, a +# non-Go file was added or deleted from a directory on which this depends. +# This target needs to be reconsidered, but Go realizes it doesn't actually +# have to be rebuilt. In that case, make will forever see the dependency as +# newer than the binary, and try to rebuild it over and over. So we touch it, +# and make is happy. +$(CONVERSION_GEN): $(k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/conversion-gen) + hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/conversion-gen + touch $@ + + ##TH### Open-api generation ##TH### ##TH### Any package that wants open-api functions generated must include a From 6387c43e8713d46237fce2ba822ad7d8123eff27 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Mon, 30 Apr 2018 08:06:04 +0100 Subject: [PATCH 10/11] Simplify build for openapi --- build/root/Makefile.generated_files | 173 +++++++++++----------------- 1 file changed, 67 insertions(+), 106 deletions(-) diff --git a/build/root/Makefile.generated_files b/build/root/Makefile.generated_files index 79e94037e9a..1718105f4fa 100644 --- a/build/root/Makefile.generated_files +++ b/build/root/Makefile.generated_files @@ -35,9 +35,9 @@ SHELL := /bin/bash # This rule collects all the generated file sets into a single rule. Other # rules should depend on this to ensure generated files are rebuilt. .PHONY: generated_files -generated_files: gen_deepcopy gen_defaulter gen_conversion +generated_files: gen_deepcopy gen_defaulter gen_conversion gen_openapi -##TH##FIXME gen_openapi gen_bindata +##TH##FIXME gen_bindata # @@ -394,110 +394,71 @@ $(CONVERSION_GEN): $(k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/conversi touch $@ -##TH### Open-api generation -##TH### -##TH### Any package that wants open-api functions generated must include a -##TH### comment-tag in column 0 of one file of the form: -##TH### // +k8s:openapi-gen=true -##TH### -##TH### The result file, in each pkg, of open-api generation. -##TH##OPENAPI_BASENAME := $(GENERATED_FILE_PREFIX)openapi -##TH##OPENAPI_FILENAME := $(OPENAPI_BASENAME).go -##TH##OPENAPI_OUTPUT_PKG := pkg/generated/openapi -##TH## -##TH### The tool used to generate open apis. -##TH##OPENAPI_GEN := $(BIN_DIR)/openapi-gen -##TH## -##TH### Find all the directories that request open-api generation. -##TH##ifeq ($(DBG_MAKEFILE),1) -##TH## $(warning ***** finding all +k8s:openapi-gen tags) -##TH##endif -##TH##OPENAPI_DIRS := $(shell \ -##TH## grep --color=never -l '+k8s:openapi-gen=' $(ALL_K8S_TAG_FILES) \ -##TH## | xargs -n1 dirname \ -##TH## | LC_ALL=C sort -u \ -##TH##) -##TH## -##TH##OPENAPI_OUTFILE := $(OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME) -##TH## -##TH### This rule is the user-friendly entrypoint for openapi generation. -##TH##.PHONY: gen_openapi -##TH##gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN) -##TH## -##TH### For each dir in OPENAPI_DIRS, this establishes a dependency between the -##TH### output file and the input files that should trigger a rebuild. -##TH### -##TH### Note that this is a deps-only statement, not a full rule (see below). This -##TH### has to be done in a distinct step because wildcards don't work in static -##TH### pattern rules. -##TH### -##TH### The '$(eval)' is needed because this has a different RHS for each LHS, and -##TH### would otherwise produce results that make can't parse. -##TH### -##TH### We depend on the $(GOFILES_META).stamp to detect when the set of input files -##TH### has changed. This allows us to detect deleted input files. -##TH##$(foreach dir, $(OPENAPI_DIRS), $(eval \ -##TH## $(OPENAPI_OUTFILE): $(META_DIR)/$(dir)/$(GOFILES_META).stamp \ -##TH## $(gofiles__$(dir)) \ -##TH##)) -##TH## -##TH### How to regenerate open-api code. This emits a single file for all results. -##TH##$(OPENAPI_OUTFILE): $(OPENAPI_GEN) $(OPENAPI_GEN) -##TH## function run_gen_openapi() { \ -##TH## ./hack/run-in-gopath.sh $(OPENAPI_GEN) \ -##TH## --v $(KUBE_VERBOSE) \ -##TH## --logtostderr \ -##TH## -i $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(OPENAPI_DIRS)) | sed 's/ /,/g') \ -##TH## -p $(PRJ_SRC_PATH)/$(OPENAPI_OUTPUT_PKG) \ -##TH## -O $(OPENAPI_BASENAME) \ -##TH## "$$@"; \ -##TH## }; \ -##TH## run_gen_openapi -##TH## -##TH### This calculates the dependencies for the generator tool, so we only rebuild -##TH### it when needed. It is PHONY so that it always runs, but it only updates the -##TH### file if the contents have actually changed. We 'sinclude' this later. -##TH##.PHONY: $(META_DIR)/$(OPENAPI_GEN).mk -##TH##$(META_DIR)/$(OPENAPI_GEN).mk: -##TH## mkdir -p $(@D); \ -##TH## (echo -n "$(OPENAPI_GEN): "; \ -##TH## ./hack/run-in-gopath.sh go list \ -##TH## -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ -##TH## ./vendor/k8s.io/code-generator/cmd/openapi-gen \ -##TH## | grep --color=never "^$(PRJ_SRC_PATH)/" \ -##TH## | xargs ./hack/run-in-gopath.sh go list \ -##TH## -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ -##TH## | paste -sd' ' - \ -##TH## | sed 's/ / \\=,/g' \ -##TH## | tr '=,' '\n\t' \ -##TH## | sed "s|$$(pwd -P)/||"; \ -##TH## ) > $@.tmp; \ -##TH## if ! cmp -s $@.tmp $@; then \ -##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ -##TH## echo "DBG: $(OPENAPI_GEN).mk changed"; \ -##TH## fi; \ -##TH## cat $@.tmp > $@; \ -##TH## rm -f $@.tmp; \ -##TH## fi -##TH## -##TH### Include dependency info for the generator tool. This will cause the rule of -##TH### the same name to be considered and if it is updated, make will restart. -##TH##sinclude $(META_DIR)/$(OPENAPI_GEN).mk -##TH## -##TH### How to build the generator tool. The deps for this are defined in -##TH### the $(OPENAPI_GEN).mk, above. -##TH### -##TH### A word on the need to touch: This rule might trigger if, for example, a -##TH### non-Go file was added or deleted from a directory on which this depends. -##TH### This target needs to be reconsidered, but Go realizes it doesn't actually -##TH### have to be rebuilt. In that case, make will forever see the dependency as -##TH### newer than the binary, and try to rebuild it over and over. So we touch it, -##TH### and make is happy. -##TH##$(OPENAPI_GEN): -##TH## hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/openapi-gen -##TH## touch $@ -##TH## -##TH### +# OpenAPI generation +# +# Any package that wants open-api functions generated must include a +# comment-tag in column 0 of one file of the form: +# // +k8s:openapi-gen=true +# +# The result file, in each pkg, of open-api generation. +OPENAPI_BASENAME := $(GENERATED_FILE_PREFIX)openapi +OPENAPI_FILENAME := $(OPENAPI_BASENAME).go +OPENAPI_OUTPUT_PKG := pkg/generated/openapi + +# The tool used to generate open apis. +OPENAPI_GEN := $(BIN_DIR)/openapi-gen + +# Find all the directories that request open-api generation. +ifeq ($(DBG_MAKEFILE),1) + $(warning ***** finding all +k8s:openapi-gen tags) +endif +OPENAPI_DIRS := $(shell \ + grep --color=never -l '+k8s:openapi-gen=' $(ALL_K8S_TAG_FILES) \ + | xargs -n1 dirname \ + | LC_ALL=C sort -u \ +) + +OPENAPI_OUTFILE := $(OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME) + +# This rule is the user-friendly entrypoint for openapi generation. +.PHONY: gen_openapi +gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN) + +# For each dir in OPENAPI_DIRS, this establishes a dependency between the +# output file and the input files that should trigger a rebuild. +# +# Note that this is a deps-only statement, not a full rule (see below for that). +# +# The '$(eval)' is needed because this has a different RHS for each LHS, and +# would otherwise produce results that make can't parse. +$(foreach dir, $(OPENAPI_DIRS), $(eval \ + $(OPENAPI_OUTFILE): $($(PRJ_SRC_PATH)/$(dir)) \ +)) + +# How to regenerate open-api code. This emits a single file for all results. +$(OPENAPI_OUTFILE): $(OPENAPI_GEN) + ./hack/run-in-gopath.sh $(OPENAPI_GEN) \ + --v $(KUBE_VERBOSE) \ + --logtostderr \ + -i $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(OPENAPI_DIRS)) | sed 's/ /,/g') \ + -p $(PRJ_SRC_PATH)/$(OPENAPI_OUTPUT_PKG) \ + -O $(OPENAPI_BASENAME) \ + "$$@" + +# How to build the generator tool. The deps for this are defined in +# the $(GO_PKGDEPS_FILE), above. +# +# A word on the need to touch: This rule might trigger if, for example, a +# non-Go file was added or deleted from a directory on which this depends. +# This target needs to be reconsidered, but Go realizes it doesn't actually +# have to be rebuilt. In that case, make will forever see the dependency as +# newer than the binary, and try to "rebuild" it over and over. So we touch +# it, and make is happy. +$(OPENAPI_GEN): $(k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/openapi-gen) + hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/openapi-gen + touch $@ + + ##TH### bindata generation ##TH### ##TH## From 0b8b7100596f6467a03eece5533c0bc5f5ab2090 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Mon, 30 Apr 2018 10:25:47 +0200 Subject: [PATCH 11/11] Simplify build for bindata --- build/root/Makefile.generated_files | 110 ++++++++++------------------ 1 file changed, 39 insertions(+), 71 deletions(-) diff --git a/build/root/Makefile.generated_files b/build/root/Makefile.generated_files index 1718105f4fa..b4d94fba725 100644 --- a/build/root/Makefile.generated_files +++ b/build/root/Makefile.generated_files @@ -35,10 +35,7 @@ SHELL := /bin/bash # This rule collects all the generated file sets into a single rule. Other # rules should depend on this to ensure generated files are rebuilt. .PHONY: generated_files -generated_files: gen_deepcopy gen_defaulter gen_conversion gen_openapi - -##TH##FIXME gen_bindata - +generated_files: gen_deepcopy gen_defaulter gen_conversion gen_openapi gen_bindata # # Helper logic to calculate Go's dependency DAG ourselves. @@ -67,17 +64,18 @@ $(META_DIR)/$(GO_PKGDEPS_FILE): FORCE echo "DBG: calculating Go dependencies"; \ fi hack/run-in-gopath.sh go install ./hack/make-rules/helpers/go2make - hack/run-in-gopath.sh go2make \ - k8s.io/kubernetes/... \ - --prune k8s.io/kubernetes/staging \ - --prune k8s.io/kubernetes/vendor \ - k8s.io/kubernetes/vendor/k8s.io/... \ + hack/run-in-gopath.sh go2make \ + k8s.io/kubernetes/... \ + --prune k8s.io/kubernetes/staging \ + --prune k8s.io/kubernetes/vendor \ + k8s.io/kubernetes/vendor/k8s.io/... \ + github.com/jteeuwen/go-bindata/go-bindata/... \ > $@.tmp - if ! cmp -s $@.tmp $@; then \ - if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ - echo "DBG: $(GO_PKGDEPS_FILE) changed"; \ - fi; \ - cat $@.tmp > $@; \ + if ! cmp -s $@.tmp $@; then \ + if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ + echo "DBG: $(GO_PKGDEPS_FILE) changed"; \ + fi; \ + cat $@.tmp > $@; \ fi rm -f $@.tmp @@ -459,60 +457,30 @@ $(OPENAPI_GEN): $(k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/openapi-gen touch $@ -##TH### bindata generation -##TH### -##TH## -##TH### The tool used to generate bindata files. -##TH##BINDATA_GEN := $(BIN_DIR)/go-bindata -##TH## -##TH### A wrapper script that generates all bindata files. It is fast enough that we -##TH### don't care. -##TH##BINDATA_SCRIPT := hack/generate-bindata.sh -##TH## -##TH### This rule is the user-friendly entrypoint for bindata generation. -##TH##.PHONY: gen_bindata -##TH##gen_bindata: $(BINDATA_GEN) FORCE -##TH## ./hack/run-in-gopath.sh $(BINDATA_SCRIPT) -##TH## -##TH### This calculates the dependencies for the generator tool, so we only rebuild -##TH### it when needed. It is PHONY so that it always runs, but it only updates the -##TH### file if the contents have actually changed. We 'sinclude' this later. -##TH##.PHONY: $(META_DIR)/$(BINDATA_GEN).mk -##TH##$(META_DIR)/$(BINDATA_GEN).mk: -##TH## mkdir -p $(@D); \ -##TH## (echo -n "$(BINDATA_GEN): "; \ -##TH## ./hack/run-in-gopath.sh go list \ -##TH## -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ -##TH## ./vendor/github.com/jteeuwen/go-bindata/go-bindata \ -##TH## | grep --color=never "^$(PRJ_SRC_PATH)/" \ -##TH## | xargs ./hack/run-in-gopath.sh go list \ -##TH## -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ -##TH## | paste -sd' ' - \ -##TH## | sed 's/ / \\=,/g' \ -##TH## | tr '=,' '\n\t' \ -##TH## | sed "s|$$(pwd -P)/||"; \ -##TH## ) > $@.tmp; \ -##TH## if ! cmp -s $@.tmp $@; then \ -##TH## if [[ "$(DBG_CODEGEN)" == 1 ]]; then \ -##TH## echo "DBG: $(BINDATA_GEN).mk changed"; \ -##TH## fi; \ -##TH## cat $@.tmp > $@; \ -##TH## rm -f $@.tmp; \ -##TH## fi -##TH## -##TH### Include dependency info for the generator tool. This will cause the rule of -##TH### the same name to be considered and if it is updated, make will restart. -##TH##sinclude $(META_DIR)/$(BINDATA_GEN).mk -##TH## -##TH### How to build the generator tool. The deps for this are defined in -##TH### the $(BINDATA_GEN).mk, above. -##TH### -##TH### A word on the need to touch: This rule might trigger if, for example, a -##TH### non-Go file was added or deleted from a directory on which this depends. -##TH### This target needs to be reconsidered, but Go realizes it doesn't actually -##TH### have to be rebuilt. In that case, make will forever see the dependency as -##TH### newer than the binary, and try to rebuild it over and over. So we touch it, -##TH### and make is happy. -##TH##$(BINDATA_GEN): -##TH## hack/make-rules/build.sh ./vendor/github.com/jteeuwen/go-bindata/go-bindata -##TH## touch $@ +# bindata generation +# + +# The tool used to generate bindata files. +BINDATA_GEN := $(BIN_DIR)/go-bindata + +# A wrapper script that generates all bindata files. It is fast enough that we +# don't care. +BINDATA_SCRIPT := hack/generate-bindata.sh + +# This rule is the user-friendly entrypoint for bindata generation. +.PHONY: gen_bindata +gen_bindata: $(BINDATA_GEN) FORCE + ./hack/run-in-gopath.sh $(BINDATA_SCRIPT) + +# How to build the generator tool. The deps for this are defined in +# the $(BINDATA_GEN).mk, above. +# +# A word on the need to touch: This rule might trigger if, for example, a +# non-Go file was added or deleted from a directory on which this depends. +# This target needs to be reconsidered, but Go realizes it doesn't actually +# have to be rebuilt. In that case, make will forever see the dependency as +# newer than the binary, and try to rebuild it over and over. So we touch it, +# and make is happy. +$(BINDATA_GEN): $(k8s.io/kubernetes/vendor/github.com/jteeuwen/go-bindata/go-bindata) + hack/make-rules/build.sh ./vendor/github.com/jteeuwen/go-bindata/go-bindata + touch $@