From f174307e49442dacf2dfd22ffadca81a53e7377d Mon Sep 17 00:00:00 2001 From: Christian Glombek Date: Thu, 6 Aug 2020 17:10:12 +0200 Subject: [PATCH 1/2] k8s.io/code-generator: Enable generate-* scripts usage as module deps So far, the generate scripts could not be required as dependencies with Go Modules (e.g. in tools.go) as they were installed without proper module context. This commit enables the generator scripts to be used as versioned dependencies with Go Modules by forcing installation into the fully-qualified module directory. --- staging/src/k8s.io/code-generator/generate-groups.sh | 6 +++--- .../src/k8s.io/code-generator/generate-internal-groups.sh | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/staging/src/k8s.io/code-generator/generate-groups.sh b/staging/src/k8s.io/code-generator/generate-groups.sh index 1c2fd7a55bd..53faad09cbd 100755 --- a/staging/src/k8s.io/code-generator/generate-groups.sh +++ b/staging/src/k8s.io/code-generator/generate-groups.sh @@ -47,10 +47,10 @@ GROUPS_WITH_VERSIONS="$4" shift 4 ( - # To support running this script from anywhere, we have to first cd into this directory - # so we can install the tools. + # To support running this script from anywhere, first cd into this directory, + # and then install with forced module mode on and fully qualified name. cd "$(dirname "${0}")" - go install ./cmd/{defaulter-gen,client-gen,lister-gen,informer-gen,deepcopy-gen} + GO111MODULE=on go install k8s.io/code-generator/cmd/{defaulter-gen,client-gen,lister-gen,informer-gen,deepcopy-gen} ) # Go installs the above commands to get installed in $GOBIN if defined, and $GOPATH/bin otherwise: GOBIN="$(go env GOBIN)" diff --git a/staging/src/k8s.io/code-generator/generate-internal-groups.sh b/staging/src/k8s.io/code-generator/generate-internal-groups.sh index e4430b27d5b..4413bd471e9 100755 --- a/staging/src/k8s.io/code-generator/generate-internal-groups.sh +++ b/staging/src/k8s.io/code-generator/generate-internal-groups.sh @@ -47,7 +47,12 @@ EXT_APIS_PKG="$4" GROUPS_WITH_VERSIONS="$5" shift 5 -go install ./"$(dirname "${0}")"/cmd/{defaulter-gen,conversion-gen,client-gen,lister-gen,informer-gen,deepcopy-gen,openapi-gen} +( + # To support running this script from anywhere, first cd into this directory, + # and then install with forced module mode on and fully qualified name. + cd "$(dirname "${0}")" + GO111MODULE=on go install k8s.io/code-generator/cmd/{defaulter-gen,conversion-gen,client-gen,lister-gen,informer-gen,deepcopy-gen,openapi-gen} +) function codegen::join() { local IFS="$1"; shift; echo "$*"; } From 326471d88061ef7bc96b1e0d779893bc60ec6a69 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Wed, 21 Jul 2021 12:33:14 -0400 Subject: [PATCH 2/2] k8s.io/code-generator: Change BoilerplatePath() to locate file or require explicit flag --- .../cmd/applyconfiguration-gen/main.go | 4 +-- .../code-generator/cmd/client-gen/main.go | 4 +-- .../code-generator/cmd/conversion-gen/main.go | 4 +-- .../code-generator/cmd/deepcopy-gen/main.go | 4 +-- .../code-generator/cmd/defaulter-gen/main.go | 4 +-- .../cmd/go-to-protobuf/protobuf/cmd.go | 2 +- .../code-generator/cmd/import-boss/main.go | 3 +- .../code-generator/cmd/informer-gen/main.go | 4 +-- .../code-generator/cmd/lister-gen/main.go | 4 +-- .../cmd/prerelease-lifecycle-gen/main.go | 4 +-- .../code-generator/cmd/register-gen/main.go | 4 +-- .../k8s.io/code-generator/cmd/set-gen/main.go | 3 +- staging/src/k8s.io/code-generator/go.mod | 2 +- .../k8s.io/code-generator/pkg/util/build.go | 36 +++++++++++++++++-- 14 files changed, 46 insertions(+), 36 deletions(-) diff --git a/staging/src/k8s.io/code-generator/cmd/applyconfiguration-gen/main.go b/staging/src/k8s.io/code-generator/cmd/applyconfiguration-gen/main.go index ba204def226..2a2f71bf39b 100644 --- a/staging/src/k8s.io/code-generator/cmd/applyconfiguration-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/applyconfiguration-gen/main.go @@ -19,10 +19,8 @@ package main import ( "flag" - "path/filepath" "github.com/spf13/pflag" - "k8s.io/gengo/args" "k8s.io/klog/v2" generatorargs "k8s.io/code-generator/cmd/applyconfiguration-gen/args" @@ -33,7 +31,7 @@ import ( func main() { klog.InitFlags(nil) genericArgs, customArgs := generatorargs.NewDefaults() - genericArgs.GoHeaderFilePath = filepath.Join(args.DefaultSourceTree(), util.BoilerplatePath()) + genericArgs.GoHeaderFilePath = util.BoilerplatePath() genericArgs.AddFlags(pflag.CommandLine) customArgs.AddFlags(pflag.CommandLine, "k8s.io/kubernetes/pkg/apis") // TODO: move this input path out of client-gen if err := flag.Set("logtostderr", "true"); err != nil { diff --git a/staging/src/k8s.io/code-generator/cmd/client-gen/main.go b/staging/src/k8s.io/code-generator/cmd/client-gen/main.go index cc0f72a6926..bdb55461385 100644 --- a/staging/src/k8s.io/code-generator/cmd/client-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/client-gen/main.go @@ -19,10 +19,8 @@ package main import ( "flag" - "path/filepath" "github.com/spf13/pflag" - "k8s.io/gengo/args" "k8s.io/klog/v2" generatorargs "k8s.io/code-generator/cmd/client-gen/args" @@ -36,7 +34,7 @@ func main() { // Override defaults. // TODO: move this out of client-gen - genericArgs.GoHeaderFilePath = filepath.Join(args.DefaultSourceTree(), util.BoilerplatePath()) + genericArgs.GoHeaderFilePath = util.BoilerplatePath() genericArgs.OutputPackagePath = "k8s.io/kubernetes/pkg/client/clientset_generated/" genericArgs.AddFlags(pflag.CommandLine) diff --git a/staging/src/k8s.io/code-generator/cmd/conversion-gen/main.go b/staging/src/k8s.io/code-generator/cmd/conversion-gen/main.go index 6e0f31f068d..feba7d02903 100644 --- a/staging/src/k8s.io/code-generator/cmd/conversion-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/conversion-gen/main.go @@ -87,10 +87,8 @@ package main import ( "flag" - "path/filepath" "github.com/spf13/pflag" - "k8s.io/gengo/args" "k8s.io/klog/v2" generatorargs "k8s.io/code-generator/cmd/conversion-gen/args" @@ -104,7 +102,7 @@ func main() { // Override defaults. // TODO: move this out of conversion-gen - genericArgs.GoHeaderFilePath = filepath.Join(args.DefaultSourceTree(), util.BoilerplatePath()) + genericArgs.GoHeaderFilePath = util.BoilerplatePath() genericArgs.AddFlags(pflag.CommandLine) customArgs.AddFlags(pflag.CommandLine) diff --git a/staging/src/k8s.io/code-generator/cmd/deepcopy-gen/main.go b/staging/src/k8s.io/code-generator/cmd/deepcopy-gen/main.go index 31a4824a5c9..888c2e306a9 100644 --- a/staging/src/k8s.io/code-generator/cmd/deepcopy-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/deepcopy-gen/main.go @@ -44,10 +44,8 @@ package main import ( "flag" - "path/filepath" "github.com/spf13/pflag" - "k8s.io/gengo/args" "k8s.io/gengo/examples/deepcopy-gen/generators" "k8s.io/klog/v2" @@ -61,7 +59,7 @@ func main() { // Override defaults. // TODO: move this out of deepcopy-gen - genericArgs.GoHeaderFilePath = filepath.Join(args.DefaultSourceTree(), util.BoilerplatePath()) + genericArgs.GoHeaderFilePath = util.BoilerplatePath() genericArgs.AddFlags(pflag.CommandLine) customArgs.AddFlags(pflag.CommandLine) diff --git a/staging/src/k8s.io/code-generator/cmd/defaulter-gen/main.go b/staging/src/k8s.io/code-generator/cmd/defaulter-gen/main.go index e9aef8d223f..f050a132788 100644 --- a/staging/src/k8s.io/code-generator/cmd/defaulter-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/defaulter-gen/main.go @@ -43,10 +43,8 @@ package main import ( "flag" - "path/filepath" "github.com/spf13/pflag" - "k8s.io/gengo/args" "k8s.io/gengo/examples/defaulter-gen/generators" "k8s.io/klog/v2" @@ -60,7 +58,7 @@ func main() { // Override defaults. // TODO: move this out of defaulter-gen - genericArgs.GoHeaderFilePath = filepath.Join(args.DefaultSourceTree(), util.BoilerplatePath()) + genericArgs.GoHeaderFilePath = util.BoilerplatePath() genericArgs.AddFlags(pflag.CommandLine) customArgs.AddFlags(pflag.CommandLine) diff --git a/staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/cmd.go b/staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/cmd.go index f7c8fb7ffca..8a92082bef1 100644 --- a/staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/cmd.go +++ b/staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/cmd.go @@ -57,7 +57,7 @@ func New() *Generator { sourceTree := args.DefaultSourceTree() common := args.GeneratorArgs{ OutputBase: sourceTree, - GoHeaderFilePath: filepath.Join(sourceTree, util.BoilerplatePath()), + GoHeaderFilePath: util.BoilerplatePath(), } defaultProtoImport := filepath.Join(sourceTree, "k8s.io", "kubernetes", "vendor", "github.com", "gogo", "protobuf", "protobuf") cwd, err := os.Getwd() diff --git a/staging/src/k8s.io/code-generator/cmd/import-boss/main.go b/staging/src/k8s.io/code-generator/cmd/import-boss/main.go index 3fd0cf80d5c..0fc56445e8c 100644 --- a/staging/src/k8s.io/code-generator/cmd/import-boss/main.go +++ b/staging/src/k8s.io/code-generator/cmd/import-boss/main.go @@ -19,7 +19,6 @@ package main import ( "os" - "path/filepath" "github.com/spf13/pflag" "k8s.io/code-generator/pkg/util" @@ -34,7 +33,7 @@ func main() { arguments := args.Default() // Override defaults. - arguments.GoHeaderFilePath = filepath.Join(args.DefaultSourceTree(), util.BoilerplatePath()) + arguments.GoHeaderFilePath = util.BoilerplatePath() pflag.CommandLine.BoolVar(&arguments.IncludeTestFiles, "include-test-files", false, "If true, include *_test.go files.") if err := arguments.Execute( diff --git a/staging/src/k8s.io/code-generator/cmd/informer-gen/main.go b/staging/src/k8s.io/code-generator/cmd/informer-gen/main.go index 6148296a043..650a3d78bd2 100644 --- a/staging/src/k8s.io/code-generator/cmd/informer-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/informer-gen/main.go @@ -18,12 +18,10 @@ package main import ( "flag" - "path/filepath" "github.com/spf13/pflag" "k8s.io/code-generator/cmd/informer-gen/generators" "k8s.io/code-generator/pkg/util" - "k8s.io/gengo/args" "k8s.io/klog/v2" generatorargs "k8s.io/code-generator/cmd/informer-gen/args" @@ -35,7 +33,7 @@ func main() { // Override defaults. // TODO: move out of informer-gen - genericArgs.GoHeaderFilePath = filepath.Join(args.DefaultSourceTree(), util.BoilerplatePath()) + genericArgs.GoHeaderFilePath = util.BoilerplatePath() genericArgs.OutputPackagePath = "k8s.io/kubernetes/pkg/client/informers/informers_generated" customArgs.VersionedClientSetPackage = "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" customArgs.InternalClientSetPackage = "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" diff --git a/staging/src/k8s.io/code-generator/cmd/lister-gen/main.go b/staging/src/k8s.io/code-generator/cmd/lister-gen/main.go index 855b4e15c73..3418005c623 100644 --- a/staging/src/k8s.io/code-generator/cmd/lister-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/lister-gen/main.go @@ -18,12 +18,10 @@ package main import ( "flag" - "path/filepath" "github.com/spf13/pflag" "k8s.io/code-generator/cmd/lister-gen/generators" "k8s.io/code-generator/pkg/util" - "k8s.io/gengo/args" "k8s.io/klog/v2" generatorargs "k8s.io/code-generator/cmd/lister-gen/args" @@ -35,7 +33,7 @@ func main() { // Override defaults. // TODO: move this out of lister-gen - genericArgs.GoHeaderFilePath = filepath.Join(args.DefaultSourceTree(), util.BoilerplatePath()) + genericArgs.GoHeaderFilePath = util.BoilerplatePath() genericArgs.OutputPackagePath = "k8s.io/kubernetes/pkg/client/listers" genericArgs.AddFlags(pflag.CommandLine) diff --git a/staging/src/k8s.io/code-generator/cmd/prerelease-lifecycle-gen/main.go b/staging/src/k8s.io/code-generator/cmd/prerelease-lifecycle-gen/main.go index 43a39f40a1a..7aeee78bf60 100644 --- a/staging/src/k8s.io/code-generator/cmd/prerelease-lifecycle-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/prerelease-lifecycle-gen/main.go @@ -34,13 +34,11 @@ package main import ( "flag" - "path/filepath" "github.com/spf13/pflag" generatorargs "k8s.io/code-generator/cmd/prerelease-lifecycle-gen/args" statusgenerators "k8s.io/code-generator/cmd/prerelease-lifecycle-gen/prerelease-lifecycle-generators" "k8s.io/code-generator/pkg/util" - "k8s.io/gengo/args" "k8s.io/klog/v2" ) @@ -50,7 +48,7 @@ func main() { // Override defaults. // TODO: move this out of prerelease-lifecycle-gen - genericArgs.GoHeaderFilePath = filepath.Join(args.DefaultSourceTree(), util.BoilerplatePath()) + genericArgs.GoHeaderFilePath = util.BoilerplatePath() genericArgs.AddFlags(pflag.CommandLine) customArgs.AddFlags(pflag.CommandLine) diff --git a/staging/src/k8s.io/code-generator/cmd/register-gen/main.go b/staging/src/k8s.io/code-generator/cmd/register-gen/main.go index 816bd81678f..b8ba66f0da0 100644 --- a/staging/src/k8s.io/code-generator/cmd/register-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/register-gen/main.go @@ -18,7 +18,6 @@ package main import ( "flag" - "path/filepath" "github.com/spf13/pflag" "k8s.io/klog/v2" @@ -26,13 +25,12 @@ import ( generatorargs "k8s.io/code-generator/cmd/register-gen/args" "k8s.io/code-generator/cmd/register-gen/generators" "k8s.io/code-generator/pkg/util" - "k8s.io/gengo/args" ) func main() { klog.InitFlags(nil) genericArgs := generatorargs.NewDefaults() - genericArgs.GoHeaderFilePath = filepath.Join(args.DefaultSourceTree(), util.BoilerplatePath()) + genericArgs.GoHeaderFilePath = util.BoilerplatePath() genericArgs.AddFlags(pflag.CommandLine) flag.Set("logtostderr", "true") pflag.CommandLine.AddGoFlagSet(flag.CommandLine) diff --git a/staging/src/k8s.io/code-generator/cmd/set-gen/main.go b/staging/src/k8s.io/code-generator/cmd/set-gen/main.go index 2d98a1b328a..f40c5e4a44e 100644 --- a/staging/src/k8s.io/code-generator/cmd/set-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/set-gen/main.go @@ -26,7 +26,6 @@ package main import ( "os" - "path/filepath" "k8s.io/code-generator/pkg/util" "k8s.io/gengo/args" @@ -40,7 +39,7 @@ func main() { arguments := args.Default() // Override defaults. - arguments.GoHeaderFilePath = filepath.Join(args.DefaultSourceTree(), util.BoilerplatePath()) + arguments.GoHeaderFilePath = util.BoilerplatePath() arguments.InputDirs = []string{"k8s.io/kubernetes/pkg/util/sets/types"} arguments.OutputPackagePath = "k8s.io/apimachinery/pkg/util/sets" diff --git a/staging/src/k8s.io/code-generator/go.mod b/staging/src/k8s.io/code-generator/go.mod index 08890d5e4fd..fb39e425a46 100644 --- a/staging/src/k8s.io/code-generator/go.mod +++ b/staging/src/k8s.io/code-generator/go.mod @@ -17,7 +17,7 @@ require ( github.com/stretchr/testify v1.7.0 // indirect golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 // indirect golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect - golang.org/x/tools v0.1.2 // indirect + golang.org/x/tools v0.1.2 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 diff --git a/staging/src/k8s.io/code-generator/pkg/util/build.go b/staging/src/k8s.io/code-generator/pkg/util/build.go index 6ed1fe497c2..72ae683d8d8 100644 --- a/staging/src/k8s.io/code-generator/pkg/util/build.go +++ b/staging/src/k8s.io/code-generator/pkg/util/build.go @@ -18,10 +18,12 @@ package util import ( gobuild "go/build" - "path" + "os" "path/filepath" "reflect" "strings" + + "golang.org/x/tools/go/packages" ) type empty struct{} @@ -55,9 +57,37 @@ func hasSubdir(root, dir string) (rel string, ok bool) { return filepath.ToSlash(dir[len(root):]), true } -// BoilerplatePath uses the boilerplate in code-generator by calculating the relative path to it. +// BoilerplatePath returns the path to the boilerplate file in code-generator, +// or "" if the default boilerplate.go.txt file cannot be located. func BoilerplatePath() string { - return path.Join(reflect.TypeOf(empty{}).PkgPath(), "/../../hack/boilerplate.go.txt") + // set up paths to check + paths := []string{ + // works when run from root of $GOPATH containing k8s.io/code-generator + filepath.Join(reflect.TypeOf(empty{}).PkgPath(), "/../../hack/boilerplate.go.txt"), + // works when run from root of module vendoring k8s.io/code-generator + "vendor/k8s.io/code-generator/hack/boilerplate.go.txt", + // works when run from root of $GOPATH containing k8s.io/kubernetes + "k8s.io/kubernetes/vendor/k8s.io/code-generator/hack/boilerplate.go.txt", + } + + // see if we can locate the module directory and add that to the list + config := packages.Config{Mode: packages.NeedModule} + if loadedPackages, err := packages.Load(&config, "k8s.io/code-generator/pkg/util"); err == nil { + for _, loadedPackage := range loadedPackages { + if loadedPackage.Module != nil && loadedPackage.Module.Dir != "" { + paths = append(paths, filepath.Join(loadedPackage.Module.Dir, "hack/boilerplate.go.txt")) + } + } + } + + // try all paths and return the first that exists + for _, path := range paths { + if _, err := os.Stat(path); err == nil { + return path + } + } + // cannot be located, invoker will have to explicitly specify boilerplate file + return "" } // Vendorless trims vendor prefix from a package path to make it canonical