From 9f0050cb44cbfe2ad85865625bed05f1412e4b8e Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Sun, 9 Jun 2019 20:39:53 -0400 Subject: [PATCH] verify import aliases - Added scripts for update and verify - golang AST code for scanning and fixing imports - default regex allows it to run on just test/e2e.* file paths - exclude verify-import-aliases.sh from running in CI jobs Change-Id: I7f9c76f5525fb9a26ea2be60ea69356362957998 Co-Authored-By: Aaron Crickenberger --- cmd/BUILD | 1 + cmd/preferredimports/BUILD | 32 +++ cmd/preferredimports/OWNERS | 8 + cmd/preferredimports/preferredimports.go | 263 +++++++++++++++++++++++ hack/.import-aliases | 2 + hack/make-rules/verify.sh | 1 + hack/update-import-aliases.sh | 34 +++ hack/verify-import-aliases.sh | 34 +++ 8 files changed, 375 insertions(+) create mode 100644 cmd/preferredimports/BUILD create mode 100644 cmd/preferredimports/OWNERS create mode 100644 cmd/preferredimports/preferredimports.go create mode 100644 hack/.import-aliases create mode 100755 hack/update-import-aliases.sh create mode 100755 hack/verify-import-aliases.sh diff --git a/cmd/BUILD b/cmd/BUILD index 64074c630d4..b63c2d9b60e 100644 --- a/cmd/BUILD +++ b/cmd/BUILD @@ -31,6 +31,7 @@ filegroup( "//cmd/kubelet:all-srcs", "//cmd/kubemark:all-srcs", "//cmd/linkcheck:all-srcs", + "//cmd/preferredimports:all-srcs", ], tags = ["automanaged"], ) diff --git a/cmd/preferredimports/BUILD b/cmd/preferredimports/BUILD new file mode 100644 index 00000000000..fd537fa2afb --- /dev/null +++ b/cmd/preferredimports/BUILD @@ -0,0 +1,32 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_binary", + "go_library", +) + +go_binary( + name = "preferredimports", + embed = [":go_default_library"], +) + +go_library( + name = "go_default_library", + srcs = ["preferredimports.go"], + importpath = "k8s.io/kubernetes/cmd/preferredimports", + deps = ["//vendor/golang.org/x/crypto/ssh/terminal:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/cmd/preferredimports/OWNERS b/cmd/preferredimports/OWNERS new file mode 100644 index 00000000000..1ee6480c5c9 --- /dev/null +++ b/cmd/preferredimports/OWNERS @@ -0,0 +1,8 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +reviewers: + - johnSchnake +approvers: + - dims + - cblecker + - spiffxp diff --git a/cmd/preferredimports/preferredimports.go b/cmd/preferredimports/preferredimports.go new file mode 100644 index 00000000000..7aa1cbfd9ca --- /dev/null +++ b/cmd/preferredimports/preferredimports.go @@ -0,0 +1,263 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// verify that all the imports have our preferred alias(es). +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "go/ast" + "go/build" + "go/format" + "go/parser" + "go/token" + "golang.org/x/crypto/ssh/terminal" + "io/ioutil" + "log" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +var ( + importAliases = flag.String("import-aliases", "hack/.import-aliases", "json file with import aliases") + confirm = flag.Bool("confirm", false, "update file with the preferred aliases for imports") + regex = flag.String("include-path", "(test/e2e/|test/e2e_node)", "only files with paths matching this regex is touched") + isTerminal = terminal.IsTerminal(int(os.Stdout.Fd())) + logPrefix = "" + aliases map[string]string +) + +type analyzer struct { + fset *token.FileSet // positions are relative to fset + ctx build.Context + failed bool + donePaths map[string]interface{} + errors []string +} + +func newAnalyzer() *analyzer { + ctx := build.Default + ctx.CgoEnabled = true + + a := &analyzer{ + fset: token.NewFileSet(), + ctx: ctx, + donePaths: make(map[string]interface{}), + } + + return a +} + +// collect extracts test metadata from a file. +func (a *analyzer) collect(dir string) { + if _, ok := a.donePaths[dir]; ok { + return + } + a.donePaths[dir] = nil + + // Create the AST by parsing src. + fs, err := parser.ParseDir(a.fset, dir, nil, parser.AllErrors|parser.ParseComments) + + if err != nil { + fmt.Fprintln(os.Stderr, "ERROR(syntax)", logPrefix, err) + a.failed = true + return + } + + for _, p := range fs { + // returns first error, but a.handleError deals with it + files := a.filterFiles(p.Files) + for _, file := range files { + replacements := make(map[string]string) + pathToFile := a.fset.File(file.Pos()).Name() + for _, imp := range file.Imports { + importPath := strings.Replace(imp.Path.Value, "\"", "", -1) + pathSegments := strings.Split(importPath, "/") + importName := pathSegments[len(pathSegments)-1] + if imp.Name != nil { + importName = imp.Name.Name + } + if alias, ok := aliases[importPath]; ok { + if alias != importName { + if !*confirm { + fmt.Fprintf(os.Stderr, "%sERROR wrong alias for import \"%s\" should be %s in file %s\n", logPrefix, importPath, alias, pathToFile) + a.failed = true + } + replacements[importName] = alias + if imp.Name != nil { + imp.Name.Name = alias + } else { + imp.Name = ast.NewIdent(alias) + } + } + } + } + + if len(replacements) > 0 { + if *confirm { + fmt.Printf("%sReplacing imports with aliases in file %s\n", logPrefix, pathToFile) + for key, value := range replacements { + renameImportUsages(file, key, value) + } + ast.SortImports(a.fset, file) + var buffer bytes.Buffer + if err = format.Node(&buffer, a.fset, file); err != nil { + panic(fmt.Sprintf("Error formatting ast node after rewriting import.\n%s\n", err.Error())) + } + + fileInfo, err := os.Stat(pathToFile) + if err != nil { + panic(fmt.Sprintf("Error stat'ing file: %s\n%s\n", pathToFile, err.Error())) + } + + err = ioutil.WriteFile(pathToFile, buffer.Bytes(), fileInfo.Mode()) + if err != nil { + panic(fmt.Sprintf("Error writing file: %s\n%s\n", pathToFile, err.Error())) + } + } + } + } + } +} + +func renameImportUsages(f *ast.File, old, new string) { + // use this to avoid renaming the package declaration, eg: + // given: package foo; import foo "bar"; foo.Baz, rename foo->qux + // yield: package foo; import qux "bar"; qux.Baz + var pkg *ast.Ident + + // Rename top-level old to new, both unresolved names + // (probably defined in another file) and names that resolve + // to a declaration we renamed. + ast.Inspect(f, func(node ast.Node) bool { + if node == nil { + return false + } + switch id := node.(type) { + case *ast.File: + pkg = id.Name + case *ast.Ident: + if pkg != nil && id == pkg { + return false + } + if id.Name == old { + id.Name = new + } + } + return true + }) +} + +func (a *analyzer) filterFiles(fs map[string]*ast.File) []*ast.File { + var files []*ast.File + for _, f := range fs { + files = append(files, f) + } + return files +} + +type collector struct { + dirs []string + regex *regexp.Regexp +} + +// handlePath walks the filesystem recursively, collecting directories, +// ignoring some unneeded directories (hidden/vendored) that are handled +// specially later. +func (c *collector) handlePath(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + // Ignore hidden directories (.git, .cache, etc) + if len(path) > 1 && path[0] == '.' || + // Staging code is symlinked from vendor/k8s.io, and uses import + // paths as if it were inside of vendor/. It fails typechecking + // inside of staging/, but works when typechecked as part of vendor/. + path == "staging" || + // OS-specific vendor code tends to be imported by OS-specific + // packages. We recursively typecheck imported vendored packages for + // each OS, but don't typecheck everything for every OS. + path == "vendor" || + path == "_output" || + // This is a weird one. /testdata/ is *mostly* ignored by Go, + // and this translates to kubernetes/vendor not working. + // edit/record.go doesn't compile without gopkg.in/yaml.v2 + // in $GOSRC/$GOROOT (both typecheck and the shell script). + path == "pkg/kubectl/cmd/testdata/edit" { + return filepath.SkipDir + } + if c.regex.MatchString(path) { + c.dirs = append(c.dirs, path) + } + } + return nil +} + +func main() { + flag.Parse() + args := flag.Args() + + if len(args) == 0 { + args = append(args, ".") + } + + regex, err := regexp.Compile(*regex) + if err != nil { + log.Fatalf("Error compiling regex: %v", err) + } + c := collector{regex: regex} + for _, arg := range args { + err := filepath.Walk(arg, c.handlePath) + if err != nil { + log.Fatalf("Error walking: %v", err) + } + } + sort.Strings(c.dirs) + + if len(*importAliases) > 0 { + bytes, err := ioutil.ReadFile(*importAliases) + if err != nil { + log.Fatalf("Error reading import aliases: %v", err) + } + err = json.Unmarshal(bytes, &aliases) + if err != nil { + log.Fatalf("Error loading aliases: %v", err) + } + } + if isTerminal { + logPrefix = "\r" // clear status bar when printing + } + fmt.Println("checking-imports: ") + + a := newAnalyzer() + for _, dir := range c.dirs { + if isTerminal { + fmt.Printf("\r\033[0m %-80s", dir) + } + a.collect(dir) + } + fmt.Println() + if a.failed { + os.Exit(1) + } +} diff --git a/hack/.import-aliases b/hack/.import-aliases new file mode 100644 index 00000000000..2c63c085104 --- /dev/null +++ b/hack/.import-aliases @@ -0,0 +1,2 @@ +{ +} diff --git a/hack/make-rules/verify.sh b/hack/make-rules/verify.sh index 9418ce8cf7d..a4e716efb61 100755 --- a/hack/make-rules/verify.sh +++ b/hack/make-rules/verify.sh @@ -35,6 +35,7 @@ EXCLUDED_PATTERNS=( "verify-linkcheck.sh" # runs in separate Jenkins job once per day due to high network usage "verify-test-owners.sh" # TODO(rmmh): figure out how to avoid endless conflicts "verify-*-dockerized.sh" # Don't run any scripts that intended to be run dockerized + "verify-import-aliases.sh" # to be run periodically by folks working on conformance tests ) # Exclude typecheck in certain cases, if they're running in a separate job. diff --git a/hack/update-import-aliases.sh b/hack/update-import-aliases.sh new file mode 100755 index 00000000000..baa9e8f86bb --- /dev/null +++ b/hack/update-import-aliases.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" + +kube::golang::verify_go_version + +cd "${KUBE_ROOT}" + +ret=0 +go run cmd/preferredimports/preferredimports.go --confirm "$@" || ret=$? +if [[ $ret -ne 0 ]]; then + echo "!!! Unable to fix imports programmatically. Please see errors above." >&2 + exit 1 +fi + diff --git a/hack/verify-import-aliases.sh b/hack/verify-import-aliases.sh new file mode 100755 index 00000000000..a80a94a2973 --- /dev/null +++ b/hack/verify-import-aliases.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" + +kube::golang::verify_go_version + +cd "${KUBE_ROOT}" + +ret=0 +go run cmd/preferredimports/preferredimports.go "$@" || ret=$? +if [[ $ret -ne 0 ]]; then + echo "!!! Please see hack/.import-aliases for the preferred aliases for imports." >&2 + exit 1 +fi +