Merge pull request #93448 from hasheddan/vendor-deps

Add dependencycheck tool to address long running no-vendor-cycles test

cmd/BUILD

@@ -14,6 +14,7 @@ filegroup(
         "//cmd/clicheck:all-srcs",
         "//cmd/cloud-controller-manager:all-srcs",
         "//cmd/controller-manager/app:all-srcs",
+        "//cmd/dependencycheck:all-srcs",
         "//cmd/gendocs:all-srcs",
         "//cmd/genkubedocs:all-srcs",
         "//cmd/genman:all-srcs",

cmd/dependencycheck/BUILD

@@ -0,0 +1,28 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["dependencycheck.go"],
+    importpath = "k8s.io/kubernetes/cmd/dependencycheck",
+    visibility = ["//visibility:private"],
+)
+
+go_binary(
+    name = "vendorcycle",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

cmd/dependencycheck/OWNERS

@@ -0,0 +1,8 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+reviewers:
+  - hasheddan
+approvers:
+  - bentheelder
+  - hasheddan
+  - liggitt

cmd/dependencycheck/dependencycheck.go

@@ -0,0 +1,115 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Checks for restricted dependencies in go packages. Does not check transitive
+// dependencies implicitly, so they must be supplied in dependencies file if
+// they are to be evaluated.
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"regexp"
+)
+
+var (
+	exclude  = flag.String("exclude", "", "skip packages regex pattern (e.g. '^k8s.io/kubernetes/')")
+	restrict = flag.String("restrict", "", "restricted dependencies regex pattern (e.g. '^k8s.io/(apimachinery|client-go)/')")
+)
+
+type goPackage struct {
+	Name         string
+	ImportPath   string
+	Imports      []string
+	TestImports  []string
+	XTestImports []string
+}
+
+func main() {
+	flag.Parse()
+
+	args := flag.Args()
+
+	if len(args) != 1 {
+		log.Fatalf("usage: dependencycheck <json-dep-file> (e.g. 'go list -mod=vendor -test -deps -json ./vendor/...')")
+	}
+	if *restrict == "" {
+		log.Fatalf("Must specify restricted regex pattern")
+	}
+	depsPattern, err := regexp.Compile(*restrict)
+	if err != nil {
+		log.Fatalf("Error compiling restricted dependencies regex: %v", err)
+	}
+	var excludePattern *regexp.Regexp
+	if *exclude != "" {
+		excludePattern, err = regexp.Compile(*exclude)
+		if err != nil {
+			log.Fatalf("Error compiling excluded package regex: %v", err)
+		}
+	}
+	b, err := ioutil.ReadFile(args[0])
+	if err != nil {
+		log.Fatalf("Error reading dependencies file: %v", err)
+	}
+
+	packages := []goPackage{}
+	decoder := json.NewDecoder(bytes.NewBuffer(b))
+	for {
+		pkg := goPackage{}
+		if err := decoder.Decode(&pkg); err != nil {
+			if err == io.EOF {
+				break
+			}
+			log.Fatalf("Error unmarshaling dependencies file: %v", err)
+		}
+		packages = append(packages, pkg)
+	}
+
+	violations := map[string][]string{}
+	for _, p := range packages {
+		if excludePattern != nil && excludePattern.MatchString(p.ImportPath) {
+			continue
+		}
+		importViolations := []string{}
+		allImports := []string{}
+		allImports = append(allImports, p.Imports...)
+		allImports = append(allImports, p.TestImports...)
+		allImports = append(allImports, p.XTestImports...)
+		for _, i := range allImports {
+			if depsPattern.MatchString(i) {
+				importViolations = append(importViolations, i)
+			}
+		}
+		if len(importViolations) > 0 {
+			violations[p.ImportPath] = importViolations
+		}
+	}
+
+	if len(violations) > 0 {
+		for k, v := range violations {
+			fmt.Printf("Found dependency violations in package %s:\n", k)
+			for _, a := range v {
+				fmt.Println("--> " + a)
+			}
+		}
+		log.Fatal("Found restricted dependency violations in packages")
+	}
+}

hack/verify-no-vendor-cycles.sh

@@ -21,7 +21,6 @@
 set -o errexit
 set -o nounset
 set -o pipefail
-set -x;
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
@@ -34,21 +33,24 @@ staging_repos_pattern=$(IFS="|"; echo "${staging_repos[*]}")
 
 cd "${KUBE_ROOT}"
 
-failed=false
-while IFS= read -r dir; do
-  deps=$(go list -f '{{range .Deps}}{{.}}{{"\n"}}{{end}}' "${dir}" 2> /dev/null || echo "")
-  deps_on_main=$(echo "${deps}" | grep -v "k8s.io/kubernetes/vendor/" | grep "k8s.io/kubernetes" || echo "")
-  if [ -n "${deps_on_main}" ]; then
-    echo "Package ${dir} has a cyclic dependency on the main repository."
-    failed=true
-  fi
-  deps_on_staging=$(echo "${deps}" | grep "k8s.io/kubernetes/vendor/k8s.io" | grep -E "k8s.io\/${staging_repos_pattern}\>" || echo "")
-  if [ -n "${deps_on_staging}" ]; then
-    echo "Package ${dir} has a cyclic dependency on staging repository packages: ${deps_on_staging}"
-    failed=true
-  fi
-done < <(find ./vendor -type d)
-
-if [[ "${failed}" == "true" ]]; then
+# Check for any module that is not main or staging and depends on main or staging
+bad_deps=$(go mod graph | grep -vE "^k8s.io\/(kubernetes|${staging_repos_pattern})" | grep -E "\sk8s.io\/(kubernetes|${staging_repos_pattern})" || true)
+if [[ -n "${bad_deps}" ]]; then
+  echo "Found disallowed dependencies that transitively depend on k8s.io/kubernetes or staging modules:"
+  echo "${bad_deps}"
   exit 1
 fi
+
+kube::util::ensure-temp-dir
+
+# Get vendored packages dependencies
+# Use -deps flag to include transitive dependencies
+go list -mod=vendor -test -deps -json ./vendor/... > "${KUBE_TEMP}/deps.json"
+
+# Check for any vendored package that imports main repo
+# Staging repos are explicitly excluded even though go list does not currently consider symlinks
+go run cmd/dependencycheck/dependencycheck.go -restrict "^k8s\.io/kubernetes/" -exclude "^k8s\.io/(${staging_repos_pattern})(/|$)" "${KUBE_TEMP}/deps.json"
+
+# Check for any vendored package that imports a staging repo
+# Staging repos are explicitly excluded even though go list does not currently consider symlinks
+go run cmd/dependencycheck/dependencycheck.go -restrict "^k8s\.io/(${staging_repos_pattern})(/|$)" -exclude "^k8s\.io/(${staging_repos_pattern})(/|$)" "${KUBE_TEMP}/deps.json"