diff --git a/hack/tools/go.mod b/hack/tools/go.mod index a93cd7c940b..c2070c0f8f7 100644 --- a/hack/tools/go.mod +++ b/hack/tools/go.mod @@ -10,8 +10,10 @@ require ( github.com/jcchavezs/porto v0.6.0 github.com/vektra/mockery/v2 v2.40.3 go.uber.org/automaxprocs v1.5.2 + golang.org/x/mod v0.20.0 gotest.tools/gotestsum v1.12.0 honnef.co/go/tools v0.5.1 + k8s.io/publishing-bot v0.5.0 sigs.k8s.io/logtools v0.8.1 ) @@ -74,6 +76,7 @@ require ( github.com/go-xmlfmt/xmlfmt v1.1.2 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gofrs/flock v0.8.1 // indirect + github.com/golang/glog v1.2.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect @@ -190,7 +193,6 @@ require ( go.uber.org/zap v1.24.0 // indirect golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect golang.org/x/exp/typeparams v0.0.0-20231219180239-dc181d75b848 // indirect - golang.org/x/mod v0.17.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/term v0.18.0 // indirect diff --git a/hack/tools/go.sum b/hack/tools/go.sum index 11c25adcf38..8ab4ad35d3e 100644 --- a/hack/tools/go.sum +++ b/hack/tools/go.sum @@ -207,6 +207,8 @@ github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= +github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -678,8 +680,8 @@ golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1024,6 +1026,8 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.5.1 h1:4bH5o3b5ZULQ4UrBmP+63W9r7qIkqJClEA9ko5YKx+I= honnef.co/go/tools v0.5.1/go.mod h1:e9irvo83WDG9/irijV44wr3tbhcFeRnfpVlRqVwpzMs= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/publishing-bot v0.5.0 h1:Hfnhltr+khEcqvoK4GBYrtaA8dHJ50Xjyi+0KGUfU3I= +k8s.io/publishing-bot v0.5.0/go.mod h1:S5+zQQhsVUEqdcaohbYf8O+2BeeWRtuYzp4tQLr5An8= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= mvdan.cc/gofumpt v0.6.0 h1:G3QvahNDmpD+Aek/bNOLrFR2XC6ZAdo62dZu65gmwGo= mvdan.cc/gofumpt v0.6.0/go.mod h1:4L0wf+kgIPZtcCWXynNS2e6bhmj73umwnuXSZarixzA= diff --git a/hack/tools/publishing-verifier/publishing-verifier.go b/hack/tools/publishing-verifier/publishing-verifier.go new file mode 100644 index 00000000000..5dffccf6537 --- /dev/null +++ b/hack/tools/publishing-verifier/publishing-verifier.go @@ -0,0 +1,240 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "golang.org/x/mod/modfile" + "k8s.io/publishing-bot/cmd/publishing-bot/config" +) + +var ( + rulesFile string + componentsDirectory string +) + +// getGoModDependencies gets all the staging dependencies for all the modules +// in the given directory +func getGoModDependencies(dir string) (map[string][]string, error) { + allDependencies := make(map[string][]string) + components, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + for _, component := range components { + componentName := component.Name() + if !component.IsDir() { + // currently there is no hard check that the staging directory should not contain + // other files + continue + } + gomodFilePath := filepath.Join(dir, componentName, "go.mod") + gomodFileContent, err := os.ReadFile(gomodFilePath) + if err != nil { + return nil, err + } + + fmt.Printf("%s dependencies", componentName) + + allDependencies[componentName] = make([]string, 0) + + gomodFile, err := modfile.Parse(gomodFilePath, gomodFileContent, nil) + if err != nil { + return nil, err + } + // get all the other dependencies from within staging, i.e all the modules in replace + // section + for _, module := range gomodFile.Replace { + dep := strings.TrimPrefix(module.Old.Path, "k8s.io/") + if dep == componentName { + continue + } + allDependencies[componentName] = append(allDependencies[componentName], dep) + } + } + return allDependencies, nil +} + +// diffSlice returns the difference of s1-s2 +func diffSlice(s1, s2 []string) []string { + var diff []string + set := make(map[string]struct{}, len(s2)) + for _, s := range s2 { + set[s] = struct{}{} + } + for _, s := range s1 { + if _, ok := set[s]; !ok { + diff = append(diff, s) + } + } + return diff +} + +// getKeys returns a slice with only the keys of the given map +func getKeys[K comparable, V any](m map[K]V) []K { + var keys []K + for k := range m { + keys = append(keys, k) + } + return keys +} + +// checkValidSourceDirectory checks if proper source directory fields are used in rules +func checkValidSourceDirectory(rule config.RepositoryRule) error { + for _, branch := range rule.Branches { + if branch.Source.Dir != "" { + return fmt.Errorf("use of deprecated `dir` field in rules for `%s`", rule.DestinationRepository) + } + if len(branch.Source.Dirs) > 1 { + return fmt.Errorf("cannot have more than one directory (%s) per source branch `%s` of `%s`", + branch.Source.Dirs, + branch.Source.Branch, + rule.DestinationRepository, + ) + } + if !strings.HasSuffix(branch.Source.Dirs[0], rule.DestinationRepository) { + return fmt.Errorf("copy/paste error `%s` refers to `%s`", rule.DestinationRepository, branch.Source.Dirs[0]) + } + } + return nil +} + +// checkMasterBranch checks if the master branch of destination repository refers to the master +// of the source +func checkMasterBranch(rule config.RepositoryRule) error { + branch := rule.Branches[0] + if branch.Name != "master" { + return fmt.Errorf("cannot find master branch for destination `%s`", rule.DestinationRepository) + } + + if branch.Source.Branch != "master" { + return fmt.Errorf("cannot find master source branch for destination `%s`", rule.DestinationRepository) + } + return nil +} + +func checkDependencies(rule config.RepositoryRule, gomodDependencies map[string][]string) error { + var processedDeps []string + branch := rule.Branches[0] + for _, dep := range gomodDependencies[rule.DestinationRepository] { + found := false + if len(branch.Dependencies) > 0 { + for _, dep2 := range branch.Dependencies { + processedDeps = append(processedDeps, dep2.Repository) + if dep2.Branch != "master" { + return fmt.Errorf("looking for master branch of %s and found : %s for destination", dep2.Repository, rule.DestinationRepository) + } + if dep2.Repository == dep { + found = true + } + } + } else { + return fmt.Errorf("Please add %s as dependencies under destination %s", gomodDependencies[rule.DestinationRepository], rule.DestinationRepository) + } + if !found { + return fmt.Errorf("Please add %s as a dependency under destination %s", dep, rule.DestinationRepository) + } else { + fmt.Printf("dependency %s found\n", dep) + } + } + // check if all deps are processed. + extraDeps := diffSlice(processedDeps, gomodDependencies[rule.DestinationRepository]) + if len(extraDeps) > 0 { + return fmt.Errorf("extra dependencies in rules for %s: %s", rule.DestinationRepository, strings.Join(extraDeps, ",")) + } + return nil +} + +func verifyPublishingBotRules() error { + rules, err := config.LoadRules(rulesFile) + if err != nil { + return fmt.Errorf("error loading rules: %v", err) + } + + gomodDependencies, err := getGoModDependencies(componentsDirectory) + + var processedRepos []string + for _, rule := range rules.Rules { + branch := rule.Branches[0] + + // if this no longer exists in master + if _, ok := gomodDependencies[rule.DestinationRepository]; !ok { + // make sure we dont include a rule to publish it from master + for _, branch := range rule.Branches { + if branch.Name == "master" { + err := fmt.Errorf("cannot find master branch for destination `%s`", rule.DestinationRepository) + panic(err) + } + } + // and skip the validation of publishing rules for it + continue + } + + if err := checkValidSourceDirectory(rule); err != nil { + return fmt.Errorf("error validating source directory: %v", err) + } + + if err := checkMasterBranch(rule); err != nil { + return fmt.Errorf("error validating master branch: %v", err) + } + + // we specify the go version for all master branches through `default-go-version` + // so ensure we don't specify explicit go version for master branch in rules + if branch.GoVersion != "" { + err := fmt.Errorf("go version must not be specified for master branch for destination `%s`", rule.DestinationRepository) + panic(err) + } + + fmt.Printf("processing : %s", rule.DestinationRepository) + if _, ok := gomodDependencies[rule.DestinationRepository]; !ok { + err := fmt.Errorf("missing go.mod for `%s`", rule.DestinationRepository) + panic(err) + } + processedRepos = append(processedRepos, rule.DestinationRepository) + + if err := checkDependencies(rule, gomodDependencies); err != nil { + return fmt.Errorf("error validating dependencies: %v", err) + } + } + + // check if all repos are processed. + items := diffSlice(getKeys(gomodDependencies), processedRepos) + if len(items) > 0 { + err := fmt.Errorf("missing rules for %s", strings.Join(items, ",")) + panic(err) + } + return nil +} + +func main() { + if len(os.Args) != 2 { + panic("invalid number of arguments") + } + + kubeRoot := os.Args[1] + stagingDirectory := kubeRoot + "/staging/" + rulesFile = stagingDirectory + "publishing/rules.yaml" + componentsDirectory = stagingDirectory + "src/k8s.io/" + + if err := verifyPublishingBotRules(); err != nil { + panic(err) + } +} diff --git a/hack/tools/tools.go b/hack/tools/tools.go index 42b52202d47..a1939294c99 100644 --- a/hack/tools/tools.go +++ b/hack/tools/tools.go @@ -36,4 +36,8 @@ import ( // tools like cpu _ "go.uber.org/automaxprocs" + + // for publishing bot + _ "golang.org/x/mod/modfile" + _ "k8s.io/publishing-bot/cmd/publishing-bot/config" ) diff --git a/hack/verify-publishing-bot.py b/hack/verify-publishing-bot.py deleted file mode 100755 index ee6b884bca8..00000000000 --- a/hack/verify-publishing-bot.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import fnmatch -import os -import sys -import json - - -def get_gomod_dependencies(rootdir, components): - all_dependencies = {} - for component in components: - with open(os.path.join(rootdir, component, "go.mod")) as f: - print((component + " dependencies")) - all_dependencies[component] = [] - lines = list(set(f)) - lines.sort() - for line in lines: - for dep in components: - if dep == component: - continue - if ("k8s.io/" + dep + " =>") not in line: - continue - print(("\t"+dep)) - if dep not in all_dependencies[component]: - all_dependencies[component].append(dep) - return all_dependencies - - -def get_rules_dependencies(rules_file): - import yaml - with open(rules_file) as f: - data = yaml.safe_load(f) - return data - - -def main(): - rootdir = os.path.dirname(__file__) + "/../" - rootdir = os.path.abspath(rootdir) - - components = [] - for component in os.listdir(rootdir + '/staging/src/k8s.io/'): - components.append(component) - components.sort() - - rules_file = "/staging/publishing/rules.yaml" - try: - import yaml - except ImportError: - print(("Please install missing pyyaml module and re-run %s" % sys.argv[0])) - sys.exit(1) - rules_dependencies = get_rules_dependencies(rootdir + rules_file) - - gomod_dependencies = get_gomod_dependencies(rootdir + '/staging/src/k8s.io/', components) - - processed_repos = [] - for rule in rules_dependencies["rules"]: - branch = rule["branches"][0] - - # If this no longer exists in master - if rule["destination"] not in gomod_dependencies: - # Make sure we don't include a rule to publish it from master - for branch in rule["branches"]: - if branch["name"] == "master": - raise Exception("cannot find master branch for destination %s" % rule["destination"]) - # And skip validation of publishing rules for it - continue - - for item in rule["branches"]: - if "dir" in item["source"]: - raise Exception("use of deprecated `dir` field in rules for `%s`" % (rule["destination"])) - if len(item["source"]["dirs"]) > 1: - raise Exception("cannot have more than one directory (`%s`) per source branch `%s` of `%s`" % - (item["source"]["dirs"], item["source"]["branch"], rule["destination"]) - ) - if not item["source"]["dirs"][0].endswith(rule["destination"]): - raise Exception("copy/paste error `%s` refers to `%s`" % (rule["destination"],item["source"]["dir"])) - - if branch["name"] != "master": - raise Exception("cannot find master branch for destination %s" % rule["destination"]) - if branch["source"]["branch"] != "master": - raise Exception("cannot find master source branch for destination %s" % rule["destination"]) - - # we specify the go version for all master branches through `default-go-version` - # so ensure we don't specify explicit go version for master branch in rules - if "go" in branch: - raise Exception("go version must not be specified for master branch for destination %s" % rule["destination"]) - - print(("processing : %s" % rule["destination"])) - if rule["destination"] not in gomod_dependencies: - raise Exception("missing go.mod for %s" % rule["destination"]) - processed_repos.append(rule["destination"]) - processed_deps = [] - for dep in set(gomod_dependencies[rule["destination"]]): - found = False - if "dependencies" in branch: - for dep2 in branch["dependencies"]: - processed_deps.append(dep2["repository"]) - if dep2["branch"] != "master": - raise Exception("Looking for master branch and found : %s for destination", dep2, - rule["destination"]) - if dep2["repository"] == dep: - found = True - else: - raise Exception( - "Please add %s as dependencies under destination %s in %s" % (gomod_dependencies[rule["destination"]], rule["destination"], rules_file)) - if not found: - raise Exception("Please add %s as a dependency under destination %s in %s" % (dep, rule["destination"], rules_file)) - else: - print((" found dependency %s" % dep)) - extraDeps = set(processed_deps) - set(gomod_dependencies[rule["destination"]]) - if len(extraDeps) > 0: - raise Exception("extra dependencies in rules for %s: %s" % (rule["destination"], ','.join(str(s) for s in extraDeps))) - items = set(gomod_dependencies.keys()) - set(processed_repos) - if len(items) > 0: - raise Exception("missing rules for %s" % ','.join(str(s) for s in items)) - print("Done.") - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/hack/verify-publishing-bot.sh b/hack/verify-publishing-bot.sh new file mode 100755 index 00000000000..9af33ec57a8 --- /dev/null +++ b/hack/verify-publishing-bot.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script checks whether staging/publishing/rules.yaml is correct +# as per the dependencies in the go.mod of the staging directories +# Usage: `hack/verify-publishing-bot.sh`. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" + +kube::golang::setup_env + +go -C "${KUBE_ROOT}/hack/tools" install ./publishing-verifier + +publishing-verifier "${KUBE_ROOT}"