From 5cfc724b9ea908eb8b444ca1366a51f4d15a3f2a Mon Sep 17 00:00:00 2001 From: shashidharatd Date: Wed, 9 Aug 2017 11:27:04 +0530 Subject: [PATCH] Simplify hack/verify-flags-underscore.py --- hack/verify-flags-underscore.py | 145 ++------------------------------ 1 file changed, 5 insertions(+), 140 deletions(-) diff --git a/hack/verify-flags-underscore.py b/hack/verify-flags-underscore.py index 073864b25f1..de9aa65699e 100755 --- a/hack/verify-flags-underscore.py +++ b/hack/verify-flags-underscore.py @@ -25,7 +25,6 @@ import argparse parser = argparse.ArgumentParser() parser.add_argument("filenames", help="list of files to check, all files if unspecified", nargs='*') -parser.add_argument("-e", "--skip-exceptions", help="ignore hack/verify-flags/exceptions.txt and print all output", action="store_true") args = parser.parse_args() # Cargo culted from http://stackoverflow.com/questions/898669/how-can-i-detect-if-a-file-is-binary-non-text-in-python @@ -81,77 +80,10 @@ def get_all_files(rootdir): all_files.append(pathname) return all_files -def normalize_files(rootdir, files): - newfiles = [] - a = ['Godeps', '_gopath', 'third_party', '.git', 'exceptions.txt', 'known-flags.txt'] - for f in files: - if any(x in f for x in a): - continue - if f.endswith(".svg"): - continue - if f.endswith(".gliffy"): - continue - if f.endswith(".md"): - continue - if f.endswith(".yaml"): - continue - newfiles.append(f) - for i, f in enumerate(newfiles): - if not os.path.isabs(f): - newfiles[i] = os.path.join(rootdir, f) - return newfiles - -def line_has_bad_flag(line, flagre): - results = flagre.findall(line) - for result in results: - if not "_" in result: - return False - # this should exclude many cases where jinja2 templates use kube flags - # as variables, except it uses _ for the variable name - if "{% set" + result + "= \"" in line: - return False - if "pillar[" + result + "]" in line: - return False - if "grains" + result in line: - return False - # something common in juju variables... - if "template_data[" + result + "]" in line: - return False - return True - return False - -def check_known_flags(rootdir): - pathname = os.path.join(rootdir, "hack/verify-flags/known-flags.txt") - f = open(pathname, 'r') - flags = set(f.read().splitlines()) - f.close() - - illegal_known_flags = set() - for flag in flags: - if len(flag) > 0: - if not "-" in flag: - illegal_known_flags.add(flag) - - if len(illegal_known_flags) != 0: - print("All flags in hack/verify-flags/known-flags.txt should contain character -, found these flags without -") - l = list(illegal_known_flags) - l.sort() - print("%s" % "\n".join(l)) - sys.exit(1) - - -# The list of files might not be the whole repo. If someone only changed a -# couple of files we don't want to run all of the golang files looking for -# flags. Instead load the list of flags from hack/verify-flags/known-flags.txt -# If running the golang files finds a new flag not in that file, return an -# error and tell the user to add the flag to the flag list. -def get_flags(rootdir, files): - # preload the 'known' flags - pathname = os.path.join(rootdir, "hack/verify-flags/known-flags.txt") - f = open(pathname, 'r') - flags = set(f.read().splitlines()) - f.close() - +# Collects all the flags used in golang files and verifies the flags do +# not contain underscore. If any flag needs to be excluded from this check, +# need to add that flag in hack/verify-flags/excluded-flags.txt. +def check_underscore_in_flags(rootdir, files): # preload the 'known' flags which don't follow the - standard pathname = os.path.join(rootdir, "hack/verify-flags/excluded-flags.txt") f = open(pathname, 'r') @@ -165,7 +97,6 @@ def get_flags(rootdir, files): re.compile('.Duration[P]?\("([^"]*)",[^,]+,[^)]+\)'), re.compile('.StringSlice[P]?\("([^"]*)",[^,]+,[^)]+\)') ] - new_flags = set() new_excluded_flags = set() # walk all the files looking for any flags being declared for pathname in files: @@ -182,10 +113,6 @@ def get_flags(rootdir, files): continue if "_" in flag: new_excluded_flags.add(flag) - if not "-" in flag: - continue - if flag not in flags: - new_flags.add(flag) if len(new_excluded_flags) != 0: print("Found a flag declared with an _ but which is not explicitly listed as a valid flag name in hack/verify-flags/excluded-flags.txt") print("Are you certain this flag should not have been declared with an - instead?") @@ -193,79 +120,17 @@ def get_flags(rootdir, files): l.sort() print("%s" % "\n".join(l)) sys.exit(1) - if len(new_flags) != 0: - print("Found flags with character - in golang files not in the list of known flags. Please add these to hack/verify-flags/known-flags.txt") - l = list(new_flags) - l.sort() - print("%s" % "\n".join(l)) - sys.exit(1) - return list(flags) - -def flags_to_re(flags): - """turn the list of all flags we found into a regex find both - and _ versions""" - dashRE = re.compile('[-_]') - flagREs = [] - for flag in flags: - # turn all flag names into regexs which will find both types - newre = dashRE.sub('[-_]', flag) - # only match if there is not a leading or trailing alphanumeric character - flagREs.append("[^\w${]" + newre + "[^\w]") - # turn that list of regex strings into a single large RE - flagRE = "|".join(flagREs) - flagRE = re.compile(flagRE) - return flagRE - -def load_exceptions(rootdir): - exceptions = set() - if args.skip_exceptions: - return exceptions - exception_filename = os.path.join(rootdir, "hack/verify-flags/exceptions.txt") - exception_file = open(exception_filename, 'r') - for exception in exception_file.read().splitlines(): - out = exception.split(":", 1) - if len(out) != 2: - print("Invalid line in exceptions file: %s" % exception) - continue - filename = out[0] - line = out[1] - exceptions.add((filename, line)) - return exceptions def main(): rootdir = os.path.dirname(__file__) + "/../" rootdir = os.path.abspath(rootdir) - exceptions = load_exceptions(rootdir) - if len(args.filenames) > 0: files = args.filenames else: files = get_all_files(rootdir) - files = normalize_files(rootdir, files) - check_known_flags(rootdir) - - flags = get_flags(rootdir, files) - flagRE = flags_to_re(flags) - - bad_lines = [] - # walk all the file looking for any flag that was declared and now has an _ - for pathname in files: - relname = os.path.relpath(pathname, rootdir) - f = open(pathname, 'r') - for line in f.read().splitlines(): - if line_has_bad_flag(line, flagRE): - if (relname, line) not in exceptions: - bad_lines.append((relname, line)) - f.close() - - if len(bad_lines) != 0: - if not args.skip_exceptions: - print("Found illegal 'flag' usage. If these are false negatives you should run `hack/verify-flags-underscore.py -e > hack/verify-flags/exceptions.txt` to update the list.") - bad_lines.sort() - for (relname, line) in bad_lines: - print("%s:%s" % (relname, line)) - return 1 + check_underscore_in_flags(rootdir, files) if __name__ == "__main__": sys.exit(main())