From 561562dd761c8268eed83a2378dbeb4f43d257e9 Mon Sep 17 00:00:00 2001 From: "David B. Kinder" Date: Tue, 13 Mar 2018 16:24:35 -0700 Subject: [PATCH] doc: filter known issues make the doc build process quiet and add filtering of known (Sphinx) issues. Scripting comes from the open source Zephyr project. Signed-off-by: David B. Kinder --- .gitignore | 2 + .known-issues/README | 55 +++++++ .known-issues/doc/hypercall.conf | 37 +++++ Makefile | 7 +- scripts/filter-doc-log.sh | 49 ++++++ scripts/filter-known-issues.py | 254 +++++++++++++++++++++++++++++++ scripts/pullsource.sh | 21 ++- 7 files changed, 416 insertions(+), 9 deletions(-) create mode 100644 .known-issues/README create mode 100644 .known-issues/doc/hypercall.conf create mode 100755 scripts/filter-doc-log.sh create mode 100755 scripts/filter-known-issues.py diff --git a/.gitignore b/.gitignore index f670f4ead..eaadf6c04 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ doxygen _build *.bak *.sav +*.log +*.warnings diff --git a/.known-issues/README b/.known-issues/README new file mode 100644 index 000000000..9875d10ce --- /dev/null +++ b/.known-issues/README @@ -0,0 +1,55 @@ +This directory contains configuration files to ignore errors found in +the build and test process which are known to the developers and for +now can be safely ignored. + +To use: + + $ cd + $ make SOMETHING >& result + $ scripts/filter-known-issues.py result + +It is included in the source tree so if anyone has to submit anything +that triggers some kind of error that is a false positive, it can +include the "ignore me" file, properly documented. + +Each file can contain one or more multiline Python regular expressions +(https://docs.python.org/2/library/re.html#regular-expression-syntax) +that match an error message. Multiple regular expressions are +separated by comment blocks (that start with #). Note that an empty +line still is considered part of the multiline regular expression. + +For example + +---beginning--- +# +# This testcase always fails, pending fix ZEP-1234 +# +.*/tests/kernel/grumpy .* FAIL +# +# Documentation issue, masks: +# +# /home/e/inaky/z/kernel.git/doc/api/io_interfaces.rst:28: WARNING: Invalid definition: Expected identifier in nested name. [error at 19] +# struct dev_config::@65 dev_config::bits +# -------------------^ +# +^(?P.+/doc/api/io_interfaces.rst):(?P[0-9]+): WARNING: Invalid definition: Expected identifier in nested name. \[error at [0-9]+] +^\s+struct dev_config::@[0-9]+ dev_config::bits.* +^\s+-+\^ +---end--- + +Note you want to: + +- use relateive paths; instead of + /home/me/mydir/zephyr/something/somewhere.c you will want + ^.*/something/somewhere.c (as they will depend on where it is being + built) + +- Replace line numbers with [0-9]+, as they will change + +- (?P[-._/\w]+/something/somewhere.c) saves the match on + that file path in a "variable" called 'filename' that later you can + match with (?P=filename) if you want to match multiple lines of the + same error message. + +Can get really twisted and interesting in terms of regexps; they are +powerful, so start small :) diff --git a/.known-issues/doc/hypercall.conf b/.known-issues/doc/hypercall.conf new file mode 100644 index 000000000..b75bbe3de --- /dev/null +++ b/.known-issues/doc/hypercall.conf @@ -0,0 +1,37 @@ +# +# Hypercall +# +# +# include/net/net_ip.h warnings +# +^(?P[-._/\w]+/api/hypercall_api.rst):(?P[0-9]+): WARNING: Invalid definition: Expected identifier in nested name. \[error at [0-9]+] +^[ \t]* +^[ \t]*\^ +^(?P=filename):(?P=lineno): WARNING: Invalid definition: Expected identifier in nested name. \[error at [0-9]+] +^[ \t]* +^[ \t]*\^ +^(?P=filename):(?P=lineno): WARNING: Invalid definition: Expected identifier in nested name. \[error at [0-9]+] +^[ \t]* +^[ \t]*\^ +^(?P=filename):(?P=lineno): WARNING: Invalid definition: Expected identifier in nested name. \[error at [0-9]+] +^[ \t]* +^[ \t]*\^ +^(?P=filename):(?P=lineno): WARNING: Invalid definition: Expected end of definition. \[error at [0-9]+] +^.*vhm_request.reqs +^[- \t]*\^ +# +^(?P[-._/\w]+/api/hypercall_api.rst):(?P[0-9]+): WARNING: Invalid definition: Expected identifier in nested name. \[error at [0-9]+] +^[ \t]* +^[ \t]*\^ +^(?P=filename):(?P=lineno): WARNING: Invalid definition: Expected identifier in nested name. \[error at [0-9]+] +^[ \t]* +^[ \t]*\^ +^(?P=filename):(?P=lineno): WARNING: Invalid definition: Expected end of definition. \[error at [0-9]+] +^.*hc_ptdev_irq.is +^[- \t]*\^ +^(?P=filename):(?P=lineno): WARNING: Invalid definition: Expected end of definition. \[error at [0-9]+] +^.*hc_ptdev_irq.is.intx +^[- \t]*\^ +^(?P=filename):(?P=lineno): WARNING: Invalid definition: Expected end of definition. \[error at [0-9]+] +^.*hc_ptdev_irq.is.msix +^[- \t]*\^ diff --git a/Makefile b/Makefile index 516b1fa98..baf8db208 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ help: .PHONY: help Makefile pullsource: - scripts/pullsource.sh + $(Q)scripts/pullsource.sh # Generate the doxygen xml (for Sphinx) and copy the doxygen html to the @@ -32,6 +32,11 @@ pullsource: doxy: pullsource $(Q)(cat acrn.doxyfile) | doxygen - 2>&1 +html: doxy + $(Q)$(SPHINXBUILD) -b html "$(SOURCEDIR)" "$(BUILDDIR)" "$(SPHINXOPTS)" $(O) > doc.log 2>&1 + $(Q)./scripts/filter-doc-log.sh doc.log + + # Remove generated content (Sphinx and doxygen) clean: diff --git a/scripts/filter-doc-log.sh b/scripts/filter-doc-log.sh new file mode 100755 index 000000000..0fea74456 --- /dev/null +++ b/scripts/filter-doc-log.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# run the filter-known-issues.py script to remove "expected" warning +# messages from the output of the document build process and write +# the filtered output to stdout +# +# Only argument is the name of the log file saved by the build. + +KI_SCRIPT=scripts/filter-known-issues.py +CONFIG_DIR=.known-issues/doc + +LOG_FILE=$1 + +if [ -z "${LOG_FILE}" ]; then + echo "Error in $0: missing input parameter " + exit 1 +fi + +# When running in background, detached from terminal jobs, tput will +# fail; we usually can tell because there is no TERM env variable. +if [ -z "${TERM:-}" -o "${TERM:-}" = dumb ]; then + TPUT="true" + red='' + green='' +else + TPUT="tput" + red='\E[31m' + green='\e[32m' +fi + +if [ -s "${LOG_FILE}" ]; then + $KI_SCRIPT --config-dir ${CONFIG_DIR} ${LOG_FILE} > doc.warnings 2>&1 + if [ -s doc.warnings ]; then + echo + echo -e "${red}New errors/warnings found, please fix them:" + echo -e "==============================================" + $TPUT sgr0 + echo + cat doc.warnings + echo + else + echo -e "${green}No new errors/warnings." + $TPUT sgr0 + fi + +else + echo "Error in $0: logfile \"${LOG_FILE}\" not found." + exit 1 +fi diff --git a/scripts/filter-known-issues.py b/scripts/filter-known-issues.py new file mode 100755 index 000000000..0f020e57c --- /dev/null +++ b/scripts/filter-known-issues.py @@ -0,0 +1,254 @@ +#! /usr/bin/env python3 +""" +Filters a file, classifying output in errors, warnings and discarding +the rest. + +Given a set of regular expresions read from files named *.conf in the +given configuration path(s), of the format: + + # + # Comments for multiline regex 1... + # + MULTILINEPYTHONREGEX + MULTILINEPYTHONREGEX + MULTILINEPYTHONREGEX + # + # Comments for multiline regex 2... + # + #WARNING + MULTILINEPYTHONREGEX2 + MULTILINEPYTHONREGEX2 + +Anything matched by MULTILINEPYTHONREGEX will be considered something +to be filtered out and not printed. + +Anything matched by MULTILINEPYHONREGEX with a #WARNING tag in the +comment means (optional) means that it describes something that is +considered to be a warning. Print it to stderr. + +Anything leftover is considred to be errors, printed to stdout. + +""" +import argparse +import logging +import mmap +import os +import re +import sre_constants +import sys +import traceback + +exclude_regexs = [] + +# first is a list of one or more comment lines +# followed by a list of non-comments which describe a multiline regex +config_regex = \ + b"(?P(^\s*#.*\n)+)" \ + b"(?P(^[^#].*\n)+)" + + +def config_import_file(filename): + """ + Imports regular expresions from any file *.conf in the given path, + format as given in the main doc + """ + try: + with open(filename, "rb") as f: + mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + # That regex basically selects any block of + # lines that is not a comment block. The + # finditer() finds all the blocks and selects + # the bits of mmapped-file that comprises + # each--we compile it into a regex and append. + for m in re.finditer(config_regex, mm, re.MULTILINE): + origin = "%s:%s-%s" % (filename, m.start(), m.end()) + gd = m.groupdict() + comment = gd['comment'] + regex = gd['regex'] + try: + r = re.compile(regex, re.MULTILINE) + except sre_constants.error as e: + logging.error("%s: bytes %d-%d: bad regex: %s", + filename, m.start(), m.end(), e) + raise + logging.debug("%s: found regex at bytes %d-%d: %s", + filename, m.start(), m.end(), regex) + if b'#WARNING' in comment: + exclude_regexs.append((r, origin, ('warning',))) + else: + exclude_regexs.append((r, origin, ())) + logging.debug("%s: loaded", filename) + except Exception as e: + logging.error("E: %s: can't load config file: %s" % (filename, e)) + raise + + +def config_import_path(path): + """ + Imports regular expresions from any file *.conf in the given path + """ + file_regex = re.compile(".*\.conf$") + try: + for dirpath, dirnames, filenames in os.walk(path): + for _filename in sorted(filenames): + filename = os.path.join(dirpath, _filename) + if not file_regex.search(_filename): + logging.debug("%s: ignored", filename) + continue + config_import_file(filename) + except Exception as e: + raise Exception( + "E: %s: can't load config files: %s %s" % + (path, e, traceback.format_exc())) + + +def config_import(paths): + """ + Imports regular expresions from any file *.conf in the list of paths. + + If a path is "" or None, the list of paths until then is flushed + and only the new ones are considered. + """ + _paths = [] + # Go over the list, flush it if the user gave an empty path ("") + for path in paths: + if path == "" or path is None: + logging.debug("flushing current config list: %s", _paths) + _paths = [] + else: + _paths.append(path) + logging.debug("config list: %s", _paths) + for path in _paths: + config_import_path(path) + + +arg_parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) +arg_parser.add_argument("-v", "--verbosity", action="count", default=0, + help="increase verbosity") +arg_parser.add_argument("-q", "--quiet", action="count", default=0, + help="decrease verbosity") +arg_parser.add_argument("-e", "--errors", action="store", default=None, + help="file where to store errors") +arg_parser.add_argument("-w", "--warnings", action="store", default=None, + help="file where to store warnings") +arg_parser.add_argument("-c", "--config-dir", action="append", nargs="?", + default=[".known-issues/"], + help="configuration directory (multiple can be " + "given; if none given, clears the current list) " + "%(default)s") +arg_parser.add_argument("FILENAMEs", nargs="+", + help="files to filter") +args = arg_parser.parse_args() + +logging.basicConfig(level=40 - 10 * (args.verbosity - args.quiet), + format="%(levelname)s: %(message)s") + +path = ".known-issues/" +logging.debug("Reading configuration from directory `%s`", path) +config_import(args.config_dir) + +exclude_ranges = [] + +if args.warnings: + warnings = open(args.warnings, "w") +else: + warnings = None +if args.errors: + errors = open(args.errors, "w") +else: + errors = None + + +def report_error(data): + sys.stdout.write(data) + if errors: + errors.write(data) + + +def report_warning(data): + sys.stderr.write(data) + if warnings: + warnings.write(data) + + +if args.warnings: + warnings = open(args.warnings, "w") +else: + warnings = None +if args.errors: + errors = open(args.errors, "w") +else: + errors = None + + +def report_error(data): + sys.stdout.write(data.decode('utf-8')) + if errors: + errors.write(data) + + +def report_warning(data): + sys.stderr.write(data) + if warnings: + warnings.write(data) + + +for filename in args.FILENAMEs: + if os.stat(filename).st_size == 0: + continue # skip empty log files + try: + with open(filename, "r+b") as f: + logging.info("%s: filtering", filename) + # Yeah, this should be more protected in case of exception + # and such, but this is a short running program... + mm = mmap.mmap(f.fileno(), 0) + for ex, origin, flags in exclude_regexs: + logging.info("%s: searching from %s: %s", + filename, origin, ex.pattern) + for m in re.finditer(ex.pattern, mm, re.MULTILINE): + logging.info("%s: %s-%s: match from from %s %s", + filename, m.start(), m.end(), origin, flags) + if 'warning' in flags: + exclude_ranges.append((m.start(), m.end(), True)) + else: + exclude_ranges.append((m.start(), m.end(), False)) + + exclude_ranges = sorted(exclude_ranges, key=lambda r: r[0]) + logging.warning( + "%s: ranges excluded: %s", + filename, + exclude_ranges) + + # Decide what to do with what has been filtered; warnings + # go to stderr and warnings file, errors to stdout, what + # is ignored is just dumped. + offset = 0 + for b, e, warning in exclude_ranges: + mm.seek(offset) + if b > offset: + # We have something not caught by a filter, an error + logging.info("%s: error range (%d, %d), from %d %dB", + filename, offset, b, offset, b - offset) + report_error(mm.read(b - offset)) + mm.seek(b) + if warning == True: # A warning, print it + mm.seek(b) + logging.info("%s: warning range (%d, %d), from %d %dB", + filename, b, e, offset, e - b) + report_warning(mm.read(e - b)) + else: # Exclude, ignore it + d = b - offset + logging.info("%s: exclude range (%d, %d), from %d %dB", + filename, b, e, offset, d) + offset = e + mm.seek(offset) + if len(mm) != offset: + logging.info("%s: error final range from %d %dB", + filename, offset, len(mm)) + report_error(mm.read(len(mm) - offset - 1)) + del mm + except Exception as e: + logging.error("%s: cannot load: %s", filename, e) + raise diff --git a/scripts/pullsource.sh b/scripts/pullsource.sh index 8a6c58920..6cb78cc82 100755 --- a/scripts/pullsource.sh +++ b/scripts/pullsource.sh @@ -1,5 +1,7 @@ #!/bin/bash +q="--quiet" + # pull fresh copies of the ACRN source for use by doxygen if [ ! -d "../acrn-hypervisor" ]; then @@ -11,15 +13,18 @@ if [ ! -d "../acrn-devicemodel" ]; then exit -1 fi +# Assumes origin (personal) and upstream (project) remote repos are +# setup + cd ../acrn-hypervisor -git checkout master; -git fetch upstream -git merge upstream/master -git push origin master +git checkout $q master; +git fetch $q upstream +git merge $q upstream/master +git push $q origin master cd ../acrn-devicemodel -git checkout master; -git fetch upstream -git merge upstream/master -git push origin master +git checkout $q master; +git fetch $q upstream +git merge $q upstream/master +git push $q origin master