Merge pull request #31 from dbkinder/filter-warnings

doc: filter known issues

.gitignore

@@ -2,3 +2,5 @@ doxygen
 _build
 *.bak
 *.sav
+*.log
+*.warnings

.known-issues/README

@@ -0,0 +1,55 @@
+This directory contains configuration files to ignore errors found in
+the build and test process which are known to the developers and for
+now can be safely ignored.
+
+To use:
+
+ $ cd <build directory>
+ $ make SOMETHING >& result
+ $ scripts/filter-known-issues.py result
+
+It is included in the source tree so if anyone has to submit anything
+that triggers some kind of error that is a false positive, it can
+include the "ignore me" file, properly documented.
+
+Each file can contain one or more multiline Python regular expressions
+(https://docs.python.org/2/library/re.html#regular-expression-syntax)
+that match an error message. Multiple regular expressions are
+separated by comment blocks (that start with #). Note that an empty
+line still is considered part of the multiline regular expression.
+
+For example
+
+---beginning---
+#
+# This testcase always fails, pending fix ZEP-1234
+#
+.*/tests/kernel/grumpy .* FAIL
+#
+# Documentation issue, masks:
+#
+# /home/e/inaky/z/kernel.git/doc/api/io_interfaces.rst:28: WARNING: Invalid definition: Expected identifier in nested name. [error at 19]
+#  struct dev_config::@65  dev_config::bits
+#  -------------------^
+#
+^(?P<filename>.+/doc/api/io_interfaces.rst):(?P<lineno>[0-9]+): WARNING: Invalid definition: Expected identifier in nested name. \[error at [0-9]+]
+^\s+struct dev_config::@[0-9]+  dev_config::bits.*
+^\s+-+\^
+---end---
+
+Note you want to:
+
+- use relateive paths; instead of
+  /home/me/mydir/zephyr/something/somewhere.c you will want
+  ^.*/something/somewhere.c (as they will depend on where it is being
+  built)
+
+- Replace line numbers with [0-9]+, as they will change
+
+- (?P<filename>[-._/\w]+/something/somewhere.c) saves the match on
+  that file path in a "variable" called 'filename' that later you can
+  match with (?P=filename) if you want to match multiple lines of the
+  same error message.
+
+Can get really twisted and interesting in terms of regexps; they are
+powerful, so start small :)

.known-issues/doc/hypercall.conf

@@ -0,0 +1,37 @@
+#
+# Hypercall
+#
+#
+# include/net/net_ip.h warnings
+#
+^(?P<filename>[-._/\w]+/api/hypercall_api.rst):(?P<lineno>[0-9]+): WARNING: Invalid definition: Expected identifier in nested name. \[error at [0-9]+]
+^[ \t]*
+^[ \t]*\^
+^(?P=filename):(?P=lineno): WARNING: Invalid definition: Expected identifier in nested name. \[error at [0-9]+]
+^[ \t]*
+^[ \t]*\^
+^(?P=filename):(?P=lineno): WARNING: Invalid definition: Expected identifier in nested name. \[error at [0-9]+]
+^[ \t]*
+^[ \t]*\^
+^(?P=filename):(?P=lineno): WARNING: Invalid definition: Expected identifier in nested name. \[error at [0-9]+]
+^[ \t]*
+^[ \t]*\^
+^(?P=filename):(?P=lineno): WARNING: Invalid definition: Expected end of definition. \[error at [0-9]+]
+^.*vhm_request.reqs
+^[- \t]*\^
+#
+^(?P<filename>[-._/\w]+/api/hypercall_api.rst):(?P<lineno>[0-9]+): WARNING: Invalid definition: Expected identifier in nested name. \[error at [0-9]+]
+^[ \t]*
+^[ \t]*\^
+^(?P=filename):(?P=lineno): WARNING: Invalid definition: Expected identifier in nested name. \[error at [0-9]+]
+^[ \t]*
+^[ \t]*\^
+^(?P=filename):(?P=lineno): WARNING: Invalid definition: Expected end of definition. \[error at [0-9]+]
+^.*hc_ptdev_irq.is
+^[- \t]*\^
+^(?P=filename):(?P=lineno): WARNING: Invalid definition: Expected end of definition. \[error at [0-9]+]
+^.*hc_ptdev_irq.is.intx
+^[- \t]*\^
+^(?P=filename):(?P=lineno): WARNING: Invalid definition: Expected end of definition. \[error at [0-9]+]
+^.*hc_ptdev_irq.is.msix
+^[- \t]*\^

Makefile

@@ -23,7 +23,7 @@ help:
 .PHONY: help Makefile
 
 pullsource:
-	scripts/pullsource.sh
+	$(Q)scripts/pullsource.sh
 
 
 # Generate the doxygen xml (for Sphinx) and copy the doxygen html to the
@@ -32,6 +32,11 @@ pullsource:
 doxy: pullsource
 	$(Q)(cat acrn.doxyfile) | doxygen -  2>&1
 
+html: doxy
+	$(Q)$(SPHINXBUILD) -b html "$(SOURCEDIR)" "$(BUILDDIR)" "$(SPHINXOPTS)" $(O) > doc.log 2>&1
+	$(Q)./scripts/filter-doc-log.sh doc.log
+
+
 # Remove generated content (Sphinx and doxygen)
 
 clean:

scripts/filter-doc-log.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# run the filter-known-issues.py script to remove "expected" warning
+# messages from the output of the document build process and write
+# the filtered output to stdout
+#
+# Only argument is the name of the log file saved by the build.
+
+KI_SCRIPT=scripts/filter-known-issues.py
+CONFIG_DIR=.known-issues/doc
+
+LOG_FILE=$1
+
+if [ -z "${LOG_FILE}" ]; then
+        echo "Error in $0: missing input parameter <logfile>"
+        exit 1
+fi
+
+# When running in background, detached from terminal jobs, tput will
+# fail; we usually can tell because there is no TERM env variable.
+if [ -z "${TERM:-}" -o "${TERM:-}" = dumb ]; then
+    TPUT="true"
+    red=''
+    green=''
+else
+    TPUT="tput"
+    red='\E[31m'
+    green='\e[32m'
+fi
+
+if [ -s "${LOG_FILE}" ]; then
+   $KI_SCRIPT --config-dir ${CONFIG_DIR} ${LOG_FILE} > doc.warnings 2>&1
+   if [ -s doc.warnings ]; then
+	   echo
+	   echo -e "${red}New errors/warnings found, please fix them:"
+	   echo -e "=============================================="
+	   $TPUT sgr0
+	   echo
+	   cat doc.warnings
+	   echo
+   else
+	   echo -e "${green}No new errors/warnings."
+	   $TPUT sgr0
+   fi
+
+else
+   echo "Error in $0: logfile \"${LOG_FILE}\" not found."
+   exit 1
+fi

scripts/filter-known-issues.py

@@ -0,0 +1,254 @@
+#! /usr/bin/env python3
+"""
+Filters a file, classifying output in errors, warnings and discarding
+the rest.
+
+Given a set of regular expresions read from files named *.conf in the
+given configuration path(s), of the format:
+
+  #
+  # Comments for multiline regex 1...
+  #
+  MULTILINEPYTHONREGEX
+  MULTILINEPYTHONREGEX
+  MULTILINEPYTHONREGEX
+  #
+  # Comments for multiline regex 2...
+  #
+  #WARNING
+  MULTILINEPYTHONREGEX2
+  MULTILINEPYTHONREGEX2
+
+Anything matched by MULTILINEPYTHONREGEX will be considered something
+to be filtered out and not printed.
+
+Anything matched by MULTILINEPYHONREGEX with a #WARNING tag in the
+comment means (optional) means that it describes something that is
+considered to be a warning. Print it to stderr.
+
+Anything leftover is considred to be errors, printed to stdout.
+
+"""
+import argparse
+import logging
+import mmap
+import os
+import re
+import sre_constants
+import sys
+import traceback
+
+exclude_regexs = []
+
+# first is a list of one or more comment lines
+# followed by a list of non-comments which describe a multiline regex
+config_regex = \
+    b"(?P<comment>(^\s*#.*\n)+)" \
+    b"(?P<regex>(^[^#].*\n)+)"
+
+
+def config_import_file(filename):
+    """
+    Imports regular expresions from any file *.conf in the given path,
+    format as given in the main doc
+    """
+    try:
+        with open(filename, "rb") as f:
+            mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+            # That regex basically selects any block of
+            # lines that is not a comment block. The
+            # finditer() finds all the blocks and selects
+            # the bits of mmapped-file that comprises
+            # each--we compile it into a regex and append.
+            for m in re.finditer(config_regex, mm, re.MULTILINE):
+                origin = "%s:%s-%s" % (filename, m.start(), m.end())
+                gd = m.groupdict()
+                comment = gd['comment']
+                regex = gd['regex']
+                try:
+                    r = re.compile(regex, re.MULTILINE)
+                except sre_constants.error as e:
+                    logging.error("%s: bytes %d-%d: bad regex: %s",
+                                  filename, m.start(), m.end(), e)
+                    raise
+                logging.debug("%s: found regex at bytes %d-%d: %s",
+                              filename, m.start(), m.end(), regex)
+                if b'#WARNING' in comment:
+                    exclude_regexs.append((r, origin, ('warning',)))
+                else:
+                    exclude_regexs.append((r, origin, ()))
+            logging.debug("%s: loaded", filename)
+    except Exception as e:
+        logging.error("E: %s: can't load config file: %s" % (filename, e))
+        raise
+
+
+def config_import_path(path):
+    """
+    Imports regular expresions from any file *.conf in the given path
+    """
+    file_regex = re.compile(".*\.conf$")
+    try:
+        for dirpath, dirnames, filenames in os.walk(path):
+            for _filename in sorted(filenames):
+                filename = os.path.join(dirpath, _filename)
+                if not file_regex.search(_filename):
+                    logging.debug("%s: ignored", filename)
+                    continue
+                config_import_file(filename)
+    except Exception as e:
+        raise Exception(
+            "E: %s: can't load config files: %s %s" %
+            (path, e, traceback.format_exc()))
+
+
+def config_import(paths):
+    """
+    Imports regular expresions from any file *.conf in the list of paths.
+
+    If a path is "" or None, the list of paths until then is flushed
+    and only the new ones are considered.
+    """
+    _paths = []
+    # Go over the list, flush it if the user gave an empty path ("")
+    for path in paths:
+        if path == "" or path is None:
+            logging.debug("flushing current config list: %s", _paths)
+            _paths = []
+        else:
+            _paths.append(path)
+    logging.debug("config list: %s", _paths)
+    for path in _paths:
+        config_import_path(path)
+
+
+arg_parser = argparse.ArgumentParser(
+    description=__doc__,
+    formatter_class=argparse.RawDescriptionHelpFormatter)
+arg_parser.add_argument("-v", "--verbosity", action="count", default=0,
+                        help="increase verbosity")
+arg_parser.add_argument("-q", "--quiet", action="count", default=0,
+                        help="decrease verbosity")
+arg_parser.add_argument("-e", "--errors", action="store", default=None,
+                        help="file where to store errors")
+arg_parser.add_argument("-w", "--warnings", action="store", default=None,
+                        help="file where to store warnings")
+arg_parser.add_argument("-c", "--config-dir", action="append", nargs="?",
+                        default=[".known-issues/"],
+                        help="configuration directory (multiple can be "
+                        "given; if none given, clears the current list) "
+                        "%(default)s")
+arg_parser.add_argument("FILENAMEs", nargs="+",
+                        help="files to filter")
+args = arg_parser.parse_args()
+
+logging.basicConfig(level=40 - 10 * (args.verbosity - args.quiet),
+                    format="%(levelname)s: %(message)s")
+
+path = ".known-issues/"
+logging.debug("Reading configuration from directory `%s`", path)
+config_import(args.config_dir)
+
+exclude_ranges = []
+
+if args.warnings:
+    warnings = open(args.warnings, "w")
+else:
+    warnings = None
+if args.errors:
+    errors = open(args.errors, "w")
+else:
+    errors = None
+
+
+def report_error(data):
+    sys.stdout.write(data)
+    if errors:
+        errors.write(data)
+
+
+def report_warning(data):
+    sys.stderr.write(data)
+    if warnings:
+        warnings.write(data)
+
+
+if args.warnings:
+    warnings = open(args.warnings, "w")
+else:
+    warnings = None
+if args.errors:
+    errors = open(args.errors, "w")
+else:
+    errors = None
+
+
+def report_error(data):
+    sys.stdout.write(data.decode('utf-8'))
+    if errors:
+        errors.write(data)
+
+
+def report_warning(data):
+    sys.stderr.write(data)
+    if warnings:
+        warnings.write(data)
+
+
+for filename in args.FILENAMEs:
+    if os.stat(filename).st_size == 0:
+        continue  # skip empty log files
+    try:
+        with open(filename, "r+b") as f:
+            logging.info("%s: filtering", filename)
+            # Yeah, this should be more protected in case of exception
+            # and such, but this is a short running program...
+            mm = mmap.mmap(f.fileno(), 0)
+            for ex, origin, flags in exclude_regexs:
+                logging.info("%s: searching from %s: %s",
+                             filename, origin, ex.pattern)
+                for m in re.finditer(ex.pattern, mm, re.MULTILINE):
+                    logging.info("%s: %s-%s: match from from %s %s",
+                                 filename, m.start(), m.end(), origin, flags)
+                    if 'warning' in flags:
+                        exclude_ranges.append((m.start(), m.end(), True))
+                    else:
+                        exclude_ranges.append((m.start(), m.end(), False))
+
+            exclude_ranges = sorted(exclude_ranges, key=lambda r: r[0])
+            logging.warning(
+                "%s: ranges excluded: %s",
+                filename,
+                exclude_ranges)
+
+            # Decide what to do with what has been filtered; warnings
+            # go to stderr and warnings file, errors to stdout, what
+            # is ignored is just dumped.
+            offset = 0
+            for b, e, warning in exclude_ranges:
+                mm.seek(offset)
+                if b > offset:
+                    # We have something not caught by a filter, an error
+                    logging.info("%s: error range (%d, %d), from %d %dB",
+                                 filename, offset, b, offset, b - offset)
+                    report_error(mm.read(b - offset))
+                    mm.seek(b)
+                if warning == True:		# A warning, print it
+                    mm.seek(b)
+                    logging.info("%s: warning range (%d, %d), from %d %dB",
+                                 filename, b, e, offset, e - b)
+                    report_warning(mm.read(e - b))
+                else:				# Exclude, ignore it
+                    d = b - offset
+                    logging.info("%s: exclude range (%d, %d), from %d %dB",
+                                 filename, b, e, offset, d)
+                offset = e
+            mm.seek(offset)
+            if len(mm) != offset:
+                logging.info("%s: error final range from %d %dB",
+                             filename, offset, len(mm))
+                report_error(mm.read(len(mm) - offset - 1))
+            del mm
+    except Exception as e:
+        logging.error("%s: cannot load: %s", filename, e)
+        raise

scripts/pullsource.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+q="--quiet"
+
 # pull fresh copies of the ACRN source for use by doxygen
 
 if [ ! -d "../acrn-hypervisor" ]; then
@@ -11,15 +13,18 @@ if [ ! -d "../acrn-devicemodel" ]; then
   exit -1
 fi
 
+# Assumes origin (personal) and upstream (project) remote repos are
+# setup
+
 cd ../acrn-hypervisor
-git checkout master;
-git fetch upstream
-git merge upstream/master
-git push origin master
+git checkout $q master;
+git fetch $q upstream
+git merge $q upstream/master
+git push $q origin master
 
 
 cd ../acrn-devicemodel
-git checkout master;
-git fetch upstream
-git merge upstream/master
-git push origin master
+git checkout $q master;
+git fetch $q upstream
+git merge $q upstream/master
+git push $q origin master