config_tools: find and report counter examples on validation error

In order for more effective and specific error reporting, this patch
enhanced the XML assertion validation mechanism by:

  - Enhancing the elementpath library at runtime to capture of quantified
    variables that causes an assertion to fail, which can be used as a
    counter example of the rule.

  - Allowing error messages (as xs:documentation in the XML schema)
    embedding XPath (up to 2.0) which will be evaluated against the counter
    example to provide more specific information about the error.

  - Adding to assertions a mandatory attribute which specifies the context
    node(s) of an error using XPath. These nodes can be further used in the
    configurator to attach the errors to the widgets where users can fix
    them.

v1 -> v2:

  * Logging the validation errors according to their defined severity

Tracked-On: #6690
Signed-off-by: Junjie Mao <junjie.mao@intel.com>

misc/config_tools/scenario_config/elementpath_overlay.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+#
+# Copyright (C), 2022 Intel Corporation.
+# Copyright (c), 2018-2021, SISSA (International School for Advanced Studies).
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+from decimal import Decimal
+from copy import copy
+import operator
+import elementpath
+
+BaseParser = elementpath.XPath2Parser
+
+class CustomParser(BaseParser):
+    SYMBOLS = BaseParser.SYMBOLS | {
+        # Bit-wise operations
+        'bitwise-and'
+        }
+
+method = CustomParser.method
+function = CustomParser.function
+
+###
+# Custom functions
+
+OPERATORS_MAP = {
+    'bitwise-and': operator.and_
+}
+
+@method(function('bitwise-and', nargs=2))
+def evaluate(self, context=None):
+    def to_int(value):
+        if isinstance(value, int):
+            return value
+        elif isinstance(value, (float, Decimal)):
+            return int(value)
+        elif isinstance(value, str) and value.startswith("0x"):
+            return int(value, base=16)
+        else:
+            raise TypeError('invalid type {!r} for xs:{}'.format(type(value), cls.name))
+
+    def aux(op):
+        op1, op2 = self.get_operands(context)
+        if op1 is not None and op2 is not None:
+            try:
+                return op(to_int(op1), to_int(op2))
+            except ValueError as err:
+                raise self.error('FORG0001', err) from None
+            except TypeError as err:
+                raise self.error('XPTY0004', err)
+
+    return aux(OPERATORS_MAP[self.symbol])
+
+###
+# Collection of counter examples
+
+class Hashable:
+    def __init__(self, obj):
+        self.obj = obj
+
+    def __hash__(self):
+        return id(self.obj)
+
+def copy_context(context):
+    ret = copy(context)
+    if hasattr(context, 'counter_example'):
+        ret.counter_example = dict()
+    return ret
+
+def add_counter_example(context, private_context, kvlist):
+    if hasattr(context, 'counter_example'):
+        context.counter_example.update(kvlist)
+        if private_context:
+            context.counter_example.update(private_context.counter_example)
+
+@method('every')
+@method('some')
+def evaluate(self, context=None):
+    if context is None:
+        raise self.missing_context()
+
+    some = self.symbol == 'some'
+    varrefs = [Hashable(self[k]) for k in range(0, len(self) - 1, 2)]
+    varnames = [self[k][0].value for k in range(0, len(self) - 1, 2)]
+    selectors = [self[k].select for k in range(1, len(self) - 1, 2)]
+
+    for results in copy(context).iter_product(selectors, varnames):
+        private_context = copy_context(context)
+        private_context.variables.update(x for x in zip(varnames, results))
+        if self.boolean_value([x for x in self[-1].select(private_context)]):
+            if some:
+                add_counter_example(context, private_context, zip(varrefs, results))
+                return True
+        elif not some:
+            add_counter_example(context, private_context, zip(varrefs, results))
+            return False
+
+    return not some
+
+elementpath.XPath2Parser = CustomParser

misc/config_tools/scenario_config/validator.py

@@ -10,8 +10,12 @@ import argparse
 import logging
 from copy import copy
 from collections import namedtuple
+import re
 
 try:
+    import elementpath
+    import elementpath_overlay
+    from elementpath.xpath_context import XPathContext
     import xmlschema
 except ImportError:
     logging.error("Python package `xmlschema` is not installed.\n" +
@@ -43,12 +47,26 @@ def log_level_type(parser):
     return aux
 
 class ValidationError(dict):
+    logging_fns = {
+        "critical": logging.critical,
+        "error": logging.error,
+        "warning": logging.warning,
+        "info": logging.info,
+        "debug": logging.debug,
+    }
+
     def __init__(self, paths, message, severity):
         super().__init__(paths = paths, message = message, severity = severity)
 
     def __str__(self):
         return f"{', '.join(self['paths'])}: {self['message']}"
 
+    def log(self):
+        try:
+            self.logging_fns[self['severity']](self)
+        except KeyError:
+            logging.debug(self)
+
 class ScenarioValidator:
     def __init__(self, schema_etree, datachecks_etree):
         """Initialize the validator with preprocessed schemas in ElementTree."""
@@ -62,7 +80,7 @@ class ScenarioValidator:
         for error in it:
             # Syntactic errors are always critical.
             e = ValidationError([error.path], error.reason, "critical")
-            logging.debug(e)
+            e.log()
             errors.append(e)
 
         return errors
@@ -71,16 +89,88 @@ class ScenarioValidator:
         errors = []
 
         unified_node = copy(scenario_etree.getroot())
+        parent_map = {c : p for p in unified_node.iter() for c in p}
         unified_node.extend(board_etree.getroot())
         it = self.datachecks.iter_errors(unified_node)
         for error in it:
-            logging.debug(f"{error.elem}: {error.message}")
+            e = self.format_error(unified_node, parent_map, error)
-            anno = error.validator.annotation
+            e.log()
-            severity = anno.elem.get("{https://projectacrn.org}severity")
+            errors.append(e)
-            errors.append(ValidationError([error.elem.tag], error.message, severity))
 
         return errors
 
+    @staticmethod
+    def format_paths(unified_node, parent_map, report_on, variables):
+        elems = elementpath.select(unified_node, report_on, variables = variables)
+        paths = []
+        for elem in elems:
+            path = []
+            while elem is not None:
+                path_segment = elem.tag
+                parent = parent_map.get(elem, None)
+                if parent is not None:
+                    children = parent.findall(elem.tag)
+                    if len(children) > 1:
+                        path_segment += f"[{children.index(elem) + 1}]"
+                path.insert(0, path_segment)
+                elem = parent
+            paths.append(f"/{'/'.join(path)}")
+        return paths
+
+    @staticmethod
+    def get_counter_example(error):
+        assertion = error.validator
+        if not isinstance(assertion, xmlschema.validators.assertions.XsdAssert):
+            return {}
+
+        elem = error.obj
+        context = XPathContext(elem, variables={'value': None})
+        context.counter_example = {}
+        result = assertion.token.evaluate(context)
+
+        if result == False:
+            return context.counter_example
+        else:
+            return {}
+
+    @staticmethod
+    def format_error(unified_node, parent_map, error):
+        def format_node(n):
+            if isinstance(n, str):
+                return n
+            elif isinstance(n, (int, float)):
+                return str(n)
+            elif isinstance(n, object) and n.__class__.__name__.endswith("Element"):
+                return n.text
+            else:
+                return str(n)
+
+        anno = error.validator.annotation
+        counter_example = ScenarioValidator.get_counter_example(error)
+        variables = {k.obj.source.strip("$"): v for k,v in counter_example.items()}
+
+        paths = ScenarioValidator.format_paths(unified_node, parent_map, anno.elem.get("{https://projectacrn.org}report-on"), variables)
+        description = anno.elem.find("{http://www.w3.org/2001/XMLSchema}documentation").text
+        severity = anno.elem.get("{https://projectacrn.org}severity")
+
+        expr_regex = re.compile("{[^{}]*}")
+        exprs = set(expr_regex.findall(description))
+        for expr in exprs:
+            result = elementpath.select(unified_node, expr.strip("{}"), variables = variables)
+            if isinstance(result, list):
+                if len(result) == 1:
+                    value = format_node(result[0])
+                elif len(result) > 1:
+                    s = ', '.join(map(format_node, result))
+                    value = f"[{s}]"
+                else:
+                    value = "{unknown}"
+            else:
+                value = str(result)
+            description = description.replace(expr, value)
+
+        return ValidationError(paths, description, severity)
+
 class ValidatorConstructionStage(PipelineStage):
     # The schema etree may still useful for schema-based transformation. Do not consume it.
     uses = {"schema_etree"}