From e266ab264928dc5dfe8604e495b8ba376a5098b0 Mon Sep 17 00:00:00 2001 From: Junjie Mao Date: Fri, 21 Jan 2022 18:01:12 +0800 Subject: [PATCH] config_tools: find and report counter examples on validation error In order for more effective and specific error reporting, this patch enhanced the XML assertion validation mechanism by: - Enhancing the elementpath library at runtime to capture of quantified variables that causes an assertion to fail, which can be used as a counter example of the rule. - Allowing error messages (as xs:documentation in the XML schema) embedding XPath (up to 2.0) which will be evaluated against the counter example to provide more specific information about the error. - Adding to assertions a mandatory attribute which specifies the context node(s) of an error using XPath. These nodes can be further used in the configurator to attach the errors to the widgets where users can fix them. v1 -> v2: * Logging the validation errors according to their defined severity Tracked-On: #6690 Signed-off-by: Junjie Mao --- .../scenario_config/elementpath_overlay.py | 102 ++++++++++++++++++ .../config_tools/scenario_config/validator.py | 100 ++++++++++++++++- 2 files changed, 197 insertions(+), 5 deletions(-) create mode 100644 misc/config_tools/scenario_config/elementpath_overlay.py diff --git a/misc/config_tools/scenario_config/elementpath_overlay.py b/misc/config_tools/scenario_config/elementpath_overlay.py new file mode 100644 index 000000000..d773bf587 --- /dev/null +++ b/misc/config_tools/scenario_config/elementpath_overlay.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# +# Copyright (C), 2022 Intel Corporation. +# Copyright (c), 2018-2021, SISSA (International School for Advanced Studies). +# +# SPDX-License-Identifier: BSD-3-Clause +# + +from decimal import Decimal +from copy import copy +import operator +import elementpath + +BaseParser = elementpath.XPath2Parser + +class CustomParser(BaseParser): + SYMBOLS = BaseParser.SYMBOLS | { + # Bit-wise operations + 'bitwise-and' + } + +method = CustomParser.method +function = CustomParser.function + +### +# Custom functions + +OPERATORS_MAP = { + 'bitwise-and': operator.and_ +} + +@method(function('bitwise-and', nargs=2)) +def evaluate(self, context=None): + def to_int(value): + if isinstance(value, int): + return value + elif isinstance(value, (float, Decimal)): + return int(value) + elif isinstance(value, str) and value.startswith("0x"): + return int(value, base=16) + else: + raise TypeError('invalid type {!r} for xs:{}'.format(type(value), cls.name)) + + def aux(op): + op1, op2 = self.get_operands(context) + if op1 is not None and op2 is not None: + try: + return op(to_int(op1), to_int(op2)) + except ValueError as err: + raise self.error('FORG0001', err) from None + except TypeError as err: + raise self.error('XPTY0004', err) + + return aux(OPERATORS_MAP[self.symbol]) + +### +# Collection of counter examples + +class Hashable: + def __init__(self, obj): + self.obj = obj + + def __hash__(self): + return id(self.obj) + +def copy_context(context): + ret = copy(context) + if hasattr(context, 'counter_example'): + ret.counter_example = dict() + return ret + +def add_counter_example(context, private_context, kvlist): + if hasattr(context, 'counter_example'): + context.counter_example.update(kvlist) + if private_context: + context.counter_example.update(private_context.counter_example) + +@method('every') +@method('some') +def evaluate(self, context=None): + if context is None: + raise self.missing_context() + + some = self.symbol == 'some' + varrefs = [Hashable(self[k]) for k in range(0, len(self) - 1, 2)] + varnames = [self[k][0].value for k in range(0, len(self) - 1, 2)] + selectors = [self[k].select for k in range(1, len(self) - 1, 2)] + + for results in copy(context).iter_product(selectors, varnames): + private_context = copy_context(context) + private_context.variables.update(x for x in zip(varnames, results)) + if self.boolean_value([x for x in self[-1].select(private_context)]): + if some: + add_counter_example(context, private_context, zip(varrefs, results)) + return True + elif not some: + add_counter_example(context, private_context, zip(varrefs, results)) + return False + + return not some + +elementpath.XPath2Parser = CustomParser diff --git a/misc/config_tools/scenario_config/validator.py b/misc/config_tools/scenario_config/validator.py index 8c3f5d171..9441287da 100755 --- a/misc/config_tools/scenario_config/validator.py +++ b/misc/config_tools/scenario_config/validator.py @@ -10,8 +10,12 @@ import argparse import logging from copy import copy from collections import namedtuple +import re try: + import elementpath + import elementpath_overlay + from elementpath.xpath_context import XPathContext import xmlschema except ImportError: logging.error("Python package `xmlschema` is not installed.\n" + @@ -43,12 +47,26 @@ def log_level_type(parser): return aux class ValidationError(dict): + logging_fns = { + "critical": logging.critical, + "error": logging.error, + "warning": logging.warning, + "info": logging.info, + "debug": logging.debug, + } + def __init__(self, paths, message, severity): super().__init__(paths = paths, message = message, severity = severity) def __str__(self): return f"{', '.join(self['paths'])}: {self['message']}" + def log(self): + try: + self.logging_fns[self['severity']](self) + except KeyError: + logging.debug(self) + class ScenarioValidator: def __init__(self, schema_etree, datachecks_etree): """Initialize the validator with preprocessed schemas in ElementTree.""" @@ -62,7 +80,7 @@ class ScenarioValidator: for error in it: # Syntactic errors are always critical. e = ValidationError([error.path], error.reason, "critical") - logging.debug(e) + e.log() errors.append(e) return errors @@ -71,16 +89,88 @@ class ScenarioValidator: errors = [] unified_node = copy(scenario_etree.getroot()) + parent_map = {c : p for p in unified_node.iter() for c in p} unified_node.extend(board_etree.getroot()) it = self.datachecks.iter_errors(unified_node) for error in it: - logging.debug(f"{error.elem}: {error.message}") - anno = error.validator.annotation - severity = anno.elem.get("{https://projectacrn.org}severity") - errors.append(ValidationError([error.elem.tag], error.message, severity)) + e = self.format_error(unified_node, parent_map, error) + e.log() + errors.append(e) return errors + @staticmethod + def format_paths(unified_node, parent_map, report_on, variables): + elems = elementpath.select(unified_node, report_on, variables = variables) + paths = [] + for elem in elems: + path = [] + while elem is not None: + path_segment = elem.tag + parent = parent_map.get(elem, None) + if parent is not None: + children = parent.findall(elem.tag) + if len(children) > 1: + path_segment += f"[{children.index(elem) + 1}]" + path.insert(0, path_segment) + elem = parent + paths.append(f"/{'/'.join(path)}") + return paths + + @staticmethod + def get_counter_example(error): + assertion = error.validator + if not isinstance(assertion, xmlschema.validators.assertions.XsdAssert): + return {} + + elem = error.obj + context = XPathContext(elem, variables={'value': None}) + context.counter_example = {} + result = assertion.token.evaluate(context) + + if result == False: + return context.counter_example + else: + return {} + + @staticmethod + def format_error(unified_node, parent_map, error): + def format_node(n): + if isinstance(n, str): + return n + elif isinstance(n, (int, float)): + return str(n) + elif isinstance(n, object) and n.__class__.__name__.endswith("Element"): + return n.text + else: + return str(n) + + anno = error.validator.annotation + counter_example = ScenarioValidator.get_counter_example(error) + variables = {k.obj.source.strip("$"): v for k,v in counter_example.items()} + + paths = ScenarioValidator.format_paths(unified_node, parent_map, anno.elem.get("{https://projectacrn.org}report-on"), variables) + description = anno.elem.find("{http://www.w3.org/2001/XMLSchema}documentation").text + severity = anno.elem.get("{https://projectacrn.org}severity") + + expr_regex = re.compile("{[^{}]*}") + exprs = set(expr_regex.findall(description)) + for expr in exprs: + result = elementpath.select(unified_node, expr.strip("{}"), variables = variables) + if isinstance(result, list): + if len(result) == 1: + value = format_node(result[0]) + elif len(result) > 1: + s = ', '.join(map(format_node, result)) + value = f"[{s}]" + else: + value = "{unknown}" + else: + value = str(result) + description = description.replace(expr, value) + + return ValidationError(paths, description, severity) + class ValidatorConstructionStage(PipelineStage): # The schema etree may still useful for schema-based transformation. Do not consume it. uses = {"schema_etree"}