config_tools: add a script that focuses on XML validation

Today the XML validation logic is embedded in scenario_cfg_gen.py which is
highly entangled with the Python-internal representation of
configurations. Such representation is used by the current configurator,
but will soon be obsolete when the new configurator is introduced.

In order to avoid unnecessary work on this internal representation when we
refine the schema of scenario XML files, this patch separates the
validation logic into a new script which can either be used from the
command line or imported in other Python-based applications. At build time
this script will be used instead to validate the XML files given by users.

This change makes it easier to refine the current configuration items for
better developer experience.

Migration of existing checks in scenario_cfg_gen.py to XML schema will be
done by a following series.

v2 -> v3:
 * Keep Invoking asl_gen.py to generate vACPI tables for pre-launched VMs.

v1 -> v2:
 * Remove "all rights reserved" from the license header
 * Upgrade the severity of the message indicating lack of xmlschema as
   error according to our latest definitions of log severities, as
   validation violations could indicate build time or boot time failures.

Tracked-On: #6690
Signed-off-by: Junjie Mao <junjie.mao@intel.com>

hypervisor/scripts/genconf.sh

@@ -22,8 +22,9 @@ apply_patch () {
 tool_dir=${base_dir}/../misc/config_tools
 diffconfig_list=${out}/.diffconfig
 
+python3 ${tool_dir}/scenario_config/validator.py ${board_xml} ${scenario_xml} &&
 python3 ${tool_dir}/board_config/board_cfg_gen.py --board ${board_xml} --scenario ${scenario_xml} --out ${out} &&
-python3 ${tool_dir}/scenario_config/scenario_cfg_gen.py --board ${board_xml} --scenario ${scenario_xml} --out ${out}
+python3 ${tool_dir}/acpi_gen/asl_gen.py --board ${board_xml} --scenario ${scenario_xml} --out ${out}
 
 if [ $? -ne 0 ]; then
     exit $?

misc/config_tools/scenario_config/validator.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2022 Intel Corporation.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+import sys, os
+import argparse
+import lxml.etree as etree
+import logging
+
+try:
+    import xmlschema
+except ImportError:
+    logging.error("Python package `xmlschema` is not installed.\n" +
+                  "The scenario XML file will NOT be validated against the schema, which may cause build-time or runtime errors.\n" +
+                  "To enable the validation, install the python package by executing: pip3 install xmlschema.")
+    sys.exit(0)
+
+from default_populator import get_node, populate
+
+def existing_file_type(parser):
+    def aux(arg):
+        if not os.path.exists(arg):
+            parser.error(f"can't open {arg}: No such file or directory")
+        elif not os.path.isfile(arg):
+            parser.error(f"can't open {arg}: Is not a file")
+        else:
+            return arg
+    return aux
+
+def log_level_type(parser):
+    def aux(arg):
+        arg = arg.lower()
+        if arg in ["critical", "error", "warning", "info", "debug"]:
+            return arg
+        else:
+            parser.error(f"{arg} is not a valid log level")
+    return aux
+
+def load_schema(xsd_xml, datachecks_xml):
+    global schema, schema_etree, schema_root, datachecks
+
+    schema_etree = etree.parse(xsd_xml)
+    schema_etree.xinclude()
+    schema = xmlschema.XMLSchema11(etree.tostring(schema_etree, encoding="unicode"))
+    schema_root = get_node(schema_etree, f"/xs:schema/xs:element")
+
+    datachecks_etree = etree.parse(datachecks_xml)
+    datachecks_etree.xinclude()
+    datachecks = xmlschema.XMLSchema11(etree.tostring(datachecks_etree, encoding="unicode"))
+
+config_tools_dir = os.path.join(os.path.dirname(__file__), "..")
+schema_dir = os.path.join(config_tools_dir, "schema")
+schema = None
+schema_etree = None
+schema_root = None
+datachecks = None
+load_schema(os.path.join(schema_dir, "config.xsd"), os.path.join(schema_dir, "datachecks.xsd"))
+
+def validate_one(board_xml, scenario_xml):
+    nr_schema_errors = 0
+    nr_check_errors = 0
+    nr_check_warnings = 0
+    board_name = os.path.basename(board_xml)
+    scenario_name = os.path.basename(scenario_xml)
+
+    scenario_etree = etree.parse(scenario_xml, etree.XMLParser(remove_blank_text=True))
+    populate(schema_etree, schema_root, scenario_etree.getroot(), False)
+
+    it = schema.iter_errors(scenario_etree)
+    for error in it:
+        logging.debug(error)
+        nr_schema_errors += 1
+
+    if nr_schema_errors == 0:
+        main_etree = etree.parse(board_xml)
+        main_etree.getroot().extend(scenario_etree.getroot()[:])
+
+        it = datachecks.iter_errors(main_etree)
+        for error in it:
+            logging.debug(error)
+
+            anno = error.validator.annotation
+            severity = anno.elem.get("{https://projectacrn.org}severity")
+
+            if severity == "error":
+                nr_check_errors += 1
+            elif severity == "warning":
+                nr_check_warnings += 1
+
+        if nr_check_errors > 0:
+            logging.error(f"Board {board_name} and scenario {scenario_name} have inconsistent data: {nr_check_errors} errors, {nr_check_warnings} warnings.")
+        elif nr_check_warnings > 0:
+            logging.warning(f"Board {board_name} and scenario {scenario_name} have inconsistent data: {nr_check_warnings} warnings.")
+        else:
+            logging.info(f"Board {board_name} and scenario {scenario_name} are valid and consistent.")
+    else:
+        logging.warning(f"Scenario {scenario_name} is invalid: {nr_schema_errors} schema errors.")
+
+    return nr_schema_errors + nr_check_errors + nr_check_warnings
+
+def validate_board(board_xml):
+    board_dir = os.path.dirname(board_xml)
+    nr_violations = 0
+
+    for f in os.listdir(board_dir):
+        if not f.endswith(".xml"):
+            continue
+        if f == os.path.basename(board_xml) or "launch" in f:
+            continue
+
+        nr_violations += validate_one(board_xml, os.path.join(board_dir, f))
+
+    return nr_violations
+
+def validate_all(data_dir):
+    nr_violations = 0
+
+    for f in os.listdir(data_dir):
+        board_xml = os.path.join(data_dir, f, f"{f}.xml")
+        if os.path.isfile(board_xml):
+            nr_violations += validate_board(board_xml)
+        else:
+            logging.warning(f"Cannot find a board XML under {os.path.join(data_dir, f)}")
+
+    return nr_violations
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("board", nargs="?", type=existing_file_type(parser), help="the board XML file to be validated")
+    parser.add_argument("scenario", nargs="?", type=existing_file_type(parser), help="the scenario XML file to be validated")
+    parser.add_argument("--loglevel", default="warning", type=log_level_type(parser), help="choose log level, e.g. debug, info, warning or error")
+    args = parser.parse_args()
+
+    logging.basicConfig(level=args.loglevel.upper())
+
+    if args.board and args.scenario:
+        nr_violations = validate_one(args.board, args.scenario)
+    elif args.board:
+        nr_violations = validate_board(args.board)
+    else:
+        nr_violations = validate_all(os.path.join(config_tools_dir, "data"))
+
+    sys.exit(1 if nr_violations > 0 else 0)