From 22a47fe7952204eb777a3c6b859a727ef22b69f9 Mon Sep 17 00:00:00 2001 From: Junjie Mao Date: Wed, 18 May 2022 13:44:13 +0800 Subject: [PATCH] config_tools: check XML file structures on load This patch validates the structure of the XML files given before they are further used by the rest of the configurator. With this validation process, the configurator confirms the XML files are well-structured and can thus access certain nodes and contents without checking their existence or data types. Upon validation failure, an alert will pop up informing the user that the given XML file is ill-formed. No further details are given as of now because we assume users should not care about the internal structure of those files. Tracked-On: #6691 Signed-off-by: Junjie Mao --- .../packages/configurator/src/lib/acrn.ts | 38 +++++++++---- .../configurator/src/pages/Config/Board.vue | 2 +- .../src/pages/Config/Scenario.vue | 2 +- .../configurator/pyodide/pyodide.py | 1 + .../configurator/pyodide/tests.py | 4 ++ .../pyodide/validateBoardStructure.py | 51 ++++++++++++++++++ .../pyodide/validateScenarioStructure.py | 53 +++++++++++++++++++ misc/config_tools/scenario_config/pipeline.py | 2 +- .../config_tools/scenario_config/validator.py | 26 +++++---- misc/config_tools/schema/board.xsd | 45 ++++++++++++++++ 10 files changed, 201 insertions(+), 23 deletions(-) create mode 100644 misc/config_tools/configurator/pyodide/validateBoardStructure.py create mode 100644 misc/config_tools/configurator/pyodide/validateScenarioStructure.py create mode 100644 misc/config_tools/schema/board.xsd diff --git a/misc/config_tools/configurator/packages/configurator/src/lib/acrn.ts b/misc/config_tools/configurator/packages/configurator/src/lib/acrn.ts index 0b37dce9f..9c5501831 100644 --- a/misc/config_tools/configurator/packages/configurator/src/lib/acrn.ts +++ b/misc/config_tools/configurator/packages/configurator/src/lib/acrn.ts @@ -11,31 +11,43 @@ enum HistoryType { export type HistoryTypeString = keyof typeof HistoryType; class PythonObject { - api(scriptName, ...params) { + api(scriptName, output_format, ...params) { // @ts-ignore let pythonFunction = window.pyodide.pyimport(`configurator.pyodide.${scriptName}`); let result = pythonFunction.main(...params); - return JSON.parse(result); + if (output_format === 'json') { + return JSON.parse(result); + } else { + return result; + } } loadBoard(boardXMLText) { - return this.api('loadBoard', boardXMLText) + return this.api('loadBoard', 'json', boardXMLText) } loadScenario(scenarioXMLText) { - return this.api('loadScenario', scenarioXMLText) + return this.api('loadScenario', 'json', scenarioXMLText) + } + + validateBoardStructure(boardXMLText) { + return this.api('validateBoardStructure', 'plaintext', boardXMLText) + } + + validateScenarioStructure(scenarioXMLText) { + return this.api('validateScenarioStructure', 'plaintext', scenarioXMLText) } validateScenario(boardXMLText, scenarioXMLText) { - return this.api('validateScenario', boardXMLText, scenarioXMLText) + return this.api('validateScenario', 'json', boardXMLText, scenarioXMLText) } generateLaunchScript(boardXMLText, scenarioXMLText) { - return this.api('generateLaunchScript', boardXMLText, scenarioXMLText) + return this.api('generateLaunchScript', 'json', boardXMLText, scenarioXMLText) } populateDefaultValues(scenarioXMLText) { - return this.api('populateDefaultValues', scenarioXMLText) + return this.api('populateDefaultValues', 'json', scenarioXMLText) } } @@ -104,12 +116,20 @@ class Configurator { loadBoard(path: String) { return this.readFile(path) .then((fileContent) => { - return this.pythonObject.loadBoard(fileContent) + let syntactical_errors = this.pythonObject.validateBoardStructure(fileContent); + if (syntactical_errors !== "") { + throw Error("The file has broken structure."); + } + return this.pythonObject.loadBoard(fileContent); }) } loadScenario(path: String): Object { return this.readFile(path).then((fileContent) => { + let syntactical_errors = this.pythonObject.validateScenarioStructure(fileContent); + if (syntactical_errors !== "") { + throw Error("The file has broken structure."); + } return this.pythonObject.loadScenario(fileContent) }) } @@ -155,4 +175,4 @@ class Configurator { } let configurator = new Configurator() -export default configurator \ No newline at end of file +export default configurator diff --git a/misc/config_tools/configurator/packages/configurator/src/pages/Config/Board.vue b/misc/config_tools/configurator/packages/configurator/src/pages/Config/Board.vue index ca3767c99..ccd9748f0 100644 --- a/misc/config_tools/configurator/packages/configurator/src/pages/Config/Board.vue +++ b/misc/config_tools/configurator/packages/configurator/src/pages/Config/Board.vue @@ -146,7 +146,7 @@ export default { .then(() => this.getBoardHistory()) }) .catch((err)=> { - alert(`Failed to load the file ${filepath}, it may not exist`) + alert(`Loading ${filepath} failed: ${err}`) console.log(err) }) } diff --git a/misc/config_tools/configurator/packages/configurator/src/pages/Config/Scenario.vue b/misc/config_tools/configurator/packages/configurator/src/pages/Config/Scenario.vue index 5f57aed03..cd15b3c37 100644 --- a/misc/config_tools/configurator/packages/configurator/src/pages/Config/Scenario.vue +++ b/misc/config_tools/configurator/packages/configurator/src/pages/Config/Scenario.vue @@ -98,7 +98,7 @@ export default { } }).catch((err) => { console.log(err) - alert(`Failed to open ${this.currentSelectedScenario}, file may not exist`) + alert(`Loading ${this.currentSelectedScenario} failed: ${err}`) }) } }, diff --git a/misc/config_tools/configurator/pyodide/pyodide.py b/misc/config_tools/configurator/pyodide/pyodide.py index f80931f08..c7688be75 100644 --- a/misc/config_tools/configurator/pyodide/pyodide.py +++ b/misc/config_tools/configurator/pyodide/pyodide.py @@ -22,6 +22,7 @@ def file_text(path): config_tools_dir = Path(__file__).absolute().parent.parent.parent configurator_dir = config_tools_dir / 'configurator' / 'packages' / 'configurator' schema_dir = config_tools_dir / 'schema' +board_xml_schema_path = schema_dir / 'board.xsd' scenario_xml_schema_path = schema_dir / 'sliced.xsd' datachecks_xml_schema_path = schema_dir / 'allchecks.xsd' diff --git a/misc/config_tools/configurator/pyodide/tests.py b/misc/config_tools/configurator/pyodide/tests.py index c4eb5178c..e4f511625 100644 --- a/misc/config_tools/configurator/pyodide/tests.py +++ b/misc/config_tools/configurator/pyodide/tests.py @@ -4,6 +4,8 @@ __package__ = 'configurator.pyodide' from .loadBoard import test as load_board_test from .loadScenario import test as load_scenario_test from .generateLaunchScript import test as generate_launch_script_test +from .validateBoardStructure import test as validate_board_structure_test +from .validateScenarioStructure import test as validate_scenario_structure_test from .validateScenario import test as validate_scenario_test from .populateDefaultValues import test as populate_default_values @@ -12,6 +14,8 @@ def main(): load_board_test() load_scenario_test() generate_launch_script_test() + validate_board_structure_test() + validate_scenario_structure_test() validate_scenario_test() populate_default_values() diff --git a/misc/config_tools/configurator/pyodide/validateBoardStructure.py b/misc/config_tools/configurator/pyodide/validateBoardStructure.py new file mode 100644 index 000000000..c7cf591d0 --- /dev/null +++ b/misc/config_tools/configurator/pyodide/validateBoardStructure.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +__package__ = 'configurator.pyodide' + +from pathlib import Path +from tempfile import TemporaryDirectory + +from scenario_config.default_populator import DefaultValuePopulatingStage +from scenario_config.pipeline import PipelineObject, PipelineEngine +from scenario_config.validator import ValidatorConstructionByFileStage, SyntacticValidationStage +from scenario_config.xml_loader import XMLLoadStage + +from .pyodide import ( + convert_result, write_temp_file, + nuc11_board, board_xml_schema_path +) + + +def main(board): + pipeline = PipelineEngine(["board_path", "schema_path", "datachecks_path"]) + pipeline.add_stages([ + ValidatorConstructionByFileStage(), + XMLLoadStage("board"), + SyntacticValidationStage(etree_tag = "board"), + ]) + + try: + with TemporaryDirectory() as tmpdir: + write_temp_file(tmpdir, { + 'board.xml': board + }) + board_file_path = Path(tmpdir) / 'board.xml' + + obj = PipelineObject( + board_path=board_file_path, + schema_path=board_xml_schema_path, + datachecks_path=None + ) + pipeline.run(obj) + + validate_result = obj.get("syntactic_errors") + return "\n\n".join(map(lambda x: x["message"], validate_result)) + except Exception as e: + return str(e) + + +def test(): + print(main(nuc11_board)) + + +if __name__ == '__main__': + test() diff --git a/misc/config_tools/configurator/pyodide/validateScenarioStructure.py b/misc/config_tools/configurator/pyodide/validateScenarioStructure.py new file mode 100644 index 000000000..b32c8ed32 --- /dev/null +++ b/misc/config_tools/configurator/pyodide/validateScenarioStructure.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +__package__ = 'configurator.pyodide' + +from pathlib import Path +from tempfile import TemporaryDirectory + +from scenario_config.default_populator import DefaultValuePopulatingStage +from scenario_config.pipeline import PipelineObject, PipelineEngine +from scenario_config.validator import ValidatorConstructionByFileStage, SyntacticValidationStage +from scenario_config.xml_loader import XMLLoadStage + +from .pyodide import ( + convert_result, write_temp_file, + nuc11_scenario, scenario_xml_schema_path, datachecks_xml_schema_path +) + + +def main(scenario): + pipeline = PipelineEngine(["scenario_path", "schema_path", "datachecks_path"]) + pipeline.add_stages([ + ValidatorConstructionByFileStage(), + XMLLoadStage("schema"), + XMLLoadStage("scenario"), + DefaultValuePopulatingStage(), + SyntacticValidationStage(), + ]) + + try: + with TemporaryDirectory() as tmpdir: + write_temp_file(tmpdir, { + 'scenario.xml': scenario + }) + scenario_file_path = Path(tmpdir) / 'scenario.xml' + + obj = PipelineObject( + scenario_path=scenario_file_path, + schema_path=scenario_xml_schema_path, + datachecks_path=None + ) + pipeline.run(obj) + + validate_result = obj.get("syntactic_errors") + return "\n\n".join(map(lambda x: x["message"], validate_result)) + except Exception as e: + return str(e) + + +def test(): + print(main(nuc11_scenario)) + + +if __name__ == '__main__': + test() diff --git a/misc/config_tools/scenario_config/pipeline.py b/misc/config_tools/scenario_config/pipeline.py index 0ff1a61ee..e6859efc0 100644 --- a/misc/config_tools/scenario_config/pipeline.py +++ b/misc/config_tools/scenario_config/pipeline.py @@ -50,7 +50,7 @@ class PipelineEngine: all_uses = consumes.union(uses) if not all_uses.issubset(self.available_data): - raise Exception(f"Data {uses - self.available_data} need by stage {stage.__class__.__name__} but not provided by the pipeline") + raise Exception(f"Data {all_uses - self.available_data} need by stage {stage.__class__.__name__} but not provided by the pipeline") self.stages.append(stage) self.available_data = self.available_data.difference(consumes).union(provides) diff --git a/misc/config_tools/scenario_config/validator.py b/misc/config_tools/scenario_config/validator.py index b1f479f8c..40cad8412 100755 --- a/misc/config_tools/scenario_config/validator.py +++ b/misc/config_tools/scenario_config/validator.py @@ -71,7 +71,7 @@ class ScenarioValidator: def __init__(self, schema_etree, datachecks_etree): """Initialize the validator with preprocessed schemas in ElementTree.""" self.schema = xmlschema.XMLSchema11(schema_etree) - self.datachecks = xmlschema.XMLSchema11(datachecks_etree) + self.datachecks = xmlschema.XMLSchema11(datachecks_etree) if datachecks_etree else None def check_syntax(self, scenario_etree): errors = [] @@ -88,14 +88,15 @@ class ScenarioValidator: def check_semantics(self, board_etree, scenario_etree): errors = [] - unified_node = copy(scenario_etree.getroot()) - parent_map = {c : p for p in unified_node.iter() for c in p} - unified_node.extend(board_etree.getroot()) - it = self.datachecks.iter_errors(unified_node) - for error in it: - e = self.format_error(unified_node, parent_map, error) - e.log() - errors.append(e) + if self.datachecks: + unified_node = copy(scenario_etree.getroot()) + parent_map = {c : p for p in unified_node.iter() for c in p} + unified_node.extend(board_etree.getroot()) + it = self.datachecks.iter_errors(unified_node) + for error in it: + e = self.format_error(unified_node, parent_map, error) + e.log() + errors.append(e) return errors @@ -190,11 +191,14 @@ class ValidatorConstructionByFileStage(PipelineStage): obj.set("validator", validator) class SyntacticValidationStage(PipelineStage): - uses = {"validator", "scenario_etree"} provides = {"syntactic_errors"} + def __init__(self, etree_tag = "scenario"): + self.etree_tag = f"{etree_tag}_etree" + self.uses = {"validator", self.etree_tag} + def run(self, obj): - errors = obj.get("validator").check_syntax(obj.get("scenario_etree")) + errors = obj.get("validator").check_syntax(obj.get(self.etree_tag)) obj.set("syntactic_errors", errors) class SemanticValidationStage(PipelineStage): diff --git a/misc/config_tools/schema/board.xsd b/misc/config_tools/schema/board.xsd new file mode 100644 index 000000000..f5c31c685 --- /dev/null +++ b/misc/config_tools/schema/board.xsd @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +