├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ └── pythonapp.yml ├── .gitignore ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── arxml_data_extractor ├── __init__.py ├── __main__.py ├── asr │ ├── __init__.py │ └── asr_parser.py ├── config_provider.py ├── data_writer.py ├── handler │ ├── __init__.py │ ├── object_handler.py │ ├── path_handler.py │ └── value_handler.py ├── output │ ├── __init__.py │ ├── excel_writer.py │ ├── tabularize.py │ └── text_writer.py ├── query │ ├── __init__.py │ ├── data_object.py │ ├── data_query.py │ └── data_value.py ├── query_builder.py ├── query_handler.py └── tests │ ├── __init__.py │ ├── asr │ ├── __init__.py │ ├── test.arxml │ └── test_asr_parser.py │ ├── output │ ├── __init__.py │ └── test_tabularize.py │ ├── query │ ├── __init__.py │ ├── test_data_object.py │ ├── test_data_query.py │ └── test_data_value.py │ ├── test.arxml │ ├── test_arxml_data_extractor.py │ ├── test_config.yaml │ ├── test_config_provider.py │ ├── test_data_writer.py │ ├── test_query_builder.py │ └── test_query_handler.py └── requirements.txt /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/pythonapp.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: ARXML Data Extractor 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Set up Python 3.8 20 | uses: actions/setup-python@v1 21 | with: 22 | python-version: 3.8 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install --upgrade pip 26 | pip install -r requirements.txt 27 | - name: Lint with flake8 28 | run: | 29 | pip install flake8 30 | # stop the build if there are Python syntax errors or undefined names 31 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 32 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 33 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 34 | - name: Test with pytest 35 | run: | 36 | pip install pytest 37 | pytest 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.formatOnSave": true, 3 | "python.pythonPath": ".venv\\Scripts\\python.exe", 4 | "python.venvPath": ".venv", 5 | "python.linting.pylintEnabled": true, 6 | "python.linting.enabled": true, 7 | "python.testing.pytestArgs": [ 8 | "arxml_data_extractor" 9 | ], 10 | "python.testing.unittestEnabled": false, 11 | "python.testing.nosetestsEnabled": false, 12 | "python.testing.pytestEnabled": true, 13 | "python.formatting.provider": "yapf", 14 | "python.formatting.yapfArgs": [ 15 | "--style", 16 | "{based_on_style: chromium, indent_width: 4, column_limit: 100}" 17 | ] 18 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Brokdar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ArxmlDataExtractor 2 | 3 | ArxmlDataExtractor makes it easy for everybody to extract data from an AUTOSAR .arxml file. It uses common .yaml files as data extraction specification, afterward referred to as configuration file. It supports the extraction of complex data structures as well as the handling of AUTOSAR references. The extracted data can then be written into three formats: '.txt', '.json' and '.xlsx'. 4 | 5 | ## Supported Features 6 | 7 | - Simple syntax to describe data extraction 8 | - Supports XPath expression with auto-handling of AUTOSAR namespaces 9 | - Supports extracting data from AUTOSAR References 10 | - Specify the output data structure within the configuration 11 | - Value conversion into integer, float or date 12 | - Config files can be shared and reused 13 | - Simple data output in a .txt file for rapid prototyping 14 | - JSON output for reusing the data in other scripts or tools 15 | - Excel output for better analytics support like filtering or sorting 16 | 17 | ## Usage 18 | 19 | In order to extract data from a given ARXML file, ArxmlDataExtractor.exe needs to be called with the following syntax in your command window. 20 | 21 | ```batch 22 | ArxmlDataExtractor.exe [-h] --config CONFIG --input INPUT --output OUTPUT 23 | ``` 24 | 25 | The order of the options is optional and can be rearranged. The table below describes the available options. 26 | 27 | | Short| Option | Description | 28 | |------|----------|--------------------------------------------------------------| 29 | | -h | --help | show help message | 30 | | -c | --config | config file that specified the data that should be extracted | 31 | | -i | --input | ARXML file from where the data should be extracted | 32 | | -o | --output | output file, possible formats are: .txt, .json or .xlsx | 33 | | -d | --debug | enables debug mode, will write a .log file | 34 | 35 | ## Configuration File 36 | 37 | ### Structure 38 | 39 | In general, every configuration file will consist of objects and values. An object is a collection of values with a given name and an anchor. Value describes the data to be extracted from the ARXML file. It consists of a freely chosen name and additional parsing instructions in the following referred to as queries. 40 | 41 | Every configuration file has one thing in common. It has to start with at least one object. This object will the entry point, specified by one of the allowed anchors, and will be the entry point for the following values. 42 | This is used to optimize query processing and therefore the parsing performance. Each query will use the anchor of its parent object as a base. The parsing instructions will be handled relatively starting from this base. Important note, an anchor can either return a single object (if only one element exists in the ARXML) or a list of all matching elements. 43 | 44 | ```yaml 45 | Object: 46 | anchor: <...> 47 | value: <...> 48 | ... 49 | ``` 50 | 51 | Below is a simple example of a configuration specification that extracts all top-level 'AR-PACKAGE' elements from the given '.arxml' file. The root object is called 'Package' with an XPath expression as an anchor. The anchor will return a list of all elements named 'AR-PACKAGE' with a parent element named 'AR-PACKAGES' started from the root element ('AUTOSAR'). More information about XPath expression can be found [here](https://www.w3schools.com/xml/xpath_syntax.asp)). More information about anchors and their types can be found in the Anchor section. 52 | 53 | ```yaml 54 | Package: 55 | _xpath: "./AR-PACKAGES/AR-PACKAGE" 56 | Name: "SHORT-NAME" 57 | ``` 58 | 59 | Underneath the anchor, the values are specified. In this case, there's only one value defined with the name 'Name'. The query for this value refers to a child element named 'SHORT-NAME'. This element is the child of the defined anchor in this case, 'AR-PACKAGE'. The query can be interpreted as: 60 | >From the object's base, go to its child element 'SHORT-NAME', extract the text value and write in a variable called 'Name' contained in the object called 'Package'. 61 | 62 | ### Nesting Objects 63 | 64 | With nesting objects, the data structure of the output data can be defined. Besides, if used cleverly, the parsing time can be reduced. This will be relevant if you want to extract multiple values from a common child element. Because every object has an anchor, it can be set to the common child to reduce the parsing depth. 65 | 66 | ```yaml 67 | object1: 68 | anchor: <...> 69 | value1: <...> 70 | object2: 71 | anchor: <...> 72 | value2: <...> 73 | ``` 74 | 75 | The following configuration shows a concrete example of nesting objects. Let's assume you want to list all PDUs and their timing specification from an ECU extract. Therefore, you can create a root object for finding all PDUs and add a nested object for the timing specification. The queries for the values of 'MinimumDelay' and 'CyclicTiming' will use the anchor of the object 'Timing Specification' as their base element. 76 | 77 | ```yaml 78 | PDU: 79 | _xpath: ".//I-SIGNAL-I-PDU" 80 | Name: "SHORT-NAME" 81 | TimingSpecification: 82 | _xpath: "./*/I-PDU-TIMING" 83 | MinimumDelay: "MINIMUM-DELAY" 84 | CyclicTiming: "TRANSMISSION-MODE-DECLARATION/TRANSMISSION-MODE-TRUE-TIMING/CYCLIC-TIMING/TIME-PERIOD/VALUE" 85 | ``` 86 | 87 | You can also extract that information without nested objects. Then the values will be part of the PDU object and the parsing time can increase slightly. 88 | 89 | ### Advanced Value Handling 90 | 91 | Value queries can be further refined by specifying the extract location and format. This can be done by prepending additional parsing information to the path. This is optional and will default to the text property of the found element specified by the path in the string format. 92 | 93 | ```yaml 94 | value: [location[>format]:] 95 | ``` 96 | 97 | Important to know is that if a format conversion is added, then also the value location needs to be set. To separate the location from the format `>` will be put in between. To further separate the parsing instructions from the path, they will be split by `:`. The following configuration extends the PDU extraction example to also convert the timing specification values in their proper format. 'MinimumDelay' will be converted to an integer and 'CyclicTiming' to a float value. 98 | 99 | ```yaml 100 | PDU: 101 | _xpath: ".//I-SIGNAL-I-PDU" 102 | Name: "SHORT-NAME" 103 | TimingSpecification: 104 | _xpath: "./*/I-PDU-TIMING" 105 | MinimumDelay: "text>int:MINIMUM-DELAY" 106 | CyclicTiming: "text>float:TRANSMISSION-MODE-DECLARATION/TRANSMISSION-MODE-TRUE-TIMING/CYCLIC-TIMING/TIME-PERIOD/VALUE" 107 | ``` 108 | 109 | If the conversion isn't possible, it will default to its textual representation. More information about the values' location and format can be found in the Syntax section. 110 | 111 | ### Syntax 112 | 113 | #### Object Anchors 114 | 115 | An object anchor is the entry point for all following value queries. The anchor is used to find the specified XML elements, e.g. all top-level 'AR-PACKAGE' elements. Therefore the anchor needs to describe where to find those elements. The following types of anchors are supported. 116 | 117 | | Syntax | Description | Usage | 118 | |--------|-------------------------------------------------------------------------------|----------------------------------------------------| 119 | | _xpath | Any XPath expression can be used to specify the object that should be parsed | `_xpath: ./AR-PACKAGES/AR-PACKAGE` | 120 | | _ref | AUTOSAR Reference to a specific object | `_ref: /PDU/Name` | 121 | | _xref | Any XPath expression that leeds to an element containing an AUTOSAR Reference | `_xref: .//I-SIGNAL-TO-I-PDU-MAPPING/I-SIGNAL-REF` | 122 | 123 | The `_xref` anchor is a special type of anchor because it is a combination of both `_xpath` and `_ref`. This is handy if you want to get data from element but from the current context, you only have access to its AUTOSAR reference. An easy example would be if you want the data type of a signal that is mapped to a PDU. The PDU only contains a reference to the signal, so to get the signals data type you need to look at the signal element itself. 124 | 125 | How does this work? First, it tries to find the element containing the AUTOSAR reference specified by the XPath. Then it grabs the reference from the elements text value and looks up the referred element which then will be the base for the child value queries. 126 | 127 | This is coming very handy if multiple values from the referenced element should be extracted. If so, the expression will only be executed once and the referenced element will be cached for all the following queries. If only one value is required of a reference than an inline reference can be used (next section). 128 | 129 | #### Value Path 130 | 131 | A values' path consists of an XPath expression that leads to the element where the data can be found. All types of XPath expressions can be used. Optionally, the path can be converted into an inline reference by prepending `&()` to the actual XPath expression. 132 | 133 | ```yaml 134 | value: [&()] 135 | ``` 136 | 137 | An inline reference is a combination of an XPath expression with an AUTOSAR reference. If the path of a value query starts with a `&`, it indicates that the path should be interpreted as inline reference. `` contains the XPath expression to the element containing the AUTOSAR reference in its text property. `` is the XPath expression to the actual value location, using the referenced element as a base. 138 | 139 | ```yaml 140 | PDU: 141 | _xpath: ".//I-SIGNAL-I-PDU" 142 | Name: "SHORT-NAME" 143 | Signal: 144 | _xpath: "//I-SIGNAL-TO-I-PDU-MAPPING" 145 | Name: "&(I-SIGNAL-REF)SHORT-NAME" 146 | ``` 147 | 148 | #### Value Location 149 | 150 | | Syntax | Description | Usage | 151 | |-----------|-------------------------------------------|------------------------| 152 | | `tag` | Gets the tag of the element | `value: tag:` | 153 | | `text` | Gets the text of the element | `value: text:` | 154 | | `@` | Gets the value of the specified attribute | `value: @UUID:` | 155 | 156 | #### Value Formats 157 | 158 | | Syntax | Description | Usage | 159 | |----------|------------------------------------|-----------------------------| 160 | | `string` | Takes the textual representation | `value: text>string:`| 161 | | `int` | Converts the value into an integer | `value: text>int:` | 162 | | `float` | Converts the value into float | `value: text>float:` | 163 | | `date` | Converts the value into a date | `value: text>date:` | 164 | 165 | ### Example Configuration 166 | 167 | Example configurations can be executed with the provided .arxml file. 168 | 169 | #### Get PDU Information 170 | 171 | This .yaml configuration will parse all PDUs present in the given .arxml file and extract the specified values. It will automatically handle situations where a PDU contains multiple signals. Therefore, all Signals will be extracted and reported. 172 | 173 | ```yaml 174 | PDU: 175 | _xpath: ".//I-SIGNAL-I-PDU" 176 | Name: "SHORT-NAME" 177 | Length: "text>int:LENGTH" 178 | CyclicTiming: "text>float:.//TRANSMISSION-MODE-TRUE-TIMING/CYCLIC-TIMING/TIME-PERIOD/VALUE" 179 | SignalMappings: 180 | _xpath: ".//I-SIGNAL-TO-I-PDU-MAPPING" 181 | Signal: "SHORT-NAME" 182 | StartPosition: "text>int:START-POSITION" 183 | ISignal: 184 | _xref: "I-SIGNAL-REF" 185 | InitValue: "text>int:.//VALUE" 186 | Length: "text>int:LENGTH" 187 | ``` 188 | 189 | #### Get CAN Cluster Specification 190 | 191 | This configuration uses an AUTOSAR Reference to get the information of the CAN Cluster. Please note, that the reference will change whenever the CAN Cluster will be renamed. You should have that in mind if you think of reusing the script. AUTOSAR References will be fast for prototyping but if you want to reuse the configuration you should aim for XPath expressions. 192 | 193 | ```yaml 194 | CanCluster: 195 | _ref: "/Cluster/CAN" 196 | Name: "SHORT-NAME" 197 | Baudrate: "text>int:CAN-CLUSTER-VARIANTS/CAN-CLUSTER-CONDITIONAL/BAUDRATE" 198 | LongName: "text:LONG-NAME/L-4" 199 | Language: "@L:LONG-NAME/L-4" 200 | ``` 201 | -------------------------------------------------------------------------------- /arxml_data_extractor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brokdar/ArxmlDataExtractor/2853112cbd4d001418b11ccb99f1db268347dfab/arxml_data_extractor/__init__.py -------------------------------------------------------------------------------- /arxml_data_extractor/__main__.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | import sys 4 | from pathlib import Path 5 | 6 | from arxml_data_extractor.config_provider import ConfigProvider 7 | from arxml_data_extractor.query_builder import QueryBuilder 8 | from arxml_data_extractor.query_handler import QueryHandler 9 | from arxml_data_extractor.data_writer import DataWriter 10 | 11 | 12 | # sets the text color to red 13 | def print_error(msg: str): 14 | print('\033[91mError: ' + msg + '\033[0m') 15 | 16 | 17 | def handle_error(msg: str): 18 | logging.getLogger().error(msg) 19 | print_error(msg) 20 | 21 | 22 | def handle_exception(msg: str, e: Exception): 23 | logging.getLogger().error(msg, exc_info=e) 24 | print_error(f'{msg}, {str(e)}') 25 | 26 | 27 | def parse_arguments(): 28 | # setup console arguments 29 | parser = argparse.ArgumentParser( 30 | description='Extracts specified data provided in a config file from an ARXML file.') 31 | parser.add_argument( 32 | '--config', 33 | '-c', 34 | help='config file that specifies the data that should be extracted', 35 | required=True) 36 | parser.add_argument( 37 | '--input', '-i', help='ARXML file from where the data should be extracted', required=True) 38 | parser.add_argument( 39 | '--output', 40 | '-o', 41 | help='output file, possible file formats are \'.txt\', \'.json\' or \'.xlsx\'.', 42 | required=True) 43 | parser.add_argument( 44 | '--debug', 45 | '-d', 46 | help='enable debug modus, this will create a log file.', 47 | action='store_true') 48 | 49 | return parser.parse_args() 50 | 51 | 52 | def setup_logging(enable: bool): 53 | logger = logging.getLogger() 54 | 55 | if enable: 56 | logging.basicConfig( 57 | filename='extraction.log', filemode='w', format='%(levelname)s: %(message)s') 58 | logger.setLevel('DEBUG') 59 | else: 60 | logger.disabled = True 61 | 62 | 63 | def validate_arguments(args): 64 | input_file = Path(args.input) 65 | if not input_file.exists() or not input_file.is_file(): 66 | handle_error(f'input file: \'{args.input}\' doesn\'t exist or isn\'t a valid file') 67 | sys.exit(-1) 68 | 69 | config_file = Path(args.config) 70 | if not config_file.exists() or not config_file.is_file(): 71 | handle_error(f'config file: \'{args.config}\' doesn\'t exist or isn\'t a valid file') 72 | sys.exit(-1) 73 | 74 | output_file = Path(args.output) 75 | allowed_suffix = ['.txt', '.json', '.xlsx'] 76 | if output_file.suffix not in allowed_suffix: 77 | handle_error( 78 | f'invalid output file extension \'{output_file.suffix}\'. Allowed extensions: \'.txt\', \'.json\' or \'.xlsx\'' 79 | ) 80 | sys.exit(-1) 81 | 82 | return input_file, config_file, output_file 83 | 84 | 85 | def load_config(file): 86 | logger = logging.getLogger() 87 | logger.info(f'START PROCESS - loading configuration \'{str(file)}\'') 88 | 89 | try: 90 | config_provider = ConfigProvider() 91 | config = config_provider.load(str(file)) 92 | except Exception as e: 93 | handle_exception(f'reading configuration file \'{str(file)}\'', e) 94 | sys.exit(-1) 95 | 96 | logger.info('END PROCESS - successfully finished loading configuration') 97 | return config 98 | 99 | 100 | def build_queries(config): 101 | logger = logging.getLogger() 102 | logger.info('START PROCESS - building queries from configuration') 103 | 104 | try: 105 | query_builder = QueryBuilder() 106 | queries = query_builder.build(config) 107 | except Exception as e: 108 | handle_exception('building queries', e) 109 | sys.exit(-1) 110 | 111 | logger.info('END PROCESS - successfully finished building queries from configuration') 112 | return queries 113 | 114 | 115 | def extract_data(file, queries): 116 | logger = logging.getLogger() 117 | logger.info('START PROCESS - handling of data queries') 118 | 119 | try: 120 | query_handler = QueryHandler() 121 | data = query_handler.handle_queries(str(file), queries) 122 | except Exception as e: 123 | handle_exception('handling queries', e) 124 | sys.exit(-1) 125 | 126 | logger.info('END PROCESS - successfully finished handling of data queries') 127 | return data 128 | 129 | 130 | def write_data(file, data): 131 | logger = logging.getLogger() 132 | logger.info(f'START PROCESS - writing results to \'{str(file)}\'') 133 | print(f'Writing results to \'{str(file)}\'') 134 | 135 | try: 136 | output_writer = DataWriter() 137 | if file.suffix == '.json': 138 | output_writer.write_json(str(file), data) 139 | elif file.suffix == '.xlsx': 140 | output_writer.write_excel(str(file), data) 141 | else: 142 | output_writer.write_text(str(file), data) 143 | except Exception as e: 144 | handle_exception(f'writing results to \'{str(file)}\'', e) 145 | sys.exit(-1) 146 | 147 | logger.info('END PROCESS - successfully finished writing results') 148 | print(f'Done.') 149 | 150 | 151 | def run(): 152 | args = parse_arguments() 153 | setup_logging(args.debug) 154 | input_file, config_file, output_file = validate_arguments(args) 155 | 156 | config = load_config(config_file) 157 | queries = build_queries(config) 158 | data = extract_data(input_file, queries) 159 | write_data(output_file, data) 160 | 161 | 162 | if __name__ == '__main__': 163 | run() 164 | -------------------------------------------------------------------------------- /arxml_data_extractor/asr/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brokdar/ArxmlDataExtractor/2853112cbd4d001418b11ccb99f1db268347dfab/arxml_data_extractor/asr/__init__.py -------------------------------------------------------------------------------- /arxml_data_extractor/asr/asr_parser.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from lxml import etree 4 | from typing import Union 5 | 6 | 7 | class AsrParser(): 8 | """Provides parsing functions for navigating within an ARXML file. 9 | """ 10 | 11 | def __init__(self, arxml: str): 12 | parser = etree.XMLParser(remove_blank_text=True) 13 | self.__tree = etree.parse(arxml, parser) 14 | self.__root = self.tree.getroot() 15 | 16 | # get namespace from arxml file 17 | AsrParser.ns = {'ar': self.__root.nsmap[None]} 18 | 19 | self._packages = { 20 | AsrParser.get_shortname(element): element 21 | for element in self.find_all_elements('AUTOSAR/AR-PACKAGES/AR-PACKAGE') 22 | } 23 | 24 | @property 25 | def tree(self): 26 | return self.__tree 27 | 28 | @property 29 | def root(self): 30 | return self.__root 31 | 32 | @property 33 | def packages(self): 34 | return self._packages 35 | 36 | def find_all_elements(self, path: str) -> list: 37 | """Finds all elements specified by the XML element path. The path can 38 | either be a simple element name, a complex xml path with namespaces 39 | or a combination of both. 40 | 41 | Arguments: 42 | element_path {str} -- element name, XML element path or a combination 43 | 44 | Returns: 45 | list -- all found elements reachable with the specified element path 46 | """ 47 | xpath = AsrParser.__assemble_xpath(path) 48 | return AsrParser.find(self.root, '//' + xpath) 49 | 50 | def find_reference(self, reference: str) -> etree.Element: 51 | """Tries to find the element described by the AUTOSAR reference 52 | 53 | Arguments: 54 | reference {str} -- AUTOSAR reference 55 | 56 | Returns: 57 | etree.Element -- xml element node or None 58 | """ 59 | ref_parts = reference.split('/') 60 | if (reference.startswith('/')): 61 | ref_parts = ref_parts[1:] 62 | 63 | element = self.packages[ref_parts[0]] 64 | if (element is None): 65 | return None 66 | 67 | for i in range(1, len(ref_parts)): 68 | xpath = f'.//*/ar:SHORT-NAME[text()="{ref_parts[i]}"]/..' 69 | element = AsrParser.__first(AsrParser.find(element, xpath)) 70 | if (element is None): 71 | return None 72 | 73 | return element 74 | 75 | @staticmethod 76 | def __append_namespace(path: str) -> str: 77 | if (path.startswith('ar:')): 78 | return path 79 | return 'ar:' + path 80 | 81 | @classmethod 82 | def __assemble_xpath(cls, path: str) -> str: 83 | if ('/' in path): 84 | return '/'.join([cls.__append_namespace(part) for part in path.split('/')]) 85 | 86 | return AsrParser.__append_namespace(path) 87 | 88 | @staticmethod 89 | def __first(list: list): 90 | return next(iter(list or []), None) 91 | 92 | @staticmethod 93 | def find(base: etree.Element, xpath: Union[str, etree.XPath]) -> list: 94 | """Evaluates an XPath expression on the given base element 95 | 96 | Arguments: 97 | base {etree.Element} -- base element, starting point 98 | xpath {str} -- XPath expression defined by W3C consortium 99 | 100 | Returns: 101 | list -- results of XPath evaluation 102 | """ 103 | if isinstance(xpath, etree.XPath): 104 | return xpath(base) 105 | return base.xpath(xpath, namespaces=AsrParser.ns) 106 | 107 | @classmethod 108 | def find_elements(cls, base: etree.Element, path: str) -> list: 109 | """Finds all child elements of the given base element defined by the given Element structure 110 | 111 | Arguments: 112 | base {etree.Element} -- base element, starting point 113 | path {str} -- XML element structure with or without namespace 114 | 115 | Returns: 116 | list -- found elements or empty list 117 | """ 118 | xpath = cls.__assemble_xpath(path) 119 | return cls.find(base, './/' + xpath) 120 | 121 | @classmethod 122 | def find_first_element(cls, 123 | base: etree.Element, 124 | path: str, 125 | attribute: str = None, 126 | text: str = None) -> etree.Element: 127 | 128 | elements = cls.find_elements(base, path) 129 | return cls.__first(elements) 130 | 131 | @classmethod 132 | def find_element_by_shortname(cls, base: etree.Element, path: str, name: str) -> etree.Element: 133 | """Finds the element of specified type with the provided name 134 | 135 | Arguments: 136 | base {etree.Element} -- base element, starting point 137 | path {str} -- element name, XML element path or a combination 138 | name {str} -- shortname of the element 139 | 140 | Returns: 141 | etree.Element -- found element or None 142 | """ 143 | xpath = cls.__assemble_xpath(path) 144 | xpath = xpath + f'/ar:SHORT-NAME[text()="{name}"]/..' 145 | return cls.__first(cls.find(base, './/' + xpath)) 146 | 147 | @classmethod 148 | def get_shortname(cls, element: etree.Element) -> Union[str, None]: 149 | """Gets the shortname of an element 150 | 151 | Arguments: 152 | element {etree.Element} -- xml element with shortname 153 | 154 | Returns: 155 | str -- shortname if found otherwise None 156 | """ 157 | return cls.__first(cls.find(element, 'ar:SHORT-NAME/text()')) 158 | 159 | @staticmethod 160 | def assemble_xpath(path: str) -> str: 161 | xpath = re.sub(r"([\/]+)(?!$)", r'\1ar:', path) 162 | if xpath.startswith('/') or xpath.startswith('.'): 163 | return xpath 164 | else: 165 | return 'ar:' + xpath 166 | -------------------------------------------------------------------------------- /arxml_data_extractor/config_provider.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | from pathlib import Path 3 | 4 | 5 | class ConfigProvider(): 6 | 7 | def load(self, file: str) -> dict: 8 | config_file = Path(file) 9 | if config_file.suffix != '.yaml': 10 | raise ValueError( 11 | f'invalid config file extension: \'{config_file.suffix}\' != \'.yaml\'') 12 | 13 | with open(str(config_file), 'r') as stream: 14 | config = yaml.safe_load(stream) 15 | 16 | return config 17 | 18 | def parse(self, text: str) -> dict: 19 | config = yaml.safe_load(text) 20 | 21 | return config 22 | -------------------------------------------------------------------------------- /arxml_data_extractor/data_writer.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from arxml_data_extractor.output.text_writer import TextWriter 4 | from arxml_data_extractor.output.excel_writer import ExcelWriter 5 | 6 | 7 | class DataWriter(): 8 | 9 | def write_text(self, file: str, data: dict): 10 | writer = TextWriter() 11 | text = writer.as_table(data) 12 | 13 | with open(file, 'w') as f: 14 | f.write(text) 15 | 16 | def write_json(self, file: str, data: dict): 17 | with open(file, 'w', encoding='utf-8') as f: 18 | json.dump(data, f, ensure_ascii=False, indent=4) 19 | 20 | def write_excel(self, file: str, data: dict): 21 | writer = ExcelWriter() 22 | writer.write(file, data) 23 | -------------------------------------------------------------------------------- /arxml_data_extractor/handler/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brokdar/ArxmlDataExtractor/2853112cbd4d001418b11ccb99f1db268347dfab/arxml_data_extractor/handler/__init__.py -------------------------------------------------------------------------------- /arxml_data_extractor/handler/object_handler.py: -------------------------------------------------------------------------------- 1 | from lxml.etree import Element, QName 2 | from typing import Union, List, Any 3 | from tqdm import tqdm 4 | import logging 5 | 6 | from arxml_data_extractor.handler import value_handler 7 | from arxml_data_extractor.handler.path_handler import PathHandler 8 | from arxml_data_extractor.asr.asr_parser import AsrParser 9 | from arxml_data_extractor.query.data_query import DataQuery 10 | from arxml_data_extractor.query.data_object import DataObject 11 | from arxml_data_extractor.query.data_value import DataValue 12 | 13 | 14 | class ObjectHandler(): 15 | 16 | def __init__(self, parser: AsrParser): 17 | self.logger = logging.getLogger() 18 | self.path_handler = PathHandler(parser) 19 | 20 | def handle(self, data_object: DataObject, node: Element = None) -> Union[list, dict]: 21 | is_not_root = True 22 | if node is None: 23 | is_not_root = False 24 | node = self.path_handler.parser.root 25 | 26 | if is_not_root: 27 | self.logger.info(f'ObjectHandler - handle DataObject(\'{data_object.name}\')') 28 | else: 29 | self.logger.info(f'ObjectHandler - [root] handle DataObject(\'{data_object.name}\')') 30 | 31 | values = [] 32 | elements = self.path_handler.elements_by_path(data_object.path, node) 33 | for element in tqdm( 34 | elements, 35 | desc=f'Handle DataObject(\'{data_object.name}\')', 36 | disable=is_not_root, 37 | bar_format="{desc:<70}{percentage:3.0f}% |{bar:70}| {n_fmt:>4}/{total_fmt}"): 38 | if element is not None: 39 | self.logger.info( 40 | f'ObjectHandler - element found: \'{QName(element).localname}\' at line {element.sourceline - 1}' 41 | ) 42 | values.append(self.__handle_values(data_object.values, element)) 43 | 44 | if not values: 45 | self.logger.warning( 46 | f'ObjectHandler - no values found for DataObject(\'{data_object.name}\')') 47 | else: 48 | self.logger.info( 49 | f'ObjectHandler - values found for DataObject(\'{data_object.name}\'): {len(values)}' 50 | ) 51 | 52 | return values[0] if len(values) == 1 else values 53 | 54 | def __handle_values(self, values: List[Union[DataValue, DataObject]], node: Element) -> dict: 55 | results = {} 56 | for value in values: 57 | if isinstance(value, DataObject): 58 | results[value.name] = self.handle(value, node) 59 | elif isinstance(value, DataValue): 60 | results[value.name] = self.__handle_value(value.query, node) 61 | if results[value.name] is None: 62 | self.logger.info( 63 | f'ObjectHandler - no value found for DataValue(\'{value.name}\')') 64 | else: 65 | self.logger.info( 66 | f'ObjectHandler - value found: DataValue(\'{value.name}\') = \'{results[value.name]}\'' 67 | ) 68 | else: 69 | error = f'ObjectHandler - invalid value type ({type(value)}). Value must be of type DataObject or DataValue' 70 | self.logger.error(error) 71 | raise TypeError(error) 72 | 73 | return results 74 | 75 | def __handle_value(self, query: DataQuery, node: Element) -> Any: 76 | if isinstance(query.path, DataQuery.XPath): 77 | if query.path.is_reference: 78 | element = self.path_handler.element_by_inline_ref(query.path, node) 79 | else: 80 | element = self.path_handler.element_by_xpath(query.path.xpath, node) 81 | else: # DataQuery.Reference isn't allowed on DataValue 82 | return None 83 | 84 | if element is None: 85 | return None 86 | 87 | return value_handler.handle(query, element) 88 | -------------------------------------------------------------------------------- /arxml_data_extractor/handler/path_handler.py: -------------------------------------------------------------------------------- 1 | from lxml.etree import Element 2 | from typing import Union, List, Tuple 3 | import logging 4 | 5 | from arxml_data_extractor.asr.asr_parser import AsrParser 6 | from arxml_data_extractor.query.data_query import DataQuery 7 | 8 | 9 | class PathHandler(): 10 | 11 | def __init__(self, parser: AsrParser): 12 | self.logger = logging.getLogger() 13 | self.parser = parser 14 | 15 | def elements_by_path(self, path: Union[DataQuery.XPath, DataQuery.Reference], 16 | node: Element) -> Union[List[Element], None]: 17 | if isinstance(path, DataQuery.XPath): 18 | elements = self.elements_by_xpath(path.xpath, node) 19 | if not elements: 20 | self.logger.warning(f'PathHandler - no elements found with XPath \'{path.xpath}\'') 21 | return elements 22 | 23 | if path.is_reference is False: 24 | return elements 25 | 26 | if len(elements) != 1: 27 | error = f'PathHandler - too many references found with XPath \'{path.xpath}\'' 28 | self.logger.error(error) 29 | raise Exception(error) 30 | return [self.element_by_ref(elements[0].text)] 31 | 32 | elif isinstance(path, DataQuery.Reference): 33 | return [self.element_by_ref(path.ref)] 34 | else: 35 | error = f'PathHandler - invalid path type (type: {type(path)}). Path must be of type DataQuery.XPath or DataQuery.Reference' 36 | self.logger.error(error) 37 | raise TypeError(error) 38 | 39 | def elements_by_xpath(self, path: str, node: Element) -> List[Element]: 40 | xpath = self.parser.assemble_xpath(path) 41 | return self.parser.find(node, xpath) 42 | 43 | def element_by_xpath(self, path: str, node: Element) -> Union[Element, None]: 44 | element = next(iter(self.elements_by_xpath(path, node)), None) 45 | if element is None: 46 | self.logger.warning(f'PathHandler - no element found with XPath \'{path}\'') 47 | return None 48 | return element 49 | 50 | def element_by_ref(self, ref: str) -> Union[Element, None]: 51 | element = self.parser.find_reference(ref) 52 | if element is None: 53 | self.logger.warning(f'PathHandler - no element found with reference \'{ref}\'') 54 | return None 55 | return element 56 | 57 | def element_by_inline_ref(self, path: DataQuery.XPath, node: Element) -> Union[Element, None]: 58 | path_to_reference, path_to_value = self.__split(path.xpath) 59 | 60 | reference = self.element_by_xpath(path_to_reference, node) 61 | if reference is None or 'REF' not in reference.tag: 62 | self.logger.warning( 63 | f'PathHandler - processing inline reference, no reference found at \'{path.xpath}\'' 64 | ) 65 | return None 66 | 67 | # if inline reference and value path is SHORT-NAME, it skips getting referenced element 68 | # to increase parsing performance. Instead the element containing the reference is returned. 69 | # This is only possible because the SHORT-NAME can be extracted directly from reference.text. 70 | # The special treatment is implemented in ValueHandler. 71 | if path_to_value == 'SHORT-NAME': 72 | return reference 73 | 74 | referred_element = self.element_by_ref(reference.text) 75 | return self.element_by_xpath(path_to_value, referred_element) 76 | 77 | def __split(self, inline_ref: str) -> Tuple[str, str]: 78 | start_inline_ref = '&(' 79 | if not inline_ref.startswith(start_inline_ref): 80 | error = f'PathHandler - invalid inline reference syntax \'{inline_ref}\'. Syntax: \'&()\'' 81 | self.logger.error(error) 82 | raise ValueError(error) 83 | 84 | parantheses_count = 1 85 | path_to_value = path_to_reference = None 86 | ref_start_idx = len(start_inline_ref) 87 | for i, c in enumerate(inline_ref[ref_start_idx:], ref_start_idx): 88 | if c == '(': 89 | parantheses_count += 1 90 | elif c == ')': 91 | parantheses_count -= 1 92 | if parantheses_count == 0: 93 | path_to_reference = inline_ref[ref_start_idx:i] 94 | path_to_value = inline_ref[i + 1:] 95 | break 96 | if path_to_value is None or path_to_reference is None: 97 | error = f'PathHandler - mismatching parantheses in inline reference \'{inline_ref}\'' 98 | self.logger.error(error) 99 | raise ValueError(error) 100 | 101 | return path_to_reference, path_to_value 102 | -------------------------------------------------------------------------------- /arxml_data_extractor/handler/value_handler.py: -------------------------------------------------------------------------------- 1 | import arrow 2 | import logging 3 | from lxml.etree import Element 4 | from typing import Any, Union 5 | 6 | from arxml_data_extractor.query.data_query import DataQuery 7 | 8 | 9 | def handle(query: DataQuery, node: Element) -> Any: 10 | # Special treatment for inline references pointing to the references SHORT-NAME 11 | if isinstance(query.path, DataQuery.XPath) \ 12 | and query.path.is_reference \ 13 | and query.path.xpath.endswith(')SHORT-NAME'): 14 | value = node.text.split('/')[-1] 15 | else: 16 | value = __get_value(query.value, node) 17 | 18 | if value is None: 19 | return value 20 | return __convert_value(value, query.format) 21 | 22 | 23 | def __get_value(value: str, node: Element) -> Union[str, None]: 24 | if value == 'text': 25 | return node.text 26 | elif value == 'tag': 27 | return node.tag 28 | elif value.startswith('@'): 29 | attribute = value[1:] 30 | if attribute in node.attrib: 31 | return node.attrib[attribute] 32 | logging.getLogger().warning(f'ValueHandler - no attribute found with name \'{attribute}\'') 33 | return None 34 | else: 35 | error = f'ValueHandler - invalid value syntax \'{value}\'. Value must be either \'tag\', \'text\' or \'@..\'' 36 | logging.getLogger().error(error) 37 | raise Exception(error) 38 | 39 | 40 | def __convert_value(value: str, format: DataQuery.Format) -> Any: 41 | try: 42 | if format == DataQuery.Format.String: 43 | return value 44 | elif format == DataQuery.Format.Integer: 45 | return int(value) 46 | elif format == DataQuery.Format.Float: 47 | return float(value) 48 | elif format == DataQuery.Format.Date: 49 | return arrow.get(value) 50 | else: 51 | logging.getLogger().warning( 52 | f'ValueHandler - convertion error {value} to {format} -> fallback to string') 53 | return value 54 | except Exception as e: 55 | logging.getLogger().exception( 56 | f'ValueHandler - error while converting {value} to {format} -> fallback to string', e) 57 | return value 58 | -------------------------------------------------------------------------------- /arxml_data_extractor/output/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brokdar/ArxmlDataExtractor/2853112cbd4d001418b11ccb99f1db268347dfab/arxml_data_extractor/output/__init__.py -------------------------------------------------------------------------------- /arxml_data_extractor/output/excel_writer.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from xlsxwriter import Workbook 3 | 4 | from arxml_data_extractor.output.tabularize import tabularize 5 | 6 | 7 | @dataclass 8 | class Cell(): 9 | row: int 10 | col: int 11 | val: any 12 | col_span: int = 0 13 | is_object: bool = False 14 | 15 | 16 | class ExcelWriter(): 17 | 18 | def write(self, file: str, data: dict): 19 | workbook = Workbook(file) 20 | self.header_format = workbook.add_format({ 21 | 'bold': True, 22 | 'align': 'center', 23 | 'bg_color': 'white' 24 | }) 25 | 26 | for key, value in data.items(): 27 | sheet = workbook.add_worksheet(key) 28 | row_count = self.write_header(sheet, value) 29 | self.write_data_frames(sheet, value, row_count) 30 | 31 | workbook.close() 32 | 33 | def write_header(self, sheet, data): 34 | headers = [] 35 | max_row, max_col = self.analyze_header(data, headers) 36 | self.preformat_header(sheet, max_row, max_col) 37 | 38 | while headers: 39 | cell = headers.pop() 40 | if cell.is_object: 41 | if cell.col_span > 0: 42 | sheet.merge_range(cell.row, cell.col, cell.row, cell.col + cell.col_span, 43 | cell.val, self.header_format) 44 | else: 45 | sheet.write(cell.row, cell.col, cell.val, self.header_format) 46 | else: 47 | sheet.write(max_row, cell.col, cell.val, self.header_format) 48 | 49 | return max_row 50 | 51 | def preformat_header(self, sheet, rows, cols): 52 | for row in range(rows): 53 | for col in range(cols): 54 | sheet.write(row, col, None, self.header_format) 55 | 56 | def analyze_header(self, data, header, row=0, col=0): 57 | max_row = row 58 | 59 | if isinstance(data, list): 60 | data = data[0] 61 | 62 | for key, value in data.items(): 63 | if isinstance(value, (list, dict)): 64 | max_row, n_col = self.analyze_header(value, header, row + 1, col) 65 | header.append(Cell(row, col, key, n_col - col, True)) 66 | col = n_col 67 | else: 68 | header.append(Cell(row, col, key)) 69 | col += 1 70 | 71 | return max_row, col - 1 72 | 73 | def write_data_frames(self, sheet, data, start_row): 74 | if isinstance(data, dict): 75 | data = [data] 76 | 77 | data_frames = tabularize(data) 78 | for row_idx, df in enumerate(data_frames, start_row + 1): 79 | sheet.write_row(row_idx, 0, df) 80 | 81 | row_count = len(data_frames) 82 | column_count = len(data_frames[0]) - 1 83 | sheet.autofilter(start_row, 0, row_count, column_count) 84 | -------------------------------------------------------------------------------- /arxml_data_extractor/output/tabularize.py: -------------------------------------------------------------------------------- 1 | def tabularize(data: list) -> list: 2 | rows = __flatten(data) 3 | return rows 4 | 5 | 6 | def __flatten(data): 7 | if isinstance(data, dict): 8 | rows = [] 9 | for value in data.values(): 10 | res = __flatten(value) 11 | if isinstance(res, list): 12 | if isinstance(res[0], list): 13 | rows = __concatenate(rows, res) 14 | else: 15 | rows.extend(res) 16 | else: 17 | rows.append(res) 18 | return rows 19 | elif isinstance(data, list): 20 | rows = [] 21 | for value in data: 22 | res = __flatten(value) 23 | if isinstance(res, list) and type(res[0]) is list: 24 | rows.extend(res) 25 | else: 26 | rows.append(res) 27 | return rows 28 | else: 29 | return data 30 | 31 | 32 | # len(left) == len(right) 33 | # left = [[a, b], [c, d]] 34 | # right = [[w, x], [y, z]] 35 | # res = [[a, b, w, x], 36 | # [c, d, y, z]] 37 | # 38 | # len(left) > len(right) 39 | # left = [[a, b], [c, d], [e, f]] 40 | # right = [[w, x], [y, z]] 41 | # res = [[a, b, w, x], 42 | # [a, b, y, z], 43 | # [c, d, w, x], 44 | # [c, d, y, z], 45 | # [e, f, w, x], 46 | # [e, f, y, z]] 47 | # 48 | # len(left) < len(right) 49 | # left = [[a, b], [c, d]] 50 | # right = [[u, v], [w, x], [y, z]] 51 | # res = [[a, b, u, v], 52 | # [a, b, w, x], 53 | # [a, b, y, z], 54 | # [c, d, u, v], 55 | # [c, d, w, x], 56 | # [c, d, y, z]] 57 | def __concatenate(left: list, right: list) -> list: 58 | if not left: 59 | return right 60 | 61 | if type(left[0]) is not list: 62 | return [left + r for r in right] 63 | 64 | if len(left) == len(right): 65 | return [l + right[i] for i, l in enumerate(left)] 66 | 67 | return [l + r for l in left for r in right] 68 | -------------------------------------------------------------------------------- /arxml_data_extractor/output/text_writer.py: -------------------------------------------------------------------------------- 1 | from tabulate import tabulate 2 | 3 | from arxml_data_extractor.output.tabularize import tabularize 4 | 5 | 6 | class TextWriter(): 7 | 8 | def as_table(self, data: dict) -> str: 9 | text = [] 10 | headers = self.analyze_headers(data) 11 | for i, values in enumerate(data.values()): 12 | if not isinstance(values, list): 13 | values = [values] 14 | rows = tabularize(values) 15 | text.append(tabulate(rows, headers=headers[i], tablefmt="orgtbl")) 16 | 17 | return '\n\n\n'.join(text) 18 | 19 | def analyze_headers(self, data: dict) -> list: 20 | headers = [] 21 | 22 | for value in data.values(): 23 | headers.append(self.__names(value)) 24 | 25 | return headers 26 | 27 | @classmethod 28 | def __names(cls, data): 29 | names = [] 30 | if isinstance(data, dict): 31 | for key, values in data.items(): 32 | if isinstance(values, (list, dict)): 33 | names.extend(cls.__names(values)) 34 | else: 35 | names.append(key) 36 | else: 37 | # first entry is enough to collect all names 38 | names.extend(cls.__names(data[0])) 39 | return names 40 | 41 | def as_dictionary(self, data: dict): 42 | text = [] 43 | for key, value in data.items(): 44 | self.__data_to_text(key, value, text) 45 | 46 | return '\n'.join(text) 47 | 48 | @classmethod 49 | def __data_to_text(cls, name, data, text, indent=0): 50 | if isinstance(data, dict): 51 | text.append(f'{" " * indent}{name}:') 52 | for key, value in data.items(): 53 | cls.__data_to_text(key, value, text, indent + 1) 54 | elif isinstance(data, list): 55 | for i, value in enumerate(data): 56 | n = f'{name}[{i}]' 57 | cls.__data_to_text(n, value, text, indent) 58 | else: 59 | text.append(f'{" " * indent}{name}: {data}') 60 | -------------------------------------------------------------------------------- /arxml_data_extractor/query/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brokdar/ArxmlDataExtractor/2853112cbd4d001418b11ccb99f1db268347dfab/arxml_data_extractor/query/__init__.py -------------------------------------------------------------------------------- /arxml_data_extractor/query/data_object.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Union, List 4 | import logging 5 | 6 | from arxml_data_extractor.query.data_query import DataQuery 7 | from arxml_data_extractor.query.data_value import DataValue 8 | 9 | 10 | class DataObject(): 11 | 12 | def __init__(self, name: str, path: Union[DataQuery.XPath, DataQuery.Reference], 13 | values: List[Union[DataValue, DataObject]]): 14 | 15 | self.logger = logging.getLogger() 16 | self.name = name 17 | self.path = self.__set_path(path) 18 | self.values = self.__set_values(values) 19 | 20 | def __set_path(self, path): 21 | if (isinstance(path, (DataQuery.Reference, DataQuery.XPath))): 22 | return path 23 | else: 24 | error = f'DataObject(\'{self.name}\') - invalid path type ({type(path)}). Path must be of type DataQuery.XPath or DataQuery.Reference' 25 | self.logger.error(error) 26 | raise TypeError(error) 27 | 28 | def __set_values(self, values): 29 | if type(values) is not list: 30 | error = f'DataObject(\'{self.name}\') - values ({type(values)}) must be of type list' 31 | self.logger.error(error) 32 | raise TypeError(error) 33 | if not values: 34 | error = f'DataObject(\'{self.name}\') - values cannot be empty' 35 | self.logger.error(error) 36 | raise ValueError(error) 37 | if all(isinstance(x, (DataObject, DataValue)) for x in values): 38 | return values 39 | 40 | error = f'DataObject(\'{self.name}\') - values can only hold items of type DataObject or DataValue' 41 | self.logger.error(error, [f'{v}: {type(v)}' for v in values]) 42 | raise ValueError(error) 43 | -------------------------------------------------------------------------------- /arxml_data_extractor/query/data_query.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | from enum import Enum 5 | from typing import Union 6 | from dataclasses import dataclass 7 | 8 | 9 | class DataQuery(): 10 | 11 | @dataclass 12 | class Reference(): 13 | ref: str 14 | 15 | @dataclass 16 | class XPath(): 17 | xpath: str 18 | is_reference: bool = False 19 | 20 | class Format(Enum): 21 | String = 0 22 | Integer = 1 23 | Float = 2 24 | Date = 3 25 | 26 | def __init__(self, 27 | path: Union[XPath, Reference], 28 | value: str = 'text', 29 | format: Format = Format.String): 30 | 31 | self.logger = logging.getLogger() 32 | self.path = self.__set_path(path) 33 | self.value = self.__set_value(value) 34 | self.format = format 35 | 36 | def __set_value(self, value): 37 | if (value == 'text' or value == 'tag' or value.startswith('@')): 38 | return value 39 | 40 | error = f'DataQuery - invalid value \'{value}\'. Value needs to be either \'tag\', \'text\' or \'@..\'' 41 | self.logger.error(error) 42 | raise ValueError(error) 43 | 44 | def __set_path(self, path): 45 | if (isinstance(path, DataQuery.XPath)): 46 | if (':' in path.xpath): 47 | error = f'DataQuery - invalid XPath \'{path.xpath}\'. XPath must not contain namespace information' 48 | self.logger.error(error) 49 | raise ValueError(error) 50 | elif isinstance(path, DataQuery.Reference): 51 | if (':' in path.ref): 52 | error = f'DataQuery - invalid Reference \'{path.ref}\'. Reference must not contain namespace information' 53 | self.logger.error(error) 54 | raise ValueError(error) 55 | else: 56 | error = f'DataQuery - invalid path type ({type(path)}). Path must be of type DataQuery.XPath or DataQuery.Reference' 57 | logging.error(error) 58 | raise TypeError(error) 59 | 60 | return path 61 | -------------------------------------------------------------------------------- /arxml_data_extractor/query/data_value.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from arxml_data_extractor.query.data_query import DataQuery 4 | 5 | 6 | class DataValue: 7 | 8 | def __init__(self, name: str, query: DataQuery): 9 | self.name = name 10 | self.logger = logging.getLogger() 11 | if (query is None): 12 | error = f'DataValue(\'{self.name}\') - invalid query type ({type(query)}). Query must be of type DataQuery' 13 | self.logger.error(error) 14 | raise TypeError(error) 15 | self.query = query 16 | -------------------------------------------------------------------------------- /arxml_data_extractor/query_builder.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from tqdm import tqdm 3 | import logging 4 | 5 | from arxml_data_extractor.query.data_object import DataObject 6 | from arxml_data_extractor.query.data_query import DataQuery 7 | from arxml_data_extractor.query.data_value import DataValue 8 | 9 | 10 | class QueryBuilder(): 11 | __path_separator = ':' 12 | __format_separator = '>' 13 | 14 | def __init__(self): 15 | self.logger = logging.getLogger() 16 | 17 | def build(self, config: dict) -> List[DataObject]: 18 | data_objects = [] 19 | for key, value in tqdm( 20 | config.items(), 21 | desc='Building Queries', 22 | bar_format="{desc:<70}{percentage:3.0f}% |{bar:70}| {n_fmt:>4}/{total_fmt}"): 23 | data_object = self.__parse_object(key, value) 24 | data_objects.append(data_object) 25 | return data_objects 26 | 27 | def __parse_object(self, name: str, values: dict) -> DataObject: 28 | required = {'_xpath', '_xref', '_ref'} 29 | path_value = required & values.keys() 30 | if len(path_value) != 1: 31 | error = f'QueryBuilder - DataObject({name}) is missing an anchor. Possible anchors are \'_xpath\', \'_ref\' or \'_xref\'' 32 | self.logger.error(error) 33 | raise ValueError(error) 34 | 35 | if '_xpath' in path_value: 36 | xpath = values['_xpath'].split(self.__path_separator)[-1] 37 | path = DataQuery.XPath(xpath) 38 | elif '_xref' in path_value: 39 | xpath = values['_xref'].split(self.__path_separator)[-1] 40 | path = DataQuery.XPath(xpath, True) 41 | else: 42 | ref = values['_ref'].split(self.__path_separator)[-1] 43 | path = DataQuery.Reference(ref) 44 | 45 | data_values = [] 46 | for key, value in values.items(): 47 | if key in required: 48 | continue 49 | 50 | if isinstance(value, dict): 51 | data_object = self.__parse_object(key, value) 52 | data_values.append(data_object) 53 | else: 54 | data_value = self.__parse_value(key, value) 55 | data_values.append(data_value) 56 | 57 | return DataObject(name, path, data_values) 58 | 59 | def __parse_value(self, name: str, value: str) -> DataValue: 60 | try: 61 | query = self.__parse_query(value) 62 | return DataValue(name, query) 63 | except Exception as e: 64 | error = f'QueryBuilder - parsing error on query for value \'{name}\': \'{value}\'' 65 | self.logger.error(error, exc_info=e) 66 | raise ValueError(error) from e 67 | 68 | def __parse_query(self, text: str) -> DataQuery: 69 | if self.__path_separator not in text: 70 | path = self.__get_path(text) 71 | return DataQuery(path) 72 | 73 | raw_value_format, raw_path = text.split(self.__path_separator, 2) 74 | path = self.__get_path(raw_path) 75 | 76 | if self.__format_separator in raw_value_format: 77 | raw_value, raw_format = raw_value_format.split(self.__format_separator) 78 | value = self.__get_value(raw_value) 79 | format = self.__get_format(raw_format) 80 | else: 81 | value = self.__get_value(raw_value_format) 82 | format = DataQuery.Format.String 83 | 84 | return DataQuery(path, value, format) 85 | 86 | def __get_path(self, path: str) -> DataQuery.XPath: 87 | illegal_character = [self.__path_separator, self.__format_separator] 88 | if any(c in illegal_character for c in path): 89 | error = f'QueryBuilder - illegal characters found on path \'{path}\'. Path must not contain any of the following characters: \':\', \'>\'' 90 | self.logger.error(error) 91 | raise ValueError(error) 92 | 93 | if path[0] != '&': 94 | return DataQuery.XPath(path) 95 | else: 96 | if path[1] != '(': 97 | error = f'QueryBuilder - invalid syntax on inline reference \'{path}\'. Inline references must start with \'&(\'' 98 | self.logger.error(error) 99 | raise ValueError(error) 100 | return DataQuery.XPath(path, True) 101 | 102 | def __get_value(self, value: str) -> str: 103 | if (value == '') or (value == 'tag') or (value == 'text'): 104 | return value 105 | elif value.startswith('@'): 106 | if len(value) > 1: 107 | return value 108 | else: 109 | error = f'QueryBuilder - invalid syntax on attribute. Attribute name must be defined' 110 | self.logger.error(error) 111 | raise ValueError(error) 112 | else: 113 | return 'text' 114 | 115 | def __get_format(self, format: str) -> DataQuery.Format: 116 | if (format == '') or (format == 'string'): 117 | return DataQuery.Format.String 118 | elif (format == 'int'): 119 | return DataQuery.Format.Integer 120 | elif (format == 'float'): 121 | return DataQuery.Format.Float 122 | elif (format == 'date'): 123 | return DataQuery.Format.Date 124 | else: 125 | return DataQuery.Format.String 126 | -------------------------------------------------------------------------------- /arxml_data_extractor/query_handler.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from pathlib import Path 3 | import logging 4 | 5 | from arxml_data_extractor.asr.asr_parser import AsrParser 6 | from arxml_data_extractor.handler.object_handler import ObjectHandler 7 | from arxml_data_extractor.query.data_object import DataObject 8 | 9 | 10 | class QueryHandler(): 11 | 12 | def __init__(self): 13 | self.logger = logging.getLogger() 14 | 15 | def handle_queries(self, input: str, queries: List[DataObject]) -> dict: 16 | arxml = Path(input) 17 | if not arxml.exists: 18 | error = f'QueryHandler - input file doesn\'t exist \'{input}\'' 19 | self.logger.error(error) 20 | raise ValueError(error) 21 | if not arxml.is_file: 22 | error = f'QueryHandler - input is not a file \'{input}\'' 23 | self.logger.error(error) 24 | raise ValueError(error) 25 | if arxml.suffix != '.arxml': 26 | error = f'QueryHandler - invalid input file extension \'{arxml.suffix}\' != \'.arxml\'' 27 | self.logger.error(error) 28 | raise ValueError(error) 29 | 30 | object_handler = ObjectHandler(AsrParser(str(arxml))) 31 | 32 | results = {} 33 | for data_object in queries: 34 | if (not isinstance(data_object, DataObject)): 35 | error = f'QueryHandler - invalid root element type \'{type(data_object)}\' != \'DataObject\'' 36 | self.logger.error(error) 37 | raise TypeError(error) 38 | 39 | results[data_object.name] = object_handler.handle(data_object) 40 | 41 | return results 42 | -------------------------------------------------------------------------------- /arxml_data_extractor/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brokdar/ArxmlDataExtractor/2853112cbd4d001418b11ccb99f1db268347dfab/arxml_data_extractor/tests/__init__.py -------------------------------------------------------------------------------- /arxml_data_extractor/tests/asr/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brokdar/ArxmlDataExtractor/2853112cbd4d001418b11ccb99f1db268347dfab/arxml_data_extractor/tests/asr/__init__.py -------------------------------------------------------------------------------- /arxml_data_extractor/tests/asr/test.arxml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | SystemSignals 8 | 9 | 10 | 11 | SignalData1 12 | false 13 | 14 | 15 | 16 | SignalData2 17 | false 18 | 19 | 20 | 21 | 22 | 23 | Pdus 24 | 25 | 26 | Bus 27 | 28 | 29 | PduCollection 30 | 31 | 32 | 33 | MyPdu 34 | 35 | 36 | 0.02 37 | 38 | 39 | 40 | 0 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | SignalData1 50 | /ISignals/Bus/PduCollection/MyPdu/SignalData1 51 | MOST-SIGNIFICANT-BYTE-LAST 52 | 0 53 | 54 | 55 | 56 | SignalData2 57 | /ISignals/Bus/PduCollection/MyPdu/SignalData2 58 | MOST-SIGNIFICANT-BYTE-LAST 59 | 8 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | ISignals 70 | 71 | 72 | 73 | Bus 74 | 75 | 76 | PduCollection 77 | 78 | 79 | MyPdu 80 | 81 | 82 | 83 | SignalData1 84 | PRIMITIVE 85 | 86 | 87 | 0 88 | 89 | 90 | 1 91 | /SystemSignals/SignalData1 92 | 93 | 94 | 95 | SignalData2 96 | PRIMITIVE 97 | 98 | 99 | 0 100 | 101 | 102 | 1 103 | /SystemSignals/SignalData2 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | -------------------------------------------------------------------------------- /arxml_data_extractor/tests/asr/test_asr_parser.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from arxml_data_extractor.asr.asr_parser import AsrParser 4 | 5 | 6 | @pytest.fixture 7 | def parser() -> AsrParser: 8 | arxml = 'arxml_data_extractor/tests/asr/test.arxml' 9 | return AsrParser(arxml) 10 | 11 | 12 | def test_lists_all_top_level_packages(parser: AsrParser): 13 | packages = parser.packages 14 | 15 | assert len(packages) == 3 16 | 17 | 18 | def test_find_all_elements_by_element_name(parser: AsrParser): 19 | packages = parser.find_all_elements('AR-PACKAGE') 20 | 21 | assert len(packages) == 6 22 | 23 | 24 | def test_find_all_elements_by_specified_path(parser: AsrParser): 25 | top_level_packages = parser.find_all_elements('AR-PACKAGES/AR-PACKAGE') 26 | 27 | assert len(top_level_packages) == 6 28 | 29 | 30 | def test_find_elements_specified_with_namespace(parser: AsrParser): 31 | signals = parser.find_all_elements('ar:I-SIGNAL') 32 | 33 | assert len(signals) == 2 34 | 35 | 36 | def test_find_elements_partly_specified_with_namespace(parser: AsrParser): 37 | pdus = parser.find_all_elements('ELEMENTS/ar:I-SIGNAL-I-PDU') 38 | 39 | assert len(pdus) == 1 40 | 41 | 42 | def test_find_elements_from_defined_base_by_path(parser: AsrParser): 43 | pdu_package = parser.packages['Pdus'] 44 | pdus = AsrParser.find_elements(pdu_package, 'I-SIGNAL-I-PDU') 45 | 46 | assert len(pdus) == 1 47 | 48 | 49 | def test_find_element_from_defined_base_by_shortname(parser: AsrParser): 50 | signal_package = parser.packages['ISignals'] 51 | signal = AsrParser.find_element_by_shortname(signal_package, 'I-SIGNAL', 'SignalData1') 52 | 53 | assert signal is not None 54 | assert AsrParser.get_shortname(signal) == 'SignalData1' 55 | 56 | 57 | def test_get_shortname(parser: AsrParser): 58 | pdu_package = parser.packages['Pdus'] 59 | shortname = AsrParser.get_shortname(pdu_package) 60 | 61 | assert shortname == 'Pdus' 62 | 63 | 64 | def test_find_element_by_reference(parser: AsrParser): 65 | ref = '/ISignals/Bus/PduCollection/MyPdu/SignalData2' 66 | signal = parser.find_reference(ref) 67 | 68 | assert signal.tag == '{http://autosar.org/schema/r4.0}I-SIGNAL' 69 | assert AsrParser.get_shortname(signal) == 'SignalData2' 70 | -------------------------------------------------------------------------------- /arxml_data_extractor/tests/output/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brokdar/ArxmlDataExtractor/2853112cbd4d001418b11ccb99f1db268347dfab/arxml_data_extractor/tests/output/__init__.py -------------------------------------------------------------------------------- /arxml_data_extractor/tests/output/test_tabularize.py: -------------------------------------------------------------------------------- 1 | from arxml_data_extractor.output.tabularize import tabularize 2 | 3 | 4 | def test_flatten_dict(): 5 | input = {'Name': 'Message', 'Length': 16, 'Cyclic Timing': 0.1} 6 | 7 | result = tabularize([input]) 8 | 9 | assert result == [['Message', 16, 0.1]] 10 | 11 | 12 | def test_flatten_nested_dict(): 13 | input = { 14 | 'Name': 'Message', 15 | 'Length': 8, 16 | 'Cyclic Timing': 0.1, 17 | 'Signal': { 18 | 'Name': 'Signal1', 19 | 'Length': 8 20 | } 21 | } 22 | 23 | result = tabularize([input]) 24 | 25 | assert result == [['Message', 8, 0.1, 'Signal1', 8]] 26 | 27 | 28 | def test_multiple_entries(): 29 | input = [{'Name': 'Request', 'Length': 8}, {'Name': 'Response', 'Length': 16}] 30 | 31 | result = tabularize(input) 32 | 33 | assert result == [['Request', 8], ['Response', 16]] 34 | 35 | 36 | def test_dict_containing_list(): 37 | input = { 38 | 'Name': 'Message', 39 | 'Length': 16, 40 | 'Signal': [{ 41 | 'Name': 'Signal1', 42 | 'Length': 8 43 | }, { 44 | 'Name': 'Signal2', 45 | 'Length': 8 46 | }] 47 | } 48 | 49 | result = tabularize([input]) 50 | 51 | assert result == [['Message', 16, 'Signal1', 8], ['Message', 16, 'Signal2', 8]] 52 | 53 | 54 | def test_dict_containing_lists_with_same_dimension(): 55 | input = { 56 | 'Name': 'Message', 57 | 'Length': 16, 58 | 'Signal': [{ 59 | 'Name': 'Signal1', 60 | 'Length': 8 61 | }, { 62 | 'Name': 'Signal2', 63 | 'Length': 8 64 | }], 65 | 'SignalGroup': [{ 66 | 'Name': 'SignalGroup1' 67 | }, { 68 | 'Name': 'SignalGroup2' 69 | }] 70 | } 71 | 72 | result = tabularize([input]) 73 | 74 | assert result == [['Message', 16, 'Signal1', 8, 'SignalGroup1'], 75 | ['Message', 16, 'Signal2', 8, 'SignalGroup2']] 76 | 77 | 78 | def test_dict_containing_lists_different_dimensions(): 79 | input = { 80 | 'Name': 81 | 'Message', 82 | 'Length': 83 | 16, 84 | 'Signal': [{ 85 | 'Name': 'Signal1', 86 | 'Length': 8 87 | }, { 88 | 'Name': 'Signal2', 89 | 'Length': 8 90 | }], 91 | 'SignalGroup': [{ 92 | 'Name': 'SignalGroup1' 93 | }, { 94 | 'Name': 'SignalGroup2' 95 | }, { 96 | 'Name': 'SignalGroup3' 97 | }] 98 | } 99 | 100 | result = tabularize([input]) 101 | 102 | assert result == [['Message', 16, 'Signal1', 8, 'SignalGroup1'], 103 | ['Message', 16, 'Signal1', 8, 'SignalGroup2'], 104 | ['Message', 16, 'Signal1', 8, 'SignalGroup3'], 105 | ['Message', 16, 'Signal2', 8, 'SignalGroup1'], 106 | ['Message', 16, 'Signal2', 8, 'SignalGroup2'], 107 | ['Message', 16, 'Signal2', 8, 'SignalGroup3']] 108 | -------------------------------------------------------------------------------- /arxml_data_extractor/tests/query/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brokdar/ArxmlDataExtractor/2853112cbd4d001418b11ccb99f1db268347dfab/arxml_data_extractor/tests/query/__init__.py -------------------------------------------------------------------------------- /arxml_data_extractor/tests/query/test_data_object.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from arxml_data_extractor.query.data_query import DataQuery 4 | from arxml_data_extractor.query.data_object import DataObject 5 | from arxml_data_extractor.query.data_value import DataValue 6 | 7 | 8 | @pytest.fixture 9 | def value(): 10 | query = DataQuery(DataQuery.XPath('/SHORT-NAME')) 11 | return DataValue('Name', query) 12 | 13 | 14 | @pytest.mark.parametrize("name, path", [ 15 | ('Signal', DataQuery.XPath('//I-SIGNALS')), 16 | ('Signal123', DataQuery.XPath('//I-SIGNALS')), 17 | ('Signal_123', DataQuery.XPath('//I-SIGNALS')), 18 | ('Signal 123', DataQuery.XPath('//I-SIGNALS')), 19 | ('Signal 123', DataQuery.Reference('/Signals')), 20 | ]) 21 | def test_create_data_object(name, path, value): 22 | data_object = DataObject(name, path, [value]) 23 | 24 | assert data_object.name == name 25 | assert data_object.path == path 26 | assert data_object.values[0] == value 27 | 28 | 29 | def test_values_contain_data_objects(value): 30 | sub_object = DataObject('Timings', DataQuery.XPath('//TIMING'), [value]) 31 | data_object = DataObject('Signals', DataQuery.XPath('//I-SIGNALS'), [sub_object]) 32 | 33 | assert data_object.name == 'Signals' 34 | data_object_value = data_object.values[0] 35 | assert isinstance(data_object_value, DataObject) 36 | assert data_object_value == sub_object 37 | assert data_object_value.values[0] == value 38 | 39 | 40 | def test_empty_values_raises_value_error(): 41 | with pytest.raises(ValueError): 42 | DataObject('Signals', DataQuery.XPath('/SHORT-NAME'), []) 43 | 44 | 45 | def test_invalid_values_raise_type_error(value): 46 | with pytest.raises(TypeError): 47 | DataObject('Signals', DataQuery.XPath('/SHORT-NAME'), value) 48 | 49 | 50 | def test_invalid_path_type_raises_type_error(value): 51 | with pytest.raises(TypeError): 52 | DataObject('Signals', None, [value]) 53 | DataObject('Signals', 'path', [value]) 54 | -------------------------------------------------------------------------------- /arxml_data_extractor/tests/query/test_data_query.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from arxml_data_extractor.query.data_query import DataQuery 4 | 5 | 6 | @pytest.mark.parametrize("path, value, format", [ 7 | (DataQuery.Reference('/Signals/Signal1'), 'text', DataQuery.Format.String), 8 | (DataQuery.Reference('/Signals/Signal1'), 'tag', DataQuery.Format.Integer), 9 | (DataQuery.Reference('/Signals/Signal1'), '@', DataQuery.Format.Float), 10 | (DataQuery.Reference('/Signals/Signal1'), '@attribute', DataQuery.Format.Date), 11 | (DataQuery.XPath('/SHORT-NAME', False), 'text', DataQuery.Format.String), 12 | (DataQuery.XPath('/SHORT-NAME', True), 'tag', DataQuery.Format.Integer), 13 | (DataQuery.XPath('/SHORT-NAME'), '@', DataQuery.Format.Float), 14 | (DataQuery.XPath('/SHORT-NAME'), '@attribute', DataQuery.Format.Date), 15 | ]) 16 | def test_create_data_query(path, value, format): 17 | query = DataQuery(path, value, format) 18 | 19 | assert query.path == path 20 | assert query.value == value 21 | assert query.format == format 22 | 23 | 24 | @pytest.mark.parametrize("path, value", [ 25 | (DataQuery.Reference('/Signals/Signal1'), 'text'), 26 | (DataQuery.Reference('/Signals/Signal1'), '@attribute'), 27 | (DataQuery.XPath('/SHORT-NAME', True), 'tag'), 28 | (DataQuery.XPath('/SHORT-NAME'), '@'), 29 | ]) 30 | def test_create_data_query_with_default_format(path, value): 31 | query = DataQuery(path, value) 32 | 33 | assert query.path == path 34 | assert query.value == value 35 | assert query.format == DataQuery.Format.String 36 | 37 | 38 | def test_create_data_query_with_default_value(): 39 | path = DataQuery.XPath('/SHORT-NAME') 40 | format = DataQuery.Format.Date 41 | 42 | query = DataQuery(path, format=format) 43 | 44 | assert query.path == path 45 | assert query.value == 'text' 46 | assert query.format == format 47 | 48 | 49 | def test_create_data_query_with_absolute_xpath(): 50 | path = DataQuery.XPath('/AR-PACKAGE/SHORT-NAME', False) 51 | 52 | query = DataQuery(path) 53 | 54 | assert query.path == path 55 | assert query.value == 'text' 56 | assert query.format == DataQuery.Format.String 57 | 58 | 59 | def test_create_data_query_with_relative_xpath(): 60 | path = DataQuery.XPath('/SHORT-NAME') 61 | 62 | query = DataQuery(path) 63 | 64 | assert query.path == path 65 | assert query.value == 'text' 66 | assert query.format == DataQuery.Format.String 67 | 68 | 69 | def test_invalid_value_raise_value_error(): 70 | with pytest.raises(ValueError): 71 | DataQuery(DataQuery.XPath('/SHORT-NAME'), 'something') 72 | 73 | 74 | def test_invalid_path_raises_exception(): 75 | with pytest.raises(TypeError): 76 | DataQuery('/SHORT-NAME') 77 | 78 | 79 | def test_xpath_containing_namespaces_raise_value_error(): 80 | with pytest.raises(ValueError): 81 | DataQuery(DataQuery.XPath('/ar:SHORT-NAME')) 82 | 83 | 84 | def test_reference_containing_namespaces_raise_value_error(): 85 | with pytest.raises(ValueError): 86 | DataQuery(DataQuery.Reference('/ar:Signals')) 87 | -------------------------------------------------------------------------------- /arxml_data_extractor/tests/query/test_data_value.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from arxml_data_extractor.query.data_query import DataQuery 4 | from arxml_data_extractor.query.data_value import DataValue 5 | 6 | 7 | @pytest.mark.parametrize("name, query", [ 8 | ('Signal', DataQuery(DataQuery.XPath('/SHORT-NAME'))), 9 | ('Signal123', DataQuery(DataQuery.XPath('/SHORT-NAME'))), 10 | ('Signal_123', DataQuery(DataQuery.XPath('/SHORT-NAME'))), 11 | ('Signal 123', DataQuery(DataQuery.XPath('/SHORT-NAME'))), 12 | ]) 13 | def test_create_data_value(name, query): 14 | value = DataValue(name, query) 15 | 16 | assert value.name == name 17 | assert value.query == query 18 | 19 | 20 | def test_none_query_raises_exception(): 21 | with pytest.raises(Exception): 22 | DataValue('name', None) 23 | -------------------------------------------------------------------------------- /arxml_data_extractor/tests/test.arxml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Cluster 6 | 7 | 8 | CAN 9 | CAN Channel 1 10 | 11 | 12 | 500000 13 | CAN 14 | 15 | 16 | 17 | 18 | 19 | 20 | PDU 21 | 22 | 23 | TxMessage 24 | 5 25 | 26 | 27 | 0 28 | 29 | 30 | 31 | 0 32 | 0.1 33 | 34 | 35 | 36 | 37 | 0 38 | 0.1 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | Signal1 47 | /ISignal/Signal1 48 | MOST-SIGNIFICANT-BYTE-LAST 49 | 0 50 | PENDING 51 | 52 | 53 | 0 54 | 55 | 56 | RxMessage 57 | 2 58 | 59 | 60 | 0 61 | 62 | 63 | 64 | 0 65 | 0.1 66 | 67 | 68 | 69 | 70 | 0 71 | 0.1 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | Signal2 80 | /ISignal/Signal2 81 | MOST-SIGNIFICANT-BYTE-LAST 82 | 0 83 | PENDING 84 | 85 | 86 | Signal3 87 | /ISignal/Signal3 88 | MOST-SIGNIFICANT-BYTE-LAST 89 | 1 90 | PENDING 91 | 92 | 93 | 0 94 | 95 | 96 | 97 | 98 | ISignal 99 | 100 | 101 | Signal1 102 | 103 | 128 104 | 105 | 5 106 | 107 | 108 | Signal2 109 | LEGACY 110 | 111 | 0 112 | 113 | 1 114 | 115 | 116 | Signal3 117 | LEGACY 118 | 119 | 0 120 | 121 | 1 122 | 123 | 124 | 125 | 126 | 127 | -------------------------------------------------------------------------------- /arxml_data_extractor/tests/test_arxml_data_extractor.py: -------------------------------------------------------------------------------- 1 | from arxml_data_extractor.config_provider import ConfigProvider 2 | from arxml_data_extractor.query_builder import QueryBuilder 3 | from arxml_data_extractor.query_handler import QueryHandler 4 | 5 | arxml = 'arxml_data_extractor/tests/test.arxml' 6 | 7 | 8 | def test_extracting_simple_object(): 9 | yaml = """ 10 | 'CAN Cluster': 11 | '_ref': '/Cluster/CAN' 12 | 'Name': 'SHORT-NAME' 13 | 'Baudrate': 'text>int:CAN-CLUSTER-VARIANTS/CAN-CLUSTER-CONDITIONAL/BAUDRATE' 14 | 'Long Name': 'text:LONG-NAME/L-4' 15 | 'Language': '@L:LONG-NAME/L-4' 16 | """ 17 | 18 | config_provider = ConfigProvider() 19 | config = config_provider.parse(yaml) 20 | 21 | query_builder = QueryBuilder() 22 | queries = query_builder.build(config) 23 | 24 | query_handler = QueryHandler() 25 | data = query_handler.handle_queries(arxml, queries) 26 | 27 | assert isinstance(data, dict) 28 | assert data['CAN Cluster']['Name'] == 'CAN' 29 | assert data['CAN Cluster']['Baudrate'] == 500000 30 | assert data['CAN Cluster']['Long Name'] == 'CAN Channel 1' 31 | assert data['CAN Cluster']['Language'] == 'FOR-ALL' 32 | 33 | 34 | def test_extracting_nested_objects(): 35 | yaml = """ 36 | 'PDUs': 37 | '_xpath': './/I-SIGNAL-I-PDU' 38 | 'Name': 'text:SHORT-NAME' 39 | 'Length': 'text>int:LENGTH' 40 | 'Cyclic Timing': 'text>float:.//TRANSMISSION-MODE-TRUE-TIMING/CYCLIC-TIMING/TIME-PERIOD/VALUE' 41 | 'Signal Mappings': 42 | '_xpath': './/I-SIGNAL-TO-I-PDU-MAPPING' 43 | 'Signal': 'SHORT-NAME' 44 | 'Start Position': 'text>int:START-POSITION' 45 | 'ISignal': 46 | '_xref': 'I-SIGNAL-REF' 47 | 'Init Value': 'text>int:.//VALUE' 48 | 'Length': 'text>int:LENGTH' 49 | """ 50 | config_provider = ConfigProvider() 51 | config = config_provider.parse(yaml) 52 | 53 | query_builder = QueryBuilder() 54 | queries = query_builder.build(config) 55 | 56 | query_handler = QueryHandler() 57 | data = query_handler.handle_queries(arxml, queries) 58 | 59 | assert isinstance(data, dict) 60 | assert len(data['PDUs']) == 2 61 | assert data['PDUs'][0]['Name'] == 'TxMessage' 62 | assert data['PDUs'][0]['Length'] == 5 63 | assert data['PDUs'][0]['Cyclic Timing'] == 0.1 64 | assert data['PDUs'][0]['Signal Mappings']['Signal'] == 'Signal1' 65 | assert data['PDUs'][0]['Signal Mappings']['Start Position'] == 0 66 | assert data['PDUs'][0]['Signal Mappings']['ISignal']['Init Value'] == 128 67 | assert data['PDUs'][0]['Signal Mappings']['ISignal']['Length'] == 5 68 | 69 | assert data['PDUs'][1]['Name'] == 'RxMessage' 70 | assert data['PDUs'][1]['Length'] == 2 71 | assert data['PDUs'][1]['Cyclic Timing'] == 0.1 72 | assert data['PDUs'][1]['Signal Mappings'][0]['Signal'] == 'Signal2' 73 | assert data['PDUs'][1]['Signal Mappings'][0]['Start Position'] == 0 74 | assert data['PDUs'][1]['Signal Mappings'][0]['ISignal']['Init Value'] == 0 75 | assert data['PDUs'][1]['Signal Mappings'][0]['ISignal']['Length'] == 1 76 | assert data['PDUs'][1]['Signal Mappings'][1]['Signal'] == 'Signal3' 77 | assert data['PDUs'][1]['Signal Mappings'][1]['Start Position'] == 1 78 | assert data['PDUs'][1]['Signal Mappings'][1]['ISignal']['Init Value'] == 0 79 | assert data['PDUs'][1]['Signal Mappings'][1]['ISignal']['Length'] == 1 80 | -------------------------------------------------------------------------------- /arxml_data_extractor/tests/test_config.yaml: -------------------------------------------------------------------------------- 1 | RootObject: 2 | _xpath: ".//*" 3 | SingleValue: "text:Value" 4 | ObjectValue: 5 | _xpath: ".//Object" 6 | Name: "text:Name" 7 | Data: "text>int:Data" 8 | -------------------------------------------------------------------------------- /arxml_data_extractor/tests/test_config_provider.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from arxml_data_extractor.config_provider import ConfigProvider 4 | 5 | 6 | def test_provides_config_from_file(): 7 | yaml = 'arxml_data_extractor/tests/test_config.yaml' 8 | 9 | provider = ConfigProvider() 10 | config = provider.load(yaml) 11 | 12 | assert isinstance(config, dict) 13 | assert len(config) == 1 14 | root = config['RootObject'] 15 | assert isinstance(root, dict) 16 | assert root['_xpath'] == './/*' 17 | assert root['SingleValue'] == 'text:Value' 18 | object_value = root['ObjectValue'] 19 | assert object_value['_xpath'] == './/Object' 20 | assert object_value['Name'] == 'text:Name' 21 | assert object_value['Data'] == 'text>int:Data' 22 | 23 | 24 | def test_raises_value_error_if_not_yaml(): 25 | yaml = 'test/config.json' 26 | 27 | provider = ConfigProvider() 28 | 29 | with pytest.raises(ValueError): 30 | provider.load(yaml) 31 | 32 | 33 | def test_raises_exception_if_file_not_exists(): 34 | not_existing_file = 'non-existing.yaml' 35 | 36 | provider = ConfigProvider() 37 | 38 | with pytest.raises(FileNotFoundError): 39 | provider.load(not_existing_file) 40 | 41 | 42 | def test_provides_config_from_string(): 43 | yaml = """ 44 | SingleValue: "text:/PathToElement" 45 | """ 46 | 47 | provider = ConfigProvider() 48 | config = provider.parse(yaml) 49 | 50 | assert isinstance(config, dict) 51 | assert config['SingleValue'] == 'text:/PathToElement' 52 | -------------------------------------------------------------------------------- /arxml_data_extractor/tests/test_data_writer.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pathlib import Path 3 | 4 | from arxml_data_extractor.data_writer import DataWriter 5 | 6 | 7 | @pytest.fixture 8 | def data(): 9 | return { 10 | 'PDU': [{ 11 | 'Name': 'TxMessage', 12 | 'Length': 5, 13 | 'Cyclic Timing': 0.1, 14 | 'Signal Mapping': { 15 | 'Signal': 'Signal1', 16 | 'Start Position': 0, 17 | 'I-Signal': { 18 | 'Init Value': 128, 19 | 'Length': 5 20 | } 21 | } 22 | }, { 23 | 'Name': 24 | 'RxMessage', 25 | 'Length': 26 | 2, 27 | 'Cyclic Timing': 28 | 0.1, 29 | 'Signal Mapping': [{ 30 | 'Signal': 'Signal2', 31 | 'Start Position': 0, 32 | 'I-Signal': { 33 | 'Init Value': 0, 34 | 'Length': 1 35 | } 36 | }, { 37 | 'Signal': 'Signal3', 38 | 'Start Position': 1, 39 | 'I-Signal': { 40 | 'Init Value': 0, 41 | 'Length': 1 42 | } 43 | }] 44 | }] 45 | } 46 | 47 | 48 | def test_write_text_file_as_table(data): 49 | file = Path('result.txt') 50 | writer = DataWriter() 51 | 52 | writer.write_text(str(file), data) 53 | 54 | assert file.exists() 55 | file.unlink() 56 | 57 | 58 | def test_write_to_json(data): 59 | file = Path('result.json') 60 | writer = DataWriter() 61 | 62 | writer.write_json(str(file), data) 63 | 64 | assert file.exists() 65 | file.unlink() 66 | 67 | 68 | def test_write_to_excel(data): 69 | file = Path('result.xlsx') 70 | writer = DataWriter() 71 | 72 | writer.write_excel(str(file), data) 73 | 74 | assert file.exists() 75 | file.unlink() 76 | -------------------------------------------------------------------------------- /arxml_data_extractor/tests/test_query_builder.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from arxml_data_extractor.query.data_object import DataObject 4 | from arxml_data_extractor.query_builder import QueryBuilder 5 | from arxml_data_extractor.query.data_query import DataQuery 6 | from arxml_data_extractor.query.data_value import DataValue 7 | 8 | 9 | def test_build_simple_data_object_with_xpath(): 10 | config = {'SimpleObject': {'_xpath': '/path/element', 'SimpleValue': '/SHORT-NAME'}} 11 | builder = QueryBuilder() 12 | 13 | data_objects = builder.build(config) 14 | 15 | assert isinstance(data_objects, list) 16 | data_object = data_objects[0] 17 | assert isinstance(data_object, DataObject) 18 | assert data_object.name == 'SimpleObject' 19 | assert isinstance(data_object.path, DataQuery.XPath) 20 | assert data_object.path.xpath == '/path/element' 21 | data_value = data_object.values[0] 22 | assert isinstance(data_value, DataValue) 23 | assert data_value.name == 'SimpleValue' 24 | assert isinstance(data_value.query, DataQuery) 25 | assert isinstance(data_value.query.path, DataQuery.XPath) 26 | assert data_value.query.path.xpath == '/SHORT-NAME' 27 | assert data_value.query.path.is_reference is False 28 | assert data_value.query.value == 'text' 29 | assert data_value.query.format == DataQuery.Format.String 30 | 31 | 32 | def test_build_simple_data_object_with_reference(): 33 | config = {'SimpleObject': {'_ref': '/path/element', 'SimpleValue': '/SHORT-NAME'}} 34 | builder = QueryBuilder() 35 | 36 | data_objects = builder.build(config) 37 | 38 | assert isinstance(data_objects, list) 39 | data_object = data_objects[0] 40 | assert isinstance(data_object, DataObject) 41 | assert data_object.name == 'SimpleObject' 42 | assert isinstance(data_object.path, DataQuery.Reference) 43 | assert data_object.path.ref == '/path/element' 44 | data_value = data_object.values[0] 45 | assert isinstance(data_value, DataValue) 46 | assert data_value.name == 'SimpleValue' 47 | assert isinstance(data_value.query, DataQuery) 48 | assert isinstance(data_value.query.path, DataQuery.XPath) 49 | assert data_value.query.path.xpath == '/SHORT-NAME' 50 | assert data_value.query.path.is_reference is False 51 | assert data_value.query.value == 'text' 52 | assert data_value.query.format == DataQuery.Format.String 53 | 54 | 55 | def test_object_is_missing_xpath_or_ref(): 56 | config = {'SimpleObject': {'SimpleValue': '/SHORT-NAME'}} 57 | builder = QueryBuilder() 58 | 59 | with pytest.raises(ValueError): 60 | builder.build(config) 61 | 62 | 63 | def test_object_with_xpath_and_ref(): 64 | config = { 65 | 'SimpleObject': { 66 | '_xpath': '/path/element', 67 | '_ref': '/ref/element', 68 | 'SimpleValue': '/SHORT-NAME' 69 | } 70 | } 71 | 72 | builder = QueryBuilder() 73 | 74 | with pytest.raises(ValueError): 75 | builder.build(config) 76 | 77 | 78 | def test_object_has_too_many_separators(): 79 | config = { 80 | 'SimpleObject': { 81 | '_xpath': '/path/element', 82 | 'SimpleValue': '/ar:AR-PACKAGE/ar:SHORT-NAME' 83 | } 84 | } 85 | builder = QueryBuilder() 86 | 87 | with pytest.raises(ValueError): 88 | builder.build(config) 89 | 90 | 91 | def test_text_value(): 92 | config = {'SimpleObject': {'_xpath': '/path/element', 'SimpleValue': 'text:/SHORT-NAME'}} 93 | builder = QueryBuilder() 94 | 95 | data_objects = builder.build(config) 96 | data_value = data_objects[0].values[0] 97 | 98 | assert isinstance(data_value, DataValue) 99 | assert data_value.name == 'SimpleValue' 100 | assert data_value.query.path == DataQuery.XPath('/SHORT-NAME') 101 | assert data_value.query.value == 'text' 102 | assert data_value.query.format == DataQuery.Format.String 103 | 104 | 105 | def test_tag_value(): 106 | config = {'SimpleObject': {'_xpath': '/path/element', 'SimpleValue': 'tag:/SHORT-NAME'}} 107 | builder = QueryBuilder() 108 | 109 | data_objects = builder.build(config) 110 | data_value = data_objects[0].values[0] 111 | 112 | assert isinstance(data_value, DataValue) 113 | assert data_value.name == 'SimpleValue' 114 | assert data_value.query.path == DataQuery.XPath('/SHORT-NAME') 115 | assert data_value.query.value == 'tag' 116 | assert data_value.query.format == DataQuery.Format.String 117 | 118 | 119 | def test_attribute_value(): 120 | config = {'SimpleObject': {'_xpath': '/path/element', 'SimpleValue': '@T:/SHORT-NAME'}} 121 | builder = QueryBuilder() 122 | 123 | data_objects = builder.build(config) 124 | data_value = data_objects[0].values[0] 125 | 126 | assert isinstance(data_value, DataValue) 127 | assert data_value.name == 'SimpleValue' 128 | assert data_value.query.path == DataQuery.XPath('/SHORT-NAME') 129 | assert data_value.query.value == '@T' 130 | assert data_value.query.format == DataQuery.Format.String 131 | 132 | 133 | def test_text_value_as_string(): 134 | config = {'SimpleObject': {'_xpath': '/path/element', 'SimpleValue': 'text>string:/SHORT-NAME'}} 135 | builder = QueryBuilder() 136 | 137 | data_objects = builder.build(config) 138 | data_value = data_objects[0].values[0] 139 | 140 | assert isinstance(data_value, DataValue) 141 | assert data_value.name == 'SimpleValue' 142 | assert data_value.query.path == DataQuery.XPath('/SHORT-NAME') 143 | assert data_value.query.value == 'text' 144 | assert data_value.query.format == DataQuery.Format.String 145 | 146 | 147 | def test_text_value_as_integer(): 148 | config = {'SimpleObject': {'_xpath': '/path/element', 'SimpleValue': 'text>int:/SHORT-NAME'}} 149 | builder = QueryBuilder() 150 | 151 | data_objects = builder.build(config) 152 | data_value = data_objects[0].values[0] 153 | 154 | assert isinstance(data_value, DataValue) 155 | assert data_value.name == 'SimpleValue' 156 | assert data_value.query.path == DataQuery.XPath('/SHORT-NAME') 157 | assert data_value.query.value == 'text' 158 | assert data_value.query.format == DataQuery.Format.Integer 159 | 160 | 161 | def test_text_value_as_float(): 162 | config = {'SimpleObject': {'_xpath': '/path/element', 'SimpleValue': 'text>float:/SHORT-NAME'}} 163 | builder = QueryBuilder() 164 | 165 | data_objects = builder.build(config) 166 | data_value = data_objects[0].values[0] 167 | 168 | assert isinstance(data_value, DataValue) 169 | assert data_value.name == 'SimpleValue' 170 | assert data_value.query.path == DataQuery.XPath('/SHORT-NAME') 171 | assert data_value.query.value == 'text' 172 | assert data_value.query.format == DataQuery.Format.Float 173 | 174 | 175 | def test_text_value_as_date(): 176 | config = {'SimpleObject': {'_xpath': '/path/element', 'SimpleValue': 'text>date:/SHORT-NAME'}} 177 | builder = QueryBuilder() 178 | 179 | data_objects = builder.build(config) 180 | data_value = data_objects[0].values[0] 181 | 182 | assert isinstance(data_value, DataValue) 183 | assert data_value.name == 'SimpleValue' 184 | assert data_value.query.path == DataQuery.XPath('/SHORT-NAME') 185 | assert data_value.query.value == 'text' 186 | assert data_value.query.format == DataQuery.Format.Date 187 | 188 | 189 | def test_text_value_with_invalid_format(): 190 | config = {'SimpleObject': {'_xpath': '/path/element', 'SimpleValue': 'text>bool:/SHORT-NAME'}} 191 | builder = QueryBuilder() 192 | 193 | data_objects = builder.build(config) 194 | data_value = data_objects[0].values[0] 195 | 196 | assert isinstance(data_value, DataValue) 197 | assert data_value.name == 'SimpleValue' 198 | assert data_value.query.path == DataQuery.XPath('/SHORT-NAME') 199 | assert data_value.query.value == 'text' 200 | assert data_value.query.format == DataQuery.Format.String 201 | 202 | 203 | def test_invalid_path_characters(): 204 | config = {'SimpleObject': {'_xpath': '/path/element', 'SimpleValue': '>/SHORT-NAME'}} 205 | builder = QueryBuilder() 206 | 207 | with pytest.raises(ValueError): 208 | builder.build(config) 209 | 210 | 211 | def test_invalid_attribute_characters(): 212 | config = {'SimpleObject': {'_xpath': '/path/element', 'SimpleValue': '@:/SHORT-NAME'}} 213 | builder = QueryBuilder() 214 | 215 | with pytest.raises(ValueError): 216 | builder.build(config) 217 | 218 | 219 | def test_data_objects_can_contain_data_object(): 220 | config = { 221 | 'SimpleObject': { 222 | '_xpath': '/path/element', 223 | 'SimpleValue': '/SHORT-NAME', 224 | 'OtherObject': { 225 | '_xpath': '/path/other_element', 226 | 'Value': '@T>date:/date_time' 227 | } 228 | } 229 | } 230 | 231 | builder = QueryBuilder() 232 | data_objects = builder.build(config) 233 | other_object = data_objects[0].values[1] 234 | 235 | assert isinstance(other_object, DataObject) 236 | assert isinstance(other_object.path, DataQuery.XPath) 237 | assert other_object.path.xpath == '/path/other_element' 238 | data_value = other_object.values[0] 239 | assert isinstance(data_value, DataValue) 240 | assert data_value.name == 'Value' 241 | assert isinstance(data_value.query.path, DataQuery.XPath) 242 | assert data_value.query.path.xpath == '/date_time' 243 | assert data_value.query.value == '@T' 244 | assert data_value.query.format == DataQuery.Format.Date 245 | 246 | 247 | def test_object_contains_xref(): 248 | config = { 249 | 'BaseObject': { 250 | '_xpath': '/path/element', 251 | 'SimpleValue': '/SHORT-NAME', 252 | 'ReferredObject': { 253 | '_xref': '/ref/object', 254 | 'Value': '/value' 255 | } 256 | } 257 | } 258 | 259 | builder = QueryBuilder() 260 | data_objects = builder.build(config) 261 | referred_object = data_objects[0].values[1] 262 | 263 | assert isinstance(referred_object, DataObject) 264 | assert isinstance(referred_object.path, DataQuery.XPath) 265 | assert referred_object.path.xpath == '/ref/object' 266 | assert referred_object.path.is_reference is True 267 | 268 | 269 | def test_inline_reference(): 270 | config = {'SimpleObject': {'_ref': '/path/element', 'RefValue': '&(path-to-element)value'}} 271 | 272 | builder = QueryBuilder() 273 | data_objects = builder.build(config) 274 | simple_object = data_objects[0] 275 | ref_value = simple_object.values[0] 276 | 277 | assert isinstance(simple_object, DataObject) 278 | assert isinstance(ref_value, DataValue) 279 | assert isinstance(ref_value.query.path, DataQuery.XPath) 280 | assert ref_value.query.path.xpath == '&(path-to-element)value' 281 | assert ref_value.query.path.is_reference is True 282 | 283 | 284 | def test_invalid_inline_reference(): 285 | config = {'SimpleObject': {'_ref': '/path/element', 'RefValue': '&path-to-elementvalue'}} 286 | builder = QueryBuilder() 287 | 288 | with pytest.raises(ValueError): 289 | builder.build(config) 290 | -------------------------------------------------------------------------------- /arxml_data_extractor/tests/test_query_handler.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from arxml_data_extractor.query.data_value import DataValue 4 | from arxml_data_extractor.query.data_query import DataQuery 5 | from arxml_data_extractor.query.data_object import DataObject 6 | from arxml_data_extractor.query_handler import QueryHandler 7 | 8 | arxml = 'arxml_data_extractor/tests/test.arxml' 9 | 10 | 11 | @pytest.fixture 12 | def only_value(): 13 | return DataValue( 14 | 'CAN Cluster', 15 | DataQuery(DataQuery.Reference('/Cluster/CAN1'), 'text', DataQuery.Format.String)) 16 | 17 | 18 | @pytest.fixture 19 | def simple_object_by_ref(): 20 | return DataObject('CAN Cluster', DataQuery.Reference('/Cluster/CAN'), [ 21 | DataValue('Name', DataQuery(DataQuery.XPath('./SHORT-NAME'))), 22 | DataValue( 23 | 'Baudrate', 24 | DataQuery( 25 | DataQuery.XPath('CAN-CLUSTER-VARIANTS/CAN-CLUSTER-CONDITIONAL/BAUDRATE'), 26 | format=DataQuery.Format.Integer)), 27 | DataValue('Language', DataQuery(DataQuery.XPath('LONG-NAME/L-4'), value='@L')), 28 | DataValue('Long Name', DataQuery(DataQuery.XPath('LONG-NAME/L-4'))) 29 | ]) 30 | 31 | 32 | @pytest.fixture 33 | def complex_object_by_ref(): 34 | return DataObject('CAN Cluster', DataQuery.Reference('/Cluster/CAN'), [ 35 | DataValue('Name', DataQuery(DataQuery.XPath('./SHORT-NAME'))), 36 | DataValue( 37 | 'Baudrate', 38 | DataQuery( 39 | DataQuery.XPath('CAN-CLUSTER-VARIANTS/CAN-CLUSTER-CONDITIONAL/BAUDRATE'), 40 | format=DataQuery.Format.Integer)), 41 | DataObject('Long Name', DataQuery.XPath('LONG-NAME/L-4'), [ 42 | DataValue('Language', DataQuery(DataQuery.XPath('.'), value='@L')), 43 | DataValue('Name', DataQuery(DataQuery.XPath('.'))) 44 | ]) 45 | ]) 46 | 47 | 48 | @pytest.fixture 49 | def simple_object_by_xpath(): 50 | return DataObject('CAN Cluster', DataQuery.XPath('.//AR-PACKAGE/ELEMENTS/CAN-CLUSTER'), [ 51 | DataValue('Name', DataQuery(DataQuery.XPath('./SHORT-NAME'))), 52 | DataValue( 53 | 'Baudrate', 54 | DataQuery( 55 | DataQuery.XPath('CAN-CLUSTER-VARIANTS/CAN-CLUSTER-CONDITIONAL/BAUDRATE'), 56 | format=DataQuery.Format.Integer)), 57 | DataValue('Language', DataQuery(DataQuery.XPath('LONG-NAME/L-4'), value='@L')), 58 | DataValue('Long Name', DataQuery(DataQuery.XPath('LONG-NAME/L-4'))) 59 | ]) 60 | 61 | 62 | @pytest.fixture 63 | def complex_object_by_xpath(): 64 | return DataObject('CAN Cluster', DataQuery.XPath('.//CAN-CLUSTER'), [ 65 | DataValue('Name', DataQuery(DataQuery.XPath('./SHORT-NAME'))), 66 | DataValue( 67 | 'Baudrate', 68 | DataQuery( 69 | DataQuery.XPath('CAN-CLUSTER-VARIANTS/CAN-CLUSTER-CONDITIONAL/BAUDRATE'), 70 | format=DataQuery.Format.Integer)), 71 | DataObject('Long Name', DataQuery.XPath('LONG-NAME/L-4'), [ 72 | DataValue('Language', DataQuery(DataQuery.XPath('.'), value='@L')), 73 | DataValue('Name', DataQuery(DataQuery.XPath('.'))) 74 | ]) 75 | ]) 76 | 77 | 78 | @pytest.fixture 79 | def multi_value_complex_object(): 80 | return DataObject('PDUs', DataQuery.XPath('.//I-SIGNAL-I-PDU'), [ 81 | DataValue('Name', DataQuery(DataQuery.XPath('SHORT-NAME'))), 82 | DataValue('Length', DataQuery(DataQuery.XPath('LENGTH'), format=DataQuery.Format.Integer)), 83 | DataValue( 84 | 'Unused Bit Pattern', 85 | (DataQuery(DataQuery.XPath('UNUSED-BIT-PATTERN'), format=DataQuery.Format.Integer))), 86 | DataObject('Timing Specification', DataQuery.XPath('./*/I-PDU-TIMING'), [ 87 | DataValue('Minimum Delay', 88 | DataQuery(DataQuery.XPath('MINIMUM-DELAY'), format=DataQuery.Format.Integer)), 89 | DataValue( 90 | 'Cyclic Timing', 91 | DataQuery( 92 | DataQuery.XPath( 93 | 'TRANSMISSION-MODE-DECLARATION/TRANSMISSION-MODE-TRUE-TIMING/CYCLIC-TIMING/TIME-PERIOD/VALUE' 94 | ), 95 | format=DataQuery.Format.Float)) 96 | ]), 97 | DataObject('Signal Mappings', DataQuery.XPath('.//I-SIGNAL-TO-I-PDU-MAPPING'), [ 98 | DataValue('Signal Name', DataQuery(DataQuery.XPath('SHORT-NAME'))), 99 | DataValue('Start Position', 100 | DataQuery(DataQuery.XPath('START-POSITION'), 101 | format=DataQuery.Format.Integer)), 102 | DataValue('Transfer Property', DataQuery(DataQuery.XPath('TRANSFER-PROPERTY'))), 103 | DataObject('Signal', DataQuery.XPath('I-SIGNAL-REF', is_reference=True), [ 104 | DataValue('Init Value', 105 | DataQuery(DataQuery.XPath('.//VALUE'), format=DataQuery.Format.Integer)), 106 | DataValue('Length', 107 | DataQuery(DataQuery.XPath('LENGTH'), format=DataQuery.Format.Integer)) 108 | ]) 109 | ]) 110 | ]) 111 | 112 | 113 | def test_find_object_by_reference(simple_object_by_ref): 114 | data_objects = [simple_object_by_ref] 115 | query_handler = QueryHandler() 116 | 117 | data_results = query_handler.handle_queries(arxml, data_objects) 118 | 119 | assert isinstance(data_results, dict) 120 | assert 'CAN Cluster' in data_results 121 | can_cluster = data_results['CAN Cluster'] 122 | assert isinstance(can_cluster, dict) 123 | assert can_cluster['Name'] == 'CAN' 124 | assert can_cluster['Baudrate'] == 500000 125 | assert can_cluster['Language'] == 'FOR-ALL' 126 | assert can_cluster['Long Name'] == 'CAN Channel 1' 127 | 128 | 129 | def test_find_complex_object_by_reference(complex_object_by_ref): 130 | data_objects = [complex_object_by_ref] 131 | query_handler = QueryHandler() 132 | 133 | data_results = query_handler.handle_queries(arxml, data_objects) 134 | 135 | assert isinstance(data_results, dict) 136 | assert 'CAN Cluster' in data_results 137 | can_cluster = data_results['CAN Cluster'] 138 | assert isinstance(can_cluster, dict) 139 | assert can_cluster['Name'] == 'CAN' 140 | assert can_cluster['Baudrate'] == 500000 141 | assert 'Long Name' in can_cluster 142 | long_name = can_cluster['Long Name'] 143 | assert long_name['Language'] == 'FOR-ALL' 144 | assert long_name['Name'] == 'CAN Channel 1' 145 | 146 | 147 | def test_find_simple_object_by_xpath(simple_object_by_xpath): 148 | data_objects = [simple_object_by_xpath] 149 | query_handler = QueryHandler() 150 | 151 | data_results = query_handler.handle_queries(arxml, data_objects) 152 | 153 | assert isinstance(data_results, dict) 154 | assert 'CAN Cluster' in data_results 155 | can_cluster = data_results['CAN Cluster'] 156 | assert isinstance(can_cluster, dict) 157 | assert can_cluster['Name'] == 'CAN' 158 | assert can_cluster['Baudrate'] == 500000 159 | assert can_cluster['Language'] == 'FOR-ALL' 160 | assert can_cluster['Long Name'] == 'CAN Channel 1' 161 | 162 | 163 | def test_find_complex_object_by_xpath(complex_object_by_xpath): 164 | data_objects = [complex_object_by_xpath] 165 | query_handler = QueryHandler() 166 | 167 | data_results = query_handler.handle_queries(arxml, data_objects) 168 | 169 | assert isinstance(data_results, dict) 170 | assert 'CAN Cluster' in data_results 171 | can_cluster = data_results['CAN Cluster'] 172 | assert isinstance(can_cluster, dict) 173 | assert can_cluster['Name'] == 'CAN' 174 | assert can_cluster['Baudrate'] == 500000 175 | assert 'Long Name' in can_cluster 176 | long_name = can_cluster['Long Name'] 177 | assert long_name['Language'] == 'FOR-ALL' 178 | assert long_name['Name'] == 'CAN Channel 1' 179 | 180 | 181 | def test_find_all_objects_by_xpath(multi_value_complex_object): 182 | data_objects = [multi_value_complex_object] 183 | query_handler = QueryHandler() 184 | 185 | data_results = query_handler.handle_queries(arxml, data_objects) 186 | 187 | assert isinstance(data_results, dict) 188 | assert 'PDUs' in data_results 189 | pdus = data_results['PDUs'] 190 | assert isinstance(pdus, list) 191 | assert len(pdus) == 2 192 | assert isinstance(pdus[0], dict) 193 | assert pdus[0]['Name'] == 'TxMessage' 194 | assert pdus[0]['Length'] == 5 195 | assert pdus[0]['Unused Bit Pattern'] == 0 196 | assert pdus[0]['Timing Specification']['Minimum Delay'] == 0 197 | assert pdus[0]['Timing Specification']['Cyclic Timing'] == 0.1 198 | assert pdus[0]['Signal Mappings']['Signal Name'] == 'Signal1' 199 | assert pdus[0]['Signal Mappings']['Start Position'] == 0 200 | assert pdus[0]['Signal Mappings']['Transfer Property'] == 'PENDING' 201 | assert pdus[0]['Signal Mappings']['Signal']['Init Value'] == 128 202 | assert pdus[0]['Signal Mappings']['Signal']['Length'] == 5 203 | 204 | assert isinstance(pdus[1], dict) 205 | assert pdus[1]['Name'] == 'RxMessage' 206 | assert pdus[1]['Length'] == 2 207 | assert pdus[1]['Unused Bit Pattern'] == 0 208 | assert pdus[1]['Timing Specification']['Minimum Delay'] == 0 209 | assert pdus[1]['Timing Specification']['Cyclic Timing'] == 0.1 210 | assert pdus[1]['Signal Mappings'][0]['Signal Name'] == 'Signal2' 211 | assert pdus[1]['Signal Mappings'][0]['Start Position'] == 0 212 | assert pdus[1]['Signal Mappings'][0]['Transfer Property'] == 'PENDING' 213 | assert pdus[1]['Signal Mappings'][0]['Signal']['Init Value'] == 0 214 | assert pdus[1]['Signal Mappings'][0]['Signal']['Length'] == 1 215 | assert pdus[1]['Signal Mappings'][1]['Signal Name'] == 'Signal3' 216 | assert pdus[1]['Signal Mappings'][1]['Start Position'] == 1 217 | assert pdus[1]['Signal Mappings'][1]['Transfer Property'] == 'PENDING' 218 | assert pdus[1]['Signal Mappings'][1]['Signal']['Init Value'] == 0 219 | assert pdus[1]['Signal Mappings'][1]['Signal']['Length'] == 1 220 | 221 | 222 | def test_handle_inline_reference(): 223 | data_object = DataObject('SignalMapping', DataQuery.Reference('/PDU/TxMessage'), [ 224 | DataValue( 225 | 'Name', 226 | DataQuery( 227 | DataQuery.XPath( 228 | '&(I-SIGNAL-TO-PDU-MAPPINGS/I-SIGNAL-TO-I-PDU-MAPPING/I-SIGNAL-REF)SHORT-NAME', 229 | True))) 230 | ]) 231 | 232 | query_handler = QueryHandler() 233 | results = query_handler.handle_queries(arxml, [data_object]) 234 | 235 | assert isinstance(results, dict) 236 | assert isinstance(results['SignalMapping'], dict) 237 | assert results['SignalMapping']['Name'] == 'Signal1' 238 | 239 | 240 | def test_invalid_file_raises_value_error(): 241 | with pytest.raises(ValueError): 242 | query_handler = QueryHandler() 243 | query_handler.handle_queries('test.json', []) 244 | 245 | 246 | def test_config_must_have_root_object(only_value): 247 | data_objects = [only_value] 248 | query_handler = QueryHandler() 249 | 250 | with pytest.raises(TypeError): 251 | query_handler.handle_queries(arxml, data_objects) 252 | 253 | 254 | def test_returns_empty_list_if_reference_not_found(only_value): 255 | data_object = DataObject('CAN Cluster', DataQuery.Reference('/Cluster/CAN1'), 256 | [DataValue('Name', DataQuery(DataQuery.XPath('./SHORT-NAME')))]) 257 | 258 | query_handler = QueryHandler() 259 | result = query_handler.handle_queries(arxml, [data_object]) 260 | 261 | assert result['CAN Cluster'] == [] 262 | 263 | 264 | def test_returns_none_value_if_no_element_found_with_xpath(): 265 | data_object = DataObject('CAN Cluster', DataQuery.Reference('/Cluster/CAN'), 266 | [DataValue('Name', DataQuery(DataQuery.XPath('/SHORT-NAME')))]) 267 | 268 | query_handler = QueryHandler() 269 | result = query_handler.handle_queries(arxml, [data_object]) 270 | 271 | assert isinstance(result['CAN Cluster'], dict) 272 | assert result['CAN Cluster']['Name'] is None 273 | 274 | 275 | def test_returns_none_value_if_no_attribute_found_with_specified_name(): 276 | data_object = DataObject('CAN Cluster', DataQuery.Reference('/Cluster/CAN'), 277 | [DataValue('UUID', DataQuery(DataQuery.XPath('.'), value='@S'))]) 278 | 279 | query_handler = QueryHandler() 280 | result = query_handler.handle_queries(arxml, [data_object]) 281 | 282 | assert isinstance(result['CAN Cluster'], dict) 283 | assert result['CAN Cluster']['UUID'] is None 284 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | pylint 3 | yapf 4 | lxml 5 | pyyaml 6 | pathlib 7 | xlsxwriter 8 | arrow 9 | tqdm 10 | tabulate --------------------------------------------------------------------------------