├── .circleci └── config.yml ├── .gitignore ├── .gitmodules ├── Dockerfile ├── LICENSE ├── README.md ├── benchmark ├── README.md ├── algo_impl.py ├── bench.py ├── start_benchmark.py └── useful_scripts │ └── load_data.py ├── example.py ├── requirements.txt ├── src ├── grammar │ ├── README.md │ ├── cnf_grammar.py │ └── rsa.py ├── graph │ ├── graph.py │ ├── index_graph.py │ ├── label_graph.py │ └── length_graph.py ├── problems │ ├── AllPaths │ │ ├── AllPaths.py │ │ └── algo │ │ │ ├── matrix_all_paths │ │ │ ├── impl │ │ │ │ ├── Grammar.cpp │ │ │ │ ├── Grammar.h │ │ │ │ ├── Graph.cpp │ │ │ │ ├── Graph.h │ │ │ │ ├── Makefile │ │ │ │ ├── apmatrix.cpp │ │ │ │ ├── apmatrix.h │ │ │ │ ├── main.cpp │ │ │ │ ├── pathindex.cpp │ │ │ │ └── pathindex.h │ │ │ └── matrix_all_paths.py │ │ │ └── tensor │ │ │ ├── tensor.py │ │ │ ├── tensor_extract_subgraph.py │ │ │ └── tensor_path.py │ ├── Base │ │ ├── Base.py │ │ └── algo │ │ │ └── matrix_base │ │ │ └── matrix_base.py │ ├── MultipleSource │ │ ├── MultipleSource.py │ │ └── algo │ │ │ ├── matrix_ms │ │ │ └── matrix_ms.py │ │ │ └── tensor_ms │ │ │ └── tensor_ms.py │ ├── SinglePath │ │ ├── SinglePath.py │ │ └── algo │ │ │ ├── matrix_shortest_path │ │ │ ├── matrix_shortest_path.py │ │ │ └── matrix_shortest_path_index.py │ │ │ └── matrix_single_path │ │ │ ├── matrix_single_path.py │ │ │ └── matrix_single_path_index.py │ └── utils.py └── utils │ ├── common.py │ ├── file_helpers.py │ ├── graph_size.py │ ├── time_profiler.py │ └── useful_paths.py └── test ├── AllPaths ├── conftest.py └── test_allpaths.py ├── Base ├── conftest.py └── test_base.py ├── MultipleSource ├── conftest.py └── test_ms.py ├── ShortestPath ├── conftest.py └── test_shortestpath.py ├── SinglePath ├── conftest.py └── test_singlepath.py ├── __init__.py ├── conftest.py ├── data ├── binary_tree │ ├── Grammars │ │ └── g.cfg │ └── Graphs │ │ └── graph_1.txt ├── cycle │ ├── Grammars │ │ └── g.cfg │ └── Graphs │ │ └── graph_1.txt ├── line │ ├── Grammars │ │ └── g.cfg │ └── Graphs │ │ └── graph_1.txt ├── loop │ ├── Grammars │ │ └── g.cfg │ └── Graphs │ │ └── graph_1.txt ├── single_vs_shortest │ ├── Grammars │ │ └── g.cfg │ └── Graphs │ │ └── graph_1.txt ├── two_cycles │ ├── Grammars │ │ └── g.cfg │ └── Graphs │ │ └── graph_1.txt └── two_nonterm │ ├── Grammars │ └── g.cfg │ └── Graphs │ └── graph_1.txt └── suites └── cfpq_data.py /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | jobs: 3 | build: 4 | docker: 5 | - image: graphblas/pygraphblas-minimal:v4.2.2 6 | steps: 7 | - checkout 8 | - run: 9 | name: Install git into container 10 | command: apt-get update && apt-get install -y git 11 | - run: 12 | name: Pull submodules 13 | command: git submodule init && git submodule update 14 | - run: 15 | name: Install requirements 16 | command: pip3 install -r requirements.txt 17 | - run: 18 | name: Install cfpq_data_devtools 19 | command: cd deps/CFPQ_Data && python3 setup.py install 20 | - run: 21 | name: Build matrix AllPaths 22 | command: cd src/problems/AllPaths/algo/matrix_all_paths/impl && make && cd - 23 | - run: 24 | name: Run tests 25 | command: python3 -m pytest test -v -m "CI" 26 | 27 | workflows: 28 | some_workflow: 29 | jobs: 30 | - build 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 98 | __pypackages__/ 99 | 100 | # Celery stuff 101 | celerybeat-schedule 102 | celerybeat.pid 103 | 104 | # SageMath parsed files 105 | *.sage.py 106 | 107 | # Environments 108 | .env 109 | .venv 110 | env/ 111 | venv/ 112 | ENV/ 113 | env.bak/ 114 | venv.bak/ 115 | 116 | # Spyder project settings 117 | .spyderproject 118 | .spyproject 119 | 120 | # Rope project settings 121 | .ropeproject 122 | 123 | # mkdocs documentation 124 | /site 125 | 126 | # mypy 127 | .mypy_cache/ 128 | .dmypy.json 129 | dmypy.json 130 | 131 | # Pyre type checker 132 | .pyre/ 133 | 134 | # pytype static type analyzer 135 | .pytype/ 136 | 137 | # Cython debug symbols 138 | cython_debug/ 139 | 140 | #IDE 141 | .idea/ -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "deps/CFPQ_Data"] 2 | path = deps/CFPQ_Data 3 | url = https://github.com/JetBrains-Research/CFPQ_Data.git 4 | [submodule "deps/pygraphblas"] 5 | path = deps/pygraphblas 6 | url = https://github.com/michelp/pygraphblas.git 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM graphblas/pygraphblas-minimal:v4.2.2 2 | 3 | ADD . /CFPQ_PyAlgo 4 | 5 | WORKDIR /CFPQ_PyAlgo 6 | RUN git submodule init 7 | RUN git submodule update 8 | 9 | WORKDIR /CFPQ_PyAlgo/deps/CFPQ_Data 10 | RUN python3 setup.py install 11 | 12 | WORKDIR /CFPQ_PyAlgo 13 | RUN pip3 install -r requirements.txt 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![CircleCI](https://circleci.com/gh/JetBrains-Research/CFPQ_PyAlgo/tree/master.svg?style=svg)](https://circleci.com/gh/JetBrains-Research/CFPQ_PyAlgo/tree/master) 2 | 3 | # CFPQ_PyAlgo 4 | The CFPQ_PyAlgo is a repository for developing, testing and benchmarking algorithms that solve Formal-Language-Constrained Path Problems, such as Context-Free Path Queries and Regular Path Queries. All algorithms are based on the [GraphBLAS](http://graphblas.org/index.php?title=Graph_BLAS_Forum) framework that allows you to represent graphs as matrices and work with them in terms of linear algebra. For convenience, all the code is written in Python using [pygraphblas](https://github.com/michelp/pygraphblas) or in C/C++ using purely [SuiteSparse](https://github.com/DrTimothyAldenDavis/SuiteSparse/tree/master/GraphBLAS) with a Python wrapper. 5 | 6 | # Installation 7 | First of all you need to clone repository with its submodules: 8 | 9 | ```bash 10 | git clone --recurse-submodules https://github.com/JetBrains-Research/CFPQ_PyAlgo.git 11 | cd CFPQ_PyAlgo/ 12 | git submodule init 13 | git submodule update 14 | ``` 15 | Then the easiest way to get started is to use Docker. An alternative, which is more correct, is to install everything directly. 16 | 17 | ## Using Docker 18 | The first way to start is to use Docker: 19 | 20 | ```bash 21 | # build docker image 22 | docker build --tag . 23 | 24 | # run docker container 25 | docker run --rm -it -v ${PWD}:/CFPQ_PyAlgo bash 26 | ``` 27 | After it you can develop everything locally and run tests and benchmarks inside the container. Also you can use PyCharm and [configure an interpreter using Docker]( https://www.jetbrains.com/help/pycharm/using-docker-as-a-remote-interpreter.html). 28 | 29 | ## Direct install 30 | The correct way is to install everything into your local python interpreter or virtual environment. 31 | 32 | First of all you need to install [pygraphblas](https://github.com/michelp/pygraphblas) package. 33 | ```bash 34 | pip3 install pygraphblas 35 | ``` 36 | Secondly you need to install cfpq_data_devtools package and other requirements: 37 | 38 | ```bash 39 | cd deps/CFPQ_Data 40 | pip3 install -r requirements.txt 41 | python3 setup.py install --user 42 | 43 | cd ../../ 44 | pip3 install -r requirements.txt 45 | ``` 46 | To check if the installation was successful you can run simple tests 47 | ```bash 48 | python3 -m pytest test -v -m "CI" 49 | ``` 50 | # Benchmark 51 | Look please [Readme](https://github.com/JetBrains-Research/CFPQ_PyAlgo/blob/master/benchmark/README.md) in *benchmark* 52 | 53 | # Usage 54 | 55 | Let's describe an example of using the implementation outside this environment. 56 | 57 | For example, you want to solve a basic problem CFPQ using the matrix algorithm. To do this, you need a context-free grammar (**Gr**), as well as a graph (**G**) in the format of "triplets". 58 | 59 | Then the matrix algorithm can be run as follows, where *PATH_TO_GRAMMAR* --- path to file with **Gr**, *PATH_TO_GRAPH* --- path to file with **G** 60 | 61 | ```cython 62 | from src.problems.Base.algo.matrix_base.matrix_base import MatrixBaseAlgo 63 | from cfpq_data import cfg_from_txt 64 | from src.graph.graph import Graph 65 | 66 | from pathlib import Path 67 | 68 | algo = MatrixBaseAlgo() 69 | algo.prepare(Graph.from_txt(Path(PATH_TO_GRAPH)), cfg_from_txt(Path(PATH_TO_GRAMMAR))) 70 | res = algo.solve() 71 | print(res.matrix_S.nvals) 72 | ``` 73 | The given fragment displays the number of pairs of vertices between which the desired path exists. 74 | 75 | More examples can be found in *test* 76 | 77 | # Project structure 78 | The global project structure is the following: 79 | 80 | ``` 81 | . 82 | ├── deps 83 | │ └── CFPQ_Data - repository with graphs and grammars suites 84 | ├───benchmark - directory for performance measurements of implementations 85 | ├── src 86 | │ ├── problems - directory where all the problems CFPQ that we know how to solve 87 | │ │ ├───AllPaths 88 | │ │ ├───Base 89 | │ │ ├───MultipleSource 90 | │ │ └───SinglePath 91 | │ ├── grammar - directory for all grammar formats representation and its loading 92 | │ ├── graph - directory for all graph formats representation and its loading 93 | │ └── utils - directory for other useful classes and methods 94 | └── test 95 | ├───AllPaths - tests for implementations in src.problems.AllPaths 96 | ├───Base - tests for implementations in src.problems.Base 97 | ├───data - dataset for tests 98 | ├───MultipleSource - tests for implementations in src.problems.MultipleSource 99 | ├───SinglePath - tests for implementations in src.problems.SinglePath 100 | └───suites 101 | 102 | ``` 103 | -------------------------------------------------------------------------------- /benchmark/README.md: -------------------------------------------------------------------------------- 1 | # How to start 2 | First, create a directory for the dataset. It should have two subdirectories for graphs (Graphs) and grammars (Grammars). In the second step, select an algorithm for benchmarking. Then run the command: 3 | ``` 4 | python3 -m benchmark.start_benchmark.py -algo ALGO -data_dir DATA_DIR 5 | ``` 6 | There are also a number of optional parameters: 7 | + -round --- number of rounds for measurements 8 | + -config --- config file in the "graph grammar" format to indicate only certain data for measurements from a directory DATA_DIR 9 | + -with_paths --- indicate additionally measure the extraction of paths 10 | + -result_dir --- specify a directory for uploading the results 11 | + -max_len_paths --- Limit on the length of the retrieved paths 12 | 13 | # Add new algorithm 14 | To add a new implementation of the algorithm to the list of available measurements, you must: 15 | 1. Add you algorithm in *algo_impl.ALGO_PROBLEM* 16 | 2. Add you implementation in *algo_impl.ALGO_IMPL* 17 | 3. Create new or use the existing pipeline or in *bench.benchmark* -------------------------------------------------------------------------------- /benchmark/algo_impl.py: -------------------------------------------------------------------------------- 1 | from typing import Final 2 | 3 | from src.problems.Base.algo.matrix_base.matrix_base import MatrixBaseAlgo 4 | 5 | from src.problems.AllPaths.algo.tensor.tensor import TensorSimpleAlgo 6 | from src.problems.AllPaths.algo.tensor.tensor import TensorDynamicAlgo 7 | 8 | from src.problems.SinglePath.algo.matrix_single_path.matrix_single_path_index import MatrixSingleAlgo 9 | 10 | from src.problems.MultipleSource.algo.matrix_ms.matrix_ms import MatrixMSOptAlgo 11 | from src.problems.MultipleSource.algo.matrix_ms.matrix_ms import MatrixMSBruteAlgo 12 | 13 | """ 14 | Correspondence of algo and problem it solves 15 | """ 16 | ALGO_PROBLEM: Final = {'TensorSimple': 'AllPaths', 17 | 'TensorDynamic': 'AllPaths', 18 | 'MatrixBase': 'Base', 19 | 'MatrixMSBrute': 'MS', 20 | 'MatrixMSOpt': 'MS', 21 | 'MatrixSingle': 'SinglePath'} 22 | 23 | """ 24 | Matching name of algo and its implementation 25 | """ 26 | ALGO_IMPL: Final = {'TensorSimple': TensorSimpleAlgo, 27 | 'TensorDynamic': TensorDynamicAlgo, 28 | 'MatrixBase': MatrixBaseAlgo, 29 | 'MatrixMSBrute': MatrixMSBruteAlgo, 30 | 'MatrixMSOpt': MatrixMSOptAlgo, 31 | 'MatrixSingle': MatrixSingleAlgo} 32 | -------------------------------------------------------------------------------- /benchmark/bench.py: -------------------------------------------------------------------------------- 1 | import csv 2 | from os import listdir 3 | from os.path import isfile, exists 4 | from tqdm import tqdm 5 | from time import time 6 | 7 | from benchmark.algo_impl import ALGO_PROBLEM, ALGO_IMPL 8 | from src.graph.label_graph import LabelGraph 9 | from src.graph.graph import Graph 10 | from cfpq_data import cfg_from_txt 11 | 12 | GRAMMAR_DIR = 'Grammars/' 13 | GRAPH_DIR = 'Graphs/' 14 | 15 | 16 | def parse_config(config): 17 | """ 18 | Returns information about which graph with which grammar to run 19 | @param config: Path to csv file with header:["Graph", "Grammar"] 20 | @return: dictionary in which keys are paths to graphs and values are paths to grammars 21 | """ 22 | graph_grammar = dict() 23 | with open(config, "r") as csv_file: 24 | reader = csv.DictReader(csv_file) 25 | for row in reader: 26 | if row['graph'] in graph_grammar: 27 | graph_grammar[row['graph']].append(row['grammar']) 28 | else: 29 | graph_grammar[row['graph']] = [row['grammar']] 30 | return graph_grammar 31 | 32 | 33 | def get_sample_mean(data): 34 | return sum(data) / float(len(data)) 35 | 36 | 37 | def get_variance(data, sample_mean): 38 | return sum([(x - sample_mean) ** 2 for x in data]) / float(len(data) - 1) 39 | 40 | 41 | def benchmark(algo, data_dir, result_dir, config, with_paths, rounds, max_len_paths): 42 | """ 43 | Pipeline builder function for measuring performance 44 | @param algo: name algorithm in string 45 | @param data_dir: path to dataset 46 | @param result_dir: path to result directory 47 | @param config: path to config file (csv) 48 | @param with_paths: flag for setting measurements for fetching paths 49 | @param rounds: number of measurement rounds 50 | """ 51 | type_problem = ALGO_PROBLEM[algo] 52 | graph_grammar = dict() 53 | if config is not None: 54 | name_graph_grammar = parse_config(config) 55 | for graph in name_graph_grammar: 56 | graph_grammar.update({data_dir.joinpath(GRAPH_DIR).joinpath(graph): []}) 57 | for grammar in name_graph_grammar[graph]: 58 | graph_grammar[data_dir.joinpath(GRAPH_DIR).joinpath(graph)].append( 59 | data_dir.joinpath(GRAMMAR_DIR).joinpath(grammar)) 60 | else: 61 | grammars = {data_dir.joinpath(GRAMMAR_DIR).joinpath(f) for f in listdir(data_dir.joinpath(GRAMMAR_DIR)) if 62 | isfile(data_dir.joinpath(GRAMMAR_DIR).joinpath(f))} 63 | graphs = {data_dir.joinpath(GRAPH_DIR).joinpath(f) for f in listdir(data_dir.joinpath(GRAPH_DIR)) if 64 | isfile(data_dir.joinpath(GRAPH_DIR).joinpath(f))} 65 | for graph in graphs: 66 | graph_grammar.update({graph: grammars}) 67 | 68 | impl_for_algo = ALGO_IMPL[algo] 69 | variances = [] 70 | if type_problem == "MS": 71 | benchmark_ms(impl_for_algo, graph_grammar, result_dir) 72 | else: 73 | variances = benchmark_index(impl_for_algo, graph_grammar, result_dir, rounds) 74 | 75 | if with_paths: 76 | if type_problem == "AllPaths": benchmark_all_paths(impl_for_algo, graph_grammar, result_dir, max_len_paths) 77 | if type_problem == "SinglePath": benchmark_single_path(impl_for_algo, graph_grammar, result_dir) 78 | 79 | 80 | def benchmark_index(algo_name, data, result_dir, rounds): 81 | """ 82 | Measurement function for finding paths between all pairs of vertices 83 | @param algo_name: concrete implementation of the algorithm 84 | @param data: dictionary in format {path to graph: list of paths to grammars} 85 | @param result_dir: directory for uploading results of measurement 86 | @param rounds: number of measurement rounds 87 | @return: variance value for each round of measurements 88 | """ 89 | header_index = ['graph', 'grammar', 'time', 'count_S', 'variance'] 90 | 91 | variances = [] 92 | for graph in data: 93 | result_index_file_path = result_dir.joinpath(f'{graph.stem}-{algo_name.__name__}-index') 94 | 95 | append_header = False 96 | if not exists(result_index_file_path): 97 | append_header = True 98 | result_csv = open(result_index_file_path, mode='a', newline='\n') 99 | csv_writer_index = csv.writer(result_csv, delimiter=',', quoting=csv.QUOTE_NONNUMERIC, escapechar=' ') 100 | 101 | if append_header: 102 | csv_writer_index.writerow(header_index) 103 | 104 | for grammar in data[graph]: 105 | algo = algo_name() 106 | algo.prepare(Graph.from_txt(graph), cfg_from_txt(grammar)) 107 | count_S = 0 108 | times = [] 109 | for _ in tqdm(range(rounds), desc=f'{graph.stem}-{grammar.stem}'): 110 | algo.prepare_for_solve() 111 | start = time() 112 | res = algo.solve() 113 | finish = time() 114 | times.append(finish - start) 115 | count_S = res.matrix_S.nvals 116 | 117 | sample_mean = get_sample_mean(times) 118 | variances.append(get_variance(times, sample_mean)) 119 | csv_writer_index.writerow( 120 | [graph.stem, grammar.stem, sample_mean, count_S, get_variance(times, sample_mean)]) 121 | 122 | return variances 123 | 124 | 125 | def benchmark_all_paths(algo_name, data, result_dir, max_len_paths): 126 | """ 127 | Measurement function for extract all paths 128 | @param algo_name: concrete implementation of the algorithm 129 | @param data: dictionary in format {path to graph: list of paths to grammars} 130 | @param result_dir: directory for uploading results of measurement 131 | """ 132 | header_paths = ['graph', 'grammar', 'count_paths', 'time'] 133 | 134 | for graph in data: 135 | result_paths_file_path = result_dir.joinpath(f'{graph.stem}-{algo_name.__name__}-allpaths') 136 | 137 | append_header = False 138 | if not exists(result_paths_file_path): 139 | append_header = True 140 | 141 | result_csv = open(result_paths_file_path, mode='a', newline='\n') 142 | csv_writer_paths = csv.writer(result_csv, delimiter=',', quoting=csv.QUOTE_NONNUMERIC, escapechar=' ') 143 | 144 | if append_header: 145 | csv_writer_paths.writerow(header_paths) 146 | 147 | for grammar in data[graph]: 148 | algo = algo_name() 149 | algo.prepare(Graph.from_txt(graph), cfg_from_txt(grammar)) 150 | res = algo.solve() 151 | for elem in tqdm(res.matrix_S, desc=f'{graph.stem}-{grammar.stem}-paths'): 152 | algo.prepare_for_exctract_paths() 153 | start = time() 154 | paths = algo.getPaths(elem[0], elem[1], "S", int(max_len_paths)) 155 | finish = time() 156 | csv_writer_paths.writerow([graph.stem, grammar.stem, len(paths), finish - start]) 157 | 158 | 159 | def benchmark_single_path(algo_name, data, result_dir): 160 | """ 161 | Measurement function for extract single path 162 | @param algo_name: concrete implementation of the algorithm 163 | @param data: dictionary in format {path to graph: list of paths to grammars} 164 | @param result_dir: directory for uploading results of measurement 165 | """ 166 | header_paths = ['graph', 'grammar', 'len_path', 'time'] 167 | 168 | for graph in data: 169 | result_paths_file_path = result_dir.joinpath(f'{graph.stem}-{algo_name.__name__}-singlepaths') 170 | 171 | append_header = False 172 | if not exists(result_paths_file_path): 173 | append_header = True 174 | 175 | result_csv = open(result_paths_file_path, mode='a', newline='\n') 176 | csv_writer_paths = csv.writer(result_csv, delimiter=',', quoting=csv.QUOTE_NONNUMERIC, escapechar=' ') 177 | 178 | if append_header: 179 | csv_writer_paths.writerow(header_paths) 180 | 181 | if not exists(result_paths_file_path): 182 | csv_writer_paths.writerow(header_paths) 183 | 184 | for grammar in data[graph]: 185 | algo = algo_name() 186 | algo.prepare(Graph.from_txt(graph), cfg_from_txt(grammar)) 187 | res = algo.solve() 188 | for elem in tqdm(res.matrix_S, desc=f'{graph.stem}-{grammar}-paths'): 189 | start = time() 190 | paths = algo.getPath(elem[0], elem[1], "S") 191 | finish = time() 192 | csv_writer_paths.writerow([graph.stem, grammar.stem, paths, finish - start]) 193 | 194 | 195 | def benchmark_ms(algo_name, data, result_dir): 196 | """ 197 | Measurement function for finding paths from set of vertices 198 | @param algo_name: concrete implementation of the algorithm 199 | @param data: dictionary in format {path to graph: list of paths to grammars} 200 | @param result_dir: directory for uploading results of measurement 201 | """ 202 | header_index = ['graph', 'grammar', 'size_chunk', 'time', 'count_S'] 203 | 204 | chunk_sizes = [1, 2, 4, 8, 16, 32, 50, 100, 500, 1000, 5000, 10000, None] 205 | 206 | for graph in data: 207 | result_index_file_path = result_dir.joinpath(f'{graph.stem}-{algo_name.__name__}-msindex') 208 | 209 | append_header = False 210 | if not exists(result_index_file_path): 211 | append_header = True 212 | 213 | result_csv = open(result_index_file_path, mode='a', newline='\n') 214 | csv_writer_index = csv.writer(result_csv, delimiter=',', quoting=csv.QUOTE_NONNUMERIC, escapechar=' ') 215 | 216 | if append_header: 217 | csv_writer_index.writerow(header_index) 218 | 219 | if not exists(result_index_file_path): 220 | csv_writer_index.writerow(header_index) 221 | 222 | g = LabelGraph.from_txt(graph) 223 | for grammar in data[graph]: 224 | algo = algo_name() 225 | algo.prepare(Graph.from_txt(graph), cfg_from_txt(grammar)) 226 | for chunk_size in chunk_sizes: 227 | chunks = [] 228 | if chunk_size is None: 229 | chunks = g.chunkify(g.matrices_size) 230 | else: 231 | chunks = g.chunkify(chunk_size) 232 | 233 | for chunk in tqdm(chunks, desc=f'{graph.stem}-{grammar.stem}'): 234 | algo.clear_src() # Attention (TODO): remove this line if you want to cache the result ! 235 | start = time() 236 | res = algo.solve(chunk) 237 | finish = time() 238 | 239 | csv_writer_index.writerow( 240 | [graph.stem, grammar.stem, chunk_size, finish - start, res.matrix_S.nvals]) 241 | -------------------------------------------------------------------------------- /benchmark/start_benchmark.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from pathlib import Path 3 | import os 4 | import datetime 5 | 6 | from benchmark.algo_impl import * 7 | from benchmark.bench import benchmark 8 | 9 | 10 | def result_folder(): 11 | """ 12 | Creates and returns an unused result directory 13 | @return: path to new directory 14 | """ 15 | results = 'results' 16 | if not os.path.exists(results): 17 | os.mkdir(f'results') 18 | 19 | now = datetime.datetime.now().strftime('%d-%m-%Y_%H:%M:%S') 20 | result_folder = os.path.join(results, now) 21 | 22 | os.mkdir(result_folder) 23 | return result_folder 24 | 25 | 26 | if __name__ == '__main__': 27 | parser = argparse.ArgumentParser(description="Benchmark implementations CFPQ's algorithms") 28 | parser.add_argument('-result_dir', dest='result_dir', default=result_folder(), 29 | help='Directory for uploading ' 30 | 'experiment results') 31 | algo_name = ALGO_PROBLEM.keys() 32 | parser.add_argument('-algo', dest='algo', required=True, choices=algo_name, 33 | help='Algorithm implementation that will be measured') 34 | parser.add_argument('-config', dest='config', default=None, help='Config file for benchmark') 35 | parser.add_argument('-data_dir', dest='data_dir', required=True, help='Directory where dataset') 36 | parser.add_argument('-with_paths', dest='with_paths', type=bool, default=False, help='Is it necessary to measure ' 37 | 'the extraction of paths?') 38 | parser.add_argument('-round', dest='rounds', type=int, default=5, help='Number of rounds for benchmarking ' 39 | '(Default: round = 5)') 40 | parser.add_argument('-max_len_paths', dest='max_len_paths', default=5, type=int, help='Limit on the length of ' 41 | 'the retrieved paths') 42 | args = parser.parse_args() 43 | benchmark(args.algo, 44 | Path(args.data_dir), 45 | Path(args.result_dir), 46 | args.config, 47 | args.with_paths, 48 | args.rounds, 49 | args.max_len_paths) 50 | -------------------------------------------------------------------------------- /benchmark/useful_scripts/load_data.py: -------------------------------------------------------------------------------- 1 | from cfpq_data import DATASET, graph_from_dataset, cfg_to_txt, nodes_to_integers, get_labels, change_edges 2 | 3 | from cfpq_data.grammars.samples.rdf import g1, g2, geo 4 | from cfpq_data.grammars.samples.cycle import a_star_0, a_star_1, a_star_2 5 | from cfpq_data.grammars.samples.barabasi_albert import an_bm_cm_dn 6 | from cfpq_data.grammars.samples.binomial import sg 7 | from cfpq_data.grammars.samples.memory_aliases import g1 as g1_ma 8 | from cfpq_data.grammars.samples.two_cycles import brackets 9 | 10 | from pathlib import Path 11 | 12 | import argparse 13 | import os 14 | 15 | DEFAULT_PATH = Path("../data/") 16 | DEFAULT_GRAPH_PATH = DEFAULT_PATH.joinpath("Graphs/") 17 | DEFAULT_GRAMMAR_PATH = DEFAULT_PATH.joinpath("Grammars/") 18 | 19 | GRAMMARS_BY_TYPE = { 20 | "rdf": [g1, g2, geo], 21 | "cycle": [a_star_0, a_star_1, a_star_2], 22 | "barabasi_albert": [an_bm_cm_dn], 23 | "bimomial": [sg], 24 | "memory_aliases": [g1_ma], 25 | "two_cycles": [brackets] 26 | } 27 | 28 | GRAMMARS_BY_NAME = { 29 | g1: "g1", 30 | g2: "g2", 31 | geo: "geo", 32 | a_star_0: "a_star_0", 33 | a_star_1: "a_star_1", 34 | a_star_2: "a_star_2", 35 | an_bm_cm_dn: "an_bm_cm_dn", 36 | sg: "sg", 37 | g1_ma: "g1_ma", 38 | brackets: "brackets" 39 | } 40 | 41 | CONFIG = { 42 | "subClassOf": "sco", 43 | "type": "t", 44 | "broaderTransitive": "bt", 45 | "http://yacc/D": "d", 46 | "http://yacc/A": "a" 47 | } 48 | 49 | 50 | def graph_to_txt(graph, path, config): 51 | with open(path, "w") as fout: 52 | for u, v, edge_labels in graph.edges(data=True): 53 | for label in edge_labels.values(): 54 | if config[str(label)] == "other": 55 | continue 56 | fout.write(f"{u} {config[str(label)]} {v}\n") 57 | fout.write(f"{v} {config[str(label)]}_r {u}\n") 58 | 59 | 60 | def load_graph_by_type(type): 61 | for name_graph in DATASET[type].keys(): 62 | load_graph_by_name(name_graph) 63 | 64 | 65 | def load_graph_by_name(name_graph): 66 | g = nodes_to_integers(graph_from_dataset(name_graph, verbose=False), verbose=False) 67 | config_cur = dict() 68 | for label in get_labels(g, verbose=False): 69 | label_str = str(label).split("#") 70 | if len(label_str) < 2 and str(label) not in CONFIG: 71 | config_cur.update({str(label): "other"}) 72 | else: 73 | if str(label) in CONFIG: 74 | config_cur.update({str(label): CONFIG[str(label)]}) 75 | else: 76 | l = CONFIG.get(label_str[1], "other") 77 | config_cur.update({str(label): l}) 78 | graph_to_txt(g, DEFAULT_GRAPH_PATH.joinpath(name_graph), config_cur) 79 | 80 | 81 | def load_grammar_by_type(name_grammar): 82 | for grammar in GRAMMARS_BY_TYPE[name_grammar]: 83 | cfg_to_txt(grammar, DEFAULT_GRAMMAR_PATH.joinpath(GRAMMARS_BY_NAME[grammar])) 84 | 85 | 86 | if __name__ == '__main__': 87 | parser = argparse.ArgumentParser(description="Tools for download graphs and grammars") 88 | 89 | types_graphs = DATASET.keys() 90 | parser.add_argument('-graph_type', dest="graph_type", default=None, choices=types_graphs) 91 | parser.add_argument('-graph_name', dest="graph_name", default=None) 92 | types_grammars = GRAMMARS_BY_TYPE.keys() 93 | parser.add_argument('-grammar_type', dest="grammar_type", default=None, choices=types_grammars) 94 | 95 | if os.path.exists(DEFAULT_PATH): 96 | if not os.path.exists(DEFAULT_GRAPH_PATH): 97 | os.mkdir(DEFAULT_GRAPH_PATH) 98 | if not os.path.exists(DEFAULT_GRAMMAR_PATH): 99 | os.mkdir(DEFAULT_GRAMMAR_PATH) 100 | else: 101 | os.mkdir(DEFAULT_PATH) 102 | os.mkdir(DEFAULT_GRAPH_PATH) 103 | os.mkdir(DEFAULT_GRAMMAR_PATH) 104 | 105 | args = parser.parse_args() 106 | if args.graph_type is not None: 107 | load_graph_by_type(args.graph_type) 108 | 109 | if args.graph_name is not None: 110 | load_graph_by_name(args.graph_name) 111 | 112 | if args.grammar_type is not None: 113 | load_grammar_by_type(args.grammar_type) 114 | -------------------------------------------------------------------------------- /example.py: -------------------------------------------------------------------------------- 1 | from src.problems.Base.algo.matrix_base.matrix_base import MatrixBaseAlgo 2 | from src.problems.MultipleSource.algo.matrix_ms.matrix_ms import MatrixMSBruteAlgo, MatrixMSOptAlgo 3 | from src.problems.MultipleSource.algo.tensor_ms.tensor_ms import TensorMSAlgo 4 | from src.problems.AllPaths.algo.tensor.tensor import TensorSimpleAlgo, TensorDynamicAlgo 5 | from src.problems.SinglePath.algo.matrix_single_path.matrix_single_path_index import MatrixSingleAlgo 6 | from src.problems.SinglePath.algo.matrix_shortest_path.matrix_shortest_path_index import MatrixShortestAlgo 7 | 8 | from src.graph.graph import Graph 9 | from cfpq_data import cfg_from_txt 10 | 11 | from src.problems.utils import ResultAlgo 12 | 13 | from pathlib import Path 14 | 15 | CASE = Path("test/data/binary_tree/") 16 | 17 | graph = Graph.from_txt(CASE.joinpath("Graphs/graph_1.txt")) 18 | grammar = cfg_from_txt(CASE.joinpath("Grammars/g.cfg")) 19 | algo = MatrixBaseAlgo() 20 | algo.prepare(graph, grammar) 21 | res:ResultAlgo = algo.solve() 22 | print(f'MatrixBaseAlgo: {res.matrix_S.nvals}') 23 | 24 | graph = Graph.from_txt(CASE.joinpath("Graphs/graph_1.txt")) 25 | grammar = cfg_from_txt(CASE.joinpath("Grammars/g.cfg")) 26 | algo = TensorSimpleAlgo() 27 | algo.prepare(graph, grammar) 28 | res:ResultAlgo = algo.solve() 29 | print(f'TensorSimpleAlgo: {res.matrix_S.nvals}') 30 | 31 | graph = Graph.from_txt(CASE.joinpath("Graphs/graph_1.txt")) 32 | grammar = cfg_from_txt(CASE.joinpath("Grammars/g.cfg")) 33 | algo = TensorDynamicAlgo() 34 | algo.prepare(graph, grammar) 35 | res:ResultAlgo = algo.solve() 36 | print(f'TensorDynamicAlgo: {res.matrix_S.nvals}') 37 | 38 | graph = Graph.from_txt(CASE.joinpath("Graphs/graph_1.txt")) 39 | grammar = cfg_from_txt(CASE.joinpath("Grammars/g.cfg")) 40 | algo = MatrixSingleAlgo() 41 | algo.prepare(graph, grammar) 42 | res:ResultAlgo = algo.solve() 43 | print(f'MatrixSingleAlgo: {res.matrix_S.nvals}') 44 | 45 | graph = Graph.from_txt(CASE.joinpath("Graphs/graph_1.txt")) 46 | grammar = cfg_from_txt(CASE.joinpath("Grammars/g.cfg")) 47 | algo = MatrixShortestAlgo() 48 | algo.prepare(graph, grammar) 49 | res:ResultAlgo = algo.solve() 50 | print(f'MatrixShortestAlgo: {res.matrix_S.nvals}') 51 | 52 | graph = Graph.from_txt(CASE.joinpath("../single_vs_shortest/Graphs/graph_1.txt")) 53 | grammar = cfg_from_txt(CASE.joinpath("../single_vs_shortest/Grammars/g.cfg")) 54 | algo = MatrixShortestAlgo() 55 | algo.prepare(graph, grammar) 56 | res:ResultAlgo = algo.solve() 57 | print(f'MatrixShortestAlgo: {res.matrix_S.nvals}') 58 | print(f'MatrixShortestAlgo shortest path 0 - 7: {algo.getPath(0, 7, "S")}') 59 | 60 | graph = Graph.from_txt(CASE.joinpath("Graphs/graph_1.txt")) 61 | grammar = cfg_from_txt(CASE.joinpath("Grammars/g.cfg")) 62 | algo = MatrixMSBruteAlgo() 63 | algo.prepare(graph, grammar) 64 | res:ResultAlgo = algo.solve([0])[0] 65 | print(f'MatrixMSBruteAlgo from 0: {res.matrix_S.nvals}') 66 | 67 | graph = Graph.from_txt(CASE.joinpath("Graphs/graph_1.txt")) 68 | grammar = cfg_from_txt(CASE.joinpath("Grammars/g.cfg")) 69 | algo = MatrixMSOptAlgo() 70 | algo.prepare(graph, grammar) 71 | res:ResultAlgo = algo.solve([0])[0] 72 | print(f'MatrixMSOptAlgo from 0: {res.matrix_S.nvals}') 73 | 74 | graph = Graph.from_txt(CASE.joinpath("Graphs/graph_1.txt")) 75 | grammar = cfg_from_txt(CASE.joinpath("Grammars/g.cfg")) 76 | algo = TensorMSAlgo() 77 | algo.prepare(graph, grammar) 78 | res:ResultAlgo = algo.solve([0])[0] 79 | print(f'TensorMSAlgo from 0: {res.matrix_S.nvals}') 80 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tqdm 2 | pytest 3 | pytest-benchmark 4 | cfpq_data -------------------------------------------------------------------------------- /src/grammar/README.md: -------------------------------------------------------------------------------- 1 | **For now, it is enough to have a context-free grammar. Conversion to the desired format will happen on its own inside the implementations.** 2 | 3 | # Presentation formats of grammars 4 | Consider presentation formats grammar CNF and RSA 5 | 6 | # Format of RSA 7 | 8 | Let the CF grammar be given: 9 | ``` 10 | S -> a S b | a b 11 | ``` 12 | Thus RSA for this grammar should be represented in the file as follows: 13 | ``` 14 | 3 15 | 1 16 | 4 17 | a 18 | 1 19 | 0 1 20 | S 21 | 1 22 | 1 2 23 | b 24 | 2 25 | 1 3 26 | 2 3 27 | S 28 | 1 29 | 0 3 30 | ``` 31 | Where 32 | * First line is number of different labels on edges 33 | * Second line is number of nonterminals 34 | * Third line is size of adjacency matrix size 35 | * Then there are triples of lines, the number of which is equal to the number on the first line in format 36 | ``` 37 | label 38 | number of edges with this label 39 | two states with this label 40 | ``` 41 | * This is followed by information about the start and finish state of the automaton in format 42 | ``` 43 | nonterminal 44 | number of start and finish states 45 | two states: start and finish for this nonterminal 46 | ``` 47 | # Format for grammar CNF 48 | 49 | This format is the same as the format in *deps/CFPQ_Data* -------------------------------------------------------------------------------- /src/grammar/cnf_grammar.py: -------------------------------------------------------------------------------- 1 | from cfpq_data import cnf_from_cfg 2 | from pyformlang.cfg import CFG 3 | 4 | 5 | class CnfGrammar: 6 | """ 7 | This class representing grammar in CNF. Supports only the functions necessary for the algorithms to work 8 | """ 9 | 10 | def __init__(self): 11 | self.start_nonterm = None 12 | self.nonterms = set() 13 | self.terms = set() 14 | self.simple_rules = [] 15 | self.complex_rules = [] 16 | self.eps_rules = [] 17 | 18 | def __setitem__(self, key, value): 19 | if (isinstance(value, tuple) or isinstance(value, list)) and 1 <= len(value) <= 2: 20 | self.nonterms.add(key) 21 | if len(value) == 1: 22 | self.simple_rules.append((key, value[0])) 23 | self.terms.add(value[0]) 24 | else: 25 | self.complex_rules.append((key, value[0], value[1])) 26 | for x in value: 27 | self.nonterms.add(x) 28 | else: 29 | raise Exception('value must be str, (str, str) or [str, str]') 30 | 31 | @classmethod 32 | def from_cfg(cls, cfg: CFG): 33 | cnf = CnfGrammar() 34 | base_cnf = cnf_from_cfg(cfg) 35 | cnf.start_nonterm = base_cnf.start_symbol.to_text() 36 | 37 | for product in base_cnf.productions: 38 | if not product.body: 39 | cnf.eps_rules.append(product.head.to_text().strip('"')) 40 | else: 41 | cnf[product.head.to_text().strip('"')] = [x.to_text().strip('"') for x in product.body] 42 | 43 | return cnf 44 | 45 | @classmethod 46 | def from_cnf(cls, path): 47 | """ 48 | Load grammar in CNF format from file 49 | @param path: path to file with grammar 50 | @return: initialized class 51 | """ 52 | grammar = CnfGrammar() 53 | with open(path, 'r') as f: 54 | lines = f.readlines() 55 | grammar.start_nonterm = lines[0].split()[0].strip() 56 | for line in lines[2:]: 57 | l, r = line.strip().split('->') 58 | l = l.strip() 59 | r = r.strip().split() 60 | grammar[l] = r 61 | return grammar 62 | -------------------------------------------------------------------------------- /src/grammar/rsa.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | from pygraphblas.matrix import Matrix 4 | from pygraphblas.types import BOOL 5 | 6 | from pathlib import Path 7 | 8 | from pyformlang.cfg import CFG 9 | from cfpq_data import rsm_from_text 10 | from cfpq_data.grammars.rsm import RSM 11 | 12 | 13 | class RecursiveAutomaton: 14 | """ 15 | This class representing recursive state automaton. Supports only the functions necessary for the algorithms to work 16 | """ 17 | def __init__(self): 18 | self.labels = set() 19 | self.nonterminals = set() 20 | self.matrices = dict() 21 | self.states = dict() 22 | self.start_and_finish = set() 23 | self.matrices_size = 0 24 | self.start_state = dict() 25 | self.finish_states = dict() 26 | self.terminals = set() 27 | self.out_states = dict() 28 | self.start_nonterm = "" 29 | self.boxes = dict() 30 | 31 | def __getitem__(self, item: str) -> Matrix: 32 | if item not in self.matrices: 33 | self.matrices[item] = Matrix.sparse(BOOL, self.matrices_size, self.matrices_size) 34 | 35 | return self.matrices[item] 36 | 37 | @classmethod 38 | def from_grammar_or_path(cls, grammar_or_path: Union[RSM, CFG, Path]): 39 | """ 40 | Build RSA from cfpq_data CFG or RSM or load it from file 41 | @param grammar_or_path: CFG or RSM on which RSA is built or path to file with RSA 42 | @return: initialized class 43 | """ 44 | if isinstance(grammar_or_path, RSM): 45 | return RecursiveAutomaton.from_rsm(grammar_or_path) 46 | elif isinstance(grammar_or_path, CFG): 47 | return RecursiveAutomaton.from_cfg(grammar_or_path) 48 | elif isinstance(grammar_or_path, Path): 49 | return RecursiveAutomaton.from_file(grammar_or_path) 50 | 51 | @classmethod 52 | def from_cfg(cls, cfg: CFG): 53 | """ 54 | Build RSA from a given cfpq_data context-free grammar 55 | @param cfg: CFG on which RSA is built 56 | @return: initialized class 57 | """ 58 | grammar = cfg.to_text() 59 | 60 | productions = dict() 61 | for line in grammar.split("\n")[:-1]: 62 | part_line = line.split(" -> ") 63 | right = part_line[1] 64 | if right == "": 65 | right = "epsilon" 66 | if part_line[0] in productions: 67 | productions[part_line[0]] += " | " + right 68 | else: 69 | productions[part_line[0]] = right 70 | 71 | grammar_new = "" 72 | for nonterminal in productions: 73 | grammar_new += nonterminal + " -> " + productions[nonterminal] + "\n" 74 | 75 | grammar_new = grammar_new[:-1] 76 | return RecursiveAutomaton.from_rsm(rsm_from_text(grammar_new)) 77 | 78 | @classmethod 79 | def from_file(cls, path: Path): 80 | """ 81 | Load RSA from file 82 | @param path: path to file with RSA 83 | @return: initialized class 84 | """ 85 | rsa = RecursiveAutomaton() 86 | with open(path, "r") as file: 87 | count_matrix = int(file.readline()) 88 | count_nonterminals = int(file.readline()) 89 | matrices_size = int(file.readline()) 90 | rsa.matrices_size = matrices_size 91 | 92 | for i in range(count_matrix): 93 | label = file.readline().replace("\n", "") 94 | rsa.labels.add(label) 95 | count_edge = int(file.readline()) 96 | for j in range(count_edge): 97 | first, second = file.readline().split() 98 | rsa[label][int(first), int(second)] = True 99 | 100 | if int(first) in rsa.out_states: 101 | rsa.out_states[int(first)].append((int(second), label)) 102 | else: 103 | rsa.out_states[int(first)] = [(int(second), label)] 104 | 105 | for i in range(count_nonterminals): 106 | label = file.readline().replace("\n", "") 107 | rsa.nonterminals.add(label) 108 | rsa.states.update({label: Matrix.sparse(BOOL, rsa.matrices_size, rsa.matrices_size)}) 109 | count_edge = int(file.readline()) 110 | for j in range(count_edge): 111 | first, second = file.readline().split() 112 | rsa.states[label][int(first), int(second)] = True 113 | rsa.start_state.update({label: int(first)}) 114 | if label in rsa.finish_states: 115 | rsa.finish_states[label].append(int(second)) 116 | else: 117 | rsa.finish_states.update({label: [int(second)]}) 118 | if first == second: 119 | rsa.start_and_finish.add(label) 120 | rsa.terminals = rsa.labels.difference(rsa.nonterminals) 121 | return rsa 122 | 123 | @classmethod 124 | def from_rsm(cls, rsm: RSM): 125 | """ 126 | Build RSA from a given cfpq_data Recursive State Machine 127 | @param rsm: RSM on which RSA is built 128 | @return: initialized class 129 | """ 130 | rsa = RecursiveAutomaton() 131 | rsa.start_nonterm = rsm.start_symbol.to_text() 132 | current_state = 0 133 | transtion_by_label = dict() 134 | for nonterm, dfa in rsm.boxes: 135 | mapping_state = dict() 136 | rsa.nonterminals.add(nonterm.to_text()) 137 | rsa.labels = rsa.labels.union(dfa.symbols) 138 | rsa.boxes[nonterm.to_text()] = [] 139 | 140 | for label in dfa.symbols: 141 | if label not in transtion_by_label: 142 | transtion_by_label.update({label: []}) 143 | 144 | dfa_dict = dfa.to_dict() 145 | for state in dfa_dict: 146 | if state not in mapping_state: 147 | mapping_state[state] = current_state 148 | rsa.boxes[nonterm.to_text()].append(current_state) 149 | current_state += 1 150 | 151 | for trans in dfa_dict[state]: 152 | if dfa_dict[state][trans] not in mapping_state: 153 | mapping_state[dfa_dict[state][trans]] = current_state 154 | rsa.boxes[nonterm.to_text()].append(current_state) 155 | current_state += 1 156 | transtion_by_label[trans].append((mapping_state[state], mapping_state[dfa_dict[state][trans]])) 157 | rsa.states[nonterm.to_text()] = [] 158 | rsa.start_state[nonterm.to_text()] = mapping_state[dfa.start_state] 159 | rsa.finish_states[nonterm.to_text()] = [] 160 | for final_state in dfa.final_states: 161 | rsa.states[nonterm.to_text()].append((mapping_state[dfa.start_state], mapping_state[final_state])) 162 | rsa.finish_states[nonterm.to_text()].append(mapping_state[final_state]) 163 | if mapping_state[dfa.start_state] == mapping_state[final_state]: 164 | rsa.start_and_finish.add(nonterm.to_text()) 165 | 166 | rsa.matrices_size = current_state 167 | for label in transtion_by_label: 168 | rsa.matrices[label] = Matrix.sparse(BOOL, rsa.matrices_size, rsa.matrices_size) 169 | for trans in transtion_by_label[label]: 170 | rsa.matrices[label][trans[0], trans[1]] = True 171 | 172 | if trans[0] in rsa.out_states: 173 | rsa.out_states[trans[0]].append((trans[1], label)) 174 | else: 175 | rsa.out_states[trans[0]] = [(trans[1], label)] 176 | 177 | rsa.terminals = rsa.labels.difference(rsa.nonterminals) 178 | return rsa 179 | -------------------------------------------------------------------------------- /src/graph/graph.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from tqdm import tqdm 3 | 4 | from pygraphblas import Matrix, BOOL 5 | from src.graph.index_graph import SAVEMIDDLETYPE 6 | from src.graph.length_graph import SAVELENGTHTYPE 7 | 8 | from src.utils.graph_size import get_graph_size 9 | 10 | 11 | class Graph: 12 | def __init__(self): 13 | self.path = "path/to/graph" 14 | self.type = None 15 | self.matrices_size = 0 16 | self.matrices = dict() 17 | 18 | def __getitem__(self, item: str) -> Matrix: 19 | if item not in self.matrices: 20 | self.matrices[item] = Matrix.sparse(self.type, self.matrices_size, self.matrices_size) 21 | return self.matrices[item] 22 | 23 | def __setitem__(self, key, value): 24 | self.matrices[key] = value 25 | 26 | def __iter__(self): 27 | return self.matrices.__iter__() 28 | 29 | def get_number_of_vertices(self): 30 | return self.matrices_size 31 | 32 | def get_number_of_edges(self): 33 | return sum([self.matrices[label].nvals for label in self.matrices]) 34 | 35 | @classmethod 36 | def from_txt(cls, path: Path): 37 | graph = Graph() 38 | graph.path = path 39 | return graph 40 | 41 | def load_bool_graph(self, verbose=False): 42 | self.type = BOOL 43 | self.matrices_size = get_graph_size(self.path) 44 | 45 | with open(self.path, "r") as f: 46 | for line in tqdm(f.readlines()) if verbose else f.readlines(): 47 | v, label, to = line.split() 48 | v, to = int(v), int(to) 49 | 50 | if label not in self.matrices: 51 | self.matrices[label] = Matrix.sparse(self.type, self.matrices_size, self.matrices_size) 52 | 53 | self.matrices[label][v, to] = True 54 | 55 | def load_save_middle_graph(self, verbose=False): 56 | self.type = SAVEMIDDLETYPE 57 | self.matrices_size = get_graph_size(self.path) 58 | 59 | with open(self.path, "r") as f: 60 | for line in tqdm(f.readlines()) if verbose else f.readlines(): 61 | v, label, to = line.split() 62 | v, to = int(v), int(to) 63 | 64 | if label not in self.matrices: 65 | self.matrices[label] = Matrix.sparse(self.type, self.matrices_size, self.matrices_size) 66 | 67 | self.matrices[label][v, to] = (v, to, v, 1, 1) 68 | 69 | def load_save_length_graph(self, verbose=False): 70 | self.type = SAVELENGTHTYPE 71 | self.matrices_size = get_graph_size(self.path) 72 | 73 | with open(self.path, "r") as f: 74 | for line in tqdm(f.readlines()) if verbose else f.readlines(): 75 | v, label, to = line.split() 76 | v, to = int(v), int(to) 77 | 78 | if label not in self.matrices: 79 | self.matrices[label] = Matrix.sparse(self.type, self.matrices_size, self.matrices_size) 80 | 81 | self.matrices[label][v, to] = (v, to, v, 1) 82 | -------------------------------------------------------------------------------- /src/graph/index_graph.py: -------------------------------------------------------------------------------- 1 | from pygraphblas import Matrix 2 | from pygraphblas.types import Type, binop 3 | 4 | MAX_MATRIX_SIZE = 10000000 5 | 6 | 7 | class SAVEMIDDLETYPE(Type): 8 | _base_name = "UDT" 9 | _numpy_t = None 10 | members = ['uint32_t left', 'uint32_t right', 'uint32_t middle', 'uint32_t height', 'uint32_t length'] 11 | one = (0, 0, 0, 0, 0) 12 | 13 | @binop(boolean=True) 14 | def EQ(z, x, y): 15 | if x.left == y.left and x.right == y.right and x.middle == y.middle and x.height == y.height \ 16 | and x.length == y.length: 17 | z = True 18 | else: 19 | z = False 20 | 21 | @binop() 22 | def PLUS(z, x, y): 23 | def is_eq_to_one(ind): 24 | return ind.left == 0 and ind.right == 0 and ind.middle == 0 and ind.height == 0 and ind.length == 0 25 | 26 | if not is_eq_to_one(x) and not is_eq_to_one(y): 27 | min_height_index = x if x.height < y.height else y 28 | z.left = min_height_index.left 29 | z.right = min_height_index.right 30 | z.middle = min_height_index.middle 31 | z.height = min_height_index.height 32 | z.length = min_height_index.length 33 | elif is_eq_to_one(x): 34 | z.left = y.left 35 | z.right = y.right 36 | z.middle = y.middle 37 | z.height = y.height 38 | z.length = y.length 39 | else: 40 | z.left = x.left 41 | z.right = x.right 42 | z.middle = x.middle 43 | z.height = x.height 44 | z.length = x.length 45 | 46 | @binop() 47 | def TIMES(z, x, y): 48 | def is_eq_to_one(ind): 49 | return ind.left == 0 and ind.right == 0 and ind.middle == 0 and ind.height == 0 and ind.length == 0 50 | 51 | if not is_eq_to_one(x) and not is_eq_to_one(y): 52 | z.left = x.left 53 | z.right = y.right 54 | z.middle = x.right 55 | z.height = (y.height if x.height < y.height else x.height) + 1 56 | z.length = x.length + y.length 57 | else: 58 | z.left = 0 59 | z.right = 0 60 | z.middle = 0 61 | z.height = 0 62 | z.length = 0 63 | 64 | 65 | class IndexGraph: 66 | def __init__(self, matrices_size=MAX_MATRIX_SIZE): 67 | self.matrices = {} 68 | self.matrices_size = matrices_size 69 | 70 | def __getitem__(self, item: str) -> Matrix: 71 | if item not in self.matrices: 72 | self.matrices[item] = Matrix.sparse(SAVEMIDDLETYPE, self.matrices_size, self.matrices_size) 73 | return self.matrices[item] 74 | 75 | def __setitem__(self, key, value): 76 | self.matrices[key] = value 77 | 78 | def __iter__(self): 79 | return self.matrices.__iter__() 80 | 81 | @classmethod 82 | def from_txt(cls, path): 83 | g = IndexGraph() 84 | with open(path, 'r') as f: 85 | for line in f.readlines(): 86 | v, label, to = line.split() 87 | v, to = int(v), int(to) 88 | g[label][v, to] = (v, to, v, 1, 1) 89 | return g 90 | -------------------------------------------------------------------------------- /src/graph/label_graph.py: -------------------------------------------------------------------------------- 1 | from pygraphblas.matrix import Matrix 2 | from pygraphblas.types import BOOL 3 | from tqdm import tqdm 4 | 5 | from src.utils.common import chunkify 6 | from src.utils.graph_size import get_graph_size 7 | 8 | MAX_MATRIX_SIZE = 1000000 9 | 10 | 11 | class LabelGraph: 12 | """ 13 | This class representing label directed graph. supports only the functions necessary for the algorithms to work 14 | """ 15 | def __init__(self, matrices_size=MAX_MATRIX_SIZE): 16 | self.matrices = {} 17 | self.matrices_size = matrices_size 18 | self.is_empty = True 19 | 20 | def __getitem__(self, item: str) -> Matrix: 21 | if item not in self.matrices: 22 | self.matrices[item] = Matrix.sparse(BOOL, self.matrices_size, self.matrices_size) 23 | return self.matrices[item] 24 | 25 | def __setitem__(self, key, value): 26 | self.is_empty = False 27 | self.matrices[key] = value 28 | 29 | def __iter__(self): 30 | return self.matrices.__iter__() 31 | 32 | def get_number_of_vertices(self): 33 | return self.matrices_size 34 | 35 | def get_number_of_edges(self): 36 | return sum([self.matrices[label].nvals for label in self.matrices]) 37 | 38 | def clone(self): 39 | obj_copy = LabelGraph(self.matrices_size) 40 | for nonterm, matr in self.matrices.items(): 41 | obj_copy[nonterm] = matr.dup() 42 | return obj_copy 43 | 44 | @classmethod 45 | def from_txt(cls, path, verbose=False): 46 | """ 47 | Load graph from file in format triplets 48 | @param path: path to file 49 | @param verbose: flag to set the output of information on download 50 | @return: initialized class 51 | """ 52 | g = LabelGraph(get_graph_size(path)) 53 | with open(path, 'r') as f: 54 | for line in tqdm(f.readlines()) if verbose else f.readlines(): 55 | v, label, to = line.split() 56 | v, to = int(v), int(to) 57 | g[label][v, to] = True 58 | return g 59 | 60 | def chunkify(self, chunk_len) -> list: 61 | return list(chunkify(list(range(self.matrices_size)), chunk_len)) 62 | 63 | def __add__(self, other): 64 | result = LabelGraph(self.matrices_size) 65 | 66 | if not self.is_empty and not other.is_empty: 67 | result.is_empty = False 68 | else: 69 | return result 70 | 71 | labels_only_in_self = set(self.matrices.keys()) - set(other.matrices.keys()) 72 | labels_only_in_other = set(other.matrices.keys()) - set(self.matrices.keys()) 73 | labels_in_both = set(self.matrices.keys()) & set(other.matrices.keys()) 74 | for label in labels_in_both: 75 | result.matrices[label] = self.matrices[label] + other.matrices[label] 76 | 77 | for label in labels_only_in_other: 78 | result.matrices[label] = other.matrices[label] 79 | 80 | for label in labels_only_in_self: 81 | result.matrices[label] = self.matrices[label] 82 | 83 | return result 84 | 85 | def __iadd__(self, other): 86 | if not other.is_empty: 87 | self.is_empty = False 88 | 89 | for label in other.matrices: 90 | if label in self.matrices: 91 | self.matrices[label] = self.matrices[label] + other.matrices[label] 92 | else: 93 | self.matrices[label] = other.matrices[label] 94 | 95 | return self 96 | -------------------------------------------------------------------------------- /src/graph/length_graph.py: -------------------------------------------------------------------------------- 1 | from pygraphblas import * 2 | from pygraphblas import Matrix 3 | from pygraphblas.types import Type, binop 4 | from pygraphblas.types import BOOL 5 | import numpy as np 6 | 7 | MAX_MATRIX_SIZE = 10000000 8 | UINT32_MAX = np.iinfo(np.uint32).max 9 | 10 | 11 | class SAVELENGTHTYPE(Type): 12 | _base_name = "UDT" 13 | _numpy_t = None 14 | members = ['uint32_t left', 'uint32_t right', 'uint32_t middle', 'uint32_t length'] 15 | one = (0, 0, 0, UINT32_MAX) 16 | 17 | @binop(boolean=True) 18 | def EQ(z, x, y): 19 | if x.left == y.left and x.right == y.right and x.middle == y.middle and x.length == y.length: 20 | z = True 21 | else: 22 | z = False 23 | 24 | @binop() 25 | def PLUS(z, x, y): 26 | def is_eq_to_one(ind): 27 | return ind.left == 0 and ind.right == 0 and ind.middle == 0 and ind.length == UINT32_MAX 28 | 29 | if not is_eq_to_one(x) and not is_eq_to_one(y): 30 | min_length_index = x if x.length < y.length else y 31 | z.left = min_length_index.left 32 | z.right = min_length_index.right 33 | z.middle = min_length_index.middle 34 | z.length = min_length_index.length 35 | elif is_eq_to_one(x): 36 | z.left = y.left 37 | z.right = y.right 38 | z.middle = y.middle 39 | z.length = y.length 40 | else: 41 | z.left = x.left 42 | z.right = x.right 43 | z.middle = x.middle 44 | z.length = x.length 45 | 46 | @binop() 47 | def TIMES(z, x, y): 48 | def is_eq_to_one(ind): 49 | return ind.left == 0 and ind.right == 0 and ind.middle == 0 and ind.length == UINT32_MAX 50 | 51 | if not is_eq_to_one(x) and not is_eq_to_one(y): 52 | z.left = x.left 53 | z.right = y.right 54 | z.middle = x.right 55 | z.length = x.length + y.length 56 | else: 57 | z.left = 0 58 | z.right = 0 59 | z.middle = 0 60 | z.length = MAX_MATRIX_SIZE 61 | 62 | @binop() 63 | def SUBTRACTION(z, x, y): 64 | if x.left == y.left and x.right == y.right and x.middle == y.middle and x.length == y.length: 65 | z.left = 0 66 | z.right = 0 67 | z.middle = 0 68 | z.length = 0 # for nonzero() function and GxB_NONZERO select operator 69 | else: 70 | z.left = UINT32_MAX 71 | z.right = UINT32_MAX 72 | z.middle = UINT32_MAX 73 | z.length = UINT32_MAX 74 | 75 | 76 | class LengthGraph: 77 | def __init__(self, matrices_size=MAX_MATRIX_SIZE): 78 | self.matrices = {} 79 | self.matrices_size = matrices_size 80 | 81 | def __getitem__(self, item: str) -> Matrix: 82 | if item not in self.matrices: 83 | self.matrices[item] = Matrix.sparse(SAVELENGTHTYPE, self.matrices_size, self.matrices_size) 84 | return self.matrices[item] 85 | 86 | def __setitem__(self, key, value): 87 | self.matrices[key] = value 88 | 89 | def __iter__(self): 90 | return self.matrices.__iter__() 91 | 92 | @classmethod 93 | def from_txt(cls, path): 94 | g = LengthGraph() 95 | with open(path, 'r') as f: 96 | for line in f.readlines(): 97 | v, label, to = line.split() 98 | v, to = int(v), int(to) 99 | g[label][v, to] = (v, to, v, 1) 100 | return g 101 | -------------------------------------------------------------------------------- /src/problems/AllPaths/AllPaths.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from pyformlang.cfg import CFG 3 | from src.graph.graph import Graph 4 | 5 | 6 | class AllPathsProblem(ABC): 7 | """ 8 | Base class for all paths problem 9 | """ 10 | 11 | @abstractmethod 12 | def prepare(self, graph: Graph, grammar: CFG): 13 | """ 14 | Prepare for the operation of the algorithm: load graph and grammar 15 | @param graph: path to file with graph 16 | @param grammar: path to file with grammar 17 | """ 18 | pass 19 | 20 | @abstractmethod 21 | def prepare_for_solve(self): 22 | pass 23 | 24 | @abstractmethod 25 | def solve(self): 26 | """ 27 | Solve problem with graph and grammar 28 | """ 29 | pass 30 | 31 | @abstractmethod 32 | def prepare_for_exctract_paths(self): 33 | pass 34 | 35 | @abstractmethod 36 | def getPaths(self, v_start: int, v_finish: int, nonterminal: str, max_len: int): 37 | """ 38 | Extract all paths between two vertices, the length of which does not exceed the given parameter 39 | @param v_start: starting vertex for paths 40 | @param v_finish: finishing vertex for paths 41 | @param nonterminal: nonterminal from which paths are being restored 42 | @param max_len: path length limitation 43 | """ 44 | pass 45 | -------------------------------------------------------------------------------- /src/problems/AllPaths/algo/matrix_all_paths/impl/Grammar.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "Grammar.h" 3 | #include 4 | #include 5 | #include 6 | 7 | using namespace std; 8 | 9 | using std::istringstream; 10 | using std::ifstream; 11 | using std::ofstream; 12 | using std::string; 13 | using std::vector; 14 | 15 | 16 | Grammar::Grammar(const string &grammar_filename) { 17 | 18 | auto chomsky_stream = ifstream(grammar_filename, ifstream::in); 19 | 20 | string line, tmp; 21 | getline(chomsky_stream, line); 22 | getline(chomsky_stream, line); 23 | while (getline(chomsky_stream, line)) { 24 | vector terms; 25 | istringstream iss(line); 26 | while (iss >> tmp) { 27 | if(tmp != "->") 28 | { 29 | terms.push_back(tmp); 30 | } 31 | } 32 | if (!nonterminal_to_index.count(terms[0])) { 33 | nonterminal_to_index[terms[0]] = nonterminals_count++; 34 | } 35 | if (terms.size() == 2) { 36 | if (!terminal_to_nonterminals.count(terms[1])) { 37 | terminal_to_nonterminals[terms[1]] = {}; 38 | } 39 | terminal_to_nonterminals[terms[1]].push_back(nonterminal_to_index[terms[0]]); 40 | } else if (terms.size() == 3) { 41 | if (!nonterminal_to_index.count(terms[1])) { 42 | nonterminal_to_index[terms[1]] = nonterminals_count++; 43 | } 44 | if (!nonterminal_to_index.count(terms[2])) { 45 | nonterminal_to_index[terms[2]] = nonterminals_count++; 46 | } 47 | rules.push_back( 48 | {nonterminal_to_index[terms[0]], {nonterminal_to_index[terms[1]], nonterminal_to_index[terms[2]]}}); 49 | } 50 | } 51 | chomsky_stream.close(); 52 | } 53 | 54 | Grammar::~Grammar() { 55 | for (unsigned int i = 0; i < nonterminals_count; ++i) 56 | delete matrices[i]; 57 | } 58 | 59 | char* Grammar::get_elements(std::string S) 60 | { 61 | return matrices[nonterminal_to_index[S]]->get_elements(); 62 | } 63 | 64 | void Grammar::print_results(const string &output_filename, int time) { 65 | ofstream out_stream; 66 | out_stream.open(output_filename, ios::app); 67 | out_stream << time / 1000.0 << std::endl; 68 | for (auto &nonterm : nonterminal_to_index) { 69 | unsigned long count = 0; 70 | out_stream << nonterm.first << " "; 71 | out_stream << matrices[nonterm.second]->get_nvals() << endl; 72 | } 73 | out_stream.close(); 74 | } 75 | 76 | -------------------------------------------------------------------------------- /src/problems/AllPaths/algo/matrix_all_paths/impl/Grammar.h: -------------------------------------------------------------------------------- 1 | #ifndef CFPQ_CFPQ_H 2 | #define CFPQ_CFPQ_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "apmatrix.h" 12 | #include "Graph.h" 13 | 14 | using nonterminals_pair = std::pair; 15 | 16 | class Grammar { 17 | 18 | public: 19 | explicit Grammar(const std::string &grammar_filename); 20 | 21 | ~Grammar(); 22 | 23 | char* get_elements(std::string S); 24 | 25 | unsigned int intersection_with_graph(Graph &graph) { 26 | vertices_count = graph.vertices_count; 27 | matrices.reserve(nonterminals_count); 28 | rules_with_nonterminal.reserve(nonterminals_count); 29 | 30 | for (unsigned int i = 0; i < nonterminals_count; ++i) { 31 | rules_with_nonterminal.emplace_back(); 32 | matrices.push_back(new ApMatrix(vertices_count)); 33 | } 34 | 35 | for (unsigned int i = 0; i < rules.size(); ++i) { 36 | rules_with_nonterminal[rules[i].second.first].push_back(i); 37 | rules_with_nonterminal[rules[i].second.second].push_back(i); 38 | to_recalculate.insert(i); 39 | } 40 | 41 | for (auto &edge : graph.edges) { 42 | for (unsigned int nonterm : terminal_to_nonterminals[edge.first]) { 43 | matrices[nonterm]->set_bit(edge.second.first, edge.second.second); 44 | } 45 | } 46 | 47 | using namespace std::chrono; 48 | high_resolution_clock::time_point begin_time = high_resolution_clock::now(); 49 | 50 | while (!to_recalculate.empty()) { 51 | unsigned int rule_index = *to_recalculate.begin(); 52 | to_recalculate.erase(to_recalculate.begin()); 53 | unsigned int C = rules[rule_index].first; 54 | unsigned int A = rules[rule_index].second.first; 55 | unsigned int B = rules[rule_index].second.second; 56 | 57 | if (matrices[C]->add_mul(matrices[A], matrices[B])) { 58 | for (unsigned int changed_rule_index: rules_with_nonterminal[C]) { 59 | to_recalculate.insert(changed_rule_index); 60 | } 61 | } 62 | } 63 | high_resolution_clock::time_point end_time = high_resolution_clock::now(); 64 | milliseconds elapsed_secs = duration_cast(end_time - begin_time); 65 | return static_cast(elapsed_secs.count()); 66 | } 67 | 68 | std::vector get_paths(unsigned int i, unsigned int j, const std::string &nonterm_name, unsigned int current_len) { 69 | if (current_len > 0) 70 | { 71 | unsigned int S = nonterminal_to_index[nonterm_name]; 72 | ApMatrix* m = matrices[S]; 73 | unsigned int size = m->get_size(); 74 | PathIndex* pindex = m->get_bit(i, j); 75 | if (PathIndex_IsIdentity(pindex)) 76 | { 77 | delete pindex; 78 | return std::vector(); 79 | } 80 | unsigned int k = 0; 81 | std::vector res; 82 | for (unsigned int mid = 0; mid < pindex->size; mid++) 83 | { 84 | k = pindex->middle[mid]; 85 | if (k == size) 86 | { 87 | res.push_back(1); 88 | } 89 | else 90 | { 91 | for (auto rule : rules) 92 | { 93 | if(rule.first == S) 94 | { 95 | unsigned int A = rule.second.first; 96 | unsigned int B = rule.second.second; 97 | std::string A_name, B_name; 98 | for (auto &nonterm : nonterminal_to_index) 99 | { 100 | if (nonterm.second == A) 101 | { 102 | A_name = nonterm.first; 103 | } 104 | else if (nonterm.second == B) 105 | { 106 | B_name = nonterm.first; 107 | } 108 | } 109 | 110 | std::vector left_paths = get_paths(i, k, A_name, current_len - 1); 111 | if(left_paths.size() > 0) 112 | { 113 | unsigned int min_len = current_len; 114 | for (auto p : left_paths) 115 | { 116 | if (p < min_len) 117 | { 118 | min_len = p; 119 | } 120 | } 121 | 122 | std::vector right_paths = get_paths(k, j, B_name, current_len - min_len); 123 | for(auto p1 : left_paths) 124 | { 125 | for(auto p2 : right_paths) 126 | { 127 | if (p1 + p2 < current_len) 128 | { 129 | res.push_back(p1 + p2); 130 | } 131 | } 132 | } 133 | } 134 | 135 | } 136 | } 137 | } 138 | 139 | } 140 | 141 | delete pindex; 142 | 143 | return res; 144 | } 145 | return std::vector(); 146 | } 147 | 148 | void print_results(const std::string &output_filename, int time); 149 | 150 | unsigned int nonterminals_count = 0; 151 | unsigned int vertices_count = 0; 152 | 153 | std::unordered_set to_recalculate; 154 | std::vector> rules_with_nonterminal; 155 | std::map nonterminal_to_index; 156 | std::unordered_map> terminal_to_nonterminals; 157 | std::vector> rules; 158 | std::vector matrices; 159 | }; 160 | 161 | #endif //CFPQ_CFPQ_H 162 | 163 | -------------------------------------------------------------------------------- /src/problems/AllPaths/algo/matrix_all_paths/impl/Graph.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include "Graph.h" 5 | 6 | using std::string; 7 | using std::ifstream; 8 | using std::max; 9 | 10 | Graph::Graph(const string &graph_filename) { 11 | auto graph_stream = ifstream(graph_filename, ifstream::in); 12 | unsigned int from, to; 13 | string terminal; 14 | while (graph_stream >> from >> terminal >> to) { 15 | edges.push_back({terminal, {from, to}}); 16 | vertices_count = max(vertices_count, max(from, to) + 1); 17 | } 18 | graph_stream.close(); 19 | } 20 | -------------------------------------------------------------------------------- /src/problems/AllPaths/algo/matrix_all_paths/impl/Graph.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef CFPQ_GRAPH_H 3 | #define CFPQ_GRAPH_H 4 | 5 | #include 6 | #include 7 | 8 | using edge = std::pair>; 9 | 10 | class Graph { 11 | public: 12 | explicit Graph(const std::string &graph_filename); 13 | 14 | virtual ~Graph() = default; 15 | 16 | std::vector edges; 17 | 18 | unsigned int vertices_count = 0; 19 | }; 20 | 21 | #endif //CFPQ_GRAPH_H 22 | -------------------------------------------------------------------------------- /src/problems/AllPaths/algo/matrix_all_paths/impl/Makefile: -------------------------------------------------------------------------------- 1 | libAllPaths.so: main.o apmatrix.o Grammar.o Graph.o pathindex.o 2 | $(CXX) -shared $^ -o $@ -Wl,--whole-archive -lgraphblas -Wl,--no-whole-archive 3 | 4 | main.o apmatrix.o Grammar.o Graph.o pathindex.o : CXXFLAGS+=-fPIC 5 | -------------------------------------------------------------------------------- /src/problems/AllPaths/algo/matrix_all_paths/impl/apmatrix.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "apmatrix.h" 3 | #include 4 | #include "pathindex.h" 5 | 6 | void ApMatrix::set_bit(unsigned int row, unsigned col) { 7 | PathIndex index; 8 | uint32_t *middles = static_cast(malloc(sizeof(uint32_t))); 9 | middles[0] = size; //size --- means that this is edge 10 | PathIndex_Init(&index, row, col, middles, 1); 11 | GrB_Matrix_setElement_UDT(m, (void *) &index, row, col); 12 | } 13 | 14 | PathIndex* ApMatrix::get_bit(unsigned int row, unsigned col) { 15 | PathIndex* index = new PathIndex; 16 | PathIndex_Init(index, 0, 0, 0, 0); 17 | GrB_Matrix_extractElement_UDT((void *) index, m, row, col); 18 | return index; 19 | } 20 | 21 | ApMatrix::~ApMatrix() { 22 | GrB_Matrix_clear(m); 23 | GrB_Matrix_free(&m); 24 | } 25 | 26 | uint32_t ApMatrix::get_nvals() 27 | { 28 | GrB_Index count; 29 | GrB_Matrix_nvals(&count, m); 30 | return count; 31 | } 32 | 33 | char* ApMatrix::get_elements() 34 | { 35 | GrB_Index nvals = get_nvals(); 36 | GrB_Index *I = static_cast(malloc(nvals * sizeof(GrB_Index))); 37 | GrB_Index *J = static_cast(malloc(nvals * sizeof(GrB_Index))); 38 | PathIndex *X = static_cast(malloc(nvals * sizeof(PathIndex))); 39 | GrB_Matrix_extractTuples_UDT(I, J, X, &nvals, m); 40 | char* result = (char *)malloc(nvals*25*sizeof(char)); //??? 41 | std::string start = ""; 42 | strcpy(result, start.c_str()); 43 | for (int k = 0; k < nvals; ++k) 44 | { 45 | 46 | std::string indexes = std::to_string(X[k].left) + " " + std::to_string(X[k].right) + "\n"; 47 | strcat(result, indexes.c_str()); 48 | } 49 | free(I); 50 | free(J); 51 | free(X); 52 | return result; 53 | } 54 | 55 | bool ApMatrix::add_mul(ApMatrix *A, ApMatrix *B) { 56 | GrB_Info info; 57 | GrB_Matrix m_old; 58 | GrB_Matrix_dup(&m_old, m); 59 | info = GrB_mxm(m, GrB_NULL, IndexType_Add, IndexType_Semiring, A->m, B->m, GrB_NULL); 60 | GrB_Index nvals_new, nvals_old; 61 | GrB_Matrix_nvals(&nvals_old, m_old); 62 | GrB_Index *I = static_cast(malloc(nvals_old * sizeof(GrB_Index))); 63 | GrB_Index *J = static_cast(malloc(nvals_old * sizeof(GrB_Index))); 64 | PathIndex *X = static_cast(malloc(nvals_old * sizeof(PathIndex))); 65 | int sum_old = 0; 66 | GrB_Matrix_extractTuples_UDT(I, J, X, &nvals_old, m_old); 67 | for (int k = 0; k < nvals_old; ++k) { 68 | sum_old += X[k].size; 69 | } 70 | free(I); 71 | free(J); 72 | free(X); 73 | 74 | GrB_Matrix_nvals(&nvals_new, m); 75 | I = static_cast(malloc(nvals_new * sizeof(GrB_Index))); 76 | J = static_cast(malloc(nvals_new * sizeof(GrB_Index))); 77 | X = static_cast(malloc(nvals_new * sizeof(PathIndex))); 78 | int sum_new = 0; 79 | GrB_Matrix_extractTuples_UDT(I, J, X, &nvals_new, m); 80 | for (int k = 0; k < nvals_new; ++k) { 81 | sum_new += X[k].size; 82 | } 83 | free(I); 84 | free(J); 85 | free(X); 86 | 87 | bool changed = sum_new != sum_old; 88 | GrB_Matrix_free(&m_old); 89 | return changed; 90 | } 91 | -------------------------------------------------------------------------------- /src/problems/AllPaths/algo/matrix_all_paths/impl/apmatrix.h: -------------------------------------------------------------------------------- 1 | #ifndef APMATRIX_H 2 | #define APMATRIX_H 3 | 4 | extern "C" { 5 | #include "GraphBLAS.h" 6 | } 7 | #include "pathindex.h" 8 | 9 | class ApMatrix 10 | { 11 | public: 12 | explicit ApMatrix(unsigned int n) { 13 | GrB_Matrix_new(&m, PathIndexType, n, n); 14 | size = n; 15 | } 16 | 17 | ~ApMatrix(); 18 | 19 | void set_bit(unsigned int row, unsigned col); 20 | uint32_t get_nvals(); 21 | char* get_elements(); 22 | 23 | PathIndex* get_bit(unsigned int row, unsigned col); 24 | 25 | bool add_mul(ApMatrix *A, ApMatrix *B); 26 | 27 | unsigned int get_size() 28 | { 29 | return size; 30 | } 31 | 32 | private: 33 | GrB_Matrix m; 34 | unsigned int size; 35 | 36 | }; 37 | 38 | #endif // APMATRIX_H 39 | -------------------------------------------------------------------------------- /src/problems/AllPaths/algo/matrix_all_paths/impl/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | extern "C" { 6 | #include "GraphBLAS.h" 7 | } 8 | #include 9 | #include "Grammar.h" 10 | #include "Graph.h" 11 | #include "apmatrix.h" 12 | #include "pathindex.h" 13 | using namespace std; 14 | 15 | extern "C" { 16 | 17 | void string_del(char *elements) 18 | { 19 | free(elements); 20 | } 21 | 22 | Grammar *grammar_new(char *name) { 23 | return new Grammar(name); 24 | } 25 | 26 | void grammar_del(Grammar* grammar) 27 | { 28 | delete grammar; 29 | } 30 | 31 | Graph *graph_new(char *name) { 32 | return new Graph(name); 33 | } 34 | 35 | void graph_del(Graph* graph) 36 | { 37 | delete graph; 38 | } 39 | 40 | void graphblas_init() 41 | { 42 | GrB_init(GrB_NONBLOCKING); 43 | InitGBSemiring(); 44 | } 45 | 46 | void graphblas_finalize() 47 | { 48 | GrB_Semiring_free(&IndexType_Semiring); 49 | GrB_Monoid_free(&IndexType_Monoid); 50 | GrB_BinaryOp_free(&IndexType_Add); 51 | GrB_BinaryOp_free(&IndexType_Mul); 52 | GrB_Type_free(&PathIndexType); 53 | GrB_finalize(); 54 | } 55 | 56 | void intersect(Grammar* grammar, Graph* graph) 57 | { 58 | auto times = grammar->intersection_with_graph(*graph); 59 | } 60 | 61 | char* get_elements(Grammar* grammar, char *S) 62 | { 63 | return grammar->get_elements(S); 64 | } 65 | 66 | int getpaths(Grammar* grammar, int i, int j, char *S, int current_len) 67 | { 68 | std::vector res = grammar->get_paths(i, j, S, current_len); 69 | int result = res.size(); 70 | 71 | return result; 72 | } 73 | 74 | } 75 | 76 | int main(int argc, char *argv[]) { 77 | return 0; 78 | } 79 | 80 | -------------------------------------------------------------------------------- /src/problems/AllPaths/algo/matrix_all_paths/impl/pathindex.cpp: -------------------------------------------------------------------------------- 1 | #include "pathindex.h" 2 | 3 | bool isChanged = false; 4 | 5 | PathIndex PathIndex_Identity = { 6 | .left = 0, 7 | .right = 0, 8 | .middle = 0, 9 | .size = 0, 10 | }; 11 | 12 | // Identity = невыводимый путь 13 | 14 | GrB_BinaryOp IndexType_Add; 15 | GrB_BinaryOp IndexType_Mul; 16 | GrB_Monoid IndexType_Monoid; 17 | GrB_Semiring IndexType_Semiring; 18 | GrB_Type PathIndexType; 19 | 20 | void InitGBSemiring() 21 | { 22 | GrB_Type_new (&PathIndexType, sizeof(PathIndex)); 23 | GrB_BinaryOp_new(&IndexType_Add, PathIndex_Add, PathIndexType, PathIndexType, PathIndexType); 24 | GrB_BinaryOp_new(&IndexType_Mul, PathIndex_Mul, PathIndexType, PathIndexType, PathIndexType); 25 | GrB_Monoid_new_UDT(&IndexType_Monoid, IndexType_Add, (void *) &PathIndex_Identity); 26 | GrB_Semiring_new(&IndexType_Semiring, IndexType_Monoid, IndexType_Mul); 27 | } 28 | 29 | void PathIndex_Init(PathIndex *index, uint32_t left, uint32_t right, uint32_t* middle, uint32_t size) { 30 | index->left = left; 31 | index->right = right; 32 | index->middle = middle; 33 | index->size = size; 34 | //index->isEdge = isEdge; 35 | } 36 | 37 | void PathIndex_InitIdentity(PathIndex *index) { 38 | index->left = 0; 39 | index->right = 0; 40 | index->middle = 0; 41 | index->size = 0; 42 | } 43 | 44 | bool PathIndex_IsIdentity(PathIndex *index) { 45 | return (index->size == 0); 46 | } 47 | 48 | void PathIndex_Copy(const PathIndex *from, PathIndex *to) { 49 | to->left = from->left; 50 | to->right = from->right; 51 | to->middle = static_cast(malloc(from->size * sizeof(uint32_t))); 52 | for (uint32_t i = 0; i < from->size; ++i) 53 | { 54 | to->middle[i] = from->middle[i]; 55 | } 56 | to->size = from->size; 57 | } 58 | 59 | void PathIndex_Mul(void *z, const void *x, const void *y) { 60 | //std::cout << "MUL" << std::endl; 61 | PathIndex *left = (PathIndex *) x; 62 | PathIndex *right = (PathIndex *) y; 63 | PathIndex *res = (PathIndex *) z; 64 | 65 | if (!PathIndex_IsIdentity(left) && !PathIndex_IsIdentity(right)) { 66 | uint32_t *middle = static_cast(malloc(sizeof(uint32_t))); 67 | middle[0] = left->right; 68 | PathIndex_Init(res, left->left, right->right, middle, 1/*, false*/); 69 | } else { 70 | PathIndex_InitIdentity(res); 71 | } 72 | } 73 | 74 | uint32_t Merge_Middles(uint32_t *res, uint32_t* left, uint32_t lsize, uint32_t* right, uint32_t rsize) 75 | { 76 | uint32_t pres = 0, pl = 0, pr = 0; 77 | while(pl != lsize || pr != rsize) 78 | { 79 | if (pl == lsize) 80 | { 81 | res[pres] = right[pr]; 82 | pres++; 83 | pr++; 84 | } 85 | else if (pr == rsize) 86 | { 87 | res[pres] = left[pl]; 88 | pres++; 89 | pl++; 90 | } 91 | else 92 | { 93 | uint32_t x1 = left[pl]; 94 | uint32_t x2 = right[pr]; 95 | if(x1 == x2) 96 | { 97 | res[pres] = x1; 98 | pres++; 99 | pr++; 100 | pl++; 101 | } 102 | else if (x1 < x2) 103 | { 104 | res[pres] = x1; 105 | pres++; 106 | pl++; 107 | } 108 | else 109 | { 110 | res[pres] = x2; 111 | pres++; 112 | pr++; 113 | } 114 | } 115 | } 116 | return pres; 117 | } 118 | 119 | void PathIndex_Add(void *z, const void *x, const void *y) { 120 | //std::cout << "ADD" << std::endl; 121 | PathIndex *left = (PathIndex *) x; 122 | PathIndex *right = (PathIndex *) y; 123 | PathIndex *res = (PathIndex *) z; 124 | 125 | if (!PathIndex_IsIdentity(left) && !PathIndex_IsIdentity(right)) { 126 | uint32_t* middle = static_cast(malloc((left->size + right->size)*sizeof(uint32_t))); 127 | uint32_t res_size = Merge_Middles(middle, left->middle, left->size, right->middle, right->size); 128 | uint32_t* new_res = static_cast(malloc(res_size*sizeof(uint32_t))); 129 | for (uint32_t i = 0; i < res_size; ++i) 130 | { 131 | new_res[i] = middle[i]; 132 | } 133 | free(middle); 134 | middle = new_res; 135 | PathIndex_Init(res, left->left, right->right, middle, res_size); 136 | } else if (PathIndex_IsIdentity(left)) { 137 | PathIndex_Copy(right, res); 138 | } else { 139 | PathIndex_Copy(left, res); 140 | } 141 | } 142 | 143 | 144 | void PathIndex_ToStr(PathIndex *index) { 145 | if (PathIndex_IsIdentity(index)) { 146 | std::cout << "( Identity )"; 147 | } else { 148 | std::cout << "(i:" << index->left << ",j:" << index->right << ",k:["; 149 | for (uint32_t i = 0; i < index->size; ++i) 150 | { 151 | std::cout << index->middle[i] << ","; 152 | } 153 | std::cout << "],size=" << index->size << ")"; 154 | } 155 | } 156 | 157 | void PathIndex_Show(PathIndex *index) { 158 | PathIndex_ToStr(index); 159 | } 160 | 161 | 162 | void PathIndex_MatrixShow(const GrB_Matrix *matrix) { 163 | GrB_Index n, m; 164 | GrB_Matrix_nrows(&n, *matrix); 165 | GrB_Matrix_ncols(&m, *matrix); 166 | 167 | 168 | for (GrB_Index i = 0; i < n; i++) { 169 | for (GrB_Index j = 0; j < m; j++) { 170 | PathIndex index; 171 | PathIndex_InitIdentity(&index); 172 | 173 | GrB_Matrix_extractElement_UDT((void *) &index, *matrix, i, j); 174 | std::cout << "(i: " << i << ", j: " << j << ", index: "; 175 | PathIndex_ToStr(&index); 176 | std::cout << ")" << std::endl; 177 | } 178 | std::cout << std::endl; 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /src/problems/AllPaths/algo/matrix_all_paths/impl/pathindex.h: -------------------------------------------------------------------------------- 1 | #ifndef PATHINDEX_H 2 | #define PATHINDEX_H 3 | #include 4 | #include 5 | extern "C" { 6 | #include "GraphBLAS.h" 7 | } 8 | 9 | typedef struct { 10 | uint32_t left; 11 | uint32_t right; 12 | uint32_t *middle; 13 | uint32_t size; 14 | } PathIndex; 15 | 16 | extern GrB_BinaryOp IndexType_Add; 17 | extern GrB_BinaryOp IndexType_Mul; 18 | extern GrB_Monoid IndexType_Monoid; 19 | extern GrB_Semiring IndexType_Semiring; 20 | extern GrB_Type PathIndexType; 21 | 22 | void InitGBSemiring(); 23 | 24 | extern bool isChanged; 25 | extern PathIndex PathIndex_Identity; 26 | 27 | void PathIndex_Init(PathIndex *index, uint32_t left, uint32_t right, uint32_t* middle, uint32_t size); 28 | void PathIndex_InitIdentity(PathIndex *index); 29 | void PathIndex_Copy(const PathIndex *from, PathIndex *to); 30 | 31 | bool PathIndex_IsIdentity(PathIndex *index); 32 | 33 | void PathIndex_Mul(void *z, const void *x, const void *y); 34 | void PathIndex_Add(void *z, const void *x, const void *y); 35 | 36 | void PathIndex_ToStr(PathIndex *index); 37 | void PathIndex_Show(PathIndex *index); 38 | 39 | void PathIndex_MatrixShow(const GrB_Matrix *matrix); 40 | #endif // PATHINDEX_H 41 | -------------------------------------------------------------------------------- /src/problems/AllPaths/algo/matrix_all_paths/matrix_all_paths.py: -------------------------------------------------------------------------------- 1 | import ctypes 2 | import ctypes.util 3 | import os.path 4 | from pathlib import Path 5 | 6 | from src.problems.AllPaths.AllPaths import AllPathsProblem 7 | 8 | PATH_TO_SO = Path('src/problems/AllPaths/algo/matrix_all_paths/impl') 9 | 10 | 11 | class LibBuilder: 12 | def __init__(self): 13 | if not os.path.isfile(PATH_TO_SO.joinpath('libAllPaths.so')): 14 | raise Exception("Please run the command 'make' in src/problems/AllPaths/algo/matrix_all_paths/impl") 15 | 16 | self.lib = ctypes.CDLL(str(PATH_TO_SO.joinpath('libAllPaths.so'))) 17 | LP_c_char = ctypes.POINTER(ctypes.c_char) 18 | self.lib.grammar_new.argtypes = [LP_c_char] 19 | self.lib.grammar_new.restype = ctypes.c_void_p 20 | 21 | self.lib.grammar_del.argtypes = [ctypes.c_void_p] 22 | 23 | self.lib.graph_new.argtypes = [LP_c_char] 24 | self.lib.graph_new.restype = ctypes.c_void_p 25 | 26 | self.lib.graph_del.argtypes = [ctypes.c_void_p] 27 | 28 | self.lib.intersect.argtypes = [ctypes.c_void_p, ctypes.c_void_p] 29 | 30 | self.lib.get_elements.argtypes = [ctypes.c_void_p, LP_c_char] 31 | self.lib.get_elements.restype = ctypes.c_void_p 32 | 33 | self.lib.string_del.argtypes = [ctypes.c_void_p] 34 | 35 | self.lib.getpaths.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int, LP_c_char, ctypes.c_int] 36 | self.lib.getpaths.restype = ctypes.c_int 37 | self.lib.graphblas_init() 38 | 39 | 40 | class MatrixAllAlgo: 41 | 42 | def prepare(self, graph: Path, grammar: Path): 43 | lib_builder = LibBuilder() 44 | self.lib = lib_builder.lib 45 | self.grammar = self.lib.grammar_new(str(grammar.with_suffix(".cnf")).encode('utf-8')) 46 | self.graph = self.lib.graph_new(str(graph.with_suffix(".txt")).encode('utf-8')) 47 | 48 | def __del__(self): 49 | if self.grammar: 50 | self.lib.grammar_del(self.grammar) 51 | if self.graph: 52 | self.lib.graph_del(self.graph) 53 | self.lib.graphblas_finalize() 54 | 55 | def get_grammar(self): 56 | return self.grammar 57 | 58 | def solve(self): 59 | self.lib.intersect(self.grammar, self.graph) 60 | 61 | def getPaths(self, v_start, v_finish, nonterminal, max_len): 62 | return self.lib.getpaths(self.grammar, v_start, v_finish, nonterminal.encode('utf-8'), max_len) 63 | 64 | def get_elements(self, label): 65 | elements = self.lib.get_elements(self.grammar, label.encode('utf-8')) 66 | result = ctypes.cast(elements, ctypes.c_char_p).value 67 | self.lib.string_del(elements) 68 | return result.decode('utf-8') 69 | -------------------------------------------------------------------------------- /src/problems/AllPaths/algo/tensor/tensor.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from cfpq_data import RSM 3 | from pyformlang.cfg import CFG 4 | from pygraphblas import Matrix, BOOL 5 | from src.graph.graph import Graph 6 | from typing import Iterable, Union 7 | 8 | from src.grammar.rsa import RecursiveAutomaton 9 | from src.graph.label_graph import LabelGraph 10 | from src.problems.AllPaths.algo.tensor.tensor_path import TensorPathsNew 11 | from src.problems.AllPaths.algo.tensor.tensor_extract_subgraph import TensorExtractSubGraph 12 | 13 | from src.problems.AllPaths.AllPaths import AllPathsProblem 14 | 15 | from src.problems.utils import ResultAlgo 16 | 17 | 18 | def restore_eps_paths(nonterminals: Iterable, graph: Graph): 19 | for label in nonterminals: 20 | for i in range(graph.matrices_size): 21 | graph[label][i, i] = True 22 | 23 | 24 | def transitive_closure(m: Matrix): 25 | prev = m.nvals 26 | degree = m 27 | with BOOL.ANY_PAIR: 28 | degree = degree @ m 29 | m += degree 30 | while prev != m.nvals: 31 | prev = m.nvals 32 | with BOOL.ANY_PAIR: 33 | degree = degree @ m 34 | m += degree 35 | 36 | 37 | class TensorSimpleAlgo(AllPathsProblem): 38 | 39 | def prepare(self, graph: Graph, grammar: Union[RSM, CFG, Path]): 40 | self.graph = graph 41 | self.graph.load_bool_graph() 42 | self.grammar = RecursiveAutomaton.from_grammar_or_path(grammar) 43 | 44 | def solve(self): 45 | restore_eps_paths(self.grammar.start_and_finish, self.graph) 46 | 47 | sizeKron = self.graph.matrices_size * self.grammar.matrices_size 48 | 49 | iter = 0 50 | changed = True 51 | while changed: 52 | iter += 1 53 | changed = False 54 | 55 | kron = Matrix.sparse(BOOL, sizeKron, sizeKron) 56 | for label in self.grammar.labels: 57 | kron += self.grammar[label].kronecker(self.graph[label]) 58 | 59 | transitive_closure(kron) 60 | 61 | # update 62 | for nonterminal in self.grammar.nonterminals: 63 | for element in self.grammar.states[nonterminal]: 64 | i = element[0] 65 | j = element[1] 66 | 67 | start_i = i * self.graph.matrices_size 68 | start_j = j * self.graph.matrices_size 69 | 70 | control_sum = self.graph[nonterminal].nvals 71 | block = kron[start_i:start_i + self.graph.matrices_size - 1, 72 | start_j:start_j + self.graph.matrices_size - 1] 73 | 74 | self.graph[nonterminal] += block 75 | new_control_sum = self.graph[nonterminal].nvals 76 | 77 | if new_control_sum != control_sum: 78 | changed = True 79 | 80 | if self.grammar.nonterminals.isdisjoint(self.grammar.labels): 81 | break 82 | 83 | return ResultAlgo(self.graph[self.grammar.start_nonterm], iter) 84 | 85 | def prepare_for_solve(self): 86 | for label in self.grammar.nonterminals: 87 | self.graph[label].clear() 88 | 89 | def prepare_for_exctract_paths(self): 90 | sizeKron = self.graph.matrices_size * self.grammar.matrices_size 91 | self.kron = Matrix.sparse(BOOL, sizeKron, sizeKron) 92 | for label in self.grammar.labels: 93 | self.kron += self.grammar[label].kronecker(self.graph[label]) 94 | 95 | def getPaths(self, v_start: int, v_finish: int, nonterminal: str, max_len: int): 96 | return TensorPathsNew(self.graph, self.grammar, self.kron).get_paths(v_start, v_finish, nonterminal, max_len) 97 | 98 | def get_sub_graph(self, v_start: int, v_finish: int, nonterminal: str, max_high: int): 99 | return TensorExtractSubGraph(self.graph, self.grammar, self.kron, max_high).get_sub_graph(v_start, 100 | v_finish, 101 | nonterminal) 102 | 103 | 104 | class TensorDynamicAlgo(AllPathsProblem): 105 | 106 | def prepare(self, graph: Graph, grammar: Union[RSM, CFG, Path]): 107 | self.graph = graph 108 | self.graph.load_bool_graph() 109 | self.grammar = RecursiveAutomaton.from_grammar_or_path(grammar) 110 | 111 | def solve(self): 112 | restore_eps_paths(self.grammar.start_and_finish, self.graph) 113 | 114 | sizeKron = self.graph.matrices_size * self.grammar.matrices_size 115 | 116 | prev_kron = Matrix.sparse(BOOL, sizeKron, sizeKron) 117 | iter = 0 118 | block = LabelGraph(self.graph.matrices_size) 119 | changed = True 120 | first_iter = True 121 | while changed: 122 | changed = False 123 | iter += 1 124 | 125 | kron = Matrix.sparse(BOOL, sizeKron, sizeKron) 126 | 127 | if first_iter: 128 | for label in self.grammar.labels: 129 | kron += self.grammar[label].kronecker(self.graph[label]) 130 | else: 131 | for nonterminal in block.matrices: 132 | kron += self.grammar[nonterminal].kronecker(block[nonterminal]) 133 | block[nonterminal] = Matrix.sparse(BOOL, self.graph.matrices_size, self.graph.matrices_size) 134 | 135 | transitive_closure(kron) 136 | 137 | if not first_iter: 138 | part = prev_kron.mxm(kron, semiring=BOOL.ANY_PAIR) 139 | with BOOL.ANY_PAIR: 140 | kron += prev_kron + part @ prev_kron + part + kron @ prev_kron 141 | 142 | prev_kron = kron 143 | 144 | for nonterminal in self.grammar.nonterminals: 145 | for element in self.grammar.states[nonterminal]: 146 | i = element[0] 147 | j = element[1] 148 | 149 | start_i = i * self.graph.matrices_size 150 | start_j = j * self.graph.matrices_size 151 | 152 | control_sum = self.graph[nonterminal].nvals 153 | 154 | if first_iter: 155 | block[nonterminal] += kron[start_i:start_i + self.graph.matrices_size - 1, 156 | start_j:start_j + self.graph.matrices_size - 1] 157 | else: 158 | new_edges = kron[start_i:start_i + self.graph.matrices_size - 1, 159 | start_j:start_j + self.graph.matrices_size - 1] 160 | part = new_edges - block[nonterminal] 161 | block[nonterminal] += part.select('==', True) 162 | 163 | self.graph[nonterminal] += block[nonterminal] 164 | new_control_sum = self.graph[nonterminal].nvals 165 | 166 | if new_control_sum != control_sum: 167 | changed = True 168 | 169 | first_iter = False 170 | 171 | if self.grammar.nonterminals.isdisjoint(self.grammar.labels): 172 | break 173 | 174 | return ResultAlgo(self.graph[self.grammar.start_nonterm], iter) 175 | 176 | def prepare_for_solve(self): 177 | for label in self.grammar.nonterminals: 178 | self.graph[label].clear() 179 | 180 | def prepare_for_exctract_paths(self): 181 | sizeKron = self.graph.matrices_size * self.grammar.matrices_size 182 | self.kron = Matrix.sparse(BOOL, sizeKron, sizeKron) 183 | for label in self.grammar.labels: 184 | self.kron += self.grammar[label].kronecker(self.graph[label]) 185 | 186 | def getPaths(self, v_start: int, v_finish: int, nonterminal: str, max_len: int): 187 | return TensorPathsNew(self.graph, self.grammar, self.kron).get_paths(v_start, v_finish, nonterminal, max_len) 188 | 189 | def get_sub_graph(self, v_start: int, v_finish: int, nonterminal: str, max_high: int): 190 | return TensorExtractSubGraph(self.graph, self.grammar, self.kron, max_high).get_sub_graph(v_start, 191 | v_finish, 192 | nonterminal) 193 | -------------------------------------------------------------------------------- /src/problems/AllPaths/algo/tensor/tensor_extract_subgraph.py: -------------------------------------------------------------------------------- 1 | from pygraphblas import Matrix, BOOL 2 | 3 | from src.graph.graph import Graph 4 | from src.grammar.rsa import RecursiveAutomaton 5 | from src.graph.label_graph import LabelGraph 6 | 7 | 8 | class TensorExtractSubGraph: 9 | 10 | def __init__(self, graph: Graph, rsa: RecursiveAutomaton, kron: Matrix): 11 | self.graph = graph 12 | self.rsa = rsa 13 | self.kron = kron 14 | self.graph_size = graph.matrices_size 15 | self.solved_triplets = set() 16 | self.visited_pairs_in_kron = set() 17 | self.negative_visited_in_kron = set() 18 | 19 | def get_sub_graph(self, i, j, nonterm): 20 | return self.get_paths(i, j, nonterm) 21 | 22 | def get_paths(self, start, finish, nonterm): 23 | 24 | if (start, finish, nonterm) in self.solved_triplets: 25 | bogus_result = LabelGraph(self.graph_size) 26 | bogus_result.is_empty = False 27 | return bogus_result 28 | 29 | self.solved_triplets.add((start, finish, nonterm)) 30 | 31 | result = LabelGraph(self.graph_size) 32 | start_state = self.rsa.start_state[nonterm] 33 | 34 | for finish_state in self.rsa.finish_states[nonterm]: 35 | result += self.bfs(start_state * self.graph_size + start, finish_state * self.graph_size + finish) 36 | 37 | self.solved_triplets.remove((start, finish, nonterm)) 38 | 39 | return result 40 | 41 | def bfs(self, i, j): 42 | 43 | if (i, j) in self.negative_visited_in_kron: 44 | return LabelGraph(self.graph_size) 45 | 46 | if (i, j) in self.visited_pairs_in_kron: 47 | bogus_result = LabelGraph(self.graph_size) 48 | bogus_result.is_empty = False 49 | return bogus_result 50 | self.visited_pairs_in_kron.add((i, j)) 51 | 52 | result_graph = LabelGraph(self.graph_size) 53 | for k in self.kron[i]: # k = (vertex, True) 54 | graph_i = i % self.graph_size 55 | graph_k = k[0] % self.graph_size 56 | 57 | rsa_i = i // self.graph_size 58 | rsa_k = k[0] // self.graph_size 59 | 60 | left = LabelGraph(self.graph_size) 61 | for label in self.rsa.labels: 62 | if self.rsa[label].get(rsa_i, rsa_k, default=False): 63 | if label in self.rsa.nonterminals: 64 | left += self.get_paths(graph_i, graph_k, label) 65 | else: 66 | if label not in left.matrices: 67 | left[label] = Matrix.sparse(BOOL, self.graph_size, self.graph_size) 68 | left[label][graph_i, graph_k] = True 69 | 70 | if left.is_empty: 71 | continue 72 | 73 | right = LabelGraph(self.graph_size) 74 | if k[0] != j: 75 | right = self.bfs(k[0], j) 76 | else: 77 | right.is_empty = False 78 | 79 | if right.is_empty: 80 | continue 81 | 82 | result_graph += left + right 83 | 84 | if result_graph.is_empty: 85 | self.negative_visited_in_kron.add((i, j)) 86 | 87 | self.visited_pairs_in_kron.remove((i, j)) 88 | 89 | return result_graph 90 | -------------------------------------------------------------------------------- /src/problems/AllPaths/algo/tensor/tensor_path.py: -------------------------------------------------------------------------------- 1 | from pygraphblas import Matrix 2 | 3 | from src.graph.graph import Graph 4 | from src.grammar.rsa import RecursiveAutomaton 5 | 6 | 7 | class Paths: 8 | def __init__(self, path, current_vertex): 9 | self.path = path 10 | self.current_vertex = current_vertex 11 | self.use = True 12 | 13 | def close(self): 14 | self.use = False 15 | 16 | 17 | class TensorPathsNew: 18 | def __init__(self, graph: Graph, rsa: RecursiveAutomaton, tc: Matrix): 19 | self.graph = graph 20 | self.rsa = rsa 21 | self.tc = tc 22 | self.graph_size = graph.matrices_size 23 | 24 | def get_paths(self, start, finish, nonterm, max_len): 25 | start_state = self.rsa.start_state[nonterm] 26 | 27 | result = [] 28 | for finish_state in self.rsa.finish_states[nonterm]: 29 | result.extend(self.bfs(start_state * self.graph_size + start, finish_state * self.graph_size + finish, max_len)) 30 | 31 | return result 32 | 33 | def bfs(self, i, j, current_len): 34 | if current_len < 1: 35 | return [] 36 | 37 | result_paths = [] 38 | for elem in self.tc[i]: 39 | graph_i = i % self.graph_size 40 | graph_j = elem[0] % self.graph_size 41 | 42 | rsa_i = i // self.graph_size 43 | rsa_j = elem[0] // self.graph_size 44 | 45 | left_paths = [] 46 | hasNonterm = False 47 | for nonterm in self.rsa.nonterminals: 48 | if self.rsa[nonterm].get(rsa_i, rsa_j, False): 49 | new_result = self.get_paths(graph_i, graph_j, nonterm, current_len - 1) 50 | left_paths.extend(new_result) 51 | if nonterm in self.rsa.start_and_finish: 52 | left_paths.append(0) 53 | hasNonterm = True 54 | 55 | if not hasNonterm: 56 | left_paths.append(1) 57 | 58 | if len(left_paths) == 0: 59 | continue 60 | 61 | min_len = current_len 62 | for path in left_paths: 63 | if path < min_len: 64 | min_len = path 65 | 66 | right_paths = [] 67 | if elem[0] != j: 68 | right_paths = self.bfs(elem[0], j, current_len - min_len) 69 | else: 70 | right_paths.append(0) 71 | 72 | for left in left_paths: 73 | for right in right_paths: 74 | if left + right < current_len: 75 | result_paths.append(left + right) 76 | 77 | return result_paths 78 | 79 | 80 | class TensorPaths: 81 | def __init__(self, graph: Graph, rsa: RecursiveAutomaton, tc: Matrix): 82 | self.graph = graph 83 | self.rsa = rsa 84 | self.tc = tc 85 | self.graph_size = graph.matrices_size 86 | 87 | def gen_paths(self, i, j, max_len): 88 | first_path = Paths([i], i) 89 | supposed_paths = [first_path] 90 | result_paths = [] 91 | for current_len in range(max_len - 1): 92 | current_size = len(supposed_paths) 93 | for i in range(current_size): 94 | if not supposed_paths[i].use: 95 | continue 96 | 97 | first_iter = True 98 | current_vertex = supposed_paths[i].current_vertex 99 | copy_paths = supposed_paths[i].path.copy() 100 | for new_vertex in self.tc[current_vertex]: 101 | if first_iter: 102 | supposed_paths[i].path.append(new_vertex[0]) 103 | supposed_paths[i].current_vertex = new_vertex[0] 104 | first_iter = False 105 | if new_vertex[0] == j: 106 | result_paths.append(supposed_paths[i].path.copy()) 107 | else: 108 | new_path = copy_paths.copy() 109 | new_path.append(new_vertex[0]) 110 | supposed_paths.append(Paths(new_path, new_vertex[0])) 111 | if new_vertex[0] == j: 112 | result_paths.append(supposed_paths[-1].path.copy()) 113 | 114 | if first_iter: 115 | supposed_paths[i].close() 116 | 117 | return result_paths 118 | 119 | def get_paths(self, start, finish, nonterm, max_len): 120 | if max_len <= 0: 121 | return [] 122 | 123 | supposed_paths = [] 124 | for finish_state in self.rsa.finish_states[nonterm]: 125 | supposed_paths += self.gen_paths(self.rsa.start_state[nonterm] * self.graph_size + start, 126 | finish_state * self.graph_size + finish, max_len) 127 | 128 | result_paths = [] 129 | for path in supposed_paths: 130 | callNonterm = [] 131 | current_size = 0 132 | for i in range(len(path) - 1): 133 | first_rsa = path[i] // self.graph_size 134 | second_rsa = path[i + 1] // self.graph_size 135 | 136 | first_graph = path[i] % self.graph_size 137 | second_graph = path[i + 1] % self.graph_size 138 | 139 | check = False 140 | for label in self.rsa.nonterminals: 141 | if (first_rsa, second_rsa) in self.rsa[label]: 142 | callNonterm.append([first_graph, second_graph, label]) 143 | check = True 144 | 145 | if not check: 146 | current_size += 1 147 | 148 | if len(callNonterm) > 0: 149 | min_size = current_size 150 | construct_paths = [current_size] 151 | stop = False 152 | for call in callNonterm: 153 | sub_paths = self.get_paths(call[0], call[1], call[2], max_len - min_size - len(callNonterm) + 1) 154 | if len(sub_paths) == 0: 155 | stop = True 156 | break 157 | 158 | first_iter = True 159 | new_min = 0 160 | new_construct_paths = [] 161 | for constr_path in construct_paths: 162 | for sub_path in sub_paths: 163 | if constr_path + sub_path < max_len: 164 | if first_iter: 165 | new_min = constr_path + sub_path 166 | else: 167 | if constr_path + sub_path < new_min: 168 | new_min = constr_path + sub_path 169 | new_construct_paths.append(constr_path + sub_path) 170 | 171 | min_size = new_min 172 | construct_paths = new_construct_paths 173 | 174 | if not stop: 175 | result_paths.extend(construct_paths) 176 | 177 | else: 178 | result_paths.append(current_size) 179 | 180 | return result_paths 181 | -------------------------------------------------------------------------------- /src/problems/Base/Base.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from pyformlang.cfg import CFG 3 | from src.graph.graph import Graph 4 | 5 | 6 | class BaseProblem(ABC): 7 | """ 8 | Base class for base problem 9 | """ 10 | 11 | @abstractmethod 12 | def prepare(self, graph: Graph, grammar: CFG): 13 | """ 14 | Prepare for the operation of the algorithm: load graph and grammar 15 | @param graph: path to file with graph 16 | @param grammar: path to file with grammar 17 | """ 18 | pass 19 | 20 | @abstractmethod 21 | def prepare_for_solve(self): 22 | pass 23 | 24 | @abstractmethod 25 | def solve(self): 26 | """ 27 | Solve problem with graph and grammar 28 | """ 29 | pass 30 | -------------------------------------------------------------------------------- /src/problems/Base/algo/matrix_base/matrix_base.py: -------------------------------------------------------------------------------- 1 | from pygraphblas import BOOL 2 | from pyformlang.cfg import CFG 3 | from src.graph.graph import Graph 4 | 5 | from src.problems.Base.Base import BaseProblem 6 | 7 | from src.grammar.cnf_grammar import CnfGrammar 8 | from src.graph.label_graph import LabelGraph 9 | from src.problems.utils import ResultAlgo 10 | 11 | 12 | class MatrixBaseAlgo(BaseProblem): 13 | 14 | def prepare(self, graph: Graph, grammar: CFG): 15 | self.graph = graph 16 | self.graph.load_bool_graph() 17 | self.grammar = CnfGrammar.from_cfg(grammar) 18 | 19 | def solve(self): 20 | m = LabelGraph(self.graph.matrices_size) 21 | 22 | for l in self.grammar.eps_rules: 23 | for i in range(m.matrices_size): 24 | m[l][i, i] = True 25 | 26 | for l, r in self.grammar.simple_rules: 27 | m[l] += self.graph[r] 28 | 29 | changed = True 30 | iter = 0 31 | while changed: 32 | iter += 1 33 | changed = False 34 | for l, r1, r2 in self.grammar.complex_rules: 35 | old_nnz = m[l].nvals 36 | m[l] += m[r1].mxm(m[r2], semiring=BOOL.ANY_PAIR) 37 | new_nnz = m[l].nvals 38 | changed |= not old_nnz == new_nnz 39 | return ResultAlgo(m[self.grammar.start_nonterm], iter) 40 | 41 | def prepare_for_solve(self): 42 | pass 43 | -------------------------------------------------------------------------------- /src/problems/MultipleSource/MultipleSource.py: -------------------------------------------------------------------------------- 1 | from typing import Iterable 2 | from abc import ABC, abstractmethod 3 | from pyformlang.cfg import CFG 4 | from src.graph.graph import Graph 5 | 6 | 7 | class MultipleSourceProblem(ABC): 8 | """ 9 | Base class for multiple-source problem 10 | """ 11 | 12 | @abstractmethod 13 | def prepare(self, graph: Graph, grammar: CFG): 14 | """ 15 | Prepare for the operation of the algorithm: load graph and grammar 16 | @param graph: path to file with graph 17 | @param grammar: path to file with grammar 18 | """ 19 | pass 20 | 21 | @abstractmethod 22 | def clear_src(self): 23 | pass 24 | 25 | @abstractmethod 26 | def solve(self, sources: Iterable): 27 | """ 28 | Solve problem with graph and grammar 29 | """ 30 | pass 31 | -------------------------------------------------------------------------------- /src/problems/MultipleSource/algo/matrix_ms/matrix_ms.py: -------------------------------------------------------------------------------- 1 | from typing import Iterable 2 | from pyformlang.cfg import CFG 3 | from src.graph.graph import Graph 4 | 5 | from pygraphblas import Matrix, descriptor 6 | from pygraphblas.types import BOOL 7 | 8 | from src.grammar.cnf_grammar import CnfGrammar 9 | from src.graph.label_graph import LabelGraph 10 | from src.problems.MultipleSource.MultipleSource import MultipleSourceProblem 11 | from src.problems.utils import ResultAlgo 12 | 13 | 14 | def update_sources(m: Matrix, dst: Matrix): 15 | """ dst += {(j, j) : (i, j) in m} by GrB_reduce src to a vector """ 16 | 17 | # Transpose src and reduce to a vector 18 | J, V = m.T.reduce_vector(BOOL.ANY_MONOID).to_lists() 19 | 20 | # If j-th column of src contains True then add (j, j) to dst 21 | for k in range(len(J)): 22 | if V[k] is True: 23 | dst[J[k], J[k]] = True 24 | 25 | 26 | def update_sources_opt(m: Matrix, mask: Matrix, res: Matrix): 27 | """ res += {(j, j): (i, j) in m and (j, j) not in mask}""" 28 | src_vec = m.reduce_vector(BOOL.ANY_MONOID, desc=descriptor.T0) 29 | for i, _ in src_vec: 30 | if (i, i) not in mask: 31 | res[i, i] = 1 32 | 33 | 34 | def init_simple_rules(rules, graph: Graph): 35 | nonterms = LabelGraph(graph.matrices_size) 36 | for l, r in rules: 37 | nonterms[l] += graph[r] 38 | 39 | return nonterms 40 | 41 | 42 | class MatrixMSBruteAlgo(MultipleSourceProblem): 43 | 44 | def prepare(self, graph: Graph, grammar: CFG): 45 | self.graph = graph 46 | self.graph.load_bool_graph() 47 | self.grammar = CnfGrammar.from_cfg(grammar) 48 | 49 | self.sources = LabelGraph(self.graph.matrices_size) 50 | 51 | # Initialize simple rules 52 | self.__initial_nonterminals = init_simple_rules(self.grammar.simple_rules, self.graph) 53 | 54 | def clear_src(self): 55 | for label in self.sources.matrices: 56 | self.sources[label].clear() 57 | 58 | def solve(self, sources: Iterable): 59 | # Creating new index per solve call 60 | # index = SingleSourceIndex(self.graph, self.grammar) 61 | 62 | nonterminals = self.__initial_nonterminals.clone() 63 | 64 | # Initialize sources and nonterms nnz 65 | # nnz: (l, r1, r2) in complex rules -> (nnz(l), nnz(r1), nnz(r2)) 66 | nnz = {} 67 | for l, r1, r2 in self.grammar.complex_rules: 68 | nnz[(l, r1, r2)] = (0, nonterminals[r1].nvals, nonterminals[r2].nvals) 69 | 70 | # Initialize source matrices masks 71 | m_src = Matrix.sparse(BOOL, self.graph.matrices_size, self.graph.matrices_size) 72 | for v in sources: 73 | m_src[v, v] = True 74 | self.sources[self.grammar.start_nonterm][v, v] = True 75 | 76 | # Create temporary matrix 77 | tmp = Matrix.sparse(BOOL, self.graph.matrices_size, self.graph.matrices_size) 78 | 79 | # Algo's body 80 | iter = 0 81 | changed = True 82 | while changed: 83 | iter += 1 84 | changed = False 85 | 86 | # Number of instances before operation 87 | # old_nnz_nonterms = {nonterm: index.nonterms[nonterm].nvals for nonterm in index.grammar.nonterms} 88 | # old_nnz_sources = {nonterm: index.sources[nonterm].nvals for nonterm in index.grammar.nonterms} 89 | 90 | # Iterate through all complex rules 91 | for l, r1, r2 in self.grammar.complex_rules: 92 | new_nnz = self.sources[l].nvals, nonterminals[r1].nvals, nonterminals[r2].nvals 93 | if nnz[(l, r1, r2)] != new_nnz: 94 | # 1) r1_src += {(j, j) : (i, j) \in l_src} 95 | update_sources(self.sources[l], self.sources[r1]) 96 | 97 | # 2) tmp = l_src * r1 98 | tmp = self.sources[l].mxm(nonterminals[r1], semiring=BOOL.ANY_PAIR) 99 | 100 | # 3) r2_src += {(j, j) : (i, j) \in tmp} 101 | update_sources(tmp, self.sources[r2]) 102 | 103 | # 4) l += tmp * r2 104 | nonterminals[l] += tmp.mxm(nonterminals[r2], semiring=BOOL.ANY_PAIR) 105 | 106 | # update nnz 107 | nnz[(l, r1, r2)] = self.sources[l].nvals, nonterminals[r1].nvals, nonterminals[r2].nvals 108 | changed = True 109 | 110 | return ResultAlgo(m_src.mxm(nonterminals[self.grammar.start_nonterm], semiring=BOOL.ANY_PAIR), iter), \ 111 | nonterminals[self.grammar.start_nonterm] 112 | 113 | 114 | class MatrixMSOptAlgo(MultipleSourceProblem): 115 | def prepare(self, graph: Graph, grammar: CFG): 116 | self.graph = graph 117 | self.graph.load_bool_graph() 118 | self.grammar = CnfGrammar.from_cfg(grammar) 119 | 120 | self.sources = LabelGraph(self.graph.matrices_size) 121 | self.nonterminals = init_simple_rules(self.grammar.simple_rules, self.graph) 122 | 123 | def clear_src(self): 124 | for label in self.sources.matrices: 125 | self.sources[label].clear() 126 | 127 | def solve(self, sources: Iterable): 128 | new_sources = LabelGraph(self.graph.matrices_size) 129 | 130 | # Initialize sources and nonterms nnz 131 | # nnz: (l, r1, r2) in complex rules -> (nnz(new[l]), nnz(index[r1]), nnz(index[r2])) 132 | nnz = {} 133 | for l, r1, r2 in self.grammar.complex_rules: 134 | nnz[(l, r1, r2)] = (0, self.nonterminals[r1].nvals, self.nonterminals[r2].nvals) 135 | 136 | # Initialize source matrices masks 137 | m_src = Matrix.sparse(BOOL, self.graph.matrices_size, self.graph.matrices_size) 138 | for i in sources: 139 | m_src[i, i] = True 140 | if (i, i) not in self.sources[self.grammar.start_nonterm]: 141 | new_sources[self.grammar.start_nonterm][i, i] = True 142 | 143 | # Create temporary matrix 144 | tmp = Matrix.sparse(BOOL, self.graph.matrices_size, self.graph.matrices_size) 145 | 146 | # Algo's body 147 | changed = True 148 | iter = 0 149 | while changed: 150 | iter += 1 151 | changed = False 152 | 153 | # Iterate through all complex rules 154 | for l, r1, r2 in self.grammar.complex_rules: 155 | # l -> r1 r2 ==> index[l] += (new[l_src] * index[r1]) * index[r2] 156 | 157 | new_nnz = new_sources[l].nvals, self.nonterminals[r1].nvals, self.nonterminals[r2].nvals 158 | if nnz[(l, r1, r2)] != new_nnz: 159 | # 1) new[r1_src] += {(j, j) : (j, j) in new[l_src] and not in index[r1_src]} 160 | for i, _, _ in new_sources[l]: 161 | if (i, i) not in self.sources[r1]: 162 | new_sources[r1][i, i] = True 163 | 164 | # 2) tmp = new[l_src] * index[r1] 165 | new_sources[l].mxm(self.nonterminals[r1], out=tmp, semiring=BOOL.ANY_PAIR) 166 | 167 | # 3) new[r2_src] += {(j, j) : (i, j) in tmp and not in index[r2_src]} 168 | update_sources_opt(tmp, self.sources[r2], new_sources[r2]) 169 | 170 | # 4) index[l] += tmp * index[r2] 171 | self.nonterminals[l] += tmp.mxm(self.nonterminals[r2], semiring=BOOL.ANY_PAIR) 172 | 173 | # update nnz 174 | nnz[(l, r1, r2)] = new_sources[l].nvals, self.nonterminals[r1].nvals, self.nonterminals[r2].nvals 175 | changed = True 176 | for n in self.grammar.nonterms: 177 | self.sources[n] += new_sources[n] 178 | return ResultAlgo(m_src.mxm(self.nonterminals[self.grammar.start_nonterm], semiring=BOOL.ANY_PAIR), iter), \ 179 | self.nonterminals[self.grammar.start_nonterm] 180 | -------------------------------------------------------------------------------- /src/problems/MultipleSource/algo/tensor_ms/tensor_ms.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from cfpq_data import RSM 3 | from pyformlang.cfg import CFG 4 | from src.graph.graph import Graph 5 | from typing import Iterable, Union 6 | 7 | from pygraphblas import Matrix, BOOL 8 | 9 | from src.problems.MultipleSource.MultipleSource import MultipleSourceProblem 10 | 11 | from src.grammar.rsa import RecursiveAutomaton 12 | from src.graph.label_graph import LabelGraph 13 | 14 | from src.problems.AllPaths.algo.tensor.tensor import restore_eps_paths, transitive_closure 15 | from src.problems.utils import ResultAlgo 16 | 17 | 18 | class TensorMSAlgo(MultipleSourceProblem): 19 | 20 | def prepare(self, graph: Graph, grammar: Union[RSM, CFG, Path]): 21 | self.graph = graph 22 | self.graph.load_bool_graph() 23 | self.grammar = RecursiveAutomaton.from_grammar_or_path(grammar) 24 | self.part_graph = LabelGraph(self.graph.matrices_size) 25 | self.src_for_states = dict() 26 | for i in range(self.grammar.matrices_size): 27 | self.src_for_states.update({i: Matrix.sparse(BOOL, self.graph.matrices_size, self.graph.matrices_size)}) 28 | 29 | def clear_src(self): 30 | for label in self.src_for_states: 31 | self.src_for_states[label].clear() 32 | 33 | for nonterm in self.grammar.nonterminals: 34 | self.graph[nonterm].clear() 35 | 36 | def solve(self, sources: Iterable): 37 | restore_eps_paths(self.grammar.start_and_finish, self.graph) 38 | 39 | # Initialize source matrices masks 40 | m_src = Matrix.sparse(BOOL, self.graph.matrices_size, self.graph.matrices_size) 41 | for v in sources: 42 | m_src[v, v] = True 43 | self.src_for_states[self.grammar.start_state[self.grammar.start_nonterm]][v, v] = True 44 | 45 | sizeKron = self.graph.matrices_size * self.grammar.matrices_size 46 | 47 | kron = Matrix.sparse(BOOL, sizeKron, sizeKron) 48 | 49 | changed = True 50 | src_changed = True 51 | iter = 0 52 | while changed or src_changed: 53 | iter += 1 54 | changed = False 55 | src_changed = False 56 | 57 | for box in self.grammar.boxes: 58 | for state in self.grammar.boxes[box]: 59 | out_state = self.grammar.out_states.get(state, []) 60 | for out in out_state: 61 | if out[1] in self.grammar.nonterminals: 62 | old_sum = self.src_for_states[self.grammar.start_state[out[1]]].nvals 63 | self.src_for_states[self.grammar.start_state[out[1]]] += self.src_for_states[state] 64 | if old_sum != self.src_for_states[self.grammar.start_state[out[1]]].nvals: 65 | src_changed = True 66 | with BOOL.ANY_PAIR: 67 | self.part_graph[out[1]] += self.src_for_states[state].mxm(self.graph[out[1]]) 68 | old_sum = self.src_for_states[out[0]].nvals 69 | for elem in self.part_graph[out[1]].T.reduce_vector(BOOL.ANY_MONOID): 70 | self.src_for_states[out[0]][elem[0], elem[0]] = True 71 | if old_sum != self.src_for_states[out[0]].nvals: 72 | src_changed = True 73 | 74 | for label in self.grammar.labels: 75 | kron += self.grammar[label].kronecker(self.part_graph[label]) 76 | 77 | transitive_closure(kron) 78 | 79 | for start in self.grammar.nonterminals: 80 | for element in self.grammar.states[start]: 81 | i = element[0] 82 | j = element[1] 83 | 84 | start_i = i * self.graph.matrices_size 85 | start_j = j * self.graph.matrices_size 86 | 87 | control_sum = self.graph[start].nvals 88 | block = kron[start_i:start_i + self.graph.matrices_size - 1, 89 | start_j: start_j + self.graph.matrices_size - 1] 90 | 91 | self.graph[start] += block 92 | new_control_sum = self.graph[start].nvals 93 | 94 | if new_control_sum != control_sum: 95 | changed = True 96 | 97 | return ResultAlgo(m_src.mxm(self.graph[self.grammar.start_nonterm], semiring=BOOL.ANY_PAIR), iter), \ 98 | self.graph[self.grammar.start_nonterm] 99 | 100 | 101 | class TensorMSAllAlgo(MultipleSourceProblem): 102 | 103 | def prepare(self, graph: Graph, grammar: Union[RSM, CFG, Path]): 104 | self.graph = graph 105 | self.graph.load_bool_graph() 106 | self.grammar = RecursiveAutomaton.from_grammar_or_path(grammar) 107 | self.part_graph = LabelGraph(self.graph.matrices_size) 108 | self.src_for_states = dict() 109 | for i in range(self.grammar.matrices_size): 110 | self.src_for_states.update({i: Matrix.sparse(BOOL, self.graph.matrices_size, self.graph.matrices_size)}) 111 | 112 | def clear_src(self): 113 | for label in self.src_for_states: 114 | self.src_for_states[label].clear() 115 | 116 | for nonterm in self.grammar.nonterminals: 117 | self.graph[nonterm].clear() 118 | 119 | def solve(self, sources: Iterable): 120 | restore_eps_paths(self.grammar.start_and_finish, self.graph) 121 | 122 | for v in sources: 123 | self.src_for_states[self.grammar.start_state[self.grammar.start_nonterm]][v, v] = True 124 | 125 | sizeKron = self.graph.matrices_size * self.grammar.matrices_size 126 | 127 | kron = Matrix.sparse(BOOL, sizeKron, sizeKron) 128 | 129 | changed = True 130 | iter = 0 131 | while changed: 132 | iter += 1 133 | changed = False 134 | src_changed = True 135 | while src_changed: 136 | src_changed = False 137 | for box in self.grammar.boxes: 138 | for state in self.grammar.boxes[box]: 139 | out_state = self.grammar.out_states.get(state, []) 140 | for out in out_state: 141 | if out[1] in self.grammar.nonterminals: 142 | old_sum = self.src_for_states[self.grammar.start_state[out[1]]].nvals 143 | self.src_for_states[self.grammar.start_state[out[1]]] += self.src_for_states[state] 144 | if old_sum != self.src_for_states[self.grammar.start_state[out[1]]].nvals: 145 | src_changed = True 146 | with BOOL.ANY_PAIR: 147 | self.part_graph[out[1]] += self.src_for_states[state].mxm(self.graph[out[1]]) 148 | old_sum = self.src_for_states[out[0]].nvals 149 | for elem in self.part_graph[out[1]].T.reduce_vector(BOOL.ANY_MONOID): 150 | self.src_for_states[out[0]][elem[0], elem[0]] = True 151 | if old_sum != self.src_for_states[out[0]].nvals: 152 | src_changed = True 153 | 154 | for label in self.grammar.labels: 155 | kron += self.grammar[label].kronecker(self.part_graph[label]) 156 | 157 | transitive_closure(kron) 158 | 159 | for start in self.grammar.nonterminals: 160 | for element in self.grammar.states[start]: 161 | i = element[0] 162 | j = element[1] 163 | 164 | start_i = i * self.graph.matrices_size 165 | start_j = j * self.graph.matrices_size 166 | 167 | control_sum = self.graph[start].nvals 168 | block = kron[start_i:start_i + self.graph.matrices_size - 1, 169 | start_j: start_j + self.graph.matrices_size - 1] 170 | 171 | self.graph[start] += block 172 | new_control_sum = self.graph[start].nvals 173 | 174 | if new_control_sum != control_sum: 175 | changed = True 176 | 177 | return ResultAlgo(self.graph[self.grammar.start_nonterm], iter) -------------------------------------------------------------------------------- /src/problems/SinglePath/SinglePath.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from pyformlang.cfg import CFG 3 | from src.graph.graph import Graph 4 | 5 | 6 | class SinglePathProblem(ABC): 7 | """ 8 | Base class for single path problem 9 | """ 10 | 11 | @abstractmethod 12 | def prepare(self, graph: Graph, grammar: CFG): 13 | """ 14 | Prepare for the operation of the algorithm: load graph and grammar 15 | @param graph: path to file with graph 16 | @param grammar: path to file with grammar 17 | """ 18 | pass 19 | 20 | @abstractmethod 21 | def solve(self): 22 | """ 23 | Solve problem with graph and grammar 24 | """ 25 | pass 26 | 27 | @abstractmethod 28 | def prepare_for_solve(self): 29 | pass 30 | 31 | @abstractmethod 32 | def getPath(self, v_start: int, v_finish: int, nonterminal: str): 33 | """ 34 | Extract one path between two vertices 35 | @param v_start: starting vertex for path 36 | @param v_finish: finishing vertex for path 37 | @param nonterminal: nonterminal from which path are being restored 38 | """ 39 | pass 40 | -------------------------------------------------------------------------------- /src/problems/SinglePath/algo/matrix_shortest_path/matrix_shortest_path.py: -------------------------------------------------------------------------------- 1 | from src.grammar.cnf_grammar import CnfGrammar 2 | from src.graph.length_graph import LengthGraph 3 | from src.graph.length_graph import MAX_MATRIX_SIZE 4 | 5 | 6 | class MatrixShortestPath: 7 | def __init__(self, graph: LengthGraph, grammar: CnfGrammar): 8 | self.length = 0 9 | self.m = graph 10 | self.grammar = grammar 11 | 12 | def get_path(self, i, j, s): 13 | def is_identity(lft, r, mid, lng): 14 | return lft == 0 and r == 0 and mid == 0 and lng == MAX_MATRIX_SIZE 15 | 16 | left, right, middle, length = self.m[s].get(i, j) 17 | if is_identity(left, right, middle, length): 18 | print("Index isn`t correct\n") 19 | 20 | if length == 1: 21 | self.length += 1 # edges can be stored here 22 | 23 | for l, r1, r2 in self.grammar.complex_rules: 24 | if s == l: 25 | left_r1, right_r1, middle_r1, length_r1 = 0, 0, 0, MAX_MATRIX_SIZE 26 | if (left, middle) in self.m[r1]: 27 | left_r1, right_r1, middle_r1, length_r1 = self.m[r1].get(left, middle) 28 | left_r2, right_r2, middle_r2, length_r2 = 0, 0, 0, MAX_MATRIX_SIZE 29 | if (middle, right) in self.m[r2]: 30 | left_r2, right_r2, middle_r2, length_r2 = self.m[r2].get(middle, right) 31 | 32 | if not is_identity(left_r1, right_r1, middle_r1, length_r1) and \ 33 | not is_identity(left_r2, right_r2, middle_r2, length_r2): 34 | if length == length_r1 + length_r2: 35 | self.get_path(left, middle, r1) 36 | self.get_path(middle, right, r2) 37 | break 38 | 39 | return self.length 40 | -------------------------------------------------------------------------------- /src/problems/SinglePath/algo/matrix_shortest_path/matrix_shortest_path_index.py: -------------------------------------------------------------------------------- 1 | from pyformlang.cfg import CFG 2 | from src.graph.graph import Graph 3 | 4 | from pygraphblas import Matrix 5 | 6 | from src.problems.SinglePath.SinglePath import SinglePathProblem 7 | from src.problems.SinglePath.algo.matrix_shortest_path.matrix_shortest_path import MatrixShortestPath 8 | 9 | from src.graph.length_graph import LengthGraph, SAVELENGTHTYPE 10 | from src.grammar.cnf_grammar import CnfGrammar 11 | from src.problems.utils import ResultAlgo 12 | 13 | 14 | class MatrixShortestAlgo(SinglePathProblem): 15 | 16 | def prepare(self, graph: Graph, grammar: CFG): 17 | self.graph = graph 18 | self.graph.load_save_length_graph() 19 | self.grammar = CnfGrammar.from_cfg(grammar) 20 | 21 | def solve(self): 22 | IndexType_monoid = SAVELENGTHTYPE.new_monoid(SAVELENGTHTYPE.PLUS, SAVELENGTHTYPE.one) 23 | IndexType_semiring = SAVELENGTHTYPE.new_semiring(IndexType_monoid, SAVELENGTHTYPE.TIMES) 24 | with IndexType_semiring, SAVELENGTHTYPE.PLUS: 25 | m = LengthGraph(self.graph.matrices_size) 26 | for l, r in self.grammar.simple_rules: 27 | m[l] += self.graph[r] 28 | 29 | iter = 0 30 | changed_nonterms = self.grammar.nonterms 31 | while len(changed_nonterms) > 0: 32 | iter += 1 33 | new_changes = set() 34 | for l, r1, r2 in self.grammar.complex_rules: 35 | if r1 in changed_nonterms or r2 in changed_nonterms: 36 | old_m = m[l].dup() 37 | old_nnz = m[l].nvals 38 | m[l] += m[r1].mxm(m[r2]) 39 | new_nnz = m[l].nvals 40 | if not old_nnz == new_nnz: 41 | new_changes.add(l) 42 | elif new_nnz > 0: 43 | C = m[l].emult(old_m, SAVELENGTHTYPE.SUBTRACTION) 44 | if C.nonzero().nvals > 0: 45 | new_changes.add(l) 46 | changed_nonterms = new_changes 47 | self.res_m = m 48 | return ResultAlgo(m[self.grammar.start_nonterm], iter) 49 | 50 | def prepare_for_solve(self): 51 | pass 52 | 53 | def getPath(self, v_start: int, v_finish: int, nonterminal: str): 54 | return MatrixShortestPath(self.res_m, self.grammar).get_path(v_start, v_finish, nonterminal) -------------------------------------------------------------------------------- /src/problems/SinglePath/algo/matrix_single_path/matrix_single_path.py: -------------------------------------------------------------------------------- 1 | from src.grammar.cnf_grammar import CnfGrammar 2 | from src.graph.index_graph import IndexGraph 3 | 4 | 5 | class MatrixSinglePath: 6 | def __init__(self, graph: IndexGraph, grammar: CnfGrammar): 7 | self.length = 0 8 | self.m = graph 9 | self.grammar = grammar 10 | 11 | def get_path(self, i, j, s): 12 | def is_identity(lft, r, mid, h, lng): 13 | return lft == 0 and r == 0 and mid == 0 and h == 0 and lng == 0 14 | 15 | left, right, middle, height, length = self.m[s].get(i, j) 16 | if is_identity(left, right, middle, height, length): 17 | print("Index isn`t correct\n") 18 | 19 | if height == 1: 20 | self.length += 1 21 | 22 | for l, r1, r2 in self.grammar.complex_rules: 23 | if s == l: 24 | left_r1, right_r1, middle_r1, height_r1, length_r1 = 0, 0, 0, 0, 0 25 | if (left, middle) in self.m[r1]: 26 | left_r1, right_r1, middle_r1, height_r1, length_r1 = self.m[r1].get(left, middle) 27 | left_r2, right_r2, middle_r2, height_r2, length_r2 = 0, 0, 0, 0, 0 28 | if (middle, right) in self.m[r2]: 29 | left_r2, right_r2, middle_r2, height_r2, length_r2 = self.m[r2].get(middle, right) 30 | 31 | if not is_identity(left_r1, right_r1, middle_r1, height_r1, length_r1) and \ 32 | not is_identity(left_r2, right_r2, middle_r2, height_r2, length_r2): 33 | max_height = height_r2 if height_r1 < height_r2 else height_r1 34 | if height == max_height + 1: 35 | self.get_path(left, middle, r1) 36 | self.get_path(middle, right, r2) 37 | break 38 | 39 | return self.length 40 | -------------------------------------------------------------------------------- /src/problems/SinglePath/algo/matrix_single_path/matrix_single_path_index.py: -------------------------------------------------------------------------------- 1 | from pyformlang.cfg import CFG 2 | from src.graph.graph import Graph 3 | 4 | from src.problems.SinglePath.SinglePath import SinglePathProblem 5 | from src.problems.SinglePath.algo.matrix_single_path.matrix_single_path import MatrixSinglePath 6 | 7 | from src.graph.index_graph import IndexGraph, SAVEMIDDLETYPE 8 | from src.grammar.cnf_grammar import CnfGrammar 9 | from src.problems.utils import ResultAlgo 10 | 11 | 12 | class MatrixSingleAlgo(SinglePathProblem): 13 | 14 | def prepare(self, graph: Graph, grammar: CFG): 15 | self.graph = graph 16 | self.graph.load_save_middle_graph() 17 | self.grammar = CnfGrammar.from_cfg(grammar) 18 | 19 | def solve(self): 20 | IndexType_monoid = SAVEMIDDLETYPE.new_monoid(SAVEMIDDLETYPE.PLUS, SAVEMIDDLETYPE.one) 21 | IndexType_semiring = SAVEMIDDLETYPE.new_semiring(IndexType_monoid, SAVEMIDDLETYPE.TIMES) 22 | with IndexType_semiring, SAVEMIDDLETYPE.PLUS: 23 | m = IndexGraph(self.graph.matrices_size) 24 | for l, r in self.grammar.simple_rules: 25 | m[l] += self.graph[r] 26 | 27 | changed = True 28 | iter = 0 29 | while changed: 30 | iter += 1 31 | changed = False 32 | for l, r1, r2 in self.grammar.complex_rules: 33 | old_nnz = m[l].nvals 34 | m[l] += m[r1].mxm(m[r2]) 35 | new_nnz = m[l].nvals 36 | if not old_nnz == new_nnz: 37 | changed = True 38 | self.res_m = m 39 | return ResultAlgo(m[self.grammar.start_nonterm], iter) 40 | 41 | def prepare_for_solve(self): 42 | pass 43 | 44 | def getPath(self, v_start: int, v_finish: int, nonterminal: str): 45 | return MatrixSinglePath(self.res_m, self.grammar).get_path(v_start, v_finish, nonterminal) -------------------------------------------------------------------------------- /src/problems/utils.py: -------------------------------------------------------------------------------- 1 | from pygraphblas import Matrix 2 | 3 | 4 | class ResultAlgo: 5 | def __init__(self, matrix_S: Matrix, iter: int): 6 | self.matrix_S = matrix_S 7 | self.number_iter = iter 8 | -------------------------------------------------------------------------------- /src/utils/common.py: -------------------------------------------------------------------------------- 1 | def chunkify(xs, chunk_size): 2 | for i in range(0, len(xs), chunk_size): 3 | yield xs[i:i+chunk_size] 4 | -------------------------------------------------------------------------------- /src/utils/file_helpers.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import subprocess as sp 3 | 4 | 5 | def get_file_name(path): 6 | return Path(path).stem 7 | 8 | 9 | def get_file_size(path): 10 | r = sp.run(f'wc -l {path}', capture_output=True, shell=True) 11 | return int(r.stdout.split()[0].decode('utf-8')) 12 | -------------------------------------------------------------------------------- /src/utils/graph_size.py: -------------------------------------------------------------------------------- 1 | def get_graph_size(path): 2 | res = -1 3 | with open(path, 'r') as f: 4 | for line in f.readlines(): 5 | v, label, to = line.split() 6 | v, to = int(v), int(to) 7 | res = max(res, v, to) 8 | return res + 1 9 | -------------------------------------------------------------------------------- /src/utils/time_profiler.py: -------------------------------------------------------------------------------- 1 | from time import time 2 | 3 | 4 | class SimpleTimer: 5 | def __enter__(self): 6 | self.start_time = time() 7 | 8 | def __exit__(self, exc_type, exc_val, exc_tb): 9 | print(f'{self.msg}: {time() - self.start_time}s') 10 | 11 | def __init__(self, msg=''): 12 | self.start_time = None 13 | self.end_time = None 14 | self.duration = None 15 | self.msg = msg 16 | 17 | def tic(self): 18 | self.start_time = time() 19 | 20 | def toc(self): 21 | self.end_time = time() 22 | self.duration = self.end_time - self.start_time 23 | return self.duration 24 | -------------------------------------------------------------------------------- /src/utils/useful_paths.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | CFPQ_PYALGO_ROOT = Path(__file__).parent.parent.parent 4 | 5 | GLOBAL_CFPQ_DATA = Path(CFPQ_PYALGO_ROOT).joinpath('deps/CFPQ_Data/data') 6 | LOCAL_CFPQ_DATA = Path(CFPQ_PYALGO_ROOT).joinpath('test/data') 7 | -------------------------------------------------------------------------------- /test/AllPaths/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from src.problems.AllPaths.algo.tensor.tensor import TensorSimpleAlgo, TensorDynamicAlgo 4 | 5 | 6 | @pytest.fixture(params=[TensorSimpleAlgo, TensorDynamicAlgo]) 7 | def algo(request): 8 | return request.param 9 | -------------------------------------------------------------------------------- /test/AllPaths/test_allpaths.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from cfpq_data import cfg_from_txt 3 | from src.graph.graph import Graph 4 | 5 | from src.problems.AllPaths.AllPaths import AllPathsProblem 6 | 7 | from src.utils.useful_paths import LOCAL_CFPQ_DATA 8 | from src.problems.utils import ResultAlgo 9 | 10 | 11 | @pytest.mark.CI 12 | def test_binary_tree(algo): 13 | test_data_path = LOCAL_CFPQ_DATA.joinpath('binary_tree') 14 | allpath_algo: AllPathsProblem = algo() 15 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 16 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 17 | allpath_algo.prepare(graph, grammar) 18 | 19 | result: ResultAlgo = allpath_algo.solve() 20 | assert result.matrix_S.nvals == 20 21 | 22 | allpath_algo.prepare_for_exctract_paths() 23 | paths = allpath_algo.getPaths(0, 3, "S", 5) 24 | assert len(paths) == 1 25 | 26 | 27 | @pytest.mark.CI 28 | def test_cycle(algo): 29 | test_data_path = LOCAL_CFPQ_DATA.joinpath('cycle') 30 | allpath_algo: AllPathsProblem = algo() 31 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 32 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 33 | allpath_algo.prepare(graph, grammar) 34 | 35 | result: ResultAlgo = allpath_algo.solve() 36 | assert result.matrix_S.nvals == 9 37 | 38 | allpath_algo.prepare_for_exctract_paths() 39 | paths = allpath_algo.getPaths(0, 1, "S", 3) 40 | assert len(paths) == 1 41 | 42 | 43 | @pytest.mark.CI 44 | def test_line(algo): 45 | test_data_path = LOCAL_CFPQ_DATA.joinpath('line') 46 | allpath_algo: AllPathsProblem = algo() 47 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 48 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 49 | allpath_algo.prepare(graph, grammar) 50 | 51 | result: ResultAlgo = allpath_algo.solve() 52 | assert result.matrix_S.nvals == 2 53 | 54 | allpath_algo.prepare_for_exctract_paths() 55 | paths = allpath_algo.getPaths(0, 4, "S", 2) 56 | assert len(paths) == 0 57 | 58 | 59 | @pytest.mark.CI 60 | def test_loop(algo): 61 | test_data_path = LOCAL_CFPQ_DATA.joinpath('loop') 62 | allpath_algo: AllPathsProblem = algo() 63 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 64 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 65 | allpath_algo.prepare(graph, grammar) 66 | 67 | result: ResultAlgo = allpath_algo.solve() 68 | assert result.matrix_S.nvals == 1 69 | 70 | allpath_algo.prepare_for_exctract_paths() 71 | paths = allpath_algo.getPaths(0, 0, "S", 1) 72 | assert len(paths) == 0 73 | 74 | 75 | @pytest.mark.CI 76 | def test_two_cycles(algo): 77 | test_data_path = LOCAL_CFPQ_DATA.joinpath('two_cycles') 78 | allpath_algo: AllPathsProblem = algo() 79 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 80 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 81 | allpath_algo.prepare(graph, grammar) 82 | 83 | result: ResultAlgo = allpath_algo.solve() 84 | assert result.matrix_S.nvals == 6 85 | 86 | allpath_algo.prepare_for_exctract_paths() 87 | paths = allpath_algo.getPaths(1, 3, "S", 3) 88 | assert len(paths) == 1 89 | 90 | 91 | @pytest.mark.CI 92 | def test_two_nonterm(algo): 93 | test_data_path = LOCAL_CFPQ_DATA.joinpath('two_nonterm') 94 | allpath_algo: AllPathsProblem = algo() 95 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 96 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 97 | allpath_algo.prepare(graph, grammar) 98 | 99 | result: ResultAlgo = allpath_algo.solve() 100 | assert result.matrix_S.nvals == 156 101 | 102 | allpath_algo.prepare_for_exctract_paths() 103 | paths = allpath_algo.getPaths(1, 1, "S", 3) 104 | assert len(paths) == 1 105 | -------------------------------------------------------------------------------- /test/Base/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from src.problems.Base.algo.matrix_base.matrix_base import MatrixBaseAlgo 4 | 5 | 6 | @pytest.fixture(params=[MatrixBaseAlgo]) 7 | def algo(request): 8 | return request.param 9 | -------------------------------------------------------------------------------- /test/Base/test_base.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from cfpq_data import cfg_from_txt 3 | from src.graph.graph import Graph 4 | 5 | from src.problems.Base.Base import BaseProblem 6 | 7 | from src.utils.useful_paths import LOCAL_CFPQ_DATA 8 | from src.problems.utils import ResultAlgo 9 | 10 | 11 | @pytest.mark.CI 12 | def test_binary_tree(algo): 13 | test_data_path = LOCAL_CFPQ_DATA.joinpath('binary_tree') 14 | base_algo: BaseProblem = algo() 15 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 16 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 17 | base_algo.prepare(graph, grammar) 18 | 19 | result: ResultAlgo = base_algo.solve() 20 | assert result.matrix_S.nvals == 20 21 | 22 | 23 | @pytest.mark.CI 24 | def test_cycle(algo): 25 | test_data_path = LOCAL_CFPQ_DATA.joinpath('cycle') 26 | base_algo: BaseProblem = algo() 27 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 28 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 29 | base_algo.prepare(graph, grammar) 30 | 31 | result: ResultAlgo = base_algo.solve() 32 | assert result.matrix_S.nvals == 9 33 | 34 | 35 | @pytest.mark.CI 36 | def test_line(algo): 37 | test_data_path = LOCAL_CFPQ_DATA.joinpath('line') 38 | base_algo: BaseProblem = algo() 39 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 40 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 41 | base_algo.prepare(graph, grammar) 42 | 43 | result: ResultAlgo = base_algo.solve() 44 | assert result.matrix_S.nvals == 2 45 | 46 | 47 | @pytest.mark.CI 48 | def test_loop(algo): 49 | test_data_path = LOCAL_CFPQ_DATA.joinpath('loop') 50 | base_algo: BaseProblem = algo() 51 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 52 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 53 | base_algo.prepare(graph, grammar) 54 | 55 | result: ResultAlgo = base_algo.solve() 56 | assert result.matrix_S.nvals == 1 57 | 58 | 59 | @pytest.mark.CI 60 | def test_two_cycles(algo): 61 | test_data_path = LOCAL_CFPQ_DATA.joinpath('two_cycles') 62 | base_algo: BaseProblem = algo() 63 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 64 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 65 | base_algo.prepare(graph, grammar) 66 | 67 | result: ResultAlgo = base_algo.solve() 68 | assert result.matrix_S.nvals == 6 69 | 70 | 71 | @pytest.mark.CI 72 | def test_two_nonterm(algo): 73 | test_data_path = LOCAL_CFPQ_DATA.joinpath('two_nonterm') 74 | base_algo: BaseProblem = algo() 75 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 76 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 77 | base_algo.prepare(graph, grammar) 78 | 79 | result: ResultAlgo = base_algo.solve() 80 | assert result.matrix_S.nvals == 156 81 | -------------------------------------------------------------------------------- /test/MultipleSource/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from src.problems.MultipleSource.algo.matrix_ms.matrix_ms import MatrixMSOptAlgo, MatrixMSBruteAlgo 4 | from src.problems.MultipleSource.algo.tensor_ms.tensor_ms import TensorMSAlgo 5 | 6 | 7 | @pytest.fixture(params=[MatrixMSOptAlgo, MatrixMSBruteAlgo, TensorMSAlgo]) 8 | def algo(request): 9 | return request.param 10 | -------------------------------------------------------------------------------- /test/MultipleSource/test_ms.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from cfpq_data import cfg_from_txt 3 | from src.graph.graph import Graph 4 | 5 | from src.problems.MultipleSource.MultipleSource import MultipleSourceProblem 6 | 7 | from src.utils.useful_paths import LOCAL_CFPQ_DATA 8 | from src.problems.utils import ResultAlgo 9 | 10 | 11 | @pytest.mark.CI 12 | def test_binary_tree(algo): 13 | test_data_path = LOCAL_CFPQ_DATA.joinpath('binary_tree') 14 | ms_algo: MultipleSourceProblem = algo() 15 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 16 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 17 | ms_algo.prepare(graph, grammar) 18 | 19 | result: ResultAlgo 20 | result, with_cache = ms_algo.solve([0]) 21 | assert result.matrix_S.nvals == 4 and with_cache.nvals == 6 22 | 23 | 24 | @pytest.mark.CI 25 | def test_line(algo): 26 | test_data_path = LOCAL_CFPQ_DATA.joinpath('line') 27 | ms_algo: MultipleSourceProblem = algo() 28 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 29 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 30 | ms_algo.prepare(graph, grammar) 31 | 32 | result: ResultAlgo 33 | result, with_cache = ms_algo.solve([1]) 34 | assert result.matrix_S.nvals == 1 and with_cache.nvals == 1 35 | 36 | 37 | @pytest.mark.CI 38 | def test_two_nonterm(algo): 39 | test_data_path = LOCAL_CFPQ_DATA.joinpath('two_nonterm') 40 | ms_algo: MultipleSourceProblem = algo() 41 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 42 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 43 | ms_algo.prepare(graph, grammar) 44 | 45 | result: ResultAlgo 46 | result, with_cache = ms_algo.solve([1]) 47 | assert result.matrix_S.nvals == 1 and with_cache.nvals == 1 48 | -------------------------------------------------------------------------------- /test/ShortestPath/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from src.problems.SinglePath.algo.matrix_shortest_path.matrix_shortest_path_index import MatrixShortestAlgo 4 | 5 | 6 | @pytest.fixture(params=[MatrixShortestAlgo]) 7 | def algo(request): 8 | return request.param -------------------------------------------------------------------------------- /test/ShortestPath/test_shortestpath.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from cfpq_data import cfg_from_txt 3 | from src.graph.graph import Graph 4 | 5 | from src.problems.SinglePath.SinglePath import SinglePathProblem 6 | 7 | from src.utils.useful_paths import LOCAL_CFPQ_DATA 8 | from src.problems.utils import ResultAlgo 9 | 10 | @pytest.mark.CI 11 | def test_binary_tree(algo): 12 | test_data_path = LOCAL_CFPQ_DATA.joinpath('single_vs_shortest') 13 | shortestpath_algo: SinglePathProblem = algo() 14 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 15 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 16 | shortestpath_algo.prepare(graph, grammar) 17 | 18 | result: ResultAlgo = shortestpath_algo.solve() 19 | assert result.matrix_S.nvals == 2 20 | 21 | length = shortestpath_algo.getPath(0, 7, "S") 22 | assert length == 6 23 | 24 | @pytest.mark.CI 25 | def test_binary_tree(algo): 26 | test_data_path = LOCAL_CFPQ_DATA.joinpath('binary_tree') 27 | shortestpath_algo: SinglePathProblem = algo() 28 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 29 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 30 | shortestpath_algo.prepare(graph, grammar) 31 | 32 | result: ResultAlgo = shortestpath_algo.solve() 33 | assert result.matrix_S.nvals == 20 34 | -------------------------------------------------------------------------------- /test/SinglePath/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from src.problems.SinglePath.algo.matrix_single_path.matrix_single_path_index import MatrixSingleAlgo 4 | 5 | 6 | @pytest.fixture(params=[MatrixSingleAlgo]) 7 | def algo(request): 8 | return request.param 9 | -------------------------------------------------------------------------------- /test/SinglePath/test_singlepath.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from cfpq_data import cfg_from_txt 3 | from src.graph.graph import Graph 4 | 5 | from src.problems.SinglePath.SinglePath import SinglePathProblem 6 | 7 | from src.utils.useful_paths import LOCAL_CFPQ_DATA 8 | from src.problems.utils import ResultAlgo 9 | 10 | 11 | @pytest.mark.CI 12 | def test_binary_tree(algo): 13 | test_data_path = LOCAL_CFPQ_DATA.joinpath('binary_tree') 14 | singlepath_algo: SinglePathProblem = algo() 15 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 16 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 17 | singlepath_algo.prepare(graph, grammar) 18 | 19 | result: ResultAlgo = singlepath_algo.solve() 20 | assert result.matrix_S.nvals == 20 21 | 22 | paths = singlepath_algo.getPath(0, 3, "S") 23 | assert paths == 4 24 | 25 | 26 | @pytest.mark.CI 27 | def test_cycle(algo): 28 | test_data_path = LOCAL_CFPQ_DATA.joinpath('cycle') 29 | singlepath_algo: SinglePathProblem = algo() 30 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 31 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 32 | singlepath_algo.prepare(graph, grammar) 33 | 34 | result: ResultAlgo = singlepath_algo.solve() 35 | assert result.matrix_S.nvals == 9 36 | 37 | paths = singlepath_algo.getPath(0, 1, "S") 38 | assert paths == 1 39 | 40 | 41 | @pytest.mark.CI 42 | def test_line(algo): 43 | test_data_path = LOCAL_CFPQ_DATA.joinpath('line') 44 | singlepath_algo: SinglePathProblem = algo() 45 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 46 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 47 | singlepath_algo.prepare(graph, grammar) 48 | 49 | result: ResultAlgo = singlepath_algo.solve() 50 | assert result.matrix_S.nvals == 2 51 | 52 | paths = singlepath_algo.getPath(0, 4, "S") 53 | assert paths == 4 54 | 55 | 56 | @pytest.mark.CI 57 | def test_loop(algo): 58 | test_data_path = LOCAL_CFPQ_DATA.joinpath('loop') 59 | singlepath_algo: SinglePathProblem = algo() 60 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 61 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 62 | singlepath_algo.prepare(graph, grammar) 63 | 64 | result: ResultAlgo = singlepath_algo.solve() 65 | assert result.matrix_S.nvals == 1 66 | 67 | paths = singlepath_algo.getPath(0, 0, "S") 68 | assert paths == 1 69 | 70 | 71 | @pytest.mark.CI 72 | def test_two_cycles(algo): 73 | test_data_path = LOCAL_CFPQ_DATA.joinpath('two_cycles') 74 | singlepath_algo: SinglePathProblem = algo() 75 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 76 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 77 | singlepath_algo.prepare(graph, grammar) 78 | 79 | result: ResultAlgo = singlepath_algo.solve() 80 | assert result.matrix_S.nvals == 6 81 | 82 | paths = singlepath_algo.getPath(1, 3, "S") 83 | assert paths == 2 84 | 85 | 86 | @pytest.mark.CI 87 | def test_two_nonterm(algo): 88 | test_data_path = LOCAL_CFPQ_DATA.joinpath('two_nonterm') 89 | singlepath_algo: SinglePathProblem = algo() 90 | graph = Graph.from_txt(test_data_path.joinpath('Graphs/graph_1.txt')) 91 | grammar = cfg_from_txt(test_data_path.joinpath('Grammars/g.cfg')) 92 | singlepath_algo.prepare(graph, grammar) 93 | 94 | result: ResultAlgo = singlepath_algo.solve() 95 | assert result.matrix_S.nvals == 156 96 | 97 | paths = singlepath_algo.getPath(1, 1, "S") 98 | assert paths == 2 99 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FormalLanguageConstrainedPathQuerying/CFPQ_PyAlgo/14ea6935cfe5772ca866f6c545b0736920ff0ffa/test/__init__.py -------------------------------------------------------------------------------- /test/conftest.py: -------------------------------------------------------------------------------- 1 | from src.utils.useful_paths import GLOBAL_CFPQ_DATA, LOCAL_CFPQ_DATA 2 | from test.suites.cfpq_data import * 3 | 4 | 5 | def pytest_configure(config): 6 | config.addinivalue_line( 7 | 'markers', 'CI: small test to run in CI' 8 | ) 9 | 10 | for suite, graph, grammar in get_all_test_cases(LOCAL_CFPQ_DATA): 11 | for mark in get_markers(suite, graph, grammar): 12 | config.addinivalue_line( 13 | 'markers', f'{mark}: generated marker' 14 | ) 15 | -------------------------------------------------------------------------------- /test/data/binary_tree/Grammars/g.cfg: -------------------------------------------------------------------------------- 1 | S -> a S b | a b -------------------------------------------------------------------------------- /test/data/binary_tree/Graphs/graph_1.txt: -------------------------------------------------------------------------------- 1 | 0 a 2 2 | 1 a 2 3 | 3 a 5 4 | 4 a 5 5 | 2 a 6 6 | 5 a 6 7 | 2 b 0 8 | 2 b 1 9 | 5 b 3 10 | 5 b 4 11 | 6 b 2 12 | 6 b 5 -------------------------------------------------------------------------------- /test/data/cycle/Grammars/g.cfg: -------------------------------------------------------------------------------- 1 | S -> a S | a -------------------------------------------------------------------------------- /test/data/cycle/Graphs/graph_1.txt: -------------------------------------------------------------------------------- 1 | 0 a 1 2 | 1 a 2 3 | 2 a 0 -------------------------------------------------------------------------------- /test/data/line/Grammars/g.cfg: -------------------------------------------------------------------------------- 1 | S -> a S b | a b -------------------------------------------------------------------------------- /test/data/line/Graphs/graph_1.txt: -------------------------------------------------------------------------------- 1 | 0 a 1 2 | 1 a 2 3 | 2 b 3 4 | 3 b 4 -------------------------------------------------------------------------------- /test/data/loop/Grammars/g.cfg: -------------------------------------------------------------------------------- 1 | S -> a | a S -------------------------------------------------------------------------------- /test/data/loop/Graphs/graph_1.txt: -------------------------------------------------------------------------------- 1 | 0 a 0 2 | 0 b 1 3 | 1 c 2 -------------------------------------------------------------------------------- /test/data/single_vs_shortest/Grammars/g.cfg: -------------------------------------------------------------------------------- 1 | S -> S1 S2 | S15 S16 2 | S1 -> S3 S4 3 | S2 -> S5 S6 4 | S3 -> S7 S8 5 | S4 -> S9 S10 6 | S5 -> S11 S12 7 | S6 -> S13 S14 8 | S16 -> S17 S18 9 | S17 -> S19 S20 10 | S18 -> S21 S22 11 | S22 -> S23 S24 12 | S7 -> a 13 | S8 -> a 14 | S9 -> a 15 | S10 -> a 16 | S11 -> b 17 | S12 -> b 18 | S13 -> b 19 | S14 -> b 20 | S15 -> a 21 | S19 -> a 22 | S20 -> a 23 | S21 -> b 24 | S23 -> b 25 | S24 -> b 26 | -------------------------------------------------------------------------------- /test/data/single_vs_shortest/Graphs/graph_1.txt: -------------------------------------------------------------------------------- 1 | 0 a 1 2 | 1 a 2 3 | 2 a 3 4 | 3 a 4 5 | 3 b 5 6 | 4 b 3 7 | 5 b 6 8 | 6 b 7 9 | -------------------------------------------------------------------------------- /test/data/two_cycles/Grammars/g.cfg: -------------------------------------------------------------------------------- 1 | S -> a S b | a b -------------------------------------------------------------------------------- /test/data/two_cycles/Graphs/graph_1.txt: -------------------------------------------------------------------------------- 1 | 0 a 1 2 | 1 a 2 3 | 2 a 0 4 | 2 b 3 5 | 3 b 2 -------------------------------------------------------------------------------- /test/data/two_nonterm/Grammars/g.cfg: -------------------------------------------------------------------------------- 1 | V2 -> 2 | V -> V1 V2 V3 3 | V1 -> 4 | V2 -> S 5 | S -> d_r V d 6 | V1 -> V2 a_r V1 7 | V3 -> 8 | V3 -> a V2 V3 -------------------------------------------------------------------------------- /test/data/two_nonterm/Graphs/graph_1.txt: -------------------------------------------------------------------------------- 1 | 0 d 1 2 | 1 d_r 0 3 | 2 d 3 4 | 3 d_r 2 5 | 3 a 71 6 | 71 a_r 3 7 | 3 a 90 8 | 90 a_r 3 9 | 4 d 5 10 | 5 d_r 4 11 | 5 d 239 12 | 239 d_r 5 13 | 6 a 7 14 | 7 a_r 6 15 | 7 d 218 16 | 218 d_r 7 17 | 7 a 6 18 | 6 a_r 7 19 | 7 a 7 20 | 7 a_r 7 21 | 8 a 9 22 | 9 a_r 8 23 | 9 a 116 24 | 116 a_r 9 25 | 9 a 9 26 | 9 a_r 9 27 | 9 a 75 28 | 75 a_r 9 29 | 10 d 11 30 | 11 d_r 10 31 | 11 d 296 32 | 296 d_r 11 33 | 12 d 13 34 | 13 d_r 12 35 | 14 d 15 36 | 15 d_r 14 37 | 15 a 211 38 | 211 a_r 15 39 | 16 d 17 40 | 17 d_r 16 41 | 18 d 19 42 | 19 d_r 18 43 | 20 d 21 44 | 21 d_r 20 45 | 21 a 222 46 | 222 a_r 21 47 | 22 d 23 48 | 23 d_r 22 49 | 24 d 25 50 | 25 d_r 24 51 | 26 d 27 52 | 27 d_r 26 53 | 27 d 215 54 | 215 d_r 27 55 | 28 d 29 56 | 29 d_r 28 57 | 29 a 7 58 | 7 a_r 29 59 | 29 a 6 60 | 6 a_r 29 61 | 30 a 19 62 | 19 a_r 30 63 | 31 a 32 64 | 32 a_r 31 65 | 32 a 32 66 | 32 a_r 32 67 | 32 a 31 68 | 31 a_r 32 69 | 33 d 34 70 | 34 d_r 33 71 | 34 d 35 72 | 35 d_r 34 73 | 36 d 37 74 | 37 d_r 36 75 | 38 a 39 76 | 39 a_r 38 77 | 39 a 38 78 | 38 a_r 39 79 | 40 a 41 80 | 41 a_r 40 81 | 42 a 43 82 | 43 a_r 42 83 | 43 a 43 84 | 43 a_r 43 85 | 43 a 42 86 | 42 a_r 43 87 | 44 d 45 88 | 45 d_r 44 89 | 45 a 214 90 | 214 a_r 45 91 | 46 d 47 92 | 47 d_r 46 93 | 47 d 209 94 | 209 d_r 47 95 | 48 d 9 96 | 9 d_r 48 97 | 49 a 50 98 | 50 a_r 49 99 | 50 a 50 100 | 50 a_r 50 101 | 50 a 70 102 | 70 a_r 50 103 | 50 a 152 104 | 152 a_r 50 105 | 51 d 52 106 | 52 d_r 51 107 | 53 d 54 108 | 54 d_r 53 109 | 55 d 56 110 | 56 d_r 55 111 | 56 d 283 112 | 283 d_r 56 113 | 57 a 58 114 | 58 a_r 57 115 | 57 a 91 116 | 91 a_r 57 117 | 57 a 206 118 | 206 a_r 57 119 | 58 d 224 120 | 224 d_r 58 121 | 59 d 6 122 | 6 d_r 59 123 | 60 a 61 124 | 61 a_r 60 125 | 61 a 115 126 | 115 a_r 61 127 | 62 d 32 128 | 32 d_r 62 129 | 63 d 64 130 | 64 d_r 63 131 | 64 a 165 132 | 165 a_r 64 133 | 65 d 66 134 | 66 d_r 65 135 | 66 d 269 136 | 269 d_r 66 137 | 67 d 68 138 | 68 d_r 67 139 | 69 a 70 140 | 70 a_r 69 141 | 70 a 70 142 | 70 a_r 70 143 | 70 a 198 144 | 198 a_r 70 145 | 71 a 90 146 | 90 a_r 71 147 | 72 d 73 148 | 73 d_r 72 149 | 74 d 75 150 | 75 d_r 74 151 | 75 a 116 152 | 116 a_r 75 153 | 75 a 9 154 | 9 a_r 75 155 | 76 d 77 156 | 77 d_r 76 157 | 77 d 94 158 | 94 d_r 77 159 | 78 a 79 160 | 79 a_r 78 161 | 79 a 167 162 | 167 a_r 79 163 | 80 d 7 164 | 7 d_r 80 165 | 81 d 82 166 | 82 d_r 81 167 | 83 a 84 168 | 84 a_r 83 169 | 85 a 79 170 | 79 a_r 85 171 | 86 d 87 172 | 87 d_r 86 173 | 88 d 89 174 | 89 d_r 88 175 | 90 a 90 176 | 90 a_r 90 177 | 90 a 71 178 | 71 a_r 90 179 | 92 d 93 180 | 93 d_r 92 181 | 94 a 7 182 | 7 a_r 94 183 | 94 a 6 184 | 6 a_r 94 185 | 94 a 276 186 | 276 a_r 94 187 | 95 d 8 188 | 8 d_r 95 189 | 96 a 97 190 | 97 a_r 96 191 | 98 d 99 192 | 99 d_r 98 193 | 100 a 8 194 | 8 a_r 100 195 | 101 d 102 196 | 102 d_r 101 197 | 102 a 143 198 | 143 a_r 102 199 | 103 a 104 200 | 104 a_r 103 201 | 105 d 106 202 | 106 d_r 105 203 | 107 d 97 204 | 97 d_r 107 205 | 108 d 109 206 | 109 d_r 108 207 | 110 d 61 208 | 61 d_r 110 209 | 111 d 39 210 | 39 d_r 111 211 | 112 a 7 212 | 7 a_r 112 213 | 112 a 205 214 | 205 a_r 112 215 | 112 a 6 216 | 6 a_r 112 217 | 113 d 114 218 | 114 d_r 113 219 | 116 a 9 220 | 9 a_r 116 221 | 117 d 90 222 | 90 d_r 117 223 | 118 d 119 224 | 119 d_r 118 225 | 119 a 130 226 | 130 a_r 119 227 | 120 a 79 228 | 79 a_r 120 229 | 121 d 38 230 | 38 d_r 121 231 | 122 d 58 232 | 58 d_r 122 233 | 123 d 124 234 | 124 d_r 123 235 | 125 a 126 236 | 126 a_r 125 237 | 127 d 128 238 | 128 d_r 127 239 | 129 a 70 240 | 70 a_r 129 241 | 129 a 198 242 | 198 a_r 129 243 | 130 a 93 244 | 93 a_r 130 245 | 130 a 188 246 | 188 a_r 130 247 | 130 a 169 248 | 169 a_r 130 249 | 131 d 132 250 | 132 d_r 131 251 | 132 d 258 252 | 258 d_r 132 253 | 133 d 134 254 | 134 d_r 133 255 | 134 a 311 256 | 311 a_r 134 257 | 135 d 42 258 | 42 d_r 135 259 | 136 a 43 260 | 43 a_r 136 261 | 136 a 42 262 | 42 a_r 136 263 | 137 d 138 264 | 138 d_r 137 265 | 139 d 140 266 | 140 d_r 139 267 | 141 d 142 268 | 142 d_r 141 269 | 144 a 145 270 | 145 a_r 144 271 | 146 d 147 272 | 147 d_r 146 273 | 148 d 129 274 | 129 d_r 148 275 | 149 d 150 276 | 150 d_r 149 277 | 151 a 77 278 | 77 a_r 151 279 | 152 a 50 280 | 50 a_r 152 281 | 153 d 79 282 | 79 d_r 153 283 | 154 d 155 284 | 155 d_r 154 285 | 155 d 196 286 | 196 d_r 155 287 | 156 a 136 288 | 136 a_r 156 289 | 157 d 70 290 | 70 d_r 157 291 | 158 a 91 292 | 91 a_r 158 293 | 159 a 61 294 | 61 a_r 159 295 | 160 d 161 296 | 161 d_r 160 297 | 161 a 152 298 | 152 a_r 161 299 | 161 a 50 300 | 50 a_r 161 301 | 162 d 163 302 | 163 d_r 162 303 | 163 a 37 304 | 37 a_r 163 305 | 164 d 31 306 | 31 d_r 164 307 | 166 d 167 308 | 167 d_r 166 309 | 168 d 169 310 | 169 d_r 168 311 | 170 d 171 312 | 171 d_r 170 313 | 172 d 136 314 | 136 d_r 172 315 | 173 d 126 316 | 126 d_r 173 317 | 174 d 85 318 | 85 d_r 174 319 | 175 d 176 320 | 176 d_r 175 321 | 177 a 178 322 | 178 a_r 177 323 | 179 d 180 324 | 180 d_r 179 325 | 181 d 182 326 | 182 d_r 181 327 | 183 d 184 328 | 184 d_r 183 329 | 185 d 186 330 | 186 d_r 185 331 | 187 d 188 332 | 188 d_r 187 333 | 189 d 190 334 | 190 d_r 189 335 | 190 a 191 336 | 191 a_r 190 337 | 192 a 193 338 | 193 a_r 192 339 | 193 a 9 340 | 9 a_r 193 341 | 194 d 195 342 | 195 d_r 194 343 | 196 a 197 344 | 197 a_r 196 345 | 198 a 70 346 | 70 a_r 198 347 | 199 a 102 348 | 102 a_r 199 349 | 200 d 201 350 | 201 d_r 200 351 | 201 a 54 352 | 54 a_r 201 353 | 202 d 104 354 | 104 d_r 202 355 | 203 d 204 356 | 204 d_r 203 357 | 207 d 143 358 | 143 d_r 207 359 | 208 d 205 360 | 205 d_r 208 361 | 209 d 210 362 | 210 d_r 209 363 | 212 d 213 364 | 213 d_r 212 365 | 213 d 325 366 | 325 d_r 213 367 | 214 a 50 368 | 50 a_r 214 369 | 216 d 217 370 | 217 d_r 216 371 | 219 d 158 372 | 158 d_r 219 373 | 220 a 3 374 | 3 a_r 220 375 | 221 d 222 376 | 222 d_r 221 377 | 222 a 21 378 | 21 a_r 222 379 | 223 d 30 380 | 30 d_r 223 381 | 224 a 87 382 | 87 a_r 224 383 | 225 d 178 384 | 178 d_r 225 385 | 226 d 227 386 | 227 d_r 226 387 | 228 d 229 388 | 229 d_r 228 389 | 229 d 237 390 | 237 d_r 229 391 | 230 d 165 392 | 165 d_r 230 393 | 231 d 84 394 | 84 d_r 231 395 | 232 d 41 396 | 41 d_r 232 397 | 233 d 234 398 | 234 d_r 233 399 | 234 a 79 400 | 79 a_r 234 401 | 235 a 43 402 | 43 a_r 235 403 | 236 d 197 404 | 197 d_r 236 405 | 238 d 71 406 | 71 d_r 238 407 | 240 d 241 408 | 241 d_r 240 409 | 242 d 198 410 | 198 d_r 242 411 | 243 d 244 412 | 244 d_r 243 413 | 245 d 246 414 | 246 d_r 245 415 | 247 a 8 416 | 8 a_r 247 417 | 248 d 145 418 | 145 d_r 248 419 | 249 d 250 420 | 250 d_r 249 421 | 251 d 252 422 | 252 d_r 251 423 | 253 a 79 424 | 79 a_r 253 425 | 254 d 255 426 | 255 d_r 254 427 | 255 d 277 428 | 277 d_r 255 429 | 256 a 129 430 | 129 a_r 256 431 | 257 a 8 432 | 8 a_r 257 433 | 259 a 205 434 | 205 a_r 259 435 | 260 d 261 436 | 261 d_r 260 437 | 262 a 79 438 | 79 a_r 262 439 | 263 a 161 440 | 161 a_r 263 441 | 264 d 253 442 | 253 d_r 264 443 | 265 d 266 444 | 266 d_r 265 445 | 267 d 268 446 | 268 d_r 267 447 | 269 d 270 448 | 270 d_r 269 449 | 271 a 41 450 | 41 a_r 271 451 | 272 d 177 452 | 177 d_r 272 453 | 273 d 91 454 | 91 d_r 273 455 | 274 d 275 456 | 275 d_r 274 457 | 278 d 211 458 | 211 d_r 278 459 | 279 d 280 460 | 280 d_r 279 461 | 281 d 130 462 | 130 d_r 281 463 | 282 a 241 464 | 241 a_r 282 465 | 282 a 90 466 | 90 a_r 282 467 | 284 a 41 468 | 41 a_r 284 469 | 285 d 50 470 | 50 d_r 285 471 | 286 d 287 472 | 287 d_r 286 473 | 288 d 276 474 | 276 d_r 288 475 | 289 a 102 476 | 102 a_r 289 477 | 290 d 291 478 | 291 d_r 290 479 | 292 a 41 480 | 41 a_r 292 481 | 293 d 294 482 | 294 d_r 293 483 | 295 d 193 484 | 193 d_r 295 485 | 297 d 191 486 | 191 d_r 297 487 | 298 d 299 488 | 299 d_r 298 489 | 300 d 151 490 | 151 d_r 300 491 | 301 a 193 492 | 193 a_r 301 493 | 302 a 41 494 | 41 a_r 302 495 | 303 a 45 496 | 45 a_r 303 497 | 304 a 41 498 | 41 a_r 304 499 | 305 a 191 500 | 191 a_r 305 501 | 306 d 125 502 | 125 d_r 306 503 | 307 d 308 504 | 308 d_r 307 505 | 309 d 115 506 | 115 d_r 309 507 | 310 d 311 508 | 311 d_r 310 509 | 312 d 313 510 | 313 d_r 312 511 | 314 a 82 512 | 82 a_r 314 513 | 315 d 144 514 | 144 d_r 315 515 | 316 d 214 516 | 214 d_r 316 517 | 317 d 152 518 | 152 d_r 317 519 | 318 d 319 520 | 319 d_r 318 521 | 320 d 321 522 | 321 d_r 320 523 | 322 a 79 524 | 79 a_r 322 525 | 323 d 120 526 | 120 d_r 323 527 | 324 d 116 528 | 116 d_r 324 529 | 326 d 206 530 | 206 d_r 326 531 | 327 d 328 532 | 328 d_r 327 533 | 329 a 41 534 | 41 a_r 329 535 | 330 a 319 536 | 319 a_r 330 537 | 331 d 43 538 | 43 d_r 331 -------------------------------------------------------------------------------- /test/suites/cfpq_data.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from cfpq_data_devtools.data_wrapper import DataWrapper 3 | from src.utils.file_helpers import get_file_name 4 | 5 | 6 | def get_markers(suite, graph, grammar): 7 | return [suite, get_file_name(graph), get_file_name(grammar)] 8 | 9 | 10 | def get_all_test_cases(cfpq_data_path): 11 | data = DataWrapper(cfpq_data_path) 12 | return [ 13 | (suite, graph, grammar) 14 | for suite in data.get_suites() 15 | for graph in data.get_graphs(suite, include_extensions=['txt']) 16 | for grammar in data.get_grammars(suite, include_extension=['cnf']) 17 | ] 18 | 19 | 20 | def get_all_test_cases_params(cfpq_data_path): 21 | return [ 22 | pytest.param(graph, grammar, 23 | marks=[getattr(pytest.mark, mark) for mark in get_markers(suite, graph, grammar)], 24 | id=f'{get_file_name(graph)}-{get_file_name(grammar)}') 25 | for suite, graph, grammar in get_all_test_cases(cfpq_data_path) 26 | ] 27 | 28 | 29 | def all_cfpq_data_test_cases(cfpq_data_path): 30 | def decorator(f): 31 | return pytest.mark.parametrize( 32 | 'graph,grammar', 33 | get_all_test_cases_params(cfpq_data_path) 34 | )(f) 35 | return decorator 36 | --------------------------------------------------------------------------------