├── .gitignore ├── LICENSE ├── README.md ├── changegraph ├── __init__.py ├── build.py ├── gumtree.py ├── models.py └── visual.py ├── collect_cgs_from_tests.py ├── conf ├── help.md └── settings.json.example ├── deployment.py ├── external ├── compiled │ └── gumtree-2.1.2 │ │ ├── bin │ │ ├── gumtree │ │ └── gumtree.bat │ │ └── lib │ │ ├── ST4-4.0.8.jar │ │ ├── animal-sniffer-annotations-1.17.jar │ │ ├── antlr-3.5.2.jar │ │ ├── antlr-runtime-3.5.2.jar │ │ ├── aopalliance-1.0.jar │ │ ├── asm-3.1.jar │ │ ├── cglib-2.2.1-v20090111.jar │ │ ├── checker-qual-2.5.2.jar │ │ ├── classindex-3.4.jar │ │ ├── client-2.1.2.jar │ │ ├── client.diff-2.1.2.jar │ │ ├── commons-codec-1.10.jar │ │ ├── commons-io-2.0.1.jar │ │ ├── commons-lang3-3.1.jar │ │ ├── commons-logging-1.2.jar │ │ ├── core-2.1.2.jar │ │ ├── error_prone_annotations-2.2.0.jar │ │ ├── failureaccess-1.0.jar │ │ ├── gen.antlr3-2.1.2.jar │ │ ├── gen.antlr3-antlr-2.1.2.jar │ │ ├── gen.antlr3-json-2.1.2.jar │ │ ├── gen.antlr3-php-2.1.2.jar │ │ ├── gen.antlr3-r-2.1.2.jar │ │ ├── gen.antlr3-xml-2.1.2.jar │ │ ├── gen.c-2.1.2.jar │ │ ├── gen.css-2.1.2.jar │ │ ├── gen.javaparser-2.1.2.jar │ │ ├── gen.jdt-2.1.2.jar │ │ ├── gen.js-2.1.2.jar │ │ ├── gen.python-2.1.2.jar │ │ ├── gen.ruby-2.1.2.jar │ │ ├── gen.srcml-2.1.2.jar │ │ ├── gson-2.8.2.jar │ │ ├── guava-27.0-jre.jar │ │ ├── guice-3.0.jar │ │ ├── j2objc-annotations-1.1.jar │ │ ├── javaparser-core-3.13.1.jar │ │ ├── javaparser-symbol-solver-core-3.13.1.jar │ │ ├── javaparser-symbol-solver-logic-3.13.1.jar │ │ ├── javaparser-symbol-solver-model-3.13.1.jar │ │ ├── javassist-3.24.0-GA.jar │ │ ├── javax.inject-1.jar │ │ ├── javax.servlet-api-3.1.0.jar │ │ ├── jetty-client-9.4.6.v20170531.jar │ │ ├── jetty-http-9.4.6.v20170531.jar │ │ ├── jetty-io-9.4.6.v20170531.jar │ │ ├── jetty-security-9.4.6.v20170531.jar │ │ ├── jetty-server-9.4.6.v20170531.jar │ │ ├── jetty-servlet-9.4.6.v20170531.jar │ │ ├── jetty-util-9.4.6.v20170531.jar │ │ ├── jetty-webapp-9.4.6.v20170531.jar │ │ ├── jetty-xml-9.4.6.v20170531.jar │ │ ├── jgrapht-core-1.0.1.jar │ │ ├── jrubyparser-0.5.3.jar │ │ ├── jsr305-3.0.2.jar │ │ ├── jtidy-r938.jar │ │ ├── junit-4.8.2.jar │ │ ├── listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar │ │ ├── org.eclipse.core.commands-3.9.700.jar │ │ ├── org.eclipse.core.contenttype-3.7.600.jar │ │ ├── org.eclipse.core.expressions-3.6.700.jar │ │ ├── org.eclipse.core.filesystem-1.7.700.jar │ │ ├── org.eclipse.core.jobs-3.10.700.jar │ │ ├── org.eclipse.core.resources-3.13.700.jar │ │ ├── org.eclipse.core.runtime-3.17.100.jar │ │ ├── org.eclipse.equinox.app-1.4.400.jar │ │ ├── org.eclipse.equinox.common-3.11.0.jar │ │ ├── org.eclipse.equinox.preferences-3.7.700.jar │ │ ├── org.eclipse.equinox.registry-3.8.700.jar │ │ ├── org.eclipse.jdt.core-3.16.0.jar │ │ ├── org.eclipse.osgi-3.15.200.jar │ │ ├── org.eclipse.text-3.10.100.jar │ │ ├── ph-commons-9.3.0.jar │ │ ├── ph-css-6.1.2.jar │ │ ├── rendersnake-1.9.0.jar │ │ ├── rhino-1.7.10.jar │ │ ├── simmetrics-core-3.2.3.jar │ │ ├── slf4j-api-1.7.25.jar │ │ ├── slf4j-nop-1.7.25.jar │ │ ├── spark-core-2.7.1.jar │ │ ├── spring-aop-4.1.6.RELEASE.jar │ │ ├── spring-beans-4.1.6.RELEASE.jar │ │ ├── spring-context-4.1.6.RELEASE.jar │ │ ├── spring-core-4.1.6.RELEASE.jar │ │ ├── spring-expression-4.1.6.RELEASE.jar │ │ ├── spring-web-4.1.6.RELEASE.jar │ │ ├── spring-webmvc-4.1.6.RELEASE.jar │ │ ├── trove4j-3.0.3.jar │ │ ├── websocket-api-9.4.6.v20170531.jar │ │ ├── websocket-client-9.4.6.v20170531.jar │ │ ├── websocket-common-9.4.6.v20170531.jar │ │ ├── websocket-server-9.4.6.v20170531.jar │ │ └── websocket-servlet-9.4.6.v20170531.jar ├── pyparser.py └── pythonparser_3.py ├── log ├── __init__.py └── logger.py ├── main.py ├── output ├── general.js ├── libs │ ├── highlight │ │ ├── default.css │ │ └── highlight.pack.js │ ├── jquery.js │ └── underscore.js ├── sample.js └── styles.css ├── patterns ├── __init__.py ├── exas.py ├── models.py └── search.py ├── pyflowgraph ├── __init__.py ├── ast_utils.py ├── build.py ├── models.py └── visual.py ├── pytest.ini ├── requirements.txt ├── research ├── process-fd-count.sh └── tools │ ├── __init__.py │ ├── email_sender.py │ ├── log_analyser.py │ ├── patterns_collector.py │ └── repo_downloader.py ├── settings.py ├── survey_patterns.tar.gz ├── tests ├── __init__.py ├── conftest.py ├── external │ ├── buggy_test_pydriller.py │ └── test_gumtree.py ├── test_change_graphs.py ├── test_exas_features.py ├── test_patterns_mining.py ├── test_patterns_output.py ├── test_pyflowgraph.py └── utils.py ├── vb_utils.py └── vcs ├── __init__.py └── traverse.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | *.py[cod] 3 | venv/ 4 | images/ 5 | examples/ 6 | .idea/ 7 | conf/settings.json 8 | progress.txt 9 | .DS_Store 10 | .pytest_cache/ 11 | storage*/ 12 | output/patterns*/ 13 | research/data 14 | *.log 15 | *.png 16 | *.dot 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | Copyright 2020 Viacheslav Bushev 179 | 180 | Licensed under the Apache License, Version 2.0 (the "License"); 181 | you may not use this file except in compliance with the License. 182 | You may obtain a copy of the License at 183 | 184 | http://www.apache.org/licenses/LICENSE-2.0 185 | 186 | Unless required by applicable law or agreed to in writing, software 187 | distributed under the License is distributed on an "AS IS" BASIS, 188 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 189 | See the License for the specific language governing permissions and 190 | limitations under the License. 191 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![JetBrains Research](https://jb.gg/badges/research.svg)](https://confluence.jetbrains.com/display/ALL/JetBrains+on+GitHub) 2 | 3 | # Python Change Miner 4 | 5 | A tool for mining graph-based change patterns in Python code. 6 | 7 | ## What it does 8 | 9 | A **program dependence graph** is a way of representing the code by showing its data dependencies and control 10 | dependencies. 11 | 12 | A **change graph** is a program dependence graph for the fragment of code changes (using the versions of code before and 13 | after the target change). 14 | 15 | Similar code changes will have similar change graphs, which means that any versioned code can be mined for **patterns** 16 | in its changes. This tool does exactly that for Python code: it can build program dependence graphs, build change graphs 17 | for changed files, mine such change graphs from Git repositories by traversing their VCS history, and discover patterns 18 | in these change graphs. 19 | 20 | This functionality can be used for empirical research of coding practices, as well as for mining the candidates for 21 | potential IDE inspections. 22 | 23 | ## Getting started 24 | 25 | 0. The tool requires Python 3.8+ to run. We have also tested it only on Linux and macOS systems. 26 | 1. Install the required dependencies: 27 | 28 | ```shell script 29 | pip3 install -r requirements.txt 30 | ``` 31 | 32 | 2. Create the settings file _settings.json_ based on 33 | [_conf/settings.json.example_](https://github.com/JetBrains-Research/code-change-miner/blob/master/conf/settings.json.example) 34 | and save it in the [same directory](https://github.com/JetBrains-Research/code-change-miner/tree/master/conf). You 35 | can find the description of individual settings in [_conf/help.md_](https://github.com/JetBrains-Research/code-change-miner/blob/master/conf/help.md). 36 | 37 | 38 | 3. If you want to use the tool for building change graphs or mining change graphs from the local repositories, you need 39 | to setup [GumTree](https://github.com/GumTreeDiff/gumtree). But you can use the compiled version of GumTree 40 | (can be found in the [_external_](https://github.com/JetBrains-Research/code-change-miner/tree/master/external) 41 | directory), it is slightly modified and uses environment variables `GUMTREE_PYTHON_BIN` 42 | (python interpreter path for GumTree pyparser calls) and `GUMTREE_PYPARSER_PATH` (python parser script path). 43 | For general cases, they are set up automatically. If you want to do it manually, set them, for instance, as follows: 44 | 45 | ```shell script 46 | GUMTREE_PYTHON_BIN=python3 47 | ``` 48 | ```shell script 49 | GUMTREE_PYPARSER_PATH={project_dir}/external/pythonparser_3.py 50 | ``` 51 | 52 | 53 | ## How to use 54 | 55 | You can run any step of the pipeline by using the following simple command: 56 | 57 | ```shell script 58 | python3 main.py 59 | ``` 60 | 61 | The tool currently supports four operation modes: 62 | 63 | 1. `pfg` — build a program dependence graph from the Python source. 64 | 65 | Arguments: 66 | - `-i` — a path to the source file. 67 | - `-o` — a path to the output file. Two files will be created, a .dot file with a graph and a .pdf file with its 68 | visualization. 69 | - `--no-closure` — **(optional)** if passed, no closure will be built for the graph. 70 | - `--show-deps` — **(optional)** if passed, edges with type _dep_ will be present in the graph, indicating the 71 | dependence of the vertices on each other. 72 | - `--hide-op-kinds` — **(optional)** if passed, the types of operations will be hidden in the graph. 73 | - `--show-data-keys` — **(optional)** if passed, IDs of the variables will be present in the graph. 74 | 75 | Typical use: 76 | 77 | ```shell script 78 | python3 main.py pfg -i examples/src.py -o images/pfg.dot 79 | ``` 80 | 81 | 2. `cg` — build a change graph from two source files (before and after change). 82 | 83 | Arguments: 84 | - `-s` — a path to the source file before changes. 85 | - `-d` — a path to the source file after changes. 86 | - `-o` — a path to the output file. Two files will be created, a .dot file with a graph and a .pdf file with its 87 | visualization. 88 | 89 | Typical use: 90 | 91 | ```shell script 92 | python3 main.py cg -s examples/0_old.py -d examples/0_new.py -o images/cg.dot 93 | ``` 94 | 95 | 3. `collect-cgs` — mine change graphs from local repositories. 96 | 97 | All the general settings for this mode are located in the JSON file, see p. 3 of **Getting started**. 98 | 99 | Use: 100 | 101 | ```shell script 102 | python3 main.py collect-cgs 103 | ``` 104 | 105 | Arguments: 106 | - `--only-tests` — **(optional)** if passed, the tool will build change graphs only for the files with filenames 107 | containing "test" substring. 108 | 109 | The tool uses [pickle](https://docs.python.org/3/library/pickle.html) to save the data, so the output files are 110 | serialized and can be only processed by pickle. Running the tool in the `patterns` mode for detecting patterns within 111 | the mined change graphs will deserialize them automatically. 112 | 113 | 4. `patterns` — search for patterns in the change graphs. 114 | 115 | This mode can be run in two ways: from the results of the previous step or from the source files. The settings are 116 | located in the JSON file, see p. 3 of **Getting started**. If you want to look for patterns in the change graphs 117 | obtained from running the tool in the `collect-sgs` mode, simply run: 118 | 119 | ```shell script 120 | python3 main.py patterns 121 | ``` 122 | and the tool will find the input automatically. Alternatively, you can mine patterns directly from files with the 123 | following arguments: 124 | 125 | - `-s` — a path to the source files before changes. 126 | - `-d` — a path to the source files after changes. 127 | - `--fake-mining` — **(optional)** if passed, no mining is carried out, the change graphs as a whole are considered 128 | to be the patterns (used in debug). 129 | 130 | Typical use: 131 | 132 | ```shell script 133 | python3 main.py patterns -s examples/0_old.py examples/1_old.py -d examples/0_new.py examples/1_new.py 134 | ``` 135 | 136 | Here, the files are automatically mapped (_0_old.py_ -> _0_new.py_, _1_old.py_ -> _1_new.py_), their change graphs 137 | are built, and the patterns between them are mined. 138 | 139 | In both usage scenarios, the `patterns` mode will produce results as shown in the picture below: 140 | 141 | drawing 142 | 143 | The patterns are organized by their size in nodes. In the output directory, a directory is created for each size, in 144 | the example, the size is 17. In each of these directories we store the patterns, once again, as directories with 145 | their ID in the name. In the example, 1379 is the ID of a pattern with size 17. 146 | 147 | Within each pattern, we store _details.html_ with its description and the listing of the pattern instances, and the 148 | instances themselves. For each instance, there are three types of files: _sample{ID}_ is the code of the instance ( 149 | before and after the change), _fragment{ID}_ is the change graph of this specific sample, and _graph{ID}_ is the 150 | larger change graph, from which this sample came from. You can also control the specifics of the output by changing 151 | the settings file. _contents.html_ on every level of the structure provides a convenient navigation. To understand 152 | the structure better, you can browse an example output in _survey_patterns.tar.gz_. 153 | 154 | ## Contacts 155 | 156 | If you have any questions or suggestions, don't hesitate to open an issue or contact the developers at 157 | stardust.skg@gmail.com. 158 | -------------------------------------------------------------------------------- /changegraph/__init__.py: -------------------------------------------------------------------------------- 1 | from . import visual 2 | from .build import ChangeGraphBuilder 3 | 4 | 5 | _builder = ChangeGraphBuilder() 6 | 7 | build_from_files = _builder.build_from_files 8 | 9 | export_graph_image = visual.export_graph_image 10 | print_out_nodes = visual.print_out_nodes 11 | -------------------------------------------------------------------------------- /changegraph/build.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from log import logger 4 | import pyflowgraph 5 | from changegraph.models import ChangeNode, ChangeGraph, ChangeEdge 6 | from pyflowgraph.models import ExtControlFlowGraph, Node 7 | from changegraph.gumtree import GumTree 8 | from changegraph import gumtree 9 | 10 | 11 | class ChangeGraphBuilder: # TODO: make gumtree optional 12 | def build_from_files(self, path1, path2, repo_info=None): 13 | logger.warning(f'Change graph building...', show_pid=True) 14 | start_building = time.time() 15 | 16 | start = time.time() 17 | fg1 = pyflowgraph.build_from_file(path1) 18 | fg2 = pyflowgraph.build_from_file(path2) 19 | logger.warning('Flow graphs... OK', start_time=start, show_pid=True) 20 | 21 | start = time.time() 22 | gt1, gt2 = gumtree.build_from_file(path1), gumtree.build_from_file(path2) 23 | GumTree.map(gt1, gt2) 24 | logger.warning('Gumtree... OK', start_time=start, show_pid=True) 25 | 26 | for node in gt1.nodes: 27 | if node.mapped: 28 | logger.info(f'Gumtree node {node} mapped to {node.mapped}', show_pid=True) 29 | 30 | start = time.time() 31 | fg1.map_to_gumtree(gt1) 32 | fg2.map_to_gumtree(gt2) 33 | ExtControlFlowGraph.map_by_gumtree(fg1, fg2, gt1.matches) 34 | logger.warning('Mapping... OK', start_time=start, show_pid=True) 35 | 36 | for node in fg1.nodes: 37 | if node.mapped: 38 | logger.log(logger.INFO, f'FG node {node} mapped to {node.mapped}, ' 39 | f'GT node {node.gt_node} to {node.mapped.gt_node}, ' 40 | f'status={node.gt_node.status} ' 41 | f'and is_changed={node.gt_node.is_changed()}', show_pid=True) 42 | 43 | for node in fg2.nodes: 44 | node.version = Node.Version.AFTER_CHANGES 45 | cg = self._create_change_graph(fg1, fg2, repo_info=repo_info) 46 | logger.warning('Change graph building... OK', start_time=start_building, show_pid=True) 47 | 48 | for node in cg.nodes: 49 | logger.info(f'Change graph has node {node}', show_pid=True) 50 | 51 | return cg 52 | 53 | @staticmethod 54 | def _create_change_graph(fg1, fg2, repo_info=None): 55 | fg1.calc_changed_nodes_by_gumtree() 56 | fg2.calc_changed_nodes_by_gumtree() 57 | 58 | fg_changed_nodes = fg1.changed_nodes.union(fg2.changed_nodes) 59 | fg_node_to_cg_node = {} 60 | 61 | cg = ChangeGraph(repo_info=repo_info) 62 | for fg_node in fg_changed_nodes: 63 | if fg_node_to_cg_node.get(fg_node): 64 | continue 65 | 66 | node = ChangeNode.create_from_fg_node(fg_node) 67 | cg.nodes.add(node) 68 | node.set_graph(cg) # todo: probably better to create method 'add_node' 69 | fg_node_to_cg_node[fg_node] = node 70 | 71 | if fg_node.mapped and fg_node.mapped in fg_changed_nodes: 72 | mapped_node = ChangeNode.create_from_fg_node(fg_node.mapped) 73 | cg.nodes.add(mapped_node) 74 | mapped_node.set_graph(cg) 75 | fg_node_to_cg_node[fg_node.mapped] = mapped_node 76 | 77 | node.mapped = mapped_node 78 | mapped_node.mapped = node 79 | 80 | for fg_node in fg_changed_nodes: 81 | for e in fg_node.in_edges: 82 | if e.node_from in fg_changed_nodes: 83 | ChangeEdge.create(e.label, fg_node_to_cg_node[e.node_from], fg_node_to_cg_node[e.node_to]) 84 | return cg 85 | 86 | 87 | class GraphBuildingException(Exception): 88 | pass 89 | -------------------------------------------------------------------------------- /changegraph/gumtree.py: -------------------------------------------------------------------------------- 1 | import json 2 | import subprocess 3 | from enum import Enum 4 | 5 | import settings 6 | 7 | 8 | def parse(src_path): 9 | gumtree_bin_path = settings.get('gumtree_bin_path') 10 | args = [gumtree_bin_path, 'parse', src_path] 11 | p = subprocess.Popen(args, stdout=subprocess.PIPE) 12 | result, _ = p.communicate() 13 | return json.loads(result) if result else {} 14 | 15 | 16 | def diff(src1_path, src2_path): 17 | gumtree_bin_path = settings.get('gumtree_bin_path') 18 | args = [gumtree_bin_path, 'jsondiff', src1_path, src2_path] 19 | p = subprocess.Popen(args, stdout=subprocess.PIPE) 20 | result, _ = p.communicate() 21 | return json.loads(result) if result else {} 22 | 23 | 24 | def get_matches_and_actions(src1_path, src2_path): 25 | result = diff(src1_path, src2_path) 26 | return result.get('matches', {}), result.get('actions', {}) 27 | 28 | 29 | def build_from_file(src_path): 30 | parsed = parse(src_path) 31 | return GumTree(src_path, parsed) 32 | 33 | 34 | class GumTree: 35 | class ActionType: 36 | UPDATE = 'update' 37 | DELETE = 'delete' 38 | INSERT = 'insert' 39 | MOVE = 'move' 40 | 41 | class TypeLabel: 42 | NAME_STORE = 'Name_Store' 43 | NAME_LOAD = 'Name_Load' 44 | FUNC_CALL = 'Call' 45 | FUNC_DEF = 'FunctionDef' 46 | ASSIGN = 'Assign' 47 | EXPR = 'Expr' 48 | SUBSCRIPT_STORE = 'Subscript_Store' 49 | SUBSCRIPT_LOAD = 'Subscript_Load' 50 | ATTRIBUTE_STORE = 'Attribute_Store' 51 | ATTRIBUTE_LOAD = 'Attribute_Load' 52 | ATTR = 'attr' 53 | RETURN = 'Return' 54 | ARGS = 'arguments' 55 | SIMPLE_ARGS = 'args' 56 | DEFAULT_ARGS = 'defaults' 57 | SIMPLE_ARG = 'arg' 58 | KEYWORD = 'keyword' 59 | LISTCOMP = 'ListComp' 60 | DICTCOMP = 'DictComp' 61 | GENERATOREXPR = 'GeneratorExp' 62 | COMPREHENSION = 'comprehension' 63 | 64 | 65 | def __init__(self, source_path, data): 66 | self.node_id_to_node = {} 67 | self.nodes = [] 68 | 69 | self._data = data 70 | self.cnt = self._read_data(self._data.get('root', {}), start_value=0) 71 | 72 | self.root = self.nodes[-1] 73 | self.source_path = source_path 74 | self.matches = {} 75 | self.actions = {} 76 | 77 | def _read_data(self, start_node, start_value=0): 78 | val = start_value 79 | 80 | child_nodes = [] 81 | children = start_node.get('children') 82 | 83 | if children: 84 | for child in children: 85 | child_node, val = self._read_data(child, start_value=val) 86 | child_nodes.append(child_node) 87 | 88 | start_node['id'] = val 89 | node = GumTreeNode(data=start_node) 90 | node.children = child_nodes 91 | 92 | for child_node in node.children: 93 | child_node.parent = node 94 | 95 | self.nodes.append(node) 96 | self.node_id_to_node[node.id] = node 97 | 98 | return node, val + 1 99 | 100 | def find_node(self, pos, length, start_node=None, type_label=None): 101 | node = start_node 102 | 103 | if node is None: 104 | node = self.root 105 | 106 | if node.pos == pos and node.length == length: 107 | if type_label is None or node.type_label == type_label: 108 | return node 109 | 110 | if node.children: 111 | for child in node.children: 112 | result = self.find_node(pos, length, start_node=child, type_label=type_label) 113 | if result: 114 | return result 115 | 116 | return False 117 | 118 | @staticmethod 119 | def map(gt_src, gt_dest): 120 | matches, actions = get_matches_and_actions(gt_src.source_path, gt_dest.source_path) 121 | 122 | GumTree._apply_matching(gt_src, gt_dest, matches) 123 | GumTree._apply_actions(gt_src, gt_dest, actions) 124 | GumTree._adjust_changes(gt_src, gt_dest) 125 | 126 | @staticmethod 127 | def _apply_matching(gt_src, gt_dest, matches): 128 | gt_src.matches = gt_dest.matches = matches 129 | 130 | for match in matches: 131 | src = gt_src.node_id_to_node[int(match.get('src'))] 132 | dest = gt_dest.node_id_to_node[int(match.get('dest'))] 133 | 134 | src.mapped = dest 135 | dest.mapped = src 136 | 137 | @staticmethod 138 | def _apply_actions(gt_src, gt_dest, actions): 139 | gt_src.actions = gt_dest.actions = actions 140 | 141 | for action in actions: 142 | action_name = action['action'] 143 | node_id = int(action['tree']) 144 | 145 | if action_name == GumTree.ActionType.UPDATE: 146 | node1 = gt_src.node_id_to_node[node_id] 147 | node1.status = GumTreeNode.STATUS.UPDATED 148 | node1.mapped.status = GumTreeNode.STATUS.UPDATED 149 | elif action_name == GumTree.ActionType.DELETE: 150 | node1 = gt_src.node_id_to_node[node_id] 151 | node1.status = GumTreeNode.STATUS.DELETED 152 | elif action_name == GumTree.ActionType.MOVE: 153 | node1 = gt_src.node_id_to_node[node_id] 154 | node1.status = GumTreeNode.STATUS.MOVED 155 | node1.mapped.status = GumTreeNode.STATUS.MOVED 156 | elif action_name == GumTree.ActionType.INSERT: 157 | node2 = gt_dest.node_id_to_node[node_id] 158 | node2.status = GumTreeNode.STATUS.INSERTED 159 | else: 160 | raise ValueError('Undefined action given by gumtree diff') 161 | 162 | @classmethod 163 | def _adjust_changes(cls, gt_src, gt_dest): 164 | gt_src.dfs(fn_before=cls._before_change_detector, fn_after=cls._change_detector) 165 | gt_dest.dfs(fn_before=cls._before_change_detector, fn_after=cls._change_detector) 166 | 167 | @classmethod 168 | def _before_change_detector(cls, node): 169 | # parent = node.parent 170 | # if parent: 171 | # if parent.type_label in [GumTree.TypeLabel.KEYWORD] and parent.status == GumTreeNode.STATUS.MOVED: 172 | # node.status = GumTreeNode.STATUS.MOVED 173 | return True 174 | 175 | @classmethod 176 | def _change_detector(cls, node): 177 | if node.status in [GumTreeNode.STATUS.INSERTED, GumTreeNode.STATUS.DELETED]: 178 | return True 179 | 180 | is_changed = not node.mapped or not node.is_equal(node.mapped) 181 | if not is_changed: 182 | if not node.children: 183 | is_changed = bool(len(node.mapped.children)) 184 | else: 185 | is_changed = not len(node.mapped.children) 186 | if not is_changed: 187 | ignore_child_ids = [] 188 | if node.type_label in [GumTree.TypeLabel.FUNC_CALL, GumTree.TypeLabel.ATTRIBUTE_LOAD]: 189 | name_node = node.children[0] 190 | is_changed = bool(name_node.status != GumTreeNode.STATUS.UNCHANGED) 191 | ignore_child_ids.append(name_node.id) 192 | 193 | if not is_changed: 194 | is_changed = cls._are_children_changed(node, ignore_child_ids=ignore_child_ids) 195 | 196 | if not is_changed: 197 | node.status = GumTreeNode.STATUS.UNCHANGED 198 | if node.mapped: 199 | node.mapped.status = GumTreeNode.STATUS.UNCHANGED 200 | 201 | return is_changed 202 | 203 | @staticmethod 204 | def _are_children_changed(node, /, *, ignore_child_ids=None): 205 | children = [child for child in node.children if child.id not in ignore_child_ids] 206 | if not children: 207 | return False 208 | 209 | for child in children: 210 | if child.status == GumTreeNode.STATUS.UNCHANGED: 211 | return False 212 | return True 213 | 214 | @classmethod 215 | def _do_dfs(cls, node, visited, fn_before=None, fn_after=None): 216 | if fn_before: 217 | if not fn_before(node): 218 | return 219 | 220 | for child in node.children: 221 | if not visited.get(child.id): 222 | cls._do_dfs(child, visited, fn_before=fn_before, fn_after=fn_after) 223 | 224 | visited[node.id] = True 225 | 226 | if fn_after: 227 | fn_after(node) 228 | 229 | def dfs(self, fn_before=None, fn_after=None, start_node=None): 230 | self._do_dfs(start_node or self.root, {}, fn_before=fn_before, fn_after=fn_after) 231 | 232 | 233 | class GumTreeNode: 234 | class STATUS(Enum): 235 | UNCHANGED = 0 236 | CHANGED = 1 237 | INSERTED = 2 238 | DELETED = 3 239 | MOVED = 4 240 | UPDATED = 5 241 | 242 | def __lt__(self, other): 243 | if self.__class__ is other.__class__: 244 | return self.value < other.value 245 | raise TypeError 246 | 247 | def __init__(self, data): 248 | self.id = data['id'] 249 | 250 | self.pos = int(data['pos']) 251 | self.length = int(data['length']) 252 | self.type_label = data['typeLabel'] 253 | self.label = data.get('label') # e.g. present in AttributeLoad.attr 254 | self.children = [] 255 | 256 | self.data = data 257 | self.mapped = None 258 | 259 | self.fg_node = None 260 | 261 | self.parent = None 262 | self.status = GumTreeNode.STATUS.CHANGED 263 | 264 | def is_changed(self): 265 | if self.status != GumTreeNode.STATUS.UNCHANGED: 266 | return True 267 | 268 | if self.parent and self.parent.type_label == GumTree.TypeLabel.EXPR: 269 | if self.parent.status != GumTreeNode.STATUS.UNCHANGED: 270 | return True 271 | 272 | return False 273 | 274 | def is_equal(self, node): 275 | fst_data = {k: self.data[k] for k in self.data.keys() if k in ['label', 'type', 'typeLabel']} 276 | snd_data = {k: node.data[k] for k in node.data.keys() if k in ['label', 'type', 'typeLabel']} 277 | return fst_data == snd_data 278 | 279 | def get_child_by_type_label(self, type_label): 280 | for child in self.children: 281 | if child.type_label == type_label: 282 | return child 283 | return None 284 | 285 | def __repr__(self): 286 | return f'#{self.id} {self.type_label} {self.label} [{self.pos}:{self.length}]' 287 | 288 | 289 | class MappingException(Exception): 290 | pass 291 | -------------------------------------------------------------------------------- /changegraph/models.py: -------------------------------------------------------------------------------- 1 | from pyflowgraph.models import DataNode, Node, OperationNode, ControlNode, LinkType 2 | 3 | 4 | class ChangeGraph: 5 | def __init__(self, repo_info=None): 6 | self.nodes = set() 7 | self.repo_info = repo_info 8 | 9 | 10 | class ChangeNode: # todo: create base class for pfg and cg 11 | _NODE_ID = 0 12 | 13 | class Property: 14 | SYNTAX_TOKEN_INTERVALS = Node.Property.SYNTAX_TOKEN_INTERVALS 15 | ALL = [SYNTAX_TOKEN_INTERVALS] 16 | 17 | def set_property(self, prop, value): 18 | self._data[prop] = value 19 | 20 | def get_property(self, prop, default=None): 21 | return self._data.get(prop, default) 22 | 23 | class CommonLabel: 24 | VARIABLE = 'var' 25 | LITERAL = 'lit' 26 | 27 | class Kind: 28 | DATA_NODE = 'data' 29 | OPERATION_NODE = 'operation' 30 | CONTROL_NODE = 'control' 31 | UNKNOWN = 'unknown' 32 | 33 | class Version(Node.Version): 34 | pass 35 | 36 | class SubKind: 37 | DATA_VARIABLE_DECL = DataNode.Kind.VARIABLE_DECL 38 | DATA_VARIABLE_USAGE = DataNode.Kind.VARIABLE_USAGE 39 | DATA_LITERAL = DataNode.Kind.LITERAL 40 | DATA_KEYWORD = DataNode.Kind.KEYWORD 41 | 42 | OP_COLLECTION = OperationNode.Kind.COLLECTION 43 | OP_FUNC_CALL = OperationNode.Kind.FUNC_CALL 44 | OP_ASSIGNMENT = OperationNode.Kind.ASSIGN 45 | OP_COMPARE = OperationNode.Kind.COMPARE 46 | OP_RETURN = OperationNode.Kind.RETURN 47 | 48 | def __init__(self, statement_num, ast, label, kind, version, sub_kind=None, original_label=None): 49 | ChangeNode._NODE_ID += 1 50 | self.id = ChangeNode._NODE_ID 51 | 52 | self.statement_num = statement_num 53 | self.ast = ast 54 | 55 | self.label = label 56 | self.original_label = original_label 57 | 58 | self.in_edges = set() 59 | self.out_edges = set() 60 | self.mapped = None 61 | self.graph = None 62 | 63 | self.kind = kind 64 | self.sub_kind = sub_kind 65 | 66 | self.version = version 67 | 68 | self._data = {} 69 | 70 | @classmethod 71 | def create_from_fg_node(cls, fg_node): 72 | label = fg_node.label 73 | if isinstance(fg_node, DataNode): 74 | kind = cls.Kind.DATA_NODE 75 | sub_kind = fg_node.kind 76 | 77 | if sub_kind in [cls.SubKind.DATA_VARIABLE_DECL, cls.SubKind.DATA_VARIABLE_USAGE]: 78 | label = cls.CommonLabel.VARIABLE 79 | elif sub_kind in [cls.SubKind.DATA_LITERAL, cls.SubKind.DATA_KEYWORD]: 80 | label = cls.CommonLabel.LITERAL 81 | 82 | elif isinstance(fg_node, OperationNode): 83 | kind = cls.Kind.OPERATION_NODE 84 | elif isinstance(fg_node, ControlNode): 85 | kind = cls.Kind.CONTROL_NODE 86 | else: 87 | kind = cls.Kind.UNKNOWN 88 | 89 | created = ChangeNode(fg_node.statement_num, fg_node.ast, label, kind, fg_node.version, 90 | sub_kind=getattr(fg_node, 'kind', None), original_label=fg_node.label) 91 | 92 | for prop in cls.Property.ALL: 93 | fg_node_prop = fg_node.get_property(prop) 94 | if fg_node_prop is None: 95 | continue 96 | created.set_property(prop, fg_node_prop) 97 | 98 | return created 99 | 100 | def get_in_nodes(self, /, *, labels=None, excluded_labels=None): 101 | return self._get_nodes_by_edges(need_out=False, labels=labels, excluded_labels=excluded_labels) 102 | 103 | def get_out_nodes(self, /, *, labels=None, excluded_labels=None): 104 | return self._get_nodes_by_edges(need_out=True, labels=labels, excluded_labels=excluded_labels) 105 | 106 | def _get_nodes_by_edges(self, need_out=False, labels=None, excluded_labels=None): 107 | if all([labels, excluded_labels]): 108 | raise ValueError('Unsupported combination of arguments') 109 | 110 | result = set() 111 | edges = self.out_edges if need_out else self.in_edges 112 | 113 | for e in edges: 114 | if excluded_labels and e.label in excluded_labels or labels and e.label not in labels: 115 | continue 116 | 117 | if need_out: 118 | result.add(e.node_to) 119 | else: 120 | result.add(e.node_from) 121 | 122 | return result 123 | 124 | def get_definitions(self): 125 | defs = set() 126 | for e in self.in_edges: 127 | if isinstance(e, ChangeEdge) and e.label == LinkType.REFERENCE: 128 | defs.add(e.node_from) 129 | return defs 130 | 131 | def set_graph(self, graph): 132 | self.graph = graph 133 | 134 | def __eq__(self, other): 135 | return self.id == other.id 136 | 137 | def __hash__(self): 138 | return self.id 139 | 140 | def __repr__(self): 141 | return f'#{self.id} v{self.version} {self.label} ({self.original_label}) {self.kind}.{self.sub_kind}' 142 | 143 | 144 | class ChangeEdge: 145 | def __init__(self, label, node_from, node_to): 146 | self.node_from = node_from 147 | self.node_to = node_to 148 | self.label = label 149 | 150 | @classmethod 151 | def create(cls, label, node_from, node_to): 152 | created = ChangeEdge(label, node_from, node_to) 153 | 154 | node_from.out_edges.add(created) 155 | node_to.in_edges.add(created) 156 | 157 | def __repr__(self): 158 | return f'#{self.node_from.id} -{self.label}> #{self.node_to.id}' 159 | -------------------------------------------------------------------------------- /changegraph/visual.py: -------------------------------------------------------------------------------- 1 | import graphviz as gv 2 | import os 3 | 4 | from changegraph.models import Node, ChangeGraph, ChangeNode 5 | 6 | 7 | def _get_label_and_attrs(node): 8 | attrs = {'shape': 'ellipse'} 9 | 10 | if node.kind == ChangeNode.Kind.DATA_NODE: 11 | attrs['shape'] = 'ellipse' 12 | elif node.kind == ChangeNode.Kind.OPERATION_NODE: 13 | attrs['shape'] = 'box' 14 | elif node.kind == ChangeNode.Kind.CONTROL_NODE: 15 | attrs['shape'] = 'diamond' 16 | 17 | if node.version == Node.Version.BEFORE_CHANGES: 18 | attrs['color'] = 'red2' # colors on https://www.graphviz.org/doc/info/colors.html 19 | else: 20 | attrs['color'] = 'green4' 21 | 22 | label = f'{node.label} ({node.original_label}) [{node.id}]' 23 | return label, attrs 24 | 25 | 26 | def _get_nodes_digraph(nodes: set, file_name, separate_mapped=True): 27 | vg = gv.Digraph(name=file_name, format='pdf') 28 | 29 | used = {} 30 | for node in nodes: 31 | if used.get(node): 32 | continue 33 | 34 | if separate_mapped and node.mapped and node.mapped in nodes: 35 | label, attrs = _get_label_and_attrs(node) 36 | mapped_label, mapped_attrs = _get_label_and_attrs(node.mapped) 37 | 38 | used[node] = used[node.mapped] = True 39 | 40 | s = gv.Digraph(f'subgraph: {node.id} to {node.mapped.id}') 41 | s.node(f'{node.id}', label=label, _attributes=attrs) 42 | s.node(f'{node.mapped.id}', label=mapped_label, _attributes=mapped_attrs) 43 | 44 | s.graph_attr.update(rank='same') 45 | vg.subgraph(s) 46 | else: 47 | label, attrs = _get_label_and_attrs(node) 48 | vg.node(f'{node.id}', label=label, _attributes=attrs) 49 | 50 | for node in nodes: 51 | for edge in node.in_edges: 52 | if edge.node_from not in nodes: 53 | continue 54 | 55 | label = edge.label 56 | attrs = {} 57 | 58 | vg.edge(f'{edge.node_from.id}', f'{edge.node_to.id}', xlabel=label, _attributes=attrs) 59 | 60 | return vg 61 | 62 | 63 | def _convert_to_visual_graph(graph: ChangeGraph, file_name: str, separate_mapped=True): 64 | return _get_nodes_digraph(graph.nodes, file_name, separate_mapped=separate_mapped) 65 | 66 | 67 | def export_graph_image(graph: ChangeGraph, path: str = 'change-graph.dot'): 68 | directory, file_name = os.path.split(path) 69 | visual_graph = _convert_to_visual_graph(graph, file_name) 70 | visual_graph.render(filename=file_name, directory=directory) 71 | 72 | 73 | def print_out_nodes(nodes, path: str = 'nodes.dot'): 74 | directory, file_name = os.path.split(path) 75 | visual_graph = _get_nodes_digraph(nodes, file_name) 76 | visual_graph.render(filename=path, directory=directory) 77 | -------------------------------------------------------------------------------- /collect_cgs_from_tests.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import multiprocessing 4 | import os 5 | import pickle 6 | import sys 7 | import tempfile 8 | import uuid 9 | from pathlib import Path 10 | from typing import List 11 | 12 | from tqdm import tqdm 13 | 14 | import changegraph 15 | import settings 16 | from changegraph.models import ChangeGraph 17 | from deployment import set_all_environment_variables 18 | from log import logger 19 | from vcs.traverse import GitAnalyzer, RepoInfo 20 | 21 | STORAGE_DIR = settings.get('change_graphs_storage_dir') 22 | 23 | finished_files = {} 24 | 25 | 26 | def init(finished): 27 | global finished_files 28 | finished_files = finished 29 | 30 | 31 | def store_change_graphs(change_graphs: List[ChangeGraph]): 32 | pickled_graphs = [] 33 | for graph in change_graphs: 34 | try: 35 | pickled = pickle.dumps(graph, protocol=5) 36 | pickled_graphs.append(pickled) 37 | except RecursionError: 38 | logger.error(f'Unable to pickle graph {graph}') 39 | filename = uuid.uuid4().hex 40 | logger.info(f'Trying to store graphs to {filename}', show_pid=True) 41 | with open(os.path.join(STORAGE_DIR, f'{filename}.pickle'), 'w+b') as f: 42 | pickle.dump(pickled_graphs, f) 43 | logger.info(f'Storing graphs to {filename} finished', show_pid=True) 44 | 45 | 46 | def mine_changes(path_to_repo_dir: str): 47 | change_graphs = [] 48 | for dirname, _, files in os.walk(path_to_repo_dir): 49 | try: 50 | old_file_path, new_file_path = None, None 51 | for filename in files: 52 | if filename.endswith('.before.py'): 53 | old_file_path = os.path.join(dirname, filename) 54 | elif filename.endswith('.after.py'): 55 | new_file_path = os.path.join(dirname, filename) 56 | 57 | if not old_file_path or not new_file_path: 58 | continue 59 | 60 | if old_file_path in finished_files: 61 | continue 62 | 63 | with open(old_file_path, 'r') as before_file, open(new_file_path, 'r') as after_file: 64 | before_src = before_file.read() 65 | after_src = after_file.read() 66 | 67 | old_method_to_new = GitAnalyzer._get_methods_mapping( 68 | GitAnalyzer._extract_methods(old_file_path, before_src), 69 | GitAnalyzer._extract_methods(new_file_path, after_src) 70 | ) 71 | 72 | for old_method, new_method in old_method_to_new.items(): 73 | old_method_src = old_method.get_source() 74 | new_method_src = new_method.get_source() 75 | 76 | if not all([old_method_src, new_method_src]) or old_method_src.strip() == new_method_src.strip(): 77 | continue 78 | 79 | line_count = max(old_method_src.count('\n'), new_method_src.count('\n')) 80 | if line_count > settings.get('traverse_file_max_line_count'): 81 | logger.info(f'Ignored files due to line limit: {old_file_path}') 82 | continue 83 | 84 | with tempfile.NamedTemporaryFile(mode='w+t', suffix='.py') as t1, \ 85 | tempfile.NamedTemporaryFile(mode='w+t', suffix='.py') as t2: 86 | 87 | t1.writelines(old_method_src) 88 | t1.seek(0) 89 | t2.writelines(new_method_src) 90 | t2.seek(0) 91 | 92 | local_repo_path = Path(old_file_path).parent.parent 93 | repo_info = RepoInfo( 94 | repo_name=local_repo_path.name, 95 | repo_path=local_repo_path, 96 | repo_url='', 97 | commit_hash='', 98 | commit_dtm='', 99 | old_file_path=old_file_path, 100 | new_file_path=new_file_path, 101 | old_method=old_method, 102 | new_method=new_method 103 | ) 104 | 105 | try: 106 | cg = changegraph.build_from_files(os.path.realpath(t1.name), 107 | os.path.realpath(t2.name), 108 | repo_info) 109 | except Exception: 110 | logger.log(logger.ERROR, 111 | f'Unable to build a change graph for ' 112 | f'method={old_method.full_name}, ' 113 | f'line={old_method.ast.lineno}', exc_info=True, show_pid=True) 114 | continue 115 | 116 | change_graphs.append(cg) 117 | 118 | if len(change_graphs) > GitAnalyzer.STORE_INTERVAL: 119 | store_change_graphs(change_graphs) 120 | change_graphs.clear() 121 | 122 | finished_files[old_file_path] = True 123 | 124 | except Exception: 125 | continue 126 | 127 | if change_graphs: 128 | store_change_graphs(change_graphs) 129 | 130 | 131 | def main(src_dir: str, parallel: bool = False): 132 | # Load (or create empty) already finished files collection 133 | global finished_files 134 | path_to_finished = Path(src_dir) / 'finished.json' 135 | if path_to_finished.exists(): 136 | try: 137 | with open(path_to_finished, 'r') as file: 138 | finished_files = json.load(file) 139 | except Exception: 140 | finished_files = {} 141 | else: 142 | finished_files = {} 143 | 144 | # Traverse all the subdirectories to find before-after pairs 145 | paths = [] 146 | for repo_name in os.listdir(src_dir): 147 | paths.append(os.path.join(src_dir, repo_name)) 148 | 149 | # Build and save change graphs 150 | try: 151 | if parallel: 152 | manager = multiprocessing.Manager() 153 | finished_files = manager.dict(finished_files) 154 | with multiprocessing.Pool(initializer=init, initargs=(finished_files,)) as pool: 155 | list(tqdm(pool.imap(mine_changes, paths), total=len(paths))) 156 | else: 157 | for path in tqdm(paths): 158 | mine_changes(path) 159 | except BaseException: 160 | with open(path_to_finished, 'w') as file: 161 | json.dump(finished_files.copy(), file) 162 | 163 | 164 | if __name__ == '__main__': 165 | set_all_environment_variables() 166 | sys.setrecursionlimit(2 ** 31 - 1) 167 | multiprocessing.set_start_method('spawn', force=True) 168 | 169 | parser = argparse.ArgumentParser() 170 | parser.add_argument('-s', '--src', help='Path to directory with before and after versions', type=str, required=True) 171 | parser.add_argument('--parallel', help='Run in parallel', action='store_true') 172 | args = parser.parse_args() 173 | 174 | main(args.src, args.parallel) 175 | -------------------------------------------------------------------------------- /conf/help.md: -------------------------------------------------------------------------------- 1 | # Settings configuration 2 | 3 | In order to run the tool, you need to create a file _settings.json_ in this directory, based on [_settings.json.example_](https://github.com/JetBrains-Research/code-change-miner/blob/master/conf/settings.json.example). 4 | Here are the detailed explanations of the settings: 5 | 6 | ### Settings for the _collect-cgs_ mode: 7 | 8 | Name | Description 9 | --- | --- 10 | **gumtree_bin_path** | path to GumTree binary file 11 | **git_repositories_dir** | path to the directory with Git repositories 12 | **traverse_file_max_line_count** | the maximum number of lines in the analyzed files (processing larger files may sometimes cause memory issues) 13 | **traverse_async** | **true** for the asynchronous processing of repositories 14 | **traverse_min_date** | **(optional)** the date in the **%d.%m.%Y** format, no changes older than this date will be processed 15 | **change_graphs_storage_dir** | path to the output directory 16 | **change_graphs_store_interval** | batch size of the number of change graphs to be saved in a single file (to prevent the files from getting too big) 17 | 18 | ### Settings for the _patterns_ mode: 19 | 20 | By default, the input for mining patterns is the output of collecting change graphs, so this step should be run after the previous one. 21 | 22 | Name | Description 23 | --- | --- 24 | **patterns_output_dir** | path to the output directory (in order to correctly visualize the results, this directory must be located [here](https://github.com/JetBrains-Research/code-change-miner/tree/master/output) or have all the same files in the parent directory) 25 | **patterns_output_details** | **true** for saving a JSON for each pattern instance with its details: repository name, commit hash, contacts of the author, and the names of the functions (please note that if you want such information for all the instances, you also need to switch **patterns_full_print** to **true**) 26 | **patterns_min_frequency** | minimum frequency of the changes graph repetition to be considered a pattern 27 | **patterns_max_frequency** | frequency of the changes graph repetition for the pattern to be considered _common_ (for such patterns, some instances are ignored for optimization purposes) 28 | **patterns_async_mining** | **true** for the asynchronous mining of patterns **(not recommended)** 29 | **patterns_full_print** | **true** for saving the information about every individual instance of a pattern, **false** for saving one instance per pattern 30 | **patterns_hide_overlapped_fragments** | **true** for ignoring pattern instances with overlapping code fragments 31 | **patterns_min_size** | minimum number of nodes that the pattern must have to be included in the output 32 | **patterns_min_date** | **(optional)** the date in the **%d.%m.%Y** format, no changes older than this date will be processed 33 | 34 | ### Additional settings: 35 | 36 | Name | Description 37 | --- | --- 38 | **logger_file_path** | path to the log output file 39 | **logger_file_log_level** | log level to be saved into the log file, possible values: **ERROR**, **WARNING**, **INFO**, **DEBUG** 40 | **logger_stdout_log_level** | log level to be printed into the console, possible values: **ERROR**, **WARNING**, **INFO**, **DEBUG** 41 | 42 | 43 | -------------------------------------------------------------------------------- /conf/settings.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "gumtree_bin_path": str, 3 | "git_repositories_dir": str, 4 | 5 | "traverse_file_max_line_count": 1500, 6 | "traverse_async": true, 7 | "traverse_min_date": str?, 8 | 9 | "change_graphs_storage_dir": str, 10 | "change_graphs_store_interval": 300, 11 | 12 | "patterns_output_dir": str, 13 | "patterns_output_details": false, 14 | 15 | "patterns_min_frequency": 3, 16 | "patterns_max_frequency": 1000, 17 | "patterns_async_mining": false, 18 | "patterns_full_print": false, 19 | "patterns_hide_overlapped_fragments": true, 20 | "patterns_min_size": 3, 21 | "patterns_min_date": str?, 22 | 23 | "logger_file_path": "miner.log", 24 | "logger_file_log_level": "INFO", 25 | "logger_stdout_log_level": "WARNING", 26 | 27 | "use_stackimpact": false, 28 | "stackimpact_agent_key": str? 29 | } -------------------------------------------------------------------------------- /deployment.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | PROJECT_DIRECTORY = os.path.dirname(__file__) 4 | 5 | PYTHON_BIN_ENV_VAR = "GUMTREE_PYTHON_BIN" 6 | PYPARSER_ENV_VAR = "GUMTREE_PYPARSER_PATH" 7 | 8 | PYTHON_BIN_VALUE = "python3" 9 | PYPARSER_NAME = 'pythonparser_3.py' 10 | 11 | 12 | def set_environment_variable(env_var, value): 13 | os.environ[env_var] = value 14 | 15 | 16 | def set_all_environment_variables(): 17 | set_environment_variable(PYTHON_BIN_ENV_VAR, PYTHON_BIN_VALUE) 18 | 19 | pyparser_path = os.path.join(PROJECT_DIRECTORY, 'external', PYPARSER_NAME) 20 | set_environment_variable(PYPARSER_ENV_VAR, pyparser_path) 21 | -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/bin/gumtree: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## gumtree start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | SAVED="`pwd`" 11 | cd "`dirname \"$0\"`/.." >/dev/null 12 | APP_HOME="`pwd -P`" 13 | cd "$SAVED" >/dev/null 14 | 15 | APP_NAME="gumtree" 16 | 17 | # Add default JVM options here. You can also use JAVA_OPTS and GUMTREE_OPTS to pass JVM options to this script. 18 | DEFAULT_JVM_OPTS="" 19 | 20 | warn () { 21 | echo "$*" 22 | } 23 | 24 | die () { 25 | echo 26 | echo "$*" 27 | echo 28 | exit 1 29 | } 30 | 31 | # OS specific support (must be 'true' or 'false'). 32 | cygwin=false 33 | msys=false 34 | darwin=false 35 | nonstop=false 36 | case "`uname`" in 37 | CYGWIN* ) 38 | cygwin=true 39 | ;; 40 | Darwin* ) 41 | darwin=true 42 | ;; 43 | MINGW* ) 44 | msys=true 45 | ;; 46 | NONSTOP* ) 47 | nonstop=true 48 | ;; 49 | esac 50 | 51 | CLASSPATH=$APP_HOME/lib/gumtree-2.1.2.jar:$APP_HOME/lib/client.diff-2.1.2.jar:$APP_HOME/lib/client-2.1.2.jar:$APP_HOME/lib/gen.antlr3-antlr-2.1.2.jar:$APP_HOME/lib/gen.antlr3-json-2.1.2.jar:$APP_HOME/lib/gen.antlr3-php-2.1.2.jar:$APP_HOME/lib/gen.antlr3-r-2.1.2.jar:$APP_HOME/lib/gen.antlr3-xml-2.1.2.jar:$APP_HOME/lib/gen.antlr3-2.1.2.jar:$APP_HOME/lib/gen.c-2.1.2.jar:$APP_HOME/lib/gen.css-2.1.2.jar:$APP_HOME/lib/gen.javaparser-2.1.2.jar:$APP_HOME/lib/gen.jdt-2.1.2.jar:$APP_HOME/lib/gen.js-2.1.2.jar:$APP_HOME/lib/gen.python-2.1.2.jar:$APP_HOME/lib/gen.ruby-2.1.2.jar:$APP_HOME/lib/gen.srcml-2.1.2.jar:$APP_HOME/lib/core-2.1.2.jar:$APP_HOME/lib/classindex-3.4.jar:$APP_HOME/lib/simmetrics-core-3.2.3.jar:$APP_HOME/lib/trove4j-3.0.3.jar:$APP_HOME/lib/gson-2.8.2.jar:$APP_HOME/lib/jgrapht-core-1.0.1.jar:$APP_HOME/lib/spark-core-2.7.1.jar:$APP_HOME/lib/slf4j-nop-1.7.25.jar:$APP_HOME/lib/rendersnake-1.9.0.jar:$APP_HOME/lib/antlr-3.5.2.jar:$APP_HOME/lib/ph-css-6.1.2.jar:$APP_HOME/lib/javaparser-symbol-solver-core-3.13.1.jar:$APP_HOME/lib/org.eclipse.jdt.core-3.16.0.jar:$APP_HOME/lib/rhino-1.7.10.jar:$APP_HOME/lib/jrubyparser-0.5.3.jar:$APP_HOME/lib/javaparser-symbol-solver-logic-3.13.1.jar:$APP_HOME/lib/javaparser-symbol-solver-model-3.13.1.jar:$APP_HOME/lib/guava-27.0-jre.jar:$APP_HOME/lib/commons-codec-1.10.jar:$APP_HOME/lib/ph-commons-9.3.0.jar:$APP_HOME/lib/slf4j-api-1.7.25.jar:$APP_HOME/lib/jetty-webapp-9.4.6.v20170531.jar:$APP_HOME/lib/websocket-server-9.4.6.v20170531.jar:$APP_HOME/lib/jetty-servlet-9.4.6.v20170531.jar:$APP_HOME/lib/jetty-security-9.4.6.v20170531.jar:$APP_HOME/lib/jetty-server-9.4.6.v20170531.jar:$APP_HOME/lib/websocket-servlet-9.4.6.v20170531.jar:$APP_HOME/lib/junit-4.8.2.jar:$APP_HOME/lib/commons-lang3-3.1.jar:$APP_HOME/lib/commons-io-2.0.1.jar:$APP_HOME/lib/spring-webmvc-4.1.6.RELEASE.jar:$APP_HOME/lib/jtidy-r938.jar:$APP_HOME/lib/guice-3.0.jar:$APP_HOME/lib/javax.inject-1.jar:$APP_HOME/lib/ST4-4.0.8.jar:$APP_HOME/lib/antlr-runtime-3.5.2.jar:$APP_HOME/lib/javassist-3.24.0-GA.jar:$APP_HOME/lib/org.eclipse.core.resources-3.13.700.jar:$APP_HOME/lib/org.eclipse.text-3.10.100.jar:$APP_HOME/lib/org.eclipse.core.expressions-3.6.700.jar:$APP_HOME/lib/org.eclipse.core.runtime-3.17.100.jar:$APP_HOME/lib/org.eclipse.core.filesystem-1.7.700.jar:$APP_HOME/lib/javax.servlet-api-3.1.0.jar:$APP_HOME/lib/websocket-client-9.4.6.v20170531.jar:$APP_HOME/lib/jetty-client-9.4.6.v20170531.jar:$APP_HOME/lib/jetty-http-9.4.6.v20170531.jar:$APP_HOME/lib/websocket-common-9.4.6.v20170531.jar:$APP_HOME/lib/jetty-io-9.4.6.v20170531.jar:$APP_HOME/lib/jetty-xml-9.4.6.v20170531.jar:$APP_HOME/lib/websocket-api-9.4.6.v20170531.jar:$APP_HOME/lib/spring-web-4.1.6.RELEASE.jar:$APP_HOME/lib/spring-context-4.1.6.RELEASE.jar:$APP_HOME/lib/spring-aop-4.1.6.RELEASE.jar:$APP_HOME/lib/spring-beans-4.1.6.RELEASE.jar:$APP_HOME/lib/spring-expression-4.1.6.RELEASE.jar:$APP_HOME/lib/spring-core-4.1.6.RELEASE.jar:$APP_HOME/lib/aopalliance-1.0.jar:$APP_HOME/lib/cglib-2.2.1-v20090111.jar:$APP_HOME/lib/jsr305-3.0.2.jar:$APP_HOME/lib/failureaccess-1.0.jar:$APP_HOME/lib/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:$APP_HOME/lib/checker-qual-2.5.2.jar:$APP_HOME/lib/error_prone_annotations-2.2.0.jar:$APP_HOME/lib/j2objc-annotations-1.1.jar:$APP_HOME/lib/animal-sniffer-annotations-1.17.jar:$APP_HOME/lib/javaparser-core-3.13.1.jar:$APP_HOME/lib/org.eclipse.osgi-3.15.200.jar:$APP_HOME/lib/org.eclipse.core.jobs-3.10.700.jar:$APP_HOME/lib/org.eclipse.core.contenttype-3.7.600.jar:$APP_HOME/lib/org.eclipse.equinox.app-1.4.400.jar:$APP_HOME/lib/org.eclipse.equinox.registry-3.8.700.jar:$APP_HOME/lib/org.eclipse.equinox.preferences-3.7.700.jar:$APP_HOME/lib/org.eclipse.core.commands-3.9.700.jar:$APP_HOME/lib/org.eclipse.equinox.common-3.11.0.jar:$APP_HOME/lib/jetty-util-9.4.6.v20170531.jar:$APP_HOME/lib/commons-logging-1.2.jar:$APP_HOME/lib/asm-3.1.jar 52 | 53 | # Determine the Java command to use to start the JVM. 54 | if [ -n "$JAVA_HOME" ] ; then 55 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 56 | # IBM's JDK on AIX uses strange locations for the executables 57 | JAVACMD="$JAVA_HOME/jre/sh/java" 58 | else 59 | JAVACMD="$JAVA_HOME/bin/java" 60 | fi 61 | if [ ! -x "$JAVACMD" ] ; then 62 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 63 | 64 | Please set the JAVA_HOME variable in your environment to match the 65 | location of your Java installation." 66 | fi 67 | else 68 | JAVACMD="java" 69 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 70 | 71 | Please set the JAVA_HOME variable in your environment to match the 72 | location of your Java installation." 73 | fi 74 | 75 | # For Darwin, add options to specify how the application appears in the dock 76 | if $darwin; then 77 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 78 | fi 79 | 80 | # For Cygwin, switch paths to Windows format before running java 81 | if $cygwin ; then 82 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 83 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 84 | JAVACMD=`cygpath --unix "$JAVACMD"` 85 | 86 | # We build the pattern for arguments to be converted via cygpath 87 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 88 | SEP="" 89 | for dir in $ROOTDIRSRAW ; do 90 | ROOTDIRS="$ROOTDIRS$SEP$dir" 91 | SEP="|" 92 | done 93 | OURCYGPATTERN="(^($ROOTDIRS))" 94 | # Add a user-defined pattern to the cygpath arguments 95 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 96 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 97 | fi 98 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 99 | i=0 100 | for arg in "$@" ; do 101 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 102 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 103 | 104 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 105 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 106 | else 107 | eval `echo args$i`="\"$arg\"" 108 | fi 109 | i=$((i+1)) 110 | done 111 | case $i in 112 | (0) set -- ;; 113 | (1) set -- "$args0" ;; 114 | (2) set -- "$args0" "$args1" ;; 115 | (3) set -- "$args0" "$args1" "$args2" ;; 116 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 117 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 118 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 119 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 120 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 121 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 122 | esac 123 | fi 124 | 125 | # Escape application args 126 | save () { 127 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 128 | echo " " 129 | } 130 | APP_ARGS=$(save "$@") 131 | 132 | # Collect all arguments for the java command, following the shell quoting and substitution rules 133 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GUMTREE_OPTS -classpath "\"$CLASSPATH\"" com.github.gumtreediff.client.Run "$APP_ARGS" 134 | 135 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 136 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 137 | cd "$(dirname "$0")" 138 | fi 139 | 140 | exec "$JAVACMD" "$@" 141 | -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/bin/gumtree.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem gumtree startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME%.. 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GUMTREE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\lib\gumtree-2.1.2.jar;%APP_HOME%\lib\client.diff-2.1.2.jar;%APP_HOME%\lib\client-2.1.2.jar;%APP_HOME%\lib\gen.antlr3-antlr-2.1.2.jar;%APP_HOME%\lib\gen.antlr3-json-2.1.2.jar;%APP_HOME%\lib\gen.antlr3-php-2.1.2.jar;%APP_HOME%\lib\gen.antlr3-r-2.1.2.jar;%APP_HOME%\lib\gen.antlr3-xml-2.1.2.jar;%APP_HOME%\lib\gen.antlr3-2.1.2.jar;%APP_HOME%\lib\gen.c-2.1.2.jar;%APP_HOME%\lib\gen.css-2.1.2.jar;%APP_HOME%\lib\gen.javaparser-2.1.2.jar;%APP_HOME%\lib\gen.jdt-2.1.2.jar;%APP_HOME%\lib\gen.js-2.1.2.jar;%APP_HOME%\lib\gen.python-2.1.2.jar;%APP_HOME%\lib\gen.ruby-2.1.2.jar;%APP_HOME%\lib\gen.srcml-2.1.2.jar;%APP_HOME%\lib\core-2.1.2.jar;%APP_HOME%\lib\classindex-3.4.jar;%APP_HOME%\lib\simmetrics-core-3.2.3.jar;%APP_HOME%\lib\trove4j-3.0.3.jar;%APP_HOME%\lib\gson-2.8.2.jar;%APP_HOME%\lib\jgrapht-core-1.0.1.jar;%APP_HOME%\lib\spark-core-2.7.1.jar;%APP_HOME%\lib\slf4j-nop-1.7.25.jar;%APP_HOME%\lib\rendersnake-1.9.0.jar;%APP_HOME%\lib\antlr-3.5.2.jar;%APP_HOME%\lib\ph-css-6.1.2.jar;%APP_HOME%\lib\javaparser-symbol-solver-core-3.13.1.jar;%APP_HOME%\lib\org.eclipse.jdt.core-3.16.0.jar;%APP_HOME%\lib\rhino-1.7.10.jar;%APP_HOME%\lib\jrubyparser-0.5.3.jar;%APP_HOME%\lib\javaparser-symbol-solver-logic-3.13.1.jar;%APP_HOME%\lib\javaparser-symbol-solver-model-3.13.1.jar;%APP_HOME%\lib\guava-27.0-jre.jar;%APP_HOME%\lib\commons-codec-1.10.jar;%APP_HOME%\lib\ph-commons-9.3.0.jar;%APP_HOME%\lib\slf4j-api-1.7.25.jar;%APP_HOME%\lib\jetty-webapp-9.4.6.v20170531.jar;%APP_HOME%\lib\websocket-server-9.4.6.v20170531.jar;%APP_HOME%\lib\jetty-servlet-9.4.6.v20170531.jar;%APP_HOME%\lib\jetty-security-9.4.6.v20170531.jar;%APP_HOME%\lib\jetty-server-9.4.6.v20170531.jar;%APP_HOME%\lib\websocket-servlet-9.4.6.v20170531.jar;%APP_HOME%\lib\junit-4.8.2.jar;%APP_HOME%\lib\commons-lang3-3.1.jar;%APP_HOME%\lib\commons-io-2.0.1.jar;%APP_HOME%\lib\spring-webmvc-4.1.6.RELEASE.jar;%APP_HOME%\lib\jtidy-r938.jar;%APP_HOME%\lib\guice-3.0.jar;%APP_HOME%\lib\javax.inject-1.jar;%APP_HOME%\lib\ST4-4.0.8.jar;%APP_HOME%\lib\antlr-runtime-3.5.2.jar;%APP_HOME%\lib\javassist-3.24.0-GA.jar;%APP_HOME%\lib\org.eclipse.core.resources-3.13.700.jar;%APP_HOME%\lib\org.eclipse.text-3.10.100.jar;%APP_HOME%\lib\org.eclipse.core.expressions-3.6.700.jar;%APP_HOME%\lib\org.eclipse.core.runtime-3.17.100.jar;%APP_HOME%\lib\org.eclipse.core.filesystem-1.7.700.jar;%APP_HOME%\lib\javax.servlet-api-3.1.0.jar;%APP_HOME%\lib\websocket-client-9.4.6.v20170531.jar;%APP_HOME%\lib\jetty-client-9.4.6.v20170531.jar;%APP_HOME%\lib\jetty-http-9.4.6.v20170531.jar;%APP_HOME%\lib\websocket-common-9.4.6.v20170531.jar;%APP_HOME%\lib\jetty-io-9.4.6.v20170531.jar;%APP_HOME%\lib\jetty-xml-9.4.6.v20170531.jar;%APP_HOME%\lib\websocket-api-9.4.6.v20170531.jar;%APP_HOME%\lib\spring-web-4.1.6.RELEASE.jar;%APP_HOME%\lib\spring-context-4.1.6.RELEASE.jar;%APP_HOME%\lib\spring-aop-4.1.6.RELEASE.jar;%APP_HOME%\lib\spring-beans-4.1.6.RELEASE.jar;%APP_HOME%\lib\spring-expression-4.1.6.RELEASE.jar;%APP_HOME%\lib\spring-core-4.1.6.RELEASE.jar;%APP_HOME%\lib\aopalliance-1.0.jar;%APP_HOME%\lib\cglib-2.2.1-v20090111.jar;%APP_HOME%\lib\jsr305-3.0.2.jar;%APP_HOME%\lib\failureaccess-1.0.jar;%APP_HOME%\lib\listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar;%APP_HOME%\lib\checker-qual-2.5.2.jar;%APP_HOME%\lib\error_prone_annotations-2.2.0.jar;%APP_HOME%\lib\j2objc-annotations-1.1.jar;%APP_HOME%\lib\animal-sniffer-annotations-1.17.jar;%APP_HOME%\lib\javaparser-core-3.13.1.jar;%APP_HOME%\lib\org.eclipse.osgi-3.15.200.jar;%APP_HOME%\lib\org.eclipse.core.jobs-3.10.700.jar;%APP_HOME%\lib\org.eclipse.core.contenttype-3.7.600.jar;%APP_HOME%\lib\org.eclipse.equinox.app-1.4.400.jar;%APP_HOME%\lib\org.eclipse.equinox.registry-3.8.700.jar;%APP_HOME%\lib\org.eclipse.equinox.preferences-3.7.700.jar;%APP_HOME%\lib\org.eclipse.core.commands-3.9.700.jar;%APP_HOME%\lib\org.eclipse.equinox.common-3.11.0.jar;%APP_HOME%\lib\jetty-util-9.4.6.v20170531.jar;%APP_HOME%\lib\commons-logging-1.2.jar;%APP_HOME%\lib\asm-3.1.jar 67 | 68 | @rem Execute gumtree 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GUMTREE_OPTS% -classpath "%CLASSPATH%" com.github.gumtreediff.client.Run %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GUMTREE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GUMTREE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/ST4-4.0.8.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/ST4-4.0.8.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/animal-sniffer-annotations-1.17.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/animal-sniffer-annotations-1.17.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/antlr-3.5.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/antlr-3.5.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/antlr-runtime-3.5.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/antlr-runtime-3.5.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/aopalliance-1.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/aopalliance-1.0.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/asm-3.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/asm-3.1.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/cglib-2.2.1-v20090111.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/cglib-2.2.1-v20090111.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/checker-qual-2.5.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/checker-qual-2.5.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/classindex-3.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/classindex-3.4.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/client-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/client-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/client.diff-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/client.diff-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/commons-codec-1.10.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/commons-codec-1.10.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/commons-io-2.0.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/commons-io-2.0.1.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/commons-lang3-3.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/commons-lang3-3.1.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/commons-logging-1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/commons-logging-1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/core-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/core-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/error_prone_annotations-2.2.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/error_prone_annotations-2.2.0.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/failureaccess-1.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/failureaccess-1.0.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/gen.antlr3-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/gen.antlr3-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/gen.antlr3-antlr-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/gen.antlr3-antlr-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/gen.antlr3-json-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/gen.antlr3-json-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/gen.antlr3-php-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/gen.antlr3-php-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/gen.antlr3-r-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/gen.antlr3-r-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/gen.antlr3-xml-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/gen.antlr3-xml-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/gen.c-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/gen.c-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/gen.css-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/gen.css-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/gen.javaparser-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/gen.javaparser-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/gen.jdt-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/gen.jdt-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/gen.js-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/gen.js-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/gen.python-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/gen.python-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/gen.ruby-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/gen.ruby-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/gen.srcml-2.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/gen.srcml-2.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/gson-2.8.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/gson-2.8.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/guava-27.0-jre.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/guava-27.0-jre.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/guice-3.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/guice-3.0.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/j2objc-annotations-1.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/j2objc-annotations-1.1.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/javaparser-core-3.13.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/javaparser-core-3.13.1.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/javaparser-symbol-solver-core-3.13.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/javaparser-symbol-solver-core-3.13.1.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/javaparser-symbol-solver-logic-3.13.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/javaparser-symbol-solver-logic-3.13.1.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/javaparser-symbol-solver-model-3.13.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/javaparser-symbol-solver-model-3.13.1.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/javassist-3.24.0-GA.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/javassist-3.24.0-GA.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/javax.inject-1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/javax.inject-1.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/javax.servlet-api-3.1.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/javax.servlet-api-3.1.0.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/jetty-client-9.4.6.v20170531.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/jetty-client-9.4.6.v20170531.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/jetty-http-9.4.6.v20170531.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/jetty-http-9.4.6.v20170531.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/jetty-io-9.4.6.v20170531.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/jetty-io-9.4.6.v20170531.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/jetty-security-9.4.6.v20170531.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/jetty-security-9.4.6.v20170531.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/jetty-server-9.4.6.v20170531.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/jetty-server-9.4.6.v20170531.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/jetty-servlet-9.4.6.v20170531.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/jetty-servlet-9.4.6.v20170531.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/jetty-util-9.4.6.v20170531.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/jetty-util-9.4.6.v20170531.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/jetty-webapp-9.4.6.v20170531.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/jetty-webapp-9.4.6.v20170531.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/jetty-xml-9.4.6.v20170531.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/jetty-xml-9.4.6.v20170531.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/jgrapht-core-1.0.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/jgrapht-core-1.0.1.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/jrubyparser-0.5.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/jrubyparser-0.5.3.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/jsr305-3.0.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/jsr305-3.0.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/jtidy-r938.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/jtidy-r938.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/junit-4.8.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/junit-4.8.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/org.eclipse.core.commands-3.9.700.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/org.eclipse.core.commands-3.9.700.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/org.eclipse.core.contenttype-3.7.600.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/org.eclipse.core.contenttype-3.7.600.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/org.eclipse.core.expressions-3.6.700.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/org.eclipse.core.expressions-3.6.700.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/org.eclipse.core.filesystem-1.7.700.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/org.eclipse.core.filesystem-1.7.700.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/org.eclipse.core.jobs-3.10.700.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/org.eclipse.core.jobs-3.10.700.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/org.eclipse.core.resources-3.13.700.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/org.eclipse.core.resources-3.13.700.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/org.eclipse.core.runtime-3.17.100.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/org.eclipse.core.runtime-3.17.100.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/org.eclipse.equinox.app-1.4.400.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/org.eclipse.equinox.app-1.4.400.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/org.eclipse.equinox.common-3.11.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/org.eclipse.equinox.common-3.11.0.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/org.eclipse.equinox.preferences-3.7.700.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/org.eclipse.equinox.preferences-3.7.700.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/org.eclipse.equinox.registry-3.8.700.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/org.eclipse.equinox.registry-3.8.700.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/org.eclipse.jdt.core-3.16.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/org.eclipse.jdt.core-3.16.0.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/org.eclipse.osgi-3.15.200.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/org.eclipse.osgi-3.15.200.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/org.eclipse.text-3.10.100.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/org.eclipse.text-3.10.100.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/ph-commons-9.3.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/ph-commons-9.3.0.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/ph-css-6.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/ph-css-6.1.2.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/rendersnake-1.9.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/rendersnake-1.9.0.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/rhino-1.7.10.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/rhino-1.7.10.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/simmetrics-core-3.2.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/simmetrics-core-3.2.3.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/slf4j-api-1.7.25.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/slf4j-api-1.7.25.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/slf4j-nop-1.7.25.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/slf4j-nop-1.7.25.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/spark-core-2.7.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/spark-core-2.7.1.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/spring-aop-4.1.6.RELEASE.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/spring-aop-4.1.6.RELEASE.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/spring-beans-4.1.6.RELEASE.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/spring-beans-4.1.6.RELEASE.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/spring-context-4.1.6.RELEASE.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/spring-context-4.1.6.RELEASE.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/spring-core-4.1.6.RELEASE.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/spring-core-4.1.6.RELEASE.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/spring-expression-4.1.6.RELEASE.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/spring-expression-4.1.6.RELEASE.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/spring-web-4.1.6.RELEASE.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/spring-web-4.1.6.RELEASE.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/spring-webmvc-4.1.6.RELEASE.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/spring-webmvc-4.1.6.RELEASE.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/trove4j-3.0.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/trove4j-3.0.3.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/websocket-api-9.4.6.v20170531.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/websocket-api-9.4.6.v20170531.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/websocket-client-9.4.6.v20170531.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/websocket-client-9.4.6.v20170531.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/websocket-common-9.4.6.v20170531.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/websocket-common-9.4.6.v20170531.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/websocket-server-9.4.6.v20170531.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/websocket-server-9.4.6.v20170531.jar -------------------------------------------------------------------------------- /external/compiled/gumtree-2.1.2/lib/websocket-servlet-9.4.6.v20170531.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/external/compiled/gumtree-2.1.2/lib/websocket-servlet-9.4.6.v20170531.jar -------------------------------------------------------------------------------- /external/pyparser.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import ast 3 | import asttokens 4 | from xml.sax.saxutils import quoteattr 5 | 6 | 7 | def read_file_to_string(filename): 8 | f = open(filename, 'rt') 9 | s = f.read() 10 | f.close() 11 | return s 12 | 13 | 14 | def parse_file(filename): 15 | tree = asttokens.ASTTokens(read_file_to_string(filename), parse=True).tree 16 | 17 | json_tree = [] 18 | 19 | def localize(node, json_node): 20 | json_node['lineno'] = str(node.first_token.start[0]) 21 | json_node['col'] = str(node.first_token.start[1]) 22 | json_node['end_line_no'] = str(node.last_token.end[0]) 23 | json_node['end_col'] = str(node.last_token.end[1]) 24 | 25 | def gen_identifier(identifier, node_type='identifier', node=None): 26 | pos = len(json_tree) 27 | json_node = {} 28 | json_tree.append(json_node) 29 | json_node['type'] = node_type 30 | json_node['value'] = identifier 31 | localize(node, json_node) 32 | return pos 33 | 34 | def traverse_list(l, node_type='list', node=None): 35 | pos = len(json_tree) 36 | json_node = {} 37 | json_tree.append(json_node) 38 | json_node['type'] = node_type 39 | localize(node, json_node) 40 | children = [] 41 | for item in l: 42 | children.append(traverse(item)) 43 | if (len(children) != 0): 44 | json_node['children'] = children 45 | return pos 46 | 47 | def traverse(node): 48 | pos = len(json_tree) 49 | json_node = {} 50 | json_tree.append(json_node) 51 | json_node['type'] = type(node).__name__ 52 | localize(node, json_node) 53 | children = [] 54 | if isinstance(node, ast.Name): 55 | json_node['value'] = node.id 56 | elif isinstance(node, ast.NameConstant): 57 | json_node['value'] = node.value 58 | elif isinstance(node, ast.Constant): 59 | json_node['value'] = node.value 60 | elif isinstance(node, ast.Num): 61 | json_node['value'] = (node.n) 62 | elif isinstance(node, ast.Str): 63 | json_node['value'] = node.s 64 | elif isinstance(node, ast.alias): 65 | json_node['value'] = (node.name) 66 | if node.asname: 67 | children.append(gen_identifier(node.asname, node=node)) 68 | elif isinstance(node, ast.FunctionDef): 69 | json_node['value'] = (node.name) 70 | elif isinstance(node, ast.ExceptHandler): 71 | if node.name: 72 | json_node['value'] = node.name 73 | elif isinstance(node, ast.ClassDef): 74 | json_node['value'] = (node.name) 75 | elif isinstance(node, ast.ImportFrom): 76 | if node.module: 77 | json_node['value'] = (node.module) 78 | elif isinstance(node, ast.Global): 79 | for n in node.names: 80 | children.append(gen_identifier(n, node=node)) 81 | elif isinstance(node, ast.keyword): 82 | json_node['value'] = (node.arg) 83 | elif isinstance(node, ast.arg): 84 | json_node['value'] = (node.arg) 85 | 86 | # Process children. 87 | if isinstance(node, ast.For): 88 | children.append(traverse(node.target)) 89 | children.append(traverse(node.iter)) 90 | children.append(traverse_list(node.body, 'body', node)) 91 | if node.orelse: 92 | children.append(traverse_list(node.orelse, 'orelse', node)) 93 | elif isinstance(node, ast.If) or isinstance(node, ast.While): 94 | children.append(traverse(node.test)) 95 | children.append(traverse_list(node.body, 'body', node)) 96 | if node.orelse: 97 | children.append(traverse_list(node.orelse, 'orelse', node)) 98 | elif isinstance(node, ast.With): 99 | children.append(traverse_list(node.items, 'items', node)) 100 | children.append(traverse_list(node.body, 'body', node)) 101 | elif isinstance(node, ast.withitem): 102 | children.append(traverse(node.context_expr)) 103 | if node.optional_vars: 104 | children.append(traverse(node.optional_vars)) 105 | elif isinstance(node, ast.Try): 106 | children.append(traverse_list(node.body, 'body', node)) 107 | children.append(traverse_list(node.handlers, 'handlers', node)) 108 | if node.orelse: 109 | children.append(traverse_list(node.orelse, 'orelse', node)) 110 | if node.finalbody: 111 | children.append(traverse_list(node.finalbody, 'finalbody', node)) 112 | elif isinstance(node, ast.arguments): 113 | children.append(traverse_list(node.args, 'args', node)) 114 | children.append(traverse_list(node.defaults, 'defaults', node)) 115 | children.append(traverse_list(node.kwonlyargs, 'defaults', node)) 116 | children.append(traverse_list(node.kw_defaults, 'defaults', node)) 117 | if node.vararg: 118 | children.append(gen_identifier(node.vararg.arg, 'vararg', node.vararg)) 119 | if node.kwarg: 120 | children.append(gen_identifier(node.kwarg.arg, 'kwarg', node.kwarg)) 121 | elif isinstance(node, ast.ExceptHandler): 122 | if node.type: 123 | children.append(traverse_list([node.type], 'type', node)) 124 | children.append(traverse_list(node.body, 'body', node)) 125 | elif isinstance(node, ast.ClassDef): 126 | children.append(traverse_list(node.bases, 'bases', node)) 127 | children.append(traverse_list(node.body, 'body', node)) 128 | children.append(traverse_list(node.decorator_list, 'decorator_list', node)) 129 | elif isinstance(node, ast.FunctionDef): 130 | children.append(traverse(node.args)) 131 | children.append(traverse_list(node.body, 'body', node)) 132 | children.append(traverse_list(node.decorator_list, 'decorator_list', node)) 133 | else: 134 | # Default handling: iterate over children. 135 | for child in ast.iter_child_nodes(node): 136 | if isinstance(child, ast.expr_context) or isinstance(child, ast.operator) \ 137 | or isinstance(child, ast.boolop) or isinstance(child, ast.unaryop) \ 138 | or isinstance(child, ast.cmpop): 139 | # Directly include expr_context, and operators into the type instead of creating a child. 140 | json_node['type'] = json_node['type'] + type(child).__name__ 141 | else: 142 | children.append(traverse(child)) 143 | 144 | if isinstance(node, ast.Attribute): 145 | children.append(gen_identifier(node.attr, 'attr', node)) 146 | 147 | if (len(children) != 0): 148 | json_node['children'] = children 149 | 150 | return pos 151 | 152 | traverse(tree) 153 | return json_tree 154 | 155 | 156 | def json2xml(tree): 157 | lines = [] 158 | 159 | def convert_node(i, indent_level=0): 160 | node = tree[i] 161 | line = "\t" * indent_level + "<{}".format(node['type']) 162 | for key in ['value', 'lineno', 'col', 'end_line_no', 'end_col']: 163 | if key in node: 164 | line += (' {}={}'.format(key, quoteattr(str(node[key])))) 165 | line += ">" 166 | lines.append(line) 167 | if "children" in node: 168 | for child in node["children"]: 169 | convert_node(int(child), indent_level + 1) 170 | lines.append("\t" * indent_level + "") 171 | return lines 172 | 173 | return "\n".join(convert_node(0)) 174 | 175 | 176 | def parse(filename): 177 | try: 178 | json_tree = parse_file(filename) 179 | return json2xml(json_tree) 180 | 181 | except (UnicodeEncodeError, UnicodeDecodeError): 182 | pass 183 | 184 | 185 | if __name__ == "__main__": 186 | default_filename = 'src1.py' 187 | filename = sys.argv[1] if len(sys.argv) > 1 else default_filename 188 | 189 | json_tree = parse_file(filename) 190 | xml = json2xml(json_tree) 191 | 192 | print(xml) 193 | -------------------------------------------------------------------------------- /external/pythonparser_3.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.8 2 | 3 | # Copyright (c) Raychev, V., Bielik, P., and Vechev, M. (see https://eth-sri.github.io/py150) 4 | # Copyright (c) Victor Quach (see https://github.com/Varal7/pythonparser) 5 | # Copyright (c) Aniskov N. 6 | 7 | 8 | import argparse 9 | import ast 10 | import json as json 11 | from typing import Dict, List, Union 12 | from xml.sax.saxutils import quoteattr 13 | 14 | import asttokens 15 | 16 | JsonNodeType = Dict[str, Union[str, List[int]]] 17 | 18 | 19 | def read_file_to_string(filename: str) -> str: 20 | """ 21 | :param filename: path to file 22 | :return: string with file content 23 | """ 24 | with open(filename, 'rt') as f: 25 | content = f.read() 26 | return content 27 | 28 | 29 | def parse_file(filename: str) -> List[JsonNodeType]: 30 | """ 31 | Produces XML in special for GumTree 2.x library 32 | format -- merge value_type and import_level into node type 33 | (Example: in standard format will become 34 | in that format) 35 | 36 | :param filename: file with python3 code to be parsed 37 | XML in special for GumTree 2.x library 38 | format -- merge value_type and import_level into node type 39 | (Example: in standard format will become 40 | in that format) 41 | :return: tree in json format 42 | """ 43 | tree = asttokens.ASTTokens(read_file_to_string(filename), parse=True).tree 44 | 45 | json_tree = [] 46 | 47 | def localize(py_node: ast.AST, json_node: JsonNodeType) -> None: 48 | if py_node is None: 49 | return 50 | 51 | location_attributes = (['first_token', 'last_token'], 52 | ['lineno', 'col_offset', 'end_lineno', 'end_col_offset']) 53 | if all(hasattr(py_node, location_attr) for location_attr in location_attributes[0]): 54 | json_node['lineno'] = str(py_node.first_token.start[0]) 55 | json_node['col'] = str(py_node.first_token.start[1]) 56 | json_node['end_line_no'] = str(py_node.last_token.end[0]) 57 | json_node['end_col'] = str(py_node.last_token.end[1]) 58 | elif all(hasattr(py_node, location_attr) for location_attr in location_attributes[1]): 59 | json_node['lineno'] = str(py_node.lineno) 60 | json_node['col'] = str(py_node.col_offset) 61 | json_node['end_line_no'] = str(py_node.end_lineno) 62 | json_node['end_col'] = str(py_node.end_col_offset) 63 | else: 64 | raise RuntimeError(f'Failed to localize {type(py_node).__name__} node. ' 65 | f'Not enough location attributes for localization') 66 | 67 | def gen_identifier(identifier: str, node_type: str = 'identifier', py_node: ast.AST = None) -> int: 68 | pos = len(json_tree) 69 | json_node = {} 70 | json_tree.append(json_node) 71 | json_node['type'] = node_type 72 | json_node['value'] = identifier 73 | localize(py_node, json_node) 74 | return pos 75 | 76 | def traverse_list(py_ast_nodes: List[ast.AST], node_type: str = 'list', py_node: ast.AST = None) -> int: 77 | pos = len(json_tree) 78 | json_node = {} 79 | json_tree.append(json_node) 80 | json_node['type'] = node_type 81 | localize(py_node, json_node) 82 | children = [] 83 | for item in py_ast_nodes: 84 | children.append(traverse(item)) 85 | if len(children) != 0: 86 | json_node['children'] = children 87 | return pos 88 | 89 | def create_child(py_node_child: ast.AST, node_type: str, py_node: ast.AST = None): 90 | pos = len(json_tree) 91 | json_node = {} 92 | json_tree.append(json_node) 93 | json_node['type'] = node_type 94 | localize(py_node, json_node) 95 | if py_node_child is not None: 96 | json_node['children'] = [traverse(py_node_child)] 97 | return pos 98 | 99 | def traverse(py_node: ast.AST) -> Union[JsonNodeType, int]: 100 | pos = len(json_tree) 101 | json_node = {} 102 | json_tree.append(json_node) 103 | json_node['type'] = type(py_node).__name__ 104 | localize(py_node, json_node) 105 | children = [] 106 | 107 | if isinstance(py_node, ast.Name): 108 | json_node['value'] = py_node.id 109 | 110 | elif isinstance(py_node, ast.NameConstant): 111 | json_node['value'] = py_node.value 112 | json_node['type'] += '-' + type(py_node.value).__name__ 113 | 114 | elif isinstance(py_node, ast.Constant): 115 | json_node['value'] = py_node.value 116 | json_node['type'] += '-' + type(py_node.value).__name__ 117 | 118 | elif isinstance(py_node, ast.Num): 119 | json_node['value'] = py_node.n 120 | json_node['type'] += '-' + type(py_node.n).__name__ 121 | 122 | elif isinstance(py_node, ast.Str): 123 | json_node['value'] = py_node.s 124 | 125 | elif isinstance(py_node, ast.alias): 126 | json_node['value'] = py_node.name 127 | if py_node.asname: 128 | children.append(gen_identifier(py_node.asname, py_node=py_node)) 129 | 130 | elif isinstance(py_node, (ast.FunctionDef, ast.AsyncFunctionDef)): 131 | json_node['value'] = py_node.name 132 | 133 | elif isinstance(py_node, ast.ExceptHandler): 134 | if py_node.name: 135 | json_node['value'] = py_node.name 136 | 137 | elif isinstance(py_node, ast.ClassDef): 138 | json_node['value'] = py_node.name 139 | 140 | elif isinstance(py_node, ast.ImportFrom): 141 | if py_node.module: 142 | json_node['value'] = py_node.module 143 | json_node['type'] += '-' + str(py_node.level) 144 | 145 | elif isinstance(py_node, (ast.Global, ast.Nonlocal)): 146 | for n in py_node.names: 147 | children.append(gen_identifier(n, py_node=py_node)) 148 | 149 | elif isinstance(py_node, ast.keyword): 150 | json_node['value'] = py_node.arg 151 | json_node['type'] += '-' + type(py_node.arg).__name__ 152 | 153 | elif isinstance(py_node, ast.arg): 154 | json_node['value'] = py_node.arg 155 | 156 | # Process children. 157 | if isinstance(py_node, (ast.For, ast.AsyncFor)): 158 | children.append(traverse(py_node.target)) 159 | children.append(traverse(py_node.iter)) 160 | children.append(traverse_list(py_node.body, 'body', py_node)) 161 | if py_node.orelse: 162 | children.append(traverse_list(py_node.orelse, 'orelse', py_node)) 163 | 164 | elif isinstance(py_node, ast.If) or isinstance(py_node, ast.While): 165 | children.append(traverse(py_node.test)) 166 | children.append(traverse_list(py_node.body, 'body', py_node)) 167 | if py_node.orelse: 168 | children.append(traverse_list(py_node.orelse, 'orelse', py_node)) 169 | 170 | elif isinstance(py_node, (ast.With, ast.AsyncWith)): 171 | children.append(traverse_list(py_node.items, 'items', py_node)) 172 | children.append(traverse_list(py_node.body, 'body', py_node)) 173 | 174 | elif isinstance(py_node, ast.withitem): 175 | children.append(traverse(py_node.context_expr)) 176 | if py_node.optional_vars: 177 | children.append(traverse(py_node.optional_vars)) 178 | 179 | elif isinstance(py_node, ast.Try): 180 | children.append(traverse_list(py_node.body, 'body', py_node)) 181 | children.append(traverse_list(py_node.handlers, 'handlers', py_node)) 182 | if py_node.orelse: 183 | children.append(traverse_list(py_node.orelse, 'orelse', py_node)) 184 | if py_node.finalbody: 185 | children.append(traverse_list(py_node.finalbody, 'finalbody', py_node)) 186 | 187 | elif isinstance(py_node, ast.arguments): 188 | children.append(traverse_list(py_node.posonlyargs, 'posonlyargs', py_node)) 189 | children.append(traverse_list(py_node.args, 'args', py_node)) 190 | children.append(traverse_list(py_node.kwonlyargs, 'kwonlyargs', py_node)) 191 | children.append(traverse_list(py_node.kw_defaults, 'kw_defaults', py_node)) 192 | children.append(traverse_list(py_node.defaults, 'defaults', py_node)) 193 | if py_node.vararg: 194 | children.append(gen_identifier(py_node.vararg.arg, 'vararg', py_node.vararg)) 195 | if py_node.kwarg: 196 | children.append(gen_identifier(py_node.kwarg.arg, 'kwarg', py_node.kwarg)) 197 | 198 | elif isinstance(py_node, ast.ExceptHandler): 199 | if py_node.type: 200 | children.append(traverse_list([py_node.type], 'type', py_node)) 201 | children.append(traverse_list(py_node.body, 'body', py_node)) 202 | 203 | elif isinstance(py_node, ast.ClassDef): 204 | children.append(traverse_list(py_node.bases, 'bases', py_node)) 205 | children.append(traverse_list(py_node.keywords, 'keywords', py_node)) 206 | children.append(traverse_list(py_node.body, 'body', py_node)) 207 | children.append(traverse_list(py_node.decorator_list, 'decorator_list', py_node)) 208 | 209 | elif isinstance(py_node, (ast.FunctionDef, ast.AsyncFunctionDef)): 210 | children.append(traverse(py_node.args)) 211 | children.append(traverse_list(py_node.body, 'body', py_node)) 212 | children.append(traverse_list(py_node.decorator_list, 'decorator_list', py_node)) 213 | 214 | elif isinstance(py_node, ast.Slice): 215 | children.append(create_child(py_node.lower, 'lower', py_node)) 216 | children.append(create_child(py_node.step, 'step', py_node)) 217 | children.append(create_child(py_node.upper, 'upper', py_node)) 218 | 219 | elif py_node is not None: 220 | # Default handling: iterate over children. 221 | for child in ast.iter_child_nodes(py_node): 222 | if isinstance(child, ast.expr_context) or\ 223 | isinstance(child, ast.operator) or\ 224 | isinstance(child, ast.boolop) or\ 225 | isinstance(child, ast.unaryop) or\ 226 | isinstance(child, ast.cmpop): 227 | # Directly include expr_context, and operators into the type instead of creating a child. 228 | json_node['type'] = json_node['type'] + '_' + type(child).__name__ 229 | else: 230 | children.append(traverse(child)) 231 | 232 | if isinstance(py_node, ast.Attribute): 233 | children.append(gen_identifier(py_node.attr, 'attr', py_node)) 234 | 235 | if len(children) != 0: 236 | json_node['children'] = children 237 | 238 | return pos 239 | 240 | traverse(tree) 241 | return json_tree 242 | 243 | 244 | def json2xml(tree: List[JsonNodeType]) -> str: 245 | """ 246 | :param tree: tree in json format produced by parser 247 | :type tree: list of dicts 248 | :return: xml file as string 249 | """ 250 | lines = [] 251 | 252 | def convert_node(i: int, indent_level: int = 0) -> List[str]: 253 | node = tree[i] 254 | line = '\t' * indent_level + '<{}'.format(node['type']) 255 | for key in ['value', 'lineno', 'col', 'end_line_no', 'end_col']: 256 | if key in node: 257 | line += (' {}={}'.format(key, quoteattr(str(node[key])))) 258 | line += '>' 259 | lines.append(line) 260 | if 'children' in node: 261 | for child in node['children']: 262 | convert_node(int(child), indent_level + 1) 263 | lines.append('\t' * indent_level + '') 264 | return lines 265 | 266 | return '\n'.join(convert_node(0)) 267 | 268 | 269 | if __name__ == '__main__': 270 | parser = argparse.ArgumentParser(description='Parse python3 file') 271 | parser.add_argument('filename', type=str, help='Filename') 272 | parser.add_argument('-f', '--format', choices=['xml', 'json'], help='Print format', default='xml') 273 | 274 | args = parser.parse_args() 275 | 276 | parsed_json_tree = parse_file(args.filename) 277 | 278 | if args.format == 'json': 279 | print(json.dumps(parsed_json_tree, separators=(',', ':'), ensure_ascii=False)) 280 | else: 281 | xml = json2xml(parsed_json_tree) 282 | print(xml) 283 | 284 | 285 | 286 | -------------------------------------------------------------------------------- /log/__init__.py: -------------------------------------------------------------------------------- 1 | import log.logger as lgr 2 | 3 | logger = lgr.CustomLogger() 4 | -------------------------------------------------------------------------------- /log/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | import os 4 | import functools 5 | 6 | import settings 7 | 8 | 9 | class CustomLogger: 10 | WARNING = logging.WARNING 11 | ERROR = logging.ERROR 12 | INFO = logging.INFO 13 | DEBUG = logging.DEBUG 14 | 15 | FILE_PATH = settings.get('logger_file_path', 'app.log') 16 | FILE_LOG_LEVEL = getattr(logging, settings.get('logger_file_log_level', 'INFO').upper()) 17 | STDOUT_LOG_LEVEL = getattr(logging, settings.get('logger_stdout_log_level', 'WARNING').upper()) 18 | 19 | def __init__(self): 20 | self._logger = None 21 | self._setup() 22 | 23 | self.error = functools.partial(self.log, self.ERROR) 24 | self.warning = functools.partial(self.log, self.WARNING) 25 | self.info = functools.partial(self.log, self.INFO) 26 | self.debug = functools.partial(self.log, self.DEBUG) 27 | 28 | def _setup(self): 29 | self._logger = logging.getLogger() 30 | self._logger.setLevel(logging.WARNING) 31 | 32 | formatter = logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s', datefmt='%d.%m.%Y %H:%M:%S') 33 | 34 | fh = logging.FileHandler(self.FILE_PATH) 35 | fh.setFormatter(formatter) 36 | fh.setLevel(self.FILE_LOG_LEVEL) 37 | self._logger.addHandler(fh) 38 | 39 | sh = logging.StreamHandler() 40 | sh.setFormatter(formatter) 41 | sh.setLevel(self.STDOUT_LOG_LEVEL) 42 | self._logger.addHandler(sh) 43 | self._logger.setLevel(min(self.FILE_LOG_LEVEL, self.STDOUT_LOG_LEVEL)) 44 | 45 | def log(self, level, text, exc_info=False, start_time=None, show_pid=False): 46 | if start_time: 47 | text = f'{text} {int((time.time() - start_time) * 1000)}ms' 48 | if show_pid: 49 | text = f'pid{os.getpid()}: {text}' 50 | 51 | self._logger.log(level, f'{text} ', exc_info=exc_info) 52 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import os 3 | import pickle 4 | import sys 5 | import stackimpact 6 | import datetime 7 | import argparse 8 | import multiprocessing 9 | 10 | from log import logger 11 | from patterns import Miner 12 | from patterns.models import Fragment, Pattern 13 | from vcs.traverse import GitAnalyzer, RepoInfo, Method 14 | from deployment import set_all_environment_variables 15 | 16 | import pyflowgraph 17 | import changegraph 18 | import settings 19 | 20 | 21 | class RunModes: 22 | BUILD_PY_FLOW_GRAPH = 'pfg' 23 | BUILD_CHANGE_GRAPH = 'cg' 24 | COLLECT_CHANGE_GRAPHS = 'collect-cgs' 25 | MINE_PATTERNS = 'patterns' 26 | ALL = [BUILD_PY_FLOW_GRAPH, BUILD_CHANGE_GRAPH, COLLECT_CHANGE_GRAPHS, MINE_PATTERNS] 27 | 28 | 29 | def main(): 30 | set_all_environment_variables() 31 | 32 | logger.info('------------------------------ Starting ------------------------------') 33 | 34 | if settings.get('use_stackimpact', required=False): 35 | _ = stackimpact.start( 36 | agent_key=settings.get('stackimpact_agent_key'), 37 | app_name='CodeChangesMiner', 38 | debug=True, 39 | app_version=str(datetime.datetime.now()) 40 | ) 41 | 42 | sys.setrecursionlimit(2 ** 31 - 1) 43 | multiprocessing.set_start_method('spawn', force=True) 44 | 45 | parser = argparse.ArgumentParser() 46 | parser.add_argument('mode', help=f'One of {RunModes.ALL}', type=str) 47 | args, _ = parser.parse_known_args() 48 | 49 | current_mode = args.mode 50 | 51 | if current_mode == RunModes.BUILD_PY_FLOW_GRAPH: 52 | parser.add_argument('-i', '--input', help='Path to source code file', type=str, required=True) 53 | parser.add_argument('-o', '--output', help='Path to output file', type=str, default='pyflowgraph.dot') 54 | parser.add_argument('--no-closure', action='store_true') 55 | parser.add_argument('--show-deps', action='store_true') 56 | parser.add_argument('--hide-op-kinds', action='store_true') 57 | parser.add_argument('--show-data-keys', action='store_true') 58 | args = parser.parse_args() 59 | 60 | fg = pyflowgraph.build_from_file( 61 | args.input, show_dependencies=args.show_deps, build_closure=not args.no_closure) 62 | pyflowgraph.export_graph_image( 63 | fg, args.output, show_op_kinds=not args.hide_op_kinds, show_data_keys=args.show_data_keys) 64 | elif current_mode == RunModes.BUILD_CHANGE_GRAPH: 65 | parser.add_argument('-s', '--src', help='Path to source code before changes', type=str, required=True) 66 | parser.add_argument('-d', '--dest', help='Path to source code after changes', type=str, required=True) 67 | parser.add_argument('-o', '--output', help='Path to output file', type=str, default='changegraph.dot') 68 | args = parser.parse_args() 69 | 70 | fg = changegraph.build_from_files(args.src, args.dest) 71 | changegraph.export_graph_image(fg, args.output) 72 | elif current_mode == RunModes.COLLECT_CHANGE_GRAPHS: 73 | parser.add_argument('--only-tests', 74 | help='Collect cgs only for the files with "test" substring in the name', 75 | action='store_true') 76 | args = parser.parse_args() 77 | 78 | GitAnalyzer().build_change_graphs(args.only_tests) 79 | elif current_mode == RunModes.MINE_PATTERNS: 80 | parser.add_argument('-s', '--src', help='Path to source code before changes', type=str, nargs='+') 81 | parser.add_argument('-d', '--dest', help='Path to source code after changes', type=str, nargs='+') 82 | parser.add_argument('--fake-mining', action='store_true') 83 | args = parser.parse_args() 84 | 85 | if args.src or args.dest or args.fake_mining: 86 | if not args.src or len(args.src) != len(args.dest): 87 | raise ValueError('src and dest have different size or unset') 88 | 89 | change_graphs = [] 90 | for old_path, new_path in zip(args.src, args.dest): 91 | methods = [] 92 | for n, path in enumerate([old_path, new_path]): 93 | with open(path, 'r+') as f: 94 | src = f.read() 95 | methods.append(Method(path, 'test_name', ast.parse(src, mode='exec').body[0], src)) 96 | 97 | mock_commit_dtm = datetime.datetime.now(tz=datetime.timezone.utc) 98 | repo_info = RepoInfo( 99 | 'mock repo path', 'mock repo name', 'mock repo url', 'mock hash', mock_commit_dtm, 100 | 'mock old file path', 'mock new file path', methods[0], methods[1]) 101 | 102 | cg = changegraph.build_from_files(old_path, new_path, repo_info=repo_info) 103 | change_graphs.append(cg) 104 | 105 | miner = Miner() 106 | if args.fake_mining: 107 | for cg in change_graphs: 108 | fragment = Fragment() 109 | fragment.graph = cg 110 | fragment.nodes = cg.nodes 111 | pattern = Pattern([fragment]) 112 | miner.add_pattern(pattern) 113 | else: 114 | miner.mine_patterns(change_graphs) 115 | miner.print_patterns() 116 | else: 117 | storage_dir = settings.get('change_graphs_storage_dir') 118 | file_names = os.listdir(storage_dir) 119 | 120 | logger.warning(f'Found {len(file_names)} files in storage directory') 121 | 122 | change_graphs = [] 123 | for file_num, file_name in enumerate(file_names): 124 | file_path = os.path.join(storage_dir, file_name) 125 | try: 126 | with open(file_path, 'rb') as f: 127 | graphs = pickle.load(f) 128 | 129 | for graph in graphs: 130 | change_graphs.append(pickle.loads(graph)) 131 | except: 132 | logger.warning(f'Incorrect file {file_path}') 133 | 134 | if file_num % 1000 == 0: 135 | logger.warning(f'Loaded [{1 + file_num}/{len(file_names)}] files') 136 | logger.warning('Pattern mining has started') 137 | 138 | miner = Miner() 139 | try: 140 | miner.mine_patterns(change_graphs) 141 | except KeyboardInterrupt: 142 | logger.warning('KeyboardInterrupt: mined patterns will be stored before exit') 143 | 144 | miner.print_patterns() 145 | else: 146 | raise ValueError 147 | 148 | 149 | if __name__ == '__main__': 150 | main() 151 | -------------------------------------------------------------------------------- /output/general.js: -------------------------------------------------------------------------------- 1 | $(document).ready(function() { 2 | $('[data-target="visibility"]').hide(); 3 | 4 | $('[data-action="copy"]').click(function () { 5 | var target = $('[data-target="copy"]', $(this).parent()); 6 | selectText(target[0]); 7 | document.execCommand('copy'); 8 | window.getSelection().removeAllRanges(); 9 | alert('Copied'); 10 | }); 11 | 12 | $('[data-action="visibility"]').click(function () { 13 | var target = $('[data-target="visibility"]', $(this).parent()); 14 | target.toggle(); 15 | }); 16 | 17 | function selectText(element) { 18 | let selection = window.getSelection(); 19 | let range = document.createRange(); 20 | range.selectNodeContents(element); 21 | selection.removeAllRanges(); 22 | selection.addRange(range); 23 | } 24 | }); 25 | -------------------------------------------------------------------------------- /output/libs/highlight/default.css: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Original highlight.js style (c) Ivan Sagalaev 4 | 5 | */ 6 | 7 | .hljs { 8 | display: block; 9 | overflow-x: auto; 10 | padding: 0.5em; 11 | background: #F0F0F0; 12 | } 13 | 14 | 15 | /* Base color: saturation 0; */ 16 | 17 | .hljs, 18 | .hljs-subst { 19 | color: #444; 20 | opacity: .4; 21 | } 22 | 23 | .hljs-comment { 24 | color: #888888; 25 | opacity: .4; 26 | } 27 | 28 | .hljs-keyword, 29 | .hljs-attribute, 30 | .hljs-selector-tag, 31 | .hljs-meta-keyword, 32 | .hljs-doctag, 33 | .hljs-name { 34 | font-weight: bold; 35 | } 36 | 37 | 38 | /* User color: hue: 0 */ 39 | 40 | .hljs-type, 41 | .hljs-string, 42 | .hljs-number, 43 | .hljs-selector-id, 44 | .hljs-selector-class, 45 | .hljs-quote, 46 | .hljs-template-tag, 47 | .hljs-deletion { 48 | color: #880000; 49 | opacity: .4; 50 | } 51 | 52 | .hljs-title, 53 | .hljs-section { 54 | color: #880000; 55 | font-weight: bold; 56 | opacity: .4; 57 | } 58 | 59 | .hljs-regexp, 60 | .hljs-symbol, 61 | .hljs-variable, 62 | .hljs-template-variable, 63 | .hljs-link, 64 | .hljs-selector-attr, 65 | .hljs-selector-pseudo { 66 | color: #BC6060; 67 | opacity: .4; 68 | } 69 | 70 | 71 | /* Language color: hue: 90; */ 72 | 73 | .hljs-literal { 74 | color: #386900; 75 | opacity: .4; 76 | } 77 | 78 | .hljs-built_in, 79 | .hljs-bullet, 80 | .hljs-code, 81 | .hljs-addition { 82 | color: #397300; 83 | opacity: .4; 84 | } 85 | 86 | 87 | /* Meta color: hue: 200 */ 88 | 89 | .hljs-meta { 90 | color: #1f7199; 91 | opacity: .4; 92 | } 93 | 94 | .hljs-meta-string { 95 | color: #4d99bf; 96 | opacity: .4; 97 | } 98 | 99 | 100 | /* Misc effects */ 101 | 102 | .hljs-emphasis { 103 | font-style: italic; 104 | } 105 | 106 | .hljs-strong { 107 | font-weight: bold; 108 | } 109 | -------------------------------------------------------------------------------- /output/libs/highlight/highlight.pack.js: -------------------------------------------------------------------------------- 1 | /*! highlight.js v9.18.1 | BSD3 License | git.io/hljslicense */ 2 | !function(e){var n="object"==typeof window&&window||"object"==typeof self&&self;"undefined"==typeof exports||exports.nodeType?n&&(n.hljs=e({}),"function"==typeof define&&define.amd&&define([],function(){return n.hljs})):e(exports)}(function(a){var f=[],i=Object.keys,_={},c={},C=!0,n=/^(no-?highlight|plain|text)$/i,l=/\blang(?:uage)?-([\w-]+)\b/i,t=/((^(<[^>]+>|\t|)+|(?:\n)))/gm,r={case_insensitive:"cI",lexemes:"l",contains:"c",keywords:"k",subLanguage:"sL",className:"cN",begin:"b",beginKeywords:"bK",end:"e",endsWithParent:"eW",illegal:"i",excludeBegin:"eB",excludeEnd:"eE",returnBegin:"rB",returnEnd:"rE",variants:"v",IDENT_RE:"IR",UNDERSCORE_IDENT_RE:"UIR",NUMBER_RE:"NR",C_NUMBER_RE:"CNR",BINARY_NUMBER_RE:"BNR",RE_STARTERS_RE:"RSR",BACKSLASH_ESCAPE:"BE",APOS_STRING_MODE:"ASM",QUOTE_STRING_MODE:"QSM",PHRASAL_WORDS_MODE:"PWM",C_LINE_COMMENT_MODE:"CLCM",C_BLOCK_COMMENT_MODE:"CBCM",HASH_COMMENT_MODE:"HCM",NUMBER_MODE:"NM",C_NUMBER_MODE:"CNM",BINARY_NUMBER_MODE:"BNM",CSS_NUMBER_MODE:"CSSNM",REGEXP_MODE:"RM",TITLE_MODE:"TM",UNDERSCORE_TITLE_MODE:"UTM",COMMENT:"C",beginRe:"bR",endRe:"eR",illegalRe:"iR",lexemesRe:"lR",terminators:"t",terminator_end:"tE"},m="",O="Could not find the language '{}', did you forget to load/include a language module?",B={classPrefix:"hljs-",tabReplace:null,useBR:!1,languages:void 0},o="of and for in not or if then".split(" ");function x(e){return e.replace(/&/g,"&").replace(//g,">")}function g(e){return e.nodeName.toLowerCase()}function u(e){return n.test(e)}function s(e){var n,t={},r=Array.prototype.slice.call(arguments,1);for(n in e)t[n]=e[n];return r.forEach(function(e){for(n in e)t[n]=e[n]}),t}function E(e){var a=[];return function e(n,t){for(var r=n.firstChild;r;r=r.nextSibling)3===r.nodeType?t+=r.nodeValue.length:1===r.nodeType&&(a.push({event:"start",offset:t,node:r}),t=e(r,t),g(r).match(/br|hr|img|input/)||a.push({event:"stop",offset:t,node:r}));return t}(e,0),a}function d(e,n,t){var r=0,a="",i=[];function o(){return e.length&&n.length?e[0].offset!==n[0].offset?e[0].offset"}function l(e){a+=""}function u(e){("start"===e.event?c:l)(e.node)}for(;e.length||n.length;){var s=o();if(a+=x(t.substring(r,s[0].offset)),r=s[0].offset,s===e){for(i.reverse().forEach(l);u(s.splice(0,1)[0]),(s=o())===e&&s.length&&s[0].offset===r;);i.reverse().forEach(c)}else"start"===s[0].event?i.push(s[0].node):i.pop(),u(s.splice(0,1)[0])}return a+x(t.substr(r))}function R(n){return n.v&&!n.cached_variants&&(n.cached_variants=n.v.map(function(e){return s(n,{v:null},e)})),n.cached_variants?n.cached_variants:function e(n){return!!n&&(n.eW||e(n.starts))}(n)?[s(n,{starts:n.starts?s(n.starts):null})]:Object.isFrozen(n)?[s(n)]:[n]}function p(e){if(r&&!e.langApiRestored){for(var n in e.langApiRestored=!0,r)e[n]&&(e[r[n]]=e[n]);(e.c||[]).concat(e.v||[]).forEach(p)}}function v(n,r){var a={};return"string"==typeof n?t("keyword",n):i(n).forEach(function(e){t(e,n[e])}),a;function t(t,e){r&&(e=e.toLowerCase()),e.split(" ").forEach(function(e){var n=e.split("|");a[n[0]]=[t,function(e,n){return n?Number(n):function(e){return-1!=o.indexOf(e.toLowerCase())}(e)?0:1}(n[0],n[1])]})}}function S(r){function s(e){return e&&e.source||e}function f(e,n){return new RegExp(s(e),"m"+(r.cI?"i":"")+(n?"g":""))}function a(a){var i,e,o={},c=[],l={},t=1;function n(e,n){o[t]=e,c.push([e,n]),t+=function(e){return new RegExp(e.toString()+"|").exec("").length-1}(n)+1}for(var r=0;r')+n+(t?"":m)}function l(){p+=null!=d.sL?function(){var e="string"==typeof d.sL;if(e&&!_[d.sL])return x(v);var n=e?T(d.sL,v,!0,R[d.sL]):w(v,d.sL.length?d.sL:void 0);return 0")+'"');if("end"===n.type){var r=function(e){var n=e[0],t=i.substr(e.index),r=o(d,t);if(r){var a=d;for(a.skip?v+=n:(a.rE||a.eE||(v+=n),l(),a.eE&&(v=n));d.cN&&(p+=m),d.skip||d.sL||(M+=d.relevance),(d=d.parent)!==r.parent;);return r.starts&&(r.endSameAsBegin&&(r.starts.eR=r.eR),u(r.starts)),a.rE?0:n.length}}(n);if(null!=r)return r}return v+=t,t.length}var g=D(n);if(!g)throw console.error(O.replace("{}",n)),new Error('Unknown language: "'+n+'"');S(g);var E,d=t||g,R={},p="";for(E=d;E!==g;E=E.parent)E.cN&&(p=c(E.cN,"",!0)+p);var v="",M=0;try{for(var b,h,N=0;d.t.lastIndex=N,b=d.t.exec(i);)h=r(i.substring(N,b.index),b),N=b.index+h;for(r(i.substr(N)),E=d;E.parent;E=E.parent)E.cN&&(p+=m);return{relevance:M,value:p,i:!1,language:n,top:d}}catch(e){if(e.message&&-1!==e.message.indexOf("Illegal"))return{i:!0,relevance:0,value:x(i)};if(C)return{relevance:0,value:x(i),language:n,top:d,errorRaised:e};throw e}}function w(t,e){e=e||B.languages||i(_);var r={relevance:0,value:x(t)},a=r;return e.filter(D).filter(L).forEach(function(e){var n=T(e,t,!1);n.language=e,n.relevance>a.relevance&&(a=n),n.relevance>r.relevance&&(a=r,r=n)}),a.language&&(r.second_best=a),r}function M(e){return B.tabReplace||B.useBR?e.replace(t,function(e,n){return B.useBR&&"\n"===e?"
":B.tabReplace?n.replace(/\t/g,B.tabReplace):""}):e}function b(e){var n,t,r,a,i,o=function(e){var n,t,r,a,i=e.className+" ";if(i+=e.parentNode?e.parentNode.className:"",t=l.exec(i)){var o=D(t[1]);return o||(console.warn(O.replace("{}",t[1])),console.warn("Falling back to no-highlight mode for this block.",e)),o?t[1]:"no-highlight"}for(n=0,r=(i=i.split(/\s+/)).length;n/g,"\n"):n=e,i=n.textContent,r=o?T(o,i,!0):w(i),(t=E(n)).length&&((a=document.createElement("div")).innerHTML=r.value,r.value=d(t,E(a),i)),r.value=M(r.value),e.innerHTML=r.value,e.className=function(e,n,t){var r=n?c[n]:t,a=[e.trim()];return e.match(/\bhljs\b/)||a.push("hljs"),-1===e.indexOf(r)&&a.push(r),a.join(" ").trim()}(e.className,o,r.language),e.result={language:r.language,re:r.relevance},r.second_best&&(e.second_best={language:r.second_best.language,re:r.second_best.relevance}))}function h(){if(!h.called){h.called=!0;var e=document.querySelectorAll("pre code");f.forEach.call(e,b)}}var N={disableAutodetect:!0};function D(e){return e=(e||"").toLowerCase(),_[e]||_[c[e]]}function L(e){var n=D(e);return n&&!n.disableAutodetect}return a.highlight=T,a.highlightAuto=w,a.fixMarkup=M,a.highlightBlock=b,a.configure=function(e){B=s(B,e)},a.initHighlighting=h,a.initHighlightingOnLoad=function(){window.addEventListener("DOMContentLoaded",h,!1),window.addEventListener("load",h,!1)},a.registerLanguage=function(n,e){var t;try{t=e(a)}catch(e){if(console.error("Language definition for '{}' could not be registered.".replace("{}",n)),!C)throw e;console.error(e),t=N}p(_[n]=t),t.rawDefinition=e.bind(null,a),t.aliases&&t.aliases.forEach(function(e){c[e]=n})},a.listLanguages=function(){return i(_)},a.getLanguage=D,a.requireLanguage=function(e){var n=D(e);if(n)return n;throw new Error("The '{}' language is required, but not loaded.".replace("{}",e))},a.autoDetection=L,a.inherit=s,a.debugMode=function(){C=!1},a.IR=a.IDENT_RE="[a-zA-Z]\\w*",a.UIR=a.UNDERSCORE_IDENT_RE="[a-zA-Z_]\\w*",a.NR=a.NUMBER_RE="\\b\\d+(\\.\\d+)?",a.CNR=a.C_NUMBER_RE="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",a.BNR=a.BINARY_NUMBER_RE="\\b(0b[01]+)",a.RSR=a.RE_STARTERS_RE="!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",a.BE=a.BACKSLASH_ESCAPE={b:"\\\\[\\s\\S]",relevance:0},a.ASM=a.APOS_STRING_MODE={cN:"string",b:"'",e:"'",i:"\\n",c:[a.BE]},a.QSM=a.QUOTE_STRING_MODE={cN:"string",b:'"',e:'"',i:"\\n",c:[a.BE]},a.PWM=a.PHRASAL_WORDS_MODE={b:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/},a.C=a.COMMENT=function(e,n,t){var r=a.inherit({cN:"comment",b:e,e:n,c:[]},t||{});return r.c.push(a.PWM),r.c.push({cN:"doctag",b:"(?:TODO|FIXME|NOTE|BUG|XXX):",relevance:0}),r},a.CLCM=a.C_LINE_COMMENT_MODE=a.C("//","$"),a.CBCM=a.C_BLOCK_COMMENT_MODE=a.C("/\\*","\\*/"),a.HCM=a.HASH_COMMENT_MODE=a.C("#","$"),a.NM=a.NUMBER_MODE={cN:"number",b:a.NR,relevance:0},a.CNM=a.C_NUMBER_MODE={cN:"number",b:a.CNR,relevance:0},a.BNM=a.BINARY_NUMBER_MODE={cN:"number",b:a.BNR,relevance:0},a.CSSNM=a.CSS_NUMBER_MODE={cN:"number",b:a.NR+"(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",relevance:0},a.RM=a.REGEXP_MODE={cN:"regexp",b:/\//,e:/\/[gimuy]*/,i:/\n/,c:[a.BE,{b:/\[/,e:/\]/,relevance:0,c:[a.BE]}]},a.TM=a.TITLE_MODE={cN:"title",b:a.IR,relevance:0},a.UTM=a.UNDERSCORE_TITLE_MODE={cN:"title",b:a.UIR,relevance:0},a.METHOD_GUARD={b:"\\.\\s*"+a.UIR,relevance:0},[a.BE,a.ASM,a.QSM,a.PWM,a.C,a.CLCM,a.CBCM,a.HCM,a.NM,a.CNM,a.BNM,a.CSSNM,a.RM,a.TM,a.UTM,a.METHOD_GUARD].forEach(function(e){!function n(t){Object.freeze(t);var r="function"==typeof t;Object.getOwnPropertyNames(t).forEach(function(e){!t.hasOwnProperty(e)||null===t[e]||"object"!=typeof t[e]&&"function"!=typeof t[e]||r&&("caller"===e||"callee"===e||"arguments"===e)||Object.isFrozen(t[e])||n(t[e])});return t}(e)}),a});hljs.registerLanguage("python",function(e){var r={keyword:"and elif is global as in if from raise for except finally print import pass return exec else break not with class assert yield try while continue del or def lambda async await nonlocal|10",built_in:"Ellipsis NotImplemented",literal:"False None True"},b={cN:"meta",b:/^(>>>|\.\.\.) /},c={cN:"subst",b:/\{/,e:/\}/,k:r,i:/#/},a={b:/\{\{/,relevance:0},l={cN:"string",c:[e.BE],v:[{b:/(u|b)?r?'''/,e:/'''/,c:[e.BE,b],relevance:10},{b:/(u|b)?r?"""/,e:/"""/,c:[e.BE,b],relevance:10},{b:/(fr|rf|f)'''/,e:/'''/,c:[e.BE,b,a,c]},{b:/(fr|rf|f)"""/,e:/"""/,c:[e.BE,b,a,c]},{b:/(u|r|ur)'/,e:/'/,relevance:10},{b:/(u|r|ur)"/,e:/"/,relevance:10},{b:/(b|br)'/,e:/'/},{b:/(b|br)"/,e:/"/},{b:/(fr|rf|f)'/,e:/'/,c:[e.BE,a,c]},{b:/(fr|rf|f)"/,e:/"/,c:[e.BE,a,c]},e.ASM,e.QSM]},n={cN:"number",relevance:0,v:[{b:e.BNR+"[lLjJ]?"},{b:"\\b(0o[0-7]+)[lLjJ]?"},{b:e.CNR+"[lLjJ]?"}]},i={cN:"params",b:/\(/,e:/\)/,c:["self",b,n,l,e.HCM]};return c.c=[l,n,b],{aliases:["py","gyp","ipython"],k:r,i:/(<\/|->|\?)|=>/,c:[b,n,{bK:"if",relevance:0},l,e.HCM,{v:[{cN:"function",bK:"def"},{cN:"class",bK:"class"}],e:/:/,i:/[${=;\n,]/,c:[e.UTM,i,{b:/->/,eW:!0,k:"None"}]},{cN:"meta",b:/^[\t ]*@/,e:/$/},{b:/\b(print|exec)\(/}]}}); -------------------------------------------------------------------------------- /output/sample.js: -------------------------------------------------------------------------------- 1 | $(document).ready(function() { 2 | function wrapLine(baseLineURL, number, content) { 3 | var className = 'line', url = '', target = ''; 4 | if (content.includes('')) { 5 | target = '_blank'; 6 | className += ' clickable'; 7 | url = (baseLineURL + number); 8 | } else { 9 | url = '#'; 10 | } 11 | 12 | return '' + 13 | '
' + number + '
' + 14 | '
' + content + '
'; 15 | } 16 | 17 | $('pre.code').each(function() { 18 | hljs.highlightBlock(this); 19 | 20 | var codeElement = $(this); 21 | var baseLineURL = codeElement.attr('data-base-line-url') || '#'; 22 | var startLineNumber = parseInt(codeElement.attr('data-line-number') || 1); 23 | 24 | var lines = []; 25 | _.each(codeElement.html().split('\n'), function(line, index) { 26 | var formattedLine = wrapLine(baseLineURL, startLineNumber + index, line || ' '); 27 | lines.push(formattedLine); 28 | }); 29 | 30 | var startHighlighted = null; 31 | var endHighlighted = null; 32 | _.each(lines.slice(), function (line, index) { 33 | if (line.includes('')) { 34 | if (startHighlighted === null) { 35 | startHighlighted = index; 36 | } 37 | endHighlighted = index; 38 | } 39 | }); 40 | 41 | startHighlighted = Math.max(startHighlighted-5, 0); 42 | endHighlighted = Math.min(endHighlighted+5, lines.length-1); 43 | 44 | var storage = window.jbr_code_changes || {}; 45 | var version = codeElement.attr('data-code-version'); 46 | storage[version] = { 47 | 'lines': lines, 48 | 'is_top_expanded': false, 49 | 'is_bottom_expanded': false, 50 | 'highlighted_start_line': startHighlighted, 51 | 'highlighted_end_line': endHighlighted 52 | }; 53 | window.jbr_code_changes = storage; 54 | 55 | if (startHighlighted > 0) { 56 | $('[data-action="toggle-expand"][data-kind="top"][data-code-version="' + version + '"]').show(); 57 | } 58 | if (endHighlighted < lines.length - 1) { 59 | $('[data-action="toggle-expand"][data-kind="bottom"][data-code-version="' + version + '"]').show(); 60 | } 61 | 62 | var newContent = lines.slice(startHighlighted, endHighlighted+1).join(''); 63 | codeElement.html(newContent); 64 | }); 65 | 66 | $('#before_code_block .title, #after_code_block .title').click(function () { 67 | var code = $('pre.code', $(this).parent()); 68 | selectText(code[0]); 69 | document.execCommand('copy'); 70 | window.getSelection().removeAllRanges(); 71 | alert('Copied'); 72 | }); 73 | 74 | function selectText(element) { 75 | var selection = window.getSelection(); 76 | var range = document.createRange(); 77 | range.selectNodeContents(element); 78 | selection.removeAllRanges(); 79 | selection.addRange(range); 80 | } 81 | 82 | $('[data-action="toggle-expand"]').click(function() { 83 | $(this).toggleClass('expanded'); 84 | var kind = $(this).attr('data-kind'); 85 | 86 | var version = $(this).attr('data-code-version'); 87 | var lines = window.jbr_code_changes[version]['lines']; 88 | 89 | var startLine, endLine; 90 | if (kind === 'top') { 91 | startLine = window.jbr_code_changes[version]['is_top_expanded'] 92 | ? window.jbr_code_changes[version]['highlighted_start_line'] : 0; 93 | endLine = window.jbr_code_changes[version]['is_bottom_expanded'] 94 | ? lines.length-1 : window.jbr_code_changes[version]['highlighted_end_line']; 95 | window.jbr_code_changes[version]['is_top_expanded'] = !window.jbr_code_changes[version]['is_top_expanded']; 96 | } else { 97 | startLine = window.jbr_code_changes[version]['is_top_expanded'] 98 | ? 0 : window.jbr_code_changes[version]['highlighted_start_line']; 99 | endLine = window.jbr_code_changes[version]['is_bottom_expanded'] 100 | ? window.jbr_code_changes[version]['highlighted_end_line'] : lines.length-1; 101 | window.jbr_code_changes[version]['is_bottom_expanded'] = !window.jbr_code_changes[version]['is_bottom_expanded']; 102 | } 103 | 104 | var codeElement = $('pre.code[data-code-version=' + version + ']'); 105 | var newContent = lines.slice(startLine, endLine+1).join(''); 106 | codeElement.html(newContent); 107 | }); 108 | }); 109 | -------------------------------------------------------------------------------- /output/styles.css: -------------------------------------------------------------------------------- 1 | html, body, div, span { 2 | padding: 0; 3 | margin: 0; 4 | } 5 | 6 | body { 7 | padding: 8px; 8 | } 9 | 10 | a { 11 | color: rebeccapurple; 12 | text-decoration: none; 13 | } 14 | 15 | a:hover { 16 | text-decoration: underline; 17 | } 18 | 19 | #commit_hash, #repo, .title { 20 | color: black; 21 | margin: 4px 0; 22 | } 23 | 24 | #before_code_block, #after_code_block { 25 | margin-top: 24px; 26 | } 27 | 28 | #before_code_block .title, #after_code_block .title { 29 | cursor: pointer; 30 | } 31 | 32 | #repo a { 33 | font-size: 16px; 34 | margin: 4px 0; 35 | } 36 | 37 | .code { 38 | background-color: #f2f2f2; 39 | border: 1px solid #f2f2f2; 40 | margin: 0; 41 | padding: 0; 42 | opacity: 1; 43 | cursor: default; 44 | } 45 | 46 | .code .highlighted { 47 | font-weight: bold; 48 | font-size: 14px; 49 | color: black; 50 | } 51 | 52 | .code .highlighted div, 53 | .code .highlighted span { 54 | opacity: 1; 55 | } 56 | 57 | .code .line { 58 | display: table-row; 59 | cursor: default; 60 | pointer-events: none; 61 | } 62 | 63 | .code .line:hover { 64 | text-decoration: none; 65 | } 66 | 67 | .code .line .line-content { 68 | display: table-cell; 69 | width: 100%; 70 | 71 | padding-left: 4px; 72 | margin: 0; 73 | 74 | color: rgba(68,68,68,.3); 75 | font-size: 14px; 76 | } 77 | 78 | .code .line .line-number { 79 | display: table-cell; 80 | vertical-align: middle; 81 | 82 | font-size: 14px; 83 | margin: 0 8px 0 0; 84 | padding: 0 4px; 85 | 86 | text-align: right; 87 | 88 | color: #0052cc; 89 | background-color: white; 90 | border-right: 1px solid #dfe1e6; 91 | 92 | user-select: none; 93 | } 94 | 95 | .code .line.clickable { 96 | pointer-events: all; 97 | } 98 | 99 | .code .line.clickable:hover { 100 | cursor: pointer; 101 | } 102 | 103 | .code .line.clickable:hover .line-content { 104 | background-color: #fcf160; 105 | } 106 | 107 | .title { 108 | width: auto; 109 | display: inline-block; 110 | } 111 | 112 | .title:hover { 113 | text-decoration: underline; 114 | } 115 | 116 | .item { 117 | float: left; 118 | clear: left; 119 | } 120 | 121 | .pattern-instance.pattern-repr::before { 122 | content: 'Representative'; 123 | color: red; 124 | } 125 | 126 | .copy-icon { 127 | cursor: pointer; 128 | } 129 | 130 | [data-action="visibility"] { 131 | cursor: pointer; 132 | -ms-user-select: none; 133 | -moz-user-select: none; 134 | -webkit-user-select: none; 135 | user-select: none; 136 | color: #1f7199; 137 | } 138 | 139 | [data-action="visibility"].title:hover { 140 | text-decoration: underline; 141 | } 142 | 143 | #sample_id { 144 | font-weight: bold; 145 | color: black; 146 | } 147 | 148 | .expand-btn { 149 | display: none; 150 | cursor: pointer; 151 | width: 1px; 152 | -ms-user-select: none; 153 | -moz-user-select: none; 154 | -webkit-user-select: none; 155 | user-select: none; 156 | font-size: 10px; 157 | color: #1f7199; 158 | } 159 | 160 | .expand-btn:hover { 161 | transform: scale(1.5); 162 | } 163 | 164 | .expand-btn[data-kind="top"]::after, .expand-btn[data-kind="bottom"]::after { 165 | content: 'expand'; 166 | } 167 | 168 | .expand-btn.expanded[data-kind="top"]::after, .expand-btn.expanded[data-kind="bottom"]::after { 169 | content: 'minimize'; 170 | } 171 | 172 | #after_code_block { 173 | margin-bottom: 32px; 174 | } -------------------------------------------------------------------------------- /patterns/__init__.py: -------------------------------------------------------------------------------- 1 | import patterns.search 2 | 3 | Miner = patterns.search.Miner 4 | -------------------------------------------------------------------------------- /patterns/exas.py: -------------------------------------------------------------------------------- 1 | from pyflowgraph.models import LinkType 2 | 3 | import sys 4 | 5 | HALF_N = sys.maxsize // 2 6 | N = HALF_N * 2 7 | 8 | 9 | def normalize(value): 10 | return (value + HALF_N) % N - HALF_N 11 | 12 | 13 | class ExasFeature: 14 | """ 15 | Features characterize code fragments 16 | Read more: "Accurate and Efficient Structural Characteristic Feature Extraction" 17 | """ 18 | MAX_LENGTH = 2 ** 3 - 1 19 | 20 | def __init__(self, nodes=None): 21 | self.node_label_to_feature_id = {} 22 | self.edge_label_to_feature_id = { 23 | LinkType.QUALIFIER: 0, 24 | LinkType.CONDITION: 1, 25 | LinkType.CONTROL: 2, 26 | LinkType.DEFINITION: 3, 27 | LinkType.MAP: 4, 28 | LinkType.PARAMETER: 5, 29 | LinkType.RECEIVER: 6, 30 | LinkType.REFERENCE: 7 31 | } 32 | 33 | if nodes is not None: 34 | self._bind_node_feature_ids(nodes) 35 | 36 | def _bind_node_feature_ids(self, nodes): 37 | for num, node in enumerate(nodes): 38 | self.node_label_to_feature_id[node.label] = num + 1 # some ids can be skipped 39 | 40 | def get_id_by_label(self, label): 41 | return self.node_label_to_feature_id.get(label) 42 | 43 | def get_id_by_labels(self, labels): 44 | result = 0 45 | 46 | for num, label in enumerate(labels): 47 | if num % 2 == 0: 48 | s = self.node_label_to_feature_id.get(label) 49 | else: 50 | s = self.edge_label_to_feature_id.get(label, 0) 51 | s = normalize(s << 5) # 2^4 types 52 | result = normalize(result << 8) 53 | 54 | result = normalize(result + s) 55 | 56 | return result 57 | -------------------------------------------------------------------------------- /pyflowgraph/__init__.py: -------------------------------------------------------------------------------- 1 | from . import visual 2 | from .build import GraphBuilder 3 | 4 | 5 | _builder = GraphBuilder() 6 | 7 | build_from_source = _builder.build_from_source 8 | build_from_file = _builder.build_from_file 9 | 10 | export_graph_image = visual.export_graph_image 11 | -------------------------------------------------------------------------------- /pyflowgraph/ast_utils.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from log import logger 3 | 4 | def get_node_key(node): 5 | if isinstance(node, ast.Name): 6 | return node.id 7 | elif isinstance(node, ast.arg): 8 | return node.arg 9 | elif isinstance(node, ast.Attribute): 10 | node_key = node.attr 11 | curr_node = node.value 12 | 13 | while isinstance(curr_node, ast.Attribute): 14 | node_key = f'{curr_node.attr}.{node_key}' 15 | curr_node = curr_node.value 16 | 17 | return f'{curr_node.id}.{node_key}' if isinstance(curr_node, ast.Name) else None 18 | elif isinstance(node, ast.FunctionDef): 19 | return node.name 20 | elif isinstance(node, ast.Call): 21 | return get_node_key(node.func) 22 | else: 23 | logger.error(f"ast_utils: get_node_key node = {node}, line = {node.first_token.line}") 24 | return "Expr" 25 | 26 | 27 | def get_node_full_name(node): 28 | if isinstance(node, ast.Name): 29 | return node.id 30 | elif isinstance(node, ast.arg): 31 | return node.arg 32 | elif isinstance(node, ast.Attribute): 33 | var_id = node.attr 34 | curr_node = node.value 35 | while isinstance(curr_node, ast.Attribute) \ 36 | or isinstance(curr_node, ast.Call) and isinstance(curr_node.func, ast.Attribute): 37 | 38 | if isinstance(curr_node, ast.Attribute): 39 | var_id = curr_node.attr + '.' + var_id 40 | curr_node = curr_node.value 41 | else: 42 | var_id = curr_node.func.attr + '()' + '.' + var_id 43 | curr_node = curr_node.func.value 44 | if isinstance(curr_node, ast.Call): 45 | id = curr_node.func.id + '()' 46 | else: 47 | try: 48 | id = curr_node.id 49 | except: 50 | id = get_node_full_name(curr_node) 51 | return id + '.' + var_id 52 | elif isinstance(node, ast.FunctionDef): 53 | return node.name 54 | elif isinstance(node, ast.Subscript): 55 | return f'{get_node_full_name(node.value)}[.]' 56 | elif isinstance(node, ast.Index): 57 | return get_node_full_name(node.value) 58 | elif isinstance(node, ast.Slice): 59 | items = ['.' for item in [node.lower, node.step, node.upper] if item] 60 | return ':'.join(items) 61 | elif isinstance(node, ast.Constant): 62 | return '.' 63 | elif isinstance(node, ast.Tuple): 64 | str = ",".join(['.' for item in node.elts]) 65 | return f"({str})" 66 | elif isinstance(node, ast.List): 67 | str = ",".join(['.' for item in node.elts]) 68 | return f"[{str}]" 69 | elif isinstance(node, ast.Dict): 70 | str = ",".join(['.' for item in node.elts]) 71 | return "{" + str + "}" 72 | elif isinstance(node, ast.UnaryOp): 73 | return "UnaryOp(.)" 74 | elif isinstance(node, ast.Call): 75 | str = ",".join(['.' for item in node.args]) 76 | id = node.func.id if hasattr(node.func, 'id') else node.func.attr 77 | return f"{id}({str})" 78 | elif isinstance(node, ast.BinOp): 79 | return "BinOp(.,.)" 80 | elif isinstance(node, ast.BoolOp): 81 | return "BoolOp(.,.)" 82 | elif isinstance(node, ast.Compare): 83 | return "Compare(.,.)" 84 | elif isinstance(node, ast.ExtSlice): 85 | return ",".join(['.' for item in node.dims]) 86 | elif isinstance(node, ast.Starred): 87 | return f"*{get_node_full_name(node.value)}" 88 | else: 89 | logger.error(f"get_node_full_name_error, expr written instead: Unable to proceed node = {node}, line = {node.first_token.line}") 90 | return "Expr" 91 | 92 | def get_node_short_name(node): 93 | if isinstance(node, ast.Name): 94 | return node.id 95 | elif isinstance(node, ast.arg): 96 | return node.arg 97 | elif isinstance(node, ast.Attribute): 98 | return node.attr 99 | elif isinstance(node, ast.FunctionDef): 100 | return node.name 101 | elif isinstance(node, ast.Call): 102 | return get_node_short_name(node.func) 103 | elif isinstance(node, ast.Subscript): 104 | return get_node_short_name(node.value) + '[.]' 105 | elif isinstance(node, ast.Call): 106 | return get_node_short_name(node.func) 107 | else: 108 | logger.error(f"ast_utils: get_node_short_name node = {node}, line = {node.first_token.line}") 109 | return "Expr" -------------------------------------------------------------------------------- /pyflowgraph/visual.py: -------------------------------------------------------------------------------- 1 | import graphviz as gv 2 | import os 3 | 4 | from pyflowgraph.models import ExtControlFlowGraph, DataNode, OperationNode, ControlNode, ControlEdge, DataEdge, \ 5 | EntryNode 6 | 7 | 8 | def _get_label_and_attrs(node, show_op_kind=True, show_data_keys=False): 9 | label = f'{node.label}' 10 | attrs = {} 11 | 12 | if isinstance(node, DataNode): 13 | attrs['shape'] = 'ellipse' 14 | if show_data_keys: 15 | label = f'{label} #{node.key}' 16 | if show_op_kind: 17 | label = f'{label} <{node.kind}>' 18 | elif isinstance(node, OperationNode): 19 | attrs['shape'] = 'box' 20 | if show_op_kind: 21 | label = f'{label} <{node.kind}>' 22 | elif isinstance(node, ControlNode): 23 | attrs['shape'] = 'diamond' 24 | 25 | label = f'{label} [{node.statement_num}]' 26 | return label, attrs 27 | 28 | 29 | def _convert_to_visual_graph(graph: ExtControlFlowGraph, file_name: str, 30 | show_op_kinds=True, show_data_keys=False, show_control_branch=False, 31 | separate_mapped=True, show_entry_node=True, 32 | min_statement_num=None, max_statement_num=None): 33 | vg = gv.Digraph(name=file_name, format='pdf') 34 | 35 | used = {} 36 | for node in graph.nodes: 37 | if isinstance(node, EntryNode) and not show_entry_node \ 38 | or min_statement_num is not None and node.statement_num < min_statement_num \ 39 | or max_statement_num is not None and node.statement_num > max_statement_num: 40 | continue 41 | 42 | if used.get(node): 43 | continue 44 | 45 | if separate_mapped and node.mapped: 46 | label, attrs = _get_label_and_attrs(node, show_op_kind=show_op_kinds, show_data_keys=show_data_keys) 47 | mapped_label, mapped_attrs = _get_label_and_attrs( 48 | node.mapped, show_op_kind=show_op_kinds, show_data_keys=show_data_keys) 49 | 50 | used[node] = used[node.mapped] = True 51 | 52 | s = gv.Digraph(f'subgraph: {node.statement_num} to {node.mapped.statement_num}') 53 | s.node(f'{node.statement_num}', label=label, _attributes=attrs) 54 | s.node(f'{node.mapped.statement_num}', label=mapped_label, _attributes=mapped_attrs) 55 | 56 | rank = 'source' if isinstance(node, EntryNode) else 'same' 57 | s.graph_attr.update(rank=rank) 58 | vg.subgraph(s) 59 | else: 60 | label, attrs = _get_label_and_attrs(node, show_op_kind=show_op_kinds, show_data_keys=show_data_keys) 61 | vg.node(f'{node.statement_num}', label=label, _attributes=attrs) 62 | 63 | for node in graph.nodes: 64 | for edge in node.in_edges: 65 | if isinstance(edge.node_from, EntryNode) and not show_entry_node \ 66 | or min_statement_num is not None and edge.node_from.statement_num < min_statement_num \ 67 | or max_statement_num is not None and edge.node_from.statement_num > max_statement_num: 68 | continue 69 | 70 | label = edge.label 71 | attrs = {} 72 | 73 | if show_control_branch and isinstance(edge, ControlEdge): 74 | label = f'{"T" if edge.branch_kind else "F"} {label}' 75 | 76 | if isinstance(edge, DataEdge): 77 | attrs['style'] = 'dotted' 78 | 79 | vg.edge(str(edge.node_from.statement_num), str(edge.node_to.statement_num), xlabel=label, _attributes=attrs) 80 | 81 | return vg 82 | 83 | 84 | def export_graph_image(graph: ExtControlFlowGraph, path: str = 'pfg.dot', show_op_kinds=True, show_data_keys=False): 85 | directory, file_name = os.path.split(path) 86 | visual_graph = _convert_to_visual_graph(graph, file_name, show_control_branch=True, 87 | show_op_kinds=show_op_kinds, show_data_keys=show_data_keys) 88 | visual_graph.render(path) 89 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | testpaths = tests 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | graphviz==0.13.2 2 | asttokens==2.0.4 3 | pydriller==1.12 4 | 5 | pytest==5.3.5 6 | 7 | # dev dependencies 8 | stackimpact==1.2.7 9 | objgraph==3.4.1 10 | matplotlib==3.2.0 11 | requests==2.31.0 12 | tqdm==4.56.0 -------------------------------------------------------------------------------- /research/process-fd-count.sh: -------------------------------------------------------------------------------- 1 | FCOUNT=`lsof -p $1 | grep -v " txt " | wc -l`;echo "PID: $1 $FCOUNT" | sort -nk3 2 | 3 | -------------------------------------------------------------------------------- /research/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/python-change-miner/272f6a4d8adc6d721a7b86872577cf0a70875faf/research/tools/__init__.py -------------------------------------------------------------------------------- /research/tools/email_sender.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import smtplib 4 | import re 5 | import logging 6 | from email.mime.multipart import MIMEMultipart 7 | from email.mime.text import MIMEText 8 | 9 | import settings 10 | 11 | _BASE_URL = settings.get('research_patterns_site_base_url') 12 | _OUT_DIR = settings.get('research_patterns_out_dir') 13 | _DATA_FILE_PATH = os.path.join(_OUT_DIR, 'email-data.json') 14 | _EMAIL_FILE_PATH = os.path.join(_OUT_DIR, 'email-text.html') 15 | 16 | 17 | def _set_default_if_needed(data, k, v): 18 | d = data.setdefault(k, v) 19 | data[k] = d 20 | 21 | 22 | def _get_sample_info(sample_dir, sample_id): 23 | with open(os.path.join(sample_dir, f'sample-details-{sample_id}.json'), 'r+') as f: 24 | sample_info = json.loads(f.read()) 25 | return sample_info 26 | 27 | 28 | def _load_email_data(): 29 | with open(_DATA_FILE_PATH, 'a+') as f: 30 | f.seek(0) 31 | data = f.read() 32 | 33 | try: 34 | email_data = json.loads(data) 35 | except: 36 | email_data = {} 37 | 38 | return email_data 39 | 40 | 41 | def _save_email_data(email_data): 42 | with open(_DATA_FILE_PATH, 'w+') as f: 43 | json.dump(email_data, f, indent=4) 44 | 45 | 46 | def send_emails(target_emails, payload): 47 | if not target_emails: 48 | print('empty target emails') 49 | return 50 | 51 | with open(_EMAIL_FILE_PATH, 'r+') as f: 52 | f.seek(0) 53 | email_text = f.read() 54 | 55 | my_email = settings.get('research_survey_sender_email') 56 | my_pass = settings.get('research_survey_sender_pass') 57 | 58 | session = smtplib.SMTP('smtp.gmail.com', 587) 59 | session.starttls() 60 | session.login(my_email, my_pass) 61 | 62 | disconnected = False 63 | sent_emails_cnt = 0 64 | processed_emails = [] 65 | for target_email in target_emails: 66 | if target_email.endswith('@users.noreply.github.com'): 67 | logging.warning('Unable to send email to @users.noreply.github.com') 68 | processed_emails.append(target_email) 69 | continue 70 | 71 | message = MIMEMultipart() 72 | message['To'] = target_email 73 | message['From'] = settings.get('research_survey_email_from') 74 | message['Subject'] = settings.get('research_survey_email_subject') 75 | 76 | mailing_text = email_text 77 | for k, v in payload[target_email].items(): 78 | mailing_text = re.sub('{' + k + '}', v, mailing_text) 79 | 80 | message.attach(MIMEText(mailing_text, 'html')) 81 | 82 | text = message.as_string() 83 | try: 84 | session.sendmail(my_email, target_email, text) 85 | except: 86 | disconnected = True 87 | logging.exception(f'Unable to send mail to {target_email}, probably, the limit exceeded') 88 | break 89 | 90 | processed_emails.append(target_email) 91 | sent_emails_cnt += 1 92 | 93 | print(f'An email was sent from {my_email} to {target_email}') 94 | 95 | if not disconnected: 96 | session.quit() 97 | 98 | print(f'Done sending emails, processed cnt = {len(processed_emails)}, sent cnt = {sent_emails_cnt}') 99 | return processed_emails 100 | 101 | 102 | def start_mailing(): 103 | email_data = _load_email_data() 104 | 105 | target_emails = [] 106 | payload = {} 107 | for email, author_data in email_data['email_to_author_data'].items(): 108 | if author_data.get('is_sent', False): 109 | continue 110 | 111 | payload[email] = { 112 | 'name': author_data['author']['name'], 113 | 'url': author_data['url'], 114 | 'base_url': f'{_BASE_URL}/contents.html' 115 | } 116 | target_emails.append(email) 117 | 118 | logging.warning(f'Target emails: {len(target_emails)}') 119 | 120 | processed_emails = send_emails(target_emails, payload) 121 | if not processed_emails: 122 | return 123 | 124 | for email in processed_emails: 125 | email_data['email_to_author_data'][email]['is_sent'] = True 126 | 127 | _save_email_data(email_data) 128 | print('New email_data was saved') 129 | 130 | 131 | if __name__ == '__main__': 132 | start_mailing() 133 | -------------------------------------------------------------------------------- /research/tools/log_analyser.py: -------------------------------------------------------------------------------- 1 | import re 2 | import os 3 | import time 4 | import matplotlib.pyplot as plt 5 | import datetime 6 | from collections import OrderedDict 7 | 8 | import logging 9 | import settings 10 | 11 | LOGS_DIR = settings.get('research_logs_dir') 12 | 13 | 14 | def search_file(pattern, from_time=None, excluded_files=None): 15 | if excluded_files is None: 16 | excluded_files = [] 17 | 18 | results = [] 19 | regexp = re.compile(pattern) 20 | time_regexp = re.compile('\\[[0-9]{2}.[0-9]{2}.[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}\\]') 21 | 22 | for filename in os.listdir(LOGS_DIR): 23 | if not filename.endswith('.log') or filename in excluded_files: 24 | continue 25 | 26 | logging.warning(f'Looking at file {filename}') 27 | file_path = os.path.join(LOGS_DIR, filename) 28 | 29 | last_result = None 30 | with open(file_path, 'r+') as f: 31 | lines = [line for line in f] 32 | 33 | for lo, line in enumerate(lines): 34 | logging.warning(f'Line [{lo + 1}/{len(lines)}]') 35 | 36 | if not line.startswith('['): 37 | if last_result: 38 | last_result['log'] += line 39 | continue 40 | else: 41 | last_result = None 42 | 43 | if from_time is not None: 44 | m = re.match(time_regexp, line) 45 | dtm_str = m.group(0).lstrip('[').rstrip(']') 46 | 47 | dtm = datetime.datetime.strptime(dtm_str, '%d.%m.%Y %H:%M:%S') 48 | if dtm < from_time: 49 | continue 50 | 51 | result = { 52 | 'matches': re.search(regexp, line), 53 | 'log': line 54 | } 55 | if result['matches']: 56 | results.append(result) 57 | last_result = result 58 | 59 | logging.warning(f'Found {len(results)} matches') 60 | return results 61 | 62 | 63 | def plt_items_for_seconds(): 64 | pattern = 'Gumtree... OK (?P