├── .coveragerc ├── .github ├── dependabot.yml └── workflows │ ├── black.yml │ ├── deploy.yml │ ├── test.yml │ └── tinyconfig.yml ├── .gitignore ├── LICENSE ├── README.md ├── clade ├── __init__.py ├── __main__.py ├── abstract.py ├── cmds.py ├── debugger.py ├── envs.py ├── extensions │ ├── __init__.py │ ├── abstract.py │ ├── alternatives.py │ ├── ar.py │ ├── assembler.py │ ├── callgraph.py │ ├── calls_by_ptr.py │ ├── cc.py │ ├── cdb.py │ ├── cl.py │ ├── cmd_graph.py │ ├── common.py │ ├── common_info.py │ ├── compiler.py │ ├── cross_ref.py │ ├── cxx.py │ ├── functions.py │ ├── info.py │ ├── info │ │ ├── info.aspect │ │ └── linux_kernel.aspect │ ├── install.py │ ├── ld.py │ ├── link.py │ ├── linker.py │ ├── ln.py │ ├── macros.py │ ├── mv.py │ ├── objcopy.py │ ├── opts.py │ ├── path.py │ ├── pid_graph.py │ ├── presets │ │ └── presets.json │ ├── src_graph.py │ ├── storage.py │ ├── typedefs.py │ ├── used_in.py │ ├── utils.py │ ├── variables.py │ └── win_copy.py ├── intercept.py ├── intercept │ ├── CMakeLists.txt │ ├── unix │ │ ├── CMakeLists.txt │ │ ├── client.c │ │ ├── client.h │ │ ├── data.c │ │ ├── data.h │ │ ├── env.c │ │ ├── env.h │ │ ├── interceptor.c │ │ ├── lock.c │ │ ├── lock.h │ │ ├── which.c │ │ ├── which.h │ │ └── wrapper.c │ └── windows │ │ ├── CMakeLists.txt │ │ ├── client.cpp │ │ ├── client.h │ │ └── debugger.cpp ├── libinterceptor.py ├── scripts │ ├── __init__.py │ ├── check.py │ ├── compilation_database.py │ ├── diff.py │ ├── file_graph.py │ ├── pid_graph.py │ ├── stats.py │ └── tracer.py ├── server.py ├── types │ ├── __init__.py │ ├── nested_dict.py │ └── path_tree.py ├── utils.py └── wrapper.py ├── docs ├── configuration.md ├── dev.md ├── extensions.md ├── pics │ ├── cmd_graph.png │ ├── libinterceptor.png │ └── pid_graph.png ├── scripts.md ├── troubleshooting.md └── usage.md ├── pytest.ini ├── ruff.toml ├── setup.py └── tests ├── __init__.py ├── conftest.py ├── test_abstract.py ├── test_ar.py ├── test_as.py ├── test_callgraph.py ├── test_calls_by_ptr.py ├── test_cc.py ├── test_cdb.py ├── test_cmd_graph.py ├── test_cmds.py ├── test_cross_ref.py ├── test_cxx.py ├── test_envs.py ├── test_functions.py ├── test_info.py ├── test_intercept.py ├── test_interface.py ├── test_ld.py ├── test_ln.py ├── test_macros.py ├── test_main.py ├── test_objcopy.py ├── test_opts.py ├── test_path.py ├── test_path_tree.py ├── test_pid_graph.py ├── test_project.py ├── test_project ├── Makefile ├── empty.s ├── main.c ├── zero.c └── zero.h ├── test_src_graph.py ├── test_storage.py ├── test_tracer.py ├── test_typedefs.py ├── test_used_in.py ├── test_utils.py ├── test_variables.py └── test_windows.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | concurrency = 3 | multiprocessing 4 | concurrent.futures 5 | source = clade 6 | omit = 7 | clade/scripts/* 8 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" # Location of package manifests 5 | schedule: 6 | interval: "weekly" 7 | -------------------------------------------------------------------------------- /.github/workflows/black.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | lint: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | - uses: psf/black@stable 11 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Build and upload to PyPI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build_wheels: 7 | name: Build wheels on ${{ matrix.os }} 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | os: [ubuntu-20.04, windows-2019, macos-12] 12 | 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Build wheels 17 | uses: pypa/cibuildwheel@v2.16.1 18 | 19 | - uses: actions/upload-artifact@v3 20 | with: 21 | path: ./wheelhouse/*.whl 22 | 23 | build_sdist: 24 | name: Build source distribution 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: actions/checkout@v4 28 | 29 | - name: Build sdist 30 | run: pipx run build --sdist 31 | 32 | - uses: actions/upload-artifact@v3 33 | with: 34 | path: dist/*.tar.gz 35 | 36 | upload_pypi: 37 | needs: [build_wheels, build_sdist] 38 | runs-on: ubuntu-latest 39 | if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') 40 | steps: 41 | - uses: actions/download-artifact@v3 42 | with: 43 | name: artifact 44 | path: dist 45 | 46 | - uses: pypa/gh-action-pypi-publish@v1.8.10 47 | with: 48 | user: ${{ secrets.PYPI_USERNAME }} 49 | password: ${{ secrets.PYPI_PASSWORD }} 50 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ${{ matrix.os }} 8 | 9 | strategy: 10 | matrix: 11 | python-version: ["3.8", "3.9", "3.10", "3.11", "3.12-dev"] 12 | os: [ubuntu-latest, macos-latest] 13 | 14 | steps: 15 | - uses: actions/checkout@v4 16 | - uses: ilammy/msvc-dev-cmd@v1 17 | 18 | - name: Set up Python ${{ matrix.python-version }} 19 | uses: actions/setup-python@v4 20 | with: 21 | python-version: ${{ matrix.python-version }} 22 | 23 | - name: Install system dependencies [Ubuntu] 24 | if: startsWith(matrix.os, 'ubuntu') 25 | run: | 26 | sudo apt-get update 27 | sudo apt-get install -y graphviz gcc-multilib 28 | 29 | - name: Install dependencies 30 | run: | 31 | python -m pip install --upgrade pip setuptools wheel 32 | python -m pip install --upgrade pytest 33 | python -m pip install -e . 34 | 35 | - name: Download CIF [Ubuntu] 36 | if: startsWith(matrix.os, 'ubuntu') 37 | run: | 38 | curl -sSfL -o cif.tar.xz https://github.com/ldv-klever/cif/releases/download/v1.2/linux-x86_64-cif-1.2.tar.xz 39 | 40 | - name: Download CIF [macOS] 41 | if: startsWith(matrix.os, 'macos') 42 | run: | 43 | curl -sSfL -o cif.tar.xz https://github.com/ldv-klever/cif/releases/download/v1.2/macos-x86_64-cif-1.2.tar.xz 44 | 45 | - name: Install CIF 46 | if: "!startsWith(matrix.os, 'windows')" 47 | run: | 48 | tar xf cif.tar.xz 49 | 50 | - name: Test with pytest 51 | if: "!startsWith(matrix.os, 'windows')" 52 | run: | 53 | PATH=$GITHUB_WORKSPACE/cif/bin:$PATH pytest 54 | 55 | - name: Test with pytest [windows] 56 | if: startsWith(matrix.os, 'windows') 57 | run: | 58 | pytest .\tests\test_windows.py 59 | -------------------------------------------------------------------------------- /.github/workflows/tinyconfig.yml: -------------------------------------------------------------------------------- 1 | name: Tinyconfig 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | - uses: actions/checkout@v4 11 | - uses: ilammy/msvc-dev-cmd@v1 12 | 13 | - name: Set up Python 14 | uses: actions/setup-python@v4 15 | with: 16 | python-version: "3.11" 17 | 18 | - name: Install system dependencies 19 | run: | 20 | sudo apt-get update 21 | sudo apt-get install -y graphviz gcc-multilib make 22 | 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install --upgrade pip setuptools wheel 26 | python -m pip install --upgrade pytest 27 | python -m pip install -e . 28 | 29 | - name: Download CIF 30 | run: | 31 | curl -sSfL -o cif.tar.xz https://github.com/ldv-klever/cif/releases/download/v1.2/linux-x86_64-cif-1.2.tar.xz 32 | 33 | - name: Install CIF 34 | run: | 35 | tar xf cif.tar.xz 36 | 37 | - name: Clone Linux kernel 38 | run: | 39 | git clone --depth 1 https://github.com/torvalds/linux 40 | 41 | - name: Configure Linux kernel 42 | run: | 43 | cd linux 44 | make tinyconfig 45 | 46 | - name: Build Linux kernel with Clade 47 | run: | 48 | cd linux 49 | PATH=$GITHUB_WORKSPACE/cif/bin:$PATH clade -p linux_kernel -e Callgraph -e Macros make -j$(nproc) 50 | zip -r clade-linux-kernel-tinyconfig.zip clade 51 | 52 | - uses: actions/upload-artifact@v3 53 | with: 54 | name: clade-linux-kernel-tinyconfig 55 | path: linux/clade-linux-kernel-tinyconfig.zip 56 | 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Clade 2 | libinterceptor.so 3 | libinterceptor.dylib 4 | wrapper 5 | debugger.exe 6 | parser.out 7 | parsetab.py 8 | *.i 9 | tests/test_project/clade 10 | 11 | # VS Code 12 | .vscode 13 | 14 | # PyCharm 15 | .idea 16 | 17 | # Static analysis 18 | .svace-dir 19 | .svacer-dir 20 | 21 | # Byte-compiled / optimized / DLL files 22 | __pycache__/ 23 | *.py[cod] 24 | *$py.class 25 | 26 | # C extensions 27 | *.so 28 | 29 | # Distribution / packaging 30 | .Python 31 | env/ 32 | build/ 33 | develop-eggs/ 34 | dist/ 35 | downloads/ 36 | eggs/ 37 | .eggs/ 38 | lib/ 39 | lib64/ 40 | parts/ 41 | sdist/ 42 | var/ 43 | wheels/ 44 | *.egg-info/ 45 | .installed.cfg 46 | *.egg 47 | Pipfile* 48 | 49 | # PyInstaller 50 | # Usually these files are written by a python script from a template 51 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 52 | *.manifest 53 | *.spec 54 | 55 | # Installer logs 56 | pip-log.txt 57 | pip-delete-this-directory.txt 58 | 59 | # Unit test / coverage reports 60 | htmlcov/ 61 | .tox/ 62 | .coverage 63 | .coverage.* 64 | .cache 65 | nosetests.xml 66 | coverage.xml 67 | *.cover 68 | .hypothesis/ 69 | .pytest_cache/ 70 | 71 | # Translations 72 | *.mo 73 | *.pot 74 | 75 | # Django stuff: 76 | *.log 77 | local_settings.py 78 | 79 | # Flask stuff: 80 | instance/ 81 | .webassets-cache 82 | 83 | # Scrapy stuff: 84 | .scrapy 85 | 86 | # Sphinx documentation 87 | docs/_build/ 88 | 89 | # PyBuilder 90 | target/ 91 | 92 | # Jupyter Notebook 93 | .ipynb_checkpoints 94 | 95 | # pyenv 96 | .python-version 97 | 98 | # celery beat schedule file 99 | celerybeat-schedule 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # dotenv 105 | .env 106 | 107 | # virtualenv 108 | .venv 109 | venv* 110 | ENV/ 111 | 112 | # Spyder project settings 113 | .spyderproject 114 | .spyproject 115 | 116 | # Rope project settings 117 | .ropeproject 118 | 119 | # mkdocs documentation 120 | /site 121 | 122 | # mypy 123 | .mypy_cache/ 124 | 125 | # macOS 126 | .DS_STORE 127 | 128 | # cProfile 129 | prof 130 | *.prof 131 | 132 | # pyright 133 | pyrightconfig.json 134 | 135 | # documents 136 | *.pdf 137 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![GitHub Actions status](https://github.com/17451k/clade/workflows/test/badge.svg)](https://github.com/17451k/clade/actions?query=workflow%3Atest) 2 | [![Supported Versions of Python](https://img.shields.io/pypi/pyversions/clade.svg)](https://pypi.org/project/clade) 3 | [![PyPI package version](https://img.shields.io/pypi/v/clade.svg)](https://pypi.org/project/clade) 4 | 5 | # Clade 6 | 7 | Clade is a tool for intercepting build commands (stuff like compilation, 8 | linking, mv, rm, and all other commands that are executed during build). 9 | Intercepted commands can be parsed (to search for input and output files, 10 | and options) and then used for various purposes: 11 | 12 | - generating [compilation database](https://clang.llvm.org/docs/JSONCompilationDatabase.html); 13 | - obtaining information about dependencies between source and object files; 14 | - obtaining information about the source code (source code querying); 15 | - generating function call graph; 16 | - running software verification tools; 17 | - visualization of all collected information; 18 | - *and for much more*. 19 | 20 | The interception of build commands is independent of the project type 21 | and used programming languages. 22 | However, all other functionality available in Clade **IS** dependent. 23 | Currently only C projects are supported, but other languages and additional 24 | functionality can be supported through the built-in *extension mechanism*. 25 | 26 | ## Prerequisites 27 | 28 | An important part of Clade - a build commands intercepting library - 29 | is written in C and it needs to be compiled before use. 30 | It will be performed automatically at the installation stage, but you will 31 | need to install some prerequisites beforehand: 32 | 33 | - Python 3 (>=3.5) 34 | - pip (Python package manager) 35 | - cmake (>=3.3) 36 | 37 | *Linux only*: 38 | 39 | - make 40 | - C **and** C++ compiler (gcc or clang) 41 | - python3-dev (Ubuntu) or python3-devel (openSUSE) package 42 | - gcc-multilib (Ubuntu) or gcc-32bit (openSUSE) package 43 | to intercept build commands of projects leveraging multilib capabilities 44 | 45 | *Windows only*: 46 | 47 | - Microsoft Visual C++ Build Tools 48 | 49 | Optional dependencies: 50 | 51 | - For obtaining information about the C code you will need [CIF](https://github.com/17451k/cif) 52 | installed. CIF is an interface to [Aspectator](https://github.com/17451k/aspectator) which in turn is a GCC 53 | based tool that implements aspect-oriented programming for the C programming 54 | language. You may download compiled CIF on [CIF releases](https://github.com/17451k/cif/releases) page. 55 | - Graphviz for some visualization capabilities. 56 | 57 | Clade works on Linux, macOS and partially on Windows. 58 | 59 | ## Hardware requirements 60 | 61 | If you want to run Clade on a large project, like the Linux kernel, 62 | you will need at least 16GB of RAM and 100GB of free disk space 63 | for temporary files. The size of generated data will be approximately 64 | 10GB, so the space used for temporary files will be freed at the end. 65 | Also several CPU cores are recommended, since in some cases Clade takes 66 | twice as long time than a typical build process. 67 | 68 | ## Installation 69 | 70 | To install the latest stable version just run the following command: 71 | 72 | ``` shell 73 | python3 -m pip install clade 74 | ``` 75 | 76 | ## Documentation 77 | 78 | Following documentation is available: 79 | * [Basic usage](docs/usage.md) 80 | * [Available configuration options](docs/configuration.md) 81 | * [Extensions](docs/extensions.md) 82 | * [Scripts](docs/scripts.md) 83 | * [Troubleshooting](docs/troubleshooting.md) 84 | * [Development documentation](docs/dev.md) 85 | 86 | You can also download an example of Clade output on the Linux kernel 87 | (configuration tinyconfig) [from here](https://github.com/17451k/clade/suites/16820630511/artifacts/960641341) (around 40MB). 88 | 89 | ## Acknowledgments 90 | 91 | Clade is inspired by the [Bear](https://github.com/rizsotto/Bear) project created by [László Nagy](https://github.com/rizsotto). 92 | -------------------------------------------------------------------------------- /clade/abstract.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import abc 17 | import os 18 | import shlex 19 | import subprocess 20 | import tempfile 21 | 22 | from clade.cmds import get_last_id 23 | from clade.utils import get_logger 24 | from clade.server import PreprocessServer 25 | 26 | 27 | class Intercept(metaclass=abc.ABCMeta): 28 | """Object for intercepting and parsing build commands. 29 | 30 | Attributes: 31 | command: A list of strings representing build command to run and intercept 32 | cwd: A path to the directory where build command should be executed 33 | output: A path to the file where intercepted commands will be saved 34 | append: A boolean allowing to append intercepted commands to already existing file with commands 35 | conf: dictionary with configuration 36 | 37 | Raises: 38 | NotImplementedError: Clade is launched on Windows 39 | RuntimeError: Clade installation is corrupted, or intercepting process failed 40 | """ 41 | 42 | def __init__( 43 | self, 44 | command, 45 | cwd=os.getcwd(), 46 | output: str = "cmds.txt", 47 | append=False, 48 | intercept_open=False, 49 | intercept_envs=False, 50 | conf=None, 51 | ): 52 | self.command = command 53 | self.cwd = cwd 54 | self.output = os.path.abspath(output) 55 | self.output_open = os.path.join(os.path.dirname(self.output), "open.txt") 56 | self.output_envs = os.path.join(os.path.dirname(self.output), "envs.txt") 57 | self.append = append 58 | self.intercept_open = intercept_open 59 | self.intercept_envs = intercept_envs 60 | self.conf = conf if conf else dict() 61 | 62 | self.clade_if_file = None 63 | self.logger = get_logger("Intercept", conf=self.conf) 64 | self.env = self._setup_env() 65 | 66 | if not self.append: 67 | if os.path.exists(self.output): 68 | os.remove(self.output) 69 | if os.path.exists(self.output_open): 70 | os.remove(self.output_open) 71 | if os.path.exists(self.output_envs): 72 | os.remove(self.output_envs) 73 | 74 | def _setup_env(self): 75 | env = dict(os.environ) 76 | 77 | self.logger.debug("Set 'CLADE_INTERCEPT' environment variable value") 78 | env["CLADE_INTERCEPT"] = str(self.output) 79 | 80 | if self.intercept_open: 81 | self.logger.debug("Set 'CLADE_INTERCEPT_OPEN' environment variable value") 82 | env["CLADE_INTERCEPT_OPEN"] = self.output_open 83 | 84 | if self.intercept_envs: 85 | self.logger.debug("Set 'CLADE_ENV_VARS' environment variable value") 86 | env["CLADE_ENV_VARS"] = self.output_envs 87 | 88 | # Prepare environment variables for PID graph 89 | if self.append: 90 | last_used_id = get_last_id(self.output) 91 | else: 92 | last_used_id = "0" 93 | 94 | f = tempfile.NamedTemporaryFile(mode="w", delete=False) 95 | f.write(last_used_id) 96 | f.flush() 97 | 98 | self.clade_if_file = f.name 99 | env["CLADE_ID_FILE"] = self.clade_if_file 100 | env["CLADE_PARENT_ID"] = "0" 101 | 102 | return env 103 | 104 | @staticmethod 105 | def preprocess(execute): 106 | """Decorator for execute() method 107 | 108 | It runs build command under socket server allowing preprocessing of intercepted build commands 109 | before their execution by a suitable extension. 110 | """ 111 | 112 | def execute_wrapper(self, *args, **kwargs): 113 | if not self.conf.get("Intercept.preprocess"): 114 | return execute(self, *args, **kwargs) 115 | 116 | server = PreprocessServer(self.conf, self.output) 117 | 118 | # self.env.update(server.env) would be wrong 119 | server_env = server.env.copy() 120 | server_env.update(self.env) 121 | self.env = server_env 122 | 123 | self.logger.debug("Start preprocess server") 124 | server.start() 125 | 126 | try: 127 | return execute(self, *args, **kwargs) 128 | finally: 129 | self.logger.debug("Terminate preprocess server") 130 | server.terminate() 131 | 132 | return execute_wrapper 133 | 134 | def execute(self): 135 | """Execute intercepting of build commands. 136 | 137 | Returns: 138 | 0 if everything went successful and error code otherwise 139 | """ 140 | 141 | shell_command = " ".join([shlex.quote(x) for x in self.command]) 142 | self.logger.debug("Execute {!r} command".format(shell_command)) 143 | r = subprocess.call(shell_command, env=self.env, shell=True, cwd=self.cwd) 144 | 145 | if self.clade_if_file and os.path.exists(self.clade_if_file): 146 | os.remove(self.clade_if_file) 147 | 148 | return r 149 | -------------------------------------------------------------------------------- /clade/cmds.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import re 18 | 19 | DELIMITER = "||" 20 | 21 | 22 | def open_cmds_file(cmds_file): 23 | """Open txt file with intercepted commands and return file object. 24 | 25 | Raises: 26 | RuntimeError: Specified file does not exist or empty. 27 | """ 28 | if not os.path.exists(cmds_file): 29 | raise RuntimeError("Specified {} file does not exist".format(cmds_file)) 30 | if not os.path.getsize(cmds_file): 31 | raise RuntimeError("Specified {} file is empty".format(cmds_file)) 32 | 33 | return open(cmds_file) 34 | 35 | 36 | def iter_cmds_by_which(cmds_file, which_list): 37 | """Get an iterator over all intercepted commands filtered by 'which' field. 38 | 39 | Args: 40 | cmds_file: Path to the txt file with intercepted commands. 41 | which_list: A list of strings to filter command by 'which' field. 42 | """ 43 | for cmd in iter_cmds(cmds_file): 44 | for which in which_list: 45 | if re.search(which, cmd["which"]): 46 | yield cmd 47 | break 48 | 49 | 50 | def number_of_cmds_by_which(cmds_file, which_list): 51 | """Return number of all intercepted commands filtered by 'which' field. 52 | 53 | Args: 54 | cmds_file: Path to the txt file with intercepted commands. 55 | which_list: A list of strings to filter command by 'which' field. 56 | """ 57 | 58 | i = 0 59 | 60 | for _ in iter_cmds_by_which(cmds_file, which_list): 61 | i += 1 62 | 63 | return i 64 | 65 | 66 | def iter_cmds(cmds_file): 67 | """Get an iterator over all intercepted commands. 68 | 69 | Args: 70 | cmds_file: Path to the txt file with intercepted commands. 71 | """ 72 | with open_cmds_file(cmds_file) as cmds_fp: 73 | for cmd_id, line in enumerate(cmds_fp): 74 | cmd = split_cmd(line) 75 | cmd["id"] = cmd_id + 1 # cmd_id should be line number in cmds_fp file 76 | yield cmd 77 | 78 | 79 | def split_cmd(line): 80 | """Convert a single intercepted command into dictionary.""" 81 | cmd = dict() 82 | cmd["cwd"], cmd["pid"], cmd["which"], *cmd["command"] = line.strip().split( 83 | DELIMITER 84 | ) 85 | cmd["pid"] = int(cmd["pid"]) 86 | return cmd 87 | 88 | 89 | def join_cmd(cmd): 90 | """Convert a single intercepted command from dictionary to cmds.txt line.""" 91 | line = DELIMITER.join([cmd["cwd"], str(cmd["pid"]), cmd["which"]] + cmd["command"]) 92 | return line 93 | 94 | 95 | def get_first_cmd(cmds_file): 96 | """Get first intercepted command.""" 97 | return next(iter_cmds(cmds_file)) 98 | 99 | 100 | def get_build_dir(cmds_file) -> str: 101 | """Get the working directory in which build process occurred.""" 102 | first_cmd = get_first_cmd(cmds_file) 103 | return first_cmd["cwd"] 104 | 105 | 106 | def get_last_cmd(cmds_file): 107 | """Get last intercepted command.""" 108 | iterable = iter_cmds(cmds_file) 109 | 110 | last_cmd = next(iterable) 111 | for last_cmd in iterable: 112 | pass 113 | 114 | return last_cmd 115 | 116 | 117 | def get_last_id(cmds_file, raise_exception=False) -> int: 118 | """Get last used id.""" 119 | try: 120 | last_cmd = get_last_cmd(cmds_file) 121 | return last_cmd["id"] 122 | except RuntimeError: 123 | if raise_exception: 124 | raise 125 | return 0 126 | 127 | 128 | def get_all_cmds(cmds_file): 129 | """Get list of all intercepted build commands.""" 130 | return list(iter_cmds(cmds_file)) 131 | 132 | 133 | def get_stats(cmds_file): 134 | """Get statistics of intercepted commands number.""" 135 | stats = dict() 136 | for cmd in iter_cmds(cmds_file): 137 | if cmd["which"] in stats: 138 | stats[cmd["which"]] += 1 139 | else: 140 | stats[cmd["which"]] = 1 141 | 142 | return stats 143 | -------------------------------------------------------------------------------- /clade/debugger.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import subprocess 18 | 19 | from clade.abstract import Intercept 20 | 21 | LIB = os.path.join(os.path.dirname(__file__), "intercept", "lib") 22 | LIB64 = os.path.join(os.path.dirname(__file__), "intercept", "lib64") 23 | 24 | 25 | class Debugger(Intercept): 26 | def __init__( 27 | self, 28 | command, 29 | cwd=os.getcwd(), 30 | output="cmds.txt", 31 | append=False, 32 | intercept_open=False, 33 | intercept_envs=False, 34 | conf=None, 35 | ): 36 | if intercept_open: 37 | raise RuntimeError("debugger can't be used to intercept open()") 38 | 39 | if intercept_envs: 40 | raise RuntimeError( 41 | "debugger can't be used to intercept environment variables" 42 | ) 43 | 44 | super().__init__( 45 | command, 46 | cwd=cwd, 47 | output=output, 48 | append=append, 49 | intercept_open=intercept_open, 50 | conf=conf, 51 | ) 52 | 53 | # self.conf["Intercept.preprocess"] = self.conf.get("Intercept.preprocess", True) 54 | self.debugger = self.__find_debugger() 55 | 56 | def __find_debugger(self): 57 | debugger = os.path.join(os.path.dirname(__file__), "intercept", "debugger.exe") 58 | 59 | if not os.path.exists(debugger): 60 | raise RuntimeError("debugger is not found in {!r}".format(debugger)) 61 | 62 | self.logger.debug("Path to the debugger: {!r}".format(debugger)) 63 | 64 | return debugger 65 | 66 | @Intercept.preprocess 67 | def execute(self): 68 | self.command.insert(0, self.debugger) 69 | self.logger.debug("Execute {!r} command".format(self.command)) 70 | return subprocess.call(self.command, env=self.env, shell=False, cwd=self.cwd) 71 | -------------------------------------------------------------------------------- /clade/envs.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | 18 | 19 | def open_envs_file(envs_file): 20 | """Open txt file with intercepted environment variables and return file object. 21 | 22 | Raises: 23 | RuntimeError: Specified file does not exist or empty. 24 | """ 25 | if not os.path.exists(envs_file): 26 | raise RuntimeError("Specified {} file does not exist".format(envs_file)) 27 | if not os.path.getsize(envs_file): 28 | raise RuntimeError("Specified {} file is empty".format(envs_file)) 29 | 30 | return open(envs_file) 31 | 32 | 33 | def iter_envs(envs_file): 34 | """Get an iterator over all intercepted environment variables. 35 | 36 | Args: 37 | envs_file: Path to the txt file with intercepted environment variables. 38 | """ 39 | with open_envs_file(envs_file) as envs_fp: 40 | cmd_id = 1 41 | envs = {"id": cmd_id, "envs": dict()} 42 | for line in envs_fp: 43 | if line.strip(): 44 | e = split_env(line) 45 | envs["envs"].update(e) 46 | else: 47 | yield envs 48 | cmd_id += 1 49 | envs = {"id": cmd_id, "envs": dict()} 50 | 51 | 52 | def split_env(line): 53 | """Convert a single intercepted environment variable into dictionary.""" 54 | env = dict() 55 | l = line.strip().split("=", maxsplit=1) 56 | env[l[0]] = l[1] 57 | return env 58 | 59 | 60 | def join_env(env): 61 | """Convert a single intercepted environment variable from dictionary to envs.txt line.""" 62 | line = "=".join(list(env.items())[0]) 63 | return line 64 | 65 | 66 | def get_first_env(envs_file): 67 | """Get environment variables for first intercepted command.""" 68 | return next(iter_envs(envs_file)) 69 | 70 | 71 | def get_last_env(envs_file): 72 | """Get environment variables for last intercepted command.""" 73 | iterable = iter_envs(envs_file) 74 | 75 | last_env = next(iterable) 76 | for last_env in iterable: 77 | pass 78 | 79 | return last_env 80 | 81 | 82 | def get_last_id(envs_file, raise_exception=False) -> str: 83 | """Get last used id.""" 84 | try: 85 | last_env = get_last_env(envs_file) 86 | return last_env["id"] 87 | except RuntimeError: 88 | if raise_exception: 89 | raise 90 | return "0" 91 | 92 | 93 | def get_all_envs(envs_file): 94 | """Get list of all intercepted environment variables.""" 95 | return list(iter_envs(envs_file)) 96 | 97 | 98 | def get_stats(envs_file): 99 | """Get statistics of intercepted environment variables number.""" 100 | stats = dict() 101 | for env in iter_envs(envs_file): 102 | stats[env["id"]] = len(env["envs"]) 103 | 104 | return stats 105 | -------------------------------------------------------------------------------- /clade/extensions/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /clade/extensions/ar.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from clade.extensions.common import Common 17 | 18 | 19 | class AR(Common): 20 | __version__ = "1" 21 | 22 | def parse(self, cmds_file): 23 | super().parse(cmds_file, self.conf.get("AR.which_list", [])) 24 | 25 | def parse_cmd(self, cmd): 26 | try: 27 | parsed_cmd = { 28 | "id": cmd["id"], 29 | "in": cmd["command"][3:], 30 | "out": [cmd["command"][2]], 31 | "opts": [cmd["command"][1]], 32 | "cwd": cmd["cwd"], 33 | "command": cmd["command"], 34 | } 35 | except IndexError: 36 | self.dump_bad_cmd_id(cmd["id"]) 37 | return 38 | 39 | if self.is_bad(parsed_cmd): 40 | self.dump_bad_cmd_id(cmd["id"]) 41 | return 42 | 43 | self.dump_cmd_by_id(cmd["id"], parsed_cmd) 44 | -------------------------------------------------------------------------------- /clade/extensions/assembler.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from clade.extensions.common import Common 17 | 18 | 19 | class AS(Common): 20 | __version__ = "1" 21 | 22 | def parse(self, cmds_file): 23 | super().parse(cmds_file, self.conf.get("AS.which_list", [])) 24 | 25 | def parse_cmd(self, cmd): 26 | parsed_cmd = super().parse_cmd(cmd, self.name) 27 | 28 | if self.is_bad(parsed_cmd): 29 | self.dump_bad_cmd_id(parsed_cmd["id"]) 30 | return 31 | 32 | self.dump_cmd_by_id(cmd["id"], parsed_cmd) 33 | -------------------------------------------------------------------------------- /clade/extensions/calls_by_ptr.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 Ilya Shchepetkov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from clade.extensions.abstract import Extension 16 | from clade.types.nested_dict import nested_dict 17 | 18 | 19 | class CallsByPtr(Extension): 20 | requires = ["Info"] 21 | 22 | __version__ = "2" 23 | 24 | def __init__(self, work_dir, conf=None): 25 | super().__init__(work_dir, conf) 26 | 27 | self.calls_by_ptr = nested_dict() 28 | self.calls_by_ptr_file = "calls_by_ptr.json" 29 | 30 | @Extension.prepare 31 | def parse(self, cmds_file): 32 | self.log("Parsing calls by pointers") 33 | 34 | for context_file, context_func, func_ptr, call_line in self.extensions[ 35 | "Info" 36 | ].iter_calls_by_pointers(): 37 | self.debug( 38 | "Processing calls by pointers: " 39 | + " ".join([context_file, context_func, func_ptr]) 40 | ) 41 | 42 | if func_ptr not in self.calls_by_ptr[context_file][context_func]: 43 | self.calls_by_ptr[context_file][context_func][func_ptr] = [ 44 | int(call_line) 45 | ] 46 | else: 47 | self.calls_by_ptr[context_file][context_func][func_ptr].append( 48 | int(call_line) 49 | ) 50 | 51 | self.dump_data(self.calls_by_ptr, self.calls_by_ptr_file) 52 | 53 | def load_calls_by_ptr(self): 54 | return self.load_data(self.calls_by_ptr_file) 55 | -------------------------------------------------------------------------------- /clade/extensions/cdb.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | 18 | from clade.extensions.abstract import Extension 19 | from clade.extensions.opts import filter_opts_for_clang 20 | 21 | 22 | class CDB(Extension): 23 | requires = ["SrcGraph"] 24 | 25 | __version__ = "1" 26 | 27 | def __init__(self, work_dir, conf=None): 28 | if not conf: 29 | conf = dict() 30 | 31 | super().__init__(work_dir, conf) 32 | 33 | self.cdb = [] 34 | # DO NOT put CDB.output option to the presets file 35 | self.cdb_file = self.conf.get( 36 | "CDB.output", os.path.join(self.work_dir, "compile_commands.json") 37 | ) 38 | 39 | @Extension.prepare 40 | def parse(self, cmds_file): 41 | cmds = self.extensions["SrcGraph"].load_compilation_cmds( 42 | with_opts=True, with_raw=True, with_deps=False 43 | ) 44 | 45 | for cmd in cmds: 46 | for i, cmd_in in enumerate(cmd["in"]): 47 | if not os.path.exists(cmd_in): 48 | continue 49 | 50 | if self.conf.get("CDB.filter_opts", False): 51 | opts = filter_opts_for_clang(cmd["opts"]) 52 | else: 53 | opts = cmd["opts"] 54 | 55 | arguments = [cmd["command"][0]] + opts + [cmd_in] 56 | if cmd["out"]: 57 | if "-c" in cmd["opts"]: 58 | cmd_out = cmd["out"][i] 59 | else: 60 | cmd_out = cmd["out"][0] 61 | 62 | arguments.extend(["-o", cmd_out]) 63 | else: 64 | cmd_out = None 65 | 66 | self.cdb.append( 67 | self.__get_cdb_dict(cmd["cwd"], arguments, cmd_in, cmd_out) 68 | ) 69 | 70 | self.dump_data(self.cdb, self.cdb_file) 71 | 72 | def load_cdb(self): 73 | """Load compilation database.""" 74 | return self.load_data(self.cdb_file) 75 | 76 | def __get_cdb_dict(self, cwd, arguments, file, output=None): 77 | cdb_dict = { 78 | "directory": cwd, 79 | "arguments": arguments, 80 | "file": file, 81 | } 82 | 83 | if output: 84 | cdb_dict["output"] = output 85 | 86 | return cdb_dict 87 | -------------------------------------------------------------------------------- /clade/extensions/compiler.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | 18 | from clade.extensions.common import Common 19 | 20 | 21 | class Compiler(Common): 22 | """Parent class for all C compiler classes.""" 23 | 24 | requires = Common.requires + ["Storage"] 25 | file_extensions = [".c", ".i", ".cpp", ".C", ".cc", ".cxx", "c++"] 26 | 27 | __version__ = "2" 28 | 29 | def __init__(self, work_dir, conf=None): 30 | super().__init__(work_dir, conf) 31 | 32 | self.deps_dir = os.path.join(self.work_dir, "deps") 33 | 34 | def parse(self, cmds_file, which_list): 35 | super().parse(cmds_file, which_list) 36 | 37 | if os.path.exists(self.cmds_file) and not os.path.exists(self.deps_dir): 38 | self.warning("All files with dependencies are empty") 39 | 40 | def store_deps_files(self, deps, cwd): 41 | self.__store_src_files(deps, cwd, self.conf.get("Compiler.deps_encoding")) 42 | 43 | def store_pre_files(self, deps, cwd, encoding=None): 44 | self.__store_src_files(deps, cwd, encoding) 45 | 46 | def __store_src_files(self, deps, cwd, encoding=None): 47 | for file in deps: 48 | if not os.path.isabs(file): 49 | file = os.path.join(cwd, file) 50 | 51 | self.extensions["Storage"].add_file(file, encoding=encoding) 52 | 53 | def load_deps_by_id(self, cmd_id): 54 | deps_file = os.path.join("deps", "{}.json".format(cmd_id)) 55 | deps = self.load_data(deps_file, raise_exception=False) 56 | 57 | # if load_data can't find file, it returns empty dict() 58 | # but deps must be a list 59 | return deps if deps else [] 60 | 61 | def dump_deps_by_id(self, cmd_id, deps, cwd): 62 | # Do not dump deps if they are empty 63 | if not deps: 64 | return 65 | 66 | # Normalize and remove duplicates 67 | deps = self.extensions["Path"].normalize_rel_paths(deps, cwd) 68 | deps = list(set(deps)) 69 | 70 | self.debug("Dependencies of command {}: {}".format(cmd_id, deps)) 71 | self.dump_data(deps, os.path.join(self.deps_dir, "{}.json".format(cmd_id))) 72 | 73 | def is_a_compilation_command(self, cmd): 74 | if any( 75 | ( 76 | True 77 | for cmd_in in cmd["in"] 78 | if os.path.splitext(os.path.basename(cmd_in))[1] in self.file_extensions 79 | ) 80 | ): 81 | return True 82 | 83 | self.debug("{} is not a compilation command".format(cmd)) 84 | return False 85 | 86 | def load_all_cmds( 87 | self, 88 | filter_by_pid=True, 89 | with_opts=False, 90 | with_raw=False, 91 | with_deps=False, 92 | compile_only=False, 93 | ): 94 | cmds = super().load_all_cmds( 95 | with_opts=with_opts, with_raw=with_raw, filter_by_pid=filter_by_pid 96 | ) 97 | 98 | # compile only - ignore linker commands, like gcc func.o main.o -o main 99 | # or cl /EP /P file.c 100 | for cmd in cmds: 101 | if compile_only and not self.is_a_compilation_command(cmd): 102 | continue 103 | 104 | if compile_only: 105 | # Remove .o files from compile-and-link commands, like gcc main.c -o main.o func.o 106 | cmd["in"] = [ 107 | cmd_in 108 | for cmd_in in cmd["in"] 109 | if os.path.splitext(os.path.basename(cmd_in))[1] 110 | in self.file_extensions 111 | ] 112 | if not cmd["in"]: 113 | continue 114 | 115 | if with_deps: 116 | cmd["deps"] = self.load_deps_by_id(cmd["id"]) 117 | 118 | yield cmd 119 | 120 | def get_all_pre_files(self): 121 | pre_files = [] 122 | 123 | for cmd in self.load_all_cmds(compile_only=True): 124 | pre_files.extend(self.get_pre_files_by_id(cmd["id"])) 125 | 126 | return pre_files 127 | 128 | def get_pre_files_by_id(self, id): 129 | cmd = self.load_cmd_by_id(id) 130 | 131 | pre_files = [] 132 | 133 | for cmd_in in cmd["in"]: 134 | pre_file = self.get_pre_file_by_path(cmd_in, cmd["cwd"]) 135 | 136 | if os.path.exists(pre_file): 137 | pre_files.append(pre_file) 138 | 139 | self.debug("Getting preprocessed files: {}".format(pre_files)) 140 | return pre_files 141 | 142 | def get_pre_file_by_path(self, path, cwd): 143 | if os.path.isabs(path): 144 | abs_path = path 145 | else: 146 | abs_path = os.path.join(cwd, path) 147 | 148 | pre_file = os.path.splitext(abs_path)[0] + ".i" 149 | pre_file = self.extensions["Storage"].get_storage_path(pre_file) 150 | 151 | self.debug("Getting preprocessed file: {}".format(pre_file)) 152 | return pre_file 153 | -------------------------------------------------------------------------------- /clade/extensions/cxx.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from clade.extensions.cc import CC 17 | 18 | 19 | class CXX(CC): 20 | """Class for parsing C++ build commands.""" 21 | 22 | __version__ = "1" 23 | 24 | def parse(self, cmds_file): 25 | which_list = list(self.conf.get("CXX.which_list", [])) 26 | 27 | if self.conf.get("CXX.process_ccache"): 28 | which_list.append("ccache") 29 | 30 | super(CC, self).parse(cmds_file, which_list) 31 | -------------------------------------------------------------------------------- /clade/extensions/info/info.aspect: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 3 | * Ivannikov Institute for System Programming of the Russian Academy of Sciences 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * ee the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | query: execution($ $(..)) { 19 | $fprintf<"$env" $path "/execution.txt", "%s %s %d %s %s\n", $env, $func_name, $decl_line, $storage_class, $signature> 20 | } 21 | 22 | query: declare_func($ $(..)) { 23 | $fprintf<"$env" $path "/declare_func.txt", "%s %s %d %s %s\n", $env, $func_name, $decl_line, $storage_class, $signature> 24 | } 25 | 26 | query: expand($) { 27 | $fprintf<"$env" $expansion_path "/CLADE-EXPAND" $path "/expand.txt", "%s %d %d\n", $macro_name, $expansion_line, $line> 28 | } 29 | 30 | query: expand($(..)) { 31 | $fprintf<"$env" $expansion_path "/CLADE-EXPAND" $path "/expand.txt", "%s %d %d\n", $macro_name, $expansion_line, $line> 32 | $fprintf<"$env" $expansion_path "/expand_args.txt", "%s %s\n", $macro_name, $actual_args> 33 | } 34 | 35 | query: define($) { 36 | $fprintf<"$env" $path "/define.txt", "%s %d\n", $macro_name, $line> 37 | } 38 | 39 | query: define($(..)) { 40 | $fprintf<"$env" $path "/define.txt", "%s %d\n", $macro_name, $line> 41 | } 42 | 43 | query: call($ $(..)) { 44 | $fprintf<"$env" $func_context_path "/call.txt", "%s %s %s %d %s %s\n", $env, $func_context_name, $func_name, $call_line, $storage_class, $actual_arg_func_names> 45 | } 46 | 47 | query: callp($ $(..)) { 48 | $fprintf<"$env" $func_context_path "/callp.txt", "%s %s %d\n", $func_context_name, $func_ptr_name, $call_line> 49 | } 50 | 51 | query: use_func($ $(..)) { 52 | $fprintf<"$env" $func_context_path "/use_func.txt", "%s %s %s %d %s\n", $env, $func_context_name, $func_name, $use_line, $storage_class> 53 | } 54 | 55 | query: use_var($ $) { 56 | $fprintf<"$env" $func_context_path "/use_var.txt", "%s %s %d\n", $func_context_name, $var_name, $use_line> 57 | } 58 | 59 | query: init_global($ $){ 60 | $fprintf<"$env/$env/init_global.txt", "\"%s\" %s %s %s\n", $signature, $env, $storage_class, $var_init_list_json> 61 | } 62 | 63 | query: introduce($ $) { 64 | $fprintf<"$env" $path "/typedefs.txt", "%s\n", $signature> 65 | } 66 | -------------------------------------------------------------------------------- /clade/extensions/info/linux_kernel.aspect: -------------------------------------------------------------------------------- 1 | around: define(likely(x)) { (x) } 2 | 3 | around: define(unlikely(x)) { (x) } 4 | 5 | query: expand(__EXPORT_SYMBOL(sym, sec)) { 6 | $fprintf<"$env" $expansion_path "/exported.txt", "%s\n", $arg_val1> 7 | } 8 | 9 | query: expand(___EXPORT_SYMBOL(sym, sec)) { 10 | $fprintf<"$env" $expansion_path "/exported.txt", "%s\n", $arg_val1> 11 | } 12 | 13 | @include "info.aspect" 14 | -------------------------------------------------------------------------------- /clade/extensions/install.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 Ilya Shchepetkov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import re 17 | 18 | from clade.extensions.common import Common 19 | from clade.extensions.opts import requires_value 20 | 21 | 22 | class Install(Common): 23 | __version__ = "1" 24 | 25 | def parse(self, cmds_file): 26 | super().parse(cmds_file, self.conf.get("Install.which_list", [])) 27 | 28 | def parse_cmd(self, cmd): 29 | parsed_cmd = { 30 | "id": cmd["id"], 31 | "in": [], 32 | "out": [], 33 | "opts": [], 34 | "cwd": cmd["cwd"], 35 | "command": cmd["command"], 36 | } 37 | 38 | # Output directory, where file will be copied 39 | out = None 40 | 41 | opts = iter(cmd["command"][1:]) 42 | for opt in opts: 43 | if re.search(r"^-", opt): 44 | if opt == "-t" or opt == "--target-directory": 45 | # Value is the next option. 46 | out = os.path.normpath(next(opts)) 47 | elif opt.startswith("--target-directory=") or opt.startswith("-t"): 48 | out = opt.replace("--target-directory=", "") 49 | out = opt.replace("-t", "") 50 | elif opt in requires_value[self.name]: 51 | parsed_cmd["opts"].extend([opt, next(opts)]) 52 | else: 53 | parsed_cmd["opts"].append(opt) 54 | elif os.path.isfile(opt) or os.path.isfile(os.path.join(cmd["cwd"], opt)): 55 | if out is not None or not parsed_cmd["in"] or opt != cmd["command"][-1]: 56 | parsed_cmd["in"].append(os.path.normpath(opt)) 57 | else: 58 | parsed_cmd["out"].append(os.path.normpath(opt)) 59 | elif os.path.isdir(opt): 60 | out = os.path.normpath(opt) 61 | else: 62 | print(parsed_cmd) 63 | self.error( 64 | f"Files from the command {cmd} probably do not exist anymore" 65 | ) 66 | return 67 | 68 | if parsed_cmd["out"] and out: 69 | self.error(f"install command {cmd} is incorrectly parsed: {parsed_cmd}") 70 | return 71 | 72 | if out: 73 | for cmd_in in parsed_cmd["in"]: 74 | parsed_cmd["out"].append(os.path.join(out, os.path.basename(cmd_in))) 75 | 76 | if self.is_bad(parsed_cmd): 77 | self.dump_bad_cmd_id(cmd["id"]) 78 | return 79 | 80 | self.dump_cmd_by_id(cmd["id"], parsed_cmd) 81 | 82 | return parsed_cmd 83 | 84 | def get_pairs(self): 85 | """Returns iterator for all (file, its symlink) pairs""" 86 | cmds = self.load_all_cmds() 87 | 88 | for cmd in cmds: 89 | for i, file in enumerate(cmd["in"]): 90 | # ln commands always have paired input and output files: 91 | # in = ["test1.c", "test2.c"] 92 | # out = ["link/test1.c", "link/test2.c"] 93 | symlink = cmd["out"][i] 94 | 95 | yield (file, symlink) 96 | -------------------------------------------------------------------------------- /clade/extensions/ld.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import functools 17 | import os 18 | import re 19 | import subprocess 20 | 21 | from clade.extensions.linker import Linker 22 | from clade.extensions.common import Common 23 | 24 | 25 | class LD(Linker): 26 | __version__ = "1" 27 | 28 | def parse(self, cmds_file): 29 | Common.parse(self, cmds_file, self.conf.get("LD.which_list", [])) 30 | 31 | def parse_cmd(self, cmd): 32 | parsed_cmd = super().parse_cmd(cmd, self.name) 33 | 34 | if self.is_bad(parsed_cmd): 35 | self.dump_bad_cmd_id(parsed_cmd["id"]) 36 | return 37 | 38 | self._parse_linker_opts(cmd["which"], parsed_cmd) 39 | 40 | self.dump_cmd_by_id(cmd["id"], parsed_cmd) 41 | 42 | @staticmethod 43 | @functools.lru_cache() 44 | def _get_default_searchdirs(which): 45 | searchdirs = [] 46 | 47 | try: 48 | r = subprocess.run( 49 | f"{which} --verbose | grep SEARCH_DIR", 50 | shell=True, 51 | text=True, 52 | stdout=subprocess.PIPE, 53 | stderr=subprocess.STDOUT, 54 | ) 55 | 56 | for line in r.stdout.split(";"): 57 | m = re.search(r"SEARCH_DIR\(\"\=(.*)\"\)", line) 58 | if not m: 59 | continue 60 | 61 | if os.path.isdir(m.group(1)): 62 | searchdirs.append(m.group(1)) 63 | except Exception: 64 | return searchdirs 65 | 66 | return searchdirs 67 | -------------------------------------------------------------------------------- /clade/extensions/link.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import re 18 | 19 | from clade.extensions.common import Common 20 | from clade.extensions.opts import requires_value 21 | 22 | 23 | class Link(Common): 24 | __version__ = "1" 25 | 26 | def __init__(self, work_dir, conf=None): 27 | super().__init__(work_dir, conf) 28 | 29 | def parse(self, cmds_file): 30 | super().parse(cmds_file, self.conf.get("Link.which_list", [])) 31 | 32 | def parse_cmd(self, cmd): 33 | self.debug("Parse: {}".format(cmd)) 34 | parsed_cmd = self._get_cmd_dict(cmd) 35 | 36 | if self.name not in requires_value: 37 | raise RuntimeError("Command type '{}' is not supported".format(self.name)) 38 | 39 | opts = iter(cmd["command"][1:]) 40 | 41 | libpaths = [] 42 | 43 | for opt in opts: 44 | if opt in requires_value[self.name]: 45 | val = next(opts) 46 | parsed_cmd["opts"].extend([opt, val]) 47 | elif re.search(r"[/-]out:", opt, re.IGNORECASE): 48 | parsed_cmd["out"].append(re.sub(r"[/-]OUT:", "", opt, flags=re.I)) 49 | elif re.search(r"^[/-]", opt): 50 | parsed_cmd["opts"].append(opt) 51 | 52 | m = re.search(r"^[/-]libpath:(.*)", opt) 53 | 54 | if m: 55 | libpaths.append(m.group(1)) 56 | else: 57 | for libpath in libpaths: 58 | joined_opt = os.path.join(libpath, opt) 59 | 60 | if os.path.exists(joined_opt): 61 | parsed_cmd["in"].append(joined_opt) 62 | break 63 | else: 64 | parsed_cmd["in"].append(opt) 65 | 66 | if self.is_bad(parsed_cmd): 67 | self.dump_bad_cmd_id(cmd["id"]) 68 | return 69 | 70 | self.dump_cmd_by_id(cmd["id"], parsed_cmd) 71 | -------------------------------------------------------------------------------- /clade/extensions/linker.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 Ilya Shchepetkov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import abc 16 | import os 17 | import re 18 | 19 | from clade.extensions.compiler import Compiler 20 | from clade.extensions.opts import requires_value 21 | 22 | 23 | class Linker(Compiler): 24 | """Parent class for all C compilers, who are also linkers""" 25 | 26 | __version__ = "1" 27 | 28 | def _parse_linker_opts(self, which, parsed_cmd): 29 | archives = [] 30 | 31 | searchdirs = self.__get_searchdirs(which, parsed_cmd) 32 | 33 | opts = iter(parsed_cmd["opts"]) 34 | for opt in opts: 35 | if opt in ["-l", "--library"]: 36 | name = next(opts) 37 | 38 | self.__find_archive(name, searchdirs, parsed_cmd) 39 | elif opt in requires_value[self.name]: 40 | continue 41 | elif opt.startswith("-l") or opt.startswith("--library="): 42 | name = re.sub(r"^-l", "", opt) 43 | name = re.sub(r"^--library=", "", name) 44 | 45 | self.__find_archive(name, searchdirs, parsed_cmd) 46 | 47 | return archives 48 | 49 | @abc.abstractmethod 50 | def _get_default_searchdirs(self, which, parse_cmd): 51 | """Returns default search dir, where linker searches for libraries""" 52 | pass 53 | 54 | def __get_searchdirs(self, which, parsed_cmd): 55 | # sysroot paths are not supported (searchdir begins with "=") 56 | default_searchdirs = self._get_default_searchdirs(which) 57 | self.debug(f"Default search dirs for {which} are: {default_searchdirs}") 58 | 59 | searchdirs = default_searchdirs + self.conf.get("Linker.searchdirs", []) 60 | 61 | opts = iter(parsed_cmd["opts"]) 62 | for opt in opts: 63 | if opt in ["-L", "--library-path"]: 64 | path = next(opts) 65 | 66 | path = os.path.normpath(os.path.join(parsed_cmd["cwd"], path)) 67 | searchdirs.append(path) 68 | elif opt.startswith("-L") or opt.startswith("--library-path="): 69 | path = re.sub(r"^-L", "", opt) 70 | path = re.sub(r"^--library-path=", "", path) 71 | 72 | path = os.path.normpath(os.path.join(parsed_cmd["cwd"], path)) 73 | searchdirs.append(os.path.normpath(path)) 74 | 75 | syslibroot = self.__get_syslibroot(parsed_cmd) 76 | 77 | return [syslibroot + s for s in searchdirs] 78 | 79 | def __get_syslibroot(self, parsed_cmd): 80 | syslibroot = "" 81 | 82 | opts = iter(parsed_cmd["opts"]) 83 | 84 | for opt in opts: 85 | if opt == "-syslibroot": 86 | syslibroot = next(opts) 87 | break 88 | 89 | return syslibroot 90 | 91 | def __find_archive(self, name, searchdirs, parsed_cmd): 92 | if not searchdirs: 93 | return 94 | 95 | names = [] 96 | 97 | if name.startswith(":"): 98 | names.append(name[1:]) 99 | else: 100 | names.append("lib" + name + ".dylib") # macOS 101 | names.append("lib" + name + ".tbd") # macOS, "text-based stub libraries" 102 | names.append("lib" + name + ".so") 103 | names.append("lib" + name + ".a") 104 | names.append(name + ".a") 105 | 106 | for searchdir in searchdirs: 107 | for basename in names: 108 | archive = os.path.normpath(os.path.join(searchdir, basename)) 109 | if os.path.exists(archive): 110 | if archive not in parsed_cmd["in"]: 111 | parsed_cmd["in"].append(archive) 112 | break 113 | else: 114 | continue 115 | break 116 | else: 117 | self.warning("Couldn't find {!r} archive in {}".format(name, searchdirs)) 118 | -------------------------------------------------------------------------------- /clade/extensions/ln.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 Ilya Shchepetkov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import pathlib 17 | import re 18 | 19 | from clade.extensions.common import Common 20 | 21 | 22 | class LN(Common): 23 | __version__ = "1" 24 | 25 | def parse(self, cmds_file): 26 | super().parse(cmds_file, self.conf.get("LN.which_list", [])) 27 | 28 | def parse_cmd(self, cmd): 29 | parsed_cmd = { 30 | "id": cmd["id"], 31 | "in": [], 32 | "out": [], 33 | "opts": [], 34 | "cwd": cmd["cwd"], 35 | "command": cmd["command"], 36 | } 37 | 38 | opts = iter(cmd["command"][1:]) 39 | files = [] 40 | out = None 41 | 42 | # First we parse only options, leaving all the files unparsed 43 | for opt in opts: 44 | if re.search(r"^-", opt): 45 | if opt == "-t" or opt == "--target-directory": 46 | # Value is the next option. 47 | out = os.path.normpath(next(opts)) 48 | elif opt.startswith("--target-directory=") or opt.startswith("-t"): 49 | out = opt.replace("--target-directory=", "") 50 | out = opt.replace("-t", "") 51 | else: 52 | parsed_cmd["opts"].append(opt) 53 | else: 54 | files.append(os.path.normpath(opt)) 55 | 56 | # ln cases (ignoring options): 57 | # - FILE 58 | # - DIR 59 | # - FILE OUT_FILE 60 | # - FILE OUT_DIR 61 | # - DIR OUT_DIR 62 | # - FILE DIR FILE DIR OUT_DIR 63 | 64 | if len(files) == 1 or len(files) > 2 or out: 65 | # Output file will be current working directory, if not specified with -t 66 | if len(files) == 1 and not out: 67 | out = parsed_cmd["cwd"] 68 | # Or it can be specified in the command itself 69 | elif len(files) > 2 and not out: 70 | out = files[-1] 71 | files = files[:-1] 72 | 73 | for file in files: 74 | if os.path.isfile(file) or not os.path.isdir(file): 75 | parsed_cmd["in"].append(file) 76 | parsed_cmd["out"].append(os.path.join(out, os.path.basename(file))) 77 | else: 78 | # If input file is a directory, we threat all the files inside it 79 | # as input files 80 | in_files = self.__get_files(file) 81 | parsed_cmd["in"].extend(in_files) 82 | 83 | for in_file in in_files: 84 | in_file = in_file.replace(file + "/", "") 85 | parsed_cmd["out"].append(os.path.join(out, in_file)) 86 | # This case is special: name of the output file differs from the name of the input one 87 | elif len(files) == 2: 88 | if os.path.isfile(files[0]) or not os.path.isdir(files[0]): 89 | parsed_cmd["in"].append(files[0]) 90 | 91 | if os.path.isdir(files[1]): 92 | parsed_cmd["out"].append( 93 | os.path.join(files[1], os.path.basename(files[0])) 94 | ) 95 | else: 96 | parsed_cmd["out"].append(files[1]) 97 | else: 98 | parsed_cmd["in"] = self.__get_files(files[0]) 99 | 100 | for in_file in parsed_cmd["in"]: 101 | in_file = in_file.replace(files[0] + "/", "") 102 | parsed_cmd["out"].append(os.path.join(files[1], in_file)) 103 | else: 104 | self.error(f"No files in {parsed_cmd['id']} command") 105 | 106 | if not parsed_cmd["out"] and not parsed_cmd["in"]: 107 | self.error(f"Command {cmd} is incorrectly parsed: {parsed_cmd}") 108 | return 109 | 110 | if self.is_bad(parsed_cmd): 111 | self.dump_bad_cmd_id(cmd["id"]) 112 | return 113 | 114 | self.dump_cmd_by_id(cmd["id"], parsed_cmd) 115 | 116 | return parsed_cmd 117 | 118 | def get_pairs(self): 119 | """Returns iterator for all (file, its symlink) pairs""" 120 | cmds = self.load_all_cmds() 121 | 122 | for cmd in cmds: 123 | for i, file in enumerate(cmd["in"]): 124 | # ln commands always have paired input and output files: 125 | # in = ["test1.c", "test2.c"] 126 | # out = ["link/test1.c", "link/test2.c"] 127 | symlink = cmd["out"][i] 128 | 129 | yield (file, symlink) 130 | 131 | def __get_files(self, path): 132 | files = [] 133 | 134 | if os.path.isfile(path): 135 | files.append(os.path.normpath(path)) 136 | else: 137 | for p in pathlib.Path(path).rglob("*"): 138 | if p.is_file(): 139 | files.append(os.path.normpath(p)) 140 | 141 | return files 142 | -------------------------------------------------------------------------------- /clade/extensions/mv.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import re 18 | 19 | from clade.extensions.common import Common 20 | 21 | 22 | class MV(Common): 23 | __version__ = "1" 24 | 25 | def parse(self, cmds_file): 26 | super().parse(cmds_file, self.conf.get("MV.which_list", [])) 27 | 28 | def parse_cmd(self, cmd): 29 | parsed_cmd = { 30 | "id": cmd["id"], 31 | "in": [], 32 | "out": [], 33 | "opts": [], 34 | "cwd": cmd["cwd"], 35 | "command": cmd["command"], 36 | } 37 | 38 | # We assume that 'MV' options always have such the form: 39 | # [-opt]... in_file out_file 40 | for opt in cmd["command"][1:]: 41 | if re.search(r"^-", opt): 42 | parsed_cmd["opts"].append(opt) 43 | elif not parsed_cmd["in"]: 44 | parsed_cmd["in"].append(os.path.normpath(opt)) 45 | else: 46 | parsed_cmd["out"].append(os.path.normpath(opt)) 47 | 48 | if self.is_bad(parsed_cmd): 49 | self.dump_bad_cmd_id(cmd["id"]) 50 | return 51 | 52 | self.dump_cmd_by_id(cmd["id"], parsed_cmd) 53 | -------------------------------------------------------------------------------- /clade/extensions/objcopy.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from clade.extensions.common import Common 17 | 18 | 19 | class Objcopy(Common): 20 | __version__ = "1" 21 | 22 | def parse(self, cmds_file): 23 | super().parse(cmds_file, self.conf.get("Objcopy.which_list", [])) 24 | 25 | def parse_cmd(self, cmd): 26 | parsed_cmd = super().parse_cmd(cmd, self.name) 27 | 28 | # objcopy has only one input file and no more than one output file. 29 | # out file is the same as in file if it didn't specified. 30 | if len(parsed_cmd["in"]) == 2: 31 | parsed_cmd["out"] = [parsed_cmd["in"].pop()] 32 | elif len(parsed_cmd["in"]) < 2: 33 | parsed_cmd["out"] = parsed_cmd["in"] 34 | 35 | if self.is_bad(parsed_cmd): 36 | self.dump_bad_cmd_id(parsed_cmd["id"]) 37 | return 38 | 39 | self.dump_cmd_by_id(cmd["id"], parsed_cmd) 40 | -------------------------------------------------------------------------------- /clade/extensions/path.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import functools 17 | import glob 18 | import os 19 | import sys 20 | 21 | from typing import List 22 | 23 | from clade.cmds import get_build_dir 24 | from clade.extensions.abstract import Extension 25 | 26 | 27 | class Path(Extension): 28 | __version__ = "4" 29 | 30 | @Extension.prepare 31 | def parse(self, cmds_file): 32 | build_cwd = get_build_dir(cmds_file) 33 | self.conf["build_dir"] = self.normalize_abs_path(build_cwd) 34 | 35 | def normalize_rel_paths(self, paths: List[str], cwd: str) -> List[str]: 36 | return [self.normalize_rel_path(path, cwd) for path in paths] 37 | 38 | @staticmethod 39 | @functools.lru_cache() 40 | def normalize_rel_path(path: str, cwd: str) -> str: 41 | cwd = cwd.strip() 42 | path = path.strip() 43 | 44 | key = Path.__get_key(path, cwd) 45 | if sys.platform == "win32": 46 | key = key.lower() 47 | 48 | if not os.path.isabs(path): 49 | abs_path = os.path.join(cwd, path) 50 | else: 51 | abs_path = path 52 | 53 | return Path.normalize_abs_path(abs_path) 54 | 55 | @staticmethod 56 | @functools.lru_cache() 57 | def normalize_abs_path(path: str) -> str: 58 | path = path.strip() 59 | 60 | key = path 61 | if sys.platform == "win32": 62 | key = key.lower() 63 | 64 | npath = os.path.normpath(path) 65 | 66 | if sys.platform == "win32": 67 | npath = Path.__get_actual_filename(npath) 68 | npath = npath.replace("\\", "/") 69 | drive, tail = os.path.splitdrive(npath) 70 | if drive: 71 | npath = "/" + drive[:-1] + tail 72 | 73 | return npath 74 | 75 | @staticmethod 76 | def __get_key(path, cwd): 77 | return cwd + "/" + path 78 | 79 | @staticmethod 80 | def __get_actual_filename(path): 81 | if path[-1] in "\\": 82 | path = path[:-1] 83 | dirs = path.split("\\") 84 | # disk letter 85 | test_path = [dirs[0].upper()] 86 | for d in dirs[1:]: 87 | test_path += ["%s[%s]" % (d[:-1], d[-1])] 88 | res = glob.glob("\\".join(test_path)) 89 | if not res: 90 | # File not found 91 | return path 92 | return res[0] 93 | -------------------------------------------------------------------------------- /clade/extensions/pid_graph.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from typing import List, Dict 17 | 18 | from clade.cmds import iter_cmds, get_last_id 19 | from clade.extensions.abstract import Extension 20 | 21 | 22 | class PidGraph(Extension): 23 | __version__ = "1" 24 | 25 | def __init__(self, work_dir, conf=None): 26 | super().__init__(work_dir, conf) 27 | 28 | self.pid_by_id = dict() 29 | self.pid_by_id_file = "pid_by_id.json" 30 | 31 | @Extension.prepare 32 | def parse(self, cmds_file): 33 | self.log( 34 | "Parsing {} commands".format(get_last_id(cmds_file, raise_exception=True)) 35 | ) 36 | 37 | for cmd in iter_cmds(cmds_file): 38 | self.pid_by_id[cmd["id"]] = cmd["pid"] 39 | 40 | self.dump_dict_with_int_keys(self.pid_by_id, self.pid_by_id_file) 41 | self.pid_by_id.clear() 42 | 43 | def load_pid_graph(self) -> Dict[int, List[int]]: 44 | pid_by_id = self.load_pid_by_id() 45 | pid_graph: Dict[int, List[int]] = dict() 46 | 47 | for key in sorted(pid_by_id.keys()): 48 | pid_graph[key] = [pid_by_id[key]] + pid_graph.get(pid_by_id[key], []) 49 | 50 | return pid_graph 51 | 52 | def load_pid_by_id(self) -> Dict[int, int]: 53 | return self.load_dict_with_int_keys(self.pid_by_id_file) 54 | 55 | def filter_cmds_by_pid(self, cmds, parsed_ids=None): 56 | pid_graph = self.load_pid_graph() 57 | 58 | if not parsed_ids: 59 | parsed_ids = set() 60 | else: 61 | parsed_ids = set(parsed_ids) 62 | 63 | filtered_cmds = [] 64 | 65 | for cmd in sorted(cmds, key=lambda x: x["id"]): 66 | if not (set(pid_graph[cmd["id"]]).intersection(parsed_ids)): 67 | filtered_cmds.append(cmd) 68 | 69 | parsed_ids.add(cmd["id"]) 70 | 71 | self.debug("Filtered out commands: {}".format(list(parsed_ids))) 72 | 73 | return filtered_cmds 74 | -------------------------------------------------------------------------------- /clade/extensions/storage.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import charset_normalizer 17 | import functools 18 | import os 19 | import shutil 20 | import tempfile 21 | 22 | from clade.extensions.abstract import Extension 23 | 24 | 25 | class Storage(Extension): 26 | requires = ["Path"] 27 | 28 | __version__ = "1" 29 | 30 | @Extension.prepare 31 | def parse(self, cmds_file): 32 | files_to_add = self.conf.get("Storage.files_to_add", []) 33 | 34 | if files_to_add: 35 | self.log("Saving files") 36 | 37 | for file in files_to_add: 38 | file = os.path.abspath(file) 39 | self.debug("Saving {!r} to the Storage".format(file)) 40 | 41 | if not os.path.exists(file): 42 | self.error("File does not exist: {!r}".format(file)) 43 | raise RuntimeError 44 | 45 | if os.path.isfile(file): 46 | self.add_file(file) 47 | continue 48 | 49 | for root, _, filenames in os.walk(file): 50 | for filename in filenames: 51 | filename = os.path.join(root, filename) 52 | if not filename.startswith(self.clade_work_dir): 53 | self.add_file(filename) 54 | 55 | def add_file(self, filename, storage_filename=None, encoding=None): 56 | """Add file to the storage. 57 | 58 | Args: 59 | filename: Path to the file 60 | storage_filename: Name by which the file will be stored 61 | encoding: encoding of the file, which may be required if you want 62 | to convert it to UTF-8 using 'Storage.convert_to_utf8' 63 | option 64 | """ 65 | 66 | storage_filename = ( 67 | storage_filename 68 | if storage_filename 69 | else self.extensions["Path"].normalize_abs_path(filename) 70 | ) 71 | 72 | dst = os.path.normpath(self.work_dir + os.sep + storage_filename) 73 | 74 | if self.__path_exists(dst): 75 | return 76 | 77 | try: 78 | self.__copy_file(filename, dst, encoding=encoding) 79 | except FileNotFoundError as e: 80 | self.debug(e) 81 | except (PermissionError, OSError) as e: 82 | self.log(e) 83 | except shutil.SameFileError: 84 | pass 85 | 86 | return dst 87 | 88 | @staticmethod 89 | @functools.lru_cache(maxsize=30000) 90 | def __path_exists(path): 91 | return os.path.exists(path) 92 | 93 | def __copy_file(self, filename, dst, encoding=None): 94 | os.makedirs(os.path.dirname(dst), exist_ok=True) 95 | 96 | if not self.conf.get("Storage.convert_to_utf8"): 97 | self.debug("Storing {!r}".format(filename)) 98 | shutil.copyfile(filename, dst) 99 | else: 100 | with open(filename, "rb") as fh: 101 | content_bytes = fh.read() 102 | 103 | if not encoding: 104 | detected = charset_normalizer.detect(content_bytes) 105 | encoding = detected["encoding"] 106 | confidence = detected["confidence"] 107 | else: 108 | # Encoding is specified by the user 109 | confidence = 1 110 | 111 | if not confidence: 112 | self.warning( 113 | "Can't confidently detect encoding of {!r}.".format(filename) 114 | ) 115 | shutil.copyfile(filename, dst) 116 | return 117 | 118 | self.debug( 119 | "Trying to store {!r}. Detected encoding: {} (confidence = {})".format( 120 | filename, encoding, confidence 121 | ) 122 | ) 123 | 124 | with tempfile.NamedTemporaryFile(mode="wb", delete=False) as f: 125 | # Encode file content to utf-8 126 | try: 127 | content_bytes = content_bytes.decode( 128 | encoding, self.conf.get("Storage.decoding_errors", "strict") 129 | ).encode("utf-8") 130 | except UnicodeDecodeError: 131 | # If user-specified encoding failed, try automatic detection 132 | if confidence == 1: 133 | self.__copy_file(filename, dst) 134 | # else: raise original exception 135 | else: 136 | raise 137 | 138 | # Convert CRLF line endings to LF 139 | content_bytes = content_bytes.replace(b"\r\n", b"\n") 140 | f.write(content_bytes) 141 | 142 | try: 143 | shutil.copymode(filename, f.name) 144 | except Exception: 145 | self.warning("Couldn't set permissions for {!r}".format(filename)) 146 | 147 | try: 148 | os.replace(f.name, dst) 149 | except OSError: 150 | os.remove(f.name) 151 | 152 | def get_storage_dir(self): 153 | return self.work_dir 154 | 155 | def get_storage_path(self, path) -> str: 156 | """Get path to the file or directory from the storage.""" 157 | path = os.path.normpath(path) 158 | return os.path.join(self.work_dir, path.lstrip(os.path.sep)) 159 | -------------------------------------------------------------------------------- /clade/extensions/typedefs.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from clade.extensions.abstract import Extension 17 | 18 | 19 | class Typedefs(Extension): 20 | requires = ["Info"] 21 | 22 | __version__ = "1" 23 | 24 | def __init__(self, work_dir, conf=None): 25 | super().__init__(work_dir, conf) 26 | 27 | self.typedefs = dict() 28 | self.typedefs_folder = "typedefs" 29 | 30 | @Extension.prepare 31 | def parse(self, cmds_file): 32 | self.log("Parsing typedefs") 33 | 34 | self.__process_typedefs() 35 | self.dump_data_by_key(self.typedefs, self.typedefs_folder) 36 | self.typedefs.clear() 37 | 38 | def __process_typedefs(self): 39 | for scope_file, declaration in self.extensions["Info"].iter_typedefs(): 40 | if scope_file not in self.typedefs: 41 | self.typedefs[scope_file] = [declaration] 42 | elif declaration not in self.typedefs[scope_file]: 43 | self.typedefs[scope_file].append(declaration) 44 | 45 | def load_typedefs(self, files=None): 46 | return self.load_data_by_key(self.typedefs_folder, files) 47 | -------------------------------------------------------------------------------- /clade/extensions/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import collections 17 | import hashlib 18 | import itertools 19 | 20 | 21 | def get_string_hash(key): 22 | return hashlib.md5(key.encode("utf-8")).hexdigest() 23 | 24 | 25 | def yield_chunk(container, chunk_size=1000): 26 | it = iter(container) 27 | 28 | while True: 29 | piece = list(itertools.islice(it, chunk_size)) 30 | 31 | if piece: 32 | yield piece 33 | else: 34 | return 35 | 36 | 37 | # Location is a file + command id, in which it was compiled 38 | Location = collections.namedtuple("Location", ["file", "cmd_id"]) 39 | -------------------------------------------------------------------------------- /clade/extensions/win_copy.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | 18 | from clade.extensions.common import Common 19 | 20 | 21 | class Copy(Common): 22 | __version__ = "1" 23 | 24 | def parse(self, cmds_file): 25 | super().parse(cmds_file, self.conf.get("Copy.which_list", [])) 26 | 27 | def parse_cmd(self, cmd): 28 | parsed_cmd = { 29 | "id": cmd["id"], 30 | "in": [], 31 | "out": [], 32 | "opts": [], 33 | "cwd": cmd["cwd"], 34 | "command": cmd["command"], 35 | } 36 | 37 | command = cmd["command"][1:] 38 | 39 | if len(cmd["command"]) >= 2: 40 | if cmd["command"][0].endswith("cmd.exe") and cmd["command"][1] in [ 41 | "/c", 42 | "-c", 43 | ]: 44 | try: 45 | copy_index = cmd["command"].index("copy") 46 | command = cmd["command"][copy_index + 1 :] 47 | except ValueError: 48 | return 49 | else: 50 | return 51 | 52 | # TODO: support patterns in names (*.h) 53 | for opt in command: 54 | # Redirects, like >nul and 2<&1 55 | if cmd["command"][0].endswith("cmd.exe") and ">" in opt or "<" in opt: 56 | continue 57 | 58 | if opt.startswith("/") or opt.startswith("-"): 59 | parsed_cmd["opts"].append(opt) 60 | elif not parsed_cmd["in"]: 61 | cmd_in = os.path.normpath(opt) 62 | 63 | if not os.path.exists(cmd_in): 64 | return 65 | 66 | parsed_cmd["in"].append(cmd_in) 67 | else: 68 | cmd_out = os.path.normpath(opt) 69 | 70 | # workaround for "cmd.exe /c if exist a copy a b" commands 71 | if os.path.isdir(cmd_out) and parsed_cmd["in"]: 72 | cmd_out = os.path.join( 73 | cmd_out, os.path.basename(parsed_cmd["in"][0]) 74 | ) 75 | parsed_cmd["out"].append(cmd_out) 76 | 77 | if self.is_bad(parsed_cmd): 78 | self.dump_bad_cmd_id(cmd["id"]) 79 | return 80 | 81 | self.dump_cmd_by_id(cmd["id"], parsed_cmd) 82 | -------------------------------------------------------------------------------- /clade/intercept.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import sys 18 | 19 | from clade.debugger import Debugger 20 | from clade.libinterceptor import Libinterceptor 21 | from clade.wrapper import Wrapper 22 | 23 | 24 | def intercept( 25 | command, 26 | cwd=os.getcwd(), 27 | output="cmds.txt", 28 | append=False, 29 | conf=None, 30 | use_wrappers=True, 31 | intercept_open=False, 32 | intercept_envs=False, 33 | ): 34 | if sys.platform in ["linux", "darwin"] and use_wrappers: 35 | cl = Wrapper 36 | elif sys.platform in ["linux", "darwin"] and not use_wrappers: 37 | cl = Libinterceptor 38 | elif sys.platform == "win32": 39 | cl = Debugger 40 | else: 41 | sys.exit("Your platform {!r} is not supported yet.".format(sys.platform)) 42 | 43 | i = cl( 44 | command=command, 45 | cwd=cwd, 46 | output=output, 47 | append=append, 48 | intercept_open=intercept_open, 49 | intercept_envs=intercept_envs, 50 | conf=conf, 51 | ) 52 | return i.execute() 53 | -------------------------------------------------------------------------------- /clade/intercept/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.3) 2 | project(interceptor) 3 | 4 | add_subdirectory(unix) 5 | add_subdirectory(windows) 6 | -------------------------------------------------------------------------------- /clade/intercept/unix/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.3) 2 | 3 | set(CMAKE_POSITION_INDEPENDENT_CODE ON) 4 | 5 | add_compile_options( 6 | -Wall 7 | -Wextra 8 | -Wno-unused-parameter 9 | -O3 10 | ) 11 | 12 | add_library(data STATIC data.c) 13 | add_library(which STATIC which.c) 14 | add_library(env STATIC env.c) 15 | add_library(client STATIC client.c) 16 | add_library(lock STATIC lock.c) 17 | target_link_libraries(data which env client lock) 18 | 19 | add_library(interceptor SHARED interceptor.c) 20 | target_link_libraries(interceptor ${CMAKE_DL_LIBS} which data env client lock) 21 | 22 | add_executable(wrapper wrapper.c) 23 | target_link_libraries(wrapper which data env client lock) 24 | 25 | set_target_properties(data which env interceptor wrapper lock PROPERTIES C_STANDARD 11) 26 | -------------------------------------------------------------------------------- /clade/intercept/unix/client.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 3 | * Ivannikov Institute for System Programming of the Russian Academy of Sciences 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #include "env.h" 28 | 29 | static void send_data_unix(const char *msg, char *address) { 30 | int sockfd; 31 | 32 | struct sockaddr_un addr; 33 | 34 | addr.sun_family = AF_UNIX; 35 | strncpy(addr.sun_path, address, sizeof(addr.sun_path)-1); 36 | 37 | sockfd = socket(AF_UNIX, SOCK_STREAM, 0); 38 | 39 | if (0 != connect(sockfd, (struct sockaddr *) &addr, sizeof(struct sockaddr_un))) { 40 | fprintf(stderr, "Couldn't connect to the socket %s: ", address); 41 | perror(""); 42 | exit(EXIT_FAILURE); 43 | } 44 | 45 | ssize_t r = write(sockfd, msg, strlen(msg)); 46 | 47 | if (r == -1) { 48 | perror("Failed to write to the socket"); 49 | } 50 | 51 | // We need to wait until the server finished message processing and close the socket 52 | char buf[1024]; 53 | while ((r = read(sockfd, buf, sizeof(buf)-1)) > 0) {} 54 | } 55 | 56 | static void send_data_inet(const char *msg, char *host, char *port) { 57 | int sockfd; 58 | 59 | struct sockaddr_in addr; 60 | 61 | addr.sin_family = AF_INET; 62 | addr.sin_port = htons(atoi(port)); 63 | 64 | if (!inet_aton(host, &(addr.sin_addr))) { 65 | perror("Invalid ip and port"); 66 | exit(EXIT_FAILURE); 67 | } 68 | 69 | sockfd = socket(AF_INET, SOCK_STREAM, 0); 70 | 71 | if (0 != connect(sockfd, (struct sockaddr*)&addr, sizeof(addr))) { 72 | fprintf(stderr, "Couldn't connect to the server %s:%s ", host, port); 73 | perror(""); 74 | exit(EXIT_FAILURE); 75 | } 76 | 77 | ssize_t r = write(sockfd, msg, strlen(msg)); 78 | 79 | if (r == -1) { 80 | perror("Failed to write to the socket"); 81 | } 82 | 83 | // We need to wait until the server finished message processing and close the socket 84 | char buf[1024]; 85 | while ((r = read(sockfd, buf, sizeof(buf)-1)) > 0) {} 86 | } 87 | 88 | void send_data(const char *msg) { 89 | char* host = getenv(CLADE_INET_HOST_ENV); 90 | char* port = getenv(CLADE_INET_PORT_ENV); 91 | char* address = getenv(CLADE_UNIX_ADDRESS_ENV); 92 | 93 | // Use UNIX sockets if address is not NULL 94 | if (address) { 95 | send_data_unix(msg, address); 96 | } 97 | // Else try to use TCP/IP sockets 98 | else if (host && port) { 99 | send_data_inet(msg, host, port); 100 | } 101 | else { 102 | perror("Server adress is not specified"); 103 | exit(EXIT_FAILURE); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /clade/intercept/unix/client.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 3 | * Ivannikov Institute for System Programming of the Russian Academy of Sciences 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef CLIENT_H 19 | #define CLIENT_H 20 | 21 | extern void send_data(const char *msg); 22 | 23 | #endif /* CLIENT_H */ 24 | -------------------------------------------------------------------------------- /clade/intercept/unix/data.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 3 | * Ivannikov Institute for System Programming of the Russian Academy of Sciences 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef DATA_H 19 | #define DATA_H 20 | 21 | extern void intercept_exec_call(const char *path, char const *const argv[], char **envp); 22 | extern void intercept_open_call(const char *path, int flags); 23 | 24 | #endif /* DATA_H */ 25 | -------------------------------------------------------------------------------- /clade/intercept/unix/env.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #ifndef ENV_H 4 | #define ENV_H 5 | 6 | extern char **copy_envp(char **envp); 7 | extern char **update_envp(char **input_envp); 8 | extern void update_environ(char **envp, bool force); 9 | 10 | extern char *get_parent_id(char **envp); 11 | extern int get_cmd_id(); 12 | 13 | extern char *getenv_or_fail(const char *name); 14 | 15 | char *getenv_from_envp(char **envp, const char *key); 16 | void setenv_to_envp(char **envp, const char *key, const char *value); 17 | 18 | // All environment variables used by clade 19 | #define CLADE_INTERCEPT_OPEN_ENV "CLADE_INTERCEPT_OPEN" 20 | #define CLADE_INTERCEPT_EXEC_ENV "CLADE_INTERCEPT" 21 | #define CLADE_ID_FILE_ENV "CLADE_ID_FILE" 22 | #define CLADE_PARENT_ID_ENV "CLADE_PARENT_ID" 23 | #define CLADE_UNIX_ADDRESS_ENV "CLADE_UNIX_ADDRESS" 24 | #define CLADE_INET_HOST_ENV "CLADE_INET_HOST" 25 | #define CLADE_INET_PORT_ENV "CLADE_INET_PORT" 26 | #define CLADE_PREPROCESS_ENV "CLADE_PREPROCESS" 27 | #define CLADE_ENV_VARS_ENV "CLADE_ENV_VARS" 28 | // Do not forget to add new variables to clade_envs inside env.c 29 | 30 | #endif /* ENV_H */ 31 | -------------------------------------------------------------------------------- /clade/intercept/unix/lock.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "env.h" 6 | 7 | static FILE *f; 8 | 9 | void clade_lock(void) { 10 | char *id_file = getenv_or_fail(CLADE_ID_FILE_ENV); 11 | 12 | f = fopen(id_file, "r"); 13 | if (!f) { 14 | fprintf(stderr, "Couldn't open %s file\n", id_file); 15 | exit(EXIT_FAILURE); 16 | } 17 | 18 | flock(fileno(f), LOCK_EX); 19 | } 20 | 21 | void clade_unlock(void) { 22 | flock(fileno(f), LOCK_UN); 23 | fclose(f); 24 | } 25 | -------------------------------------------------------------------------------- /clade/intercept/unix/lock.h: -------------------------------------------------------------------------------- 1 | #ifndef LOCK_H 2 | #define LOCK_H 3 | 4 | extern void clade_lock(void); 5 | extern void clade_unlock(void); 6 | 7 | #endif /* LOCK_H */ 8 | -------------------------------------------------------------------------------- /clade/intercept/unix/which.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 3 | * Ivannikov Institute for System Programming of the Russian Academy of Sciences 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "which.h" 24 | 25 | // Lookup executable `name` within the PATH environment variable 26 | char *which(const char *name) { 27 | return which_path(name, getenv("PATH")); 28 | } 29 | 30 | // Lookup executable `name` within `path` 31 | char *which_path(const char *name, const char *_path) { 32 | char *path = strdup(_path); 33 | 34 | if (!path) 35 | return NULL; 36 | 37 | char *tok = strtok(path, WHICH_DELIMITER); 38 | 39 | while (tok) { 40 | // path 41 | int len = strlen(tok) + 2 + strlen(name); 42 | char *file = malloc(len); 43 | 44 | if (!file) { 45 | free(path); 46 | return NULL; 47 | } 48 | 49 | sprintf(file, "%s/%s", tok, name); 50 | 51 | // executable 52 | if (!access(file, X_OK)) { 53 | free(path); 54 | return file; 55 | } 56 | 57 | // next token 58 | tok = strtok(NULL, WHICH_DELIMITER); 59 | free(file); 60 | } 61 | 62 | free(path); 63 | 64 | return NULL; 65 | } 66 | -------------------------------------------------------------------------------- /clade/intercept/unix/which.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 3 | * Ivannikov Institute for System Programming of the Russian Academy of Sciences 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef WHICH_H 19 | #define WHICH_H 20 | 21 | 22 | // delimiter 23 | #ifdef _WIN32 24 | #define WHICH_DELIMITER ";" 25 | #else 26 | #define WHICH_DELIMITER ":" 27 | #endif 28 | 29 | extern char *which(const char *name); 30 | extern char *which_path(const char *name, const char *path); 31 | 32 | 33 | #endif /* WHICH_H */ 34 | -------------------------------------------------------------------------------- /clade/intercept/unix/wrapper.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 3 | * Ivannikov Institute for System Programming of the Russian Academy of Sciences 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "env.h" 25 | #include "data.h" 26 | #include "which.h" 27 | 28 | #define wrapper_postfix ".clade" 29 | 30 | 31 | int main(int argc, char **argv, char **envp) { 32 | char *original_exe = malloc(strlen(argv[0]) + strlen(wrapper_postfix) + 1); 33 | sprintf(original_exe, "%s%s", argv[0], wrapper_postfix); 34 | 35 | // Copy envp, so we can safely modify it later 36 | // All missing Clade environment variables will be added to "new_envp" 37 | // from "environ" if they were absent in "evnp" 38 | char **new_envp = copy_envp((char **)envp); 39 | 40 | /* First case: original executable file was renamed 41 | * (.clade extension was added to its name) 42 | * and symlink to the wrapper was added instead. 43 | */ 44 | if(access(original_exe, F_OK) != -1) { 45 | // Intercept call only if clade-intercept is working 46 | if (getenv_from_envp(new_envp, CLADE_INTERCEPT_EXEC_ENV)) { 47 | char *which = realpath(original_exe, NULL); 48 | 49 | if (!which) { 50 | fprintf(stderr, "which is empty\n"); 51 | exit(EXIT_FAILURE); 52 | } 53 | 54 | // strip wrapper_postfix extension 55 | which[strlen(which) - strlen(wrapper_postfix)] = 0; 56 | intercept_exec_call(which, (char const *const *)argv, new_envp); 57 | } 58 | 59 | // First argument must be a valid path, not just a filename 60 | argv[0] = original_exe; 61 | // Execute original file 62 | return execve(original_exe, argv, new_envp); 63 | } else { 64 | // Otherwise directory with wrappers is located in the PATH variable 65 | char *path = strstr(strdup(getenv("PATH")), WHICH_DELIMITER); 66 | char *which = which_path(basename(argv[0]), path); 67 | 68 | if (!which) { 69 | fprintf(stderr, "which is empty\n"); 70 | exit(EXIT_FAILURE); 71 | } 72 | 73 | if (getenv_from_envp(new_envp, CLADE_INTERCEPT_EXEC_ENV)) { 74 | intercept_exec_call(which, (char const *const *)argv, new_envp); 75 | } 76 | 77 | // First argument must be a valid path, not just a filename 78 | argv[0] = which; 79 | return execve(which, argv, new_envp); 80 | } 81 | 82 | fprintf(stderr, "Something went wrong\n"); 83 | exit(EXIT_FAILURE); 84 | } 85 | -------------------------------------------------------------------------------- /clade/intercept/windows/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.3) 2 | 3 | add_executable(debugger debugger.cpp) 4 | add_library(client_win STATIC client.cpp) 5 | target_link_libraries(debugger client_win) 6 | 7 | if(MSVC) 8 | add_definitions(-D_CRT_SECURE_NO_WARNINGS) 9 | add_compile_options(/O2) 10 | endif() 11 | 12 | set_target_properties(debugger client_win PROPERTIES CXX_STANDARD 11) 13 | -------------------------------------------------------------------------------- /clade/intercept/windows/client.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 3 | * Ivannikov Institute for System Programming of the Russian Academy of Sciences 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #define WIN32_LEAN_AND_MEAN 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | // For socket client functionality need to link with Ws2_32.lib, Mswsock.lib, and Advapi32.lib 27 | #pragma comment(lib, "Ws2_32.lib") 28 | #pragma comment(lib, "Mswsock.lib") 29 | #pragma comment(lib, "AdvApi32.lib") 30 | 31 | #define DEFAULT_BUFLEN 1024 32 | 33 | 34 | void SendData(const wchar_t *wdata) 35 | { 36 | WSADATA wsaData; 37 | SOCKET ConnectSocket = INVALID_SOCKET; 38 | struct addrinfo *result = NULL, 39 | *ptr = NULL, 40 | hints; 41 | char recvbuf[DEFAULT_BUFLEN]; 42 | int r; 43 | int recvbuflen = DEFAULT_BUFLEN; 44 | 45 | char *host = getenv("CLADE_INET_HOST"); 46 | char *port = getenv("CLADE_INET_PORT"); 47 | 48 | if (!host || !port) 49 | { 50 | std::cerr << "Server adress is not specified" << std::endl; 51 | exit(EXIT_FAILURE); 52 | } 53 | 54 | // Initialize Winsock 55 | r = WSAStartup(MAKEWORD(2, 2), &wsaData); 56 | if (r != 0) 57 | { 58 | std::cerr << "WSAStartup failed: error code " << r << std::endl; 59 | exit(EXIT_FAILURE); 60 | } 61 | 62 | ZeroMemory(&hints, sizeof(hints)); 63 | hints.ai_family = AF_UNSPEC; 64 | hints.ai_socktype = SOCK_STREAM; 65 | hints.ai_protocol = IPPROTO_TCP; 66 | 67 | // Resolve the server address and port 68 | r = getaddrinfo(host, port, &hints, &result); 69 | if (r != 0) 70 | { 71 | std::cerr << "getaddrinfo failed: error code " << r << std::endl; 72 | WSACleanup(); 73 | exit(EXIT_FAILURE); 74 | } 75 | 76 | // Attempt to connect to an address until one succeeds 77 | for (ptr = result; ptr != NULL; ptr = ptr->ai_next) 78 | { 79 | 80 | // Create a SOCKET for connecting to server 81 | ConnectSocket = socket(ptr->ai_family, ptr->ai_socktype, 82 | ptr->ai_protocol); 83 | if (ConnectSocket == INVALID_SOCKET) 84 | { 85 | std::cerr << "Socket failed: error code " << GetLastError() << std::endl; 86 | WSACleanup(); 87 | exit(EXIT_FAILURE); 88 | } 89 | 90 | // Connect to server. 91 | r = connect(ConnectSocket, ptr->ai_addr, (int)ptr->ai_addrlen); 92 | if (r == SOCKET_ERROR) 93 | { 94 | closesocket(ConnectSocket); 95 | ConnectSocket = INVALID_SOCKET; 96 | continue; 97 | } 98 | break; 99 | } 100 | 101 | freeaddrinfo(result); 102 | 103 | if (ConnectSocket == INVALID_SOCKET) 104 | { 105 | std::cerr << "Unable to connect to server" << std::endl; 106 | WSACleanup(); 107 | exit(EXIT_FAILURE); 108 | } 109 | 110 | // convert wchar_t* to char* 111 | size_t dataLen = wcslen(wdata); 112 | size_t charsConverted; 113 | char *data = new char[dataLen + 1]; 114 | 115 | wcstombs_s(&charsConverted, data, dataLen + 1, wdata, dataLen); 116 | data[dataLen] = 0; 117 | 118 | // Send data 119 | r = send(ConnectSocket, data, (int)strlen(data), 0); 120 | if (r == SOCKET_ERROR) 121 | { 122 | std::cerr << "Send failed: error code " << GetLastError() << std::endl; 123 | closesocket(ConnectSocket); 124 | WSACleanup(); 125 | exit(EXIT_FAILURE); 126 | } 127 | 128 | // shutdown the connection since no more data will be sent 129 | r = shutdown(ConnectSocket, SD_SEND); 130 | if (r == SOCKET_ERROR) 131 | { 132 | std::cerr << "Shutdown failed: error code " << GetLastError() << std::endl; 133 | closesocket(ConnectSocket); 134 | WSACleanup(); 135 | exit(EXIT_FAILURE); 136 | } 137 | 138 | // Receive until the peer closes the connection 139 | do 140 | { 141 | r = recv(ConnectSocket, recvbuf, recvbuflen, 0); 142 | } while (r > 0); 143 | 144 | // cleanup 145 | closesocket(ConnectSocket); 146 | WSACleanup(); 147 | 148 | return; 149 | } 150 | -------------------------------------------------------------------------------- /clade/intercept/windows/client.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 3 | * Ivannikov Institute for System Programming of the Russian Academy of Sciences 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef CLIENT_H 19 | #define CLIENT_H 20 | 21 | extern void SendData(const wchar_t *wdata); 22 | 23 | #endif /* CLIENT_H */ 24 | -------------------------------------------------------------------------------- /clade/libinterceptor.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import sys 18 | 19 | from clade.abstract import Intercept 20 | 21 | LIB = os.path.join(os.path.dirname(__file__), "intercept", "lib") 22 | LIB64 = os.path.join(os.path.dirname(__file__), "intercept", "lib64") 23 | 24 | 25 | class Libinterceptor(Intercept): 26 | def _setup_env(self): 27 | env = super()._setup_env() 28 | 29 | libinterceptor = self.__find_libinterceptor() 30 | 31 | if sys.platform == "darwin": 32 | self.logger.debug("Set 'DYLD_INSERT_LIBRARIES' environment variable value") 33 | env["DYLD_INSERT_LIBRARIES"] = libinterceptor 34 | env["DYLD_FORCE_FLAT_NAMESPACE"] = "1" 35 | elif sys.platform == "linux": 36 | existing_preload = env.get("LD_PRELOAD") 37 | env["LD_PRELOAD"] = libinterceptor 38 | 39 | if existing_preload: 40 | env["LD_PRELOAD"] += " " + existing_preload 41 | 42 | existing_lpath = env.get("LD_LIBRARY_PATH") 43 | env["LD_LIBRARY_PATH"] = LIB64 + ":" + LIB 44 | 45 | if existing_lpath: 46 | env["LD_LIBRARY_PATH"] += ":" + existing_lpath 47 | 48 | self.logger.debug( 49 | "Set 'LD_PRELOAD' environment variable value as {!r}".format( 50 | env["LD_PRELOAD"] 51 | ) 52 | ) 53 | self.logger.debug( 54 | "Set LD_LIBRARY_PATH environment variable value as {!r}".format( 55 | env["LD_LIBRARY_PATH"] 56 | ) 57 | ) 58 | 59 | return env 60 | 61 | def __find_libinterceptor(self): 62 | if sys.platform == "linux": 63 | libinterceptor_name = "libinterceptor.so" 64 | elif sys.platform == "darwin": 65 | libinterceptor_name = "libinterceptor.dylib" 66 | else: 67 | raise NotImplementedError( 68 | "Libinterceptor doesn't work on {!r}".format(sys.platform) 69 | ) 70 | 71 | libinterceptor = os.path.join( 72 | os.path.dirname(__file__), "intercept", libinterceptor_name 73 | ) 74 | 75 | if not os.path.exists(libinterceptor): 76 | raise RuntimeError( 77 | "libinterceptor is not found in {!r}".format(libinterceptor) 78 | ) 79 | 80 | # Multilib support, Linux only 81 | path = os.path.join(LIB, libinterceptor_name) 82 | path64 = os.path.join(LIB64, libinterceptor_name) 83 | 84 | if os.path.exists(path) and os.path.exists(path64): 85 | libinterceptor = libinterceptor_name 86 | self.logger.debug( 87 | "Path to libinterceptor library locations: {!r}, {!r}".format( 88 | path, path64 89 | ) 90 | ) 91 | else: 92 | self.logger.debug( 93 | "Path to libinterceptor library location: {!r}".format(libinterceptor) 94 | ) 95 | 96 | return libinterceptor 97 | 98 | @Intercept.preprocess 99 | def execute(self): 100 | return super().execute() 101 | -------------------------------------------------------------------------------- /clade/scripts/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /clade/scripts/check.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import argparse 17 | import sys 18 | 19 | from clade import Clade 20 | 21 | 22 | def main(args=sys.argv[1:]): 23 | parser = argparse.ArgumentParser( 24 | description="Check that Clade working directory exists and not corrupted." 25 | ) 26 | 27 | parser.add_argument(dest="work_dir", help="path to the Clade working directory") 28 | 29 | args = parser.parse_args(args) 30 | 31 | c = Clade(args.work_dir) 32 | sys.exit(not c.work_dir_ok(log=True)) 33 | 34 | 35 | if __name__ == "__main__": 36 | main(sys.argv[1:]) 37 | -------------------------------------------------------------------------------- /clade/scripts/compilation_database.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import argparse 17 | import os 18 | import shutil 19 | import sys 20 | import tempfile 21 | 22 | from clade import Clade 23 | from clade.utils import load 24 | 25 | 26 | def parse_args(args, work_dir): 27 | parser = argparse.ArgumentParser() 28 | 29 | parser.add_argument( 30 | "-o", 31 | "--output", 32 | help="path to the FILE where compilation database will be saved", 33 | metavar="FILE", 34 | default="compile_commands.json", 35 | ) 36 | parser.add_argument( 37 | "-w", 38 | "--wrappers", 39 | help="enable intercepting mode based on wrappers (not available on Windows)", 40 | action="store_true", 41 | ) 42 | parser.add_argument( 43 | "-c", 44 | "--config", 45 | help="path to the JSON file with configuration", 46 | metavar="JSON", 47 | default=None, 48 | ) 49 | parser.add_argument( 50 | "-p", 51 | "--preset", 52 | help="name of the preset configuration", 53 | metavar="NAME", 54 | default="base", 55 | ) 56 | parser.add_argument( 57 | "--cmds", 58 | help="path to the file with intercepted commands", 59 | ) 60 | parser.add_argument( 61 | "-f", 62 | "--filter", 63 | help="filter irrelevant options", 64 | action="store_true", 65 | default=False, 66 | ) 67 | parser.add_argument( 68 | dest="command", nargs=argparse.REMAINDER, help="build command to run" 69 | ) 70 | 71 | args = parser.parse_args(args) 72 | 73 | if not args.command and not args.cmds: 74 | sys.exit("Build command is missing") 75 | 76 | if not args.cmds: 77 | args.cmds = os.path.join(work_dir, "cmds.txt") 78 | 79 | return args 80 | 81 | 82 | def prepare_conf(args): 83 | conf = dict() 84 | 85 | if args.config: 86 | try: 87 | conf = load(args.config) 88 | except FileNotFoundError: 89 | print("Configuration file is not found") 90 | sys.exit(-1) 91 | 92 | conf["log_level"] = "ERROR" 93 | conf["preset"] = args.preset 94 | conf["CDB.output"] = os.path.abspath(args.output) 95 | conf["CDB.filter_opts"] = args.filter 96 | conf["SrcGraph.requires"] = ["CC", "CL", "CXX"] 97 | conf["Compiler.get_deps"] = False 98 | 99 | return conf 100 | 101 | 102 | def main(args=sys.argv[1:]): 103 | work_dir = tempfile.mkdtemp() 104 | args = parse_args(args, work_dir) 105 | conf = prepare_conf(args) 106 | 107 | try: 108 | c = Clade(work_dir, args.cmds, conf, args.preset) 109 | except RuntimeError as e: 110 | raise SystemExit(e) 111 | 112 | if args.command and not os.path.isfile(args.cmds): 113 | c.intercept(args.command, use_wrappers=args.wrappers) 114 | 115 | if not os.path.exists(args.cmds): 116 | print("Something is wrong: file with intercepted commands is empty") 117 | sys.exit(-1) 118 | 119 | c.parse("CDB") 120 | 121 | shutil.rmtree(work_dir) 122 | 123 | 124 | if __name__ == "__main__": 125 | main(sys.argv[1:]) 126 | -------------------------------------------------------------------------------- /clade/scripts/pid_graph.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 Ilya Shchepetkov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | import os 17 | import re 18 | import sys 19 | 20 | from graphviz import Digraph 21 | 22 | from clade.cmds import iter_cmds 23 | 24 | 25 | class PidGraph: 26 | def __init__(self, args): 27 | self.output = args.output 28 | self.cmds_file = args.cmds_file 29 | 30 | def print(self): 31 | dot = Digraph(graph_attr={"rankdir": "LR"}, node_attr={"shape": "rectangle"}) 32 | 33 | cmds = list(iter_cmds(self.cmds_file)) 34 | 35 | for cmd in cmds: 36 | cmd_node = "[{}] {}".format(cmd["id"], cmd["which"]) 37 | dot.node(str(cmd["id"]), label=re.escape(cmd_node)) 38 | 39 | for cmd in cmds: 40 | for parent_cmd in [x for x in cmds if x["id"] == cmd["pid"]]: 41 | dot.edge(str(parent_cmd["id"]), str(cmd["id"])) 42 | 43 | dot.render("pid_graph", directory=self.output, cleanup=True) 44 | 45 | 46 | def parse_args(args): 47 | parser = argparse.ArgumentParser( 48 | description="Create a pid graph based on ids and parent ids of intercepted commands." 49 | ) 50 | 51 | parser.add_argument( 52 | "-o", 53 | "--output", 54 | help="path to the output directory where generated graphs will be saved", 55 | metavar="DIR", 56 | default=os.path.curdir, 57 | ) 58 | 59 | parser.add_argument( 60 | "cmds_file", 61 | help="path to the Clade cmds.txt file", 62 | metavar="FILE", 63 | ) 64 | 65 | args = parser.parse_args(args) 66 | 67 | return args 68 | 69 | 70 | def main(args=None): 71 | if not args: 72 | args = sys.argv[1:] 73 | 74 | f = PidGraph(parse_args(args)) 75 | f.print() 76 | 77 | 78 | if __name__ == "__main__": 79 | main() 80 | -------------------------------------------------------------------------------- /clade/scripts/stats.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | 18 | 19 | from clade.cmds import get_stats 20 | 21 | 22 | def print_cmds_stats(args=sys.argv[1:]): 23 | if not args: 24 | sys.exit("Path to the json file with intercepted commands is missing") 25 | 26 | stats = get_stats(args[0]) 27 | 28 | total_count = sum(stats.values()) 29 | for key in sorted(stats, key=stats.get): 30 | print("{}: {}".format(stats[key], key)) 31 | 32 | print("-------------" + "-" * len(str(total_count))) 33 | print("Total count: {}".format(total_count)) 34 | -------------------------------------------------------------------------------- /clade/server.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import multiprocessing 17 | import threading 18 | import os 19 | import socketserver 20 | import sys 21 | import tempfile 22 | 23 | from clade.utils import get_logger 24 | from clade.extensions.abstract import Extension 25 | from clade.cmds import split_cmd, join_cmd 26 | 27 | if sys.platform == "linux" or sys.platform == "darwin": 28 | parent = socketserver.UnixStreamServer 29 | else: 30 | parent = socketserver.TCPServer 31 | 32 | 33 | # Forking and threading versions can be created using 34 | # the ForkingMixIn and ThreadingMixIn mix-in classes. 35 | # For instance, a forking CladeSocketServer class is created as follows: 36 | # class SocketServer(ForkingMixIn, parent): 37 | class SocketServer(parent): 38 | class RequestHandler(socketserver.StreamRequestHandler): 39 | def handle(self): 40 | data = self.rfile.readline().strip().decode("utf-8") 41 | cmd = split_cmd(data) 42 | 43 | for ext in self.extensions: 44 | ext.preprocess(cmd) 45 | 46 | data = join_cmd(cmd) 47 | 48 | with open(self.output, "a") as clade_fh: 49 | clade_fh.write(data + "\n") 50 | 51 | def __init__(self, address, output, conf): 52 | self.process = None 53 | # Variable to store file object of UNIX socket parent directory 54 | self.socket_fh = None 55 | 56 | rh = SocketServer.RequestHandler 57 | 58 | rh.output = output 59 | 60 | # Request handler must have access to extensions 61 | extensions = [] 62 | for cls in Extension.get_all_extensions(): 63 | try: 64 | extensions.append(cls(conf.get("work_dir", "Clade"), conf)) 65 | except Exception: 66 | # Some extension classes are abstract and can't be instantiated 67 | continue 68 | rh.extensions = extensions 69 | 70 | super().__init__(address, rh) 71 | 72 | def start(self): 73 | if sys.platform == "win32" or ( 74 | sys.platform == "darwin" and sys.version_info[1] >= 8 75 | ): 76 | self.process = threading.Thread(target=self.serve_forever) 77 | else: 78 | self.process = multiprocessing.Process(target=self.serve_forever) 79 | self.process.daemon = True 80 | self.process.start() 81 | 82 | def terminate(self): 83 | # if UNIX socket was used, it's parent directory needs to be closed 84 | if self.socket_fh: 85 | self.socket_fh.close() 86 | 87 | 88 | class PreprocessServer: 89 | def __init__(self, conf, output): 90 | self.conf = conf 91 | self.output = output 92 | self.logger = get_logger("Server", conf=self.conf) 93 | self.server = self.__prepare() 94 | self.env = self.__setup_env() 95 | 96 | def __prepare(self): 97 | if sys.platform == "linux" or sys.platform == "darwin": 98 | self.logger.debug("UNIX socket will be used") 99 | server = self.__prepare_unix() 100 | else: 101 | self.logger.debug("INET socket will be used") 102 | server = self.__prepare_inet() 103 | 104 | return server 105 | 106 | def __prepare_unix(self): 107 | # Create temporary directory with random name to store UNIX socket 108 | f = tempfile.TemporaryDirectory() 109 | name = os.path.join(f.name, "clade.sock") 110 | self.conf["Server.address"] = name 111 | 112 | server = SocketServer(name, self.output, self.conf) 113 | 114 | # Without this file object will be closed automatically after exiting from this function 115 | server.sock_fh = f 116 | 117 | return server 118 | 119 | def __prepare_inet(self): 120 | self.conf["Server.host"] = self.conf.get("Server.host", "localhost") 121 | self.conf["Server.port"] = self.conf.get("Server.port", "0") 122 | 123 | server = SocketServer( 124 | (self.conf["Server.host"], int(self.conf["Server.port"])), 125 | self.output, 126 | self.conf, 127 | ) 128 | 129 | # If "Server.port" is 0, than dynamic port assignment is used and the value needs to be updated 130 | self.conf["Server.port"] = str(server.server_address[1]) 131 | 132 | return server 133 | 134 | def __setup_env(self): 135 | env = os.environ.copy() 136 | # Windows doesn't support UNIX sockets 137 | if sys.platform == "linux" or sys.platform == "darwin": 138 | env.update({"CLADE_UNIX_ADDRESS": self.conf["Server.address"]}) 139 | else: 140 | env.update({"CLADE_INET_HOST": self.conf["Server.host"]}) 141 | env.update({"CLADE_INET_PORT": self.conf["Server.port"]}) 142 | 143 | env.update({"CLADE_PREPROCESS": "true"}) 144 | 145 | return env 146 | 147 | def start(self): 148 | # Create separate server process 149 | self.server.start() 150 | 151 | def terminate(self): 152 | self.server.terminate() 153 | -------------------------------------------------------------------------------- /clade/types/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /clade/types/nested_dict.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import collections 17 | 18 | 19 | def nested_dict(): 20 | return collections.defaultdict(nested_dict) 21 | 22 | 23 | def traverse(ndict, depth, restrict=None, allow_smaller=False): 24 | """Traverse nested dictionary and yield list of its elements. 25 | 26 | Args: 27 | depth: limit depth of the dictionary to traverse. 28 | restrict: ability to restrict output by specifying the exact value 29 | that must be at a specified position of the output list. 30 | Example: restrict={3: "calls"}. 31 | allow_smaller: allow to return list of the size smaller then required 32 | (if the dictionary is not uniform). 33 | """ 34 | 35 | if not restrict: 36 | restrict = dict() 37 | 38 | for l in __traverse(ndict, depth): 39 | if not restrict: 40 | if allow_smaller or len(l) == depth: 41 | yield l 42 | continue 43 | 44 | allow = True 45 | 46 | for key in restrict: 47 | if key <= len(l) and l[key - 1] != restrict[key]: 48 | allow = False 49 | 50 | if allow and (allow_smaller or len(l) == depth): 51 | yield l 52 | 53 | 54 | def __traverse(ndict, depth): 55 | if depth == 0: 56 | return [] 57 | 58 | if not isinstance(ndict, dict): 59 | yield [ndict] 60 | return 61 | 62 | for key in ndict: 63 | r = False 64 | 65 | for l in __traverse(ndict[key], depth - 1): 66 | yield [key] + l 67 | r = True 68 | 69 | if not r: 70 | yield [key] 71 | -------------------------------------------------------------------------------- /clade/types/path_tree.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | class PathTree: 18 | KEY = "__val__" 19 | 20 | def __init__(self): 21 | self.data = dict() 22 | 23 | def __setitem__(self, key: str, value): 24 | data = self.data 25 | 26 | for new_key in key.split("/"): 27 | if not new_key: 28 | continue 29 | 30 | data = self.__get_or_create(data, new_key, dict()) 31 | 32 | data[self.KEY] = value 33 | 34 | def __getitem__(self, key): 35 | return self.__get_or_create(self.__getitem_deep(key), self.KEY, None) 36 | 37 | def __getitem_deep(self, key): 38 | data = self.data 39 | 40 | for new_key in key.split("/"): 41 | if not new_key: 42 | continue 43 | 44 | data = data[new_key] 45 | 46 | return data 47 | 48 | def __get_or_create(self, data, key, value=None): 49 | if key not in data: 50 | data[key] = value 51 | 52 | return data[key] 53 | 54 | def __contains__(self, key): 55 | return self.get(key) is not None 56 | 57 | def __iter__(self): 58 | yield from self.__deep_iter() 59 | 60 | def __deep_iter(self, k=""): 61 | data = self.__getitem_deep(k) 62 | 63 | for key in data: 64 | if key == self.KEY: 65 | yield k 66 | continue 67 | 68 | if k: 69 | yield from self.__deep_iter(k + "/" + key) 70 | else: 71 | yield from self.__deep_iter("/" + key) 72 | 73 | def keys(self): 74 | return [x for x in self.__deep_iter()] 75 | 76 | def update(self, path_tree): 77 | raise NotImplementedError 78 | 79 | # if path_tree contains "/usr", and self contains "/usr/local" paths 80 | # then the following line of code will overwrite it: 81 | # self.data.update(path_tree.data) 82 | # TODO: fix it 83 | 84 | def get(self, key, default_value=None): 85 | try: 86 | return self.__getitem__(key) 87 | except KeyError: 88 | return default_value 89 | -------------------------------------------------------------------------------- /clade/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import array 17 | import logging 18 | import pkg_resources 19 | import os 20 | import re 21 | import subprocess 22 | import sys 23 | import orjson 24 | 25 | 26 | def get_logger(name, with_name=True, conf=None): 27 | if not conf: 28 | conf = dict() 29 | 30 | logger = logging.getLogger(name) 31 | 32 | # Remove all old handlers 33 | for handler in logger.handlers[:]: 34 | logger.removeHandler(handler) 35 | 36 | if with_name: 37 | formatter = logging.Formatter( 38 | "%(asctime)s clade {}: %(message)s".format(name), "%H:%M:%S" 39 | ) 40 | else: 41 | formatter = logging.Formatter("%(asctime)s clade: %(message)s", "%H:%M:%S") 42 | 43 | stream_handler = logging.StreamHandler(stream=sys.stdout) 44 | stream_handler.setFormatter(formatter) 45 | logger.addHandler(stream_handler) 46 | 47 | if conf.get("work_dir") and os.access(conf.get("work_dir", "."), os.W_OK): 48 | try: 49 | log_file = os.path.join(conf["work_dir"], "clade.log") 50 | log_file = os.path.abspath(log_file) 51 | os.makedirs(os.path.dirname(log_file), exist_ok=True) 52 | 53 | file_handler = logging.FileHandler(log_file, delay=True) 54 | file_handler.setFormatter(formatter) 55 | logger.addHandler(file_handler) 56 | except (PermissionError, OSError): 57 | # Can't write to clade.log file if working directory is read only 58 | pass 59 | 60 | logger.setLevel(conf.get("log_level", "INFO")) 61 | 62 | return logger 63 | 64 | 65 | def merge_preset_to_conf(preset_name, conf): 66 | preset_file = os.path.join( 67 | os.path.dirname(__file__), "extensions", "presets", "presets.json" 68 | ) 69 | 70 | presets = load(preset_file) 71 | 72 | if preset_name not in presets: 73 | raise RuntimeError("Preset {!r} is not found".format(preset_name)) 74 | 75 | preset_conf = presets[preset_name] 76 | parent_preset = preset_conf.get("extends") 77 | 78 | if parent_preset: 79 | preset_conf = merge_preset_to_conf(parent_preset, preset_conf) 80 | 81 | preset_conf.update(conf) 82 | 83 | return preset_conf 84 | 85 | 86 | def get_clade_version(): 87 | version = pkg_resources.get_distribution("clade").version 88 | location = pkg_resources.get_distribution("clade").location 89 | 90 | if not os.path.exists(os.path.join(location, ".git")): 91 | return version 92 | 93 | try: 94 | desc = ["git", "describe", "--tags", "--dirty"] 95 | version = subprocess.check_output( 96 | desc, cwd=location, stderr=subprocess.DEVNULL, universal_newlines=True 97 | ).strip() 98 | finally: 99 | return version 100 | 101 | 102 | def get_program_version(program, version_arg="--version"): 103 | version = "unknown" 104 | try: 105 | version = subprocess.check_output( 106 | [program, version_arg], stderr=subprocess.DEVNULL, universal_newlines=True 107 | ).strip() 108 | finally: 109 | if version.startswith("gcc"): 110 | version = re.sub(r"\nCopyright[\s\S]*", "", version) 111 | return version 112 | 113 | 114 | def array_hook(obj): 115 | if isinstance(obj, array.array): 116 | return list(obj) 117 | raise TypeError 118 | 119 | 120 | def dump(data, path): 121 | with open(path, "wb") as fh: 122 | fh.write(orjson.dumps(data, default=array_hook)) 123 | 124 | 125 | def load(path): 126 | with open(path, "rb") as f: 127 | return orjson.loads(f.read()) 128 | -------------------------------------------------------------------------------- /docs/dev.md: -------------------------------------------------------------------------------- 1 | # Documentation for Clade developers 2 | 3 | ## Installing prerequisites for development 4 | 5 | The following sections assume that you already installed all the required 6 | Python packages for testing, measuring code coverage and profiling, using the 7 | following command: 8 | 9 | ``` shell 10 | python3 -m pip install -e ".[dev]" 11 | ``` 12 | 13 | Note that this command installs Clade in "editable" mode directly from the 14 | repository (you need to clone it on your computer beforehand and execute 15 | the command from the root of the repository). 16 | 17 | ## Testing 18 | 19 | You can check that Clade works as expected on your machine by running 20 | the test suite from the repository (doesn't work on Windows yet): 21 | 22 | ``` shell 23 | pytest 24 | ``` 25 | 26 | ## Disable parallelism 27 | 28 | Some issues in Clade are hard to debug, because a lot of stuff happens inside 29 | child processes. You can disable parallelism by setting *CLADE_DEBUG* 30 | environment value. 31 | 32 | ## Measuring code coverage 33 | 34 | To measure coverage you need to execute the following commands: 35 | 36 | ``` shell 37 | coverage run -m pytest && coverage combine && coverage html 38 | ``` 39 | 40 | Results can be observed by opening generated *htmlcov/index.html* file in the browser. 41 | 42 | ## Extensions 43 | 44 | Most of the functionality in Clade is implemented as *extensions*. Each 45 | extension implements and uses a simple API: 46 | 47 | - Extension class must be a child of an `clade.extensions.abstract.Abstract` class. 48 | - Extension must implement a `parse(self, cmds_file)` method, 49 | which will serve as an entry point. `cmds_file` may be used to 50 | parse intercepted commands, but this is not required. 51 | - Extension may interact with other extensions by specifying them in the 52 | `requires` class attribute. Its value should be the list of names of 53 | required extensions, by default it is empty. 54 | This will regulate the correct order between extensions: of one extensions 55 | is required by other, then it will be executed first. 56 | Interaction with other extensions is possible via `self.extensions` 57 | dictionary, where keys are the names of required extensions, and values 58 | are corresponding objects. This way you can easily access their API. 59 | - Each extension has a *working directory*, which can be used to store files. 60 | It is available via `self.work_dir`. 61 | - `Abstract` class implements a bunch of helpful methods, which can be used to 62 | simplify various things. For example, it implements an API to execute jobs 63 | on intercepted commands in parallel. 64 | -------------------------------------------------------------------------------- /docs/extensions.md: -------------------------------------------------------------------------------- 1 | # Extensions 2 | 3 | Most of the functionality in Clade is implemented as *extensions*. 4 | These extensions can be separated into the following groups: 5 | 6 | ## Extensions that parse a particular type of build commands 7 | 8 | - C compilation commands: `CC` and `CL` extensions. 9 | - C++ compilation commands: `CXX` extension. 10 | - `ar` commands, which are used to create, modify, and extract 11 | from archives: `AR` extension. 12 | - Assembler commands: `AS` extension 13 | - Link commands: `LN` extension. 14 | - Move commands: `MV` extension. 15 | - Object copy commands: `Objcopy` extension. 16 | - Copy commands (Windows-specific): `Copy` extension. 17 | 18 | Each of these extensions has a corresponding [configuration](configuration.md) 19 | option, which precisely describes which commands it parse. 20 | These options can be found on the [presets.json](../clade/extensions/presets/presets.json) file. 21 | 22 | These extensions output a list of json files with parsed commands. 23 | Each parsed command consists of a list of input files, a list of 24 | output files, and a list of options. Some extensions may add 25 | additional data, for example, `CC` extension also adds a list 26 | of dependencies (header files that were used by the intercepted 27 | compile command). 28 | 29 | ## Extensions that generate information by using a list of all intercepted commands 30 | 31 | - `PidGraph` extension, which produces parent-child graph between intercepted 32 | commands. 33 | 34 | More about it you can read in the [usage docs](usage.md). 35 | 36 | ## Extensions that generate information using data from other extensions 37 | 38 | Extensions may interact with each other, and thus refine and produce data 39 | from several sources. Extensions in this group are: 40 | 41 | - `CmdGraph` extension, which uses parsed commands from other extensions, 42 | and connects them by their input and output files, creating dependency 43 | graph between these commands. 44 | - `SrcGraph` extension, which shows for each source file (C or C++ only) 45 | a list of commands in which it was compiled, and a list of commands 46 | in which it was indirectly used. 47 | - `Alternatives` extension, which parses build commands that create 48 | *identical* file copies (ln, cp, install, etc.) and allows to determine 49 | that different paths are in fact related to the same file. 50 | - `CDB` extension, which is used to create compilation databases. 51 | 52 | ## Helper extensions 53 | 54 | These are extensions that implement functionality useful for other extensions. 55 | 56 | - `Path` extension: provides functions to normalize file paths. 57 | - `Storage` extension: adds ability to save files to the Clade 58 | working directory, and to retrieve them later. This allows the following 59 | scenario: Clade intercepts build process on one computer, and then parses 60 | the ouput on another, using cmds.txt file and file copies stored in the 61 | Storage. 62 | 63 | ## Extensions that use information about the source code (C only) 64 | 65 | - `Info` is the main extension here: it uses [CIF](https://github.com/17451k/cif) to parse source code and extract various info about it. All extensions 66 | in this group are using data from the `Info` extension. 67 | - `Functions` extension parses information about function definitions and 68 | declarations. 69 | - `Callgraph` extension creates a function callgraph. 70 | - `Macros` extension pares information about macros definitions and expansions. 71 | - `Typedefs` extension parses information about typedefs. 72 | - `UsedIn` extension parses information about functional pointers 73 | - `CallsByPtr` extension parses information about function calls by pointers. 74 | - `Variables` extension parses information about global variable initializations. 75 | 76 | ## Parent extensions 77 | 78 | These extensions are not meant to be used as is, they are parent classes for other extensions. They usually implement some common functionality. 79 | 80 | - `Abstract` extension (parent of all other extensions). 81 | - `Common` extension (for extensions that parse intercepted 82 | commands into input, output files, and a list of options. 83 | - `Link` extension (parent of compiler extensions that also 84 | parse linker commands) 85 | - `CommonInfo` extension, implements common functionality for extensions 86 | that parse output of `Info` extension. 87 | 88 | -------------------------------------------------------------------------------- /docs/pics/cmd_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/17451k/clade/b06da1c41cf14e58781b5735fbcab36190eaf0d3/docs/pics/cmd_graph.png -------------------------------------------------------------------------------- /docs/pics/libinterceptor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/17451k/clade/b06da1c41cf14e58781b5735fbcab36190eaf0d3/docs/pics/libinterceptor.png -------------------------------------------------------------------------------- /docs/pics/pid_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/17451k/clade/b06da1c41cf14e58781b5735fbcab36190eaf0d3/docs/pics/pid_graph.png -------------------------------------------------------------------------------- /docs/scripts.md: -------------------------------------------------------------------------------- 1 | # Scripts 2 | 3 | In the [scripts](../clade/scripts/) directory there are several scripts that 4 | showcase the usage of the Clade API (both public and 5 | internal ones). Thus, these scripts may be seen as examples, but they may 6 | be useful on their own as well. 7 | 8 | The following scripts are available: 9 | 10 | - `clade-check` simply checks that Clade working directory exists and is not 11 | corrupted. 12 | - `clade-cdb` can be used to create compilation database. 13 | - `clade-trace` can be used to visualize callgraph between specified functions. 14 | - `clade-file-graph` can be used to visualize file dependencies between 15 | intercepted commands. 16 | - `clade-pid-graph` can be used to visualize parent-child relationship between 17 | intercepted commands. 18 | - `clade-cmds` outputs some statistics based on the `cmds.txt` file. 19 | - `clade-diff` can output diff between 2 Clade working directories. 20 | (Though, this one probably isn't working right now). 21 | -------------------------------------------------------------------------------- /docs/troubleshooting.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting 2 | 3 | ## File with intercepted commands is empty 4 | 5 | Access control mechanisms on different operating systems might disable 6 | library injection that is used by Clade to intercept build commands: 7 | 8 | - SELinux on Fedora, CentOS, RHEL; 9 | - System Integrity Protection on macOS; 10 | - Mandatory Integrity Control on Windows (disables similar mechanisms) 11 | 12 | A solution is to use another intercepting mechanism that is based on 13 | *wrappers* (see [usage documentation](usage.md)). 14 | 15 | ## File with intercepted commands is not complete 16 | 17 | Sometimes some commands are intercepted, so file `cmds.txt` is present and not 18 | empty, but other commands are clearly missing. 19 | Such behaviour should be reported so the issue can be fixed, but until then 20 | you can try to use another intercepting mechanism that is based on 21 | *wrappers*. 22 | 23 | ## Wrong ELF class 24 | 25 | Build command intercepting may result in the following error: 26 | 27 | ``` 28 | ERROR: ld.so: object 'libinterceptor.so' from LD_PRELOAD cannot be preloaded (wrong ELF class: ELFCLASS64): ignored. 29 | ``` 30 | 31 | It is because your project leverages multilib capabilities, but 32 | `libinterceptor` library that is used to intercept build commands is 33 | compiled without multilib support. 34 | You need to install `gcc-multilib` (Ubuntu) or `gcc-32bit` (openSUSE) package 35 | and **reinstall Clade**. `libinterceptor` library will be recompiled and your 36 | issue will be fixed. 37 | 38 | ## Not all intercepted compilation commands are parsed 39 | 40 | The reason is because `CC` extension that parse intercepted commands cannot 41 | identify a command as a compilation command. You can help it by specifying 42 | `CC.which_list` [configuration](configuration.md) option, in which you should write a list of 43 | [regexes](https://en.wikipedia.org/wiki/Regular_expression) that will match your compiler. For example, if path to your compiler 44 | is `~/.local/bin/c_compiler`, than `CC.which_list` may be set like this: 45 | 46 | ``` 47 | "CC.which_list": ["c_compiler$"] 48 | ``` 49 | 50 | If you want to parse not only commands executed by your compiler, but by system 51 | `gcc` as well, then you can add it to the list too: 52 | 53 | ``` 54 | "CC.which_list": ["c_compiler$", ""gcc$"] 55 | ``` 56 | 57 | How to set configuration option is described in the [configuration](configuration.md) section of 58 | this documentation. 59 | 60 | ## Compilation database miss some commands 61 | 62 | Same as above. 63 | 64 | ## Command graph is not connected properly 65 | 66 | Most certainly it is due to the fact that some type of commands is unparsed. 67 | If there is an extension in Clade that can parse them, then you will need 68 | to specify it via the option "CmdGraph.requires": 69 | 70 | 71 | ``` json 72 | { 73 | "CmdGraph.requires": ["CC", "LD", "MV", "AR", "Objcopy"] 74 | } 75 | ``` 76 | 77 | Otherwise such extension should be developed. 78 | 79 | Similar problems with the *source graph* and the *call graph* can be fixed 80 | via the same option, since they use the *command graph* internally. 81 | 82 | ## BitBake support 83 | 84 | BitBake limits environment of the worker processes it creates, which 85 | doesn't allow Clade to correctly intercept build commands. To overcome it, 86 | you can use [BB_ENV_EXTRAWHITE](https://www.yoctoproject.org/docs/1.6/bitbake-user-manual/bitbake-user-manual.html#var-BB_ENV_EXTRAWHITE) 87 | BitBake environment variable, which specifies a set of variables to pass 88 | to the build processes: 89 | 90 | ``` shell 91 | export BB_ENV_EXTRAWHITE="CLADE_INTERCEPT CLADE_ID_FILE CLADE_PARENT_ID LD_PRELOAD LD_LIBRARY_PATH $BB_ENV_EXTRAWHITE" 92 | clade bitbake 93 | ``` 94 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | markers = 3 | cif: tests that require CIF 4 | -------------------------------------------------------------------------------- /ruff.toml: -------------------------------------------------------------------------------- 1 | [tool.ruff] 2 | ignore = ["E501"] 3 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import pytest 18 | import shutil 19 | import tempfile 20 | 21 | from clade import Clade 22 | from clade.intercept import intercept 23 | from tests.test_intercept import test_project_make, test_project 24 | 25 | 26 | @pytest.fixture(scope="session") 27 | def cmds_file(): 28 | # Disable multiprocessing 29 | os.environ["CLADE_DEBUG"] = "1" 30 | 31 | with tempfile.NamedTemporaryFile() as fh: 32 | intercept(command=test_project_make, output=fh.name, use_wrappers=True) 33 | yield fh.name 34 | 35 | 36 | @pytest.fixture(scope="session") 37 | def envs_file(): 38 | # Disable multiprocessing 39 | os.environ["CLADE_DEBUG"] = "1" 40 | 41 | c = Clade(work_dir=test_project + "/clade") 42 | c.intercept(command=test_project_make, use_wrappers=True, intercept_envs=True) 43 | yield os.path.join(c.work_dir, "envs.txt") 44 | 45 | 46 | @pytest.fixture(scope="session") 47 | def clade_api(tmpdir_factory): 48 | tmpdir = tmpdir_factory.mktemp("Clade") 49 | 50 | c = Clade(tmpdir) 51 | c.intercept(command=test_project_make, use_wrappers=True, intercept_envs=True) 52 | c.parse_list(["CrossRef", "Variables", "Macros", "Typedefs", "CDB"]) 53 | 54 | yield c 55 | 56 | 57 | def pytest_collection_modifyitems(config, items): 58 | skip_cif = pytest.mark.skipif( 59 | not shutil.which("cif"), reason="cif is not installed" 60 | ) 61 | 62 | for item in items: 63 | if "cif" in item.keywords: 64 | item.add_marker(skip_cif) 65 | -------------------------------------------------------------------------------- /tests/test_abstract.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import pytest 17 | import os 18 | import unittest.mock 19 | 20 | from clade import Clade 21 | 22 | 23 | def test_cc_parallel(tmpdir, cmds_file): 24 | del os.environ["CLADE_DEBUG"] 25 | 26 | try: 27 | c = Clade(tmpdir, cmds_file) 28 | e = c.parse("CC") 29 | 30 | assert e.load_all_cmds() 31 | finally: 32 | os.environ["CLADE_DEBUG"] = "1" 33 | 34 | 35 | def test_cc_parallel_with_exception(tmpdir, cmds_file): 36 | del os.environ["CLADE_DEBUG"] 37 | 38 | try: 39 | # Force results() method of a future object to raise Exception 40 | with unittest.mock.patch("concurrent.futures.Future.result") as result_mock: 41 | result_mock.side_effect = Exception 42 | 43 | c = Clade(tmpdir, cmds_file) 44 | with pytest.raises(Exception): 45 | c.parse("CC") 46 | finally: 47 | os.environ["CLADE_DEBUG"] = "1" 48 | 49 | 50 | def test_cc_parallel_with_print(tmpdir, cmds_file): 51 | del os.environ["CLADE_DEBUG"] 52 | 53 | try: 54 | with unittest.mock.patch("sys.stdout.isatty") as isatty_mock: 55 | isatty_mock.return_value = True 56 | 57 | c = Clade(tmpdir, cmds_file) 58 | e = c.parse("CC") 59 | 60 | assert e.load_all_cmds() 61 | finally: 62 | os.environ["CLADE_DEBUG"] = "1" 63 | 64 | 65 | @pytest.mark.parametrize("force", [True, False]) 66 | def test_force(tmpdir, cmds_file, force): 67 | conf = {"force": force} 68 | 69 | c1 = Clade(tmpdir, cmds_file, conf=conf) 70 | c1.parse("CC") 71 | 72 | p_work_dir = os.path.join(str(tmpdir), "PidGraph") 73 | c_work_dir = os.path.join(str(tmpdir), "CC") 74 | 75 | p_mtime1 = os.stat(p_work_dir).st_mtime 76 | c_mtime1 = os.stat(c_work_dir).st_mtime 77 | 78 | c2 = Clade(tmpdir, cmds_file, conf=conf) 79 | c2.parse("CC") 80 | 81 | p_mtime2 = os.stat(p_work_dir).st_mtime 82 | c_mtime2 = os.stat(c_work_dir).st_mtime 83 | 84 | assert force != (p_mtime1 == p_mtime2) 85 | assert force != (c_mtime1 == c_mtime2) 86 | 87 | 88 | @pytest.mark.parametrize("clean", [True, False]) 89 | def test_parse_clean(tmpdir, cmds_file, clean): 90 | c = Clade(tmpdir, cmds_file) 91 | c.parse("CC", clean=clean) 92 | 93 | p_work_dir = os.path.join(str(tmpdir), "PidGraph") 94 | c_work_dir = os.path.join(str(tmpdir), "CC") 95 | 96 | p_mtime1 = os.stat(p_work_dir).st_mtime 97 | c_mtime1 = os.stat(c_work_dir).st_mtime 98 | 99 | c.parse("CC", clean=clean) 100 | 101 | p_mtime2 = os.stat(p_work_dir).st_mtime 102 | c_mtime2 = os.stat(c_work_dir).st_mtime 103 | 104 | assert clean != (c_mtime1 == c_mtime2) 105 | assert p_mtime1 == p_mtime2 106 | 107 | 108 | def test_check_conf_consistency(tmpdir, cmds_file): 109 | conf = {"PidGraph.filter_cmds_by_pid": True} 110 | 111 | c = Clade(tmpdir, cmds_file, conf=conf) 112 | c.parse("PidGraph") 113 | 114 | changed_conf = {"PidGraph.filter_cmds_by_pid": False} 115 | 116 | c = Clade(tmpdir, cmds_file, conf=changed_conf) 117 | with pytest.raises(RuntimeError): 118 | c.parse("CC") 119 | -------------------------------------------------------------------------------- /tests/test_ar.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from clade import Clade 17 | 18 | 19 | def test_ar(tmpdir, cmds_file): 20 | c = Clade(tmpdir, cmds_file) 21 | e = c.parse("AR") 22 | 23 | cmds = e.load_all_cmds(with_opts=True, with_raw=True) 24 | assert len(cmds) == 1 25 | assert len(cmds[0]["in"]) == 2 26 | assert len(cmds[0]["out"]) == 1 27 | assert len(cmds[0]["opts"]) == 1 28 | assert len(cmds[0]["command"]) == 5 29 | -------------------------------------------------------------------------------- /tests/test_as.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import re 17 | 18 | from clade import Clade 19 | 20 | 21 | def test_as(tmpdir, cmds_file): 22 | c = Clade(tmpdir, cmds_file) 23 | e = c.parse("AS") 24 | 25 | cmds = e.load_all_cmds(with_opts=True, with_raw=True) 26 | target_cmd = dict() 27 | 28 | for cmd in cmds: 29 | for cmd_in in cmd["in"]: 30 | if re.search("empty.s", cmd_in): 31 | target_cmd = cmd 32 | 33 | assert len(cmds) >= 1 34 | assert len(target_cmd["in"]) == 1 35 | assert len(target_cmd["out"]) == 1 36 | assert len(target_cmd["opts"]) == 2 37 | assert len(target_cmd["command"]) == 6 38 | -------------------------------------------------------------------------------- /tests/test_callgraph.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import pytest 17 | 18 | from clade import Clade 19 | from tests.test_project import main_c, zero_c 20 | 21 | 22 | def callgraph_is_ok(callgraph): 23 | call_line = 10 24 | match_type = 4 25 | 26 | assert {"match": match_type, "line": call_line} in callgraph[zero_c]["zero"][ 27 | "called_in" 28 | ][main_c]["main"] 29 | assert {"match": match_type, "line": call_line} in callgraph[main_c]["main"][ 30 | "calls" 31 | ][zero_c]["zero"] 32 | 33 | 34 | def callgraph_by_file_is_ok(callgraph, callgraph_by_zero_c): 35 | for file in callgraph: 36 | if file == zero_c: 37 | assert callgraph_by_zero_c[zero_c] == callgraph[zero_c] 38 | else: 39 | assert file not in callgraph_by_zero_c 40 | 41 | 42 | @pytest.mark.cif 43 | def test_callgraph(tmpdir, cmds_file): 44 | conf = {"CmdGraph.requires": ["CC", "MV"]} 45 | 46 | c = Clade(tmpdir, cmds_file, conf) 47 | e = c.parse("Callgraph") 48 | 49 | callgraph = e.load_callgraph() 50 | callgraph_by_zero_c = e.load_callgraph([zero_c]) 51 | 52 | callgraph_is_ok(callgraph) 53 | callgraph_by_file_is_ok(callgraph, callgraph_by_zero_c) 54 | -------------------------------------------------------------------------------- /tests/test_calls_by_ptr.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 Ilya Shchepetkov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | 17 | from clade import Clade 18 | from tests.test_project import zero_c 19 | 20 | 21 | def calls_by_ptr_is_ok(calls_by_ptr): 22 | assert calls_by_ptr[zero_c]["func_with_pointers"]["fp1"] == [17] 23 | assert calls_by_ptr[zero_c]["func_with_pointers"]["fp2"] == [17] 24 | 25 | 26 | @pytest.mark.cif 27 | def test_calls_by_ptr(tmpdir, cmds_file): 28 | conf = {"CmdGraph.requires": ["CC", "MV"]} 29 | 30 | c = Clade(tmpdir, cmds_file, conf) 31 | e = c.parse("CallsByPtr") 32 | 33 | calls_by_ptr = e.load_calls_by_ptr() 34 | 35 | calls_by_ptr_is_ok(calls_by_ptr) 36 | -------------------------------------------------------------------------------- /tests/test_cdb.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import pytest 18 | 19 | from clade import Clade 20 | from clade.scripts.compilation_database import main 21 | 22 | 23 | @pytest.mark.parametrize("filter_opts", [True, False]) 24 | def test_cdb(tmpdir, cmds_file, filter_opts): 25 | cdb_json = os.path.join(str(tmpdir), "cdb.json") 26 | 27 | c = Clade( 28 | tmpdir, cmds_file, conf={"CDB.output": cdb_json, "CDB.filter_opts": filter_opts} 29 | ) 30 | e = c.parse("CDB") 31 | 32 | cdb = e.load_cdb() 33 | assert cdb 34 | 35 | cc = c.parse("CC") 36 | assert len(cdb) >= len(list(cc.load_all_cmds())) 37 | 38 | for cmd in cdb: 39 | assert "directory" in cmd 40 | assert "arguments" in cmd 41 | assert "file" in cmd 42 | 43 | for arg in cmd["arguments"]: 44 | assert isinstance(arg, str) 45 | 46 | if filter_opts: 47 | assert "-fsyntax-only" not in cmd["arguments"] 48 | 49 | 50 | def test_cdb_main(tmpdir, cmds_file): 51 | main(["-o", os.path.join(str(tmpdir), "cdb.json"), "--cmds", cmds_file]) 52 | -------------------------------------------------------------------------------- /tests/test_cmd_graph.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import pytest 18 | import shutil 19 | 20 | from clade import Clade 21 | from tests.test_project import main_c, zero_c, tmp_main 22 | 23 | 24 | def test_cmd_graph_requires(tmpdir, cmds_file): 25 | conf = {"CmdGraph.requires": ["CC", "MV"]} 26 | 27 | c = Clade(tmpdir, cmds_file, conf) 28 | e = c.parse("CmdGraph") 29 | 30 | cmd_graph = e.load_cmd_graph() 31 | cmd_type = e.load_cmd_type() 32 | 33 | cmd_id = None 34 | for cmd in e.extensions["CC"].load_all_cmds(): 35 | if main_c in cmd["in"] and zero_c in cmd["in"] and tmp_main in cmd["out"]: 36 | cmd_id = cmd["id"] 37 | 38 | assert cmd_id 39 | assert cmd_graph 40 | assert cmd_type[cmd_id] == "CC" 41 | assert len(cmd_graph[cmd_id]["used_by"]) == 1 42 | assert cmd_graph[cmd_id]["using"] == [] 43 | 44 | used_by_id = cmd_graph[cmd_id]["used_by"][0] 45 | assert cmd_graph[used_by_id]["using"] == [cmd_id] 46 | 47 | 48 | def test_cmd_graph_empty_requires(tmpdir, cmds_file): 49 | conf = {"CmdGraph.requires": []} 50 | 51 | c = Clade(tmpdir, cmds_file, conf) 52 | 53 | with pytest.raises(RuntimeError): 54 | c.parse("CmdGraph") 55 | 56 | 57 | @pytest.mark.parametrize("as_picture", [True, False]) 58 | @pytest.mark.skipif(not shutil.which("dot"), reason="dot is not installed") 59 | def test_cmd_graph_as_picture(tmpdir, cmds_file, as_picture): 60 | conf = {"CmdGraph.as_picture": as_picture} 61 | 62 | c = Clade(tmpdir, cmds_file, conf) 63 | e = c.parse("CmdGraph") 64 | 65 | assert os.path.exists(e.pdf_file + ".pdf") == as_picture 66 | 67 | 68 | def test_cmd_graph_empty_conf(tmpdir, cmds_file): 69 | c = Clade(tmpdir, cmds_file) 70 | e = c.parse("CmdGraph") 71 | 72 | assert e.load_cmd_graph() 73 | -------------------------------------------------------------------------------- /tests/test_cmds.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import pytest 18 | import shutil 19 | 20 | from clade.cmds import ( 21 | iter_cmds, 22 | iter_cmds_by_which, 23 | open_cmds_file, 24 | get_build_dir, 25 | get_last_id, 26 | get_stats, 27 | join_cmd, 28 | get_all_cmds, 29 | ) 30 | from clade.scripts.stats import print_cmds_stats 31 | 32 | # TODO: Replace >= by == 33 | number_of_cmds = 5 34 | number_of_gcc_cmds = 2 35 | gcc_which = shutil.which("gcc") 36 | 37 | 38 | def test_bad_open(): 39 | with pytest.raises(RuntimeError): 40 | open_cmds_file("do_not_exist.txt") 41 | 42 | 43 | def test_iter(cmds_file): 44 | assert len(list(iter_cmds(cmds_file))) >= number_of_cmds 45 | 46 | 47 | def test_iter_by_which(cmds_file): 48 | assert len(list(iter_cmds_by_which(cmds_file, [gcc_which]))) >= number_of_gcc_cmds 49 | 50 | 51 | def test_get_build_dir(cmds_file): 52 | assert get_build_dir(cmds_file) == os.getcwd() 53 | 54 | 55 | def test_get_last_id(cmds_file): 56 | assert int(get_last_id(cmds_file)) >= number_of_cmds 57 | 58 | 59 | def test_get_stats(cmds_file): 60 | assert get_stats(cmds_file)[gcc_which] >= number_of_gcc_cmds 61 | 62 | 63 | def test_print_stats(cmds_file): 64 | print_cmds_stats([cmds_file]) 65 | 66 | 67 | def test_print_stats_bad(): 68 | with pytest.raises(SystemExit): 69 | print_cmds_stats([]) 70 | 71 | 72 | def test_join_cmd(cmds_file): 73 | with open(cmds_file, "r") as cmds_fh: 74 | for cmd in iter_cmds(cmds_file): 75 | assert join_cmd(cmd) == cmds_fh.readline().strip() 76 | 77 | 78 | def test_get_all_cmds(cmds_file): 79 | cmds = [] 80 | for cmd in iter_cmds(cmds_file): 81 | cmds.append(cmd) 82 | 83 | assert cmds == get_all_cmds(cmds_file) 84 | -------------------------------------------------------------------------------- /tests/test_cross_ref.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import pytest 17 | 18 | from clade import Clade 19 | from tests.test_project import main_c, zero_c, zero_h 20 | 21 | 22 | def ref_to_are_ok(ref_to): 23 | assert ref_to[main_c] 24 | assert ref_to[zero_c] 25 | 26 | assert ref_to[main_c]["decl_func"] 27 | assert ref_to[main_c]["def_func"] 28 | assert "def_macro" not in ref_to[main_c] 29 | 30 | assert ref_to[zero_c]["decl_func"] 31 | assert "def_func" not in ref_to[zero_c] 32 | assert ref_to[zero_c]["def_macro"] 33 | 34 | assert [[10, 11, 15], [zero_h, 1]] in ref_to[main_c]["decl_func"] 35 | assert [[9, 4, 9], [main_c, 4]] in ref_to[main_c]["def_func"] 36 | assert [[10, 11, 15], [zero_c, 6]] in ref_to[main_c]["def_func"] 37 | assert [[7, 11, 15], [zero_c, 4]] in ref_to[zero_c]["def_macro"] 38 | 39 | 40 | def filtered_ref_to_are_ok(rel_to, rel_to_main_c): 41 | for file in rel_to: 42 | if file == main_c: 43 | assert rel_to[main_c] == rel_to_main_c[main_c] 44 | else: 45 | assert file not in rel_to_main_c 46 | 47 | 48 | def ref_from_are_ok(ref_from): 49 | assert ref_from[main_c] 50 | assert ref_from[zero_c] 51 | assert ref_from[zero_h] 52 | 53 | assert ref_from[main_c]["call"] 54 | assert ref_from[zero_c]["call"] 55 | assert ref_from[zero_c]["expand"] 56 | assert ref_from[zero_h]["call"] 57 | 58 | assert [[4, 12, 17], [main_c, [9]]] in ref_from[main_c]["call"] 59 | assert [[6, 4, 8], [main_c, [10]]] in ref_from[zero_c]["call"] 60 | assert [[1, 11, 15], [main_c, [10]]] in ref_from[zero_h]["call"] 61 | assert [[3, 8, 18], [zero_c, [7]]] in ref_from[zero_c]["expand"] 62 | assert [[4, 8, 12], [zero_c, [7]]] in ref_from[zero_c]["expand"] 63 | 64 | 65 | def filtered_ref_from_are_ok(rel_from, rel_from_main_c): 66 | for file in rel_from: 67 | if file == main_c: 68 | assert rel_from[main_c] == rel_from_main_c[main_c] 69 | else: 70 | assert file not in rel_from_main_c 71 | 72 | 73 | @pytest.mark.cif 74 | def test_cross_ref(tmpdir, cmds_file): 75 | c = Clade(tmpdir, cmds_file) 76 | e = c.parse("CrossRef") 77 | 78 | ref_to = e.load_ref_to_by_file() 79 | ref_to_are_ok(ref_to) 80 | 81 | ref_to_main_c = e.load_ref_to_by_file([main_c]) 82 | filtered_ref_to_are_ok(ref_to, ref_to_main_c) 83 | 84 | ref_from = e.load_ref_from_by_file() 85 | ref_from_are_ok(ref_from) 86 | 87 | ref_from_main_c = e.load_ref_from_by_file([main_c]) 88 | filtered_ref_from_are_ok(ref_from, ref_from_main_c) 89 | -------------------------------------------------------------------------------- /tests/test_cxx.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from clade import Clade 18 | 19 | 20 | def test_cxx(tmpdir, cmds_file): 21 | c = Clade(tmpdir, cmds_file) 22 | e = c.parse("CXX") 23 | 24 | assert len(list(e.load_all_cmds(compile_only=True))) >= 1 25 | -------------------------------------------------------------------------------- /tests/test_envs.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | import pytest 18 | 19 | from clade.envs import ( 20 | open_envs_file, 21 | iter_envs, 22 | join_env, 23 | get_last_id, 24 | get_all_envs, 25 | get_stats, 26 | ) 27 | 28 | 29 | def test_bad_open(): 30 | with pytest.raises(RuntimeError): 31 | open_envs_file("do_not_exist.txt") 32 | 33 | 34 | def test_iter(envs_file): 35 | assert len(list(iter_envs(envs_file))) > 0 36 | 37 | 38 | def test_get_last_id(envs_file): 39 | assert int(get_last_id(envs_file)) > 0 40 | 41 | 42 | def test_get_stats(envs_file): 43 | assert get_stats(envs_file)[1] > 0 44 | 45 | 46 | def test_join_env(envs_file): 47 | lines = [] 48 | for envs in iter_envs(envs_file): 49 | for name, value in envs["envs"].items(): 50 | lines.append(join_env({name: value})) 51 | 52 | with open(envs_file, "r") as envs_fh: 53 | for line in envs_fh: 54 | line = line.strip() 55 | 56 | assert not line or line in lines 57 | 58 | 59 | def test_get_all_envs(envs_file): 60 | envs = [] 61 | for env in iter_envs(envs_file): 62 | envs.append(env) 63 | 64 | assert envs == get_all_envs(envs_file) 65 | -------------------------------------------------------------------------------- /tests/test_functions.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import pytest 17 | 18 | from clade import Clade 19 | from tests.test_project import main_c, zero_c, zero_h 20 | 21 | 22 | def funcs_are_ok(funcs): 23 | assert len(funcs["main"]) == 1 24 | definition = funcs["main"][0] 25 | 26 | assert not definition["declarations"] 27 | assert definition["line"] == 8 28 | assert definition["signature"] == "int main(void);" 29 | assert definition["type"] == "extern" 30 | assert len(funcs["print"]) >= 2 31 | 32 | 33 | def funcs_by_file_are_ok(funcs_by_file): 34 | assert funcs_by_file 35 | assert sorted([d["name"] for d in funcs_by_file[zero_c]]) == sorted( 36 | [ 37 | "zero", 38 | "print", 39 | "func_with_pointers", 40 | ] 41 | ) 42 | 43 | for definition in funcs_by_file[zero_c]: 44 | if definition["name"] != "zero": 45 | continue 46 | for declaration in definition["declarations"]: 47 | if declaration["file"] != zero_h: 48 | continue 49 | assert declaration["line"] == 1 50 | assert declaration["signature"] == "int zero();" 51 | assert declaration["type"] == "extern" 52 | 53 | 54 | def funcs_are_consistent(funcs, funcs_by_file): 55 | for func in funcs: 56 | for definition in funcs[func]: 57 | definition = dict(definition) 58 | file = definition["file"] 59 | definition["name"] = func 60 | del definition["file"] 61 | assert definition in funcs_by_file[file] 62 | 63 | for file in funcs_by_file: 64 | for definition in funcs_by_file[file]: 65 | definition = dict(definition) 66 | func = definition["name"] 67 | definition["file"] = file 68 | del definition["name"] 69 | assert definition in funcs[func] 70 | 71 | 72 | def filtered_funcs_by_file_are_ok(funcs_by_file, funcs_by_main_c): 73 | for file in funcs_by_file: 74 | if file == main_c: 75 | assert funcs_by_main_c[main_c] == funcs_by_file[main_c] 76 | else: 77 | assert file not in funcs_by_main_c 78 | 79 | 80 | @pytest.mark.cif 81 | def test_functions(tmpdir, cmds_file): 82 | conf = {"CmdGraph.requires": ["CC", "MV"]} 83 | 84 | c = Clade(tmpdir, cmds_file, conf) 85 | e = c.parse("Functions") 86 | 87 | funcs = e.load_functions() 88 | funcs_by_file = e.load_functions_by_file() 89 | funcs_by_main_c = e.load_functions_by_file([main_c]) 90 | 91 | funcs_are_ok(funcs) 92 | funcs_by_file_are_ok(funcs_by_file) 93 | funcs_are_consistent(funcs, funcs_by_file) 94 | filtered_funcs_by_file_are_ok(funcs_by_file, funcs_by_main_c) 95 | -------------------------------------------------------------------------------- /tests/test_info.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import pytest 17 | 18 | from clade import Clade 19 | 20 | 21 | @pytest.mark.cif 22 | def test_info(tmpdir, cmds_file): 23 | conf = {"CC.filter_deps": False, "Info.extra_CIF_pts": ["-hello"]} 24 | 25 | c = Clade(tmpdir, cmds_file, conf) 26 | e = c.parse("Info") 27 | 28 | assert list(e.iter_definitions()) 29 | assert list(e.iter_declarations()) 30 | assert not list(e.iter_exported()) 31 | assert list(e.iter_calls()) 32 | assert list(e.iter_calls_by_pointers()) 33 | assert list(e.iter_functions_usages()) 34 | assert list(e.iter_macros_definitions()) 35 | assert list(e.iter_macros_expansions()) 36 | assert list(e.iter_typedefs()) 37 | -------------------------------------------------------------------------------- /tests/test_intercept.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import shutil 18 | import sys 19 | 20 | from clade.intercept import intercept 21 | 22 | 23 | test_project = os.path.join(os.path.dirname(__file__), "test_project") 24 | test_project_make = ["make", "-C", test_project] 25 | 26 | 27 | def calculate_loc(file): 28 | with open(file, "rb") as f: 29 | for i, _ in enumerate(f): 30 | pass 31 | return i + 1 32 | 33 | 34 | def test_no_fallback(tmpdir): 35 | output = os.path.join(str(tmpdir), "cmds.txt") 36 | 37 | assert not intercept(command=test_project_make, output=output, use_wrappers=False) 38 | 39 | # LD_PRELOAD may not work on certain systems 40 | # Due to SELinux or System Integrity Protection 41 | 42 | if sys.platform != "darwin": 43 | assert os.path.isfile(output) 44 | assert calculate_loc(output) > 1 45 | 46 | 47 | def test_no_fallback_with_server(tmpdir): 48 | output = os.path.join(str(tmpdir), "cmds.txt") 49 | conf = {"Intercept.preprocess": True} 50 | 51 | assert not intercept( 52 | command=test_project_make, output=output, use_wrappers=False, conf=conf 53 | ) 54 | 55 | if sys.platform != "darwin": 56 | assert os.path.isfile(output) 57 | assert calculate_loc(output) > 1 58 | 59 | 60 | def test_fallback(tmpdir): 61 | output = os.path.join(str(tmpdir), "cmds.txt") 62 | 63 | assert not intercept(command=test_project_make, output=output, use_wrappers=True) 64 | assert os.path.isfile(output) 65 | assert calculate_loc(output) > 1 66 | 67 | 68 | def test_fallback_with_exe_wrappers(tmpdir): 69 | output = os.path.join(str(tmpdir), "cmds.txt") 70 | cc_path = shutil.which("cc") 71 | 72 | assert cc_path 73 | 74 | conf = { 75 | "Wrapper.wrap_list": [cc_path, os.path.dirname(cc_path)], 76 | "Wrapper.recursive_wrap": False, 77 | } 78 | 79 | assert not intercept( 80 | command=test_project_make, output=output, use_wrappers=True, conf=conf 81 | ) 82 | assert os.path.isfile(output) 83 | assert calculate_loc(output) > 1 84 | 85 | 86 | def test_fallback_with_exe_wrappers_recursive(tmpdir): 87 | output = os.path.join(str(tmpdir), "cmds.txt") 88 | conf = { 89 | "Wrapper.wrap_list": [os.path.dirname(__file__), __file__], 90 | "Wrapper.recursive_wrap": True, 91 | } 92 | 93 | assert not intercept( 94 | command=test_project_make, output=output, use_wrappers=True, conf=conf 95 | ) 96 | assert os.path.isfile(output) 97 | assert calculate_loc(output) > 1 98 | 99 | 100 | def test_fallback_with_unix_server(tmpdir): 101 | output = os.path.join(str(tmpdir), "cmds.txt") 102 | conf = {"Intercept.preprocess": True} 103 | 104 | assert not intercept( 105 | command=test_project_make, output=output, use_wrappers=True, conf=conf 106 | ) 107 | assert os.path.isfile(output) 108 | assert calculate_loc(output) > 1 109 | -------------------------------------------------------------------------------- /tests/test_ld.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import re 17 | 18 | from clade import Clade 19 | 20 | 21 | def test_ld(tmpdir, cmds_file): 22 | c = Clade(tmpdir, cmds_file) 23 | e = c.parse("LD") 24 | 25 | cmds = list(e.load_all_cmds(with_opts=True, with_raw=True)) 26 | 27 | target_cmd = dict() 28 | 29 | for cmd in cmds: 30 | for cmd_out in cmd["out"]: 31 | if re.search("main.o2", cmd_out): 32 | target_cmd = cmd 33 | 34 | assert len(cmds) >= 1 35 | assert len(target_cmd["in"]) == 2 36 | assert len(target_cmd["out"]) == 1 37 | assert len(target_cmd["opts"]) == 7 38 | assert len(target_cmd["command"]) == 11 39 | -------------------------------------------------------------------------------- /tests/test_ln.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 Ilya Shchepetkov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import subprocess 17 | import pathlib 18 | 19 | from clade.extensions.ln import LN 20 | 21 | 22 | def get_cmd(tmp_path, command): 23 | cwd = tmp_path / "cwd" 24 | cwd.mkdir() 25 | 26 | subprocess.run(command, cwd=cwd, shell=True) 27 | 28 | return { 29 | "cwd": str(cwd), 30 | "pid": "0", 31 | "id": "1", 32 | "which": "/usr/bin/ln", 33 | "command": command.split(" "), 34 | } 35 | 36 | 37 | def create_empty_file(path): 38 | with open(path, "w"): 39 | pass 40 | 41 | return path 42 | 43 | 44 | def check_ln(parsed_cmd): 45 | assert len(parsed_cmd["in"]) == len(parsed_cmd["out"]) 46 | 47 | for i, cmd_in in enumerate(parsed_cmd["in"]): 48 | assert os.path.basename(cmd_in) == os.path.basename(parsed_cmd["out"][i]) 49 | 50 | 51 | def test_ln_cwd(tmp_path: pathlib.Path): 52 | ln = LN(tmp_path) 53 | 54 | in_file = create_empty_file(tmp_path / "test.txt") 55 | cmd = get_cmd(tmp_path, f"ln {in_file}") 56 | 57 | parsed_cmd = ln.parse_cmd(cmd) 58 | 59 | assert parsed_cmd 60 | assert len(parsed_cmd["in"]) == len(parsed_cmd["out"]) 61 | assert len(parsed_cmd["in"]) == 1 62 | assert os.path.basename(parsed_cmd["in"][0]) == os.path.basename(in_file) 63 | assert os.path.basename(parsed_cmd["out"][0]) == os.path.basename(in_file) 64 | assert cmd["cwd"] in parsed_cmd["out"][0] 65 | 66 | 67 | def test_ln_single(tmp_path: pathlib.Path): 68 | ln = LN(tmp_path) 69 | 70 | in_file = create_empty_file(tmp_path / "test1.txt") 71 | out_file = tmp_path / "test2.txt" 72 | cmd = get_cmd(tmp_path, f"ln {in_file} {out_file}") 73 | 74 | parsed_cmd = ln.parse_cmd(cmd) 75 | 76 | assert parsed_cmd 77 | assert len(parsed_cmd["in"]) == len(parsed_cmd["out"]) 78 | assert len(parsed_cmd["in"]) == 1 79 | assert os.path.basename(parsed_cmd["in"][0]) == os.path.basename(in_file) 80 | assert os.path.basename(parsed_cmd["out"][0]) == os.path.basename(out_file) 81 | 82 | 83 | def test_ln_multiple(tmp_path: pathlib.Path): 84 | ln = LN(tmp_path) 85 | 86 | in_file1 = create_empty_file(tmp_path / "test1.txt") 87 | in_file2 = create_empty_file(tmp_path / "test2.txt") 88 | out_dir = tmp_path / "output" 89 | out_dir.mkdir() 90 | 91 | cmd = get_cmd(tmp_path, f"ln {in_file1} {in_file2} {out_dir}") 92 | 93 | parsed_cmd = ln.parse_cmd(cmd) 94 | 95 | assert parsed_cmd 96 | assert len(parsed_cmd["in"]) == len(parsed_cmd["out"]) 97 | assert len(parsed_cmd["in"]) == 2 98 | assert os.path.basename(parsed_cmd["in"][0]) == os.path.basename(in_file1) 99 | assert os.path.basename(parsed_cmd["in"][1]) == os.path.basename(in_file2) 100 | assert os.path.basename(parsed_cmd["out"][0]) == os.path.basename(in_file1) 101 | assert os.path.basename(parsed_cmd["out"][1]) == os.path.basename(in_file2) 102 | assert str(out_dir) in parsed_cmd["out"][0] 103 | assert str(out_dir) in parsed_cmd["out"][1] 104 | 105 | 106 | def check_target(parsed_cmd, in_file, out_dir): 107 | assert parsed_cmd 108 | assert len(parsed_cmd["in"]) == len(parsed_cmd["out"]) 109 | assert len(parsed_cmd["in"]) == 1 110 | assert os.path.basename(parsed_cmd["in"][0]) == os.path.basename(in_file) 111 | assert os.path.basename(parsed_cmd["out"][0]) == os.path.basename(in_file) 112 | assert str(out_dir) in parsed_cmd["out"][0] 113 | 114 | 115 | def test_ln_t1(tmp_path: pathlib.Path): 116 | ln = LN(tmp_path) 117 | 118 | in_file = create_empty_file(tmp_path / "test.txt") 119 | out_dir = tmp_path / "output" 120 | out_dir.mkdir() 121 | 122 | cmd = get_cmd(tmp_path, f"ln -t {out_dir} {in_file}") 123 | 124 | parsed_cmd = ln.parse_cmd(cmd) 125 | check_target(parsed_cmd, in_file, out_dir) 126 | 127 | 128 | def test_ln_t2(tmp_path: pathlib.Path): 129 | ln = LN(tmp_path) 130 | 131 | in_file = create_empty_file(tmp_path / "test.txt") 132 | out_dir = tmp_path / "output" 133 | out_dir.mkdir() 134 | 135 | cmd = get_cmd(tmp_path, f"ln -t{out_dir} {in_file}") 136 | 137 | parsed_cmd = ln.parse_cmd(cmd) 138 | check_target(parsed_cmd, in_file, out_dir) 139 | 140 | 141 | def test_ln_target1(tmp_path: pathlib.Path): 142 | ln = LN(tmp_path) 143 | 144 | in_file = create_empty_file(tmp_path / "test.txt") 145 | out_dir = tmp_path / "output" 146 | out_dir.mkdir() 147 | 148 | cmd = get_cmd(tmp_path, f"ln --target-directory={out_dir} {in_file}") 149 | 150 | parsed_cmd = ln.parse_cmd(cmd) 151 | check_target(parsed_cmd, in_file, out_dir) 152 | 153 | 154 | def test_ln_target2(tmp_path: pathlib.Path): 155 | ln = LN(tmp_path) 156 | 157 | in_file = create_empty_file(tmp_path / "test.txt") 158 | out_dir = tmp_path / "output" 159 | out_dir.mkdir() 160 | 161 | cmd = get_cmd(tmp_path, f"ln --target-directory {out_dir} {in_file}") 162 | 163 | parsed_cmd = ln.parse_cmd(cmd) 164 | check_target(parsed_cmd, in_file, out_dir) 165 | -------------------------------------------------------------------------------- /tests/test_macros.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import pytest 18 | 19 | from clade import Clade 20 | 21 | zero_c = os.path.abspath("tests/test_project/zero.c") 22 | 23 | 24 | def definitions_are_ok(definitions): 25 | assert definitions[zero_c]["WEIRD_ZERO"] == [3] 26 | assert definitions[zero_c]["ZERO"] == [4] 27 | 28 | 29 | def expansions_are_ok(expansions): 30 | assert expansions[zero_c]["WEIRD_ZERO"]["args"] == [["10"]] 31 | assert expansions[zero_c]["ZERO"] 32 | 33 | 34 | @pytest.mark.cif 35 | def test_macros(tmpdir, cmds_file): 36 | c = Clade(tmpdir, cmds_file) 37 | c.parse("Macros") 38 | 39 | definitions_are_ok(c.get_macros_definitions()) 40 | expansions_are_ok(c.get_macros_expansions()) 41 | -------------------------------------------------------------------------------- /tests/test_main.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import pytest 17 | import os 18 | import sys 19 | 20 | from clade.__main__ import main 21 | 22 | test_project = os.path.join(os.path.dirname(__file__), "test_project") 23 | test_project_make = ["make", "-C", test_project] 24 | 25 | 26 | @pytest.mark.skipif(sys.platform == "darwin", reason="test doesn't work on macOS") 27 | def test_intercept(tmpdir): 28 | cmds_file = os.path.join(str(tmpdir), "cmds.txt") 29 | 30 | with pytest.raises(SystemExit) as e: 31 | main(["--cmds", cmds_file, "-i"] + test_project_make) 32 | 33 | assert "0" == str(e.value) 34 | 35 | 36 | def test_intercept_no_command(): 37 | with pytest.raises(SystemExit) as e: 38 | main(["-i"]) 39 | 40 | assert "-1" == str(e.value) 41 | 42 | 43 | def test_main_cc(tmpdir, cmds_file): 44 | with pytest.raises(SystemExit) as e: 45 | main(["-w", str(tmpdir), "--cmds", cmds_file, "-e", "CC"]) 46 | 47 | assert "0" == str(e.value) 48 | 49 | 50 | def test_main_bad_conf(tmpdir, cmds_file): 51 | with pytest.raises(SystemExit) as e: 52 | main(["-w", str(tmpdir), "--cmds", cmds_file, "-c", "does_not_exist.conf"]) 53 | 54 | assert "-1" == str(e.value) 55 | 56 | 57 | def test_main_bad_preset(tmpdir, cmds_file): 58 | with pytest.raises(SystemExit): 59 | main(["-w", str(tmpdir), "--cmds", cmds_file, "-p", "does_not_exist"]) 60 | 61 | 62 | def test_main_good_json(tmpdir, cmds_file): 63 | with pytest.raises(SystemExit) as e: 64 | main( 65 | [ 66 | "-w", 67 | str(tmpdir), 68 | "--cmds", 69 | cmds_file, 70 | "-e", 71 | "PidGraph", 72 | "--conf", 73 | '{"CmdGraph.requires": []}', 74 | ] 75 | ) 76 | 77 | assert "0" == str(e.value) 78 | 79 | 80 | def test_main_bad_json(tmpdir, cmds_file): 81 | with pytest.raises(SystemExit) as e: 82 | main( 83 | [ 84 | "-w", 85 | str(tmpdir), 86 | "--cmds", 87 | cmds_file, 88 | "-e", 89 | "PidGraph", 90 | "--conf", 91 | '{"CmdGraph.requires": [}', 92 | ] 93 | ) 94 | 95 | assert "-1" == str(e.value) 96 | -------------------------------------------------------------------------------- /tests/test_objcopy.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import pytest 17 | import sys 18 | 19 | from clade import Clade 20 | 21 | 22 | @pytest.mark.skipif(sys.platform != "linux", reason="test only for Linux") 23 | def test_objcopy(tmpdir, cmds_file): 24 | c = Clade(tmpdir, cmds_file) 25 | e = c.parse("Objcopy") 26 | 27 | cmds = e.load_all_cmds(with_opts=True, with_raw=True) 28 | assert len(cmds) == 2 29 | for cmd in cmds: 30 | assert len(cmd["in"]) == 1 31 | assert len(cmd["out"]) == 1 32 | assert len(cmd["opts"]) == 1 33 | -------------------------------------------------------------------------------- /tests/test_opts.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from clade import Clade 17 | from clade.extensions.opts import filter_opts 18 | 19 | 20 | def test_isysroot(tmpdir): 21 | c = Clade(tmpdir) 22 | 23 | opts = ["-isysroot=/test/path", "-I/usr/include"] 24 | 25 | filtered_opts = filter_opts(opts, c.get_storage_path) 26 | 27 | assert len(filtered_opts) == len(opts) 28 | assert filtered_opts[0] == "-isysroot={}/test/path".format(c.storage_dir) 29 | assert filtered_opts[1] == opts[1] 30 | 31 | 32 | def test_no_isysroot(tmpdir): 33 | c = Clade(tmpdir) 34 | 35 | opts = ["-I/usr/include"] 36 | 37 | filtered_opts = filter_opts(opts, c.get_storage_path) 38 | 39 | assert len(filtered_opts) == len(opts) 40 | assert filtered_opts[0] == f"-I{c.storage_dir}/usr/include" 41 | 42 | 43 | def test_no_get_storage_path(): 44 | opts = ["-I/usr/include"] 45 | 46 | assert filter_opts(opts) == opts 47 | 48 | 49 | def test_bad_opt(): 50 | opts = ["-ABC", "-Dtest"] 51 | 52 | assert filter_opts(opts) == ["-Dtest"] 53 | -------------------------------------------------------------------------------- /tests/test_path.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import pathlib 18 | 19 | from clade import Clade 20 | 21 | test_file_rel = "tests/test_project/main.c" 22 | test_file_abs = os.path.abspath(test_file_rel) 23 | 24 | 25 | def test_path(tmpdir, cmds_file): 26 | c = Clade(tmpdir, cmds_file) 27 | c.parse("SrcGraph") 28 | 29 | assert c.Path.normalize_rel_path(test_file_rel, os.getcwd()) == test_file_abs 30 | 31 | 32 | def test_path_capital(tmpdir, cmds_file): 33 | c = Clade(tmpdir, cmds_file) 34 | c.parse("SrcGraph") 35 | 36 | tmpdir = str(tmpdir) 37 | 38 | test_small = pathlib.Path(tmpdir) / "test.c" 39 | test_capital = pathlib.Path(tmpdir) / "TEST.c" 40 | 41 | test_small.touch() 42 | test_capital.touch() 43 | 44 | assert "test.c" in c.Path.normalize_rel_path("test.c", tmpdir) 45 | assert "TEST.c" in c.Path.normalize_rel_path("TEST.c", tmpdir) 46 | -------------------------------------------------------------------------------- /tests/test_path_tree.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from clade.types.path_tree import PathTree 17 | 18 | 19 | def test_path_tree(): 20 | pt = PathTree() 21 | 22 | pt[__file__] = 1 23 | assert __file__ in pt 24 | assert pt[__file__] == 1 25 | assert pt.keys() == [__file__] 26 | assert pt.get(__file__) == 1 27 | assert not pt.get("do/not/exist") 28 | assert pt.get("do/not/exist", 2) == 2 29 | 30 | for key in pt: 31 | assert key == __file__ 32 | -------------------------------------------------------------------------------- /tests/test_pid_graph.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from clade import Clade 18 | from clade.cmds import get_last_id 19 | 20 | 21 | def test_pid_graph(tmpdir, cmds_file): 22 | c = Clade(tmpdir, cmds_file) 23 | e = c.parse("PidGraph") 24 | 25 | last_id = get_last_id(cmds_file) 26 | 27 | pid_graph = e.load_pid_graph() 28 | pid_by_id = e.load_pid_by_id() 29 | 30 | cmd_ids = list(x for x in range(1, int(last_id) + 1)) 31 | assert len(pid_graph) == len(cmd_ids) 32 | 33 | for cmd_id in cmd_ids: 34 | assert cmd_id in pid_graph 35 | assert len(pid_graph[cmd_id]) >= 1 36 | 37 | for pid in pid_graph[cmd_id]: 38 | assert int(pid) < int(cmd_id) 39 | 40 | assert len(pid_by_id) == len(cmd_ids) 41 | 42 | for cmd_id in cmd_ids: 43 | assert cmd_id in pid_by_id 44 | assert int(pid_by_id[cmd_id]) < int(cmd_id) 45 | -------------------------------------------------------------------------------- /tests/test_project.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | 18 | zero_c = os.path.abspath("tests/test_project/zero.c") 19 | zero_h = os.path.abspath("tests/test_project/zero.h") 20 | main_c = os.path.abspath("tests/test_project/main.c") 21 | tmp_main = os.path.abspath("tests/test_project/tmp_main") 22 | -------------------------------------------------------------------------------- /tests/test_project/Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | gcc zero.c main.c -o tmp_main -D TEST_MACRO -O3 3 | mv tmp_main main 4 | gcc zero.c main.c -o /dev/null -fsyntax-only 5 | gcc zero.c -M -MF zero.txt 6 | -clang -cc1 /dev/null 7 | gcc -c zero.c main.c 8 | gcc zero.o main.o -o main 9 | -as empty.s -o zero.o -I /usr/include 10 | -ar rcs zero.a zero.o main.o 11 | -objcopy main.o --strip-all 12 | -objcopy main.o zero.o --strip-all 13 | -ld main.o -o main.o2 -g -l interceptor -linterceptor -L../../clade/intercept -L clade/intercept 14 | -rm main zero.txt zero.o* main.o* zero.a 15 | g++ zero.c main.c -o /dev/null -D TEST_MACRO -O3 16 | -ccache gcc zero.c -I --ccache-skip /usr/include 17 | -ccache g++ zero.c -o /dev/null 18 | -------------------------------------------------------------------------------- /tests/test_project/empty.s: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/17451k/clade/b06da1c41cf14e58781b5735fbcab36190eaf0d3/tests/test_project/empty.s -------------------------------------------------------------------------------- /tests/test_project/main.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "zero.h" 3 | 4 | static void print(const char *msg) { 5 | printf("%s\n", msg); 6 | } 7 | 8 | int main() { 9 | print("Hello"); 10 | return zero(); 11 | } 12 | -------------------------------------------------------------------------------- /tests/test_project/zero.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #define WEIRD_ZERO(X) X - X 4 | #define ZERO WEIRD_ZERO(10) 5 | 6 | int zero() { 7 | return ZERO; 8 | } 9 | 10 | static void print(char *msg) { 11 | printf("%s\n", msg); 12 | } 13 | 14 | int func_with_pointers() { 15 | int (*fp1)(void) = zero; 16 | int (*fp2)(void) = zero; 17 | return fp1() + fp2(); 18 | } 19 | 20 | typedef unsigned char super_char; 21 | 22 | int (*fp3[])(void) = {zero}; 23 | 24 | static int (*fp4[])(void) = {zero}; 25 | -------------------------------------------------------------------------------- /tests/test_project/zero.h: -------------------------------------------------------------------------------- 1 | extern int zero(); 2 | -------------------------------------------------------------------------------- /tests/test_src_graph.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import itertools 18 | 19 | from clade import Clade 20 | 21 | test_file = os.path.abspath("tests/test_project/main.c") 22 | 23 | 24 | def test_src_graph(tmpdir, cmds_file): 25 | conf = {"CmdGraph.requires": ["CC", "MV"]} 26 | 27 | c = Clade(tmpdir, cmds_file, conf) 28 | e = c.parse("SrcGraph") 29 | 30 | src_graph = e.load_src_graph() 31 | 32 | assert src_graph 33 | assert len(src_graph[test_file].keys()) == 3 34 | assert len(list(itertools.chain(*src_graph[test_file].values()))) == 2 35 | 36 | src_info = e.load_src_info() 37 | assert src_info 38 | assert src_info[test_file]["loc"] == 11 39 | 40 | graph_part = e.load_src_graph([test_file]) 41 | assert graph_part 42 | assert len(list(itertools.chain(*graph_part[test_file].values()))) == 2 43 | assert len(src_graph[test_file].keys()) == 3 44 | 45 | 46 | def test_src_graph_empty_conf(tmpdir, cmds_file): 47 | c = Clade(tmpdir, cmds_file) 48 | e = c.parse("SrcGraph") 49 | 50 | src_graph = e.load_src_graph() 51 | assert src_graph 52 | assert len(list(itertools.chain(*src_graph[test_file].values()))) >= 1 53 | -------------------------------------------------------------------------------- /tests/test_storage.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import pytest 18 | import shutil 19 | import stat 20 | import unittest.mock 21 | 22 | from clade import Clade 23 | 24 | test_file = os.path.abspath("tests/test_project/main.c") 25 | 26 | 27 | def test_storage(tmpdir): 28 | c = Clade(tmpdir) 29 | 30 | returned_storage_path = c.add_file_to_storage(__file__) 31 | c.add_file_to_storage("do_not_exist.c") 32 | 33 | storage_path = c.get_storage_path(__file__) 34 | 35 | assert storage_path 36 | assert os.path.exists(storage_path) 37 | assert storage_path.startswith(c.storage_dir) 38 | assert returned_storage_path == storage_path 39 | 40 | # Test possible race condition 41 | with unittest.mock.patch("shutil.copyfile") as copyfile_mock: 42 | copyfile_mock.side_effect = shutil.SameFileError 43 | c.add_file_to_storage(test_file) 44 | 45 | 46 | def test_storage_with_conversion(tmpdir): 47 | c = Clade(tmpdir, conf={"Storage.convert_to_utf8": True}) 48 | 49 | with unittest.mock.patch("os.replace") as replace_mock: 50 | replace_mock.side_effect = OSError 51 | c.add_file_to_storage(test_file) 52 | 53 | 54 | def test_files_to_add(tmpdir, cmds_file): 55 | c = Clade(tmpdir, cmds_file, conf={"Storage.files_to_add": [__file__]}) 56 | c.parse("Storage") 57 | 58 | storage_path = c.get_storage_path(__file__) 59 | assert storage_path 60 | assert os.path.exists(storage_path) 61 | 62 | 63 | def test_folders_to_add(tmpdir, cmds_file): 64 | c = Clade( 65 | tmpdir, cmds_file, conf={"Storage.files_to_add": [os.path.dirname(__file__)]} 66 | ) 67 | c.parse("Storage") 68 | 69 | storage_path = c.get_storage_path(__file__) 70 | assert storage_path 71 | assert os.path.exists(storage_path) 72 | 73 | 74 | @pytest.mark.parametrize("encoding", ["cp1251", "utf8"]) 75 | def test_storage_encoding(tmpdir, encoding): 76 | c = Clade(tmpdir, conf={"Storage.convert_to_utf8": True}) 77 | 78 | bstr = "мир".encode("cp1251") 79 | 80 | test_file = os.path.join(str(tmpdir), "test") 81 | with open(test_file, "wb") as fh: 82 | fh.write(bstr) 83 | 84 | c.add_file_to_storage(test_file, encoding=encoding) 85 | 86 | 87 | @pytest.mark.parametrize("convert", [False, True]) 88 | def test_storage_permissions(tmpdir, convert): 89 | c = Clade(tmpdir, conf={"Storage.convert_to_utf8": convert}) 90 | storage_path = c.add_file_to_storage(__file__) 91 | assert os.stat(storage_path)[stat.ST_MODE] == os.stat(__file__)[stat.ST_MODE] 92 | -------------------------------------------------------------------------------- /tests/test_tracer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import pytest 17 | 18 | from clade import Clade 19 | from clade.scripts.tracer import Tracer 20 | 21 | 22 | @pytest.mark.cif 23 | def test_tracer(tmpdir, cmds_file): 24 | c = Clade(tmpdir, cmds_file, preset="klever_linux_kernel") 25 | c.parse_list(c.conf["extensions"]) 26 | 27 | print(c.work_dir) 28 | t = Tracer(c.work_dir) 29 | 30 | from_func = t.find_functions(["main"])[0] 31 | to_func = t.find_functions(["printf"])[0] 32 | trace = t.trace(from_func, to_func) 33 | assert len(trace) == 2 34 | -------------------------------------------------------------------------------- /tests/test_typedefs.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import pytest 17 | 18 | from clade import Clade 19 | from tests.test_project import zero_c 20 | 21 | 22 | def typedefs_are_ok(typedefs): 23 | assert "unsigned char super_char;" in typedefs[zero_c] 24 | 25 | 26 | @pytest.mark.cif 27 | def test_typedefs(tmpdir, cmds_file): 28 | c = Clade(tmpdir, cmds_file) 29 | e = c.parse("Typedefs") 30 | 31 | typedefs_are_ok(e.load_typedefs()) 32 | -------------------------------------------------------------------------------- /tests/test_used_in.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 Ilya Shchepetkov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | 17 | from clade import Clade 18 | from tests.test_project import zero_c 19 | 20 | 21 | def used_in_is_ok(used_in): 22 | assert not used_in[zero_c]["zero"]["used_in_file"] 23 | assert {"line": 15, "match": 3} in used_in[zero_c]["zero"]["used_in_func"][zero_c][ 24 | "func_with_pointers" 25 | ] 26 | assert {"line": 16, "match": 3} in used_in[zero_c]["zero"]["used_in_func"][zero_c][ 27 | "func_with_pointers" 28 | ] 29 | 30 | 31 | @pytest.mark.cif 32 | def test_used_in(tmpdir, cmds_file): 33 | conf = {"CmdGraph.requires": ["CC", "MV"]} 34 | 35 | c = Clade(tmpdir, cmds_file, conf) 36 | e = c.parse("UsedIn") 37 | 38 | used_in = e.load_used_in() 39 | 40 | used_in_is_ok(used_in) 41 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from clade.utils import get_clade_version, get_program_version, merge_preset_to_conf 17 | 18 | 19 | def test_get_clade_version(): 20 | version = get_clade_version() 21 | 22 | assert version 23 | assert type(version) == str 24 | 25 | 26 | def test_get_program_version(): 27 | version = get_program_version("gcc") 28 | 29 | assert version 30 | assert type(version) == str 31 | assert get_clade_version() in get_program_version("clade") 32 | 33 | 34 | def test_merge_preset_to_conf(): 35 | conf = {} 36 | 37 | assert merge_preset_to_conf("klever_linux_kernel", conf) 38 | -------------------------------------------------------------------------------- /tests/test_variables.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import pytest 17 | 18 | from clade import Clade 19 | from tests.test_project import zero_c 20 | 21 | 22 | def variables_are_ok(variables): 23 | assert variables[zero_c] 24 | assert variables[zero_c][0]["declaration"] == "int (*fp3[1U])(void)" 25 | assert variables[zero_c][0]["path"] == zero_c 26 | assert variables[zero_c][0]["type"] == "extern" 27 | assert variables[zero_c][0]["value"][0]["index"] == 0 28 | assert variables[zero_c][0]["value"][0]["value"] == " & zero" 29 | assert variables[zero_c][1]["type"] == "static" 30 | 31 | 32 | def used_in_vars_is_ok(used_in_vars): 33 | assert used_in_vars["zero"][zero_c] == [zero_c] 34 | 35 | 36 | @pytest.mark.cif 37 | def test_variables(tmpdir, cmds_file): 38 | c = Clade(tmpdir, cmds_file) 39 | e = c.parse("Variables") 40 | 41 | variables_are_ok(e.load_variables()) 42 | used_in_vars_is_ok(e.load_used_in_vars()) 43 | -------------------------------------------------------------------------------- /tests/test_windows.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 ISP RAS (http://www.ispras.ru) 2 | # Ivannikov Institute for System Programming of the Russian Academy of Sciences 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import pytest 18 | import sys 19 | 20 | from clade import Clade 21 | 22 | test_build = ["pip", "install", "--user", "--no-binary", ":all:", "--force", "cchardet"] 23 | 24 | 25 | @pytest.mark.skipif(sys.platform != "win32", reason="tests only for Windows") 26 | def test_windows(tmpdir): 27 | work_dir = os.path.join(str(tmpdir), "clade") 28 | output = os.path.join(str(tmpdir), "cmds.txt") 29 | 30 | c = Clade(work_dir, cmds_file=output) 31 | 32 | assert not c.intercept(command=test_build) 33 | 34 | c.parse("SrcGraph") 35 | 36 | assert c.pid_graph 37 | assert c.cmds 38 | assert c.cmd_graph 39 | assert c.src_graph 40 | --------------------------------------------------------------------------------