├── .editorconfig ├── .github ├── dependabot.yml └── workflows │ ├── docs.yml │ ├── lint.yml │ ├── pip-audit.yml │ ├── release.yml │ └── tests.yml ├── .gitignore ├── .trunk ├── .gitignore ├── configs │ ├── .flake8 │ ├── .isort.cfg │ └── .markdownlint.yaml └── trunk.yaml ├── CODEOWNERS ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── pyproject.toml ├── src └── blight │ ├── __init__.py │ ├── _cli.py │ ├── action.py │ ├── actions │ ├── __init__.py │ ├── benchmark.py │ ├── cc_for_cxx.py │ ├── demo.py │ ├── embed_bitcode.py │ ├── find_inputs.py │ ├── find_outputs.py │ ├── ignore_flags.py │ ├── ignore_flto.py │ ├── ignore_werror.py │ ├── inject_flags.py │ ├── lint.py │ ├── record.py │ └── skip_strip.py │ ├── constants.py │ ├── enums.py │ ├── exceptions.py │ ├── protocols.py │ ├── tool.py │ └── util.py └── test ├── actions ├── test_cc_for_cxx.py ├── test_demo.py ├── test_embed_bitcode.py ├── test_find_inputs.py ├── test_find_outputs.py ├── test_ignore_flags.py ├── test_ignore_flto.py ├── test_ignore_werror.py ├── test_inject_flags.py ├── test_lint.py ├── test_record.py └── test_skip_strip.py ├── conftest.py ├── test_action.py ├── test_enums.py ├── test_tool.py └── test_util.py /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | charset = utf-8 7 | 8 | [*.py] 9 | indent_style = space 10 | indent_size = 4 11 | 12 | [Makefile] 13 | indent_style = tab 14 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: pip 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "10:00" 8 | open-pull-requests-limit: 10 9 | - package-ecosystem: "github-actions" 10 | directory: "/" 11 | schedule: 12 | # Check for updates to GitHub Actions every weekday 13 | interval: "daily" 14 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: Deploy Documentation 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v3.3.0 13 | 14 | - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 15 | with: 16 | # NOTE: We use 3.10+ typing syntax via future, which pdoc only 17 | # understands if it's actually run with Python 3.10 or newer. 18 | python-version: ">= 3.10" 19 | cache: "pip" 20 | cache-dependency-path: pyproject.toml 21 | 22 | - name: setup 23 | run: | 24 | make dev PIP_AUDIT_EXTRA=doc 25 | - name: build docs 26 | run: | 27 | make doc 28 | - name: upload docs artifact 29 | uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 30 | with: 31 | path: ./html/ 32 | 33 | # This is copied from the official `pdoc` example: 34 | # https://github.com/mitmproxy/pdoc/blob/main/.github/workflows/docs.yml 35 | # 36 | # Deploy the artifact to GitHub pages. 37 | # This is a separate job so that only actions/deploy-pages has the necessary permissions. 38 | deploy: 39 | needs: build 40 | runs-on: ubuntu-latest 41 | permissions: 42 | # NOTE: Needed to push to the repository. 43 | pages: write 44 | id-token: write 45 | environment: 46 | name: github-pages 47 | url: ${{ steps.deployment.outputs.page_url }} 48 | steps: 49 | - id: deployment 50 | uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 51 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | merge_group: 9 | 10 | concurrency: 11 | group: ${{ github.head_ref || github.run_id }} 12 | cancel-in-progress: true 13 | 14 | jobs: 15 | lint: 16 | uses: trailofbits/.github/.github/workflows/lint.yml@v0.1.3 17 | permissions: 18 | contents: read 19 | pull-requests: read 20 | checks: write 21 | -------------------------------------------------------------------------------- /.github/workflows/pip-audit.yml: -------------------------------------------------------------------------------- 1 | name: Scan dependencies for vulnerabilities with pip-audit 2 | 3 | on: 4 | push: 5 | branches: ["master"] 6 | pull_request: 7 | branches: ["master"] 8 | schedule: 9 | - cron: "0 12 * * *" 10 | 11 | jobs: 12 | pip-audit: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - name: Checkout repository 17 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 18 | 19 | - name: Install Python 20 | uses: actions/setup-python@v5 21 | with: 22 | python-version: "3.x" 23 | 24 | - name: Install project 25 | run: make dev 26 | 27 | - name: Run pip-audit 28 | uses: trailofbits/gh-action-pip-audit@v1.1.0 29 | with: 30 | virtual-environment: env/ 31 | ignore-vulns: | 32 | GHSA-w596-4wvx-j9j6 33 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | on: 2 | release: 3 | types: 4 | - published 5 | 6 | name: release 7 | 8 | jobs: 9 | pypi: 10 | name: upload release to PyPI 11 | runs-on: ubuntu-latest 12 | permissions: 13 | # For OIDC publishing + Sigstore signing 14 | id-token: write 15 | 16 | # For signature + release asset uploading 17 | contents: write 18 | steps: 19 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 20 | 21 | - uses: actions/setup-python@v5 22 | with: 23 | python-version: "3.8" 24 | 25 | - name: deps 26 | run: python -m pip install -U setuptools build wheel 27 | 28 | - name: build 29 | run: python -m build 30 | 31 | - name: publish 32 | uses: pypa/gh-action-pypi-publish@release/v1 33 | 34 | - name: sign 35 | uses: sigstore/gh-action-sigstore-python@v3.0.0 36 | with: 37 | inputs: ./dist/*.tar.gz ./dist/*.whl 38 | release-signing-artifacts: true 39 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | merge_group: 9 | 10 | jobs: 11 | test: 12 | strategy: 13 | matrix: 14 | python: 15 | - "3.8" 16 | - "3.9" 17 | - "3.10" 18 | - "3.11" 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 22 | 23 | - uses: actions/setup-python@v5 24 | with: 25 | python-version: ${{ matrix.python }} 26 | 27 | - name: deps 28 | run: make dev INSTALL_EXTRA=test 29 | 30 | - name: test 31 | run: make test 32 | 33 | all-tests-pass: 34 | if: always() 35 | 36 | needs: 37 | - test 38 | 39 | runs-on: ubuntu-latest 40 | 41 | steps: 42 | - name: check test jobs 43 | uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 44 | with: 45 | jobs: ${{ toJSON(needs) }} 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | env/ 2 | pip-wheel-metadata/ 3 | *.egg-info/ 4 | __pycache__/ 5 | .coverage 6 | .idea 7 | html/ 8 | dist/ 9 | -------------------------------------------------------------------------------- /.trunk/.gitignore: -------------------------------------------------------------------------------- 1 | *out 2 | *logs 3 | *actions 4 | *notifications 5 | plugins 6 | user_trunk.yaml 7 | user.yaml 8 | shims 9 | -------------------------------------------------------------------------------- /.trunk/configs/.flake8: -------------------------------------------------------------------------------- 1 | # Autoformatter friendly flake8 config (all formatting rules disabled) 2 | [flake8] 3 | extend-ignore = D1, D2, E1, E2, E3, E501, W1, W2, W3, W5 4 | -------------------------------------------------------------------------------- /.trunk/configs/.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | profile=black 3 | -------------------------------------------------------------------------------- /.trunk/configs/.markdownlint.yaml: -------------------------------------------------------------------------------- 1 | # Autoformatter friendly markdownlint config (all formatting rules disabled) 2 | default: true 3 | blank_lines: false 4 | bullet: false 5 | html: false 6 | indentation: false 7 | line_length: false 8 | spaces: false 9 | url: false 10 | whitespace: false 11 | -------------------------------------------------------------------------------- /.trunk/trunk.yaml: -------------------------------------------------------------------------------- 1 | version: 0.1 2 | cli: 3 | version: 1.9.1 4 | plugins: 5 | sources: 6 | - id: trunk 7 | ref: v0.0.14 8 | uri: https://github.com/trunk-io/plugins 9 | lint: 10 | disabled: 11 | - isort 12 | ignore: 13 | - linters: [mypy] 14 | paths: 15 | - "test/**" 16 | enabled: 17 | - ruff@0.0.264 18 | - mypy@1.2.0 19 | - prettier@2.8.8 20 | - black@23.3.0 21 | - git-diff-check 22 | - markdownlint@0.34.0 23 | - gitleaks@8.16.3 24 | - actionlint@1.6.24 25 | - taplo@0.7.0 26 | runtimes: 27 | enabled: 28 | - go@1.18.3 29 | - node@18.12.1 30 | - python@3.10.8 31 | actions: 32 | enabled: 33 | - trunk-announce 34 | - trunk-check-pre-push 35 | - trunk-fmt-pre-commit 36 | - trunk-upgrade-available 37 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @woodruffw 2 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to `blight` 2 | 3 | Thank you for your interest in contributing to `blight`! 4 | 5 | The information below will help you set up a local development environment, 6 | as well as performing common development tasks. 7 | 8 | - [Requirements](#requirements) 9 | - [Development steps](#development-steps) 10 | - [Linting](#linting) 11 | - [Testing](#testing) 12 | - [Documentation](#documentation) 13 | - [Contributing a new action](#contributing-a-new-action) 14 | 15 | ## Requirements 16 | 17 | `blight`'s only requirement is Python 3.8 or newer. 18 | 19 | Development and testing is actively performed on macOS and Linux, but Windows 20 | and other supported platforms that are supported by Python should also work. 21 | 22 | If you're on a system that has GNU Make, you can use the convenience targets 23 | included in the Makefile that comes in the `blight` repository detailed below. 24 | But this isn't required; all steps can be done without Make. 25 | 26 | ## Development steps 27 | 28 | First, clone this repository: 29 | 30 | ```bash 31 | git clone https://github.com/trailofbits/blight 32 | cd blight 33 | ``` 34 | 35 | Then, use one of the `Makefile` targets to run a task. The first time this is 36 | run, this will also set up the local development virtual environment, and will 37 | install `blight` as an editable package into this environment. 38 | 39 | Any changes you make to the `src/blight` source tree will take effect 40 | immediately in the virtual environment. 41 | 42 | ### Linting 43 | 44 | This repository uses [trunk.io](https://trunk.io) and the `trunk` CLI for 45 | linting. 46 | 47 | If you don't already have `trunk`, you can download it with one of the 48 | following: 49 | 50 | ```bash 51 | # macOS (Homebrew Cask) 52 | brew install trunk-io 53 | 54 | # all platforms 55 | curl https://get.trunk.io -fsSL | bash 56 | ``` 57 | 58 | Once installed, you can run `trunk check` and `trunk fmt` for all linting 59 | and auto-formatting, or use the Makefile: 60 | 61 | ```bash 62 | # run all linters 63 | make lint 64 | 65 | # run all auto-formatters 66 | make format 67 | ``` 68 | 69 | By default, only modified files are checked. 70 | 71 | ### Testing 72 | 73 | You can run the tests locally with: 74 | 75 | ```bash 76 | make test 77 | ``` 78 | 79 | You can also filter by a pattern (uses `pytest -k`): 80 | 81 | ```bash 82 | make test TESTS=test_audit_dry_run 83 | ``` 84 | 85 | To test a specific file: 86 | 87 | ```bash 88 | make test T=path/to/file.py 89 | ``` 90 | 91 | `blight` has a [`pytest`](https://docs.pytest.org/)-based unit test suite, 92 | including code coverage with [`coverage.py`](https://coverage.readthedocs.io/). 93 | 94 | ### Documentation 95 | 96 | If you're running Python 3.7 or newer, you can run the documentation build locally: 97 | 98 | ```bash 99 | make doc 100 | ``` 101 | 102 | `blight` uses [`pdoc`](https://github.com/mitmproxy/pdoc) to generate HTML 103 | documentation for its public Python APIs. 104 | 105 | Live documentation for the `master` branch is hosted 106 | [here](https://trailofbits.github.io/blight/). 107 | 108 | ## Contributing a new action 109 | 110 | New blight actions are easy to write. For example, the following prints a 111 | message before every `ld` invocation: 112 | 113 | ```python 114 | # src/blight/actions/printld.py 115 | 116 | from blight.action import LDAction 117 | 118 | 119 | class PrintLD(LDAction): 120 | def before_run(self, tool): 121 | print(f"ld was run with: {tool.args}") 122 | ``` 123 | 124 | ```python 125 | # src/blight/actions/__init__.py 126 | 127 | # bring PrintLD into blight.actions so that `BLIGHT_ACTIONS` can find it 128 | from printld import PrintLD # noqa: F401 129 | ``` 130 | 131 | ```bash 132 | eval $(blight-env --guess-wrapped) 133 | export BLIGHT_ACTIONS="PrintLD" 134 | make 135 | ``` 136 | 137 | Check out blight's [API documentation](https://trailofbits.github.io/blight) 138 | for more details, including the kinds of available actions. 139 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PY_MODULE := blight 2 | 3 | # Optionally overriden by the user, if they're using a virtual environment manager. 4 | VENV ?= env 5 | VENV_EXISTS := $(VENV)/pyvenv.cfg 6 | 7 | # On Windows, venv scripts/shims are under `Scripts` instead of `bin`. 8 | VENV_BIN := $(VENV)/bin 9 | ifeq ($(OS),Windows_NT) 10 | VENV_BIN := $(VENV)/Scripts 11 | endif 12 | 13 | # Optionally overridden by the user/CI, to limit the installation to a specific 14 | # subset of development dependencies. 15 | INSTALL_EXTRA ?= dev 16 | 17 | ALL_PY_SRCS := $(shell find src -name '*.py') \ 18 | $(shell find test -name '*.py') 19 | 20 | .PHONY: all 21 | all: 22 | @echo "Run my targets individually!" 23 | 24 | $(VENV)/pyvenv.cfg: pyproject.toml 25 | python -m venv env 26 | . $(VENV_BIN)/activate && \ 27 | pip install --upgrade pip setuptools && \ 28 | pip install -e .[$(INSTALL_EXTRA)] 29 | 30 | .PHONY: dev 31 | dev: $(VENV)/pyvenv.cfg 32 | 33 | .PHONY: lint 34 | lint: 35 | trunk check 36 | 37 | .PHONY: format 38 | format: 39 | trunk fmt 40 | 41 | .PHONY: test 42 | test: $(VENV_EXISTS) 43 | . $(VENV_BIN)/activate && \ 44 | pytest --cov=$(PY_MODULE) test/ && \ 45 | python -m coverage report -m --fail-under 100 46 | 47 | .PHONY: doc 48 | doc: $(VENV_EXISTS) 49 | . $(VENV_BIN)/activate && \ 50 | pdoc -o html $(PY_MODULE) 51 | 52 | .PHONY: package 53 | package: $(VENV_EXISTS) 54 | . $(VENV_BIN)/activate && \ 55 | python -m build && \ 56 | twine upload --repository pypi dist/* 57 | 58 | .PHONY: edit 59 | edit: 60 | $(EDITOR) $(ALL_PY_SRCS) 61 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # blight 2 | 3 | ![CI](https://github.com/trailofbits/blight/workflows/CI/badge.svg) 4 | [![PyPI version](https://badge.fury.io/py/blight.svg)](https://badge.fury.io/py/blight) 5 | [![Downloads](https://pepy.tech/badge/blight)](https://pepy.tech/project/blight) 6 | 7 | `blight` is a framework for wrapping and instrumenting build tools and build 8 | systems. It contains: 9 | 10 | 1. A collection of high-fidelity models for various common build tools (e.g. 11 | the C and C++ compilers, the standard linker, the preprocessor, etc.); 12 | 1. A variety of "actions" that can be run on each build tool or specific 13 | classes of tools (e.g. "whenever the build system invokes `$CC`, add this 14 | flag"); 15 | 1. Command-line wrappers (`blight-env` and `blight-exec`) for instrumenting 16 | builds. 17 | 18 | - [Installation](#installation) 19 | - [Usage](#usage) 20 | - [Quickstart](#quickstart) 21 | - [Cookbook](#cookbook) 22 | - [Goals](#goals) 23 | - [Anti-goals](#anti-goals) 24 | - [Contributing](#contributing) 25 | 26 | ## Installation 27 | 28 | `blight` is available on PyPI and is installable via `pip`: 29 | 30 | ```bash 31 | python -m pip install blight 32 | ``` 33 | 34 | Python 3.8 or newer is required. 35 | 36 | ## Usage 37 | 38 | `blight` comes with two main entrypoints: 39 | 40 | - `blight-exec`: directly execute a command within a `blight`-instrumented 41 | environment 42 | - `blight-env`: write a `sh`-compatible environment definition to `stdout`, 43 | which the shell or other tools can consume to enter a `blight`-instrumented 44 | environment 45 | 46 | In most cases, you'll probably want `blight-exec`. `blight-env` can be thought 47 | of as the "advanced" or "plumbing" interface. 48 | 49 | 50 | 51 | ```text 52 | Usage: blight-exec [OPTIONS] TARGET [ARGS]... 53 | 54 | Options: 55 | --guess-wrapped Attempt to guess the appropriate programs to wrap 56 | --swizzle-path Wrap via PATH swizzling 57 | --stub STUB Stub a command out while swizzling 58 | --shim SHIM Add a custom shim while swizzling 59 | --action ACTION Enable an action 60 | --journal-path PATH The path to use for action journaling 61 | --help Show this message and exit. 62 | ``` 63 | 64 | 65 | 66 | 67 | 68 | ```text 69 | Usage: blight-env [OPTIONS] 70 | 71 | Options: 72 | --guess-wrapped Attempt to guess the appropriate programs to wrap 73 | --swizzle-path Wrap via PATH swizzling 74 | --stub TEXT Stub a command out while swizzling 75 | --shim TEXT Add a custom shim while swizzling 76 | --unset Unset the tool variables instead of setting them 77 | --help Show this message and exit. 78 | ``` 79 | 80 | 81 | 82 | ## Quickstart 83 | 84 | The easiest way to get started with `blight` is to use `blight-exec` with 85 | `--guess-wrapped` and `--swizzle-path`. These flags tell `blight` to configure 86 | the environment with some common-sense defaults: 87 | 88 | - `--guess-wrapped`: guess the appropriate underlying tools to invoke from 89 | the current `PATH` and other runtime environment; 90 | - `--swizzle-path`: rewrite the `PATH` to put some common build tool shims 91 | first, e.g. redirecting `cc` to `blight-cc`. 92 | 93 | For example, the following will run `cc -v` under `blight`'s instrumentation, 94 | with the [`Demo`](https://trailofbits.github.io/blight/blight/actions.html#Demo) 95 | action: 96 | 97 | ```bash 98 | blight-exec --action Demo --swizzle-path --guess-wrapped -- cc -v 99 | ``` 100 | 101 | which should produce something like: 102 | 103 | ```console 104 | [demo] before-run: /usr/bin/cc 105 | Apple clang version 14.0.0 (clang-1400.0.29.202) 106 | Target: x86_64-apple-darwin22.2.0 107 | Thread model: posix 108 | InstalledDir: /Library/Developer/CommandLineTools/usr/bin 109 | [demo] after-run: /usr/bin/cc 110 | ``` 111 | 112 | We can also see the effect of `--swizzle-path` by running `which cc` under 113 | `blight`, and observing that it points to a temporary shim rather than the 114 | normal `cc` location: 115 | 116 | ```bash 117 | $ blight-exec --swizzle-path --guess-wrapped -- which cc 118 | /var/folders/zj/hy934vnj5xs68zv6w4b_f6s40000gn/T/tmp5uahp6tg@blight-swizzle@/cc 119 | 120 | $ which cc 121 | /usr/bin/cc 122 | ``` 123 | 124 | All the `Demo` action does is print a message before and after each tool run, 125 | allowing you to diagnose when a tool is or isn't correctly instrumented. 126 | See the [actions documentation below](#enabling-actions) for information on 127 | using and configuring more interesting actions. 128 | 129 | ## Cookbook 130 | 131 | ### Running `blight` against a `make`-based build 132 | 133 | Most `make`-based builds use `$(CC)`, `$(CXX)`, etc., which means that they 134 | should work out of the box with `blight-exec`: 135 | 136 | ```bash 137 | blight-exec --guess-wrapped -- make 138 | ``` 139 | 140 | In some cases, poorly written builds may hard-code `cc`, `clang`, `gcc`, etc. 141 | rather than using their symbolic counterparts. For these, you can use 142 | `--swizzle-path` to interpose shims that redirect those hardcoded tool 143 | invocations back to `blight`'s wrappers: 144 | 145 | ```bash 146 | blight-exec --guess-wrapped --swizzle-path -- make 147 | ``` 148 | 149 | See [Taming an uncooperative build with shims and stubs](#taming-an-uncooperative-build-with-shims-and-stubs) 150 | for more advanced techniques for dealing with poorly written build systems. 151 | 152 | ### Enabling actions 153 | 154 | Actions are where `blight` really shines: they allow you to run arbitrary Python 155 | code before and after each build tool invocation. 156 | 157 | `blight` comes with built-in actions, which are 158 | [documented here](https://trailofbits.github.io/blight/blight/actions.html). 159 | See each action's Python module for its documentation. 160 | 161 | Actions can be enabled in two different ways: 162 | 163 | - With the `--action` flag, which can be passed multiple times. For example, 164 | `--action SkipStrip --action Record` enables both the 165 | [`SkipStrip`](https://trailofbits.github.io/blight/blight/actions.html#SkipStrip) 166 | and [`Record`](https://trailofbits.github.io/blight/blight/actions.html#Record) 167 | actions. 168 | 169 | - With the `BLIGHT_ACTIONS` environment variable, which can take multiple 170 | actions delimited by `:`. For example, `BLIGHT_ACTIONS=SkipStrip:Record` 171 | is equivalent to `--action SkipStrip --action Record`. 172 | 173 | Actions are run in the order of specification with duplicates removed, meaning 174 | that `BLIGHT_ACTIONS=Foo:Bar:Foo` is equivalent to `BLIGHT_ACTIONS=Foo:Bar` 175 | but **not** `BLIGHT_ACTIONS=Bar:Foo`. This is important if actions have side 176 | effects, which they may (such as modifying the tool's flags). 177 | 178 | #### Action configuration 179 | 180 | Some actions accept or require additional configuration, which is passed 181 | through the `BLIGHT_ACTION_{ACTION}` environment variable in `key=value` 182 | format, where `{ACTION}` is the uppercased name of the action. 183 | 184 | For example, to configure `Record`'s output file: 185 | 186 | ```bash 187 | BLIGHT_ACTION_RECORD="output=/tmp/output.jsonl" 188 | ``` 189 | 190 | #### Action outputs 191 | 192 | There are two ways to get output from actions under `blight`: 193 | 194 | - Many actions support an `output` configuration value, which should be a 195 | filename to write to. This allows each action to write its own output 196 | file. 197 | - `blight` supports a "journaling" mode, in which all action outputs 198 | are written to a single file, keyed by action name. 199 | 200 | The "journaling" mode is generally encouraged over individual outputs, 201 | and can be enabled with either `BLIGHT_JOURNAL_PATH=/path/to/output.jsonl` 202 | in the environment or `blight-exec --journal-path /path/to/output.jsonl`. 203 | 204 | ### Configuring an environment with `blight-env` 205 | 206 | `blight-env` behaves exactly the same as `blight-exec`, except that it 207 | stops before actually executing anything. You can use it to set up an 208 | environment for use across multiple build system runs. 209 | 210 | By default, `blight-env` will just export the appropriate environment 211 | for replacing `CC`, etc., with their `blight` wrappers: 212 | 213 | ```bash 214 | $ blight-env 215 | export CC=blight-cc 216 | export CXX=blight-c++ 217 | export CPP=blight-cpp 218 | export LD=blight-ld 219 | export AS=blight-as 220 | export AR=blight-ar 221 | export STRIP=blight-strip 222 | export INSTALL=blight-install 223 | ``` 224 | 225 | `--guess-wrapped` augments this by adding a best-guess underlying tool for 226 | each wrapper: 227 | 228 | ```bash 229 | $ blight-env --guess-wrapped 230 | export BLIGHT_WRAPPED_CC=/usr/bin/cc 231 | export BLIGHT_WRAPPED_CXX=/usr/bin/c++ 232 | export BLIGHT_WRAPPED_CPP=/usr/bin/cpp 233 | export BLIGHT_WRAPPED_LD=/usr/bin/ld 234 | export BLIGHT_WRAPPED_AS=/usr/bin/as 235 | export BLIGHT_WRAPPED_AR=/usr/bin/ar 236 | export BLIGHT_WRAPPED_STRIP=/usr/bin/strip 237 | export BLIGHT_WRAPPED_INSTALL=/usr/bin/install 238 | export CC=blight-cc 239 | export CXX=blight-c++ 240 | export CPP=blight-cpp 241 | export LD=blight-ld 242 | export AS=blight-as 243 | export AR=blight-ar 244 | export STRIP=blight-strip 245 | export INSTALL=blight-install 246 | ``` 247 | 248 | `--guess-wrapped` also respects `CC`, etc. in the environment: 249 | 250 | ```bash 251 | $ CC=/some/custom/cc blight-env --guess-wrapped 252 | export BLIGHT_WRAPPED_CC=/some/custom/cc 253 | export BLIGHT_WRAPPED_CXX=/usr/bin/c++ 254 | export BLIGHT_WRAPPED_CPP=/usr/bin/cpp 255 | export BLIGHT_WRAPPED_LD=/usr/bin/ld 256 | export BLIGHT_WRAPPED_AS=/usr/bin/as 257 | export BLIGHT_WRAPPED_AR=/usr/bin/ar 258 | export BLIGHT_WRAPPED_STRIP=/usr/bin/strip 259 | export BLIGHT_WRAPPED_INSTALL=/usr/bin/install 260 | export CC=blight-cc 261 | export CXX=blight-c++ 262 | export CPP=blight-cpp 263 | export LD=blight-ld 264 | export AS=blight-as 265 | export AR=blight-ar 266 | export STRIP=blight-strip 267 | export INSTALL=blight-install 268 | ``` 269 | 270 | `--swizzle-path` further modifies the environment by rewriting `PATH`: 271 | 272 | ```bash 273 | $ blight-env --guess-wrapped-swizzle-path 274 | export BLIGHT_WRAPPED_CC=/usr/bin/cc 275 | export BLIGHT_WRAPPED_CXX=/usr/bin/c++ 276 | export BLIGHT_WRAPPED_CPP=/usr/bin/cpp 277 | export BLIGHT_WRAPPED_LD=/usr/bin/ld 278 | export BLIGHT_WRAPPED_AS=/usr/bin/as 279 | export BLIGHT_WRAPPED_AR=/usr/bin/ar 280 | export BLIGHT_WRAPPED_STRIP=/usr/bin/strip 281 | export BLIGHT_WRAPPED_INSTALL=/usr/bin/install 282 | export PATH='/var/folders/zj/hy934vnj5xs68zv6w4b_f6s40000gn/T/tmpxh5ryu22@blight-swizzle@:/bin:/usr/bin:/usr/local/bin' 283 | export CC=blight-cc 284 | export CXX=blight-c++ 285 | export CPP=blight-cpp 286 | export LD=blight-ld 287 | export AS=blight-as 288 | export AR=blight-ar 289 | export STRIP=blight-strip 290 | export INSTALL=blight-install 291 | ``` 292 | 293 | The swizzled addition can be identified by its `@blight-swizzle@` directory name. 294 | 295 | ### Taming an uncooperative build with shims and stubs 296 | 297 | Sometimes build systems need more coaxing than just `--guess-wrapped` and 298 | `--swizzle-path`. Common examples include: 299 | 300 | - Hard-coding a particular tool or tool version rather than using the symbolic 301 | name (e.g. `clang-7 example.c` instead of `$(CC) example.c`); 302 | - Running lots of "junk" commands that can be suppressed (e.g. lots of `echo` 303 | invocations) 304 | 305 | You can use _shims_ and _stubs_ to smooth out these cases: 306 | 307 | - _shims_ replace a command with a build tool that `blight` knows about, e.g. 308 | `clang-3.8` with `cc`. 309 | - _stubs_ replace a command with an invocation of `true`, meaning that it 310 | does nothing and never fails. 311 | 312 | Shims are specified with `--shim cmd:tool`, while stubs are specified with 313 | `--stub cmd`. Both require `--swizzle-path`, since the `PATH` must be rewritten 314 | to inject additional commands. 315 | 316 | For example, to instrument a build system that hardcodes `tcc` everywhere 317 | and that spews way too much output with `echo`: 318 | 319 | ```bash 320 | blight-exec --guess-wrapped --swizzle-path --shim tcc:cc --stub echo -- make 321 | ``` 322 | 323 | ## Goals 324 | 325 | - Wrapping `CC`, `CXX`, `CPP`, `LD`, `AS`, `AR`, `STRIP`, and `INSTALL`. 326 | - Providing a visitor-style API for each of the above, pre- and post-execution. 327 | - Providing a nice set of default actions. 328 | - Being as non-invasive as possible. 329 | 330 | ## Anti-goals 331 | 332 | - Using `LD_PRELOAD` to capture every `exec` in a build system, 333 | a la [Bear](https://github.com/rizsotto/Bear). 334 | - Supporting `cl.exe`. 335 | - Detailed support for non C/C++ languages. 336 | 337 | ## Contributing 338 | 339 | Check out our [CONTRIBUTING.md](./CONTRIBUTING.md)! 340 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["flit_core >=3.2,<4"] 3 | build-backend = "flit_core.buildapi" 4 | 5 | [project] 6 | name = "blight" 7 | dynamic = ["version"] 8 | description = "A catch-all compile-tool wrapper" 9 | readme = "README.md" 10 | license = { file = "LICENSE" } 11 | authors = [{ name = "William Woodruff", email = "william@trailofbits.com" }] 12 | classifiers = [ 13 | "Programming Language :: Python :: 3", 14 | "License :: OSI Approved :: Apache Software License", 15 | "Intended Audience :: Developers", 16 | "Topic :: Software Development :: Build Tools", 17 | ] 18 | dependencies = ["click >= 7.1,< 9.0", "pydantic ~= 1.7"] 19 | requires-python = ">=3.8" 20 | 21 | [project.optional-dependencies] 22 | doc = ["pdoc"] 23 | test = ["pytest", "pytest-cov", "pretend", "coverage[toml]"] 24 | dev = ["blight[doc,test]", "twine"] 25 | 26 | [project.scripts] 27 | "blight-env" = "blight._cli:env" 28 | "blight-exec" = "blight._cli:exec_" 29 | "blight-cc" = "blight._cli:tool" 30 | "blight-c++" = "blight._cli:tool" 31 | "blight-cpp" = "blight._cli:tool" 32 | "blight-ld" = "blight._cli:tool" 33 | "blight-as" = "blight._cli:tool" 34 | "blight-ar" = "blight._cli:tool" 35 | "blight-strip" = "blight._cli:tool" 36 | "blight-install" = "blight._cli:tool" 37 | 38 | [project.urls] 39 | Homepage = "https://pypi.org/project/blight" 40 | Documentation = "https://trailofbits.github.io/blight/" 41 | Issues = "https://github.com/trailofbits/blight/issues" 42 | Source = "https://github.com/trailofbits/blight" 43 | 44 | [tool.black] 45 | line-length = 100 46 | 47 | [tool.coverage.run] 48 | # don't attempt code coverage for the CLI entrypoints 49 | omit = ["src/blight/_cli.py"] 50 | 51 | [tool.mypy] 52 | allow_redefinition = true 53 | check_untyped_defs = true 54 | disallow_incomplete_defs = true 55 | disallow_untyped_defs = true 56 | ignore_missing_imports = true 57 | no_implicit_optional = true 58 | show_error_codes = true 59 | sqlite_cache = true 60 | strict_equality = true 61 | warn_no_return = true 62 | warn_redundant_casts = true 63 | warn_return_any = true 64 | warn_unreachable = true 65 | warn_unused_configs = true 66 | warn_unused_ignores = true 67 | 68 | [tool.ruff] 69 | line-length = 100 70 | select = ["E", "F", "W", "UP", "I", "N", "YTT", "BLE", "C4", "SIM"] 71 | target-version = "py38" 72 | -------------------------------------------------------------------------------- /src/blight/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | `blight`'s public APIs. 3 | 4 | Some specific APIs of interest: 5 | 6 | * `blight.tool`: APIs for interacting with specific tool models (e.g. `blight.tool.CC`) 7 | * `blight.action`: Core interfaces for defining actions (instantiated under `blight.actions`) 8 | """ 9 | 10 | __version__ = "0.0.53" 11 | -------------------------------------------------------------------------------- /src/blight/_cli.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import shlex 4 | import shutil 5 | import stat 6 | import sys 7 | import tempfile 8 | from pathlib import Path 9 | from typing import Iterator, List, Tuple 10 | 11 | import click 12 | 13 | import blight.tool 14 | from blight.enums import BlightTool, BuildTool 15 | from blight.exceptions import BlightError 16 | from blight.util import die, unswizzled_path 17 | 18 | logging.basicConfig(level=os.environ.get("BLIGHT_LOGLEVEL", "INFO").upper()) 19 | logger = logging.getLogger(__name__) 20 | 21 | # Particularly common clang version suffixes. 22 | CLANG_VERSION_SUFFIXES = ["3.8", "7", "9", "10", "11", "12", "13", "14", "15", "16"] 23 | 24 | # A mapping of shim name -> tool name for default shim generation. 25 | # Each shim will ultimately call `blight-{tool}`, completing the interposition. 26 | # fmt: off 27 | SHIM_MAP = { 28 | # Standard build tool names. 29 | "cc": BuildTool.CC, 30 | "c++": BuildTool.CXX, 31 | "cpp": BuildTool.CPP, 32 | "ld": BuildTool.LD, 33 | "as": BuildTool.AS, 34 | "ar": BuildTool.AR, 35 | "strip": BuildTool.STRIP, 36 | "install": BuildTool.INSTALL, 37 | 38 | # GNU shims. 39 | "gcc": BuildTool.CC, 40 | "g++": BuildTool.CXX, 41 | "gold": BuildTool.LD, 42 | "gas": BuildTool.AS, 43 | 44 | # Clang shims. 45 | "clang": BuildTool.CC, 46 | **{f"clang-{v}": BuildTool.CC for v in CLANG_VERSION_SUFFIXES}, 47 | "clang++": BuildTool.CXX, 48 | **{f"clang++-{v}": BuildTool.CXX for v in CLANG_VERSION_SUFFIXES}, 49 | "lld": BuildTool.LD, 50 | } 51 | # fmt: on 52 | 53 | 54 | def _unset(variable: str) -> None: 55 | print(f"unset {variable}") 56 | 57 | 58 | def _export(variable: str, value: str) -> None: 59 | value = shlex.quote(value) 60 | print(f"export {variable}={value}") 61 | 62 | 63 | def _guess_wrapped() -> Iterator[Tuple[str, str]]: 64 | for tool in BuildTool: 65 | tool_path = os.getenv(tool.env) 66 | if tool_path is None: 67 | tool_path = shutil.which(tool.cmd) 68 | if tool_path is None: 69 | die(f"Couldn't locate {tool} on the $PATH") 70 | 71 | yield (tool.blight_tool.env, tool_path) 72 | 73 | 74 | def _swizzle_path(stubs: List[str], shim_specs: List[str]) -> str: 75 | blight_dir = Path(tempfile.mkdtemp(suffix=blight.util.SWIZZLE_SENTINEL)) 76 | 77 | for shim, tool in SHIM_MAP.items(): 78 | shim_path = blight_dir / shim 79 | with open(shim_path, "w+") as io: 80 | print("#!/bin/sh", file=io) 81 | print(f'{tool.blight_tool.value} "${{@}}"', file=io) 82 | 83 | shim_path.chmod(shim_path.stat().st_mode | stat.S_IEXEC) 84 | 85 | for shim_spec in shim_specs: 86 | try: 87 | (shim, tool_name) = shim_spec.split(":", 1) 88 | tool = BuildTool(tool_name.upper()) 89 | except ValueError: 90 | die( 91 | f"Malformatted custom shim spec: expected `shim:tool`, got {shim_spec} " 92 | f"(supported tools: {[t.value for t in BuildTool]})" 93 | ) 94 | 95 | if shim in SHIM_MAP: 96 | logger.warning(f"overriding default shim ({shim}) with custom tool ({tool})") 97 | 98 | shim_path = blight_dir / shim 99 | with open(shim_path, "w+") as io: 100 | print("#!/bin/sh", file=io) 101 | print(f'{tool.blight_tool.value} "${{@}}"', file=io) 102 | 103 | shim_path.chmod(shim_path.stat().st_mode | stat.S_IEXEC) 104 | 105 | for stub in stubs: 106 | stub_path = blight_dir / stub 107 | with open(stub_path, "w+") as io: 108 | print("#!/bin/sh", file=io) 109 | print("true", file=io) 110 | 111 | stub_path.chmod(stub_path.stat().st_mode | stat.S_IEXEC) 112 | 113 | return f"{blight_dir}:{unswizzled_path()}" 114 | 115 | 116 | @click.command() 117 | @click.option( 118 | "--guess-wrapped", help="Attempt to guess the appropriate programs to wrap", is_flag=True 119 | ) 120 | @click.option("--swizzle-path", help="Wrap via PATH swizzling", is_flag=True) 121 | @click.option("--stub", "stubs", help="Stub a command out while swizzling", multiple=True) 122 | @click.option("--shim", "shims", help="Add a custom shim while swizzling", multiple=True) 123 | @click.option("--unset", help="Unset the tool variables instead of setting them", is_flag=True) 124 | def env( 125 | unset: bool, guess_wrapped: bool, swizzle_path: bool, stubs: List[str], shims: List[str] 126 | ) -> None: 127 | if guess_wrapped: 128 | for variable, value in _guess_wrapped(): 129 | if variable not in os.environ: 130 | _export(variable, value) 131 | 132 | if swizzle_path: 133 | _export("PATH", _swizzle_path(stubs, shims)) 134 | 135 | for tool in BuildTool: 136 | if unset: 137 | _unset(tool.env) 138 | else: 139 | _export(tool.env, tool.blight_tool.value) 140 | 141 | 142 | @click.command() 143 | @click.option( 144 | "--guess-wrapped", help="Attempt to guess the appropriate programs to wrap", is_flag=True 145 | ) 146 | @click.option("--swizzle-path", help="Wrap via PATH swizzling", is_flag=True) 147 | @click.option( 148 | "--stub", "stubs", metavar="STUB", help="Stub a command out while swizzling", multiple=True 149 | ) 150 | @click.option( 151 | "--shim", "shims", metavar="SHIM", help="Add a custom shim while swizzling", multiple=True 152 | ) 153 | @click.option("--action", "actions", metavar="ACTION", help="Enable an action", multiple=True) 154 | @click.option( 155 | "--journal-path", 156 | metavar="PATH", 157 | help="The path to use for action journaling", 158 | type=click.Path(dir_okay=False, exists=False, path_type=Path), 159 | ) 160 | @click.argument("target") 161 | @click.argument("args", nargs=-1) 162 | def exec_( 163 | guess_wrapped: bool, 164 | swizzle_path: bool, 165 | stubs: List[str], 166 | shims: List[str], 167 | actions: List[str], 168 | journal_path: click.Path, 169 | target: str, 170 | args: List[str], 171 | ) -> None: 172 | env = dict(os.environ) 173 | 174 | if guess_wrapped: 175 | env.update( 176 | {variable: value for (variable, value) in _guess_wrapped() if variable not in env} 177 | ) 178 | 179 | if swizzle_path: 180 | env["PATH"] = _swizzle_path(stubs, shims) 181 | 182 | if len(actions) > 0: 183 | env["BLIGHT_ACTIONS"] = ":".join(actions) 184 | 185 | if journal_path is not None: 186 | env["BLIGHT_JOURNAL_PATH"] = str(journal_path) 187 | 188 | env.update({tool.env: tool.blight_tool.value for tool in BuildTool}) 189 | 190 | logger.debug(f"built environment: {env}") 191 | 192 | os.execvpe(target, [target, *args], env) 193 | 194 | 195 | def tool() -> None: 196 | # NOTE(ww): Specifically *not* a click command! 197 | wrapped_basename = os.path.basename(sys.argv[0]) 198 | 199 | try: 200 | blight_tool = BlightTool(wrapped_basename) 201 | except ValueError: 202 | die(f"Unknown blight wrapper requested: {wrapped_basename}") 203 | 204 | tool_class = getattr(blight.tool, blight_tool.build_tool.value) 205 | tool = tool_class(sys.argv[1:]) 206 | try: 207 | tool.run() 208 | except BlightError as e: 209 | die(str(e)) 210 | -------------------------------------------------------------------------------- /src/blight/action.py: -------------------------------------------------------------------------------- 1 | """ 2 | The different actions supported by blight. 3 | """ 4 | 5 | from typing import Any, Dict, Optional 6 | 7 | import blight.tool 8 | from blight.tool import Tool 9 | 10 | 11 | class Action: 12 | """ 13 | A generic action, run with every tool (both before and after the tool's execution). 14 | """ 15 | 16 | def __init__(self, config: Dict[str, str]): 17 | self._config = config 18 | self._result: Optional[Dict[str, Any]] = None 19 | 20 | def _should_run_on(self, tool: Tool) -> bool: 21 | return True 22 | 23 | def before_run(self, tool: Tool) -> None: # pragma: no cover 24 | """ 25 | Invoked right before the underlying tool is run. 26 | 27 | Args: 28 | tool: The tool about to run 29 | """ 30 | pass 31 | 32 | def _before_run(self, tool: Tool) -> None: 33 | if self._should_run_on(tool): 34 | self.before_run(tool) 35 | 36 | def after_run(self, tool: Tool, *, run_skipped: bool = False) -> None: # pragma: no cover 37 | """ 38 | Invoked right after the underlying tool is run. 39 | 40 | Args: 41 | tool: The tool that just ran 42 | """ 43 | pass 44 | 45 | def _after_run(self, tool: Tool, *, run_skipped: bool = False) -> None: 46 | if self._should_run_on(tool): 47 | self.after_run(tool, run_skipped=run_skipped) 48 | 49 | @property 50 | def result(self) -> Optional[Dict[str, Any]]: 51 | """ 52 | Returns the result computed by this action, if there is one. 53 | """ 54 | return self._result 55 | 56 | 57 | class CCAction(Action): 58 | """ 59 | A `cc` action, run whenever the tool is a `blight.tool.CC` instance. 60 | """ 61 | 62 | def _should_run_on(self, tool: Tool) -> bool: 63 | return isinstance(tool, blight.tool.CC) 64 | 65 | 66 | class CXXAction(Action): 67 | """ 68 | A `c++` action, run whenever the tool is a `blight.tool.CXX` instance. 69 | """ 70 | 71 | def _should_run_on(self, tool: Tool) -> bool: 72 | return isinstance(tool, blight.tool.CXX) 73 | 74 | 75 | class CompilerAction(CCAction, CXXAction): 76 | """ 77 | A generic compiler action, run whenever the tool is a `blight.tool.CC` 78 | or `blight.tool.CXX` instance. 79 | 80 | **NOTE:** Action writers should generally prefer this over `CCAction` and `CXXAction`, 81 | as messy builds may use `cc` to compile C++ sources (via `-x c`) and vice versa. 82 | """ 83 | 84 | def _should_run_on(self, tool: Tool) -> bool: 85 | return isinstance(tool, (blight.tool.CC, blight.tool.CXX)) 86 | 87 | 88 | class CPPAction(Action): 89 | """ 90 | A `cpp` action, run whenever the tool is a `blight.tool.CPP` instance. 91 | """ 92 | 93 | def _should_run_on(self, tool: Tool) -> bool: 94 | return isinstance(tool, blight.tool.CPP) 95 | 96 | 97 | class LDAction(Action): 98 | """ 99 | An `ld` action, run whenever the tool is a `blight.tool.LD` instance. 100 | """ 101 | 102 | def _should_run_on(self, tool: Tool) -> bool: 103 | return isinstance(tool, blight.tool.LD) 104 | 105 | 106 | class ASAction(Action): 107 | """ 108 | An `as` action, run whenever the tool is a `blight.tool.AS` instance. 109 | """ 110 | 111 | def _should_run_on(self, tool: Tool) -> bool: 112 | return isinstance(tool, blight.tool.AS) 113 | 114 | 115 | class ARAction(Action): 116 | """ 117 | An `ar` action, run whenever the tool is a `blight.tool.AR` instance. 118 | """ 119 | 120 | def _should_run_on(self, tool: Tool) -> bool: 121 | return isinstance(tool, blight.tool.AR) 122 | 123 | 124 | class STRIPAction(Action): 125 | """ 126 | A `strip` action, run whenever the tool is a `blight.tool.STRIP` instance. 127 | """ 128 | 129 | def _should_run_on(self, tool: Tool) -> bool: 130 | return isinstance(tool, blight.tool.STRIP) 131 | -------------------------------------------------------------------------------- /src/blight/actions/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Actions supported by blight. 3 | """ 4 | 5 | from .benchmark import Benchmark 6 | from .cc_for_cxx import CCForCXX 7 | from .demo import Demo 8 | from .embed_bitcode import EmbedBitcode 9 | from .find_inputs import FindInputs 10 | from .find_outputs import FindOutputs 11 | from .ignore_flags import IgnoreFlags 12 | from .ignore_flto import IgnoreFlto 13 | from .ignore_werror import IgnoreWerror 14 | from .inject_flags import InjectFlags 15 | from .lint import Lint 16 | from .record import Record 17 | from .skip_strip import SkipStrip 18 | 19 | __all__ = [ 20 | "Benchmark", 21 | "CCForCXX", 22 | "Demo", 23 | "EmbedBitcode", 24 | "FindInputs", 25 | "FindOutputs", 26 | "IgnoreFlags", 27 | "IgnoreFlto", 28 | "IgnoreWerror", 29 | "InjectFlags", 30 | "Lint", 31 | "Record", 32 | "SkipStrip", 33 | ] 34 | -------------------------------------------------------------------------------- /src/blight/actions/benchmark.py: -------------------------------------------------------------------------------- 1 | """ 2 | The `Benchmark` action. 3 | """ 4 | 5 | import time 6 | from pathlib import Path 7 | 8 | from pydantic import BaseModel 9 | 10 | from blight.action import Action 11 | from blight.tool import Tool 12 | from blight.util import flock_append 13 | 14 | 15 | class BenchmarkRecord(BaseModel): 16 | """ 17 | Represents a single `Benchmark` record. Each record contains a representation 18 | of the tool invocation, the elapsed time between the `before_run` 19 | and `after_run` handlers (in milliseconds), and a flag indicating whether 20 | the underlying tool was actually run. 21 | """ 22 | 23 | tool: Tool 24 | """ 25 | The `Tool` invocation. 26 | """ 27 | 28 | elapsed: int 29 | """ 30 | The invocation's runtime, in milliseconds. 31 | """ 32 | 33 | run_skipped: bool 34 | """ 35 | Whether or not the tool was actually run. 36 | """ 37 | 38 | class Config: 39 | arbitrary_types_allowed = True 40 | json_encoders = {Tool: lambda t: t.asdict()} 41 | 42 | 43 | class Benchmark(Action): 44 | def before_run(self, tool: Tool) -> None: 45 | self._start_nanos = time.monotonic_ns() 46 | 47 | def after_run(self, tool: Tool, *, run_skipped: bool = False) -> None: 48 | elapsed = (time.monotonic_ns() - self._start_nanos) // 1000 49 | bench = BenchmarkRecord(tool=tool, elapsed=elapsed, run_skipped=run_skipped) 50 | 51 | if tool.is_journaling(): 52 | self._result = bench.dict() 53 | else: 54 | bench_file = Path(self._config["output"]) 55 | with flock_append(bench_file) as io: 56 | print(bench.json(), file=io) 57 | -------------------------------------------------------------------------------- /src/blight/actions/cc_for_cxx.py: -------------------------------------------------------------------------------- 1 | """ 2 | The `CCForCXX` action. 3 | """ 4 | 5 | from blight.action import CCAction 6 | from blight.tool import CC 7 | 8 | 9 | class CCForCXX(CCAction): 10 | """ 11 | An action for detecting whether the C compiler is being used as if it's 12 | a C++ compiler, and correcting the build when so. 13 | 14 | This action is used to fix a particular kind of misconfigured C++ build, 15 | where the C++ compiler is referred to as if it were a C compiler. 16 | 17 | For example, in Make: 18 | 19 | ```make 20 | CC := clang++ 21 | CFLAGS := -std=c++17 22 | 23 | all: 24 | $(CC) $(CFLAGS) -o whatever foo.cpp bar.cpp 25 | ``` 26 | 27 | Whereas the correct use would be: 28 | 29 | ```make 30 | CXX := clang++ 31 | CXXFLAGS := -std=c++17 32 | 33 | all: 34 | $(CXX) $(CXXFLAGS) -o whatever foo.cpp bar.cpp 35 | ``` 36 | 37 | This action fixes these builds by checking whether `CC` is being used 38 | as a C++ compiler. If it is, it explicitly injects additional flags 39 | to force the compiler into C++ mode. 40 | """ 41 | 42 | # NOTE(ww): type ignore here because mypy thinks this is a Liskov 43 | # substitution principle violation -- it can't see that `CompilerAction` 44 | # is safely specialized for `CompilerTool`. 45 | def before_run(self, tool: CC) -> None: # type: ignore 46 | # NOTE(ww): Currently, the only way we check whether CC is being used 47 | # as a C++ compiler is by checking whether one of the `-std=c++XX` 48 | # flags has been passed. This won't catch all cases; someone could use 49 | # CC as a C++ compiler with the default C++ standard. 50 | # Other options for detecting this: 51 | # * Check for common C++-only linkages, like -lstdc++fs 52 | # * Check whether tool.inputs contains files that look like C++ 53 | if tool.std.is_cxxstd(): 54 | tool.args[:0] = ["-x", "c++"] 55 | -------------------------------------------------------------------------------- /src/blight/actions/demo.py: -------------------------------------------------------------------------------- 1 | """ 2 | The `Demo` action. 3 | """ 4 | 5 | import sys 6 | 7 | from blight.action import Action 8 | from blight.tool import Tool 9 | 10 | 11 | class Demo(Action): 12 | def before_run(self, tool: Tool) -> None: 13 | print(f"[demo] before-run: {tool.wrapped_tool()}", file=sys.stderr) 14 | 15 | def after_run(self, tool: Tool, *, run_skipped: bool = False) -> None: 16 | print(f"[demo] after-run: {tool.wrapped_tool()}", file=sys.stderr) 17 | -------------------------------------------------------------------------------- /src/blight/actions/embed_bitcode.py: -------------------------------------------------------------------------------- 1 | """ 2 | The `EmbedBitcode` action. 3 | """ 4 | 5 | import logging 6 | 7 | from blight.action import CompilerAction 8 | from blight.tool import CompilerTool 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class EmbedBitcode(CompilerAction): 14 | """ 15 | An action to embed bitcode in compiler tool outputs. 16 | 17 | This action assumes that the compiler toolchain is LLVM based, and supports 18 | the `-fembed-bitcode` option. It injects `-fembed-bitcode` into each invocation, 19 | and lets the compiler tools take care of the rest. 20 | 21 | Example: 22 | 23 | ```bash 24 | export BLIGHT_ACTIONS="EmbedBitcode" 25 | make CC=blight-cc 26 | ``` 27 | """ 28 | 29 | def before_run(self, tool: CompilerTool) -> None: # type: ignore 30 | # TODO(ww): It probably makes sense to sanity check the arguments here, 31 | # just in case the build is being run with some other flags that are 32 | # relevant to bitcode generation (e.g. `-emit-llvm` or `-flto`). 33 | tool.args = ["-fembed-bitcode", *tool.args] 34 | -------------------------------------------------------------------------------- /src/blight/actions/find_inputs.py: -------------------------------------------------------------------------------- 1 | """ 2 | The `FindInputs` action. 3 | """ 4 | 5 | from pathlib import Path 6 | from typing import List, Optional 7 | 8 | from pydantic import BaseModel 9 | 10 | from blight.action import Action 11 | from blight.constants import INPUT_SUFFIX_KIND_MAP 12 | from blight.enums import InputKind 13 | from blight.tool import Tool 14 | from blight.util import flock_append 15 | 16 | 17 | class Input(BaseModel): 18 | """ 19 | Represents a single input to a tool. 20 | """ 21 | 22 | kind: InputKind 23 | """ 24 | The kind of input. 25 | """ 26 | 27 | prenormalized_path: str 28 | """ 29 | The path to the input, as passed directly to the tool itself. 30 | 31 | This copy of the path may be relative; consumers should prefer 32 | `path` for an absolute copy. 33 | """ 34 | 35 | path: Path 36 | """ 37 | The path to the input, as created by the tool. 38 | 39 | **NOTE**: This path may not actually exist, as a build system may arbitrarily 40 | choose to rename or relocate any inputs consumed by individual tools. 41 | """ 42 | 43 | store_path: Optional[Path] 44 | """ 45 | An optional stable path to the input, as preserved by the `FindInputs` action. 46 | 47 | `store_path` is only present if the `store=/some/dir/` setting is passed in the 48 | `FindInputs` configuration **and** the tool actually consumes the expected input. 49 | """ 50 | 51 | content_hash: Optional[str] 52 | """ 53 | A SHA256 hash of the input's content. 54 | 55 | `content_hash` is only present if the `store=/some/dir/` setting is passed in the 56 | `FindInputs` configuration **and** the tool actually consumes the expected input. 57 | """ 58 | 59 | 60 | class InputsRecord(BaseModel): 61 | """ 62 | Represents a single `FindInputs` record. Each record contains a representation 63 | of the tool invocation that consumes the inputs, as well as the list of `Input`s 64 | associated with the invocation. 65 | """ 66 | 67 | tool: Tool 68 | """ 69 | The `Tool` invocation. 70 | """ 71 | 72 | inputs: List[Input] 73 | """ 74 | A list of `Input`s. 75 | """ 76 | 77 | class Config: 78 | arbitrary_types_allowed = True 79 | json_encoders = {Tool: lambda t: t.asdict()} 80 | 81 | 82 | class FindInputs(Action): 83 | def before_run(self, tool: Tool) -> None: 84 | inputs = [] 85 | for input in tool.inputs: 86 | input_path = Path(input) 87 | if not input_path.is_absolute(): 88 | input_path = tool.cwd / input_path 89 | 90 | kind = INPUT_SUFFIX_KIND_MAP.get(input_path.suffix, InputKind.Unknown) 91 | 92 | inputs.append(Input(prenormalized_path=input, kind=kind, path=input_path)) 93 | 94 | self._inputs = inputs 95 | 96 | def after_run(self, tool: Tool, *, run_skipped: bool = False) -> None: 97 | inputs = InputsRecord(tool=tool, inputs=self._inputs) 98 | 99 | if tool.is_journaling(): 100 | # NOTE(ms): The `tool` member is excluded to avoid journal bloat. 101 | self._result = inputs.dict(exclude={"tool"}) 102 | else: 103 | output_path = Path(self._config["output"]) 104 | with flock_append(output_path) as io: 105 | print(inputs.json(), file=io) 106 | -------------------------------------------------------------------------------- /src/blight/actions/find_outputs.py: -------------------------------------------------------------------------------- 1 | """ 2 | The `FindOutputs` action. 3 | """ 4 | 5 | import hashlib 6 | import logging 7 | import re 8 | import shutil 9 | from pathlib import Path 10 | from typing import List, Optional 11 | 12 | from pydantic import BaseModel 13 | 14 | from blight.action import Action 15 | from blight.constants import OUTPUT_SUFFIX_KIND_MAP, OUTPUT_SUFFIX_PATTERN_MAP 16 | from blight.enums import OutputKind 17 | from blight.tool import CC, CXX, INSTALL, LD, Tool 18 | from blight.util import flock_append 19 | 20 | logger = logging.getLogger(__name__) 21 | 22 | 23 | class Output(BaseModel): 24 | """ 25 | Represents a single output from a tool. 26 | """ 27 | 28 | kind: OutputKind 29 | """ 30 | The kind of output. 31 | """ 32 | 33 | prenormalized_path: str 34 | """ 35 | The path to the output, as passed directly to the tool itself. 36 | 37 | This copy of the path may be relative; consumers should prefer 38 | `path` for an absolute copy. 39 | """ 40 | 41 | path: Path 42 | """ 43 | The path to the output, as created by the tool. 44 | 45 | **NOTE**: This path may not actually exist, as a build system may arbitrarily 46 | choose to rename or relocate any outputs produced by individual tools. 47 | """ 48 | 49 | store_path: Optional[Path] 50 | """ 51 | An optional stable path to the output, as preserved by the `FindOutputs` action. 52 | 53 | `store_path` is only present if the `store=/some/dir/` setting is passed in the 54 | `FindOutputs` configuration **and** the tool actually produces the expected output. 55 | """ 56 | 57 | content_hash: Optional[str] 58 | """ 59 | A SHA256 hash of the output's content. 60 | 61 | `content_hash` is only present if the `store=/some/dir/` setting is passed in the 62 | `FindOuputs` configuration **and** the tool actually produces the expected output. 63 | """ 64 | 65 | 66 | class OutputsRecord(BaseModel): 67 | """ 68 | Represents a single `FindOuputs` record. Each record contains a representation 69 | of the tool invocation that produced the outputs, as well as the list of `Output`s 70 | associated with the invocation. 71 | """ 72 | 73 | tool: Tool 74 | """ 75 | The `Tool` invocation. 76 | """ 77 | 78 | outputs: List[Output] 79 | """ 80 | A list of `Output`s. 81 | """ 82 | 83 | class Config: 84 | arbitrary_types_allowed = True 85 | json_encoders = {Tool: lambda t: t.asdict()} 86 | 87 | 88 | class FindOutputs(Action): 89 | def before_run(self, tool: Tool) -> None: 90 | outputs = [] 91 | for output in tool.outputs: 92 | output_path = Path(output) 93 | if not output_path.is_absolute(): 94 | output_path = tool.cwd / output_path 95 | 96 | # Special cases: a.out is produced by both the linker and compiler tools by default, 97 | # and some tools (like `install`) have modes that produce directories as outputs. 98 | if output_path.name == "a.out" and isinstance(tool, (CC, CXX, LD)): 99 | kind = OutputKind.Executable 100 | elif tool.__class__ == INSTALL and tool.directory_mode: # type: ignore 101 | kind = OutputKind.Directory 102 | else: 103 | kind = OUTPUT_SUFFIX_KIND_MAP.get(output_path.suffix, OutputKind.Unknown) 104 | 105 | # Last attempt: try some common patterns for output kinds if we can't 106 | # match the suffix precisely. 107 | if kind == OutputKind.Unknown: 108 | kind = next( 109 | ( 110 | k 111 | for (p, k) in OUTPUT_SUFFIX_PATTERN_MAP.items() 112 | if re.match(p, str(output_path)) 113 | ), 114 | OutputKind.Unknown, 115 | ) 116 | 117 | outputs.append(Output(prenormalized_path=output, kind=kind, path=output_path)) 118 | 119 | self._outputs = outputs 120 | 121 | def after_run(self, tool: Tool, *, run_skipped: bool = False) -> None: 122 | store = self._config.get("store") 123 | if store is not None: 124 | store_path = Path(store) 125 | store_path.mkdir(parents=True, exist_ok=True) 126 | 127 | for output in self._outputs: 128 | # We don't copy output directories into the store, for now. 129 | if output.path.is_dir(): 130 | continue 131 | 132 | if not output.path.exists(): 133 | logger.warning(f"tool={tool}'s output ({output.path}) does not exist") 134 | continue 135 | 136 | # Outputs aren't guaranteed to have unique basenames and subsequent 137 | # steps in the build system could even modify a particular output 138 | # in-place, so we give each output a `store_path` based on a hash 139 | # of its content. 140 | content_hash = hashlib.sha256(output.path.read_bytes()).hexdigest() 141 | # Append hash to the filename unless `append_hash=false` is specified in the config 142 | append_hash = self._config.get("append_hash") != "false" 143 | filename = f"{output.path.name}-{content_hash}" if append_hash else output.path.name 144 | output_store_path = store_path / filename 145 | if not output_store_path.exists(): 146 | shutil.copy(output.path, output_store_path) 147 | output.store_path = output_store_path 148 | output.content_hash = content_hash 149 | 150 | outputs = OutputsRecord(tool=tool, outputs=self._outputs) 151 | 152 | if tool.is_journaling(): 153 | # NOTE(ms): The `tool` member is excluded to avoid journal bloat. 154 | self._result = outputs.dict(exclude={"tool"}) 155 | else: 156 | output_path = Path(self._config["output"]) 157 | with flock_append(output_path) as io: 158 | print(outputs.json(), file=io) 159 | -------------------------------------------------------------------------------- /src/blight/actions/ignore_flags.py: -------------------------------------------------------------------------------- 1 | """ 2 | The `IgnoreFlags` action. 3 | """ 4 | 5 | import logging 6 | import shlex 7 | 8 | from blight.action import CompilerAction 9 | from blight.enums import Lang 10 | from blight.tool import CompilerTool 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | class IgnoreFlags(CompilerAction): 16 | """ 17 | An action for ignoring specific flags passed to compiler 18 | commands (specifically, `cc` and `c++`). 19 | 20 | For example: 21 | 22 | ```bash 23 | export BLIGHT_WRAPPED_CC=clang 24 | export BLIGHT_ACTIONS="IgnoreFlags" 25 | export BLIGHT_ACTIONS_IGNOREFLAGS="FLAGS='-Werror -ffunction-sections'" 26 | make CC=blight-cc 27 | ``` 28 | 29 | will cause blight to remove `-Werror` and `--ffunction-sections` arguments 30 | from each `clang` invocation. 31 | """ 32 | 33 | # NOTE(ww): type ignore here because mypy thinks this is a Liskov 34 | # substitution principle violation -- it can't see that `CompilerAction` 35 | # is safely specialized for `CompilerTool`. 36 | def before_run(self, tool: CompilerTool) -> None: # type: ignore 37 | ignore_flags = shlex.split(self._config.get("FLAGS", "")) 38 | if tool.lang in [Lang.C, Lang.Cxx]: 39 | tool.args = [a for a in tool.args if a not in ignore_flags] 40 | else: 41 | logger.debug("not ignoring flags for an unknown language") 42 | -------------------------------------------------------------------------------- /src/blight/actions/ignore_flto.py: -------------------------------------------------------------------------------- 1 | """ 2 | The `IgnoreFlto` action. 3 | """ 4 | 5 | import logging 6 | 7 | from blight.action import CompilerAction 8 | from blight.tool import CompilerTool 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class IgnoreFlto(CompilerAction): 14 | """ 15 | An action for ignoring the `-flto` flag passed to compiler 16 | commands (specifically, `cc` and `c++`). Related commands that 17 | control LTO (`-flto=...`) are also ignored. 18 | 19 | For example: 20 | 21 | ```bash 22 | export BLIGHT_WRAPPED_CC=clang 23 | export BLIGHT_ACTIONS="IgnoreFlto" 24 | make CC=blight-cc 25 | ``` 26 | 27 | will cause blight to remove `-flto` arguments from each `clang` 28 | invocation. 29 | """ 30 | 31 | # NOTE(ww): type ignore here because mypy thinks this is a Liskov 32 | # substitution principle violation -- it can't see that `CompilerAction` 33 | # is safely specialized for `CompilerTool`. 34 | def before_run(self, tool: CompilerTool) -> None: # type: ignore 35 | tool.args = [a for a in tool.args if not a.startswith("-flto")] 36 | -------------------------------------------------------------------------------- /src/blight/actions/ignore_werror.py: -------------------------------------------------------------------------------- 1 | """ 2 | The `IgnoreWerror` action. 3 | """ 4 | 5 | import logging 6 | 7 | from blight.action import CompilerAction 8 | from blight.enums import Lang 9 | from blight.tool import CompilerTool 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | class IgnoreWerror(CompilerAction): 15 | """ 16 | An action for ignoring the `-Werror` flag passed to compiler 17 | commands (specifically, `cc` and `c++`). 18 | 19 | For example: 20 | 21 | ```bash 22 | export BLIGHT_WRAPPED_CC=clang 23 | export BLIGHT_ACTIONS="IgnoreWerror" 24 | make CC=blight-cc 25 | ``` 26 | 27 | will cause blight to remove `-Werror` arguments from each `clang` 28 | invocation. 29 | """ 30 | 31 | # NOTE(ww): type ignore here because mypy thinks this is a Liskov 32 | # substitution principle violation -- it can't see that `CompilerAction` 33 | # is safely specialized for `CompilerTool`. 34 | def before_run(self, tool: CompilerTool) -> None: # type: ignore 35 | if tool.lang in [Lang.C, Lang.Cxx]: 36 | tool.args = [a for a in tool.args if a != "-Werror"] 37 | else: 38 | logger.debug("not injecting flags for an unknown language") 39 | -------------------------------------------------------------------------------- /src/blight/actions/inject_flags.py: -------------------------------------------------------------------------------- 1 | """ 2 | The `InjectFlags` action. 3 | """ 4 | 5 | import logging 6 | import shlex 7 | 8 | from blight.action import CompilerAction 9 | from blight.enums import CompilerStage, Lang 10 | from blight.tool import CompilerTool 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | class InjectFlags(CompilerAction): 16 | """ 17 | An action for injecting flags into compiler commands (specifically, `cc` and `c++`). 18 | 19 | This action takes the following flags in its configuration and appends them 20 | (after shell-splitting) as appropriate: 21 | - `CFLAGS`: Flags to append to every C compiler call 22 | - `CFLAGS_LINKER`: Flags to append to every C compiler call that runs the linking 23 | stage (i.e: no `-c`, `-e`, `-S`, etc. flags present) 24 | - `CXXFLAGS`: Same as `CFLAGS` but for C++ 25 | - `CFLAGS_LINKER`: Same as `CFLAGS_LINKER`, but for C++ 26 | - `CPPFLAGS`: Flags to append for the preprocessor stage 27 | 28 | For example: 29 | 30 | ```bash 31 | export BLIGHT_WRAPPED_CC=clang 32 | export BLIGHT_ACTIONS="InjectFlags" 33 | export BLIGHT_ACTION_INJECTFLAGS="CFLAGS='-g -O0' CPPFLAGS='-DWHATEVER'" 34 | make CC=blight-cc 35 | ``` 36 | 37 | will cause blight to add `-g -O0 -DWHATEVER` to each `clang` invocation 38 | (unless it's a C++ invocation, e.g. via `-x c++`). 39 | """ 40 | 41 | # NOTE(ww): type ignore here because mypy thinks this is a Liskov 42 | # substitution principle violation -- it can't see that `CompilerAction` 43 | # is safely specialized for `CompilerTool`. 44 | def before_run(self, tool: CompilerTool) -> None: # type: ignore 45 | cflags = shlex.split(self._config.get("CFLAGS", "")) 46 | cflags_linker = shlex.split(self._config.get("CFLAGS_LINKER", "")) 47 | cxxflags = shlex.split(self._config.get("CXXFLAGS", "")) 48 | cxxflags_linker = shlex.split(self._config.get("CXXFLAGS_LINKER", "")) 49 | cppflags = shlex.split(self._config.get("CPPFLAGS", "")) 50 | 51 | if tool.lang == Lang.C: 52 | tool.args += cflags 53 | tool.args += cppflags 54 | if tool.stage is CompilerStage.AllStages: 55 | tool.args += cflags_linker 56 | elif tool.lang == Lang.Cxx: 57 | tool.args += cxxflags 58 | tool.args += cppflags 59 | if tool.stage is CompilerStage.AllStages: 60 | tool.args += cxxflags_linker 61 | else: 62 | logger.debug("not injecting flags for an unknown language") 63 | -------------------------------------------------------------------------------- /src/blight/actions/lint.py: -------------------------------------------------------------------------------- 1 | """ 2 | The `Lint` action. 3 | """ 4 | 5 | import logging 6 | 7 | from blight.action import CompilerAction 8 | from blight.tool import CompilerTool 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class Lint(CompilerAction): 14 | """ 15 | A "linting" action for common command-line mistakes. 16 | 17 | At the moment, this catches mistakes like: 18 | 19 | * `-DFORTIFY_SOURCE=...` instead of `-D_FORTIFY_SOURCE=...` 20 | 21 | For example: 22 | 23 | ```bash 24 | export BLIGHT_WRAPPED_CC=clang 25 | export BLIGHT_ACTIONS="Lint" 26 | make CC=blight-cc 27 | ``` 28 | """ 29 | 30 | # NOTE(ww): type ignore here because mypy thinks this is a Liskov 31 | # substitution principle violation -- it can't see that `CompilerAction` 32 | # is safely specialized for `CompilerTool`. 33 | def before_run(self, tool: CompilerTool) -> None: # type: ignore 34 | for name, _ in tool.defines: 35 | # TODO: Maybe do something more drastic here, like stopping the run. 36 | if name == "FORTIFY_SOURCE": 37 | logger.warning("found -DFORTIFY_SOURCE; you probably meant: -D_FORTIFY_SOURCE") 38 | -------------------------------------------------------------------------------- /src/blight/actions/record.py: -------------------------------------------------------------------------------- 1 | """ 2 | The `Record` action. 3 | 4 | `Record` produces a transcript of each tool's invocation. It supports 5 | both journaling and a custom output path. 6 | 7 | Configuration: 8 | 9 | * `output`: a path on disk that each `Record` step's output will be written to 10 | 11 | """ 12 | 13 | import json 14 | from pathlib import Path 15 | 16 | from blight.action import Action 17 | from blight.tool import Tool 18 | from blight.util import flock_append 19 | 20 | 21 | class Record(Action): 22 | def after_run(self, tool: Tool, *, run_skipped: bool = False) -> None: 23 | # TODO(ww): Restructure this dictionary; it should be more like: 24 | # { run: {...}, tool: {...}} 25 | tool_record = tool.asdict() 26 | tool_record["run_skipped"] = run_skipped 27 | 28 | if tool.is_journaling(): 29 | self._result = tool_record 30 | else: 31 | record_file = Path(self._config["output"]) 32 | with flock_append(record_file) as io: 33 | print(json.dumps(tool_record), file=io) 34 | -------------------------------------------------------------------------------- /src/blight/actions/skip_strip.py: -------------------------------------------------------------------------------- 1 | """ 2 | The `SkipStrip` action. 3 | 4 | All `SkipStrip` does is skip invocations of `strip`. No other commands 5 | are affected. 6 | """ 7 | 8 | 9 | from blight.action import STRIPAction 10 | from blight.exceptions import SkipRun 11 | from blight.tool import STRIP 12 | 13 | 14 | class SkipStrip(STRIPAction): 15 | # NOTE(ww): type ignore here because mypy thinks this is a Liskov 16 | # substitution principle violation -- it can't see that `CompilerAction` 17 | # is safely specialized for `CompilerTool`. 18 | def before_run(self, tool: STRIP) -> None: # type: ignore 19 | raise SkipRun 20 | -------------------------------------------------------------------------------- /src/blight/constants.py: -------------------------------------------------------------------------------- 1 | """ 2 | Constant tables and maps for various blight APIs and actions. 3 | """ 4 | 5 | from blight.enums import InputKind, OutputKind 6 | 7 | COMPILER_FLAG_INJECTION_VARIABLES = {"CL", "_CL_", "CCC_OVERRIDE_OPTIONS"} 8 | """ 9 | Environment variables that some compiler frontends use to do their own 10 | flag injection. 11 | """ 12 | 13 | OUTPUT_SUFFIX_KIND_MAP = { 14 | ".o": OutputKind.Object, 15 | ".obj": OutputKind.Object, 16 | ".so": OutputKind.SharedLibrary, 17 | ".dylib": OutputKind.SharedLibrary, 18 | ".dll": OutputKind.SharedLibrary, 19 | ".a": OutputKind.StaticLibrary, 20 | ".lib": OutputKind.StaticLibrary, 21 | "": OutputKind.Executable, 22 | ".exe": OutputKind.Executable, 23 | ".bin": OutputKind.Executable, 24 | ".elf": OutputKind.Executable, 25 | ".com": OutputKind.Executable, 26 | ".ko": OutputKind.KernelModule, 27 | ".sys": OutputKind.KernelModule, 28 | } 29 | """ 30 | A mapping of common output suffixes to their (expected) file kinds. 31 | 32 | This mapping is not exhaustive. 33 | """ 34 | 35 | 36 | OUTPUT_SUFFIX_PATTERN_MAP = { 37 | r".*\.so\.\d+\.\d+\.\d+$": OutputKind.SharedLibrary, # anything with libtool 38 | r".*\.so\.\d+\.\d+$": OutputKind.SharedLibrary, # libssl.so.1.1 39 | r".*\.so\.\d+$": OutputKind.SharedLibrary, # libc.so.6 40 | } 41 | """ 42 | A mapping of common output suffix patterns to their (expected) file kinds. 43 | 44 | This mapping is not exhaustive. 45 | """ 46 | 47 | INPUT_SUFFIX_KIND_MAP = { 48 | ".c": InputKind.Source, 49 | ".cc": InputKind.Source, 50 | ".cpp": InputKind.Source, 51 | ".cxx": InputKind.Source, 52 | ".o": InputKind.Object, 53 | ".obj": InputKind.Object, 54 | ".so": InputKind.SharedLibrary, 55 | ".dylib": InputKind.SharedLibrary, 56 | ".dll": InputKind.SharedLibrary, 57 | ".a": InputKind.StaticLibrary, 58 | ".lib": InputKind.StaticLibrary, 59 | } 60 | """ 61 | A mapping of common input suffixes to their (expected) file kinds. 62 | 63 | This mapping is not exhaustive. 64 | """ 65 | -------------------------------------------------------------------------------- /src/blight/enums.py: -------------------------------------------------------------------------------- 1 | """ 2 | Enumerations for blight. 3 | """ 4 | 5 | from __future__ import annotations 6 | 7 | import enum 8 | 9 | from blight.util import assert_never 10 | 11 | 12 | @enum.unique 13 | class BuildTool(str, enum.Enum): 14 | """ 15 | An enumeration of standard build tools. 16 | """ 17 | 18 | CC: str = "CC" 19 | CXX: str = "CXX" 20 | CPP: str = "CPP" 21 | LD: str = "LD" 22 | AS: str = "AS" 23 | AR: str = "AR" 24 | STRIP: str = "STRIP" 25 | INSTALL: str = "INSTALL" 26 | 27 | @property 28 | def cmd(self) -> str: 29 | """ 30 | Returns the standard command associated with this tool. 31 | """ 32 | if self is BuildTool.CC: 33 | return "cc" 34 | elif self is BuildTool.CXX: 35 | return "c++" 36 | elif self is BuildTool.CPP: 37 | return "cpp" 38 | elif self is BuildTool.LD: 39 | return "ld" 40 | elif self is BuildTool.AS: 41 | return "as" 42 | elif self is BuildTool.AR: 43 | return "ar" 44 | elif self is BuildTool.STRIP: 45 | return "strip" 46 | elif self is BuildTool.INSTALL: 47 | return "install" 48 | else: 49 | assert_never(self) # pragma: no cover 50 | 51 | @property 52 | def env(self) -> str: 53 | return self.value 54 | 55 | @property 56 | def blight_tool(self) -> BlightTool: 57 | if self is BuildTool.CC: 58 | return BlightTool.CC 59 | elif self is BuildTool.CXX: 60 | return BlightTool.CXX 61 | elif self is BuildTool.CPP: 62 | return BlightTool.CPP 63 | elif self is BuildTool.LD: 64 | return BlightTool.LD 65 | elif self is BuildTool.AS: 66 | return BlightTool.AS 67 | elif self is BuildTool.AR: 68 | return BlightTool.AR 69 | elif self is BuildTool.STRIP: 70 | return BlightTool.STRIP 71 | elif self is BuildTool.INSTALL: 72 | return BlightTool.INSTALL 73 | else: 74 | assert_never(self) # pragma: no cover 75 | 76 | def __str__(self) -> str: 77 | return self.value 78 | 79 | 80 | @enum.unique 81 | class BlightTool(str, enum.Enum): 82 | """ 83 | An enumeration of blight wrappers for the standard build tools. 84 | """ 85 | 86 | CC: str = "blight-cc" 87 | CXX: str = "blight-c++" 88 | CPP: str = "blight-cpp" 89 | LD: str = "blight-ld" 90 | AS: str = "blight-as" 91 | AR: str = "blight-ar" 92 | STRIP: str = "blight-strip" 93 | INSTALL: str = "blight-install" 94 | 95 | @property 96 | def build_tool(self) -> BuildTool: 97 | """ 98 | Returns the `BuildTool` corresponding to this blight wrapper tool. 99 | """ 100 | if self is BlightTool.CC: 101 | return BuildTool.CC 102 | elif self is BlightTool.CXX: 103 | return BuildTool.CXX 104 | elif self is BlightTool.CPP: 105 | return BuildTool.CPP 106 | elif self is BlightTool.LD: 107 | return BuildTool.LD 108 | elif self is BlightTool.AS: 109 | return BuildTool.AS 110 | elif self is BlightTool.AR: 111 | return BuildTool.AR 112 | elif self is BlightTool.STRIP: 113 | return BuildTool.STRIP 114 | elif self is BlightTool.INSTALL: 115 | return BuildTool.INSTALL 116 | else: 117 | assert_never(self) # pragma: no cover 118 | 119 | @property 120 | def env(self) -> str: 121 | return f"BLIGHT_WRAPPED_{self.build_tool.env}" 122 | 123 | def __str__(self) -> str: 124 | return self.value 125 | 126 | 127 | @enum.unique 128 | class CompilerFamily(enum.Enum): 129 | """ 130 | Models known compiler families (e.g. GCC, Clang, etc.) 131 | """ 132 | 133 | Gcc = enum.auto() 134 | """ 135 | The GCC family of compilers. 136 | """ 137 | 138 | MainlineLlvm = enum.auto() 139 | """ 140 | The "mainline" LLVM family, corresponding to upstream releases of LLVM. 141 | """ 142 | 143 | AppleLlvm = enum.auto() 144 | """ 145 | The "Apple" LLVM family, corresponding to Apple's builds of LLVM. 146 | """ 147 | 148 | Tcc = enum.auto() 149 | """ 150 | The Tiny C Compiler. 151 | """ 152 | 153 | Unknown = enum.auto() 154 | """ 155 | An unknown compiler family. 156 | """ 157 | 158 | def __str__(self) -> str: 159 | return self.name 160 | 161 | 162 | @enum.unique 163 | class CompilerStage(enum.Enum): 164 | """ 165 | Models the known stages that a compiler tool can be in. 166 | """ 167 | 168 | # TODO(ww): Maybe handle -v, -###, --help, --help=..., etc. 169 | 170 | Preprocess = enum.auto() 171 | """ 172 | Preprocess only (e.g., `cc -E`) 173 | """ 174 | 175 | SyntaxOnly = enum.auto() 176 | """ 177 | Preprocess, parse, and typecheck only (e.g., `cc -fsyntax-only`) 178 | """ 179 | 180 | Assemble = enum.auto() 181 | """ 182 | Compile to assembly but don't run the assembler (e.g., `cc -S`) 183 | """ 184 | 185 | CompileObject = enum.auto() 186 | """ 187 | Compile and assemble to an individual object (e.g. `cc -c`) 188 | """ 189 | 190 | AllStages = enum.auto() 191 | """ 192 | All stages, including the linker (e.g. `cc`) 193 | """ 194 | 195 | Unknown = enum.auto() 196 | """ 197 | An unknown or unqualified stage. 198 | """ 199 | 200 | def __str__(self) -> str: 201 | return self.name 202 | 203 | 204 | @enum.unique 205 | class Lang(enum.Enum): 206 | """ 207 | Models the known languages for a tool. 208 | """ 209 | 210 | # TODO(ww): Maybe add each of the following: 211 | # * Asm (assembly) 212 | # * PreprocessedC (C that's already gone through the preprocessor) 213 | # * PreprocessedCxx (C++ that's already gone through the preprocessor) 214 | 215 | C = enum.auto() 216 | """ 217 | The C programming language. 218 | """ 219 | 220 | Cxx = enum.auto() 221 | """ 222 | The C++ programming language. 223 | """ 224 | 225 | Unknown = enum.auto() 226 | """ 227 | An unknown language. 228 | """ 229 | 230 | def __str__(self) -> str: 231 | return self.name 232 | 233 | 234 | @enum.unique 235 | class Std(enum.Enum): 236 | """ 237 | Models the various language standards for a tool. 238 | """ 239 | 240 | def is_unknown(self) -> bool: 241 | """ 242 | Returns: 243 | `True` if the standard is unknown 244 | """ 245 | return self in [Std.CUnknown, Std.CxxUnknown, Std.GnuUnknown, Std.GnuxxUnknown, Std.Unknown] 246 | 247 | def is_cstd(self) -> bool: 248 | """ 249 | Returns: 250 | `True` if the standard is a C standard 251 | """ 252 | return self in [ 253 | Std.C89, 254 | Std.C94, 255 | Std.C99, 256 | Std.C11, 257 | Std.C17, 258 | Std.C2x, 259 | Std.Gnu89, 260 | Std.Gnu99, 261 | Std.Gnu11, 262 | Std.Gnu17, 263 | Std.Gnu2x, 264 | Std.CUnknown, 265 | Std.GnuUnknown, 266 | ] 267 | 268 | def is_cxxstd(self) -> bool: 269 | """ 270 | Returns: 271 | `True` if the standard is a C++ standard 272 | """ 273 | return self in [ 274 | Std.Cxx03, 275 | Std.Cxx11, 276 | Std.Cxx14, 277 | Std.Cxx17, 278 | Std.Cxx2a, 279 | Std.Gnuxx03, 280 | Std.Gnuxx11, 281 | Std.Gnuxx14, 282 | Std.Gnuxx17, 283 | Std.Gnuxx2a, 284 | Std.CxxUnknown, 285 | Std.GnuxxUnknown, 286 | ] 287 | 288 | def lang(self) -> Lang: 289 | """ 290 | Returns: a `Lang` corresponding to this `Std`. 291 | """ 292 | if self.is_cstd(): 293 | return Lang.C 294 | elif self.is_cxxstd(): 295 | return Lang.Cxx 296 | else: 297 | return Lang.Unknown 298 | 299 | C89 = enum.auto() 300 | """ 301 | C89 (a.k.a. C90, iso9899:1990) 302 | """ 303 | 304 | C94 = enum.auto() 305 | """ 306 | C94 (a.k.a. iso9899:199409) 307 | """ 308 | 309 | C99 = enum.auto() 310 | """ 311 | C99 (a.k.a. C9x, iso9899:1999, iso9899:199x) 312 | """ 313 | 314 | C11 = enum.auto() 315 | """ 316 | C11 (a.k.a. C1x, iso9899:2011) 317 | """ 318 | 319 | C17 = enum.auto() 320 | """ 321 | C17 (a.k.a. C18, iso9899:2017, iso9899:2018) 322 | """ 323 | 324 | C2x = enum.auto() 325 | """ 326 | C2x 327 | """ 328 | 329 | Gnu89 = enum.auto() 330 | """ 331 | GNU C89 (a.k.a. GNU C 90) 332 | """ 333 | 334 | Gnu99 = enum.auto() 335 | """ 336 | GNU C99 (a.k.a. GNU C 9x) 337 | """ 338 | 339 | Gnu11 = enum.auto() 340 | """ 341 | GNU C11 (a.k.a. GNU C11x) 342 | """ 343 | 344 | Gnu17 = enum.auto() 345 | """ 346 | GNU C17 (a.k.a. GNU C18) 347 | """ 348 | 349 | Gnu2x = enum.auto() 350 | """ 351 | GNU C2x 352 | """ 353 | 354 | Cxx03 = enum.auto() 355 | """ 356 | C++03 (a.k.a. C++98) 357 | """ 358 | 359 | Cxx11 = enum.auto() 360 | """ 361 | C++11 (a.k.a. C++0x) 362 | """ 363 | 364 | Cxx14 = enum.auto() 365 | """ 366 | C++14 (a.k.a. C++1y) 367 | """ 368 | 369 | Cxx17 = enum.auto() 370 | """ 371 | C++17 (a.k.a. C++1z) 372 | """ 373 | 374 | Cxx2a = enum.auto() 375 | """ 376 | C++2a (a.k.a. C++20) 377 | """ 378 | 379 | Gnuxx03 = enum.auto() 380 | """ 381 | GNU C++03 (a.k.a. GNU C++98) 382 | """ 383 | 384 | Gnuxx11 = enum.auto() 385 | """ 386 | GNU C++11 (a.k.a. GNU C++0x) 387 | """ 388 | 389 | Gnuxx14 = enum.auto() 390 | """ 391 | GNU C++14 (a.k.a. GNU C++1y) 392 | """ 393 | 394 | Gnuxx17 = enum.auto() 395 | """ 396 | GNU C++17 (a.k.a. GNU C++1z) 397 | """ 398 | 399 | Gnuxx2a = enum.auto() 400 | """ 401 | GNU C++2a (a.k.a. GNU C++20) 402 | """ 403 | 404 | CUnknown = enum.auto() 405 | """ 406 | Standard C, but an unknown version. 407 | """ 408 | 409 | CxxUnknown = enum.auto() 410 | """ 411 | Standard C++, but an unknown version. 412 | """ 413 | 414 | GnuUnknown = enum.auto() 415 | """ 416 | GNU C, but an unknown version. 417 | """ 418 | 419 | GnuxxUnknown = enum.auto() 420 | """ 421 | GNU C++, but an unknown version. 422 | """ 423 | 424 | Unknown = enum.auto() 425 | """ 426 | A completely unknown language standard. 427 | """ 428 | 429 | def __str__(self) -> str: 430 | return self.name 431 | 432 | 433 | @enum.unique 434 | class OptLevel(enum.Enum): 435 | """ 436 | Models the known optimization levels. 437 | """ 438 | 439 | def for_size(self) -> bool: 440 | """ 441 | Returns: 442 | `True` if the optimization is for compiled size 443 | """ 444 | return self == OptLevel.OSize or self == OptLevel.OSizeZ 445 | 446 | def for_performance(self) -> bool: 447 | """ 448 | Returns: 449 | `True` if the optimization is for performance 450 | """ 451 | return self in [OptLevel.O1, OptLevel.O2, OptLevel.O3, OptLevel.OFast] 452 | 453 | def for_debug(self) -> bool: 454 | """ 455 | Returns: 456 | `True` if the optimization is for debugging experience 457 | """ 458 | return self == OptLevel.ODebug 459 | 460 | O0 = enum.auto() 461 | """ 462 | No optimizations. 463 | """ 464 | 465 | O1 = enum.auto() 466 | """ 467 | Minimal performance optimizations. 468 | """ 469 | 470 | O2 = enum.auto() 471 | """ 472 | Some performance optimizations. 473 | """ 474 | 475 | O3 = enum.auto() 476 | """ 477 | Aggressive performance optimizations. 478 | """ 479 | 480 | OFast = enum.auto() 481 | """ 482 | Aggressive, possibly standards-breaking performance optimizations. 483 | """ 484 | 485 | OSize = enum.auto() 486 | """ 487 | Size optimizations. 488 | """ 489 | 490 | OSizeZ = enum.auto() 491 | """ 492 | More aggressive size optimizations (Clang only). 493 | """ 494 | 495 | ODebug = enum.auto() 496 | """ 497 | Debugging experience optimizations. 498 | """ 499 | 500 | Unknown = enum.auto() 501 | """ 502 | An unknown optimization level. 503 | """ 504 | 505 | def __str__(self) -> str: 506 | return self.name 507 | 508 | 509 | @enum.unique 510 | class CodeModel(enum.Enum): 511 | """ 512 | Models the known machine code models. 513 | """ 514 | 515 | Small = enum.auto() 516 | """ 517 | The `small` machine code model (also `medlow` in Clang). 518 | """ 519 | 520 | Medium = enum.auto() 521 | """ 522 | The `medium` machine code model (also `medany` in Clang). 523 | """ 524 | 525 | Large = enum.auto() 526 | """ 527 | The `large` machine code model. 528 | """ 529 | 530 | Kernel = enum.auto() 531 | """ 532 | The `kernel` machine code model. 533 | """ 534 | 535 | Unknown = enum.auto() 536 | """ 537 | An unknown machine code model. 538 | """ 539 | 540 | def __str__(self) -> str: 541 | return self.name 542 | 543 | 544 | @enum.unique 545 | class OutputKind(str, enum.Enum): 546 | """ 547 | A collection of common output kinds for build tools. 548 | 549 | This enumeration is not exhaustive. 550 | """ 551 | 552 | Object: str = "object" 553 | SharedLibrary: str = "shared" 554 | StaticLibrary: str = "static" 555 | Executable: str = "executable" 556 | KernelModule: str = "kernel" 557 | Directory: str = "directory" 558 | Unknown: str = "unknown" 559 | 560 | def __str__(self) -> str: 561 | return self.value 562 | 563 | 564 | @enum.unique 565 | class InputKind(str, enum.Enum): 566 | """ 567 | A collection of common input kinds for build tools. 568 | 569 | This enumeration is not exhaustive. 570 | """ 571 | 572 | Source: str = "source" 573 | Object: str = "object" 574 | SharedLibrary: str = "shared" 575 | StaticLibrary: str = "static" 576 | Directory: str = "directory" 577 | Unknown: str = "unknown" 578 | 579 | def __str__(self) -> str: 580 | return self.value 581 | -------------------------------------------------------------------------------- /src/blight/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exceptions for blight. 3 | """ 4 | 5 | 6 | class BlightError(ValueError): 7 | """ 8 | Raised whenever an internal condition isn't met. 9 | """ 10 | 11 | pass 12 | 13 | 14 | class BuildError(BlightError): 15 | """ 16 | Raised whenever a wrapped tool fails. 17 | """ 18 | 19 | pass 20 | 21 | 22 | class SkipRun(BlightError): # noqa: N818 23 | """ 24 | A special error that `before_run` actions can raise to tell the underlying 25 | tool not to actually run. 26 | """ 27 | 28 | pass 29 | -------------------------------------------------------------------------------- /src/blight/protocols.py: -------------------------------------------------------------------------------- 1 | """ 2 | Substructural typing protocols for blight. 3 | 4 | These are, generally speaking, an implementation detail. 5 | """ 6 | 7 | from pathlib import Path 8 | from typing import Dict, List, Protocol 9 | 10 | from blight.enums import Lang 11 | 12 | 13 | class CwdProtocol(Protocol): 14 | @property 15 | def cwd(self) -> Path: 16 | ... # pragma: no cover 17 | 18 | 19 | class ArgsProtocol(CwdProtocol, Protocol): 20 | @property 21 | def args(self) -> List[str]: 22 | ... # pragma: no cover 23 | 24 | 25 | class CanonicalizedArgsProtocol(ArgsProtocol, Protocol): 26 | @property 27 | def canonicalized_args(self) -> List[str]: 28 | ... # pragma: no cover 29 | 30 | 31 | class LangProtocol(CanonicalizedArgsProtocol, Protocol): 32 | @property 33 | def lang(self) -> Lang: 34 | ... # pragma: no cover 35 | 36 | 37 | class IndexedUndefinesProtocol(CanonicalizedArgsProtocol, Protocol): 38 | @property 39 | def indexed_undefines(self) -> Dict[str, int]: 40 | ... # pragma: no cover 41 | -------------------------------------------------------------------------------- /src/blight/tool.py: -------------------------------------------------------------------------------- 1 | """ 2 | Encapsulations of the tools supported by blight. 3 | """ 4 | 5 | import itertools 6 | import json 7 | import logging 8 | import os 9 | import re 10 | import shlex 11 | import subprocess 12 | from pathlib import Path 13 | from typing import Any, Dict, List, Optional, Tuple 14 | 15 | from blight import util 16 | from blight.constants import COMPILER_FLAG_INJECTION_VARIABLES 17 | from blight.enums import ( 18 | BlightTool, 19 | BuildTool, 20 | CodeModel, 21 | CompilerFamily, 22 | CompilerStage, 23 | Lang, 24 | OptLevel, 25 | Std, 26 | ) 27 | from blight.exceptions import BlightError, BuildError, SkipRun 28 | from blight.protocols import ( 29 | CanonicalizedArgsProtocol, 30 | IndexedUndefinesProtocol, 31 | LangProtocol, 32 | ) 33 | from blight.util import json_helper 34 | 35 | logger = logging.getLogger(__name__) 36 | 37 | 38 | RESPONSE_FILE_RECURSION_LIMIT = 64 39 | """ 40 | Response files can contain further `@file` arguments, because of course they can. 41 | 42 | Neither clang nor GCC is explicit in their documentation about their recursion limits, 43 | if they have any. We choose an arbitrary limit here. 44 | """ 45 | 46 | 47 | class Tool: 48 | """ 49 | Represents a generic tool wrapped by blight. 50 | 51 | Every `Tool` has two views of its supplied arguments: 52 | 53 | * An "effective" view, provided by `Tool.args` 54 | * A "canonicalized" view, provided by `Tool.canonicalized_args` 55 | 56 | The "effective" view is used to invoke the underlying wrapped tool. It should 57 | never differ from the original arguments supplied to the invocation, **except** 58 | for when a user configures an action that **intentionally** modifies the 59 | arguments. 60 | 61 | The "canonicalized" view is used to model the behavior of the underlying wrapped 62 | tool. Specific `Tool` subclasses may specialize the canonicalized view to improve 63 | modeling fidelity. For example, tools that support the `@file` syntax (see 64 | `ResponseFileMixin`) for expanding arguments may augment `canonicalized_args` 65 | to reflect a fully expanded and normalized version of the original arguments. 66 | 67 | The "canonicalized" view always derives directly from the "effective" view: 68 | any modifications made to the "effective" arguments by an action will be 69 | propagated to the "canonicalized" arguments. 70 | 71 | `Tool` instances cannot be created directory; a specific subclass must be used. 72 | """ 73 | 74 | @classmethod 75 | def build_tool(cls) -> BuildTool: 76 | """ 77 | Returns the `BuildTool` enum associated with this `Tool`. 78 | """ 79 | return BuildTool(cls.__name__) 80 | 81 | @classmethod 82 | def blight_tool(cls) -> BlightTool: 83 | """ 84 | Returns the `BlightTool` enum associated with this `Tool`. 85 | """ 86 | return cls.build_tool().blight_tool 87 | 88 | @classmethod 89 | def wrapped_tool(cls) -> str: 90 | """ 91 | Returns the executable name or path of the tool that this blight tool wraps. 92 | """ 93 | wrapped_tool = os.getenv(cls.blight_tool().env) 94 | if wrapped_tool is None: 95 | raise BlightError(f"No wrapped tool found for {cls.build_tool()}") 96 | return wrapped_tool 97 | 98 | def __init__(self, args: List[str]) -> None: 99 | if self.__class__ == Tool: 100 | raise NotImplementedError(f"can't instantiate {self.__class__.__name__} directly") 101 | self._args = args 102 | self._canonicalized_args = args.copy() 103 | self._env = self._fixup_env() 104 | self._cwd = Path(os.getcwd()).resolve() 105 | self._actions = util.load_actions() 106 | self._skip_run = False 107 | self._action_results: Dict[str, Optional[Dict[str, Any]]] = {} 108 | self._journal_path = os.getenv("BLIGHT_JOURNAL_PATH") 109 | 110 | def _fixup_env(self) -> Dict[str, str]: 111 | """ 112 | Fixes up `os.environ` to remove any references to blight's swizzled paths, 113 | if any are present. 114 | """ 115 | env = dict(os.environ) 116 | env["PATH"] = util.unswizzled_path() 117 | return env 118 | 119 | def _before_run(self) -> None: 120 | for action in self._actions: 121 | try: 122 | action._before_run(self) 123 | except SkipRun: 124 | self._skip_run = True 125 | 126 | def _after_run(self) -> None: 127 | for action in self._actions: 128 | action._after_run(self, run_skipped=self._skip_run) 129 | 130 | if self.is_journaling(): 131 | self._action_results[action.__class__.__name__] = action.result 132 | 133 | def _commit_journal(self) -> None: 134 | if self.is_journaling(): 135 | with util.flock_append(self._journal_path) as io: # type: ignore 136 | json.dump(self._action_results, io, default=json_helper) 137 | # NOTE(ww): `json.dump` doesn't do this for us. 138 | io.write("\n") 139 | 140 | def run(self) -> None: 141 | """ 142 | Runs the wrapped tool with the original arguments. 143 | """ 144 | self._before_run() 145 | 146 | if not self._skip_run: 147 | status = subprocess.run([self.wrapped_tool(), *self.args], env=self._env) 148 | if status.returncode != 0: 149 | raise BuildError( 150 | f"{self.wrapped_tool()} exited with status code {status.returncode}" 151 | ) 152 | 153 | self._after_run() 154 | 155 | self._commit_journal() 156 | 157 | def is_journaling(self) -> bool: 158 | """ 159 | Returns: 160 | `True` if this `Tool` is in "journaling" mode. 161 | """ 162 | return self._journal_path is not None 163 | 164 | def asdict(self) -> Dict[str, Any]: 165 | """ 166 | Returns: 167 | A dictionary representation of this tool 168 | """ 169 | 170 | return { 171 | "name": self.__class__.__name__, 172 | "wrapped_tool": self.wrapped_tool(), 173 | "args": self.args, 174 | "canonicalized_args": self.canonicalized_args, 175 | "cwd": str(self._cwd), 176 | "env": self._env, 177 | } 178 | 179 | @property 180 | def args(self) -> List[str]: 181 | return self._args 182 | 183 | @args.setter 184 | def args(self, args_: List[str]) -> None: 185 | self._args = args_ 186 | 187 | # NOTE(ww): Modifying the effective arguments also propagates 188 | # those changes to the canonicalized arguments. This shouldn't be a problem, 189 | # since mixins that specialize `canonicalized_args` call 190 | # `super.canonicalized_args` to get the most recent copy. 191 | self._canonicalized_args = args_.copy() 192 | 193 | @property 194 | def canonicalized_args(self) -> List[str]: 195 | # NOTE(ww): `canonicalized_args` doesn't need an explicit setter property, 196 | # since all specializations of it are expected to modify the underlying 197 | # list. 198 | return self._canonicalized_args 199 | 200 | @property 201 | def cwd(self) -> Path: 202 | """ 203 | Returns the directory that this tool was run in. 204 | """ 205 | return self._cwd 206 | 207 | @property 208 | def inputs(self) -> List[str]: 209 | """ 210 | Returns all explicit "inputs" to the tool. "Inputs" is subjectively 211 | defined to be the "main" inputs to a tool, i.e. source files and **not** 212 | additional files that *may* be passed in via options. 213 | 214 | Tools may further refine the behavior of this property 215 | by overriding it with their own, more specific behavior. 216 | 217 | **NOTE**: This property, more so than others, relies on heuristics. 218 | 219 | Returns: 220 | A list of `str`s, representing the tool's inputs. 221 | """ 222 | 223 | # Our strategy here is as follows: 224 | # * Filter out any arguments that begin with "-" or "@" and 225 | # aren't *just" "-" (since that indicates stdin). 226 | # * Then, look for arguments that are files in the tool's current 227 | # directory. 228 | inputs = [] 229 | for idx, arg in enumerate(self.canonicalized_args): 230 | if arg.startswith("-") or arg.startswith("@"): 231 | if arg == "-": 232 | inputs.append(arg) 233 | continue 234 | 235 | candidate = Path(arg) 236 | if not candidate.is_file() and not (self.cwd / candidate).is_file(): 237 | # NOTE(ww): pathlib's is_file returns False for device files, e.g. /dev/stdin. 238 | # It would be perverse for a build system to use these, but maybe worth 239 | # handling. 240 | continue 241 | 242 | # Annoying edge cases: most other flags that take filenames do so in 243 | # -flag=filename form, but -aux-info does it without the "=". 244 | # Similarly, we need to make sure not to catch an output flag's 245 | # argument here. 246 | if idx == 0 or self.canonicalized_args[idx - 1] not in ["-aux-info", "-o"]: 247 | inputs.append(arg) 248 | 249 | return inputs 250 | 251 | @property 252 | def outputs(self) -> List[str]: 253 | """ 254 | Returns all "outputs" produced by the tool. "Outputs" is subjectively 255 | defined to be the "main" products of a tool, i.e. results of a particular 256 | stage or invocation and **not** any incidental or metadata files that 257 | might otherwise be created in the process. 258 | 259 | Tools may further refine the behavior of this mixin-supplied property 260 | by overriding it with their own, more specific behavior. 261 | 262 | Returns: 263 | A list of `str`, each of which is an output 264 | """ 265 | 266 | o_flag_index = util.rindex_prefix(self.canonicalized_args, "-o") 267 | if o_flag_index is None: 268 | return [] 269 | 270 | if self.canonicalized_args[o_flag_index] == "-o": 271 | return [self.canonicalized_args[o_flag_index + 1]] 272 | 273 | # NOTE(ww): Outputs like -ofoo. Gross, but valid according to GCC. 274 | return [self.canonicalized_args[o_flag_index][2:]] 275 | 276 | 277 | class LangMixin: 278 | """ 279 | A mixin for tools that have a "language" component, i.e. 280 | those that change their behavior based on the language that they're used with. 281 | """ 282 | 283 | @property 284 | def lang(self: CanonicalizedArgsProtocol) -> Lang: 285 | """ 286 | Returns: 287 | A `blight.enums.Lang` value representing the tool's language 288 | """ 289 | logger.warning( 290 | "this API might not do what you expect; see: https://github.com/trailofbits/blight/issues/43493" 291 | ) 292 | 293 | x_lang_map = {"c": Lang.C, "c-header": Lang.C, "c++": Lang.Cxx, "c++-header": Lang.Cxx} 294 | 295 | # First, check for `-x lang`. This overrides the language determined by 296 | # the frontend's binary name (e.g. `g++`). 297 | x_flag_index = util.rindex_prefix(self.canonicalized_args, "-x") 298 | if x_flag_index is not None: 299 | if self.canonicalized_args[x_flag_index] == "-x": 300 | # TODO(ww): Maybe bounds check. 301 | x_lang = self.canonicalized_args[x_flag_index + 1] 302 | else: 303 | # NOTE(ww): -xc and -xc++ both work, at least on GCC. 304 | x_lang = self.canonicalized_args[x_flag_index][2:] 305 | return x_lang_map.get(x_lang, Lang.Unknown) 306 | 307 | # No `-x lang` means that we're operating in the frontend's default mode. 308 | if self.__class__ == CC: 309 | return Lang.C 310 | elif self.__class__ == CXX: 311 | return Lang.Cxx 312 | else: 313 | logger.debug(f"unknown default language mode for {self.__class__.__name__}") 314 | return Lang.Unknown 315 | 316 | 317 | class StdMixin(LangMixin): 318 | """ 319 | A mixin for tools that have a "standard" component, i.e. 320 | those that change their behavior based on a particular language standard. 321 | """ 322 | 323 | @property 324 | def std(self: LangProtocol) -> Std: 325 | """ 326 | Returns: 327 | A `blight.enums.Std` value representing the tool's standard 328 | """ 329 | 330 | # First, a special case: if -ansi is present, we're in 331 | # C89 mode for C code and C++03 mode for C++ code. 332 | if "-ansi" in self.canonicalized_args: 333 | if self.lang == Lang.C: 334 | return Std.C89 335 | elif self.lang == Lang.Cxx: 336 | return Std.Cxx03 337 | else: 338 | logger.debug(f"-ansi passed but unknown language: {self.lang}") 339 | return Std.Unknown 340 | 341 | # Experimentally, both GCC and clang respect the last -std=XXX flag passed. 342 | # See: https://stackoverflow.com/questions/40563269/passing-multiple-std-switches-to-g 343 | std_flag_index = util.rindex_prefix(self.canonicalized_args, "-std=") 344 | 345 | # No -std=XXX flags? The tool is operating in its default standard mode, 346 | # which is determined by its language. 347 | if std_flag_index is None: 348 | if self.lang == Lang.C: 349 | return Std.GnuUnknown 350 | elif self.lang == Lang.Cxx: 351 | return Std.GnuxxUnknown 352 | else: 353 | logger.debug(f"no -std= flag and unknown language: {self.lang}") 354 | return Std.Unknown 355 | 356 | last_std_flag = self.canonicalized_args[std_flag_index] 357 | std_flag_map = { 358 | # C89 flags. 359 | "-std=c89": Std.C89, 360 | "-std=c90": Std.C89, 361 | "-std=iso9899:1990": Std.C89, 362 | # C94 flags. 363 | "-std=iso9899:199409": Std.C94, 364 | # C99 flags. 365 | "-std=c99": Std.C99, 366 | "-std=c9x": Std.C99, 367 | "-std=iso9899:1999": Std.C99, 368 | "-std=iso9899:199x": Std.C99, 369 | # C11 flags. 370 | "-std=c11": Std.C11, 371 | "-std=c1x": Std.C11, 372 | "-std=iso9899:2011": Std.C11, 373 | # C17 flags. 374 | "-std=c17": Std.C17, 375 | "-std=c18": Std.C17, 376 | "-std=iso9899:2017": Std.C17, 377 | "-std=iso9899:2018": Std.C17, 378 | # C20 (presumptive) flags. 379 | "-std=c2x": Std.C2x, 380 | # GNU89 flags. 381 | "-std=gnu89": Std.Gnu89, 382 | "-std=gnu90": Std.Gnu89, 383 | # GNU99 flags. 384 | "-std=gnu99": Std.Gnu99, 385 | "-std=gnu9x": Std.Gnu99, 386 | # GNU11 flags. 387 | "-std=gnu11": Std.Gnu11, 388 | "-std=gnu1x": Std.Gnu11, 389 | # GNU17 flags. 390 | "-std=gnu17": Std.Gnu17, 391 | "-std=gnu18": Std.Gnu17, 392 | # GNU20 (presumptive) flags. 393 | "-std=gnu2x": Std.Gnu2x, 394 | # C++03 flags. 395 | # NOTE(ww): Both gcc and clang treat C++98 mode as C++03 mode. 396 | "-std=c++98": Std.Cxx03, 397 | "-std=c++03": Std.Cxx03, 398 | # C++11 flags. 399 | "-std=c++11": Std.Cxx11, 400 | "-std=c++0x": Std.Cxx11, 401 | # C++14 flags. 402 | "-std=c++14": Std.Cxx14, 403 | "-std=c++1y": Std.Cxx14, 404 | # C++17 flags. 405 | "-std=c++17": Std.Cxx17, 406 | "-std=c++1z": Std.Cxx17, 407 | # C++20 (presumptive) flags. 408 | "-std=c++2a": Std.Cxx2a, 409 | "-std=c++20": Std.Cxx2a, 410 | # GNU++03 flags. 411 | "-std=gnu++98": Std.Gnuxx03, 412 | "-std=gnu++03": Std.Gnuxx03, 413 | # GNU++11 flags. 414 | "-std=gnu++11": Std.Gnuxx11, 415 | "-std=gnu++0x": Std.Gnuxx11, 416 | # GNU++14 flags. 417 | "-std=gnu++14": Std.Gnuxx14, 418 | "-std=gnu++1y": Std.Gnuxx14, 419 | # GNU++17 flags. 420 | "-std=gnu++17": Std.Gnuxx17, 421 | "-std=gnu++1z": Std.Gnuxx17, 422 | # GNU++20 (presumptive) flags. 423 | "-std=gnu++2a": Std.Gnuxx2a, 424 | "-std=gnu++20": Std.Gnuxx2a, 425 | } 426 | 427 | std = std_flag_map.get(last_std_flag) 428 | if std is not None: 429 | return std 430 | 431 | # If we've made it here, then we've reached a -std=XXX flag that we 432 | # don't know yet. Make an effort to guess at it. 433 | std_name = last_std_flag.split("=")[1] 434 | if std_name.startswith("c++"): 435 | logger.debug(f"partially unrecognized c++ std: {last_std_flag}") 436 | return Std.CxxUnknown 437 | elif std_name.startswith("gnu++"): 438 | logger.debug(f"partially unrecognized gnu++ std: {last_std_flag}") 439 | return Std.GnuxxUnknown 440 | elif std_name.startswith("gnu"): 441 | logger.debug(f"partially unrecognized gnu c std: {last_std_flag}") 442 | return Std.GnuUnknown 443 | elif std_name.startswith("c") or std_name.startswith("iso9899"): 444 | logger.debug(f"partially unrecognized c std: {last_std_flag}") 445 | return Std.CUnknown 446 | 447 | logger.debug(f"completely unrecognized -std= flag: {last_std_flag}") 448 | return Std.Unknown 449 | 450 | 451 | class OptMixin: 452 | """ 453 | A mixin for tools that have an optimization level. 454 | """ 455 | 456 | @property 457 | def opt(self: CanonicalizedArgsProtocol) -> OptLevel: 458 | """ 459 | Returns: 460 | A `blight.enums.OptLevel` value representing the optimization level 461 | """ 462 | 463 | opt_flag_map = { 464 | "-O0": OptLevel.O0, 465 | "-O": OptLevel.O1, 466 | "-O1": OptLevel.O1, 467 | "-O2": OptLevel.O2, 468 | "-O3": OptLevel.O3, 469 | "-Ofast": OptLevel.OFast, 470 | "-Os": OptLevel.OSize, 471 | "-Oz": OptLevel.OSizeZ, 472 | "-Og": OptLevel.ODebug, 473 | } 474 | 475 | # The last optimization flag takes precedence, so iterate over the arguments 476 | # in reverse order. 477 | for arg in reversed(self.canonicalized_args): 478 | opt = opt_flag_map.get(arg) 479 | if opt is not None: 480 | return opt 481 | 482 | if not arg.startswith("-O"): 483 | continue 484 | 485 | # Special case: -O4 and above are currently equivalent to -O3 in 486 | # GCC and Clang. Identify these and map them to -O3. 487 | if re.fullmatch(r"^-O[1-9]\d*$", arg): 488 | return OptLevel.O3 489 | 490 | # Otherwise: We've found an argument that looks like -Osomething, 491 | # but we don't know what it is. Treat it as an unknown. 492 | logger.debug(f"unknown optimization level: {arg}") 493 | return OptLevel.Unknown 494 | 495 | # If we've made it here, then the arguments don't mention an explicit 496 | # optimization level. Both GCC and Clang use -O0 by default, so return that here. 497 | return OptLevel.O0 498 | 499 | 500 | class ResponseFileMixin: 501 | """ 502 | A mixin for tools that support the `@file` syntax for adding command-line arguments 503 | via an input file. 504 | 505 | These appear to originate from Windows and are called "response files" there, hence 506 | the name of this mixin. 507 | """ 508 | 509 | def _expand_response_file( 510 | self, response_file: Path, working_dir: Path, level: int 511 | ) -> List[str]: 512 | if level >= RESPONSE_FILE_RECURSION_LIMIT: 513 | logger.debug(f"recursion limit exceeded: {response_file} in {working_dir}") 514 | return [] 515 | 516 | # Non-absolute response files are resolved relative to `working_dir`, which 517 | # begins at the CWD initially and changes to the parent directory of the 518 | # including file for nested response files. 519 | if not response_file.is_absolute(): 520 | response_file = working_dir / response_file 521 | 522 | if not response_file.is_file(): 523 | logger.debug(f"response file {response_file} does not exist") 524 | # TODO(ww): Instead of returning empty here, maybe return `@response_file`? 525 | return [] 526 | 527 | args = shlex.split(response_file.read_text()) 528 | response_files = [(idx, arg) for (idx, arg) in enumerate(args) if arg.startswith("@")] 529 | for idx, nested_rf in response_files: 530 | args = util.insert_items_at_idx( 531 | args, 532 | idx, 533 | self._expand_response_file( 534 | Path(nested_rf[1:]), response_file.parent.resolve(), level + 1 535 | ), 536 | ) 537 | 538 | return args 539 | 540 | @property 541 | def canonicalized_args(self) -> List[str]: 542 | """ 543 | Overrides the behavior of `Tool.canonicalized_args`, expanding any response file arguments 544 | in a depth-first manner. 545 | """ 546 | 547 | # NOTE(ww): This method badly needs some typechecking TLC. 548 | # The `super()` call to `canonicalized_args` probably needs to be handled 549 | # with a `self: CanonicalizedArgsProtocol` hint, but that causes other problems 550 | # related to mypy's ability to see `_expand_response_file`. 551 | 552 | response_files = [ 553 | (idx, arg) 554 | for (idx, arg) in enumerate(super().canonicalized_args) # type: ignore 555 | if arg.startswith("@") 556 | ] 557 | expanded_args = super().canonicalized_args # type: ignore 558 | for idx, response_file in response_files: 559 | expanded_args = util.insert_items_at_idx( 560 | expanded_args, 561 | idx, 562 | self._expand_response_file(Path(response_file[1:]), self.cwd, 0), # type: ignore 563 | ) 564 | 565 | self._canonicalized_args = expanded_args 566 | return self._canonicalized_args # type: ignore[no-any-return] 567 | 568 | 569 | class DefinesMixin: 570 | """ 571 | A mixin for tools that support the `-Dname[=value]` and `-Uname` syntaxes for defining 572 | and undefining C preprocessor macros. 573 | """ 574 | 575 | @property 576 | def indexed_undefines(self: IndexedUndefinesProtocol) -> Dict[str, int]: 577 | """ 578 | Returns a dictionary of indices for undefined macros. This is used in 579 | `defines` to ensure that we don't incorrectly report a subsequently undefined 580 | macro as defined. Only the rightmost index of each undefined macro is saved. 581 | 582 | Returns: 583 | A dict of `name: index` for each undefined macro. 584 | """ 585 | indexed_undefines = {} 586 | for idx, arg in enumerate(self.canonicalized_args): 587 | if not arg.startswith("-U"): 588 | continue 589 | 590 | # Both `-Uname` and `-U name` work in GCC and Clang. 591 | undefine = self.canonicalized_args[idx + 1] if arg == "-U" else arg[2:] 592 | 593 | indexed_undefines[undefine] = idx 594 | 595 | return indexed_undefines 596 | 597 | @property 598 | def defines(self: IndexedUndefinesProtocol) -> List[Tuple[str, str]]: 599 | """ 600 | The list of **effective** defines for this tool invocation. An "effective" 601 | define is one that is not canceled out by a subsequent undefine. 602 | 603 | Returns: 604 | A list of tuples of (name, value) for each effectively defined macro. 605 | """ 606 | defines = [] 607 | for idx, arg in enumerate(self.canonicalized_args): 608 | if not arg.startswith("-D"): 609 | continue 610 | 611 | # Both `-Dname[=value]` and `-D name[=value]` work in GCC and Clang. 612 | define = self.canonicalized_args[idx + 1] if arg == "-D" else arg[2:] 613 | 614 | components = define.split("=", 1) 615 | name = components[0] 616 | 617 | # NOTE(ww): 1 is the default macro value. 618 | # It's actually an integer at the preprocessor level, but we model everything 619 | # as strings here to avoid complicating things. 620 | value = "1" if len(components) == 1 else components[1] 621 | 622 | # Is this macro subsequently undefined? If so, don't include it in 623 | # the defines list. 624 | if self.indexed_undefines.get(name, -1) > idx: 625 | continue 626 | 627 | defines.append((name, value)) 628 | 629 | return defines 630 | 631 | 632 | class CodeModelMixin: 633 | """ 634 | A mixin for tools that support the `-mcmodel=MODEL` syntax for declaring their 635 | code model. 636 | """ 637 | 638 | @property 639 | def code_model(self: CanonicalizedArgsProtocol) -> CodeModel: 640 | """ 641 | Returns: 642 | A `blight.enums.CodeModel` value representing the tool's code model 643 | """ 644 | code_model_map = { 645 | "-mcmodel=small": CodeModel.Small, 646 | "-mcmodel=medlow": CodeModel.Small, 647 | "-mcmodel=medium": CodeModel.Medium, 648 | "-mcmodel=medany": CodeModel.Medium, 649 | "-mcmodel=large": CodeModel.Large, 650 | "-mcmodel=kernel": CodeModel.Kernel, 651 | } 652 | 653 | # NOTE(ww): Both Clang and GCC seem to default to the "small" code model 654 | # when none is specified, at least on x86-64. But this might not be consistent 655 | # across architectures, so maybe we should return `CodeModel.Unknown` here 656 | # instead. 657 | code_model = util.ritem_prefix(self.canonicalized_args, "-mcmodel=") 658 | if code_model is None: 659 | return CodeModel.Small 660 | 661 | return code_model_map.get(code_model, CodeModel.Unknown) 662 | 663 | 664 | class LinkSearchMixin: 665 | """ 666 | A mixin for tools that support the `-Lpath` and `-llib` syntaxes for specifying 667 | library paths and libraries, respectively. 668 | """ 669 | 670 | @property 671 | def explicit_library_search_paths(self: CanonicalizedArgsProtocol) -> List[Path]: 672 | """ 673 | Returns a list of library search paths that are explicitly specified in 674 | the tool's invocation. Semantically, these paths are (normally) given 675 | priority over all other search paths. 676 | 677 | NOTE: This is **not** the same as the complete list of library search paths, 678 | which is tool-specific and host-dependent. 679 | """ 680 | 681 | shorts = util.collect_option_values(self.canonicalized_args, "-L") 682 | longs = util.collect_option_values( 683 | self.canonicalized_args, "--library-path", style=util.OptionValueStyle.EqualOrSpace 684 | ) 685 | 686 | sorted_values = sorted(itertools.chain(shorts, longs), key=lambda v: v[0]) 687 | 688 | return [(self.cwd / value[1]).resolve() for value in sorted_values] 689 | 690 | @property 691 | def library_names(self: CanonicalizedArgsProtocol) -> List[str]: 692 | """ 693 | Returns a list of library names (without suffixes) for libraries that 694 | are explicitly specified in the tool's invocation. 695 | 696 | NOTE: This list does not include any libraries that are 697 | listed as "inputs" to the tool rather than as linkage specifications. 698 | """ 699 | 700 | shorts = util.collect_option_values(self.canonicalized_args, "-l") 701 | longs = util.collect_option_values( 702 | self.canonicalized_args, "--library", style=util.OptionValueStyle.EqualOrSpace 703 | ) 704 | 705 | sorted_values = sorted(itertools.chain(shorts, longs), key=lambda v: v[0]) 706 | 707 | return [f"lib{value[1]}" for value in sorted_values] 708 | 709 | 710 | # NOTE(ww): The funny mixin order here (`ResponseFileMixin` before `Tool`) and elsewhere 711 | # is because Python defines its class hierarchy from right to left. `ResponseFileMixin` 712 | # therefore needs to come first in order to properly override `canonicalized_args`. 713 | class CompilerTool( 714 | LinkSearchMixin, ResponseFileMixin, Tool, StdMixin, OptMixin, DefinesMixin, CodeModelMixin 715 | ): 716 | """ 717 | Represents a generic (C or C++) compiler frontend. 718 | 719 | Like `Tool`, `CompilerTool` cannot be instantiated directly. 720 | """ 721 | 722 | def __init__(self, args: List[str]) -> None: 723 | if self.__class__ == CompilerTool: 724 | raise NotImplementedError(f"can't instantiate {self.__class__.__name__} directly") 725 | 726 | super().__init__(args) 727 | 728 | # #40 and #41: These should be handled in an overridden implementation 729 | # of `canonicalized_args`. 730 | injection_vars = COMPILER_FLAG_INJECTION_VARIABLES & self._env.keys() 731 | if injection_vars: 732 | logger.warning(f"not tracking compiler's own instrumentation: {injection_vars}") 733 | 734 | @property 735 | def family(self) -> CompilerFamily: 736 | """ 737 | Returns: 738 | A `blight.enums.CompilerFamily` value representing the "family" of compilers 739 | that this tool belongs to. 740 | """ 741 | 742 | # NOTE(ww): Both GCC and Clang support -### as an alias for -v, but 743 | # with additional guarantees around argument quoting. Do other families support it? 744 | 745 | result = subprocess.run([self.wrapped_tool(), "-###"], capture_output=True) 746 | 747 | # If the command exited with an error, we're likely dealing with a frontend 748 | # that doesn't understand `-###`. 749 | if result.returncode != 0: 750 | logger.warning("compiler fingerprint failed: frontend didn't recognize -###?") 751 | # ...but even still, we can infer a bit from the error message. 752 | if b"tcc: error" in result.stderr: 753 | return CompilerFamily.Tcc 754 | else: 755 | return CompilerFamily.Unknown 756 | 757 | # We expect the relevant parts of `-###` on stderr. The lack of any output 758 | # again suggests that the frontend doesn't understand the flag. 759 | if not result.stderr: 760 | logger.warning("compiler fingerprint failed: frontend didn't produce output for -###?") 761 | return CompilerFamily.Unknown 762 | 763 | # Finally, we do some silly substring checks. 764 | # TODO(ww): Better heuristics here? 765 | if b"Apple clang version" in result.stderr: 766 | return CompilerFamily.AppleLlvm 767 | elif b"clang version" in result.stderr: 768 | return CompilerFamily.MainlineLlvm 769 | elif b"gcc version" in result.stderr: 770 | return CompilerFamily.Gcc 771 | else: 772 | return CompilerFamily.Unknown 773 | 774 | @property 775 | def stage(self) -> CompilerStage: 776 | """ 777 | Returns: 778 | A `blight.enums.CompilerStage` value representing the stage that this tool is on 779 | """ 780 | 781 | # TODO(ww): Refactor this entire method. Both GCC and Clang can actually 782 | # run multiple stages per invocation, e.g. `-x c foo.c -x c++ bar.cpp`, 783 | # so we should model this as "stages" instead. This, in turn, will require 784 | # us to reevaluate our output guesswork below. 785 | 786 | if len(self.canonicalized_args) == 0: 787 | return CompilerStage.Unknown 788 | 789 | stage_flag_map = { 790 | # NOTE(ww): See the TODO in CompilerStage. 791 | "-v": CompilerStage.Unknown, 792 | "-###": CompilerStage.Unknown, 793 | "-E": CompilerStage.Preprocess, 794 | "-fsyntax-only": CompilerStage.SyntaxOnly, 795 | "-S": CompilerStage.Assemble, 796 | "-c": CompilerStage.CompileObject, 797 | } 798 | 799 | for flag, stage in stage_flag_map.items(): 800 | if flag in self.canonicalized_args: 801 | return stage 802 | 803 | # TODO(ww): Handle header precompilation here. GCC doesn't seem to 804 | # consider this a real "stage", but it's different enough from every 805 | # other stage to warrant special treatment. 806 | 807 | # No explicit stage flag? Both gcc and clang treat this as 808 | # "run all stages", so we do too. 809 | return CompilerStage.AllStages 810 | 811 | @property 812 | def outputs(self) -> List[str]: 813 | """ 814 | Specializes `Tool.outputs` for compiler tools. 815 | """ 816 | outputs = super().outputs 817 | if outputs != []: 818 | return outputs 819 | 820 | # Without an explicit `-o outfile`, the default output name(s) 821 | # depends on the compiler's stage. 822 | if self.stage == CompilerStage.Preprocess: 823 | # NOTE(ww): The preprocessor stage emits to stdout, but returning "-" as 824 | # a sentinel for that is very meh. If only Python had Rust-style enums. 825 | return ["-"] 826 | elif self.stage == CompilerStage.Assemble: 827 | # NOTE(ww): Outputs are created relative to the current working directory, 828 | # not relative to their input. We return them as relative paths to 829 | # indicate this (maybe we should just fully resolve them?) 830 | return [Path(input_).with_suffix(".s").name for input_ in self.inputs] 831 | elif self.stage == CompilerStage.CompileObject: 832 | return [Path(input_).with_suffix(".o").name for input_ in self.inputs] 833 | elif self.stage == CompilerStage.AllStages: 834 | # NOTE(ww): This will be wrong when we're doing header precompilation; 835 | # see the TODO in `stage`. 836 | return ["a.out"] 837 | else: 838 | return [] 839 | 840 | def asdict(self) -> Dict[str, Any]: 841 | return { 842 | **super().asdict(), 843 | "lang": self.lang.name, 844 | "std": self.std.name, 845 | "stage": self.stage.name, 846 | "opt": self.opt.name, 847 | } 848 | 849 | 850 | class CC(CompilerTool): 851 | """ 852 | A specialization of `CompilerTool` for the C compiler frontend. 853 | """ 854 | 855 | def __repr__(self) -> str: 856 | return f"" 857 | 858 | 859 | class CXX(CompilerTool): 860 | """ 861 | A specialization of `CompilerTool` for the C++ compiler frontend. 862 | """ 863 | 864 | def __repr__(self) -> str: 865 | return f"" 866 | 867 | 868 | class CPP(Tool, StdMixin, DefinesMixin): 869 | """ 870 | Represents the C preprocessor tool. 871 | """ 872 | 873 | def __repr__(self) -> str: 874 | return f"" 875 | 876 | def asdict(self) -> Dict[str, Any]: 877 | return {**super().asdict(), "lang": self.lang.name, "std": self.std.name} 878 | 879 | 880 | class LD(LinkSearchMixin, ResponseFileMixin, Tool): 881 | """ 882 | Represents the linker. 883 | """ 884 | 885 | @property 886 | def outputs(self) -> List[str]: 887 | """ 888 | Specializes `Tool.outputs` for the linker. 889 | """ 890 | 891 | outputs = super().outputs 892 | if outputs != []: 893 | return outputs 894 | 895 | # The GNU linker additionally supports --output=OUTFILE and 896 | # --output OUTFILE. Handle them here. 897 | output_flag_index = util.rindex_prefix(self.canonicalized_args, "--output") 898 | if output_flag_index is None: 899 | return ["a.out"] 900 | 901 | # Split option form. 902 | if self.canonicalized_args[output_flag_index] == "--output": 903 | return [self.canonicalized_args[output_flag_index + 1]] 904 | 905 | # Assignment form. 906 | return [self.canonicalized_args[output_flag_index].split("=")[1]] 907 | 908 | def __repr__(self) -> str: 909 | return f"" 910 | 911 | 912 | class AS(ResponseFileMixin, Tool): 913 | """ 914 | Represents the assembler. 915 | """ 916 | 917 | def __repr__(self) -> str: 918 | return f"" 919 | 920 | 921 | class AR(ResponseFileMixin, Tool): 922 | """ 923 | Represents the archiver. 924 | """ 925 | 926 | @property 927 | def outputs(self) -> List[str]: 928 | """ 929 | Specializes `Tool.outputs` for the archiver. 930 | """ 931 | 932 | # TODO(ww): This doesn't support `ar x`, which explodes the archive 933 | # (i.e., treats it as input) instead of treats it as output. 934 | # It would be pretty strange for a build system to do this, but it's 935 | # probably something we should detect at the very least. 936 | 937 | # TODO(ww): We also don't support `ar t`, which queries the given 938 | # archive to provide a table listing of its contents. 939 | 940 | # NOTE(ww): `ar`'s POSIX and GNU CLIs are annoyingly complicated. 941 | # We save ourselves some pain by scanning from left-to-right, looking 942 | # for the first argument that looks like an archive output 943 | # (since the archiver only ever produces one output at a time). 944 | for arg in self.canonicalized_args: 945 | if arg.startswith("-"): 946 | continue 947 | 948 | maybe_archive_suffixes = Path(arg).suffixes 949 | if len(maybe_archive_suffixes) > 0 and maybe_archive_suffixes[0] == ".a": 950 | return [arg] 951 | 952 | logger.debug("couldn't infer output for archiver") 953 | return [] 954 | 955 | def __repr__(self) -> str: 956 | return f"" 957 | 958 | 959 | class STRIP(ResponseFileMixin, Tool): 960 | """ 961 | Represents the stripping tool. 962 | """ 963 | 964 | def __repr__(self) -> str: 965 | return f"" 966 | 967 | 968 | class INSTALL(Tool): 969 | """ 970 | Represents the install tool. 971 | """ 972 | 973 | def _install_parser(self) -> util.ArgumentParser: 974 | parser = util.ArgumentParser( 975 | prog=self.build_tool().value, add_help=False, allow_abbrev=False 976 | ) 977 | 978 | def add_flag(short: str, dest: str, **kwargs: Any) -> None: 979 | parser.add_argument(short, action="store_true", dest=dest, **kwargs) 980 | 981 | add_flag("-b", "overwrite") 982 | add_flag("-C", "copy_no_mtime") 983 | add_flag("-c", "copy", default=True) 984 | add_flag("-d", "directory_mode") 985 | add_flag("-M", "disable_mmap") 986 | add_flag("-p", "preserve_mtime") 987 | add_flag("-S", "safe_copy") 988 | add_flag("-s", "exec_strip") 989 | add_flag("-v", "verbose") 990 | parser.add_argument("-f", dest="flags") 991 | parser.add_argument("-g", dest="group") 992 | parser.add_argument("-m", dest="mode") 993 | parser.add_argument("-o", dest="owner") 994 | parser.add_argument("trailing", nargs="+", default=[]) 995 | 996 | return parser 997 | 998 | def __init__(self, args: List[str]) -> None: 999 | super().__init__(args) 1000 | self._parser = self._install_parser() 1001 | 1002 | try: 1003 | (self._matches, self._unknown) = self._parser.parse_known_args(args) 1004 | except ValueError as e: 1005 | logger.error(f"argparse error: {e}") 1006 | self._matches = self._parser.default_namespace() 1007 | self._unknown = args 1008 | 1009 | @property 1010 | def directory_mode(self) -> bool: 1011 | """ 1012 | Returns whether this `install` invocation is in "directory mode," i.e. 1013 | is creating directories instead of installing files. 1014 | """ 1015 | return self._matches.directory_mode # type: ignore[no-any-return] 1016 | 1017 | @property 1018 | def inputs(self) -> List[str]: 1019 | """ 1020 | Specializes `Tool.inputs` for the install tool. 1021 | """ 1022 | 1023 | # Directory mode: all positionals are new directories, i.e. outputs. 1024 | if self.directory_mode: 1025 | return [] 1026 | 1027 | # `install` requires at least two positionals outside of directory mode, 1028 | # so this probably indicates an unknown GNUism like `--help`. 1029 | if len(self._matches.trailing) < 2: 1030 | logger.debug(f"install called with no positionals (hint: unknown args: {self._unknown}") 1031 | return [] 1032 | 1033 | # Otherwise, we're either installing one file to another or we're 1034 | # installing multiple files to a directory. Test the last positional 1035 | # to determine which mode we're in. 1036 | maybe_dir = self._cwd / self._matches.trailing[-1] 1037 | if maybe_dir.is_dir(): 1038 | return self._matches.trailing[0:-1] # type: ignore[no-any-return] 1039 | else: 1040 | return [self._matches.trailing[0]] 1041 | 1042 | @property 1043 | def outputs(self) -> List[str]: 1044 | """ 1045 | Specializes `Tool.outputs` for the install tool. 1046 | """ 1047 | 1048 | # Directory mode: treat created directories as outputs. 1049 | if self.directory_mode: 1050 | return self._matches.trailing # type: ignore[no-any-return] 1051 | 1052 | # `install` requires at least two positionals outside of directory mode, 1053 | # so this probably indicates an unknown GNUism like `--help`. 1054 | if len(self._matches.trailing) < 2: 1055 | logger.debug(f"install called with no positionals (hint: unknown args: {self._unknown}") 1056 | return [] 1057 | 1058 | # If we're installing multiple files to a destination directory, 1059 | # then our outputs are every input, under the destination. 1060 | # Otherwise, our output is a single file. 1061 | maybe_dir = self._cwd / self._matches.trailing[-1] 1062 | if maybe_dir.is_dir(): 1063 | inputs = [Path(input_) for input_ in self._matches.trailing[0:-1]] 1064 | return [str(maybe_dir / input_.name) for input_ in inputs] 1065 | else: 1066 | return [self._matches.trailing[-1]] 1067 | 1068 | def __repr__(self) -> str: 1069 | return f"" 1070 | -------------------------------------------------------------------------------- /src/blight/util.py: -------------------------------------------------------------------------------- 1 | """ 2 | Helper utilities for blight. 3 | """ 4 | 5 | from __future__ import annotations 6 | 7 | import argparse 8 | import contextlib 9 | import enum 10 | import fcntl 11 | import os 12 | import shlex 13 | import sys 14 | from pathlib import Path 15 | from typing import IO, TYPE_CHECKING, Any, Iterator, NoReturn, Sequence 16 | 17 | if TYPE_CHECKING: 18 | from blight.action import Action # pragma: no cover 19 | from blight.exceptions import BlightError 20 | 21 | SWIZZLE_SENTINEL = "@blight-swizzle@" 22 | 23 | 24 | @enum.unique 25 | class OptionValueStyle(enum.Enum): 26 | """ 27 | A collection of common option formatting styles in build tools. 28 | 29 | This enumeration is not exhaustive. 30 | """ 31 | 32 | Space = enum.auto() 33 | """ 34 | Options that look like `-O foo`. 35 | """ 36 | 37 | Mash = enum.auto() 38 | """ 39 | Options that look like `-Ofoo`. 40 | """ 41 | 42 | MashOrSpace = enum.auto() 43 | """ 44 | Options that look like `-Ofoo` or `-O foo`. 45 | """ 46 | 47 | Equal = enum.auto() 48 | """ 49 | Options that look like `-O=foo`. 50 | """ 51 | 52 | EqualOrSpace = enum.auto() 53 | """ 54 | Options that look like `-O=foo` or `-O foo`. 55 | """ 56 | 57 | def permits_equal(self) -> bool: 58 | return self in [OptionValueStyle.Equal, OptionValueStyle.EqualOrSpace] 59 | 60 | def permits_mash(self) -> bool: 61 | return self in [OptionValueStyle.Mash, OptionValueStyle.MashOrSpace] 62 | 63 | def permits_space(self) -> bool: 64 | return self in [ 65 | OptionValueStyle.Space, 66 | OptionValueStyle.MashOrSpace, 67 | OptionValueStyle.EqualOrSpace, 68 | ] 69 | 70 | 71 | def die(message: str) -> NoReturn: 72 | """ 73 | Aborts the program with a final message. 74 | 75 | Args: 76 | message (str): The message to print 77 | """ 78 | print(f"Fatal: {message}", file=sys.stderr) 79 | sys.exit(1) 80 | 81 | 82 | def assert_never(x: NoReturn) -> NoReturn: 83 | """ 84 | A hint to the typechecker that a branch can never occur. 85 | """ 86 | assert False, f"unhandled type: {type(x).__name__}" # pragma: no cover 87 | 88 | 89 | def collect_option_values( 90 | args: Sequence[str], 91 | option: str, 92 | *, 93 | style: OptionValueStyle = OptionValueStyle.MashOrSpace, 94 | ) -> list[tuple[int, str]]: 95 | """ 96 | Given a list of arguments, collect the ones that look like options with values. 97 | 98 | Supports multiple option "styles" via `OptionValueStyle`. 99 | 100 | Args: 101 | args (sequence): The arguments to search 102 | option (str): The option prefix to search for 103 | style: (OptionValueStyle): The option style to search for 104 | 105 | Returns: 106 | A list of tuples of (index, value) for matching options. The index in each 107 | tuple is the argument index for the option itself. 108 | """ 109 | 110 | # TODO(ww): There are a lot of error cases here. They should be thought out more. 111 | 112 | values: list[tuple[int, str]] = [] 113 | for idx, arg in enumerate(args): 114 | if not arg.startswith(option): 115 | continue 116 | 117 | is_exact = arg == option 118 | if is_exact and style.permits_space(): 119 | # -o foo is the only style that make sense here. 120 | values.append((idx, args[idx + 1])) 121 | elif not is_exact: 122 | # We have -oSOMETHING, where SOMETHING might be: 123 | # * A "mash", like `-Dfoo` 124 | # * An equals, like `-D=foo` 125 | if style.permits_mash(): 126 | # NOTE(ww): Assignment to work around black's confusing formatting. 127 | suff = len(option) 128 | values.append((idx, arg[suff:])) 129 | elif style.permits_equal(): 130 | values.append((idx, arg.split("=", 1)[1])) 131 | 132 | return values 133 | 134 | 135 | def rindex(items: Sequence[Any], needle: Any) -> int | None: 136 | """ 137 | Args: 138 | items (sequence): The items to search 139 | needle (object): The object to search for 140 | 141 | Returns: 142 | The rightmost index of `needle`, or `None`. 143 | """ 144 | for idx, item in enumerate(reversed(items)): 145 | if item == needle: 146 | return len(items) - idx - 1 147 | return None 148 | 149 | 150 | def rindex_prefix(items: Sequence[str], prefix: str) -> int | None: 151 | """ 152 | Args: 153 | items (sequence of str): The items to search 154 | prefix (str): The prefix to find 155 | 156 | Returns: 157 | The rightmost index of the element that starts with `prefix`, or `None` 158 | """ 159 | for idx, item in enumerate(reversed(items)): 160 | if item.startswith(prefix): 161 | return len(items) - idx - 1 162 | return None 163 | 164 | 165 | def ritem_prefix(items: Sequence[str], prefix: str) -> str | None: 166 | """ 167 | Args: 168 | items (sequence of str): The items to search 169 | prefix (str): The prefix to find 170 | 171 | Returns: 172 | The rightmost element that starts with `prefix`, or `None` 173 | """ 174 | for item in reversed(items): 175 | if item.startswith(prefix): 176 | return item 177 | return None 178 | 179 | 180 | def insert_items_at_idx(parent_items: Sequence[Any], idx: int, items: Sequence[Any]) -> list[Any]: 181 | """ 182 | Inserts `items` at `idx` in `parent_items`. 183 | 184 | Args: 185 | parent_items (sequence of any): The parent sequence to insert within 186 | idx (int): The index to insert at 187 | items (sequence of any): The items to insert 188 | 189 | Returns: 190 | A new list containing both the parent and inserted items 191 | """ 192 | 193 | def _insert_items_at_idx( 194 | parent_items: Sequence[Any], idx: int, items: Sequence[Any] 195 | ) -> Iterator[Any]: 196 | for pidx, item in enumerate(parent_items): 197 | if pidx != idx: 198 | print(item) 199 | yield item 200 | else: 201 | for item in items: 202 | yield item 203 | 204 | return list(_insert_items_at_idx(parent_items, idx, items)) 205 | 206 | 207 | @contextlib.contextmanager 208 | def flock_append(filename: os.PathLike) -> Iterator[IO]: 209 | """ 210 | Open the given file for appending, acquiring an exclusive lock on it in 211 | the process. 212 | 213 | Args: 214 | filename (str): The file to open for appending 215 | 216 | Yields: 217 | An open fileobject for `filename` 218 | """ 219 | with open(filename, "a") as io: 220 | try: 221 | fcntl.flock(io, fcntl.LOCK_EX) 222 | yield io 223 | finally: 224 | fcntl.flock(io, fcntl.LOCK_UN) 225 | 226 | 227 | def unswizzled_path() -> str: 228 | """ 229 | Returns a version of the current `$PATH` with any blight shim paths removed. 230 | """ 231 | paths = os.getenv("PATH", "").split(os.pathsep) 232 | paths = [p for p in paths if not Path(p).name.endswith(SWIZZLE_SENTINEL)] 233 | 234 | return os.pathsep.join(paths) 235 | 236 | 237 | def load_actions() -> list[Action]: 238 | """ 239 | Loads any blight actions requested via the environment. 240 | 241 | Each action is loaded from the `BLIGHT_ACTIONS` environment variable, 242 | separated by colons. Duplicate actions are removed. 243 | 244 | For example, the following loads the `Record` and `Benchmark` actions: 245 | 246 | ```bash 247 | BLIGHT_ACTIONS="Record:Benchmark" 248 | ``` 249 | 250 | Each action additionally receives a configuration dictionary from 251 | `BLIGHT_ACTION_{UPPERCASE_NAME}`. The contents of each of these variables 252 | is shell-quoted, in `key=value` format. 253 | 254 | For example, the following: 255 | 256 | ```bash 257 | BLIGHT_ACTION_RECORD="output=/tmp/whatever.jsonl" 258 | ``` 259 | 260 | yields the following configuration dictionary: 261 | 262 | ```python 263 | {"output": "/tmp/whatever.jsonl"} 264 | ``` 265 | 266 | Returns: 267 | A list of `blight.action.Action`s. 268 | """ 269 | import blight.actions 270 | 271 | action_names = os.getenv("BLIGHT_ACTIONS") 272 | if not action_names: 273 | return [] 274 | 275 | seen = set() 276 | actions = [] 277 | for action_name in action_names.split(":"): 278 | if action_name in seen: 279 | continue 280 | seen.add(action_name) 281 | 282 | action_class = getattr(blight.actions, action_name, None) 283 | if action_class is None: 284 | raise BlightError(f"Unknown action: {action_name}") 285 | 286 | action_config_raw = os.getenv(f"BLIGHT_ACTION_{action_name.upper()}", None) 287 | if action_config_raw is not None: 288 | action_config = shlex.split(action_config_raw) 289 | action_config = dict(c.split("=", 1) for c in action_config) 290 | else: 291 | action_config = {} 292 | 293 | actions.append(action_class(action_config)) 294 | return actions 295 | 296 | 297 | def json_helper(value: Any) -> Any: 298 | """ 299 | A `default` helper for Python's `json`, intended to facilitate 300 | serialization of blight classes. 301 | """ 302 | 303 | if hasattr(value, "asdict"): 304 | return value.asdict() 305 | 306 | if isinstance(value, Path): 307 | return str(value) 308 | 309 | raise TypeError 310 | 311 | 312 | class ArgumentParser(argparse.ArgumentParser): 313 | """ 314 | A wrapper around `argparse.ArgumentParser` with non-exiting error behavior. 315 | 316 | Parsing errors raise `ValueError` instead. 317 | """ 318 | 319 | def error(self, message: str) -> NoReturn: 320 | raise ValueError(message) 321 | 322 | def default_namespace(self) -> argparse.Namespace: 323 | """ 324 | Returns a default `argparse.Namespace`, suitable for contexts where 325 | argument parsing fails completely. 326 | """ 327 | defaults = {action.dest: self.get_default(action.dest) for action in self._actions} 328 | return argparse.Namespace(**defaults) 329 | -------------------------------------------------------------------------------- /test/actions/test_cc_for_cxx.py: -------------------------------------------------------------------------------- 1 | import shlex 2 | 3 | from blight.actions import CCForCXX 4 | from blight.tool import CC 5 | 6 | 7 | def test_cc_for_cxx(): 8 | cc_for_cxx = CCForCXX({}) 9 | cc = CC(["-std=c++17", "foo.cpp"]) 10 | 11 | cc_for_cxx.before_run(cc) 12 | 13 | assert cc.args == shlex.split("-x c++ -std=c++17 foo.cpp") 14 | 15 | 16 | def test_cc_for_cxx_does_not_inject(): 17 | cc_for_cxx = CCForCXX({}) 18 | cc = CC(["-std=c99", "foo.c"]) 19 | 20 | cc_for_cxx.before_run(cc) 21 | 22 | assert cc.args == shlex.split("-std=c99 foo.c") 23 | -------------------------------------------------------------------------------- /test/actions/test_demo.py: -------------------------------------------------------------------------------- 1 | from blight.actions import Demo 2 | from blight.tool import CC 3 | 4 | 5 | def test_demo(capfd): 6 | demo = Demo({}) 7 | 8 | demo.before_run(CC(["-fake", "-flags"])) 9 | demo.after_run(CC(["-fake", "-flags"]), run_skipped=False) 10 | 11 | out, err = capfd.readouterr() 12 | assert "before-run" in err 13 | assert "after-run" in err 14 | -------------------------------------------------------------------------------- /test/actions/test_embed_bitcode.py: -------------------------------------------------------------------------------- 1 | from blight.actions import EmbedBitcode 2 | from blight.tool import CC 3 | 4 | 5 | def test_embed_bitcode(): 6 | embed_bitcode = EmbedBitcode({}) 7 | cc = CC(["-o", "foo"]) 8 | embed_bitcode.before_run(cc) 9 | 10 | assert cc.args[0] == "-fembed-bitcode" 11 | -------------------------------------------------------------------------------- /test/actions/test_find_inputs.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from pathlib import Path 4 | 5 | from blight.actions import FindInputs 6 | from blight.actions.find_inputs import Input 7 | from blight.enums import InputKind 8 | from blight.tool import CC 9 | from blight.util import json_helper 10 | 11 | 12 | def test_find_inputs(tmp_path): 13 | output = tmp_path / "outputs.jsonl" 14 | 15 | foo_input = (tmp_path / "foo.c").resolve() 16 | foo_input.touch() 17 | 18 | find_inputs = FindInputs({"output": output}) 19 | cwd_path = Path(os.getcwd()).resolve() 20 | os.chdir(tmp_path) 21 | cc = CC(["-o", "foo", "foo.c"]) 22 | os.chdir(cwd_path) 23 | find_inputs.before_run(cc) 24 | find_inputs.after_run(cc) 25 | 26 | inputs = json.loads(output.read_text())["inputs"] 27 | assert inputs == [ 28 | { 29 | "kind": InputKind.Source.value, 30 | "prenormalized_path": "foo.c", 31 | "path": str(foo_input), 32 | "store_path": None, 33 | "content_hash": None, 34 | } 35 | ] 36 | 37 | 38 | def test_find_inputs_journaling(monkeypatch, tmp_path): 39 | journal_output = tmp_path / "journal.jsonl" 40 | monkeypatch.setenv("BLIGHT_JOURNAL_PATH", str(journal_output)) 41 | 42 | foo_input = (tmp_path / "foo.c").resolve() 43 | foo_input.touch() 44 | 45 | find_inputs = FindInputs({}) 46 | cc = CC(["-c", str(foo_input), "-o", "foo"]) 47 | find_inputs.before_run(cc) 48 | find_inputs.after_run(cc) 49 | 50 | inputs = find_inputs._result["inputs"] 51 | assert len(inputs) == 1 52 | assert inputs[0] == { 53 | "kind": InputKind.Source.value, 54 | "prenormalized_path": str(foo_input), 55 | "path": foo_input, 56 | "store_path": None, 57 | "content_hash": None, 58 | } 59 | 60 | 61 | def test_serialize_input(tmp_path): 62 | foo_input = (tmp_path / "foo.c").resolve() 63 | input = Input(prenormalized_path="foo.c", kind=InputKind.Source, path=foo_input) 64 | input_json = json.dumps(input.dict(), default=json_helper) 65 | kwargs = json.loads(input_json) 66 | assert Input(**kwargs).dict() == input.dict() 67 | -------------------------------------------------------------------------------- /test/actions/test_find_outputs.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import json 3 | 4 | import pytest 5 | 6 | from blight.actions import FindOutputs 7 | from blight.actions.find_outputs import OutputKind 8 | from blight.tool import CC, INSTALL 9 | 10 | 11 | def test_find_outputs(tmp_path): 12 | output = tmp_path / "outputs.jsonl" 13 | 14 | find_outputs = FindOutputs({"output": output}) 15 | cc = CC(["-o", "foo", "foo.c"]) 16 | find_outputs.before_run(cc) 17 | find_outputs.after_run(cc) 18 | 19 | outputs = json.loads(output.read_text())["outputs"] 20 | assert outputs == [ 21 | { 22 | "kind": OutputKind.Executable.value, 23 | "prenormalized_path": "foo", 24 | "path": str(cc.cwd / "foo"), 25 | "store_path": None, 26 | "content_hash": None, 27 | } 28 | ] 29 | 30 | 31 | def test_find_outputs_journaling(monkeypatch, tmp_path): 32 | journal_output = tmp_path / "journal.jsonl" 33 | monkeypatch.setenv("BLIGHT_JOURNAL_PATH", str(journal_output)) 34 | 35 | store = tmp_path / "store" 36 | contents = b"not a real object file" 37 | contents_digest = hashlib.sha256(contents).hexdigest() 38 | dummy_foo_o = tmp_path / "foo.o" 39 | dummy_foo_o_store = store / f"{dummy_foo_o.name}-{contents_digest}" 40 | 41 | find_outputs = FindOutputs({"store": store}) 42 | cc = CC(["-c", "foo.c", "-o", str(dummy_foo_o)]) 43 | find_outputs.before_run(cc) 44 | # Pretend to be the compiler: write some junk to dummy_foo_o 45 | dummy_foo_o.write_bytes(contents) 46 | 47 | find_outputs.after_run(cc) 48 | 49 | outputs = find_outputs._result["outputs"] 50 | assert len(outputs) == 1 51 | assert outputs[0] == { 52 | "kind": OutputKind.Object.value, 53 | "prenormalized_path": str(dummy_foo_o), 54 | "path": dummy_foo_o, 55 | "store_path": dummy_foo_o_store, 56 | "content_hash": contents_digest, 57 | } 58 | assert dummy_foo_o_store.read_bytes() == contents 59 | 60 | 61 | def test_find_outputs_multiple(tmp_path): 62 | fake_cs = [tmp_path / fake_c for fake_c in ["foo.c", "bar.c", "baz.c"]] 63 | [fake_c.touch() for fake_c in fake_cs] 64 | 65 | output = tmp_path / "outputs.jsonl" 66 | 67 | find_outputs = FindOutputs({"output": output}) 68 | cc = CC(["-c", *[str(fake_c) for fake_c in fake_cs]]) 69 | find_outputs.before_run(cc) 70 | find_outputs.after_run(cc) 71 | 72 | outputs = json.loads(output.read_text())["outputs"] 73 | assert outputs == [ 74 | { 75 | "kind": OutputKind.Object.value, 76 | "prenormalized_path": fake_c.with_suffix(".o").name, 77 | "path": str(cc.cwd / fake_c.with_suffix(".o").name), 78 | "store_path": None, 79 | "content_hash": None, 80 | } 81 | for fake_c in fake_cs 82 | ] 83 | 84 | 85 | def test_find_outputs_handles_a_out(tmp_path): 86 | output = tmp_path / "outputs.jsonl" 87 | 88 | find_outputs = FindOutputs({"output": output}) 89 | cc = CC(["foo.c"]) 90 | find_outputs.before_run(cc) 91 | find_outputs.after_run(cc) 92 | 93 | outputs = json.loads(output.read_text())["outputs"] 94 | assert outputs == [ 95 | { 96 | "kind": OutputKind.Executable.value, 97 | "prenormalized_path": "a.out", 98 | "path": str(cc.cwd / "a.out"), 99 | "store_path": None, 100 | "content_hash": None, 101 | } 102 | ] 103 | 104 | 105 | def test_find_outputs_store(tmp_path): 106 | output = tmp_path / "outputs.jsonl" 107 | store = tmp_path / "store" 108 | contents = b"not a real object file" 109 | contents_digest = hashlib.sha256(contents).hexdigest() 110 | dummy_foo_o = tmp_path / "foo.o" 111 | dummy_foo_o_store = store / f"{dummy_foo_o.name}-{contents_digest}" 112 | 113 | find_outputs = FindOutputs({"output": output, "store": store}) 114 | cc = CC(["-c", "foo.c", "-o", str(dummy_foo_o)]) 115 | find_outputs.before_run(cc) 116 | # Pretend to be the compiler: write some junk to dummy_foo_o 117 | dummy_foo_o.write_bytes(contents) 118 | find_outputs.after_run(cc) 119 | 120 | outputs = json.loads(output.read_text())["outputs"] 121 | assert outputs == [ 122 | { 123 | "kind": OutputKind.Object.value, 124 | "prenormalized_path": str(dummy_foo_o), 125 | "path": str(dummy_foo_o), 126 | "store_path": str(dummy_foo_o_store), 127 | "content_hash": contents_digest, 128 | } 129 | ] 130 | assert dummy_foo_o_store.read_bytes() == contents 131 | 132 | 133 | def test_find_outputs_store_no_hash(tmp_path): 134 | output = tmp_path / "outputs.jsonl" 135 | store = tmp_path / "store" 136 | contents = b"not a real object file" 137 | contents_digest = hashlib.sha256(contents).hexdigest() 138 | dummy_foo_o = tmp_path / "foo.o" 139 | # Store filename should not have its hash appended 140 | dummy_foo_o_store = store / dummy_foo_o.name 141 | 142 | find_outputs = FindOutputs({"output": output, "store": store, "append_hash": "false"}) 143 | cc = CC(["-c", "foo.c", "-o", str(dummy_foo_o)]) 144 | find_outputs.before_run(cc) 145 | # Pretend to be the compiler: write some junk to dummy_foo_o 146 | dummy_foo_o.write_bytes(contents) 147 | find_outputs.after_run(cc) 148 | 149 | outputs = json.loads(output.read_text())["outputs"] 150 | assert outputs == [ 151 | { 152 | "kind": OutputKind.Object.value, 153 | "prenormalized_path": str(dummy_foo_o), 154 | "path": str(dummy_foo_o), 155 | "store_path": str(dummy_foo_o_store), 156 | "content_hash": contents_digest, 157 | } 158 | ] 159 | assert dummy_foo_o_store.read_bytes() == contents 160 | 161 | 162 | def test_find_outputs_store_output_does_not_exist(tmp_path): 163 | output = tmp_path / "outputs.jsonl" 164 | store = tmp_path / "store" 165 | dummy_foo_o = tmp_path / "foo.o" 166 | 167 | find_outputs = FindOutputs({"output": output, "store": store}) 168 | cc = CC(["-c", "foo.c", "-o", str(dummy_foo_o)]) 169 | find_outputs.before_run(cc) 170 | find_outputs.after_run(cc) 171 | 172 | outputs = json.loads(output.read_text())["outputs"] 173 | assert outputs == [ 174 | { 175 | "kind": OutputKind.Object.value, 176 | "prenormalized_path": str(dummy_foo_o), 177 | "path": str(dummy_foo_o), 178 | "store_path": None, 179 | "content_hash": None, 180 | } 181 | ] 182 | 183 | 184 | @pytest.mark.parametrize( 185 | ("soname",), 186 | [ 187 | ("foo.so",), 188 | ("foo.so.1",), 189 | ("foo.so.1.2",), 190 | ("foo.so.1.2.3",), 191 | ], 192 | ) 193 | def test_find_outputs_annoying_so_prefixes(tmp_path, soname): 194 | output = tmp_path / "outputs.jsonl" 195 | 196 | find_outputs = FindOutputs({"output": output}) 197 | cc = CC(["-shared", "-o", soname, "foo.c"]) 198 | find_outputs.before_run(cc) 199 | find_outputs.after_run(cc) 200 | 201 | outputs = json.loads(output.read_text())["outputs"] 202 | assert outputs == [ 203 | { 204 | "kind": OutputKind.SharedLibrary.value, 205 | "prenormalized_path": soname, 206 | "path": str(cc.cwd / soname), 207 | "store_path": None, 208 | "content_hash": None, 209 | } 210 | ] 211 | 212 | 213 | def test_find_outputs_install(tmp_path): 214 | output = tmp_path / "outputs.jsonl" 215 | 216 | find_outputs = FindOutputs({"output": output}) 217 | install = INSTALL(["-c", "foo", "bar", "baz", "/tmp"]) 218 | find_outputs.before_run(install) 219 | find_outputs.after_run(install) 220 | 221 | outputs = json.loads(output.read_text())["outputs"] 222 | assert outputs == [ 223 | { 224 | "kind": OutputKind.Executable.value, 225 | "prenormalized_path": f"/tmp/{name}", 226 | "path": f"/tmp/{name}", 227 | "store_path": None, 228 | "content_hash": None, 229 | } 230 | for name in ["foo", "bar", "baz"] 231 | ] 232 | 233 | 234 | def test_find_outputs_install_directory_mode(tmp_path): 235 | output = tmp_path / "outputs.jsonl" 236 | 237 | find_outputs = FindOutputs({"output": output}) 238 | install = INSTALL(["-d", "foo", "bar", "baz"]) 239 | find_outputs.before_run(install) 240 | find_outputs.after_run(install) 241 | 242 | outputs = json.loads(output.read_text())["outputs"] 243 | assert outputs == [ 244 | { 245 | "kind": OutputKind.Directory.value, 246 | "prenormalized_path": name, 247 | "path": str(install.cwd / name), 248 | "store_path": None, 249 | "content_hash": None, 250 | } 251 | for name in ["foo", "bar", "baz"] 252 | ] 253 | 254 | 255 | def test_find_outputs_install_directory_mode_skip_copy(tmp_path): 256 | output = tmp_path / "outputs.jsonl" 257 | dummy = tmp_path / "dummy" 258 | store = tmp_path / "store" 259 | 260 | find_outputs = FindOutputs({"output": output, "store": store}) 261 | install = INSTALL(["-d", str(dummy)]) 262 | find_outputs.before_run(install) 263 | # Simulate `install -d`. 264 | dummy.mkdir() 265 | find_outputs.after_run(install) 266 | 267 | outputs = json.loads(output.read_text())["outputs"] 268 | assert outputs == [ 269 | { 270 | "kind": OutputKind.Directory.value, 271 | "prenormalized_path": str(dummy), 272 | "path": str(dummy), 273 | "store_path": None, 274 | "content_hash": None, 275 | } 276 | ] 277 | -------------------------------------------------------------------------------- /test/actions/test_ignore_flags.py: -------------------------------------------------------------------------------- 1 | import shlex 2 | 3 | from blight.actions import IgnoreFlags 4 | from blight.tool import CC, CXX 5 | 6 | 7 | def test_ignore_flags(): 8 | ignore_flags = IgnoreFlags({"FLAGS": "-Wextra -ffunction-sections"}) 9 | 10 | for tool in [CC, CXX]: 11 | tool = CC(["-Wall", "-ffunction-sections", "-O3", "-ffunction-sections", "-Wextra"]) 12 | ignore_flags.before_run(tool) 13 | assert tool.args == shlex.split("-Wall -O3") 14 | 15 | 16 | def test_ignore_werror_unknown_lang(): 17 | ignore_flags = IgnoreFlags({"FLAGS": "-Wextra -ffunction-sections"}) 18 | cxx = CXX(["-x", "-unknownlanguage", "-Werror"]) 19 | 20 | ignore_flags.before_run(cxx) 21 | 22 | assert cxx.args == shlex.split("-x -unknownlanguage -Werror") 23 | -------------------------------------------------------------------------------- /test/actions/test_ignore_flto.py: -------------------------------------------------------------------------------- 1 | import shlex 2 | 3 | from blight.actions import IgnoreFlto 4 | from blight.tool import CC 5 | 6 | 7 | def test_ignore_werror(): 8 | ignore_flto = IgnoreFlto({}) 9 | cc = CC(["-Wall", "-Werror", "-flto", "-flto=thin", "-O3"]) 10 | 11 | ignore_flto.before_run(cc) 12 | 13 | assert cc.args == shlex.split("-Wall -Werror -O3") 14 | -------------------------------------------------------------------------------- /test/actions/test_ignore_werror.py: -------------------------------------------------------------------------------- 1 | import shlex 2 | 3 | from blight.actions import IgnoreWerror 4 | from blight.tool import CC, CXX 5 | 6 | 7 | def test_ignore_werror(): 8 | ignore_werror = IgnoreWerror({}) 9 | cc = CC(["-Wall", "-Werror", "-O3", "-Werror"]) 10 | 11 | ignore_werror.before_run(cc) 12 | 13 | assert cc.args == shlex.split("-Wall -O3") 14 | 15 | 16 | def test_ignore_werror_cxx(): 17 | ignore_werror = IgnoreWerror({}) 18 | cxx = CXX(["-Wall", "-Werror", "-O3", "-Werror"]) 19 | 20 | ignore_werror.before_run(cxx) 21 | 22 | assert cxx.args == shlex.split("-Wall -O3") 23 | 24 | 25 | def test_ignore_werror_unknown_lang(): 26 | ignore_werror = IgnoreWerror({}) 27 | cxx = CXX(["-x", "-unknownlanguage", "-Werror"]) 28 | 29 | ignore_werror.before_run(cxx) 30 | 31 | assert cxx.args == shlex.split("-x -unknownlanguage -Werror") 32 | -------------------------------------------------------------------------------- /test/actions/test_inject_flags.py: -------------------------------------------------------------------------------- 1 | import shlex 2 | 3 | from blight.actions import InjectFlags 4 | from blight.tool import CC, CXX 5 | 6 | 7 | def test_inject_flags(): 8 | inject_flags = InjectFlags( 9 | {"CFLAGS": "-more -flags", "CXXFLAGS": "-these -are -ignored", "CPPFLAGS": "-foo"} 10 | ) 11 | cc = CC(["-fake", "-flags"]) 12 | 13 | inject_flags.before_run(cc) 14 | 15 | assert cc.args == shlex.split("-fake -flags -more -flags -foo") 16 | 17 | 18 | def test_inject_flags_cxx(): 19 | inject_flags = InjectFlags( 20 | {"CFLAGS": "-these -are -ignored", "CXXFLAGS": "-more -flags", "CPPFLAGS": "-bar"} 21 | ) 22 | cxx = CXX(["-fake", "-flags"]) 23 | 24 | inject_flags.before_run(cxx) 25 | 26 | assert cxx.args == shlex.split("-fake -flags -more -flags -bar") 27 | 28 | 29 | def test_inject_linker_flags(): 30 | inject_flags = InjectFlags( 31 | { 32 | "CFLAGS": "-cc-flags", 33 | "CFLAGS_LINKER": "-c-linker-flags", 34 | "CXXFLAGS": "-cxx-flags", 35 | "CXXFLAGS_LINKER": "-cxx-linker-flags", 36 | } 37 | ) 38 | 39 | cc_nolink = CC(["-c"]) 40 | cc_link = CC(["-fake", "-flags"]) 41 | cxx_nolink = CXX(["-c"]) 42 | cxx_link = CXX(["-fake", "-flags"]) 43 | 44 | inject_flags.before_run(cc_nolink) 45 | inject_flags.before_run(cc_link) 46 | inject_flags.before_run(cxx_nolink) 47 | inject_flags.before_run(cxx_link) 48 | 49 | assert cc_nolink.args == shlex.split("-c -cc-flags") 50 | assert cc_link.args == shlex.split("-fake -flags -cc-flags -c-linker-flags") 51 | assert cxx_nolink.args == shlex.split("-c -cxx-flags") 52 | assert cxx_link.args == shlex.split("-fake -flags -cxx-flags -cxx-linker-flags") 53 | 54 | 55 | def test_inject_flags_unknown_lang(): 56 | inject_flags = InjectFlags( 57 | {"CFLAGS": "-these -are -ignored", "CXXFLAGS": "-so -are -these", "CPPFLAGS": "-and -this"} 58 | ) 59 | cxx = CXX(["-x", "-unknownlanguage"]) 60 | 61 | inject_flags.before_run(cxx) 62 | 63 | assert cxx.args == shlex.split("-x -unknownlanguage") 64 | -------------------------------------------------------------------------------- /test/actions/test_lint.py: -------------------------------------------------------------------------------- 1 | import shlex 2 | 3 | import pretend 4 | import pytest 5 | from blight.actions import lint 6 | from blight.tool import CC 7 | 8 | 9 | @pytest.mark.parametrize( 10 | "macro", ["-DFORTIFY_SOURCE", "-D FORTIFY_SOURCE", "-DFORTIFY_SOURCE=1", "-D FORTIFY_SOURCE=2"] 11 | ) 12 | def test_lint(monkeypatch, macro): 13 | logger = pretend.stub(warning=pretend.call_recorder(lambda s: None)) 14 | monkeypatch.setattr(lint, "logger", logger) 15 | 16 | lint_ = lint.Lint({}) 17 | cc = CC([*shlex.split(macro), "-std=c++17", "foo.cpp"]) 18 | 19 | lint_.before_run(cc) 20 | assert logger.warning.calls == [ 21 | pretend.call("found -DFORTIFY_SOURCE; you probably meant: -D_FORTIFY_SOURCE") 22 | ] 23 | -------------------------------------------------------------------------------- /test/actions/test_record.py: -------------------------------------------------------------------------------- 1 | import json 2 | import shutil 3 | 4 | from blight.actions import Record 5 | from blight.tool import CC 6 | 7 | 8 | def test_record(tmp_path): 9 | output = tmp_path / "record.jsonl" 10 | record = Record({"output": output}) 11 | 12 | record.after_run(CC(["-fake", "-flags"]), run_skipped=False) 13 | 14 | record_contents = json.loads(output.read_text()) 15 | assert record_contents["wrapped_tool"] == shutil.which("cc") 16 | assert record_contents["args"] == ["-fake", "-flags"] 17 | assert not record_contents["run_skipped"] 18 | -------------------------------------------------------------------------------- /test/actions/test_skip_strip.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from blight.actions import SkipStrip 4 | from blight.exceptions import SkipRun 5 | from blight.tool import CC, STRIP 6 | 7 | 8 | def test_skip_strip_before_run_raises(): 9 | strip = STRIP(["--help"]) 10 | skip_strip = SkipStrip({}) 11 | 12 | with pytest.raises(SkipRun): 13 | skip_strip.before_run(strip) 14 | 15 | 16 | def test_skip_strip(monkeypatch): 17 | monkeypatch.setenv("BLIGHT_ACTIONS", "SkipStrip") 18 | monkeypatch.setenv("BLIGHT_WRAPPED_STRIP", "true") 19 | 20 | # SkipStrip causes strip runs to be skipped 21 | strip = STRIP([]) 22 | assert SkipStrip in [a.__class__ for a in strip._actions] 23 | assert not strip._skip_run 24 | strip.run() 25 | assert strip._skip_run 26 | 27 | # SkipStrip doesn't affect other tools 28 | cc = CC(["-v"]) 29 | assert SkipStrip in [a.__class__ for a in cc._actions] 30 | assert not cc._skip_run 31 | cc.run() 32 | assert not cc._skip_run 33 | -------------------------------------------------------------------------------- /test/conftest.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | 3 | import pytest 4 | 5 | 6 | @pytest.fixture(autouse=True) 7 | def blight_env(monkeypatch): 8 | monkeypatch.setenv("BLIGHT_WRAPPED_CC", shutil.which("cc")) 9 | monkeypatch.setenv("BLIGHT_WRAPPED_CXX", shutil.which("c++")) 10 | monkeypatch.setenv("BLIGHT_WRAPPED_CPP", shutil.which("cpp")) 11 | monkeypatch.setenv("BLIGHT_WRAPPED_LD", shutil.which("ld")) 12 | monkeypatch.setenv("BLIGHT_WRAPPED_AS", shutil.which("as")) 13 | monkeypatch.setenv("BLIGHT_WRAPPED_AR", shutil.which("ar")) 14 | monkeypatch.setenv("BLIGHT_WRAPPED_STRIP", shutil.which("strip")) 15 | monkeypatch.setenv("BLIGHT_WRAPPED_INSTALL", shutil.which("install")) 16 | -------------------------------------------------------------------------------- /test/test_action.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from blight import action, tool 4 | 5 | 6 | @pytest.mark.parametrize( 7 | ("action_class", "tool_class", "should_run_on"), 8 | [ 9 | (action.Action, tool.CC, True), 10 | (action.CCAction, tool.CC, True), 11 | (action.CCAction, tool.CXX, False), 12 | (action.CXXAction, tool.CXX, True), 13 | (action.CXXAction, tool.CC, False), 14 | (action.CompilerAction, tool.CC, True), 15 | (action.CompilerAction, tool.CXX, True), 16 | (action.CompilerAction, tool.CPP, False), 17 | (action.CPPAction, tool.CC, False), 18 | (action.CPPAction, tool.CPP, True), 19 | (action.LDAction, tool.CC, False), 20 | (action.LDAction, tool.LD, True), 21 | (action.ASAction, tool.CC, False), 22 | (action.ASAction, tool.AS, True), 23 | (action.ARAction, tool.CC, False), 24 | (action.ARAction, tool.AR, True), 25 | (action.STRIPAction, tool.CC, False), 26 | (action.STRIPAction, tool.STRIP, True), 27 | ], 28 | ) 29 | def test_should_run_on(action_class, tool_class, should_run_on): 30 | action = action_class({}) 31 | tool = tool_class([]) 32 | 33 | assert action._should_run_on(tool) == should_run_on 34 | -------------------------------------------------------------------------------- /test/test_enums.py: -------------------------------------------------------------------------------- 1 | from blight import enums 2 | 3 | 4 | def test_buildtool_cmd(): 5 | # the cmd property is only used in `blight.cli`, so we test it exhaustively here 6 | assert enums.BuildTool.CC.cmd == "cc" 7 | assert enums.BuildTool.CXX.cmd == "c++" 8 | assert enums.BuildTool.CPP.cmd == "cpp" 9 | assert enums.BuildTool.LD.cmd == "ld" 10 | assert enums.BuildTool.AS.cmd == "as" 11 | assert enums.BuildTool.AR.cmd == "ar" 12 | assert enums.BuildTool.STRIP.cmd == "strip" 13 | assert enums.BuildTool.INSTALL.cmd == "install" 14 | 15 | 16 | def test_optlevel_predictates(): 17 | assert enums.OptLevel.OSize.for_size() 18 | assert enums.OptLevel.OSizeZ.for_size() 19 | 20 | assert enums.OptLevel.O1.for_performance() 21 | assert enums.OptLevel.O2.for_performance() 22 | assert enums.OptLevel.O3.for_performance() 23 | assert enums.OptLevel.OFast.for_performance() 24 | 25 | assert enums.OptLevel.ODebug.for_debug() 26 | 27 | 28 | def test_std_predicates_and_lang(): 29 | for std in enums.Std: 30 | if std.is_cstd(): 31 | assert std.lang() == enums.Lang.C 32 | assert not std.is_cxxstd() 33 | elif std.is_cxxstd(): 34 | assert std.lang() == enums.Lang.Cxx 35 | assert not std.is_cstd() 36 | else: 37 | assert std.lang() == enums.Lang.Unknown 38 | assert std == enums.Std.Unknown 39 | 40 | 41 | def test_enum_stringification(): 42 | for tool in enums.BuildTool: 43 | assert str(tool) == tool.value 44 | 45 | for tool in enums.BlightTool: 46 | assert str(tool) == tool.value 47 | 48 | for family in enums.CompilerFamily: 49 | assert str(family) == family.name 50 | 51 | for stage in enums.CompilerStage: 52 | assert str(stage) == stage.name 53 | 54 | for lang in enums.Lang: 55 | assert str(lang) == lang.name 56 | 57 | for std in enums.Std: 58 | assert str(std) == std.name 59 | 60 | for opt in enums.OptLevel: 61 | assert str(opt) == opt.name 62 | 63 | for model in enums.CodeModel: 64 | assert str(model) == model.name 65 | 66 | for output in enums.OutputKind: 67 | assert str(output) == output.value 68 | 69 | for input in enums.InputKind: 70 | assert str(input) == input.value 71 | -------------------------------------------------------------------------------- /test/test_tool.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import shlex 4 | import shutil 5 | import sys 6 | from pathlib import Path 7 | 8 | import pretend 9 | import pytest 10 | 11 | from blight import tool, util 12 | from blight.enums import CodeModel, CompilerFamily, CompilerStage, Lang, OptLevel, Std 13 | from blight.exceptions import BlightError, BuildError 14 | 15 | needs_clang = pytest.mark.skipif(not shutil.which("clang"), reason="test requires clang") 16 | needs_gcc = pytest.mark.skipif(not shutil.which("gcc"), reason="test requires gcc") 17 | 18 | 19 | def test_tool_doesnt_instantiate(): 20 | with pytest.raises(NotImplementedError): 21 | tool.Tool([]) 22 | 23 | 24 | def test_compilertool_doesnt_instantiate(): 25 | with pytest.raises(NotImplementedError): 26 | tool.CompilerTool([]) 27 | 28 | 29 | def test_compilertool_env_warns_on_injection(monkeypatch): 30 | monkeypatch.setenv("_CL_", "foo") 31 | 32 | logger = pretend.stub(warning=pretend.call_recorder(lambda s: None)) 33 | monkeypatch.setattr(tool, "logger", logger) 34 | 35 | _ = tool.CC([]) 36 | assert logger.warning.calls == [ 37 | pretend.call("not tracking compiler's own instrumentation: {'_CL_'}") 38 | ] 39 | 40 | 41 | @needs_clang 42 | def test_compilertool_family_clang(monkeypatch): 43 | monkeypatch.setenv("BLIGHT_WRAPPED_CC", "clang") 44 | cc = tool.CC([]) 45 | 46 | # The host `clang` can be either one of these, depending on the OS or 47 | # user's configuration. 48 | assert cc.family in [CompilerFamily.AppleLlvm, CompilerFamily.MainlineLlvm] 49 | 50 | 51 | @needs_gcc 52 | @pytest.mark.skipif(sys.platform.startswith("darwin"), reason="clang is aliased as gcc on macOS") 53 | def test_compilertool_family_gcc(monkeypatch): 54 | monkeypatch.setenv("BLIGHT_WRAPPED_CC", "gcc") 55 | cc = tool.CC([]) 56 | 57 | assert cc.family == CompilerFamily.Gcc 58 | 59 | 60 | @pytest.mark.parametrize( 61 | ("stderr", "family"), 62 | [ 63 | (b"Apple clang version 13.1.6 (clang-1316.0.21.2.5)", CompilerFamily.AppleLlvm), 64 | (b"clang version 10.0.0-4ubuntu1", CompilerFamily.MainlineLlvm), 65 | (b"gcc version 9.4.0 (Ubuntu 9.4.0-1ubuntu1~20.04.1)", CompilerFamily.Gcc), 66 | (b"mystery compiler version 1.0.0", CompilerFamily.Unknown), 67 | (b"", CompilerFamily.Unknown), 68 | ], 69 | ) 70 | def test_compilertool_family(monkeypatch, stderr, family): 71 | logger = pretend.stub(warning=pretend.call_recorder(lambda s: None)) 72 | monkeypatch.setattr(tool, "logger", logger) 73 | 74 | result = pretend.stub(returncode=0, stderr=stderr) 75 | subprocess = pretend.stub(run=pretend.call_recorder(lambda args, **kw: result)) 76 | monkeypatch.setattr(tool, "subprocess", subprocess) 77 | 78 | cc = tool.CC([]) 79 | assert cc.family == family 80 | 81 | if stderr == b"": 82 | assert logger.warning.calls == [ 83 | pretend.call("compiler fingerprint failed: frontend didn't produce output for -###?") 84 | ] 85 | else: 86 | assert logger.warning.calls == [] 87 | 88 | 89 | def test_compilertool_family_tcc(monkeypatch): 90 | logger = pretend.stub(warning=pretend.call_recorder(lambda s: None)) 91 | monkeypatch.setattr(tool, "logger", logger) 92 | 93 | result = pretend.stub(returncode=1, stderr=b"tcc: error: invalid option -- '-###'") 94 | subprocess = pretend.stub(run=pretend.call_recorder(lambda args, **kw: result)) 95 | monkeypatch.setattr(tool, "subprocess", subprocess) 96 | 97 | cc = tool.CC([]) 98 | assert cc.family == CompilerFamily.Tcc 99 | assert logger.warning.calls == [ 100 | pretend.call("compiler fingerprint failed: frontend didn't recognize -###?") 101 | ] 102 | 103 | 104 | def test_compilertool_family_fingerprint_fails(monkeypatch): 105 | logger = pretend.stub(warning=pretend.call_recorder(lambda s: None)) 106 | monkeypatch.setattr(tool, "logger", logger) 107 | 108 | result = pretend.stub(returncode=1, stderr=b"") 109 | subprocess = pretend.stub(run=pretend.call_recorder(lambda args, **kw: result)) 110 | monkeypatch.setattr(tool, "subprocess", subprocess) 111 | 112 | cc = tool.CC([]) 113 | assert cc.family == CompilerFamily.Unknown 114 | assert logger.warning.calls == [ 115 | pretend.call("compiler fingerprint failed: frontend didn't recognize -###?") 116 | ] 117 | 118 | 119 | def test_tool_missing_wrapped_tool(monkeypatch): 120 | monkeypatch.delenv("BLIGHT_WRAPPED_CC") 121 | with pytest.raises(BlightError): 122 | tool.CC.wrapped_tool() 123 | 124 | 125 | def test_tool_fails(monkeypatch): 126 | monkeypatch.setenv("BLIGHT_WRAPPED_CC", "false") 127 | with pytest.raises(BuildError): 128 | tool.CC([]).run() 129 | 130 | 131 | def test_tool_env_filters_swizzle_path(monkeypatch): 132 | path = os.getenv("PATH") 133 | monkeypatch.setenv("PATH", f"/tmp/does-not-exist-{util.SWIZZLE_SENTINEL}:{path}") 134 | 135 | cc = tool.CC(["-v"]) 136 | 137 | env = cc.asdict()["env"] 138 | assert util.SWIZZLE_SENTINEL not in env["PATH"] 139 | 140 | 141 | def test_tool_run(monkeypatch, tmp_path): 142 | bench_output = tmp_path / "bench.jsonl" 143 | monkeypatch.setenv("BLIGHT_ACTIONS", "Benchmark") 144 | monkeypatch.setenv("BLIGHT_ACTION_BENCHMARK", f"output={bench_output}") 145 | 146 | cc = tool.CC(["-v"]) 147 | cc.run() 148 | 149 | bench_record = json.loads(bench_output.read_text()) 150 | assert bench_record["tool"] == cc.asdict() 151 | assert isinstance(bench_record["elapsed"], int) 152 | 153 | 154 | def test_tool_run_journaling(monkeypatch, tmp_path): 155 | journal_output = tmp_path / "journal.jsonl" 156 | monkeypatch.setenv("BLIGHT_ACTIONS", "Record:Benchmark:FindOutputs") 157 | monkeypatch.setenv("BLIGHT_JOURNAL_PATH", str(journal_output)) 158 | 159 | cc = tool.CC(["-v"]) 160 | cc.run() 161 | 162 | journal = json.loads(journal_output.read_text()) 163 | assert set(journal.keys()) == {"Record", "Benchmark", "FindOutputs"} 164 | assert all(isinstance(v, dict) for v in journal.values()) 165 | 166 | 167 | def test_tool_run_journaling_multiple(monkeypatch, tmp_path): 168 | journal_output = tmp_path / "journal.jsonl" 169 | monkeypatch.setenv("BLIGHT_ACTIONS", "Record:Benchmark:FindOutputs") 170 | monkeypatch.setenv("BLIGHT_JOURNAL_PATH", str(journal_output)) 171 | 172 | for _ in range(0, 10): 173 | cc = tool.CC(["-v"]) 174 | cc.run() 175 | 176 | count = 0 177 | with journal_output.open() as journal: 178 | for line in journal: 179 | count += 1 180 | record = json.loads(line) 181 | assert set(record.keys()) == {"Record", "Benchmark", "FindOutputs"} 182 | assert all(isinstance(v, dict) for v in record.values()) 183 | 184 | assert count == 10 185 | 186 | 187 | def test_tool_args_property(): 188 | cpp = tool.CPP(["a", "b", "c"]) 189 | 190 | assert cpp.args == ["a", "b", "c"] 191 | 192 | cpp.args.append("d") 193 | cpp.args += ["e"] 194 | 195 | assert cpp.args == ["a", "b", "c", "d", "e"] 196 | 197 | 198 | def test_tool_inputs(tmp_path): 199 | foo_input = (tmp_path / "foo.c").resolve() 200 | foo_input.touch() 201 | 202 | bar_input = (tmp_path / "bar.c").resolve() 203 | bar_input.touch() 204 | 205 | cc = tool.CC([str(foo_input), str(bar_input), "-", "-o", "foo"]) 206 | 207 | assert cc.inputs == [str(foo_input), str(bar_input), "-"] 208 | 209 | 210 | def test_tool_output(tmp_path): 211 | assert tool.CC(["-ofoo"]).outputs == ["foo"] 212 | assert tool.CC(["-o", "foo"]).outputs == ["foo"] 213 | assert tool.CC(["foo.c"]).outputs == ["a.out"] 214 | assert tool.CC(["-E"]).outputs == ["-"] 215 | 216 | foo_input = (tmp_path / "foo.c").resolve() 217 | foo_input.touch() 218 | 219 | assert tool.CC(["-c", str(foo_input)]).outputs == [str(foo_input.with_suffix(".o").name)] 220 | assert tool.CC(["-S", str(foo_input)]).outputs == [str(foo_input.with_suffix(".s").name)] 221 | 222 | bar_input = (tmp_path / "bar.c").resolve() 223 | bar_input.touch() 224 | 225 | assert tool.CC(["-c", str(foo_input), str(bar_input)]).outputs == [ 226 | str(foo_input.with_suffix(".o").name), 227 | str(bar_input.with_suffix(".o").name), 228 | ] 229 | assert tool.CC(["-S", str(foo_input), str(bar_input)]).outputs == [ 230 | str(foo_input.with_suffix(".s").name), 231 | str(bar_input.with_suffix(".s").name), 232 | ] 233 | 234 | assert tool.CC([]).outputs == [] 235 | assert tool.CC(["-v"]).outputs == [] 236 | 237 | 238 | def test_tool_response_file(tmp_path): 239 | response_file = (tmp_path / "args").resolve() 240 | response_file.write_text("-some -flags -O3") 241 | 242 | cc = tool.CC([f"@{response_file}"]) 243 | assert cc.args == [f"@{response_file}"] 244 | assert cc.canonicalized_args == ["-some", "-flags", "-O3"] 245 | assert cc.opt == OptLevel.O3 246 | 247 | 248 | def test_tool_response_file_nested(tmp_path): 249 | response_file1 = (tmp_path / "args").resolve() 250 | response_file1.write_text("-some -flags @args2 -more -flags") 251 | response_file2 = (tmp_path / "args2").resolve() 252 | response_file2.write_text("-nested -flags -O3") 253 | 254 | cc = tool.CC([f"@{response_file1}"]) 255 | assert cc.args == [f"@{response_file1}"] 256 | assert cc.canonicalized_args == [ 257 | "-some", 258 | "-flags", 259 | "-nested", 260 | "-flags", 261 | "-O3", 262 | "-more", 263 | "-flags", 264 | ] 265 | assert cc.opt == OptLevel.O3 266 | 267 | 268 | def test_tool_response_file_invalid_file(): 269 | cc = tool.CC(["@/this/file/does/not/exist"]) 270 | 271 | assert cc.args == ["@/this/file/does/not/exist"] 272 | assert cc.canonicalized_args == [] 273 | 274 | 275 | def test_tool_response_file_recursion_limit(tmp_path): 276 | response_file = (tmp_path / "args").resolve() 277 | response_file.write_text(f"-foo @{response_file}") 278 | 279 | cc = tool.CC([f"@{response_file}"]) 280 | assert cc.args == [f"@{response_file}"] 281 | assert cc.canonicalized_args == ["-foo"] * tool.RESPONSE_FILE_RECURSION_LIMIT 282 | 283 | 284 | def test_tool_explicit_library_search_paths(): 285 | cc = tool.CC( 286 | [ 287 | "-L.", 288 | "--library-path", 289 | "./bar", 290 | "-L..", 291 | "-L../foo", 292 | "-L", 293 | "foo", 294 | "-L/lib", 295 | "--library-path=../../baz", 296 | ] 297 | ) 298 | assert cc.explicit_library_search_paths == [ 299 | cc.cwd, 300 | cc.cwd / "bar", 301 | cc.cwd.parent, 302 | cc.cwd.parent / "foo", 303 | cc.cwd / "foo", 304 | Path("/lib").resolve(), 305 | cc.cwd.parent.parent / "baz", 306 | ] 307 | 308 | 309 | def test_tool_library_names(): 310 | cc = tool.CC(["-lfoo", "-l", "bar", "-liberty"]) 311 | assert cc.library_names == ["libfoo", "libbar", "libiberty"] 312 | 313 | 314 | @pytest.mark.parametrize( 315 | ("flags", "defines", "undefines"), 316 | [ 317 | ("-Dfoo -Dbar -Dbaz", [("foo", "1"), ("bar", "1"), ("baz", "1")], {}), 318 | ("-Dfoo -Ufoo -Dbar", [("bar", "1")], {"foo": 1}), 319 | ("-Dfoo -Dbar -Ufoo", [("bar", "1")], {"foo": 2}), 320 | ("-Ufoo -Dfoo", [("foo", "1")], {"foo": 0}), 321 | ("-U foo -Dfoo", [("foo", "1")], {"foo": 0}), 322 | ("-Ufoo -D foo", [("foo", "1")], {"foo": 0}), 323 | ("-Dkey=value", [("key", "value")], {}), 324 | ("-Dkey=value=x", [("key", "value=x")], {}), 325 | ("-Dkey='value'", [("key", "value")], {}), 326 | ("-Dkey='value=x'", [("key", "value=x")], {}), 327 | ("-D'FOO(x)=x+1'", [("FOO(x)", "x+1")], {}), 328 | ("-D 'FOO(x)=x+1'", [("FOO(x)", "x+1")], {}), 329 | ], 330 | ) 331 | def test_defines_mixin(flags, defines, undefines): 332 | cc = tool.CC(shlex.split(flags)) 333 | 334 | assert cc.defines == defines 335 | assert cc.indexed_undefines == undefines 336 | 337 | 338 | @pytest.mark.parametrize( 339 | ("flags", "code_model"), 340 | [ 341 | ("", CodeModel.Small), 342 | ("-mcmodel", CodeModel.Small), 343 | ("-mcmodel=small", CodeModel.Small), 344 | ("-mcmodel=medlow", CodeModel.Small), 345 | ("-mcmodel=medium", CodeModel.Medium), 346 | ("-mcmodel=medany", CodeModel.Medium), 347 | ("-mcmodel=large", CodeModel.Large), 348 | ("-mcmodel=kernel", CodeModel.Kernel), 349 | ("-mcmodel=unknown", CodeModel.Unknown), 350 | ], 351 | ) 352 | def test_codemodel_mixin(flags, code_model): 353 | cc = tool.CC(shlex.split(flags)) 354 | 355 | assert cc.code_model == code_model 356 | 357 | 358 | @pytest.mark.parametrize( 359 | ("flags", "lang", "std", "stage", "opt"), 360 | [ 361 | ("", Lang.C, Std.GnuUnknown, CompilerStage.Unknown, OptLevel.O0), 362 | ("-x c++ -O1", Lang.Cxx, Std.GnuxxUnknown, CompilerStage.AllStages, OptLevel.O1), 363 | ("-xc++ -O1", Lang.Cxx, Std.GnuxxUnknown, CompilerStage.AllStages, OptLevel.O1), 364 | ("-ansi -O2", Lang.C, Std.C89, CompilerStage.AllStages, OptLevel.O2), 365 | ("-ansi -x c++ -O3", Lang.Cxx, Std.Cxx03, CompilerStage.AllStages, OptLevel.O3), 366 | ("-std=c99 -O4", Lang.C, Std.C99, CompilerStage.AllStages, OptLevel.O3), 367 | ("-x unknown -Ofast", Lang.Unknown, Std.Unknown, CompilerStage.AllStages, OptLevel.OFast), 368 | ("-xunknown -Ofast", Lang.Unknown, Std.Unknown, CompilerStage.AllStages, OptLevel.OFast), 369 | ( 370 | "-ansi -x unknown -Os", 371 | Lang.Unknown, 372 | Std.Unknown, 373 | CompilerStage.AllStages, 374 | OptLevel.OSize, 375 | ), 376 | ("-std=cunknown -Oz", Lang.C, Std.CUnknown, CompilerStage.AllStages, OptLevel.OSizeZ), 377 | ("-std=c++unknown -Og", Lang.C, Std.CxxUnknown, CompilerStage.AllStages, OptLevel.ODebug), 378 | ( 379 | "-std=gnuunknown -Omadeup", 380 | Lang.C, 381 | Std.GnuUnknown, 382 | CompilerStage.AllStages, 383 | OptLevel.Unknown, 384 | ), 385 | ("-std=gnu++unknown -O", Lang.C, Std.GnuxxUnknown, CompilerStage.AllStages, OptLevel.O1), 386 | ("-std=nonsense", Lang.C, Std.Unknown, CompilerStage.AllStages, OptLevel.O0), 387 | ("-v", Lang.C, Std.GnuUnknown, CompilerStage.Unknown, OptLevel.O0), 388 | ("-###", Lang.C, Std.GnuUnknown, CompilerStage.Unknown, OptLevel.O0), 389 | ("-E", Lang.C, Std.GnuUnknown, CompilerStage.Preprocess, OptLevel.O0), 390 | ("-fsyntax-only", Lang.C, Std.GnuUnknown, CompilerStage.SyntaxOnly, OptLevel.O0), 391 | ("-S", Lang.C, Std.GnuUnknown, CompilerStage.Assemble, OptLevel.O0), 392 | ("-c", Lang.C, Std.GnuUnknown, CompilerStage.CompileObject, OptLevel.O0), 393 | ], 394 | ) 395 | def test_cc(flags, lang, std, stage, opt): 396 | flags = shlex.split(flags) 397 | cc = tool.CC(flags) 398 | 399 | assert cc.wrapped_tool() == shutil.which("cc") 400 | assert cc.lang == lang 401 | assert cc.std == std 402 | assert cc.stage == stage 403 | assert cc.opt == opt 404 | assert repr(cc) == f"" 405 | assert cc.asdict() == { 406 | "name": cc.__class__.__name__, 407 | "wrapped_tool": cc.wrapped_tool(), 408 | "args": flags, 409 | "canonicalized_args": flags, 410 | "cwd": str(cc.cwd), 411 | "lang": lang.name, 412 | "std": std.name, 413 | "stage": stage.name, 414 | "opt": opt.name, 415 | "env": dict(os.environ), 416 | } 417 | 418 | 419 | @pytest.mark.parametrize( 420 | ("flags", "lang", "std", "stage", "opt"), 421 | [ 422 | ("", Lang.Cxx, Std.GnuxxUnknown, CompilerStage.Unknown, OptLevel.O0), 423 | ("-x c", Lang.C, Std.GnuUnknown, CompilerStage.AllStages, OptLevel.O0), 424 | ("-xc", Lang.C, Std.GnuUnknown, CompilerStage.AllStages, OptLevel.O0), 425 | ("-std=c++17", Lang.Cxx, Std.Cxx17, CompilerStage.AllStages, OptLevel.O0), 426 | ], 427 | ) 428 | def test_cxx(flags, lang, std, stage, opt): 429 | flags = shlex.split(flags) 430 | cxx = tool.CXX(flags) 431 | 432 | assert cxx.wrapped_tool() == shutil.which("c++") 433 | assert cxx.lang == lang 434 | assert cxx.std == std 435 | assert cxx.stage == stage 436 | assert repr(cxx) == f"" 437 | assert cxx.asdict() == { 438 | "name": cxx.__class__.__name__, 439 | "wrapped_tool": cxx.wrapped_tool(), 440 | "args": flags, 441 | "canonicalized_args": flags, 442 | "cwd": str(cxx.cwd), 443 | "lang": lang.name, 444 | "std": std.name, 445 | "stage": stage.name, 446 | "opt": opt.name, 447 | "env": dict(os.environ), 448 | } 449 | 450 | 451 | @pytest.mark.parametrize( 452 | ("flags", "lang", "std"), 453 | [ 454 | ("", Lang.Unknown, Std.Unknown), 455 | ("-x c", Lang.C, Std.GnuUnknown), 456 | ("-ansi", Lang.Unknown, Std.Unknown), 457 | ], 458 | ) 459 | def test_cpp(flags, lang, std): 460 | flags = shlex.split(flags) 461 | cpp = tool.CPP(flags) 462 | 463 | assert cpp.wrapped_tool() == shutil.which("cpp") 464 | assert cpp.lang == lang 465 | assert cpp.std == std 466 | assert cpp.std.is_unknown() 467 | assert repr(cpp) == f"" 468 | assert cpp.asdict() == { 469 | "name": cpp.__class__.__name__, 470 | "wrapped_tool": cpp.wrapped_tool(), 471 | "args": flags, 472 | "canonicalized_args": flags, 473 | "cwd": str(cpp.cwd), 474 | "lang": lang.name, 475 | "std": std.name, 476 | "env": dict(os.environ), 477 | } 478 | 479 | 480 | def test_ld(): 481 | ld = tool.LD([]) 482 | 483 | assert ld.wrapped_tool() == shutil.which("ld") 484 | assert repr(ld) == f"" 485 | assert ld.asdict() == { 486 | "name": ld.__class__.__name__, 487 | "wrapped_tool": ld.wrapped_tool(), 488 | "args": [], 489 | "canonicalized_args": [], 490 | "cwd": str(ld.cwd), 491 | "env": dict(os.environ), 492 | } 493 | 494 | 495 | @pytest.mark.parametrize( 496 | ("flags", "output"), 497 | [ 498 | ("", "a.out"), 499 | ("-o foo", "foo"), 500 | ("-ofoo", "foo"), 501 | ("--output foo", "foo"), 502 | ("--output=foo", "foo"), 503 | ], 504 | ) 505 | def test_ld_output_forms(flags, output): 506 | ld = tool.LD(shlex.split(flags)) 507 | 508 | assert ld.outputs == [output] 509 | 510 | 511 | def test_as(): 512 | as_ = tool.AS([]) 513 | 514 | assert as_.wrapped_tool() == shutil.which("as") 515 | assert repr(as_) == f"" 516 | assert as_.asdict() == { 517 | "name": as_.__class__.__name__, 518 | "wrapped_tool": as_.wrapped_tool(), 519 | "args": [], 520 | "canonicalized_args": [], 521 | "cwd": str(as_.cwd), 522 | "env": dict(os.environ), 523 | } 524 | 525 | 526 | def test_ar(): 527 | ar = tool.AR([]) 528 | 529 | assert ar.wrapped_tool() == shutil.which("ar") 530 | assert repr(ar) == f"" 531 | assert ar.asdict() == { 532 | "name": ar.__class__.__name__, 533 | "wrapped_tool": ar.wrapped_tool(), 534 | "args": [], 535 | "canonicalized_args": [], 536 | "cwd": str(ar.cwd), 537 | "env": dict(os.environ), 538 | } 539 | 540 | 541 | @pytest.mark.parametrize( 542 | ("flags", "outputs"), 543 | [ 544 | ("cr foo.a a.o b.o", ["foo.a"]), 545 | ("-X64_32 cr foo.a a.o b.o", ["foo.a"]), 546 | ("r foo.a bar.o", ["foo.a"]), 547 | ("ru foo.a bar.o", ["foo.a"]), 548 | ("d foo.a bar.o", ["foo.a"]), 549 | ("--help", []), 550 | ], 551 | ) 552 | def test_ar_output_forms(flags, outputs): 553 | ar = tool.AR(shlex.split(flags)) 554 | 555 | assert ar.outputs == outputs 556 | 557 | 558 | def test_strip(): 559 | strip = tool.STRIP([]) 560 | 561 | assert strip.wrapped_tool() == shutil.which("strip") 562 | assert repr(strip) == f"" 563 | assert strip.asdict() == { 564 | "name": strip.__class__.__name__, 565 | "wrapped_tool": strip.wrapped_tool(), 566 | "args": [], 567 | "canonicalized_args": [], 568 | "cwd": str(strip.cwd), 569 | "env": dict(os.environ), 570 | } 571 | 572 | 573 | def test_install(): 574 | install = tool.INSTALL([]) 575 | 576 | assert install.wrapped_tool() == shutil.which("install") 577 | assert repr(install) == f"" 578 | assert install.asdict() == { 579 | "name": install.__class__.__name__, 580 | "wrapped_tool": install.wrapped_tool(), 581 | "args": [], 582 | "canonicalized_args": [], 583 | "cwd": str(install.cwd), 584 | "env": dict(os.environ), 585 | } 586 | 587 | 588 | @pytest.mark.parametrize( 589 | ("flags", "directory_mode", "inputs", "outputs"), 590 | [ 591 | # Reasonable inputs. 592 | ("-d foo bar baz", True, [], ["foo", "bar", "baz"]), 593 | ("-d -g wheel -o root -m 0755 foo bar baz", True, [], ["foo", "bar", "baz"]), 594 | ("-c foo bar", False, ["foo"], ["bar"]), 595 | ("-c foo bar baz /tmp", False, ["foo", "bar", "baz"], ["/tmp/foo", "/tmp/bar", "/tmp/baz"]), 596 | ( 597 | "-cv foo bar baz /tmp", 598 | False, 599 | ["foo", "bar", "baz"], 600 | ["/tmp/foo", "/tmp/bar", "/tmp/baz"], 601 | ), 602 | ( 603 | "-c -v foo bar baz /tmp", 604 | False, 605 | ["foo", "bar", "baz"], 606 | ["/tmp/foo", "/tmp/bar", "/tmp/baz"], 607 | ), 608 | ( 609 | "-cbCM foo bar baz /tmp", 610 | False, 611 | ["foo", "bar", "baz"], 612 | ["/tmp/foo", "/tmp/bar", "/tmp/baz"], 613 | ), 614 | ( 615 | "-g wheel -o root -m 0755 foo bar baz /tmp", 616 | False, 617 | ["foo", "bar", "baz"], 618 | ["/tmp/foo", "/tmp/bar", "/tmp/baz"], 619 | ), 620 | # Broken inputs. 621 | ("", False, [], []), 622 | ("foo", False, [], []), 623 | ], 624 | ) 625 | def test_install_inputs_and_outputs(flags, directory_mode, inputs, outputs): 626 | install = tool.INSTALL(shlex.split(flags)) 627 | 628 | assert install.directory_mode == directory_mode 629 | assert install.inputs == inputs 630 | assert install.outputs == outputs 631 | -------------------------------------------------------------------------------- /test/test_util.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | import pretend 5 | import pytest 6 | 7 | from blight import util 8 | from blight.actions import Record 9 | from blight.exceptions import BlightError 10 | 11 | 12 | def test_die(): 13 | with pytest.raises(SystemExit): 14 | util.die(":(") 15 | 16 | 17 | def test_collect_option_values(): 18 | args = ["foo", "-foo", "baz", "-fooquux", "-Dfoo"] 19 | 20 | assert util.collect_option_values(args, "-foo") == [(1, "baz"), (3, "quux")] 21 | assert util.collect_option_values(args, "-foo", style=util.OptionValueStyle.Mash) == [ 22 | (3, "quux") 23 | ] 24 | assert util.collect_option_values(args, "-foo", style=util.OptionValueStyle.Space) == [ 25 | (1, "baz") 26 | ] 27 | 28 | args = ["foo", "-foo=bar", "-foo", "baz"] 29 | assert util.collect_option_values(args, "-foo", style=util.OptionValueStyle.EqualOrSpace) == [ 30 | (1, "bar"), 31 | (2, "baz"), 32 | ] 33 | 34 | 35 | def test_rindex(): 36 | assert util.rindex([1, 1, 2, 3, 4, 5], 1) == 1 37 | assert util.rindex([1, 1, 2, 3, 4, 5], 6) is None 38 | assert util.rindex([1, 1, 2, 3, 4, 5], 5) == 5 39 | 40 | 41 | def test_load_actions(monkeypatch): 42 | monkeypatch.setenv("BLIGHT_ACTIONS", "Record") 43 | monkeypatch.setenv("BLIGHT_ACTION_RECORD", "key=value key2='a=b'") 44 | 45 | actions = util.load_actions() 46 | assert len(actions) == 1 47 | assert actions[0].__class__ == Record 48 | assert actions[0]._config == {"key": "value", "key2": "a=b"} 49 | 50 | 51 | def test_load_actions_dedupes(monkeypatch): 52 | monkeypatch.setenv("BLIGHT_ACTIONS", "Record:Record") 53 | 54 | actions = util.load_actions() 55 | assert len(actions) == 1 56 | 57 | 58 | def test_load_actions_preserves_order(monkeypatch): 59 | monkeypatch.setenv("BLIGHT_ACTIONS", "Benchmark:Record:FindOutputs") 60 | 61 | actions = util.load_actions() 62 | assert [a.__class__.__name__ for a in actions] == ["Benchmark", "Record", "FindOutputs"] 63 | 64 | 65 | def test_load_actions_nonexistent(monkeypatch): 66 | monkeypatch.setenv("BLIGHT_ACTIONS", "ThisActionDoesNotExist") 67 | 68 | with pytest.raises(BlightError): 69 | util.load_actions() 70 | 71 | 72 | def test_load_actions_empty_variable(monkeypatch): 73 | monkeypatch.setenv("BLIGHT_ACTIONS", "") 74 | 75 | actions = util.load_actions() 76 | assert actions == [] 77 | 78 | 79 | def test_load_actions_empty_config(monkeypatch): 80 | monkeypatch.setenv("BLIGHT_ACTIONS", "Record") 81 | 82 | actions = util.load_actions() 83 | assert len(actions) == 1 84 | assert actions[0].__class__ == Record 85 | assert actions[0]._config == {} 86 | 87 | 88 | def test_json_helper_asdict(): 89 | has_asdict = pretend.stub(asdict=lambda: {"foo": "bar"}) 90 | 91 | assert util.json_helper(has_asdict) == {"foo": "bar"} 92 | 93 | 94 | def test_json_helper_path(): 95 | cwd = os.getcwd() 96 | path = cwd / Path("foo") 97 | result = util.json_helper(path) 98 | assert isinstance(result, str) 99 | assert result == f"{cwd}/foo" 100 | 101 | 102 | def test_json_helper_typeerror(): 103 | junk = pretend.stub() 104 | 105 | with pytest.raises(TypeError): 106 | util.json_helper(junk) 107 | --------------------------------------------------------------------------------